tsql_parser 0.1.4 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0473885bdca58523ffe440e748c72c6918e0c549e33a65d9287d3c48bf63e6b9'
-  data.tar.gz: fe2f61520f1ee60df0ed980af63f187273a8d275d245d5aeeb688ce52e3c8760
+  metadata.gz: d67275fb1f5b1ff478b0e8a79d99af69a2d33b53059822e882b6a0c94e68d0a1
+  data.tar.gz: 50b27357c58d8c0ca89b069ddef3a03e4565b1e354a7e322d79c9ba64440895d
 SHA512:
-  metadata.gz: c0286ddb36e63aa05587d22ebc8bd26578de851779934fe590c77f43d7ff646379922d6eae4b5f265dbbf6a609b3e12de3bb829f865ef8850fc2314076c19d11
-  data.tar.gz: 48797d07c9d3dc5615ea692692717c0ac5312af185a930bf12e089eab2a4f2e80ca347790514e62dfbcf6238617ffc61b0ef248769ad3de4bb411c8dbe22399f
+  metadata.gz: dd4314846a33bd5b4c0e99dd41ecb67c65494faf015fc5ec81ffcf74cd5f98c52215bc28fd1a8729e0a1694ad7a055b09f51b724b0d7f27ae945a3ed2f778d13
+  data.tar.gz: 664dce679b765bf73e229f5cafc4eb0059f540c85674d4b6d41f1e3508e7536d512c457b64075d6c0f12b70bf7d3bfabdf4887466894beceec4182f444a5b986

data/lib/parsing/config/defaults.rb

@@ -0,0 +1,56 @@
+#   __               .__
+# _/  |_  ___________|  |           ___________ _______  ______ ___________
+# \   __\/  ___/ ____/  |    ______ \____ \__  \\_  __ \/  ___// __ \_  __ \
+#  |  |  \___ < <_|  |  |__ /_____/ |  |_> > __ \|  | \/\___ \\  ___/|  | \/
+#  |__| /____  >__   |____/         |   __(____  /__|  /____  >\___  >__|
+#            \/   |__|              |__|       \/           \/     \/
+#
+# A very light-weight and opinionated T-SQL parser and formatter.
+#
+# github.com/scstauf
+#
+# path:
+#   parsing/config/defaults.rb
+# object:
+#   TSqlParser::Parsing::Defaults
+
+module TSqlParser::Parsing
+  class Defaults
+    @@default_single_char_tokens = ["(", ",", ")", "=", "+", "-", "%", "/", "*", "<", "!", ">", "'", "[", "]", ";"]
+    @@default_delimiters = [" ", "\n", "\t"]
+    @@default_tab_count = 0
+    @@default_tab = "    "
+
+    def self.set_default_tab_count(tab_count = 0)
+      @@default_tab_count = tab_count
+    end
+
+    def self.set_default_tab(tab = "    ")
+      @@default_tab = tab
+    end
+
+    def self.set_default_single_char_tokens(single_char_tokens = ["(", ",", ")", "=", "+", "-", "%", "/", "*", "<", "!", ">", "'", "[", "]", ";"])
+      @@default_single_char_tokens = single_char_tokens
+    end
+
+    def self.set_default_delimiters(delimiters = [" ", "\n", "\t"])
+      @@default_delimiters = delimiters
+    end
+
+    def self.get_default_single_char_tokens
+      @@default_single_char_tokens
+    end
+
+    def self.get_default_delimiters
+      @@default_delimiters
+    end
+
+    def self.get_default_tab_count
+      @@default_tab_count
+    end
+
+    def self.get_default_tab
+      @@default_tab
+    end
+  end
+end

data/lib/parsing/formatter.rb

@@ -15,26 +15,25 @@
 #   TSqlParser::Parsing::Formatter
 
 module TSqlParser::Parsing
-  require_relative "iterator"
+  require_relative "config/defaults"
   require_relative "parser"
   require_relative "formatters/text_formatter"
-  require_relative "model/sql_container"
-  require_relative "model/flat_sql_container"
-
+  require_relative "transformers/token_transformer"
+  
   class Formatter
-    def self.format(tokens, tab_count = 0, tab = "    ")
-      containers = self.as_containers(tokens)
-      lines = self.combine_containers(containers)
+    def self.format(tokens, tab_count = Defaults.get_default_tab_count, tab = Defaults.get_default_tab)
+      lines = TokenTransformer.transform(tokens)
       lines = self.cleanup_whitespace(lines)
       lines = self.insert_indentation(lines, tab_count, tab)
-      lines = self.insert_newlines(lines)
+      #lines = self.insert_newlines(lines)
       text = lines.join("\n")
-      text = TextFormatter.format_inserts(text, tab)
-      text = TextFormatter.format_updates(text, tab)
-      text = TextFormatter.format_joins(text, tab)
-      text = TextFormatter.format_wheres(text, tab)
-      text = TextFormatter.format_selects(text, tab)
-      text = TextFormatter.format_sets(text, tab)
+      text = TextFormatter.new(JOIN, text, tab).format
+      text = TextFormatter.new(INSERT, text, tab).format
+      text = TextFormatter.new(UPDATE, text, tab).format
+      text = TextFormatter.new(WHERE, text, tab).format
+      text = TextFormatter.new(SELECT, text, tab).format
+      text = TextFormatter.new(SET, text, tab).format
+      puts text
       text
     end
 
@@ -63,7 +62,7 @@ module TSqlParser::Parsing
       new_lines
     end
 
-    def self.insert_indentation(lines, tab_count = 0, tab = "    ")
+    def self.insert_indentation(lines, tab_count = Defaults.get_default_tab_count, tab = Defaults.get_default_tab)
       indented_lines = []
       work_lines = []
       lines.each do |line|
@@ -116,66 +115,6 @@ module TSqlParser::Parsing
       lines
     end
 
-    def self.combine_containers(containers)
-      lines = []
-      containers.each do |c|
-        ct = c.get_token
-
-        builder = []
-        builder << ct[:value]
-
-        if c.has_siblings?
-          c.get_siblings.each do |sibling|
-            st = sibling.get_token
-
-            if st[:comment]
-              builder << "\n#{st[:value]}"
-              next
-            end
-
-            builder << st[:value]
-          end
-        end
-
-        lines << builder.join(" ")
-      end
-
-      lines
-    end
-
-    def self.as_containers(tokens)
-      containers = []
-      container = nil
-      skip_count = 0
-      tokens.each_with_index do |t, index|
-        if skip_count > 0
-          skip_count -= 1
-          next
-        end
-
-        next_token = tokens[index + 1]
-        if Parser.is_new_node_keyword? t[:value]
-          if not next_token.nil? and Parser.is_new_node_keyword? next_token[:value]
-            if Parser.is_new_node_composite?(t[:value], next_token[:value])
-              containers << container unless container.nil?
-              container = SqlContainer.combine(t, next_token)
-              skip_count = 1
-              next
-            end
-          end
-          containers << container unless container.nil?
-          container = SqlContainer.new(t)
-        elsif t[:label]
-          containers << container unless container.nil?
-          container = SqlContainer.new(t)
-        else
-          container.add t unless container.nil?
-        end
-      end
-      containers << container unless container.nil?
-      FlatSqlContainer.flatten_containers(containers)
-    end
-
     def self.safe_ws_cleanup(line)
       parts = []
       builder = ""

data/lib/parsing/formatters/format_factory.rb

@@ -0,0 +1,32 @@
+#   __               .__
+# _/  |_  ___________|  |           ___________ _______  ______ ___________
+# \   __\/  ___/ ____/  |    ______ \____ \__  \\_  __ \/  ___// __ \_  __ \
+#  |  |  \___ < <_|  |  |__ /_____/ |  |_> > __ \|  | \/\___ \\  ___/|  | \/
+#  |__| /____  >__   |____/         |   __(____  /__|  /____  >\___  >__|
+#            \/   |__|              |__|       \/           \/     \/
+#
+# A very light-weight and opinionated T-SQL parser and formatter.
+#
+# github.com/scstauf
+#
+# path:
+#   parsing/formatters/format_factory.rb
+# object:
+#   TSqlParser::Parsing::FormatFactory
+
+module TSqlParser::Parsing
+    require_relative "strategy/__formatters"
+    class FormatFactory
+        def self.get(type)
+            case type
+            when CTE then Formatters::CommonTableExpressionFormatter.new
+            when INSERT then Formatters::InsertFormatter.new
+            when JOIN then Formatters::JoinFormatter.new
+            when SELECT then Formatters::SelectFormatter.new
+            when SET then Formatters::SetFormatter.new
+            when UPDATE then Formatters::UpdateFormatter.new
+            when WHERE then Formatters::WhereFormatter.new
+            end
+        end
+    end
+end

data/lib/parsing/formatters/strategy/__defaults.rb

@@ -0,0 +1,16 @@
+#   __               .__
+# _/  |_  ___________|  |           ___________ _______  ______ ___________
+# \   __\/  ___/ ____/  |    ______ \____ \__  \\_  __ \/  ___// __ \_  __ \
+#  |  |  \___ < <_|  |  |__ /_____/ |  |_> > __ \|  | \/\___ \\  ___/|  | \/
+#  |__| /____  >__   |____/         |   __(____  /__|  /____  >\___  >__|
+#            \/   |__|              |__|       \/           \/     \/
+#
+# A very light-weight and opinionated T-SQL parser and formatter.
+#
+# github.com/scstauf
+#
+# path:
+#   parsing/formatters/strategy/__defaults.rb
+
+require_relative "../../config/defaults"
+Defaults = TSqlParser::Parsing::Defaults

data/lib/parsing/formatters/strategy/__formatters.rb

@@ -0,0 +1,31 @@
+#   __               .__
+# _/  |_  ___________|  |           ___________ _______  ______ ___________
+# \   __\/  ___/ ____/  |    ______ \____ \__  \\_  __ \/  ___// __ \_  __ \
+#  |  |  \___ < <_|  |  |__ /_____/ |  |_> > __ \|  | \/\___ \\  ___/|  | \/
+#  |__| /____  >__   |____/         |   __(____  /__|  /____  >\___  >__|
+#            \/   |__|              |__|       \/           \/     \/
+#
+# A very light-weight and opinionated T-SQL parser and formatter.
+#
+# github.com/scstauf
+#
+# path:
+#   parsing/formatters/strategy/__formatters.rb
+
+module TSqlParser::Parsing
+  require_relative "cte_formatter"
+  require_relative "set_formatter"
+  require_relative "join_formatter"
+  require_relative "insert_formatter"
+  require_relative "select_formatter"
+  require_relative "update_formatter"
+  require_relative "where_formatter"
+
+  CTE = 0
+  INSERT = 1
+  JOIN = 2
+  SELECT = 3
+  SET = 4
+  UPDATE = 5
+  WHERE = 6
+end

data/lib/parsing/formatters/strategy/base_formatter.rb

@@ -15,14 +15,24 @@
 #   TSqlParser::Parsing::Formatters::SetFormatter
 
 module TSqlParser::Parsing::Formatters
+  require_relative "__defaults"
+
   class BaseFormatter
-    def self.get_tab_count(line, tab = "    ")
-      tab_count = 0
+    def get_tab_count(line, tab = Defaults.get_default_tab)
+      tab_count = Defaults.get_default_tab_count
       while line.start_with? tab
         tab_count += 1
         line = line.sub(tab, "")
       end
       tab_count
     end
+
+    # @abstract
+    #
+    # @param [String] text
+    # @param [String] tab
+    def format(text, tab = Defaults.get_default_tab)
+      raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+    end
   end
 end

data/lib/parsing/formatters/strategy/cte_formatter.rb

@@ -0,0 +1,25 @@
+#   __               .__
+# _/  |_  ___________|  |           ___________ _______  ______ ___________
+# \   __\/  ___/ ____/  |    ______ \____ \__  \\_  __ \/  ___// __ \_  __ \
+#  |  |  \___ < <_|  |  |__ /_____/ |  |_> > __ \|  | \/\___ \\  ___/|  | \/
+#  |__| /____  >__   |____/         |   __(____  /__|  /____  >\___  >__|
+#            \/   |__|              |__|       \/           \/     \/
+#
+# A very light-weight and opinionated T-SQL parser and formatter.
+#
+# github.com/scstauf
+#
+# path:
+#   parsing/formatters/cte_formatter.rb
+# object:
+#   TSqlParser::Parsing::Formatters::CommonTableExpressionFormatter
+
+module TSqlParser::Parsing::Formatters
+    require_relative "base_formatter"
+
+    class CommonTableExpressionFormatter < BaseFormatter
+      def format(text, tab = Defaults.get_default_tab)
+      end
+    end
+  end
+  

data/lib/parsing/formatters/strategy/insert_formatter.rb

@@ -18,7 +18,7 @@ module TSqlParser::Parsing::Formatters
   require_relative "base_formatter"
 
   class InsertFormatter < BaseFormatter
-    def self.format(text, tab = "    ")
+    def format(text, tab = Defaults.get_default_tab)
       formatted = []
       lines = text.split("\n")
       search = "INSERT INTO"
@@ -43,7 +43,7 @@ module TSqlParser::Parsing::Formatters
 
     private
 
-    def self.format_insert(s, tab_count = 0, tab = "    ")
+    def format_insert(s, tab_count = Defaults.get_default_tab_count, tab = Defaults.get_default_tab)
       return s if s.nil?
       formatted = []
       if s.include? ") VALUES ("

data/lib/parsing/formatters/strategy/join_formatter.rb

@@ -18,15 +18,17 @@ module TSqlParser::Parsing::Formatters
   require_relative "base_formatter"
 
   class JoinFormatter < BaseFormatter
-    def self.format(text, tab = "    ")
+    def format(text, tab = Defaults.get_default_tab)
       text = text.gsub(/INNER\s+JOIN/, "INNER JOIN")
                  .gsub(/LEFT\s+JOIN/, "LEFT JOIN")
+                 .gsub(/RIGHT\s+JOIN/, "RIGHT JOIN")
+                 .gsub(/CROSS\s+JOIN/, "CROSS JOIN")
       lines = text.split("\n")
       new_text = []
 
       lines.each do |line|
         first = line.strip.split(" ").first
-        if line.include? " WHERE " and first != "WHERE" and not first.start_with? "--" and not first.start_with? "/*"
+        if line.include? " WHERE " and first != "WHERE" and not first.start_with? "--" and not first.start_with? "/*" and not line.strip.end_with? "'"
           tab_count = self.get_tab_count(line, tab)
           where_parts = line.strip.split(" WHERE ")
           where_text = []

data/lib/parsing/formatters/strategy/select_formatter.rb

@@ -18,7 +18,7 @@ module TSqlParser::Parsing::Formatters
   require_relative "base_formatter"
 
   class SelectFormatter < BaseFormatter
-    def self.format(text, tab = "    ")
+    def format(text, tab = Defaults.get_default_tab)
       formatted = []
       lines = text.split("\n")
       lines.each do |line|
@@ -42,7 +42,7 @@ module TSqlParser::Parsing::Formatters
 
     private
 
-    def self.format_select(s, tab_count = 0, tab = "    ")
+    def format_select(s, tab_count = Defaults.get_default_tab_count, tab = Defaults.get_default_tab)
       return s if s.nil?
 
       tokens = s.split(", ")

data/lib/parsing/formatters/strategy/set_formatter.rb

@@ -15,10 +15,10 @@
 #   TSqlParser::Parsing::Formatters::SetFormatter
 
 module TSqlParser::Parsing::Formatters
-  require_relative 'base_formatter'
+  require_relative "base_formatter"
 
   class SetFormatter < BaseFormatter
-    def self.format(text, tab = "    ")
+    def format(text, tab = Defaults.get_default_tab)
       formatted = []
       lines = text.split("\n")
       wait = false
@@ -71,7 +71,7 @@ module TSqlParser::Parsing::Formatters
 
     private
 
-    def self.format_set(s, tab_count = 0, tab = "    ")
+    def format_set(s, tab_count = Defaults.get_default_tab_count, tab = Defaults.get_default_tab)
       return s if s.nil?
       parts = []
       builder = ""

data/lib/parsing/formatters/strategy/update_formatter.rb

@@ -18,7 +18,7 @@ module TSqlParser::Parsing::Formatters
   require_relative "base_formatter"
 
   class UpdateFormatter < BaseFormatter
-    def self.format(text, tab = "    ")
+    def format(text, tab = Defaults.get_default_tab)
       formatted = []
       lines = text.split("\n")
       lines.each do |line|
@@ -42,7 +42,7 @@ module TSqlParser::Parsing::Formatters
 
     private
 
-    def self.format_update(s, tab_count = 0, tab = "    ")
+    def format_update(s, tab_count = Defaults.get_default_tab_count, tab = Defaults.get_default_tab)
       return s if s.nil?
       "\n#{tab * (tab_count + 1)}#{s}"
     end

data/lib/parsing/formatters/strategy/where_formatter.rb

@@ -18,7 +18,7 @@ module TSqlParser::Parsing::Formatters
   require_relative "base_formatter"
 
   class WhereFormatter < BaseFormatter
-    def self.format(text, tab = "   ")
+    def format(text, tab = Defaults.get_default_tab)
       formatted = []
       text.split("\n").each do |line|
         first = line.strip.split(" ").first
@@ -42,7 +42,7 @@ module TSqlParser::Parsing::Formatters
 
     private
 
-    def self.format_predicate(s, tab_count = 0, tab = "    ")
+    def format_predicate(s, tab_count = Defaults.get_default_tab_count, tab = Defaults.get_default_tab)
       return s if s.nil?
       indented = []
       formatted = []

data/lib/parsing/formatters/text_formatter.rb

@@ -15,36 +15,20 @@
 #   TSqlParser::Parsing::Formatter
 
 module TSqlParser::Parsing
-  require_relative 'strategy/set_formatter'
-  require_relative 'strategy/join_formatter'
-  require_relative 'strategy/insert_formatter'
-  require_relative 'strategy/select_formatter'
-  require_relative 'strategy/update_formatter'
-  require_relative 'strategy/where_formatter'
+  require_relative "strategy/__defaults"
+  require_relative "format_factory"
 
   class TextFormatter
-    def self.format_sets(text, tab = "    ")
-      Formatters::SetFormatter.format(text, tab)
-    end
-
-    def self.format_joins(text, tab = "    ")
-      Formatters::JoinFormatter.format(text, tab)
-    end
-
-    def self.format_updates(text, tab = "    ")
-      Formatters::UpdateFormatter.format(text, tab)
-    end
-
-    def self.format_inserts(text, tab = "    ")
-      Formatters::InsertFormatter.format(text, tab)
-    end
+    attr_writer :strategy
 
-    def self.format_selects(text, tab = "    ")
-      Formatters::SelectFormatter.format(text, tab)
+    def initialize(strategy, text, tab = Defaults.get_default_tab)
+      @strategy = FormatFactory.get(strategy)
+      @text = text
+      @tab = tab
     end
 
-    def self.format_wheres(text, tab = "   ")
-      Formatters::WhereFormatter.format(text, tab)
+    def format
+      @strategy.format(@text, @tab)
     end
   end
 end

data/lib/parsing/keyword.rb

@@ -24,7 +24,7 @@ module TSqlParser::Parsing
     end
 
     def self.get_new_node_keywords
-      %w[CREATE ALTER DROP RENAME SELECT INSERT UPDATE DELETE WHILE IF ELSE DECLARE SET WITH BEGIN FROM WHERE INNER LEFT JOIN END GO GROUP ORDER CASE PRINT RETURN GOTO OPEN CLOSE DEALLOCATE FETCH] \
+      %w[CREATE ALTER DROP RENAME SELECT INSERT UPDATE DELETE WHILE IF ELSE DECLARE SET WITH BEGIN FROM WHERE INNER LEFT JOIN END GO GROUP ORDER CASE PRINT RETURN GOTO OPEN CLOSE DEALLOCATE FETCH USE EXEC] \
         - %w[WITH LEFT RIGHT]
     end
 

data/lib/parsing/parser.rb

@@ -86,8 +86,8 @@ module TSqlParser::Parsing
       s == "]"
     end
 
-    def self.is_string_mark?(s)
-      s == "'"
+    def self.is_string?(s)
+      s.start_with? "'" and s.end_with? "'"
     end
 
     def self.is_comma?(s)

data/lib/parsing/tokenizer.rb

@@ -15,137 +15,166 @@
 #   TSqlParser::Parsing::Tokenizer
 
 module TSqlParser::Parsing
-  require_relative "parser"
+  require_relative "config/defaults"
+  require_relative "transformers/token_categorizer"
 
   class Tokenizer
     def self.tokenize(tsql_string)
-      tokens = basic_tokenize(
-        tsql_string,
-        ["(", ",", ")", "=", "+", "-", "%", "/", "*", "<", "!", ">", "'", "[", "]", ";"],
-        [" ", "\n", "\t"]
-      )
-      tokens.map do |t|
-        categorize(t)
-      end
+      Tokenizer.new.basic_tokenize(tsql_string).map { |t| TokenCategorizer.categorize(t) }
     end
 
-    def self.categorize(s)
-      data = {}
-      data[:value] = s
-      data[:keyword] = true if Parser.is_keyword? s
-      data[:operator] = true if Parser.is_operator? s
-      data[:function] = true if Parser.is_function? s
-      data[:type] = true if Parser.is_type? s
-      data[:comment] = true if Parser.is_comment? s
-      data[:numeric] = true if Parser.is_numeric? s
-      data[:special_variable] = true if Parser.is_special_variable? s
-      data[:variable] = true if Parser.is_variable? s
-      data[:temporary_table] = true if Parser.is_temp_table? s
-      data[:label] = true if Parser.is_label? s
-      data[:parenthesis] = true if Parser.is_parenthesis? s
-      data[:open_parenthesis] = true if Parser.is_open_parenthesis? s
-      data[:close_parenthesis] = true if Parser.is_close_parenthesis? s
-      data[:bracket] = true if Parser.is_bracket? s
-      data[:open_bracket] = true if Parser.is_open_bracket? s
-      data[:close_bracket] = true if Parser.is_close_bracket? s
-      data[:string_mark] = true if Parser.is_string_mark? s
-      data[:comma] = true if Parser.is_comma? s
-      data[:join] = true if Parser.is_join? s
-      data[:join_type] = true if Parser.is_join_type? s
-      data[:begin] = true if Parser.is_begin? s
-      data[:end] = true if Parser.is_end? s
-      data[:terminator] = true if Parser.is_terminator? s
-      data[:value] = data[:value].upcase if data[:keyword] or data[:function] or data[:type]
-      data[:needs_newline] = true if data[:keyword] and Parser.is_newline_required? s
-      data
-    end
-
-    def self.basic_tokenize(tsql_string, char_delimiters, skip_delimiters)
-      specific_tokens = []
-      delimiters = ([] << char_delimiters << skip_delimiters).flatten
-      builder = ""
+    def basic_tokenize(tsql_string)
+      self.reset
       tsql_chars = tsql_string.split("")
-      multiline_comment = false
-      comment = false
-      string = false
-      string_count = 0
-      skip_count = 0
+
       tsql_chars.each_with_index do |c, i|
-        if skip_count > 0
-          skip_count -= 1
+        if @skip_count > 0
+          @skip_count -= 1
           next
         end
 
-        next_c = tsql_chars[i + 1] unless i + 1 > tsql_chars.size
+        # get last and next char
+        @c = c
+        @last_c = tsql_chars[i - 1] unless i - 1 < 0
+        @next_c = tsql_chars[i + 1] unless i + 1 > tsql_chars.size
 
-        if Parser.is_multiline_comment_start?(c, next_c)
-          multiline_comment = true
-          specific_tokens << builder unless builder.empty?
-          builder = c
-          next
+        # if we aren't in a string.
+        unless @string
+          next if self.handle_multicomment_start
+          next if self.handle_multicomment_end
+          next if self.handle_singlecomment_start
+          next if self.build_comment
         end
 
-        if Parser.is_multiline_comment_end?(c, next_c)
-          skip_count = 1
-          multiline_comment = false
-          builder << c << next_c
-          specific_tokens << builder unless builder.empty?
-          builder = ""
-          next
+        unless @multiline_comment or @comment
+          next if self.handle_string_char
+          next if self.handle_two_char_op
+          next if self.handle_delimiter
         end
 
-        if Parser.is_comment_start?(c, next_c) and not comment
-          comment = true
-          skip_count = 1
-          specific_tokens << builder unless builder.empty?
-          builder = "--"
-          next
-        end
+        self.build
+      end
 
-        if comment and c != "\n"
-          builder << c
-          next
-        elsif comment and c == "\n"
-          specific_tokens << builder unless builder.empty?
-          builder = ""
-          comment = false
-          next
-        end
+      self.flush_builder
+      @tokens
+    end
 
-        if c == "'" and not multiline_comment and not comment
-          if not string
-            string = true
-            specific_tokens << builder unless builder.empty?
-            builder = c
-            next
-          else
-            string = false
-            builder << c
-            specific_tokens << builder unless builder.empty?
-            builder = ""
-            next
-          end
-        end
+    private
 
-        if Parser.is_two_char_op?(c, next_c)
-          skip_count = 1
-          specific_tokens << builder unless builder.empty?
-          specific_tokens << "#{c}#{next_c}"
-          builder = ""
-          next
-        end
+    def handle_multicomment_start
+      if Parser.is_multiline_comment_start?(@c, @next_c)
+        @multiline_comment = true
+        self.flush_builder(c)
+        return true
+      end
+    end
 
-        if delimiters.include? c and !multiline_comment and !string and !comment
-          specific_tokens << builder unless builder.empty?
-          specific_tokens << c unless skip_delimiters.include? c
-          builder = ""
-          next
-        end
+    def handle_multicomment_end
+      if Parser.is_multiline_comment_end?(@c, @next_c)
+        @skip_count = 1
+        @multiline_comment = false
+        self.build
+        self.build(@next_c)
+        self.flush_builder("")
+        return true
+      end
+    end
+
+    def handle_singlecomment_start
+      if Parser.is_comment_start?(@c, @next_c) and not @comment
+        @comment = true
+        @skip_count = 1
+        self.flush_builder("--")
+        return true
+      end
+    end
+
+    def handle_singlecomment_end
+      if @c == "\n"
+        self.flush_builder("")
+        @comment = false
+        return true
+      end
+    end
+
+    def handle_string_char
+      if @c == "'"
+        @string_count += 1
+        return true if self.handle_string_start
+        @string_count += 1 if @next_c == "'"
+        return true if self.handle_string_end
+      end
+    end
+
+    def handle_string_start
+      if not @string
+        @string = true
+        self.flush_builder(@c)
+        return true
+      end
+    end
+
+    def handle_string_end
+      if @string_count % 2 == 0
+        @string = false
+        @string_count = 0
+        self.build
+        self.flush_builder("")
+        return true
+      end
+    end
+
+    def handle_two_char_op
+      if Parser.is_two_char_op?(@c, @next_c)
+        @skip_count = 1
+        self.flush_builder("")
+        @tokens << "#{@c}#{@next_c}"
+        return true
+      end
+    end
 
-        builder << c
+    def handle_delimiter
+      if @delimiters.include? @c and !@multiline_comment and !@comment and !@string
+        self.flush_builder("")
+        @tokens << @c unless @skip_delimiters.include? @c
+        return true
       end
-      specific_tokens << builder unless builder.empty?
-      specific_tokens
+    end
+
+    def build(value = nil)
+      @builder << @c if value.nil?
+      @builder << value unless value.nil?
+    end
+
+    def build_comment
+      if @comment
+        return true if self.handle_singlecomment_end
+
+        self.build
+        return true
+      end
+    end
+
+    def flush_builder(value = nil)
+      @tokens << @builder unless @builder.empty?
+      @builder = value
+    end
+
+    def initialize
+      self.reset
+    end
+
+    def reset
+      @tokens = []
+      @multiline_comment = false
+      @comment = false
+      @string = false
+      @string_count = 0
+      @skip_count = 0
+      @char_delimiters = Defaults.get_default_single_char_tokens
+      @skip_delimiters = Defaults.get_default_delimiters
+      @delimiters = ([] << @char_delimiters << @skip_delimiters).flatten.uniq
+      @builder = ""
     end
   end
 end

data/lib/parsing/transformers/token_categorizer.rb

@@ -0,0 +1,52 @@
+#   __               .__
+# _/  |_  ___________|  |           ___________ _______  ______ ___________
+# \   __\/  ___/ ____/  |    ______ \____ \__  \\_  __ \/  ___// __ \_  __ \
+#  |  |  \___ < <_|  |  |__ /_____/ |  |_> > __ \|  | \/\___ \\  ___/|  | \/
+#  |__| /____  >__   |____/         |   __(____  /__|  /____  >\___  >__|
+#            \/   |__|              |__|       \/           \/     \/
+#
+# A very light-weight and opinionated T-SQL parser and formatter.
+#
+# github.com/scstauf
+#
+# path:
+#   parsing/transformers/token_categorizer.rb
+# object:
+#   TSqlParser::Parsing::TokenCategorizer
+
+module TSqlParser::Parsing
+  require_relative "../parser"
+
+  class TokenCategorizer
+    def self.categorize(s)
+      data = {}
+      data[:value] = s
+      data[:keyword] = true if Parser.is_keyword? s
+      data[:operator] = true if Parser.is_operator? s
+      data[:function] = true if Parser.is_function? s
+      data[:type] = true if Parser.is_type? s
+      data[:comment] = true if Parser.is_comment? s
+      data[:numeric] = true if Parser.is_numeric? s
+      data[:special_variable] = true if Parser.is_special_variable? s
+      data[:variable] = true if Parser.is_variable? s
+      data[:temporary_table] = true if Parser.is_temp_table? s
+      data[:label] = true if Parser.is_label? s
+      data[:parenthesis] = true if Parser.is_parenthesis? s
+      data[:open_parenthesis] = true if Parser.is_open_parenthesis? s
+      data[:close_parenthesis] = true if Parser.is_close_parenthesis? s
+      data[:bracket] = true if Parser.is_bracket? s
+      data[:open_bracket] = true if Parser.is_open_bracket? s
+      data[:close_bracket] = true if Parser.is_close_bracket? s
+      data[:string] = true if Parser.is_string? s
+      data[:comma] = true if Parser.is_comma? s
+      data[:join] = true if Parser.is_join? s
+      data[:join_type] = true if Parser.is_join_type? s
+      data[:begin] = true if Parser.is_begin? s
+      data[:end] = true if Parser.is_end? s
+      data[:terminator] = true if Parser.is_terminator? s
+      data[:value] = data[:value].upcase if data[:keyword] or data[:function] or data[:type]
+      data[:needs_newline] = true if data[:keyword] and Parser.is_newline_required? s
+      data
+    end
+  end
+end

data/lib/parsing/transformers/token_transformer.rb

@@ -0,0 +1,153 @@
+#   __               .__
+# _/  |_  ___________|  |           ___________ _______  ______ ___________
+# \   __\/  ___/ ____/  |    ______ \____ \__  \\_  __ \/  ___// __ \_  __ \
+#  |  |  \___ < <_|  |  |__ /_____/ |  |_> > __ \|  | \/\___ \\  ___/|  | \/
+#  |__| /____  >__   |____/         |   __(____  /__|  /____  >\___  >__|
+#            \/   |__|              |__|       \/           \/     \/
+#
+# A very light-weight and opinionated T-SQL parser and formatter.
+#
+# github.com/scstauf
+#
+# path:
+#   parsing/model/token_transformer.rb
+# object:
+#   TSqlParser::Parsing::TokenTransformer
+
+module TSqlParser::Parsing
+  require_relative "../models/sql_container"
+  require_relative "../models/flat_sql_container"
+
+  class TokenTransformer
+    def initialize(tokens = [])
+      @containers = []
+      @container = nil
+      @skip_count = 0
+      @cte = ""
+      @parenthesis = 0
+      @tokens = tokens
+    end
+
+    def self.transform(tokens)
+      TokenTransformer.new(tokens).transform
+    end
+
+    def transform
+      @tokens.each_with_index do |t, index|
+        @parenthesis += 1 if t[:open_parenthesis]
+        @parenthesis -= 1 if t[:close_parenthesis]
+
+        next if self.handle_skip  
+
+        @t = t
+        @last_token = @tokens[index - 1] unless index - 1 < 0
+        @next_token = @tokens[index + 1] unless index + 1 > @tokens.size
+
+        next if self.handle_cte_end   
+
+        if t[:string]
+          self.add_to_container
+        elsif Parser.is_new_node_keyword? t[:value]
+          if not @next_token.nil? and Parser.is_new_node_keyword? @next_token[:value]
+            if Parser.is_new_node_composite?(@t[:value], @next_token[:value])
+              self.add_container
+              @container = SqlContainer.combine(@t, @next_token)
+              @skip_count = 1
+              next
+            end
+          end
+          self.add_container
+          self.new_container
+        elsif t[:value] == "WITH"
+          if not @next_token.nil? and not @next_token[:keyword] and not @next_token[:parenthesis]
+            self.add_container
+            self.new_container
+            @cte = @next_token[:value]
+          else
+            self.add_to_container
+          end
+        elsif @t[:label]
+          self.add_container
+          self.new_container
+        else
+          self.add_to_container
+        end
+      end
+      self.add_container
+      self.get_lines_from_containers
+    end
+
+    private
+
+    def handle_skip
+      if @skip_count > 0
+        @skip_count -= 1
+        return true
+      end
+    end
+
+    def add_to_container
+      @container.add @t unless @container.nil?
+    end
+
+    def add_container
+      @containers << @container unless @container.nil?
+    end
+
+    def new_container
+      @container = SqlContainer.new(@t)
+    end
+
+    def handle_cte_end
+      if not @cte.empty? 
+        if not @last_token.nil? and @last_token[:close_parenthesis] and @parenthesis == 0
+          if @t[:keyword]
+            self.add_container
+            self.new_container
+            @cte = ""
+          else
+            self.add_to_container
+          end
+        elsif not @last_token.nil? and not @last_token[:keyword] and @t[:value] == "AS"
+          self.add_to_container
+        elsif not @last_token.nil? and @last_token[:comma] and not @t[:keyword] and not @next_token.nil? and @next_token[:value] == "AS"
+          self.add_container
+          self.new_container
+          @cte = @t[:value]
+        else
+          self.add_to_container
+        end
+        return true
+      end
+    end
+
+    def get_lines_from_containers
+      @containers = FlatSqlContainer.flatten_containers(@containers)
+      
+      lines = []
+      @containers.each do |c|
+        ct = c.get_token
+
+        builder = []
+        builder << ct[:value]
+
+        if c.has_siblings?
+          c.get_siblings.each do |sibling|
+            st = sibling.get_token
+
+            if st[:comment]
+              builder << "\n#{st[:value]}"
+              next
+            end
+
+            builder << st[:value]
+          end
+        end
+
+        lines << builder.join(" ")
+      end
+
+      lines
+    end
+  end
+end

data/lib/tsql_parser.rb

@@ -15,13 +15,15 @@
 #   TSqlParser
 
 module TSqlParser
+  require_relative "parsing/config/defaults"
+
   # Formats a SQL string.
   #
   # @param sql [String] the SQL string to format.
   # @param tab_count [Integer] the number of tabs to start with.
   # @param tab [String] the tab string.
   # @return [String] the formatted SQL string.
-  def self.format(sql, tab_count = 0, tab = "    ")
+  def self.format(sql, tab_count = Parsing::Defaults.get_default_tab_count, tab = Parsing::Defaults.get_default_tab)
     require_relative "parsing/formatter"
     tokens = self.parse(sql)
     Parsing::Formatter.format(tokens, tab_count, tab)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tsql_parser
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.6
 platform: ruby
 authors:
 - Scott Stauffer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-03-28 00:00:00.000000000 Z
+date: 2023-04-03 00:00:00.000000000 Z
 dependencies: []
 description: A very light-weight and opinionated T-SQL parser and formatter.
 email: scott@fuseraft.com
@@ -16,8 +16,13 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- lib/parsing/config/defaults.rb
 - lib/parsing/formatter.rb
+- lib/parsing/formatters/format_factory.rb
+- lib/parsing/formatters/strategy/__defaults.rb
+- lib/parsing/formatters/strategy/__formatters.rb
 - lib/parsing/formatters/strategy/base_formatter.rb
+- lib/parsing/formatters/strategy/cte_formatter.rb
 - lib/parsing/formatters/strategy/insert_formatter.rb
 - lib/parsing/formatters/strategy/join_formatter.rb
 - lib/parsing/formatters/strategy/select_formatter.rb
@@ -25,12 +30,13 @@ files:
 - lib/parsing/formatters/strategy/update_formatter.rb
 - lib/parsing/formatters/strategy/where_formatter.rb
 - lib/parsing/formatters/text_formatter.rb
-- lib/parsing/iterator.rb
 - lib/parsing/keyword.rb
-- lib/parsing/model/flat_sql_container.rb
-- lib/parsing/model/sql_container.rb
+- lib/parsing/models/flat_sql_container.rb
+- lib/parsing/models/sql_container.rb
 - lib/parsing/parser.rb
 - lib/parsing/tokenizer.rb
+- lib/parsing/transformers/token_categorizer.rb
+- lib/parsing/transformers/token_transformer.rb
 - lib/tsql_parser.rb
 homepage: https://rubygems.org/gems/tsql_parser
 licenses:

data/lib/parsing/iterator.rb

@@ -1,54 +0,0 @@
-#   __               .__
-# _/  |_  ___________|  |           ___________ _______  ______ ___________
-# \   __\/  ___/ ____/  |    ______ \____ \__  \\_  __ \/  ___// __ \_  __ \
-#  |  |  \___ < <_|  |  |__ /_____/ |  |_> > __ \|  | \/\___ \\  ___/|  | \/
-#  |__| /____  >__   |____/         |   __(____  /__|  /____  >\___  >__|
-#            \/   |__|              |__|       \/           \/     \/
-#
-# A very light-weight and opinionated T-SQL parser and formatter.
-#
-# github.com/scstauf
-#
-# path:
-#   parsing/iterator.rb
-# object:
-#   TSqlParser::Parsing::TokenIterator
-
-module TSqlParser::Parsing
-  class TokenIterator
-    def initialize(tokens)
-      @tokens = tokens
-      @size = tokens.size
-      @iter = -1
-    end
-
-    def has_next?
-      @iter < @size - 1
-    end
-
-    def get!
-      @tokens[@iter]
-    end
-
-    def peek!
-      @tokens[@iter + 1]
-    end
-
-    def peek_ahead!(length)
-      @tokens[@iter + length]
-    end
-
-    def peek_value!
-      self.peek![:value]
-    end
-
-    def peek_ahead_value!(length)
-      self.peek_ahead!(length)[:value]
-    end
-
-    def next!
-      @iter += 1
-      self.get!
-    end
-  end
-end