tsql_parser 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0473885bdca58523ffe440e748c72c6918e0c549e33a65d9287d3c48bf63e6b9'
-  data.tar.gz: fe2f61520f1ee60df0ed980af63f187273a8d275d245d5aeeb688ce52e3c8760
+  metadata.gz: 81a892f94d3a26de7e1b77b16c723b67ec1ccddc8a8c6dc4251300792be75c75
+  data.tar.gz: 1a326f8c1f465444144d436d69c95180c4a6d6ff19cc90e3db93afa9e7f7484e
 SHA512:
-  metadata.gz: c0286ddb36e63aa05587d22ebc8bd26578de851779934fe590c77f43d7ff646379922d6eae4b5f265dbbf6a609b3e12de3bb829f865ef8850fc2314076c19d11
-  data.tar.gz: 48797d07c9d3dc5615ea692692717c0ac5312af185a930bf12e089eab2a4f2e80ca347790514e62dfbcf6238617ffc61b0ef248769ad3de4bb411c8dbe22399f
+  metadata.gz: 240fb09638d5005e9cce7ba057666c8fc3a5167e10071e0affd78af39a962b253914b392022ee54449f3126abbea32e70ef6a3d2c3c151607b0ac46703c9605e
+  data.tar.gz: 71dc6419e1a96499521158f1e25023afa53488f4a94fd5ff33cc46e15664533f0e6dd19a57c71fd866537c8b12ca70c64e7b72d08e0841f9aa5cbaa5c6e12667

data/lib/parsing/formatter.rb

@@ -15,16 +15,13 @@
 #   TSqlParser::Parsing::Formatter
 
 module TSqlParser::Parsing
-  require_relative "iterator"
   require_relative "parser"
   require_relative "formatters/text_formatter"
-  require_relative "model/sql_container"
-  require_relative "model/flat_sql_container"
-
+  require_relative "transformers/token_transformer"
+  
   class Formatter
     def self.format(tokens, tab_count = 0, tab = "    ")
-      containers = self.as_containers(tokens)
-      lines = self.combine_containers(containers)
+      lines = TokenTransformer.transform(tokens)
       lines = self.cleanup_whitespace(lines)
       lines = self.insert_indentation(lines, tab_count, tab)
       lines = self.insert_newlines(lines)
@@ -116,66 +113,6 @@ module TSqlParser::Parsing
       lines
     end
 
-    def self.combine_containers(containers)
-      lines = []
-      containers.each do |c|
-        ct = c.get_token
-
-        builder = []
-        builder << ct[:value]
-
-        if c.has_siblings?
-          c.get_siblings.each do |sibling|
-            st = sibling.get_token
-
-            if st[:comment]
-              builder << "\n#{st[:value]}"
-              next
-            end
-
-            builder << st[:value]
-          end
-        end
-
-        lines << builder.join(" ")
-      end
-
-      lines
-    end
-
-    def self.as_containers(tokens)
-      containers = []
-      container = nil
-      skip_count = 0
-      tokens.each_with_index do |t, index|
-        if skip_count > 0
-          skip_count -= 1
-          next
-        end
-
-        next_token = tokens[index + 1]
-        if Parser.is_new_node_keyword? t[:value]
-          if not next_token.nil? and Parser.is_new_node_keyword? next_token[:value]
-            if Parser.is_new_node_composite?(t[:value], next_token[:value])
-              containers << container unless container.nil?
-              container = SqlContainer.combine(t, next_token)
-              skip_count = 1
-              next
-            end
-          end
-          containers << container unless container.nil?
-          container = SqlContainer.new(t)
-        elsif t[:label]
-          containers << container unless container.nil?
-          container = SqlContainer.new(t)
-        else
-          container.add t unless container.nil?
-        end
-      end
-      containers << container unless container.nil?
-      FlatSqlContainer.flatten_containers(containers)
-    end
-
     def self.safe_ws_cleanup(line)
       parts = []
       builder = ""

data/lib/parsing/formatters/strategy/join_formatter.rb

@@ -26,7 +26,7 @@ module TSqlParser::Parsing::Formatters
 
       lines.each do |line|
         first = line.strip.split(" ").first
-        if line.include? " WHERE " and first != "WHERE" and not first.start_with? "--" and not first.start_with? "/*"
+        if line.include? " WHERE " and first != "WHERE" and not first.start_with? "--" and not first.start_with? "/*" and not line.strip.end_with? "'"
           tab_count = self.get_tab_count(line, tab)
           where_parts = line.strip.split(" WHERE ")
           where_text = []

data/lib/parsing/keyword.rb

@@ -24,7 +24,7 @@ module TSqlParser::Parsing
     end
 
     def self.get_new_node_keywords
-      %w[CREATE ALTER DROP RENAME SELECT INSERT UPDATE DELETE WHILE IF ELSE DECLARE SET WITH BEGIN FROM WHERE INNER LEFT JOIN END GO GROUP ORDER CASE PRINT RETURN GOTO OPEN CLOSE DEALLOCATE FETCH] \
+      %w[CREATE ALTER DROP RENAME SELECT INSERT UPDATE DELETE WHILE IF ELSE DECLARE SET WITH BEGIN FROM WHERE INNER LEFT JOIN END GO GROUP ORDER CASE PRINT RETURN GOTO OPEN CLOSE DEALLOCATE FETCH USE EXEC] \
         - %w[WITH LEFT RIGHT]
     end
 

data/lib/parsing/parser.rb

@@ -86,8 +86,8 @@ module TSqlParser::Parsing
       s == "]"
     end
 
-    def self.is_string_mark?(s)
-      s == "'"
+    def self.is_string?(s)
+      s.start_with? "'" and s.end_with? "'"
     end
 
     def self.is_comma?(s)

data/lib/parsing/tokenizer.rb

@@ -15,137 +15,176 @@
 #   TSqlParser::Parsing::Tokenizer
 
 module TSqlParser::Parsing
-  require_relative "parser"
+  require_relative "transformers/token_categorizer"
 
   class Tokenizer
+    @@default_char_delimiters = ["(", ",", ")", "=", "+", "-", "%", "/", "*", "<", "!", ">", "'", "[", "]", ";"]
+    @@default_skip_delimiters = [" ", "\n", "\t"]
+
+    def self.set_default_char_delimiters(char_delimiters = [])
+      @@default_char_delimiters = char_delimiters
+    end
+
+    def self.set_default_skip_delimiters(skip_delimiters = [])
+      @@default_skip_delimiters = skip_delimiters
+    end
+
     def self.tokenize(tsql_string)
-      tokens = basic_tokenize(
-        tsql_string,
-        ["(", ",", ")", "=", "+", "-", "%", "/", "*", "<", "!", ">", "'", "[", "]", ";"],
-        [" ", "\n", "\t"]
-      )
-      tokens.map do |t|
-        categorize(t)
-      end
+      Tokenizer.new.basic_tokenize(tsql_string).map { |t| TokenCategorizer.categorize(t) }
     end
 
-    def self.categorize(s)
-      data = {}
-      data[:value] = s
-      data[:keyword] = true if Parser.is_keyword? s
-      data[:operator] = true if Parser.is_operator? s
-      data[:function] = true if Parser.is_function? s
-      data[:type] = true if Parser.is_type? s
-      data[:comment] = true if Parser.is_comment? s
-      data[:numeric] = true if Parser.is_numeric? s
-      data[:special_variable] = true if Parser.is_special_variable? s
-      data[:variable] = true if Parser.is_variable? s
-      data[:temporary_table] = true if Parser.is_temp_table? s
-      data[:label] = true if Parser.is_label? s
-      data[:parenthesis] = true if Parser.is_parenthesis? s
-      data[:open_parenthesis] = true if Parser.is_open_parenthesis? s
-      data[:close_parenthesis] = true if Parser.is_close_parenthesis? s
-      data[:bracket] = true if Parser.is_bracket? s
-      data[:open_bracket] = true if Parser.is_open_bracket? s
-      data[:close_bracket] = true if Parser.is_close_bracket? s
-      data[:string_mark] = true if Parser.is_string_mark? s
-      data[:comma] = true if Parser.is_comma? s
-      data[:join] = true if Parser.is_join? s
-      data[:join_type] = true if Parser.is_join_type? s
-      data[:begin] = true if Parser.is_begin? s
-      data[:end] = true if Parser.is_end? s
-      data[:terminator] = true if Parser.is_terminator? s
-      data[:value] = data[:value].upcase if data[:keyword] or data[:function] or data[:type]
-      data[:needs_newline] = true if data[:keyword] and Parser.is_newline_required? s
-      data
-    end
-
-    def self.basic_tokenize(tsql_string, char_delimiters, skip_delimiters)
-      specific_tokens = []
-      delimiters = ([] << char_delimiters << skip_delimiters).flatten
-      builder = ""
+    def basic_tokenize(tsql_string)
+      self.reset
       tsql_chars = tsql_string.split("")
-      multiline_comment = false
-      comment = false
-      string = false
-      string_count = 0
-      skip_count = 0
+
       tsql_chars.each_with_index do |c, i|
-        if skip_count > 0
-          skip_count -= 1
+        if @skip_count > 0
+          @skip_count -= 1
           next
         end
 
-        next_c = tsql_chars[i + 1] unless i + 1 > tsql_chars.size
+        # get last and next char
+        @c = c
+        @last_c = tsql_chars[i - 1] unless i - 1 < 0
+        @next_c = tsql_chars[i + 1] unless i + 1 > tsql_chars.size
 
-        if Parser.is_multiline_comment_start?(c, next_c)
-          multiline_comment = true
-          specific_tokens << builder unless builder.empty?
-          builder = c
-          next
+        # if we aren't in a string.
+        unless @string
+          next if self.handle_multicomment_start
+          next if self.handle_multicomment_end
+          next if self.handle_singlecomment_start
+          next if self.build_comment
         end
 
-        if Parser.is_multiline_comment_end?(c, next_c)
-          skip_count = 1
-          multiline_comment = false
-          builder << c << next_c
-          specific_tokens << builder unless builder.empty?
-          builder = ""
-          next
+        unless @multiline_comment or @comment
+          next if self.handle_string_char
+          next if self.handle_two_char_op
+          next if self.handle_delimiter
         end
 
-        if Parser.is_comment_start?(c, next_c) and not comment
-          comment = true
-          skip_count = 1
-          specific_tokens << builder unless builder.empty?
-          builder = "--"
-          next
-        end
+        self.build
+      end
 
-        if comment and c != "\n"
-          builder << c
-          next
-        elsif comment and c == "\n"
-          specific_tokens << builder unless builder.empty?
-          builder = ""
-          comment = false
-          next
-        end
+      self.flush_builder
+      @tokens
+    end
 
-        if c == "'" and not multiline_comment and not comment
-          if not string
-            string = true
-            specific_tokens << builder unless builder.empty?
-            builder = c
-            next
-          else
-            string = false
-            builder << c
-            specific_tokens << builder unless builder.empty?
-            builder = ""
-            next
-          end
-        end
+    private
 
-        if Parser.is_two_char_op?(c, next_c)
-          skip_count = 1
-          specific_tokens << builder unless builder.empty?
-          specific_tokens << "#{c}#{next_c}"
-          builder = ""
-          next
-        end
+    def handle_multicomment_start
+      if Parser.is_multiline_comment_start?(@c, @next_c)
+        @multiline_comment = true
+        self.flush_builder(c)
+        return true
+      end
+    end
 
-        if delimiters.include? c and !multiline_comment and !string and !comment
-          specific_tokens << builder unless builder.empty?
-          specific_tokens << c unless skip_delimiters.include? c
-          builder = ""
-          next
-        end
+    def handle_multicomment_end
+      if Parser.is_multiline_comment_end?(@c, @next_c)
+        @skip_count = 1
+        @multiline_comment = false
+        self.build
+        self.build(@next_c)
+        self.flush_builder("")
+        return true
+      end
+    end
+
+    def handle_singlecomment_start
+      if Parser.is_comment_start?(@c, @next_c) and not @comment
+        @comment = true
+        @skip_count = 1
+        self.flush_builder("--")
+        return true
+      end
+    end
 
-        builder << c
+    def handle_singlecomment_end
+      if @c == "\n"
+        self.flush_builder("")
+        @comment = false
+        return true
       end
-      specific_tokens << builder unless builder.empty?
-      specific_tokens
+    end
+
+    def handle_string_char
+      if @c == "'"
+        @string_count += 1
+        return true if self.handle_string_start
+        @string_count += 1 if @next_c == "'"
+        return true if self.handle_string_end
+      end
+    end
+
+    def handle_string_start
+      if not @string
+        @string = true
+        self.flush_builder(@c)
+        return true
+      end
+    end
+
+    def handle_string_end
+      if @string_count % 2 == 0
+        @string = false
+        @string_count = 0
+        self.build
+        self.flush_builder("")
+        return true
+      end
+    end
+
+    def handle_two_char_op
+      if Parser.is_two_char_op?(@c, @next_c)
+        @skip_count = 1
+        self.flush_builder("")
+        @tokens << "#{@c}#{@next_c}"
+        return true
+      end
+    end
+
+    def handle_delimiter
+      if @delimiters.include? @c and !@multiline_comment and !@comment and !@string
+        self.flush_builder("")
+        @tokens << @c unless @skip_delimiters.include? @c
+        return true
+      end
+    end
+
+    def build(value = nil)
+      @builder << @c if value.nil?
+      @builder << value unless value.nil?
+    end
+
+    def build_comment
+      if @comment
+        return true if self.handle_singlecomment_end
+
+        self.build
+        return true
+      end
+    end
+
+    def flush_builder(value = nil)
+      @tokens << @builder unless @builder.empty?
+      @builder = value
+    end
+
+    def initialize
+      self.reset
+    end
+
+    def reset
+      @tokens = []
+      @multiline_comment = false
+      @comment = false
+      @string = false
+      @string_count = 0
+      @skip_count = 0
+      @char_delimiters = @@default_char_delimiters
+      @skip_delimiters = @@default_skip_delimiters
+      @delimiters = ([] << @char_delimiters << @skip_delimiters).flatten.uniq
+      @builder = ""
     end
   end
 end

data/lib/parsing/transformers/token_categorizer.rb

@@ -0,0 +1,52 @@
+#   __               .__
+# _/  |_  ___________|  |           ___________ _______  ______ ___________
+# \   __\/  ___/ ____/  |    ______ \____ \__  \\_  __ \/  ___// __ \_  __ \
+#  |  |  \___ < <_|  |  |__ /_____/ |  |_> > __ \|  | \/\___ \\  ___/|  | \/
+#  |__| /____  >__   |____/         |   __(____  /__|  /____  >\___  >__|
+#            \/   |__|              |__|       \/           \/     \/
+#
+# A very light-weight and opinionated T-SQL parser and formatter.
+#
+# github.com/scstauf
+#
+# path:
+#   parsing/transformers/token_categorizer.rb
+# object:
+#   TSqlParser::Parsing::TokenCategorizer
+
+module TSqlParser::Parsing
+  require_relative "../parser"
+
+  class TokenCategorizer
+    def self.categorize(s)
+      data = {}
+      data[:value] = s
+      data[:keyword] = true if Parser.is_keyword? s
+      data[:operator] = true if Parser.is_operator? s
+      data[:function] = true if Parser.is_function? s
+      data[:type] = true if Parser.is_type? s
+      data[:comment] = true if Parser.is_comment? s
+      data[:numeric] = true if Parser.is_numeric? s
+      data[:special_variable] = true if Parser.is_special_variable? s
+      data[:variable] = true if Parser.is_variable? s
+      data[:temporary_table] = true if Parser.is_temp_table? s
+      data[:label] = true if Parser.is_label? s
+      data[:parenthesis] = true if Parser.is_parenthesis? s
+      data[:open_parenthesis] = true if Parser.is_open_parenthesis? s
+      data[:close_parenthesis] = true if Parser.is_close_parenthesis? s
+      data[:bracket] = true if Parser.is_bracket? s
+      data[:open_bracket] = true if Parser.is_open_bracket? s
+      data[:close_bracket] = true if Parser.is_close_bracket? s
+      data[:string] = true if Parser.is_string? s
+      data[:comma] = true if Parser.is_comma? s
+      data[:join] = true if Parser.is_join? s
+      data[:join_type] = true if Parser.is_join_type? s
+      data[:begin] = true if Parser.is_begin? s
+      data[:end] = true if Parser.is_end? s
+      data[:terminator] = true if Parser.is_terminator? s
+      data[:value] = data[:value].upcase if data[:keyword] or data[:function] or data[:type]
+      data[:needs_newline] = true if data[:keyword] and Parser.is_newline_required? s
+      data
+    end
+  end
+end

data/lib/parsing/transformers/token_transformer.rb

@@ -0,0 +1,99 @@
+#   __               .__
+# _/  |_  ___________|  |           ___________ _______  ______ ___________
+# \   __\/  ___/ ____/  |    ______ \____ \__  \\_  __ \/  ___// __ \_  __ \
+#  |  |  \___ < <_|  |  |__ /_____/ |  |_> > __ \|  | \/\___ \\  ___/|  | \/
+#  |__| /____  >__   |____/         |   __(____  /__|  /____  >\___  >__|
+#            \/   |__|              |__|       \/           \/     \/
+#
+# A very light-weight and opinionated T-SQL parser and formatter.
+#
+# github.com/scstauf
+#
+# path:
+#   parsing/model/token_transformer.rb
+# object:
+#   TSqlParser::Parsing::TokenTransformer
+
+module TSqlParser::Parsing
+  require_relative "../models/sql_container"
+  require_relative "../models/flat_sql_container"
+
+  class TokenTransformer
+    def self.transform(tokens)
+      containers = self.as_containers(tokens)
+      self.combine_containers(containers)
+    end
+
+    private
+
+    def self.as_containers(tokens)
+      containers = []
+      container = nil
+      skip_count = 0
+      tokens.each_with_index do |t, index|
+        if skip_count > 0
+          skip_count -= 1
+          next
+        end
+
+        next_token = tokens[index + 1]
+
+        if t[:string]
+          container.add t unless container.nil?
+        elsif Parser.is_new_node_keyword? t[:value]
+          if not next_token.nil? and Parser.is_new_node_keyword? next_token[:value]
+            if Parser.is_new_node_composite?(t[:value], next_token[:value])
+              containers << container unless container.nil?
+              container = SqlContainer.combine(t, next_token)
+              skip_count = 1
+              next
+            end
+          end
+          containers << container unless container.nil?
+          container = SqlContainer.new(t)
+        elsif t[:value] == "WITH"
+          if not next_token.nil? and not next_token[:keyword] and not next_token[:parenthesis]
+            containers << container unless container.nil?
+            container = SqlContainer.new(t)
+          else
+            container.add t unless container.nil?
+          end
+        elsif t[:label]
+          containers << container unless container.nil?
+          container = SqlContainer.new(t)
+        else
+          container.add t unless container.nil?
+        end
+      end
+      containers << container unless container.nil?
+      FlatSqlContainer.flatten_containers(containers)
+    end
+
+    def self.combine_containers(containers)
+      lines = []
+      containers.each do |c|
+        ct = c.get_token
+
+        builder = []
+        builder << ct[:value]
+
+        if c.has_siblings?
+          c.get_siblings.each do |sibling|
+            st = sibling.get_token
+
+            if st[:comment]
+              builder << "\n#{st[:value]}"
+              next
+            end
+
+            builder << st[:value]
+          end
+        end
+
+        lines << builder.join(" ")
+      end
+
+      lines
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tsql_parser
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - Scott Stauffer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-03-28 00:00:00.000000000 Z
+date: 2023-04-03 00:00:00.000000000 Z
 dependencies: []
 description: A very light-weight and opinionated T-SQL parser and formatter.
 email: scott@fuseraft.com
@@ -25,12 +25,13 @@ files:
 - lib/parsing/formatters/strategy/update_formatter.rb
 - lib/parsing/formatters/strategy/where_formatter.rb
 - lib/parsing/formatters/text_formatter.rb
-- lib/parsing/iterator.rb
 - lib/parsing/keyword.rb
-- lib/parsing/model/flat_sql_container.rb
-- lib/parsing/model/sql_container.rb
+- lib/parsing/models/flat_sql_container.rb
+- lib/parsing/models/sql_container.rb
 - lib/parsing/parser.rb
 - lib/parsing/tokenizer.rb
+- lib/parsing/transformers/token_categorizer.rb
+- lib/parsing/transformers/token_transformer.rb
 - lib/tsql_parser.rb
 homepage: https://rubygems.org/gems/tsql_parser
 licenses:
@@ -53,7 +54,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.5
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: A very light-weight and opinionated T-SQL parser and formatter.

data/lib/parsing/iterator.rb

@@ -1,54 +0,0 @@
-#   __               .__
-# _/  |_  ___________|  |           ___________ _______  ______ ___________
-# \   __\/  ___/ ____/  |    ______ \____ \__  \\_  __ \/  ___// __ \_  __ \
-#  |  |  \___ < <_|  |  |__ /_____/ |  |_> > __ \|  | \/\___ \\  ___/|  | \/
-#  |__| /____  >__   |____/         |   __(____  /__|  /____  >\___  >__|
-#            \/   |__|              |__|       \/           \/     \/
-#
-# A very light-weight and opinionated T-SQL parser and formatter.
-#
-# github.com/scstauf
-#
-# path:
-#   parsing/iterator.rb
-# object:
-#   TSqlParser::Parsing::TokenIterator
-
-module TSqlParser::Parsing
-  class TokenIterator
-    def initialize(tokens)
-      @tokens = tokens
-      @size = tokens.size
-      @iter = -1
-    end
-
-    def has_next?
-      @iter < @size - 1
-    end
-
-    def get!
-      @tokens[@iter]
-    end
-
-    def peek!
-      @tokens[@iter + 1]
-    end
-
-    def peek_ahead!(length)
-      @tokens[@iter + length]
-    end
-
-    def peek_value!
-      self.peek![:value]
-    end
-
-    def peek_ahead_value!(length)
-      self.peek_ahead!(length)[:value]
-    end
-
-    def next!
-      @iter += 1
-      self.get!
-    end
-  end
-end