sql_beautifier 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 74ce2777ba2e1a7635aa92623cbb6e252bef41abafa62fff7b3f9f651aad05af
-  data.tar.gz: 83f543391d4e60f2ab928eb44c6975d4f935b84c7e63bcbb05685331bcdf59a2
+  metadata.gz: ae905b6c37cef236a53ae510743055f7c5fb5c184a1513152920aeaa23b6f79b
+  data.tar.gz: ed5e4c3e0207d1cdf9f600852661fca974bd4f435d1e6c32fd3d471ab46e6288
 SHA512:
-  metadata.gz: 3bbfe00044b32a468fb019a9d32010ce355803000d627d6b16146398b54515531d0a6b76a84022facaec39b1afb849307b08ee96b8859b00ad99f7f2895b2668
-  data.tar.gz: 2252b3c944667589b17c0bde79e2eca176ced558c6bf8d19795607490788dc468200a00a2dedbce667adee966956389b3957f70cc39fa317d865435ceee22007
+  metadata.gz: ef28e0c0853fb740d677be87061c284aeae9c8e5f0ed62f4d34417577d58c5ac87bcc0072c88a6f8bcfdac52a4cae3ff335cf8f1de07a9d1c7b3ba67533ed820
+  data.tar.gz: 3f8f35358960a0c0015026251799621ee0a775aaa066a4dc53de36c4a973920f0f76052cfced812a861e4b372e88ca22430c432f77c1537642d6303c0468501a

data/CHANGELOG.md

@@ -2,6 +2,22 @@
 
 ## [X.X.X] - YYYY-MM-DD
 
+## [0.6.0] - 2026-03-28
+
+- **Breaking**: comments are now preserved by default. Set `removable_comment_types = :all` to restore previous behavior of stripping all comments
+- Add `removable_comment_types` configuration option (default: `:none`) — controls which SQL comment types are stripped during formatting. Accepts `:none`, `:all`, or an array of specific types (`:inline`, `:separate_line`, `:blocks`)
+- Add multi-statement support — input containing multiple statements (separated by `;` or concatenated) is split and formatted independently
+- Add `trailing_semicolon` configuration option (default: `true`) — automatically appends `;` to each formatted statement
+- Add per-call configuration overrides via `SqlBeautifier.call(value, trailing_semicolon: false)` — overrides take precedence over global config for the duration of the call
+- Change `inline_group_threshold` default from `100` to `0` — parenthesized condition groups are now always expanded to multiple lines
+- Fix `StatementSplitter` incorrectly splitting `INSERT INTO ... SELECT` as two separate statements
+- Fix inline comments after a trailing semicolon (e.g. `SELECT 1; -- done`) being silently dropped during formatting
+- Fix infinite loop in `Normalizer#consume_sentinel!` when a malformed sentinel prefix has no closing `*/`
+- Fix `CommentStripper#resolve_removal_set` returning `nil` for unrecognized `removable_types` values — now raises `ArgumentError` with a descriptive message
+- Fix `CommentStripper#resolve_removal_set` silently accepting invalid entries in Array-typed `removable_types` (e.g. `[:inlne]`) — now validates each element against known comment types
+- Fix `CommentStripper` not inserting token-separating whitespace around sentinels when preserving block comments between adjacent tokens (e.g. `SELECT/*comment*/id`)
+- Strengthen end-to-end specs with exact full-output assertions and add coverage for JOINs, subqueries, CTEs, CREATE TABLE AS, DISTINCT, complex WHERE conditions, and configuration variations
+
 ## [0.5.0] - 2026-03-28
 
 - Add support for Create Table As (CTA) formatting

data/README.md

@@ -45,7 +45,7 @@ from    Users u
 
 where   active = true
 
-order by name
+order by name;
 ```
 
 Single-word keywords are lowercased and padded so their clause bodies start at an 8-character column. Multi-word clauses such as `order by` and `group by`, and short clauses like `limit`, use a single space between the keyword and the clause body instead of padding. Clause spacing is compact by default for simple one-column / one-table / one-condition queries, and otherwise uses blank lines between top-level clauses. Multi-column SELECT lists place each column on its own line with continuation indentation. Table names are PascalCased and automatically aliased.
@@ -74,7 +74,7 @@ select  u.id,
 
 from    Users u
 
-where   u.active = true
+where   u.active = true;
 ```
 
 When two tables produce the same initials, a counter is appended for disambiguation (e.g. `u1`, `u2`).
@@ -108,7 +108,7 @@ from    Users u
 where   u.active = true
         and o.total > 100
 
-order by o.total desc
+order by o.total desc;
 ```
 
 Supported join types: `inner join`, `left join`, `right join`, `full join`, `left outer join`, `right outer join`, `full outer join`, `cross join`.
@@ -129,7 +129,7 @@ select  distinct
         name,
         email
 
-from    Users u
+from    Users u;
 ```
 
 `DISTINCT ON` preserves the full expression:
@@ -145,7 +145,7 @@ select  distinct on (user_id)
         id,
         name
 
-from    Events e
+from    Events e;
 ```
 
 ### WHERE and HAVING Conditions
@@ -165,10 +165,10 @@ from    Users u
 
 where   active = true
         and role = 'admin'
-        and created_at > '2024-01-01'
+        and created_at > '2024-01-01';
 ```
 
-Short parenthesized groups stay inline:
+Parenthesized condition groups are expanded to multiple lines with indentation:
 
 ```ruby
 SqlBeautifier.call("SELECT * FROM users WHERE active = true AND (role = 'admin' OR role = 'moderator')")
@@ -182,7 +182,10 @@ select  *
 from    Users u
 
 where   active = true
-        and (role = 'admin' or role = 'moderator')
+        and (
+            role = 'admin'
+            or role = 'moderator'
+        );
 ```
 
 ### GROUP BY and HAVING
@@ -206,7 +209,7 @@ from    Users u
 
 group by status
 
-having  count(*) > 5
+having  count(*) > 5;
 ```
 
 ### LIMIT
@@ -221,7 +224,7 @@ Produces:
 select  id
 from    Users u
 order by created_at desc
-limit 25
+limit 25;
 ```
 
 ### String Literals
@@ -240,7 +243,7 @@ select  *
 from    Users u
 
 where   name = 'O''Brien'
-        and status = 'Active'
+        and status = 'Active';
 ```
 
 ### Double-Quoted Identifiers
@@ -257,7 +260,7 @@ Produces:
 select  user_id,
         full_name
 
-from    Users u
+from    Users u;
 ```
 
 ### Subqueries
@@ -277,17 +280,17 @@ where   id in (
             select  user_id
             from    Orders o
             where   total > 100
-        )
+        );
 ```
 
 Nested subqueries increase indentation at each level.
 
-### Comments and Semicolons
+### Trailing Semicolons
 
-SQL comments (`--` line comments and `/* */` block comments) and trailing semicolons are automatically stripped during normalization. Comments inside string literals are preserved:
+By default, each formatted statement ends with a `;`:
 
 ```ruby
-SqlBeautifier.call("SELECT id /* primary key */ FROM users -- main table\nWHERE active = true;")
+SqlBeautifier.call("SELECT id FROM users WHERE active = true")
 ```
 
 Produces:
@@ -295,45 +298,141 @@ Produces:
 ```sql
 select  id
 from    Users u
-where   active = true
+where   active = true;
+```
+
+Disable with `config.trailing_semicolon = false` to omit the trailing `;`.
+
+### Multiple Statements
+
+Input containing multiple SQL statements is split and formatted independently. Statements can be separated by `;` or simply concatenated:
+
+```ruby
+SqlBeautifier.call("SELECT id FROM constituents; SELECT id FROM departments")
 ```
 
+Produces:
+
+```sql
+select  id
+from    Constituents c;
+
+select  id
+from    Departments d;
+```
+
+Concatenated statements without `;` are also detected:
+
+```ruby
+SqlBeautifier.call("SELECT id FROM constituents SELECT id FROM departments")
+```
+
+Produces the same output. Subqueries and CTE bodies are not mistakenly split.
+
+### Comments
+
+By default, SQL comments are preserved in formatted output. Line comments (`--`) and block comments (`/* */`) are classified by position and passed through formatting:
+
+```ruby
+SqlBeautifier.call("-- Base Query\nSELECT id /* primary key */ FROM users WHERE active = true")
+```
+
+Produces:
+
+```sql
+-- Base Query
+select  id /* primary key */
+from    Users u
+where   active = true;
+```
+
+Configure `removable_comment_types` to control which comment types are stripped. See the `removable_comment_types` configuration option for details. Comments inside string literals are always preserved regardless of configuration.
+
 ### Configuration
 
 Customize formatting behavior with `SqlBeautifier.configure`:
 
 ```ruby
 SqlBeautifier.configure do |config|
-  config.keyword_case = :upper           # :lower (default), :upper
-  config.keyword_column_width = 10       # default: 8
-  config.indent_spaces = 4               # default: 4
-  config.clause_spacing_mode = :spacious # :compact (default), :spacious
-  config.table_name_format = :lowercase  # :pascal_case (default), :lowercase
-  config.inline_group_threshold = 80     # default: 100
-  config.alias_strategy = :none          # :initials (default), :none, or a callable
+  config.keyword_case = :upper
+  config.keyword_column_width = 10
+  config.indent_spaces = 4
+  config.clause_spacing_mode = :spacious
+  config.table_name_format = :lowercase
+  config.inline_group_threshold = 80
+  config.alias_strategy = :none
+  config.trailing_semicolon = false
+  config.removable_comment_types = :all
 end
 ```
 
-#### Clause Spacing Modes
-
-- `:compact` (default) keeps top-level clauses on single newlines only when the query is simple:
-  - exactly one SELECT column
-  - exactly one FROM table (no JOINs)
-  - zero or one top-level WHERE condition
-  - only `select`, `from`, optional `where`, optional `order by`, and optional `limit`
-- `:spacious` always separates top-level clauses with blank lines
-
 Reset to defaults:
 
 ```ruby
 SqlBeautifier.reset_configuration!
 ```
 
-#### Alias Strategies
+#### Per-Call Overrides
 
-- `:initials` (default) — automatic aliases using table initials (`users` → `u`, `active_storage_blobs` → `asb`)
-- `:none` — no automatic aliases (explicit aliases in the SQL are still preserved)
-- Callable — provide a proc/lambda for custom alias generation:
+Pass configuration overrides directly to `SqlBeautifier.call` to override global settings for a single invocation:
+
+```ruby
+SqlBeautifier.call(query, trailing_semicolon: false, keyword_case: :upper)
+```
+
+Per-call overrides take precedence over the global `SqlBeautifier.configure` block. Any keys not included in the override hash fall back to the global configuration. The global configuration is never mutated. Unknown keys raise `ArgumentError`.
+
+#### `keyword_case`
+
+Controls the case of SQL keywords in the output. Default: `:lower`.
+
+- `:lower` — lowercases all keywords (`select`, `from`, `where`, `inner join`, etc.)
+- `:upper` — uppercases all keywords (`SELECT`, `FROM`, `WHERE`, `INNER JOIN`, etc.)
+
+#### `keyword_column_width`
+
+Sets the column width for single-word keyword alignment. Keywords shorter than this width are right-padded with spaces so clause bodies start at this column position. Default: `8`.
+
+For example, with the default width of 8, `select` (6 chars) gets 2 spaces of padding, `where` (5 chars) gets 3 spaces, and `from` (4 chars) gets 4 spaces. Multi-word keywords like `order by` and `group by` use a single space instead of padding.
+
+#### `indent_spaces`
+
+Number of spaces used for indentation within subqueries and CTE bodies. Each nesting level adds this many spaces of indentation. Default: `4`.
+
+#### `clause_spacing_mode`
+
+Controls whether blank lines are inserted between top-level clauses. Default: `:compact`.
+
+- `:compact` — omits blank lines when the query is simple (single SELECT column, single FROM table with no JOINs, at most one WHERE condition, and only basic clauses like `select`, `from`, `where`, `order by`, `limit`). Complex queries automatically get blank lines regardless.
+- `:spacious` — always inserts blank lines between every top-level clause.
+
+#### `table_name_format`
+
+Controls how table names are formatted in the output. Default: `:pascal_case`.
+
+- `:pascal_case` — capitalizes each underscore-separated segment (`users` → `Users`, `active_storage_blobs` → `Active_Storage_Blobs`)
+- `:lowercase` — keeps table names lowercase as-is
+
+#### `inline_group_threshold`
+
+Maximum character length for a parenthesized condition group to remain on a single line. Groups whose inline representation exceeds this length are expanded to multiple lines with indented conditions. Default: `0` (always expand).
+
+Set to a positive integer to allow short groups to stay inline. For example, with a threshold of `80`, the group `(role = 'admin' or role = 'moderator')` would stay on one line since it's under 80 characters.
+
+#### `trailing_semicolon`
+
+Controls whether a trailing `;` is appended to each formatted statement. Default: `true`.
+
+- `true` — appends `;` at the end of each statement
+- `false` — omits the trailing `;`
+
+#### `alias_strategy`
+
+Controls automatic table aliasing in FROM and JOIN clauses. Default: `:initials`.
+
+- `:initials` — generates aliases from the first letter of each underscore-separated segment (`users` → `u`, `active_storage_blobs` → `asb`). When two tables produce the same initials, a counter is appended for disambiguation (`u1`, `u2`). All `table.column` references throughout the query are replaced with `alias.column`.
+- `:none` — disables automatic aliasing. Explicit aliases written in the SQL are still preserved.
+- Callable — provide a proc/lambda that receives the table name and returns a custom alias string:
 
 ```ruby
 SqlBeautifier.configure do |config|
@@ -341,6 +440,26 @@ SqlBeautifier.configure do |config|
 end
 ```
 
+#### `removable_comment_types`
+
+Controls which SQL comment types are stripped during formatting. Default: `:none`.
+
+- `:none` — preserves all comments in the formatted output
+- `:all` — strips all comments (equivalent to `[:inline, :separate_line, :blocks]`)
+- Array of specific types — strips only the listed types, preserving the rest
+
+The three comment types:
+
+- `:separate_line` — `--` comments on their own line (only whitespace before `--`), including banner-style dividers
+- `:inline` — `--` comments at the end of a line that contains SQL
+- `:blocks` — `/* ... */` block comments (single or multi-line)
+
+```ruby
+SqlBeautifier.configure do |config|
+  config.removable_comment_types = [:inline, :blocks]
+end
+```
+
 ### Callable Interface
 
 `SqlBeautifier.call` is the public API, making it a valid callable for Rails `normalizes` and anywhere a proc-like object is expected:

data/lib/sql_beautifier/clauses/from.rb

@@ -141,9 +141,15 @@ module SqlBeautifier
         table_name = Util.first_word(table_text)
         formatted_table_name = Util.format_table_name(table_name)
         table_alias = @table_registry.alias_for(table_name)
-        return formatted_table_name unless table_alias
+        trailing_sentinels = extract_trailing_sentinels(table_text)
 
-        "#{formatted_table_name} #{table_alias}"
+        formatted = table_alias ? "#{formatted_table_name} #{table_alias}" : formatted_table_name
+        trailing_sentinels.empty? ? formatted : "#{formatted} #{trailing_sentinels}"
+      end
+
+      def extract_trailing_sentinels(text)
+        sentinels = text.scan(CommentStripper::SENTINEL_PATTERN).map { |match| "#{CommentStripper::SENTINEL_PREFIX}#{match[0]}#{CommentStripper::SENTINEL_SUFFIX}" }
+        sentinels.join(" ")
       end
     end
   end

data/lib/sql_beautifier/comment_restorer.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module SqlBeautifier
+  class CommentRestorer
+    def self.call(formatted_sql, comment_map)
+      return formatted_sql if comment_map.empty?
+
+      new(formatted_sql, comment_map).call
+    end
+
+    def initialize(formatted_sql, comment_map)
+      @formatted_sql = formatted_sql
+      @comment_map = comment_map
+    end
+
+    def call
+      result = @formatted_sql
+
+      @comment_map.each do |index, entry|
+        sentinel = "#{CommentStripper::SENTINEL_PREFIX}#{index}#{CommentStripper::SENTINEL_SUFFIX}"
+
+        result = begin
+          case entry[:type]
+          when :blocks
+            restore_block_comment(result, sentinel, entry[:text])
+          when :separate_line
+            restore_separate_line_comment(result, sentinel, entry[:text])
+          when :inline
+            restore_inline_comment(result, sentinel, entry[:text])
+          else
+            result
+          end
+        end
+      end
+
+      result
+    end
+
+    private
+
+    def restore_block_comment(sql, sentinel, comment_text)
+      sql.sub(sentinel, comment_text)
+    end
+
+    def restore_separate_line_comment(sql, sentinel, comment_text)
+      sql.sub(%r{#{Regexp.escape(sentinel)}[ \n]?}, "#{comment_text}\n")
+    end
+
+    def restore_inline_comment(sql, sentinel, comment_text)
+      pattern = %r{ ?#{Regexp.escape(sentinel)}([^\n]*)}
+      sql.sub(pattern) do
+        trailing_content = Regexp.last_match(1)
+
+        if trailing_content.strip.empty?
+          " #{comment_text}"
+        else
+          "#{trailing_content.strip} #{comment_text}"
+        end
+      end
+    end
+  end
+end

data/lib/sql_beautifier/comment_stripper.rb

@@ -0,0 +1,222 @@
+# frozen_string_literal: true
+
+module SqlBeautifier
+  class CommentStripper
+    SENTINEL_PREFIX = "/*__sqlb_"
+    SENTINEL_SUFFIX = "__*/"
+    SENTINEL_PATTERN = %r{/\*__sqlb_(\d+)__\*/}
+
+    Result = Struct.new(:stripped_sql, :comment_map)
+
+    def self.call(sql, removable_types)
+      new(sql, removable_types).call
+    end
+
+    def initialize(sql, removable_types)
+      @sql = sql
+      @removal_set = resolve_removal_set(removable_types)
+      @output = +""
+      @comment_map = {}
+      @sentinel_index = 0
+      @position = 0
+      @pending_separate_line_comments = []
+    end
+
+    def call
+      while @position < @sql.length
+        character = @sql[@position]
+
+        if @in_single_quoted_string
+          consume_single_quoted_character!(character)
+        elsif @in_double_quoted_identifier
+          consume_double_quoted_character!(character)
+        elsif character == Constants::SINGLE_QUOTE
+          flush_pending_separate_line_comments!
+          @in_single_quoted_string = true
+          @output << character
+          @position += 1
+        elsif character == Constants::DOUBLE_QUOTE
+          flush_pending_separate_line_comments!
+          @in_double_quoted_identifier = true
+          @output << character
+          @position += 1
+        elsif line_comment_start?
+          handle_line_comment!
+        elsif block_comment_start?
+          flush_pending_separate_line_comments!
+          handle_block_comment!
+        else
+          flush_pending_separate_line_comments! unless character == "\n"
+          @output << character
+          @position += 1
+        end
+      end
+
+      flush_pending_separate_line_comments!
+
+      Result.new(@output, @comment_map)
+    end
+
+    private
+
+    def resolve_removal_set(removable_types)
+      case removable_types
+      when :none
+        []
+      when :all
+        Configuration::COMMENT_TYPES.dup
+      when Array
+        invalid_types = removable_types - Configuration::COMMENT_TYPES
+        raise ArgumentError, "Unsupported removable_types entries: #{invalid_types.inspect}. Expected elements of #{Configuration::COMMENT_TYPES.inspect}" if invalid_types.any?
+
+        removable_types
+      when *Configuration::COMMENT_TYPES
+        [removable_types]
+      else
+        raise ArgumentError, "Unsupported removable_types: #{removable_types.inspect}. Expected :none, :all, an Array, or one of #{Configuration::COMMENT_TYPES.inspect}"
+      end
+    end
+
+    def consume_single_quoted_character!(character)
+      @output << character
+
+      if character == Constants::SINGLE_QUOTE && @sql[@position + 1] == Constants::SINGLE_QUOTE
+        @position += 1
+        @output << @sql[@position]
+      elsif character == Constants::SINGLE_QUOTE
+        @in_single_quoted_string = false
+      end
+
+      @position += 1
+    end
+
+    def consume_double_quoted_character!(character)
+      @output << character
+
+      if character == Constants::DOUBLE_QUOTE && @sql[@position + 1] == Constants::DOUBLE_QUOTE
+        @position += 1
+        @output << @sql[@position]
+      elsif character == Constants::DOUBLE_QUOTE
+        @in_double_quoted_identifier = false
+      end
+
+      @position += 1
+    end
+
+    def line_comment_start?
+      @sql[@position] == "-" && @sql[@position + 1] == "-"
+    end
+
+    def block_comment_start?
+      @sql[@position] == "/" && @sql[@position + 1] == "*"
+    end
+
+    def handle_line_comment!
+      comment_type = separate_line_comment? ? :separate_line : :inline
+      comment_text = extract_line_comment_text
+
+      if removable?(comment_type)
+        strip_line_comment!
+      else
+        preserve_line_comment!(comment_type, comment_text)
+      end
+    end
+
+    def separate_line_comment?
+      line_start = @output.rindex("\n")
+      preceding_content = begin
+        if line_start
+          @output[(line_start + 1)..]
+        else
+          @output
+        end
+      end
+
+      preceding_content.match?(%r{\A[[:space:]]*\z})
+    end
+
+    def extract_line_comment_text
+      start_position = @position
+      @position += 2
+
+      @position += 1 while @position < @sql.length && @sql[@position] != "\n"
+
+      @sql[start_position...@position]
+    end
+
+    def strip_line_comment!; end
+
+    def preserve_line_comment!(comment_type, comment_text)
+      if comment_type == :separate_line
+        @pending_separate_line_comments << comment_text
+        @position += 1 if @position < @sql.length && @sql[@position] == "\n"
+      else
+        flush_pending_separate_line_comments!
+        sentinel = build_sentinel(comment_type, comment_text)
+        @output << sentinel
+      end
+    end
+
+    def handle_block_comment!
+      comment_text = extract_block_comment_text
+
+      if removable?(:blocks)
+        strip_block_comment!
+      else
+        sentinel = build_sentinel(:blocks, comment_text)
+        @output << " " unless @output.empty? || @output[-1] =~ Constants::WHITESPACE_CHARACTER_REGEX
+        @output << sentinel
+
+        next_character = @sql[@position]
+        @output << " " if next_character && next_character !~ Constants::WHITESPACE_CHARACTER_REGEX
+      end
+    end
+
+    def extract_block_comment_text
+      start_position = @position
+      @position += 2
+
+      while @position < @sql.length
+        if @sql[@position] == "*" && @sql[@position + 1] == "/"
+          @position += 2
+          break
+        end
+
+        @position += 1
+      end
+
+      @sql[start_position...@position]
+    end
+
+    def strip_block_comment!
+      @output << " " unless @output.empty? || @output[-1] =~ Constants::WHITESPACE_CHARACTER_REGEX
+    end
+
+    def flush_pending_separate_line_comments!
+      return if @pending_separate_line_comments.empty?
+
+      grouped_text = @pending_separate_line_comments.join("\n")
+      sentinel = build_sentinel(:separate_line, grouped_text)
+
+      @output << sentinel
+      @output << "\n"
+
+      @pending_separate_line_comments.clear
+    end
+
+    def build_sentinel(comment_type, comment_text)
+      index = @sentinel_index
+      @sentinel_index += 1
+      @comment_map[index] = {
+        type: comment_type,
+        text: comment_text,
+      }
+
+      "#{SENTINEL_PREFIX}#{index}#{SENTINEL_SUFFIX}"
+    end
+
+    def removable?(comment_type)
+      @removal_set.include?(comment_type)
+    end
+  end
+end

data/lib/sql_beautifier/configuration.rb

@@ -8,10 +8,14 @@ module SqlBeautifier
       indent_spaces: 4,
       clause_spacing_mode: :compact,
       table_name_format: :pascal_case,
-      inline_group_threshold: 100,
+      inline_group_threshold: 0,
       alias_strategy: :initials,
+      trailing_semicolon: true,
+      removable_comment_types: :none,
     }.freeze
 
+    COMMENT_TYPES = %i[inline separate_line blocks].freeze
+
     attr_accessor :keyword_case
     attr_accessor :keyword_column_width
     attr_accessor :indent_spaces
@@ -19,6 +23,8 @@ module SqlBeautifier
     attr_accessor :table_name_format
     attr_accessor :inline_group_threshold
     attr_accessor :alias_strategy
+    attr_accessor :trailing_semicolon
+    attr_accessor :removable_comment_types
 
     def initialize
       reset!

data/lib/sql_beautifier/formatter.rb

@@ -17,14 +17,17 @@ module SqlBeautifier
       @normalized_value = Normalizer.call(@value)
       return unless @normalized_value.present?
 
+      @leading_sentinels = extract_leading_sentinels!
+      return unless @normalized_value.present?
+
       cte_result = CteFormatter.format(@normalized_value, depth: @depth)
-      return cte_result if cte_result
+      return prepend_sentinels(cte_result) if cte_result
 
       create_table_as_result = CreateTableAsFormatter.format(@normalized_value, depth: @depth)
-      return create_table_as_result if create_table_as_result
+      return prepend_sentinels(create_table_as_result) if create_table_as_result
 
       first_clause_position = Tokenizer.first_clause_position(@normalized_value)
-      return "#{@normalized_value}\n" if first_clause_position.nil? || first_clause_position.positive?
+      return prepend_sentinels("#{@normalized_value}\n") if first_clause_position.nil? || first_clause_position.positive?
 
       @clauses = Tokenizer.split_into_clauses(@normalized_value)
       @table_registry = TableRegistry.new(@clauses[:from]) if @clauses[:from].present?
@@ -39,15 +42,36 @@ module SqlBeautifier
       append_clause!(:limit, Clauses::Limit)
 
       output = @parts.join(clause_separator)
-      return "#{@normalized_value}\n" if output.empty?
+      return prepend_sentinels("#{@normalized_value}\n") if output.empty?
 
       output = SubqueryFormatter.format(output, @depth)
       output = @table_registry.apply_aliases(output) if @table_registry
-      "#{output}\n"
+      prepend_sentinels("#{output}\n")
     end
 
     private
 
+    def extract_leading_sentinels!
+      leading_sentinel_text = +""
+      remaining_value = @normalized_value
+
+      while remaining_value.match?(%r{\A#{CommentStripper::SENTINEL_PATTERN}[[:space:]]*})
+        match = remaining_value.match(%r{\A(#{CommentStripper::SENTINEL_PATTERN}[[:space:]]*)})
+        leading_sentinel_text << match[1]
+        remaining_value = remaining_value[match[1].length..]
+      end
+
+      @normalized_value = remaining_value
+
+      leading_sentinel_text
+    end
+
+    def prepend_sentinels(output)
+      return output if @leading_sentinels.empty?
+
+      "#{@leading_sentinels}#{output}"
+    end
+
     def append_clause!(clause_key, formatter_class)
       value = @clauses[clause_key]
       return unless value.present?

data/lib/sql_beautifier/normalizer.rb

@@ -18,7 +18,6 @@ module SqlBeautifier
       @source = @value.strip
       return unless @source.present?
 
-      @source = strip_comments(@source)
       @source = strip_trailing_semicolons(@source)
       @source = @source.strip
       return unless @source.present?
@@ -34,6 +33,14 @@ module SqlBeautifier
         when Constants::DOUBLE_QUOTE
           consume_quoted_identifier!
 
+        when "/"
+          if sentinel_at_position?
+            consume_sentinel!
+          else
+            @output << current_character.downcase
+            @position += 1
+          end
+
         when Constants::WHITESPACE_CHARACTER_REGEX
           collapse_whitespace!
 
@@ -52,6 +59,24 @@ module SqlBeautifier
       @source[@position]
     end
 
+    def sentinel_at_position?
+      @source[@position, CommentStripper::SENTINEL_PREFIX.length] == CommentStripper::SENTINEL_PREFIX
+    end
+
+    def consume_sentinel!
+      sentinel_end = @source.index("*/", @position + CommentStripper::SENTINEL_PREFIX.length)
+
+      unless sentinel_end
+        @output << current_character.downcase
+        @position += 1
+        return
+      end
+
+      end_position = sentinel_end + 2
+      @output << @source[@position...end_position]
+      @position = end_position
+    end
+
     def collapse_whitespace!
       @output << " "
       @position += 1
@@ -119,67 +144,5 @@ module SqlBeautifier
     def strip_trailing_semicolons(sql)
       sql.sub(%r{;[[:space:]]*\z}, "")
     end
-
-    def strip_comments(sql)
-      output = +""
-      position = 0
-      in_single_quoted_string = false
-      in_double_quoted_identifier = false
-
-      while position < sql.length
-        character = sql[position]
-
-        if in_single_quoted_string
-          output << character
-
-          if character == Constants::SINGLE_QUOTE && sql[position + 1] == Constants::SINGLE_QUOTE
-            position += 1
-            output << sql[position]
-          elsif character == Constants::SINGLE_QUOTE
-            in_single_quoted_string = false
-          end
-
-          position += 1
-          next
-        end
-
-        if in_double_quoted_identifier
-          output << character
-
-          if character == Constants::DOUBLE_QUOTE && sql[position + 1] == Constants::DOUBLE_QUOTE
-            position += 1
-            output << sql[position]
-          elsif character == Constants::DOUBLE_QUOTE
-            in_double_quoted_identifier = false
-          end
-
-          position += 1
-          next
-        end
-
-        if character == Constants::SINGLE_QUOTE
-          in_single_quoted_string = true
-          output << character
-          position += 1
-        elsif character == Constants::DOUBLE_QUOTE
-          in_double_quoted_identifier = true
-          output << character
-          position += 1
-        elsif character == "-" && sql[position + 1] == "-"
-          position += 2
-          position += 1 while position < sql.length && sql[position] != "\n"
-        elsif character == "/" && sql[position + 1] == "*"
-          output << " " unless output.empty? || output[-1] =~ Constants::WHITESPACE_CHARACTER_REGEX
-          position += 2
-          position += 1 while position < sql.length && !(sql[position] == "*" && sql[position + 1] == "/")
-          position += 2 if position < sql.length
-        else
-          output << character
-          position += 1
-        end
-      end
-
-      output
-    end
   end
 end

data/lib/sql_beautifier/statement_assembler.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module SqlBeautifier
+  class StatementAssembler
+    def self.call(value)
+      new(value).call
+    end
+
+    def initialize(value)
+      @value = value
+    end
+
+    def call
+      removable_types = SqlBeautifier.config_for(:removable_comment_types)
+      comment_result = CommentStripper.call(@value, removable_types)
+
+      statements = StatementSplitter.split(comment_result.stripped_sql)
+      formatted_statements = statements.filter_map do |statement|
+        Formatter.call(statement)&.chomp
+      end
+      return if formatted_statements.empty?
+
+      trailing_semicolon = SqlBeautifier.config_for(:trailing_semicolon)
+      separator = trailing_semicolon ? ";\n\n" : "\n\n"
+      terminator = trailing_semicolon ? ";\n" : "\n"
+
+      output = formatted_statements.join(separator) + terminator
+      CommentRestorer.call(output, comment_result.comment_map)
+    end
+  end
+end

data/lib/sql_beautifier/statement_splitter.rb

@@ -0,0 +1,269 @@
+# frozen_string_literal: true
+
+module SqlBeautifier
+  module StatementSplitter
+    STATEMENT_KEYWORDS = %w[select with create insert update delete].freeze
+    BOUNDARY_KEYWORDS = %w[from where having limit into set values].freeze
+    CONTINUATION_PAIRS = { "insert" => "select" }.freeze
+
+    module_function
+
+    def split(sql)
+      semicolon_chunks = split_on_semicolons(sql)
+      statements = semicolon_chunks.flat_map { |chunk| split_concatenated_statements(chunk) }.reject(&:empty?)
+      merge_trailing_sentinel_segments(statements)
+    end
+
+    def split_on_semicolons(sql)
+      segments = []
+      current_segment = +""
+      inside_string_literal = false
+      inside_quoted_identifier = false
+      inside_dollar_quoted_string = false
+      dollar_quote_delimiter = nil
+      parenthesis_depth = 0
+      position = 0
+
+      while position < sql.length
+        character = sql[position]
+
+        if inside_string_literal
+          current_segment << character
+
+          if character == Constants::SINGLE_QUOTE && sql[position + 1] == Constants::SINGLE_QUOTE
+            position += 1
+            current_segment << sql[position]
+          elsif character == Constants::SINGLE_QUOTE
+            inside_string_literal = false
+          end
+
+          position += 1
+          next
+        end
+
+        if inside_dollar_quoted_string
+          if sql[position, dollar_quote_delimiter.length] == dollar_quote_delimiter
+            current_segment << dollar_quote_delimiter
+            position += dollar_quote_delimiter.length
+            inside_dollar_quoted_string = false
+            dollar_quote_delimiter = nil
+          else
+            current_segment << character
+            position += 1
+          end
+          next
+        end
+
+        if inside_quoted_identifier
+          current_segment << character
+
+          if character == Constants::DOUBLE_QUOTE && sql[position + 1] == Constants::DOUBLE_QUOTE
+            position += 1
+            current_segment << sql[position]
+          elsif character == Constants::DOUBLE_QUOTE
+            inside_quoted_identifier = false
+          end
+
+          position += 1
+          next
+        end
+
+        if Tokenizer.sentinel_at?(sql, position)
+          end_position = Tokenizer.sentinel_end_position(sql, position)
+          current_segment << sql[position...end_position]
+          position = end_position
+          next
+        end
+
+        delimiter = Tokenizer.dollar_quote_delimiter_at(sql, position)
+
+        if delimiter
+          inside_dollar_quoted_string = true
+          dollar_quote_delimiter = delimiter
+          current_segment << delimiter
+          position += delimiter.length
+          next
+        end
+
+        case character
+        when Constants::SINGLE_QUOTE
+          inside_string_literal = true
+          current_segment << character
+        when Constants::DOUBLE_QUOTE
+          inside_quoted_identifier = true
+          current_segment << character
+        when Constants::OPEN_PARENTHESIS
+          parenthesis_depth += 1
+          current_segment << character
+        when Constants::CLOSE_PARENTHESIS
+          parenthesis_depth = [parenthesis_depth - 1, 0].max
+          current_segment << character
+        when ";"
+          if parenthesis_depth.zero?
+            stripped_segment = current_segment.strip
+            segments << stripped_segment unless stripped_segment.empty?
+            current_segment = +""
+          else
+            current_segment << character
+          end
+        else
+          current_segment << character
+        end
+
+        position += 1
+      end
+
+      stripped_segment = current_segment.strip
+      segments << stripped_segment unless stripped_segment.empty?
+      segments
+    end
+
+    def split_concatenated_statements(sql)
+      boundaries = detect_statement_boundaries(sql)
+      return [sql.strip] if boundaries.length <= 1
+
+      statements = []
+
+      boundaries.each_with_index do |boundary_position, index|
+        end_position = begin
+          if index + 1 < boundaries.length
+            boundaries[index + 1]
+          else
+            sql.length
+          end
+        end
+
+        statement = sql[boundary_position...end_position].strip
+        statements << statement unless statement.empty?
+      end
+
+      statements
+    end
+
+    def detect_statement_boundaries(sql)
+      boundaries = []
+      clause_seen = false
+      current_statement_keyword = nil
+      inside_string_literal = false
+      inside_quoted_identifier = false
+      inside_dollar_quoted_string = false
+      dollar_quote_delimiter = nil
+      parenthesis_depth = 0
+      position = 0
+
+      while position < sql.length
+        character = sql[position]
+
+        if inside_string_literal
+          if character == Constants::SINGLE_QUOTE && sql[position + 1] == Constants::SINGLE_QUOTE
+            position += 2
+          else
+            inside_string_literal = false if character == Constants::SINGLE_QUOTE
+            position += 1
+          end
+          next
+        end
+
+        if inside_dollar_quoted_string
+          if sql[position, dollar_quote_delimiter.length] == dollar_quote_delimiter
+            position += dollar_quote_delimiter.length
+            inside_dollar_quoted_string = false
+            dollar_quote_delimiter = nil
+          else
+            position += 1
+          end
+          next
+        end
+
+        if inside_quoted_identifier
+          if character == Constants::DOUBLE_QUOTE && sql[position + 1] == Constants::DOUBLE_QUOTE
+            position += 2
+          else
+            inside_quoted_identifier = false if character == Constants::DOUBLE_QUOTE
+            position += 1
+          end
+          next
+        end
+
+        if Tokenizer.sentinel_at?(sql, position)
+          position = Tokenizer.sentinel_end_position(sql, position)
+          next
+        end
+
+        delimiter = Tokenizer.dollar_quote_delimiter_at(sql, position)
+
+        if delimiter
+          inside_dollar_quoted_string = true
+          dollar_quote_delimiter = delimiter
+          position += delimiter.length
+          next
+        end
+
+        case character
+        when Constants::SINGLE_QUOTE
+          inside_string_literal = true
+        when Constants::DOUBLE_QUOTE
+          inside_quoted_identifier = true
+        when Constants::OPEN_PARENTHESIS
+          parenthesis_depth += 1
+        when Constants::CLOSE_PARENTHESIS
+          parenthesis_depth = [parenthesis_depth - 1, 0].max
+        else
+          if parenthesis_depth.zero?
+            matched_statement_keyword = keyword_match_at(sql, position, STATEMENT_KEYWORDS)
+
+            if matched_statement_keyword
+              if clause_seen && !continuation_keyword?(current_statement_keyword, matched_statement_keyword)
+                boundaries << position
+                clause_seen = false
+                current_statement_keyword = matched_statement_keyword
+              elsif boundaries.empty?
+                boundaries << position
+                current_statement_keyword = matched_statement_keyword
+              end
+
+              position += matched_statement_keyword.length
+              next
+            end
+
+            matched_boundary_keyword = keyword_match_at(sql, position, BOUNDARY_KEYWORDS)
+
+            if matched_boundary_keyword
+              clause_seen = true
+              position += matched_boundary_keyword.length
+              next
+            end
+          end
+        end
+
+        position += 1
+      end
+
+      boundaries
+    end
+
+    def merge_trailing_sentinel_segments(statements)
+      return statements if statements.length <= 1
+
+      statements.each_with_object([]) do |statement, merged|
+        if sentinel_only?(statement) && merged.any?
+          merged[-1] = "#{merged[-1]} #{statement}"
+        else
+          merged << statement
+        end
+      end
+    end
+
+    def sentinel_only?(segment)
+      segment.gsub(CommentStripper::SENTINEL_PATTERN, "").strip.empty?
+    end
+
+    def continuation_keyword?(current_keyword, next_keyword)
+      CONTINUATION_PAIRS[current_keyword] == next_keyword
+    end
+
+    def keyword_match_at(sql, position, keywords)
+      keywords.detect { |keyword| Tokenizer.keyword_at?(sql, position, keyword) }
+    end
+  end
+end

data/lib/sql_beautifier/table_registry.rb

@@ -216,7 +216,7 @@ module SqlBeautifier
     end
 
     def extract_explicit_alias(table_specification)
-      words = table_specification.strip.split(Constants::WHITESPACE_REGEX)
+      words = table_specification.strip.split(Constants::WHITESPACE_REGEX).grep_v(CommentStripper::SENTINEL_PATTERN)
       return nil if words.length < 2
 
       if words[1] == "as"

data/lib/sql_beautifier/tokenizer.rb

@@ -3,6 +3,7 @@
 module SqlBeautifier
   module Tokenizer
     IDENTIFIER_CHARACTER = %r{[[:alnum:]_$]}
+    SENTINEL_MAX_LOOKBACK = 20
 
     module_function
 
@@ -16,6 +17,11 @@ module SqlBeautifier
 
         match_position = match.begin(0)
 
+        if inside_sentinel?(sql, match_position)
+          search_position = sentinel_end_position(sql, match_position) || (match_position + 1)
+          next
+        end
+
         previous_character = character_before(sql, match_position)
         next_character = character_after(sql, match_position, keyword.length)
 
@@ -104,6 +110,13 @@ module SqlBeautifier
           next
         end
 
+        if sentinel_at?(text, position)
+          end_position = sentinel_end_position(text, position)
+          current_segment << text[position...end_position]
+          position = end_position
+          next
+        end
+
         case character
         when Constants::SINGLE_QUOTE
           inside_string_literal = true
@@ -201,6 +214,11 @@ module SqlBeautifier
           next
         end
 
+        if sentinel_at?(text, position)
+          position = sentinel_end_position(text, position)
+          next
+        end
+
         case character
         when Constants::SINGLE_QUOTE
           inside_string_literal = true
@@ -252,6 +270,40 @@ module SqlBeautifier
       text[position] == Constants::DOUBLE_QUOTE && text[position + 1] == Constants::DOUBLE_QUOTE
     end
 
+    def dollar_quote_delimiter_at(text, position)
+      return "$$" if text[position, 2] == "$$"
+      return unless text[position] == "$"
+
+      closing_dollar_position = text.index("$", position + 1)
+      return unless closing_dollar_position
+
+      delimiter = text[position..closing_dollar_position]
+      tag = delimiter[1..-2]
+      return unless tag.match?(%r{\A[[:alpha:]_][[:alnum:]_]*\z})
+
+      delimiter
+    end
+
+    def sentinel_at?(text, position)
+      text[position, CommentStripper::SENTINEL_PREFIX.length] == CommentStripper::SENTINEL_PREFIX
+    end
+
+    def sentinel_end_position(text, position)
+      closing = text.index(CommentStripper::SENTINEL_SUFFIX, position + CommentStripper::SENTINEL_PREFIX.length)
+      return position + 1 unless closing
+
+      closing + CommentStripper::SENTINEL_SUFFIX.length
+    end
+
+    def inside_sentinel?(text, position)
+      search_start = [position - SENTINEL_MAX_LOOKBACK, 0].max
+      prefix_position = text.rindex(CommentStripper::SENTINEL_PREFIX, position)
+      return false unless prefix_position && prefix_position >= search_start
+
+      end_position = sentinel_end_position(text, prefix_position)
+      position < end_position
+    end
+
     def top_level?(sql, target_position)
       parenthesis_depth = 0
       inside_string_literal = false
@@ -285,6 +337,11 @@ module SqlBeautifier
           next
         end
 
+        if sentinel_at?(sql, position)
+          position = sentinel_end_position(sql, position)
+          next
+        end
+
         case character
         when Constants::SINGLE_QUOTE
           inside_string_literal = true
@@ -337,6 +394,11 @@ module SqlBeautifier
           next
         end
 
+        if sentinel_at?(text, position)
+          position = sentinel_end_position(text, position)
+          next
+        end
+
         case character
         when Constants::SINGLE_QUOTE
           inside_string_literal = true

data/lib/sql_beautifier/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SqlBeautifier
-  VERSION = "0.5.0"
+  VERSION = "0.6.0"
 end

data/lib/sql_beautifier.rb

@@ -7,8 +7,11 @@ require_relative "sql_beautifier/constants"
 require_relative "sql_beautifier/util"
 require_relative "sql_beautifier/configuration"
 
+require_relative "sql_beautifier/comment_stripper"
+require_relative "sql_beautifier/comment_restorer"
 require_relative "sql_beautifier/normalizer"
 require_relative "sql_beautifier/tokenizer"
+require_relative "sql_beautifier/statement_splitter"
 require_relative "sql_beautifier/table_registry"
 require_relative "sql_beautifier/condition_formatter"
 require_relative "sql_beautifier/subquery_formatter"
@@ -24,16 +27,19 @@ require_relative "sql_beautifier/clauses/order_by"
 require_relative "sql_beautifier/clauses/having"
 require_relative "sql_beautifier/clauses/limit"
 require_relative "sql_beautifier/formatter"
+require_relative "sql_beautifier/statement_assembler"
 
 module SqlBeautifier
   class Error < StandardError; end
 
   module_function
 
-  def call(value)
+  def call(value, config = {})
     return unless value.present?
 
-    Formatter.call(value)
+    with_configuration(config) do
+      StatementAssembler.call(value)
+    end
   end
 
   def configuration
@@ -45,10 +51,31 @@ module SqlBeautifier
   end
 
   def config_for(key)
+    overrides = Thread.current[:sql_beautifier_config]
+    return overrides[key] if overrides&.key?(key)
+
     configuration.public_send(key)
   end
 
   def reset_configuration!
     @configuration = Configuration.new
   end
+
+  def with_configuration(config)
+    raise ArgumentError, "Expected a Hash for configuration overrides, got #{config.class}" unless config.is_a?(Hash)
+
+    return yield if config.empty?
+
+    previous = Thread.current[:sql_beautifier_config]
+    validate_configuration_keys!(config)
+    Thread.current[:sql_beautifier_config] = config
+    yield
+  ensure
+    Thread.current[:sql_beautifier_config] = previous if config.is_a?(Hash) && config.any?
+  end
+
+  def validate_configuration_keys!(config)
+    invalid_keys = config.keys - Configuration::DEFAULTS.keys
+    raise ArgumentError, "Unknown configuration keys: #{invalid_keys.join(', ')}" if invalid_keys.any?
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sql_beautifier
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Kinnell Shah
@@ -46,6 +46,8 @@ files:
 - lib/sql_beautifier/clauses/order_by.rb
 - lib/sql_beautifier/clauses/select.rb
 - lib/sql_beautifier/clauses/where.rb
+- lib/sql_beautifier/comment_restorer.rb
+- lib/sql_beautifier/comment_stripper.rb
 - lib/sql_beautifier/condition_formatter.rb
 - lib/sql_beautifier/configuration.rb
 - lib/sql_beautifier/constants.rb
@@ -53,6 +55,8 @@ files:
 - lib/sql_beautifier/cte_formatter.rb
 - lib/sql_beautifier/formatter.rb
 - lib/sql_beautifier/normalizer.rb
+- lib/sql_beautifier/statement_assembler.rb
+- lib/sql_beautifier/statement_splitter.rb
 - lib/sql_beautifier/subquery_formatter.rb
 - lib/sql_beautifier/table_registry.rb
 - lib/sql_beautifier/tokenizer.rb