collie 0.1.0 → 1.0.0

data/lib/collie/parser/lexer.rb

@@ -6,12 +6,13 @@ module Collie
   module Parser
     # Token representation
     class Token
-      attr_accessor :type, :value, :location
+      attr_accessor :type, :value, :location, :raw_value
 
-      def initialize(type:, value:, location:)
+      def initialize(type:, value:, location:, raw_value: nil)
         @type = type
         @value = value
         @location = location
+        @raw_value = raw_value
       end
 
       def to_s
@@ -24,7 +25,7 @@ module Collie
       KEYWORDS = %w[
         %token %type %left %right %nonassoc %prec %union %start
         %rule %inline %code %expect %define %param %parse-param
-        %lex-param %initial-action %destructor %printer
+        %lex-param %initial-action %destructor %printer %empty
       ].freeze
 
       def initialize(source, filename: "<input>")
@@ -34,6 +35,7 @@ module Collie
         @line = 1
         @column = 1
         @tokens = []
+        @section_separator_count = 0
       end
 
       def tokenize
@@ -44,15 +46,20 @@ module Collie
           if current_char == "/" && peek_char == "/"
             skip_line_comment
           elsif current_char == "/" && peek_char == "*"
-            skip_block_comment
+            token = tokenize_block_comment
+            @tokens << token if token
           elsif current_char == "%" && peek_char == "{"
             @tokens << tokenize_prologue
           elsif current_char == "%" && peek_char == "}"
             advance(2)
             @tokens << make_token(:PROLOGUE_END, "%}")
           elsif current_char == "%" && peek_char == "%"
-            advance(2)
-            @tokens << make_token(:SECTION_SEPARATOR, "%%")
+            @tokens << tokenize_section_separator
+            if @section_separator_count == 2
+              epilogue = tokenize_epilogue
+              @tokens << epilogue if epilogue
+              break
+            end
           elsif current_char == "%" && alpha?(peek_char)
             @tokens << tokenize_directive
           elsif current_char == "{"
@@ -140,15 +147,34 @@ module Collie
         advance unless eof? # skip \n
       end
 
-      def skip_block_comment
+      def tokenize_block_comment
+        start_line = @line
+        start_column = @column
+        buffer = +"/*"
         advance(2) # skip /*
+
         until eof?
           if current_char == "*" && peek_char == "/"
+            buffer << "*/"
             advance(2)
             break
           end
+
+          buffer << current_char
           advance
         end
+
+        return unless empty_comment?(buffer)
+
+        Token.new(
+          type: :EMPTY,
+          value: buffer,
+          location: make_location(start_line, start_column, buffer.length)
+        )
+      end
+
+      def empty_comment?(comment)
+        comment.match?(%r{\A/\*\s*empty\s*\*/\z}i)
       end
 
       def tokenize_prologue
@@ -190,7 +216,9 @@ module Collie
                when "%start" then :START
                when "%rule" then :RULE
                when "%inline" then :INLINE
-               else :DIRECTIVE
+               when "%empty" then :EMPTY
+               else
+                 return tokenize_unknown_declaration(start_line, start_column, buffer)
                end
 
         Token.new(
@@ -200,6 +228,81 @@ module Collie
         )
       end
 
+      def tokenize_section_separator
+        start_line = @line
+        start_column = @column
+        advance(2)
+        @section_separator_count += 1
+
+        Token.new(
+          type: :SECTION_SEPARATOR,
+          value: "%%",
+          location: make_location(start_line, start_column, 2)
+        )
+      end
+
+      def tokenize_epilogue
+        consume_single_line_break
+        return nil if eof?
+
+        start_line = @line
+        start_column = @column
+        buffer = @source[@pos..]
+        advance(buffer.length)
+
+        Token.new(
+          type: :EPILOGUE,
+          value: buffer,
+          location: make_location(start_line, start_column, buffer.length)
+        )
+      end
+
+      def consume_single_line_break
+        if current_char == "\r" && peek_char == "\n"
+          advance(2)
+        elsif current_char == "\n"
+          advance
+        end
+      end
+
+      def tokenize_unknown_declaration(start_line, start_column, directive)
+        buffer = +directive
+
+        append_unknown_declaration_content(buffer)
+
+        Token.new(
+          type: :UNKNOWN_DECLARATION,
+          value: buffer.rstrip,
+          location: make_location(start_line, start_column, buffer.length)
+        )
+      end
+
+      def append_unknown_declaration_content(buffer)
+        action_depth = 0
+
+        until eof?
+          break if action_depth.zero? && current_char == "\n"
+
+          if action_depth.positive? && (current_char == '"' || current_char == "'")
+            append_quoted_action_content(buffer, current_char)
+            next
+          elsif action_depth.positive? && current_char == "/" && peek_char == "/"
+            append_line_comment_action_content(buffer)
+            next
+          elsif action_depth.positive? && current_char == "/" && peek_char == "*"
+            append_block_comment_action_content(buffer)
+            next
+          elsif current_char == "{"
+            action_depth += 1
+          elsif current_char == "}" && action_depth.positive?
+            action_depth -= 1
+          end
+
+          buffer << current_char
+          advance
+        end
+      end
+
       def tokenize_action
         start_line = @line
         start_column = @column
@@ -209,7 +312,16 @@ module Collie
         loop do
           break if eof?
 
-          if current_char == "{"
+          if current_char == '"' || current_char == "'"
+            append_quoted_action_content(buffer, current_char)
+            next
+          elsif current_char == "/" && peek_char == "/"
+            append_line_comment_action_content(buffer)
+            next
+          elsif current_char == "/" && peek_char == "*"
+            append_block_comment_action_content(buffer)
+            next
+          elsif current_char == "{"
             depth += 1
           elsif current_char == "}"
             depth -= 1
@@ -232,6 +344,7 @@ module Collie
       end
 
       def tokenize_char_literal
+        start_pos = @pos
         start_line = @line
         start_column = @column
         buffer = +""
@@ -247,15 +360,18 @@ module Collie
         end
 
         advance unless eof? # skip closing '
+        raw_value = @source[start_pos...@pos]
 
         Token.new(
           type: :CHAR,
           value: buffer,
-          location: make_location(start_line, start_column, buffer.length + 2)
+          raw_value: raw_value,
+          location: make_location(start_line, start_column, raw_value.length)
         )
       end
 
       def tokenize_string_literal
+        start_pos = @pos
         start_line = @line
         start_column = @column
         buffer = +""
@@ -271,14 +387,69 @@ module Collie
         end
 
         advance unless eof? # skip closing "
+        raw_value = @source[start_pos...@pos]
 
         Token.new(
           type: :STRING,
           value: buffer,
-          location: make_location(start_line, start_column, buffer.length + 2)
+          raw_value: raw_value,
+          location: make_location(start_line, start_column, raw_value.length)
         )
       end
 
+      def append_quoted_action_content(buffer, quote)
+        buffer << current_char
+        advance
+
+        until eof?
+          buffer << current_char
+
+          if current_char == "\\"
+            advance
+            next if eof?
+
+            buffer << current_char
+          elsif current_char == quote
+            advance
+            break
+          end
+
+          advance
+        end
+      end
+
+      def append_line_comment_action_content(buffer)
+        buffer << current_char
+        advance
+        buffer << current_char
+        advance
+
+        until eof? || current_char == "\n"
+          buffer << current_char
+          advance
+        end
+      end
+
+      def append_block_comment_action_content(buffer)
+        buffer << current_char
+        advance
+        buffer << current_char
+        advance
+
+        until eof?
+          if current_char == "*" && peek_char == "/"
+            buffer << current_char
+            advance
+            buffer << current_char
+            advance
+            break
+          end
+
+          buffer << current_char
+          advance
+        end
+      end
+
       def tokenize_type_tag
         start_line = @line
         start_column = @column

data/lib/collie/parser/parser.rb

@@ -86,6 +86,8 @@ module Collie
             # %inline for Lrama extensions
             advance
             declarations << parse_inline_declaration
+          when :UNKNOWN_DECLARATION
+            declarations << parse_unknown_declaration
           else
             advance # Skip unknown declarations for now
           end
@@ -127,12 +129,35 @@ module Collie
       end
 
       def parse_inline_declaration
-        # %inline followed by rule name
-        rule_name = expect(:IDENTIFIER).value
+        # %inline followed by a rule name, optionally with a full inline rule body.
+        rule_token = expect(:IDENTIFIER)
+        rule_name = rule_token.value
+        parameters = []
+        alternatives = []
+
+        if match?(:LPAREN)
+          advance
+          parameters = parse_parameter_list
+          expect(:RPAREN)
+        end
+
+        if match?(:COLON)
+          advance
+          alternatives << parse_alternative
+
+          while match?(:PIPE)
+            advance
+            alternatives << parse_alternative
+          end
+
+          expect(:SEMICOLON) if match?(:SEMICOLON)
+        end
 
         AST::InlineRule.new(
           rule: rule_name,
-          location: current_token.location
+          parameters: parameters,
+          alternatives: alternatives,
+          location: rule_token.location
         )
       end
 
@@ -147,7 +172,7 @@ module Collie
         end
 
         while match?(:IDENTIFIER) || match?(:STRING) || match?(:CHAR)
-          names << current_token.value
+          names << token_value(current_token)
           advance
         end
 
@@ -191,7 +216,7 @@ module Collie
 
         tokens = []
         while match?(:IDENTIFIER) || match?(:STRING) || match?(:CHAR)
-          tokens << current_token.value
+          tokens << token_value(current_token)
           advance
         end
 
@@ -227,6 +252,15 @@ module Collie
         )
       end
 
+      def parse_unknown_declaration
+        token = expect(:UNKNOWN_DECLARATION)
+
+        AST::UnknownDeclaration.new(
+          source: token.value,
+          location: token.location
+        )
+      end
+
       def parse_rules
         rules = []
 
@@ -300,7 +334,8 @@ module Collie
 
         if match?(:IDENTIFIER) || match?(:STRING) || match?(:CHAR)
           symbol_token = current_token
-          kind = if symbol_token.value.match?(/^[A-Z]/) || match?(:STRING) || match?(:CHAR)
+          name = token_value(symbol_token)
+          kind = if name.match?(/^[A-Z]/) || literal_token?(symbol_token)
                    :terminal
                  else
                    :nonterminal
@@ -308,7 +343,7 @@ module Collie
           advance
 
           args << AST::Symbol.new(
-            name: symbol_token.value,
+            name: name,
             kind: kind,
             location: symbol_token.location
           )
@@ -316,7 +351,8 @@ module Collie
           while match?(:COMMA)
             advance
             symbol_token = current_token
-            kind = if symbol_token.value.match?(/^[A-Z]/) || match?(:STRING) || match?(:CHAR)
+            name = token_value(symbol_token)
+            kind = if name.match?(/^[A-Z]/) || literal_token?(symbol_token)
                      :terminal
                    else
                      :nonterminal
@@ -324,7 +360,7 @@ module Collie
             advance
 
             args << AST::Symbol.new(
-              name: symbol_token.value,
+              name: name,
               kind: kind,
               location: symbol_token.location
             )
@@ -338,17 +374,24 @@ module Collie
         symbols = []
         action = nil
         prec = nil
+        explicit_empty = false
+        empty_marker = nil
         start_location = current_token.location
 
         until match?(:PIPE) || match?(:SEMICOLON) || match?(:ACTION) ||
               match?(:SECTION_SEPARATOR) || match?(:EOF)
           if match?(:PREC)
             advance
-            prec = current_token.value
+            prec = token_value(current_token)
+            advance
+          elsif match?(:EMPTY)
+            explicit_empty = true
+            empty_marker = token_value(current_token)
             advance
           elsif match?(:IDENTIFIER) || match?(:STRING) || match?(:CHAR)
             symbol_token = current_token
-            kind = if symbol_token.value.match?(/^[A-Z]/) || match?(:STRING) || match?(:CHAR)
+            name = token_value(symbol_token)
+            kind = if name.match?(/^[A-Z]/) || literal_token?(symbol_token)
                      :terminal
                    else
                      :nonterminal
@@ -371,7 +414,7 @@ module Collie
             end
 
             symbols << AST::Symbol.new(
-              name: symbol_token.value,
+              name: name,
               kind: kind,
               alias_name: alias_name,
               arguments: arguments,
@@ -394,6 +437,8 @@ module Collie
           symbols: symbols,
           action: action,
           prec: prec,
+          explicit_empty: explicit_empty,
+          empty_marker: empty_marker,
           location: symbols.first&.location || start_location
         )
       end
@@ -402,15 +447,30 @@ module Collie
         return nil unless match?(:SECTION_SEPARATOR)
 
         advance
+
+        if match?(:EPILOGUE)
+          token = current_token
+          advance
+          return AST::Epilogue.new(code: token.value, location: token.location) unless token.value.empty?
+        end
+
         code = +""
 
         until match?(:EOF)
-          code << current_token.value
+          code << token_value(current_token)
           advance
         end
 
         AST::Epilogue.new(code: code, location: current_token.location) unless code.empty?
       end
+
+      def token_value(token)
+        token.raw_value || token.value
+      end
+
+      def literal_token?(token)
+        token.type == :STRING || token.type == :CHAR
+      end
     end
   end
 end

data/lib/collie/reporter/github.rb

@@ -12,14 +12,27 @@ module Collie
 
       def format_offense(offense)
         level = github_level(offense.severity)
-        file = offense.location.file
+        file = escape_property(offense.location.file)
         line = offense.location.line
         col = offense.location.column
-        message = offense.message.gsub(",", "%2C") # Escape commas
+        message = escape_data(offense.message)
 
         "::#{level} file=#{file},line=#{line},col=#{col}::#{message}"
       end
 
+      def escape_data(value)
+        value.to_s
+             .gsub("%", "%25")
+             .gsub("\r", "%0D")
+             .gsub("\n", "%0A")
+      end
+
+      def escape_property(value)
+        escape_data(value)
+          .gsub(":", "%3A")
+          .gsub(",", "%2C")
+      end
+
       def github_level(severity)
         case severity
         when :error

data/lib/collie/reporter/json.rb

@@ -40,10 +40,13 @@ module Collie
           rule: offense.rule.rule_name,
           severity: offense.severity,
           message: offense.message,
+          autocorrectable: offense.autocorrectable?,
           location: {
             line: offense.location.line,
             column: offense.location.column,
-            length: offense.location.length
+            length: offense.location.length,
+            end_line: offense.location.line,
+            end_column: offense.location.column + offense.location.length
           }
         }
       end

data/lib/collie/reporter/sarif.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Collie
+  module Reporter
+    # SARIF 2.1.0 reporter for code scanning integrations.
+    class Sarif
+      def report(offenses)
+        JSON.pretty_generate(
+          version: "2.1.0",
+          "$schema": "https://json.schemastore.org/sarif-2.1.0.json",
+          runs: [
+            {
+              tool: {
+                driver: {
+                  name: "Collie",
+                  informationUri: "https://github.com/ydah/collie",
+                  rules: rules(offenses)
+                }
+              },
+              results: offenses.map { |offense| result(offense) }
+            }
+          ]
+        )
+      end
+
+      private
+
+      def rules(offenses)
+        offenses.map(&:rule).uniq(&:rule_name).map do |rule|
+          {
+            id: rule.rule_name,
+            name: rule.rule_name,
+            shortDescription: {
+              text: rule.description
+            },
+            defaultConfiguration: {
+              level: level(rule.severity)
+            }
+          }
+        end
+      end
+
+      def result(offense)
+        {
+          ruleId: offense.rule.rule_name,
+          level: level(offense.severity),
+          message: {
+            text: offense.message
+          },
+          locations: [
+            {
+              physicalLocation: {
+                artifactLocation: {
+                  uri: offense.location.file
+                },
+                region: {
+                  startLine: offense.location.line,
+                  startColumn: offense.location.column,
+                  charLength: offense.location.length
+                }
+              }
+            }
+          ]
+        }
+      end
+
+      def level(severity)
+        case severity
+        when :error
+          "error"
+        when :warning, :convention
+          "warning"
+        else
+          "note"
+        end
+      end
+    end
+  end
+end

data/lib/collie/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Collie
-  VERSION = "0.1.0"
+  VERSION = "1.0.0"
 end

data/lib/collie.rb

@@ -1,22 +1,27 @@
 # frozen_string_literal: true
 
 require_relative "collie/version"
-require_relative "collie/cli"
 require_relative "collie/config"
+require_relative "collie/config/schema"
+require_relative "collie/cli"
 require_relative "collie/ast"
 require_relative "collie/parser/lexer"
 require_relative "collie/parser/parser"
+require_relative "collie/parser/debug_serializer"
 require_relative "collie/analyzer/symbol_table"
+require_relative "collie/analyzer/symbol_resolver"
 require_relative "collie/analyzer/reachability"
 require_relative "collie/analyzer/recursion"
 require_relative "collie/analyzer/conflict"
 require_relative "collie/linter/base"
 require_relative "collie/linter/registry"
+require_relative "collie/formatter/signature"
 require_relative "collie/formatter/formatter"
 require_relative "collie/formatter/options"
 require_relative "collie/reporter/text"
 require_relative "collie/reporter/json"
 require_relative "collie/reporter/github"
+require_relative "collie/reporter/sarif"
 
 # Collie is a linter and formatter for Lrama Style BNF grammar files (.y files).
 #