kumi-parser 0.0.2 → 0.0.4

data/lib/kumi/parser/token_metadata.rb

@@ -0,0 +1,370 @@
+# frozen_string_literal: true
+
+module Kumi
+  module Parser
+    # Token types
+    module TokenType
+      # Literals
+      INTEGER = :integer
+      FLOAT = :float
+      STRING = :string
+      BOOLEAN = :boolean
+
+      # Identifiers and symbols
+      IDENTIFIER = :identifier
+      SYMBOL = :symbol           # :name
+
+      # Keywords
+      SCHEMA = :schema
+      INPUT = :input
+      VALUE = :value
+      TRAIT = :trait
+      DO = :do
+      END_KW = :end
+      ON = :on
+      BASE = :base
+
+      # Type keywords
+      INTEGER_TYPE = :integer_type   # integer
+      FLOAT_TYPE = :float_type       # float
+      STRING_TYPE = :string_type     # string
+      BOOLEAN_TYPE = :boolean_type   # boolean
+      ANY_TYPE = :any_type           # any
+      ARRAY_TYPE = :array_type       # array
+
+      # Function keywords
+      FN = :fn
+
+      # Operators (by precedence)
+      MULTIPLY = :multiply       # *
+      DIVIDE = :divide          # /
+      MODULO = :modulo          # %
+      ADD = :add                # +
+      SUBTRACT = :subtract      # -
+      GTE = :gte                # >=
+      LTE = :lte                # <=
+      GT = :gt                  # >
+      LT = :lt                  # <
+      EQ = :eq                  # ==
+      NE = :ne                  # !=
+      AND = :and                # &
+      OR = :or                  # |
+
+      # Punctuation
+      DOT = :dot                # .
+      COMMA = :comma            # ,
+      COLON = :colon            # :
+      LPAREN = :lparen          # (
+      RPAREN = :rparen          # )
+      LBRACKET = :lbracket      # [
+      RBRACKET = :rbracket      # ]
+
+      # Special
+      NEWLINE = :newline
+      EOF = :eof
+      COMMENT = :comment        # # comment
+    end
+
+    # Rich metadata for each token type
+    TOKEN_METADATA = {
+      # Keywords with parsing hints
+      schema: {
+        category: :keyword,
+        expects_block: true,
+        block_terminator: :end
+      },
+      input: {
+        category: :keyword,
+        expects_block: true,
+        block_terminator: :end,
+        context: :input_declarations
+      },
+      value: {
+        category: :keyword,
+        expects_expression: true,
+        declaration_type: :value
+      },
+      trait: {
+        category: :keyword,
+        expects_expression: true,
+        declaration_type: :trait
+      },
+      do: {
+        category: :keyword,
+        block_opener: true
+      },
+      end: {
+        category: :keyword,
+        block_closer: true,
+        terminates_expression: true
+      },
+      on: {
+        category: :keyword,
+        cascade_keyword: true,
+        expects_condition: true
+      },
+      base: {
+        category: :keyword,
+        cascade_keyword: true,
+        is_base_case: true
+      },
+
+      # Type keywords
+      integer_type: {
+        category: :type_keyword,
+        starts_declaration: true,
+        type_name: :integer
+      },
+      float_type: {
+        category: :type_keyword,
+        starts_declaration: true,
+        type_name: :float
+      },
+      string_type: {
+        category: :type_keyword,
+        starts_declaration: true,
+        type_name: :string
+      },
+      boolean_type: {
+        category: :type_keyword,
+        starts_declaration: true,
+        type_name: :boolean
+      },
+      any_type: {
+        category: :type_keyword,
+        starts_declaration: true,
+        type_name: :any
+      },
+      array_type: {
+        category: :type_keyword,
+        starts_declaration: true,
+        type_name: :array
+      },
+
+      # Function keyword
+      fn: {
+        category: :keyword,
+        function_keyword: true,
+        starts_expression: true
+      },
+
+      # Operators with precedence and associativity
+      multiply: {
+        category: :operator,
+        precedence: 6,
+        associativity: :left,
+        arity: :binary
+      },
+      divide: {
+        category: :operator,
+        precedence: 6,
+        associativity: :left,
+        arity: :binary
+      },
+      modulo: {
+        category: :operator,
+        precedence: 6,
+        associativity: :left,
+        arity: :binary
+      },
+      add: {
+        category: :operator,
+        precedence: 5,
+        associativity: :left,
+        arity: :binary
+      },
+      subtract: {
+        category: :operator,
+        precedence: 5,
+        associativity: :left,
+        arity: :binary
+      },
+      gte: {
+        category: :operator,
+        precedence: 4,
+        associativity: :left,
+        arity: :binary,
+        returns_boolean: true
+      },
+      lte: {
+        category: :operator,
+        precedence: 4,
+        associativity: :left,
+        arity: :binary,
+        returns_boolean: true
+      },
+      gt: {
+        category: :operator,
+        precedence: 4,
+        associativity: :left,
+        arity: :binary,
+        returns_boolean: true
+      },
+      lt: {
+        category: :operator,
+        precedence: 4,
+        associativity: :left,
+        arity: :binary,
+        returns_boolean: true
+      },
+      eq: {
+        category: :operator,
+        precedence: 4,
+        associativity: :left,
+        arity: :binary,
+        returns_boolean: true
+      },
+      ne: {
+        category: :operator,
+        precedence: 4,
+        associativity: :left,
+        arity: :binary,
+        returns_boolean: true
+      },
+      and: {
+        category: :operator,
+        precedence: 3,
+        associativity: :left,
+        arity: :binary,
+        requires_boolean: true
+      },
+      or: {
+        category: :operator,
+        precedence: 2,
+        associativity: :left,
+        arity: :binary,
+        requires_boolean: true
+      },
+
+      # Literals with type information
+      integer: {
+        category: :literal,
+        starts_expression: true,
+        ast_class: 'Kumi::Syntax::Literal'
+      },
+      float: {
+        category: :literal,
+        starts_expression: true,
+        ast_class: 'Kumi::Syntax::Literal'
+      },
+      string: {
+        category: :literal,
+        starts_expression: true,
+        ast_class: 'Kumi::Syntax::Literal'
+      },
+      boolean: {
+        category: :literal,
+        starts_expression: true,
+        ast_class: 'Kumi::Syntax::Literal'
+      },
+
+      # Identifiers and references
+      identifier: {
+        category: :identifier,
+        starts_expression: true,
+        can_be_reference: true
+      },
+      symbol: {
+        category: :identifier,
+        starts_expression: true,
+        is_declaration_name: true
+      },
+
+      # Punctuation with parser hints
+      dot: {
+        category: :punctuation,
+        indicates_member_access: true
+      },
+      comma: {
+        category: :punctuation,
+        separates_items: true
+      },
+      colon: {
+        category: :punctuation,
+        indicates_symbol: true
+      },
+      lparen: {
+        category: :punctuation,
+        opens_group: true,
+        group_closer: :rparen,
+        starts_expression: true
+      },
+      rparen: {
+        category: :punctuation,
+        closes_group: true,
+        terminates_expression: true
+      },
+      lbracket: {
+        category: :punctuation,
+        opens_group: true,
+        group_closer: :rbracket,
+        starts_expression: true,
+        indicates_array: true
+      },
+      rbracket: {
+        category: :punctuation,
+        closes_group: true,
+        terminates_expression: true
+      },
+
+      # Special tokens
+      newline: {
+        category: :whitespace,
+        separates_statements: true
+      },
+      eof: {
+        category: :special,
+        terminates_input: true
+      },
+      comment: {
+        category: :whitespace,
+        ignored_by_parser: true
+      }
+    }.freeze
+
+    # Character to token mappings
+    CHAR_TO_TOKEN = {
+      '(' => :lparen,
+      ')' => :rparen,
+      '[' => :lbracket,
+      ']' => :rbracket,
+      ',' => :comma,
+      '.' => :dot,
+      ':' => :colon,
+      '+' => :add,
+      '-' => :subtract,
+      '*' => :multiply,
+      '/' => :divide,
+      '%' => :modulo,
+      '&' => :and,
+      '|' => :or
+    }.freeze
+
+    # Keywords mapping
+    KEYWORDS = {
+      'schema' => :schema,
+      'input' => :input,
+      'value' => :value,
+      'trait' => :trait,
+      'do' => :do,
+      'end' => :end,
+      'on' => :on,
+      'base' => :base,
+      'fn' => :fn,
+      'true' => :boolean,
+      'false' => :boolean,
+      'integer' => :integer_type,
+      'float' => :float_type,
+      'string' => :string_type,
+      'boolean' => :boolean_type,
+      'any' => :any_type,
+      'array' => :array_type
+    }.freeze
+
+    # Opener to closer mappings for error recovery
+    OPENER_FOR_CLOSER = {
+      rparen: :lparen,
+      rbracket: :lbracket
+    }.freeze
+  end
+end

data/lib/kumi/parser/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Kumi
+  module Parser
+    VERSION = '0.0.4'
+  end
+end

data/lib/kumi/text_parser.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require_relative 'parser/text_parser'
+
+module Kumi
+  # Top-level text parser module with same interface as Ruby DSL
+  module TextParser
+    extend self
+    
+    # Parse text schema and return AST (same interface as RubyParser::Dsl.build_syntax_tree)
+    def parse(text, source_file: '<input>')
+      Parser::TextParser.parse(text, source_file: source_file)
+    end
+    
+    # Validate text schema
+    def valid?(text, source_file: '<input>')
+      Parser::TextParser.valid?(text, source_file: source_file)
+    end
+    
+    # Get validation diagnostics
+    def validate(text, source_file: '<input>')
+      Parser::TextParser.validate(text, source_file: source_file)
+    end
+    
+    # Get Monaco Editor format diagnostics
+    def diagnostics_for_monaco(text, source_file: '<input>')
+      Parser::TextParser.diagnostics_for_monaco(text, source_file: source_file)
+    end
+    
+    # Get CodeMirror format diagnostics
+    def diagnostics_for_codemirror(text, source_file: '<input>')
+      Parser::TextParser.diagnostics_for_codemirror(text, source_file: source_file)
+    end
+    
+    # Get JSON format diagnostics
+    def diagnostics_as_json(text, source_file: '<input>')
+      Parser::TextParser.diagnostics_as_json(text, source_file: source_file)
+    end
+  end
+end

data/lib/kumi/text_schema.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'kumi'
+require_relative 'text_parser'
+
+module Kumi
+  # Text-based schema that extends Kumi::Schema with text parsing capabilities
+  class TextSchema
+    extend Kumi::Schema
+    
+    # Create a schema from text using the same pipeline as Ruby DSL
+    def self.from_text(text, source_file: '<input>')
+      # Parse text to AST (same as RubyParser::Dsl.build_syntax_tree)
+      @__syntax_tree__ = Kumi::TextParser.parse(text, source_file: source_file).freeze
+      @__analyzer_result__ = Analyzer.analyze!(@__syntax_tree__).freeze
+      @__compiled_schema__ = Compiler.compile(@__syntax_tree__, analyzer: @__analyzer_result__).freeze
+
+      Inspector.new(@__syntax_tree__, @__analyzer_result__, @__compiled_schema__)
+    end
+    
+    # Validate text schema
+    def self.valid?(text, source_file: '<input>')
+      Kumi::TextParser.valid?(text, source_file: source_file)
+    end
+    
+    # Get validation diagnostics
+    def self.validate(text, source_file: '<input>')
+      Kumi::TextParser.validate(text, source_file: source_file)
+    end
+  end
+end

data/lib/kumi-parser.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require 'kumi'
+require 'kumi/syntax/node'
+require 'zeitwerk'
+require 'parslet'
+
+loader = Zeitwerk::Loader.for_gem(warn_on_extra_files: false)
+loader.ignore("#{__dir__}/kumi-parser.rb")
+loader.ignore("#{__dir__}/kumi/parser/version.rb")
+loader.setup
+
+require_relative 'kumi/parser/version'
+
+module Kumi
+  module Parser
+    # Parser extension for Kumi DSL
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kumi-parser
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.4
 platform: ruby
 authors:
 - Kumi Team
@@ -128,7 +128,31 @@ email:
 executables: []
 extensions: []
 extra_rdoc_files: []
-files: []
+files:
+- ".rspec"
+- CLAUDE.md
+- LICENSE
+- README.md
+- Rakefile
+- examples/debug_text_parser.rb
+- examples/debug_transform_rule.rb
+- examples/text_parser_comprehensive_test.rb
+- examples/text_parser_test_with_comments.rb
+- kumi-parser.gemspec
+- lib/kumi-parser.rb
+- lib/kumi/parser/base.rb
+- lib/kumi/parser/direct_parser.rb
+- lib/kumi/parser/error_extractor.rb
+- lib/kumi/parser/errors.rb
+- lib/kumi/parser/smart_tokenizer.rb
+- lib/kumi/parser/syntax_validator.rb
+- lib/kumi/parser/text_parser.rb
+- lib/kumi/parser/text_parser/api.rb
+- lib/kumi/parser/token.rb
+- lib/kumi/parser/token_metadata.rb
+- lib/kumi/parser/version.rb
+- lib/kumi/text_parser.rb
+- lib/kumi/text_schema.rb
 homepage: https://github.com/amuta/kumi-parser
 licenses:
 - MIT