kumi-parser 0.0.33 → 0.1.0

data/lib/kumi/parser/parser.rb

@@ -0,0 +1,692 @@
+# frozen_string_literal: true
+
+module Kumi
+  module Parser
+    # Recursive-descent parser for declarations and a Pratt parser for
+    # expressions, producing kumi-core's Kumi::Syntax::* AST directly.
+    #
+    # The parser is the boundary of the parse phase: it reports *shape* errors
+    # (a missing `end`, an unexpected token, a malformed hash pair) with exact
+    # locations, and it does not attempt to resolve names, check types, or know
+    # anything about axes — those are semantic concerns owned by the analyzer.
+    class Parser
+      include Syntax = Kumi::Syntax
+
+      def initialize(tokens, source)
+        @tokens = tokens
+        @source = source
+        @pos = 0
+        @imported_names = Set.new
+      end
+
+      def parse
+        skip_separators
+        imports = parse_imports
+        @imported_names.merge(imports.flat_map(&:names))
+
+        root = parse_schema(imports)
+
+        skip_separators
+        expect(:eof, 'end of input')
+        root
+      end
+
+      private
+
+      # ---- cursor --------------------------------------------------------
+
+      def current
+        @tokens[@pos]
+      end
+
+      def peek(offset = 1)
+        @tokens[@pos + offset] || @tokens.last
+      end
+
+      def at?(*kinds)
+        kinds.include?(current.kind)
+      end
+
+      def advance
+        token = current
+        @pos += 1 unless current.kind == :eof
+        token
+      end
+
+      # Consume a token of the expected kind, or raise a clear parse error
+      # naming what was expected and what was found.
+      def expect(kind, description = nil)
+        return advance if current.kind == kind
+
+        want = description || describe_kind(kind)
+        error("expected #{want}, but found #{describe_token(current)}")
+      end
+
+      def skip_separators
+        @pos += 1 while at?(:newline, :comment)
+      end
+
+      def loc(token)
+        @source.location(token.offset)
+      end
+
+      # ---- schema --------------------------------------------------------
+
+      def parse_schema(root_imports)
+        schema_token = expect(:schema, '`schema do`')
+        expect(:do, '`do` after `schema`')
+        skip_separators
+
+        inner_imports = parse_imports
+        @imported_names.merge(inner_imports.flat_map(&:names))
+        hints = parse_codegen_directives
+
+        inputs = parse_input_block
+        values, traits = parse_declarations
+
+        expect(:end, '`end` to close the schema')
+
+        Syntax::Root.new(
+          inputs,
+          values,
+          traits,
+          root_imports + inner_imports,
+          hints: hints,
+          loc: loc(schema_token)
+        )
+      end
+
+      def parse_declarations
+        values = []
+        traits = []
+        skip_separators
+        while at?(:value, :let, :trait)
+          case current.kind
+          when :value then values << parse_value(inline: false)
+          when :let   then values << parse_value(inline: true)
+          when :trait then traits << parse_trait
+          end
+          skip_separators
+        end
+        [values, traits]
+      end
+
+      # ---- imports -------------------------------------------------------
+
+      def parse_imports
+        imports = []
+        skip_separators
+        while at?(:import)
+          imports << parse_import
+          skip_separators
+        end
+        imports
+      end
+
+      def parse_import
+        import_token = advance
+        names = [expect(:symbol, 'a `:name` to import').value]
+
+        while at?(:comma)
+          advance
+          skip_separators
+          break if from_label?
+
+          names << expect(:symbol, 'a `:name` to import').value
+        end
+
+        skip_separators
+        error('expected `from:` to name the module the import comes from') unless from_label?
+        advance # consume `from:`
+        skip_separators
+
+        module_ref = parse_constant_path
+        Syntax::ImportDeclaration.new(names, module_ref, loc: loc(import_token))
+      end
+
+      def from_label?
+        current.kind == :label && current.value == 'from'
+      end
+
+      def parse_constant_path
+        token = expect(:constant, 'a module name like `Foo::Bar`')
+        token.value
+      end
+
+      # ---- codegen directives -------------------------------------------
+
+      def parse_codegen_directives
+        hints = {}
+        skip_separators
+        while at?(:codegen)
+          advance
+          opts = parse_codegen_options
+          hints[:codegen] = (hints[:codegen] || {}).merge(opts)
+          skip_separators
+        end
+        hints
+      end
+
+      def parse_codegen_options
+        opts = {}
+        loop do
+          key = expect(:label, 'a codegen option like `streaming:`')
+          error("Unknown codegen option '#{key.value}'", at: key) unless key.value == 'streaming'
+          value = expect(:boolean, '`true` or `false`')
+          opts[:streaming] = value.value
+          break unless at?(:comma)
+
+          advance
+          skip_separators
+        end
+        opts
+      end
+
+      # ---- input block ---------------------------------------------------
+
+      def parse_input_block
+        expect(:input, 'an `input do` block')
+        expect(:do, '`do` after `input`')
+        declarations = []
+        skip_separators
+        while at?(:type_keyword)
+          declarations << parse_input_declaration
+          skip_separators
+        end
+        expect(:end, '`end` to close the `input` block')
+        declarations
+      end
+
+      def parse_input_declaration
+        type_token = expect(:type_keyword)
+        type = type_token.value
+        name = expect(:symbol, "a `:name` for the #{type} input").value
+
+        domain, index_name = parse_input_options(type)
+
+        children = parse_input_children(type)
+
+        Syntax::InputDeclaration.new(name, domain, type, children, index_name, loc: loc(type_token))
+      end
+
+      # Optional ", domain: …" and ", index: :sym" in any order.
+      def parse_input_options(type)
+        domain = nil
+        index_name = nil
+
+        while at?(:comma) && peek.kind == :label
+          advance # comma
+          key = advance # label
+          case key.value
+          when 'domain'
+            domain = parse_domain
+          when 'index'
+            error('`index:` is only valid on array declarations', at: key) unless type == :array
+            index_name = expect(:symbol, 'an index name like `:i`').value
+          else
+            error("unknown option `#{key.value}:` on an input declaration", at: key)
+          end
+        end
+
+        [domain, index_name]
+      end
+
+      def parse_input_children(type)
+        return [] unless Grammar::CONTAINER_TYPES.include?(type) && at?(:do)
+
+        advance # do
+        children = []
+        skip_separators
+        while at?(:type_keyword)
+          children << parse_input_declaration
+          skip_separators
+        end
+        expect(:end, '`end` to close the nested input block')
+        children
+      end
+
+      # Domains: a bracketed literal list `[a, b]` or a numeric range `lo..hi`.
+      def parse_domain
+        case current.kind
+        when :lbracket
+          array = parse_array_literal
+          array.elements.map { |e| e.is_a?(Syntax::Literal) ? e.value : e }
+        when :integer, :float
+          parse_range_domain
+        else
+          error('expected a domain: a list like `[1, 2]` or a range like `0..10`')
+        end
+      end
+
+      def parse_range_domain
+        lo = advance.value
+        case current.kind
+        when :dot_dot
+          advance
+          (lo..numeric_bound)
+        when :dot_dot_dot
+          advance
+          (lo...numeric_bound)
+        else
+          [lo]
+        end
+      end
+
+      def numeric_bound
+        token = expect(current.kind == :float ? :float : :integer, 'a number')
+        token.value
+      end
+
+      # ---- value / let / trait ------------------------------------------
+
+      def parse_value(inline:)
+        keyword = advance
+        name = expect(:symbol, "a `:name` for the #{inline ? 'let' : 'value'}").value
+
+        expression =
+          if at?(:do)
+            parse_cascade
+          else
+            expect(:comma, '`,` then an expression')
+            parse_expression
+          end
+
+        hints = inline ? { inline: true } : {}
+        Syntax::ValueDeclaration.new(name, expression, hints: hints, loc: loc(keyword))
+      end
+
+      def parse_trait
+        keyword = advance
+        name = expect(:symbol, 'a `:name` for the trait').value
+        expect(:comma, '`,` then a boolean expression')
+        expression = parse_expression
+        Syntax::TraitDeclaration.new(name, expression, loc: loc(keyword))
+      end
+
+      # ---- cascade -------------------------------------------------------
+
+      def parse_cascade
+        do_token = expect(:do)
+        cases = []
+        skip_separators
+        while at?(:on, :base)
+          cases << parse_case
+          skip_separators
+        end
+        expect(:end, '`end` to close the cascade')
+        Syntax::CascadeExpression.new(cases, loc: loc(do_token))
+      end
+
+      def parse_case
+        if at?(:on)
+          on_token = advance
+          exprs = [parse_expression]
+          while at?(:comma)
+            advance
+            exprs << parse_expression
+          end
+          result = exprs.pop
+          condition = build_case_condition(exprs, on_token)
+          Syntax::CaseExpression.new(condition, result, loc: loc(on_token))
+        else
+          base_token = advance
+          result = parse_expression
+          true_literal = Syntax::Literal.new(true, loc: loc(base_token))
+          Syntax::CaseExpression.new(true_literal, result, loc: loc(base_token))
+        end
+      end
+
+      # A single trait reference is wrapped in `cascade_and([ref])` so the
+      # condition is uniformly an all-of over traits; multiple conditions become
+      # `cascade_and([...])` directly.
+      def build_case_condition(conditions, on_token)
+        if conditions.length == 1
+          c = conditions.first
+          if c.is_a?(Syntax::DeclarationReference)
+            Syntax::CallExpression.new(:cascade_and, [c], loc: loc(on_token))
+          else
+            c
+          end
+        else
+          Syntax::CallExpression.new(:cascade_and, conditions, loc: loc(on_token))
+        end
+      end
+
+      # ---- expressions (Pratt) ------------------------------------------
+
+      def parse_expression(min_precedence = 0)
+        left = parse_postfix(parse_primary)
+        skip_separators
+
+        while Grammar.binary_operator?(current.kind) && Grammar.precedence(current.kind) >= min_precedence
+          op = advance
+          skip_separators
+          next_min = Grammar.right_associative?(op.kind) ? Grammar.precedence(op.kind) : Grammar.precedence(op.kind) + 1
+          right = parse_expression(next_min)
+          left = Syntax::CallExpression.new(Grammar.operator_fn(op.kind), [left, right], loc: loc(op))
+          left = parse_postfix(left)
+          skip_separators
+        end
+
+        left
+      end
+
+      # Trailing `[index]` accesses lower to `at(base, index)`.
+      def parse_postfix(base)
+        skip_separators
+        while at?(:lbracket)
+          advance
+          index = parse_expression
+          expect(:rbracket, '`]` to close the index')
+          base = Syntax::CallExpression.new(:at, [base, index], loc: base.loc)
+          skip_separators
+        end
+        base
+      end
+
+      def parse_primary
+        skip_separators
+        token = current
+
+        case token.kind
+        when :integer, :float, :string, :boolean, :symbol
+          advance
+          Syntax::Literal.new(token.value, loc: loc(token))
+        when :constant
+          advance
+          Syntax::Literal.new(resolve_constant(token), loc: loc(token))
+        when :function_sugar
+          parse_function_sugar
+        when :fn
+          parse_fn_call
+        when :input
+          parse_input_reference
+        when :identifier
+          parse_identifier_expression
+        when :lparen
+          advance
+          expr = parse_expression
+          expect(:rparen, '`)` to close the group')
+          expr
+        when :lbracket
+          parse_array_literal
+        when :lbrace
+          parse_hash_literal
+        when :subtract
+          parse_unary_minus
+        else
+          error("expected an expression, but found #{describe_token(token)}")
+        end
+      end
+
+      def parse_unary_minus
+        token = advance
+        operand = parse_postfix(parse_primary)
+        zero = Syntax::Literal.new(0, loc: loc(token))
+        Syntax::CallExpression.new(:subtract, [zero, operand], loc: loc(token))
+      end
+
+      def parse_identifier_expression
+        token = current
+        if token.value == 'input' && peek.kind == :dot
+          parse_input_reference
+        elsif peek.kind == :lparen
+          parse_named_call
+        else
+          advance
+          Syntax::DeclarationReference.new(token.value.to_sym, loc: loc(token))
+        end
+      end
+
+      # `input` is a keyword token; `input.field…` builds an input reference.
+      def parse_input_reference
+        input_token = advance
+        error('expected `input`', at: input_token) if input_token.kind == :identifier && input_token.value != 'input'
+        expect(:dot, '`.` after `input`')
+        path = [expect_field_name]
+        while at?(:dot)
+          advance
+          path << expect_field_name
+        end
+        if path.length == 1
+          Syntax::InputReference.new(path.first, loc: loc(input_token))
+        else
+          Syntax::InputElementReference.new(path, loc: loc(input_token))
+        end
+      end
+
+      def expect_field_name
+        token = current
+        if token.kind == :identifier || token.kind == :type_keyword || keyword_token?(token)
+          advance
+          field_name_value(token).to_sym
+        else
+          error("expected an input field name after `.`, but found #{describe_token(token)}")
+        end
+      end
+
+      def keyword_token?(token)
+        Grammar::KEYWORDS.value?(token.kind)
+      end
+
+      def field_name_value(token)
+        # type_keyword tokens carry the type symbol as value; reconstruct the word.
+        return Grammar::TYPE_KEYWORDS.key(token.value) if token.kind == :type_keyword
+
+        token.value
+      end
+
+      # ---- calls ---------------------------------------------------------
+
+      # `name(args)` sugar: `select(...)`, `cross(...)`, `index(:i)`, etc.
+      # Keyword args here are function options (`policy: :clamp`, `axis_offset: 1`)
+      # and are stored as raw scalars, the form the IR lowering reads.
+      def parse_function_sugar
+        token = advance
+        expect(:lparen, "`(` after #{Grammar::FUNCTION_SUGAR.key(token.value)}")
+        args, opts = parse_call_arguments(keyword_mode: :literal)
+        Syntax::CallExpression.new(token.value, args, opts, loc: loc(token))
+      end
+
+      # `fn(:name, args)` explicit form. Keyword args are function options (raw).
+      def parse_fn_call
+        fn_token = advance
+        expect(:lparen, '`(` after `fn`')
+        name = expect(:symbol, 'a `:function_name` inside `fn(...)`').value
+        args = []
+        opts = {}
+        if at?(:comma)
+          advance
+          args, opts = parse_call_arguments(keyword_mode: :literal)
+        else
+          expect(:rparen, '`)` to close `fn(...)`')
+        end
+        build_call(name, args, opts, fn_token)
+      end
+
+      # `name(...)` where `name` is a bare identifier — an imported schema call.
+      # Its keyword args are the input mapping, whose values are full
+      # expressions (e.g. `subtotal(items: input.order_items)`).
+      def parse_named_call
+        name_token = advance
+        name = name_token.value.to_sym
+        expect(:lparen, "`(` after `#{name}`")
+        args, opts = parse_call_arguments(keyword_mode: :expression)
+        build_call(name, args, opts, name_token)
+      end
+
+      def build_call(name, args, opts, token)
+        if @imported_names.include?(name) && args.empty? && !opts.empty?
+          Syntax::ImportCall.new(name, opts, loc: loc(token))
+        else
+          Syntax::CallExpression.new(name, args, opts, loc: loc(token))
+        end
+      end
+
+      # Parse the contents of a `(...)` argument list up to and including the
+      # closing paren: positional expression args first, then `label: value`
+      # keyword args. `keyword_mode` selects how keyword values are read:
+      # `:literal` (raw scalars, for function options) or `:expression` (full
+      # AST, for import input mappings).
+      def parse_call_arguments(keyword_mode:)
+        args = []
+        opts = {}
+        skip_separators
+        return [args, opts] if consume_rparen
+
+        loop do
+          if at?(:label)
+            key = advance.value.to_sym
+            opts[key] = keyword_mode == :literal ? parse_keyword_literal : parse_expression
+          else
+            args << parse_expression
+          end
+          skip_separators
+          break unless at?(:comma)
+
+          advance
+          skip_separators
+        end
+
+        expect(:rparen, '`)` to close the argument list')
+        [args, opts]
+      end
+
+      def consume_rparen
+        return false unless at?(:rparen)
+
+        advance
+        true
+      end
+
+      # A function option's value: a raw scalar, not an AST node. `:clamp` stays
+      # a symbol, `1` stays an Integer, a bare `label:`-style word becomes a
+      # symbol. The IR lowering reads these directly.
+      def parse_keyword_literal
+        token = current
+        case token.kind
+        when :integer, :float, :string, :boolean
+          advance.value
+        when :symbol
+          advance.value
+        when :label
+          advance.value.to_sym
+        when :subtract
+          advance
+          v = parse_keyword_literal
+          error('expected a number after unary `-`') unless v.is_a?(Numeric)
+          -v
+        else
+          error('a function option value must be a literal or symbol, ' \
+                "but found #{describe_token(token)}")
+        end
+      end
+
+      # ---- array / hash literals ----------------------------------------
+
+      def parse_array_literal
+        open = expect(:lbracket)
+        elements = []
+        skip_separators
+        until at?(:rbracket)
+          elements << parse_expression
+          skip_separators
+          break unless at?(:comma)
+
+          advance
+          skip_separators
+        end
+        expect(:rbracket, '`]` to close the array')
+        Syntax::ArrayExpression.new(elements, loc: loc(open))
+      end
+
+      def parse_hash_literal
+        open = expect(:lbrace)
+        pairs = []
+        skip_separators
+        until at?(:rbrace)
+          pairs << parse_hash_pair
+          skip_separators
+          break unless at?(:comma)
+
+          advance
+          skip_separators
+        end
+        expect(:rbrace, '`}` to close the hash')
+        Syntax::HashExpression.new(pairs, loc: loc(open))
+      end
+
+      def parse_hash_pair
+        key_token = current
+        key_value =
+          case key_token.kind
+          when :label  then advance && key_token.value.to_sym
+          when :string then advance && key_token.value
+          when :symbol then advance && key_token.value
+          else
+            error('a hash key must be a `name:` label, a `:symbol`, or a "string", ' \
+                  "but found #{describe_token(key_token)}")
+          end
+        key = Syntax::Literal.new(key_value, loc: loc(key_token))
+
+        skip_separators
+        if at?(:arrow)
+          advance
+        elsif key_token.kind != :label
+          error('expected `=>` after the hash key')
+        end
+        skip_separators
+        value = parse_expression
+        [key, value]
+      end
+
+      # ---- constants -----------------------------------------------------
+
+      # The parse phase resolves only the handful of constants whose value is
+      # part of the language surface; everything else is left to the analyzer.
+      KNOWN_CONSTANTS = { 'Float::INFINITY' => Float::INFINITY }.freeze
+
+      def resolve_constant(token)
+        KNOWN_CONSTANTS.fetch(token.value) do
+          error("unknown constant `#{token.value}` (use an inline value instead)", at: token)
+        end
+      end
+
+      # ---- errors --------------------------------------------------------
+
+      def error(message, at: current)
+        raise ParseError.new(message, source: @source, offset: at.offset)
+      end
+
+      def describe_token(token)
+        case token.kind
+        when :eof        then 'end of input'
+        when :newline    then 'a line break'
+        when :type_keyword then "`#{Grammar::TYPE_KEYWORDS.key(token.value)}`"
+        when :function_sugar then "`#{Grammar::FUNCTION_SUGAR.key(token.value)}`"
+        when :symbol     then "`:#{token.value}`"
+        when :label      then "`#{token.value}:`"
+        when :string     then token.value.inspect
+        when :integer, :float, :boolean then "`#{token.value}`"
+        else "`#{token.value}`"
+        end
+      end
+
+      def describe_kind(kind)
+        {
+          symbol: 'a `:name`',
+          comma: '`,`',
+          do: '`do`',
+          end: '`end`',
+          rparen: '`)`',
+          rbracket: '`]`',
+          rbrace: '`}`',
+          lparen: '`(`',
+          dot: '`.`',
+          arrow: '`=>`',
+          eof: 'end of input'
+        }.fetch(kind, "`#{kind}`")
+      end
+    end
+  end
+end