rjq 0.1.0

data/lib/rjq/compiler.rb

@@ -0,0 +1,392 @@
+# frozen_string_literal: true
+
+module Rjq
+  class CompiledProgram
+    attr_reader :ast, :program
+
+    def initialize(ast, program:)
+      @ast = ast
+      @program = program
+      freeze
+    end
+
+    def instructions
+      program.instructions
+    end
+
+    def run(input_value, opts = {})
+      VM.new(self, Runtime.normalize_options(opts)).run(input_value)
+    end
+
+    def run_with_instruction_budget(input_value, opts, budget)
+      normalized = Runtime.normalize_options(opts)
+      unless budget.is_a?(VM::InstructionBudget) && budget.maximum == normalized[:max_instructions]
+        raise ArgumentError, 'instruction budget must match max_instructions'
+      end
+
+      VM.new(self, normalized, instruction_budget: budget).run(input_value)
+    end
+
+    def disasm
+      program.disasm
+    end
+  end
+
+  class BytecodeCompiler
+    def initialize(allow_comments: true, max_filter_depth: Parser::DEFAULT_MAX_FILTER_DEPTH)
+      @constants = []
+      @constant_indices = {}
+      @current_nodes = []
+      @allow_comments = allow_comments
+      @max_filter_depth = max_filter_depth
+    end
+
+    def compile(ast, module_metadata: {}, module_variables: {})
+      instructions = compile_node(ast.body)
+      Rjq::Program.new(
+        instructions: instructions,
+        constants: @constants,
+        definitions: ast.definitions.map { |definition| compile_definition(definition) },
+        module_metadata: module_metadata,
+        module_variables: module_variables
+      )
+    end
+
+    private
+
+    def compile_node(node)
+      @current_nodes << node
+      case node
+      when AST::Identity
+        [instruction(:load_input)]
+      when AST::Literal
+        [instruction(:load_const, const(node.value))]
+      when AST::StringLiteral
+        compile_string(node)
+      when AST::Format
+        compile_format(node)
+      when AST::Variable
+        [instruction(:variable, node.name, nil, loc: node.source_span)]
+      when AST::Field
+        compile_node(node.base) + [instruction(:field, node.name)]
+      when AST::Index
+        compile_index(node)
+      when AST::Slice
+        compile_slice(node)
+      when AST::Iterate
+        compile_node(node.base) + [instruction(:each)]
+      when AST::Optional
+        [instruction(:optional, block_for(node.node))]
+      when AST::Pipe
+        compile_node(node.left) + [instruction(:pipe, block_for(node.right))]
+      when AST::Comma
+        compile_node(node.left) + [instruction(:append, block_for(node.right))]
+      when AST::Binding
+        [instruction(:binding, {
+                       source: block_for(node.source),
+                       pattern: node.pattern,
+                       body: block_for(node.body)
+                     })]
+      when AST::ArrayLiteral
+        expression = node.expression
+        [instruction(:array, expression ? block_for(expression) : nil)]
+      when AST::ObjectLiteral
+        [instruction(:object, compile_object_pairs(node.pairs))]
+      when AST::FunctionCall
+        compile_call(node)
+      when AST::BinaryOp
+        compile_binary(node)
+      when AST::UnaryOp
+        [instruction(:unary, node.op, block_for(node.expression))]
+      when AST::If
+        compile_if(node)
+      when AST::Try
+        [instruction(:try, { body: block_for(node.body), handler: optional_block(node.handler) })]
+      when AST::Reduce
+        [instruction(:reduce, {
+                       generator: block_for(node.generator),
+                       pattern: node.variable,
+                       initial: block_for(node.initial),
+                       update: block_for(node.update)
+                     })]
+      when AST::Foreach
+        [instruction(:foreach, {
+                       generator: block_for(node.generator),
+                       pattern: node.variable,
+                       initial: block_for(node.initial),
+                       update: block_for(node.update),
+                       extract: optional_block(node.extract)
+                     })]
+      when AST::Label
+        [instruction(:label, node.label, block_for(node.body))]
+      when AST::Break
+        [instruction(:break, node.label)]
+      when AST::Assignment
+        [instruction(:assign,
+                     { left: block_for(node.left), op: node.op, right: block_for(node.right) })]
+      when AST::ScopedDefinition
+        [instruction(:scoped_def, compile_definition(node.definition), block_for(node.body))]
+      when AST::Recurse
+        [instruction(:recurse)]
+      else
+        raise CompileError, "unsupported AST node #{node.class}"
+      end
+    ensure
+      @current_nodes.pop
+    end
+
+    def compile_string(node)
+      value = node.value
+      return [instruction(:load_const, const(value))] if value.is_a?(String)
+
+      [instruction(:string_interp, value.map { |kind, segment| compile_string_segment(kind, segment) })]
+    end
+
+    def compile_string_segment(kind, value)
+      return { kind: :text, value: value } if kind == :text
+
+      fragment = value.is_a?(SourceFragment) ? value : SourceFragment.new(
+        source: value, filename: '<top-level>', line: 1, column: 1, start_offset: 0
+      )
+      parsed = Parser.new(fragment.source, allow_comments: @allow_comments, source_name: fragment.filename,
+                                          initial_line: fragment.line, initial_column: fragment.column,
+                                          start_offset: fragment.start_offset,
+                                          max_filter_depth: @max_filter_depth).parse
+      { kind: :expr, block: block_for(parsed.body), definitions: parsed.definitions.map do |definition|
+        compile_definition(definition)
+      end }
+    end
+
+    def compile_format(node)
+      expression = node.expression
+      return [instruction(:format, node.name, nil)] unless expression
+
+      if expression.is_a?(AST::StringLiteral) && expression.value.is_a?(Array)
+        return [instruction(:format, node.name, { segments: expression.value.map do |kind, value|
+          compile_string_segment(kind, value)
+        end })]
+      end
+
+      [instruction(:format, node.name, { block: block_for(expression) })]
+    end
+
+    def compile_index(node)
+      base = compile_node(node.base)
+      index = node.index
+      return base + [instruction(:index_const, index.value)] if index.is_a?(AST::Literal)
+
+      base + [instruction(:index_filter, block_for(index))]
+    end
+
+    def compile_slice(node)
+      start_node = node.start_node
+      finish_node = node.finish_node
+      base = compile_node(node.base)
+      if literal_or_nil?(start_node) && literal_or_nil?(finish_node)
+        return base + [instruction(:slice_const, literal_value(start_node), literal_value(finish_node))]
+      end
+
+      base + [instruction(:slice_filter, { start: optional_block(start_node), finish: optional_block(finish_node) })]
+    end
+
+    def compile_call(node)
+      arg_blocks = node.args.map { |arg| block_for(arg) }
+      return [instruction(:path, arg_blocks.first)] if node.name == 'path' && arg_blocks.length == 1
+
+      [instruction(:call, node.name, arg_blocks)]
+    end
+
+    def compile_binary(node)
+      [instruction(:binary, node.op, [block_for(node.left), block_for(node.right)])]
+    end
+
+    def compile_if(node)
+      [instruction(
+        :branch,
+        block_for(node.condition),
+        [block_for(node.then_branch), block_for(node.else_branch)]
+      )]
+    end
+
+    def block_for(node)
+      BytecodeBlock.new(instructions: compile_node(node))
+    end
+
+    def optional_block(node)
+      node ? block_for(node) : nil
+    end
+
+    def compile_definition(definition)
+      compiled = BytecodeFunctionDefinition.new(
+        name: definition.name,
+        params: definition.params,
+        body: block_for(definition.body)
+      )
+      mark_tail_calls(compiled.body)
+      compiled
+    end
+
+    def mark_tail_calls(block)
+      instruction = block.instructions.last
+      return unless instruction
+
+      case instruction.op
+      when :call
+        instruction.op = :tail_call
+      when :pipe
+        mark_tail_calls(instruction.arg1)
+      when :branch
+        instruction.arg2.each { |branch| mark_tail_calls(branch) }
+      when :binding
+        mark_tail_calls(instruction.arg1.fetch(:body))
+      when :append
+        mark_tail_calls(instruction.arg1)
+      when :scoped_def
+        mark_tail_calls(instruction.arg2)
+      end
+    end
+
+    def compile_object_pairs(pairs)
+      pairs.map do |pair|
+        {
+          key: compile_object_key(pair.key),
+          value: block_for(pair.value)
+        }
+      end
+    end
+
+    def compile_object_key(key)
+      return { type: :filter, block: block_for(key) } if key.is_a?(AST::Node)
+      return { type: :filter, block: block_for(AST::StringLiteral.new(key)) } if key.is_a?(Array)
+
+      { type: :literal, value: key }
+    end
+
+    def literal_or_nil?(node)
+      node.nil? || node.is_a?(AST::Literal)
+    end
+
+    def literal_value(node)
+      node&.value
+    end
+
+    def const(value)
+      copy = Value.deep_copy(value)
+      key = JSON::Dumper.dump(copy, indent: nil, sort_keys: true)
+      return @constant_indices.fetch(key) if @constant_indices.key?(key)
+
+      @constants << copy
+      @constant_indices[key] = @constants.length - 1
+    end
+
+    def instruction(op, arg1 = nil, arg2 = nil, loc: nil)
+      inherited_location = @current_nodes.reverse_each.lazy.filter_map(&:source_span).first
+      location = loc || inherited_location || AST::SourceSpan.new(
+        filename: '<top-level>', line: 1, column: 1, start_offset: 0, end_offset: 0
+      )
+      Instruction.new(op: op, arg1: arg1, arg2: arg2, loc: location)
+    end
+  end
+
+  class Compiler
+    OPTION_KEYS = %i[allow_comments library_path max_filter_depth module_resolver source_path].freeze
+
+    class << self
+      def options_from(opts)
+        opts.slice(*OPTION_KEYS)
+      end
+
+      def validate_options!(opts)
+        raise ArgumentError, 'options must be a Hash' unless opts.is_a?(Hash)
+
+        unknown = opts.keys - OPTION_KEYS
+        raise ArgumentError, "unknown compiler option: #{unknown.first.inspect}" unless unknown.empty?
+
+        validate_boolean!(opts, :allow_comments)
+        validate_optional_string!(opts, :source_path)
+        validate_positive_integer!(opts, :max_filter_depth)
+        validate_library_path!(opts[:library_path]) if opts.key?(:library_path)
+        validate_module_resolver!(opts[:module_resolver]) if opts.key?(:module_resolver)
+        opts
+      end
+
+      def normalize_options(opts)
+        validate_options!(opts)
+        normalized = opts.dup
+        normalized[:library_path] = normalized[:library_path].dup.freeze if normalized[:library_path]
+        normalized.freeze
+      end
+
+      private
+
+      def validate_boolean!(opts, key)
+        return unless opts.key?(key)
+        return if opts[key] == true || opts[key] == false
+
+        raise ArgumentError, "#{key} must be true or false"
+      end
+
+      def validate_optional_string!(opts, key)
+        return unless opts.key?(key)
+        return if opts[key].nil? || opts[key].is_a?(String)
+
+        raise ArgumentError, "#{key} must be a String or nil"
+      end
+
+      def validate_positive_integer!(opts, key)
+        return unless opts.key?(key)
+        return if opts[key].is_a?(Integer) && opts[key].positive?
+
+        raise ArgumentError, "#{key} must be a positive Integer"
+      end
+
+      def validate_library_path!(paths)
+        return if paths.is_a?(Array) && paths.all? { |path| path.is_a?(String) }
+
+        raise ArgumentError, 'library_path must be an Array of Strings'
+      end
+
+      def validate_module_resolver!(resolver)
+        return if resolver.nil?
+        return if resolver.respond_to?(:resolve) && resolver.respond_to?(:initial_metadata)
+
+        raise ArgumentError, 'module_resolver must be nil or respond to resolve and initial_metadata'
+      end
+    end
+
+    def initialize(opts = {})
+      @opts = self.class.normalize_options(opts)
+    end
+
+    def compile(filter_string)
+      max_filter_depth = @opts.fetch(:max_filter_depth, Parser::DEFAULT_MAX_FILTER_DEPTH)
+      source_path = @opts[:source_path] || '<top-level>'
+      parsed = Parser.new(filter_string, allow_comments: @opts.fetch(:allow_comments, true),
+                                         source_name: source_path, max_filter_depth: max_filter_depth).parse
+      resolver = @opts[:module_resolver] || default_module_resolver
+      allow_comments = @opts.fetch(:allow_comments, true)
+      loaded = ModuleLoader.new(resolver, allow_comments: allow_comments,
+                                          max_filter_depth: max_filter_depth).load(parsed,
+                                                                                   source_path: @opts[:source_path])
+      ast = loaded.program
+      program = BytecodeCompiler.new(allow_comments: allow_comments, max_filter_depth: max_filter_depth).compile(
+        ast,
+        module_metadata: loaded.metadata,
+        module_variables: loaded.variables
+      )
+      SemanticAnalyzer.new(program).validate!
+      program.finalize!
+      CompiledProgram.new(ast, program: program)
+    rescue SystemStackError
+      raise CompileError, "filter nesting exceeds safe parser/compiler depth in #{source_path}"
+    end
+
+    private
+
+    def default_module_resolver
+      paths = @opts.fetch(:library_path, [])
+      ModuleResolver.new(paths: paths, use_default_paths: paths.empty?)
+    end
+
+  end
+end

data/lib/rjq/errors.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Rjq
+  class Error < StandardError; end
+
+  class ParseError < Error; end
+
+  class CompileError < Error; end
+
+  class RuntimeError < Error
+    attr_reader :outputs
+
+    def prepend_outputs(values)
+      @outputs = Array(values) + Array(@outputs)
+      self
+    end
+
+    def take_outputs
+      values = Array(@outputs)
+      @outputs = nil
+      values
+    end
+  end
+
+  # Execution budgets are process-safety boundaries and cannot be caught by
+  # jq filters such as `try` or `?`.
+  class ResourceLimitError < RuntimeError; end
+
+  class TypeError < RuntimeError; end
+
+  class InvalidPathError < TypeError
+    attr_reader :result
+
+    def initialize(message, result, outputs: nil)
+      @result = result
+      prepend_outputs(outputs)
+      super(message)
+    end
+  end
+
+  class ErrorValue < RuntimeError
+    attr_reader :value
+
+    def initialize(value, outputs: nil)
+      @value = value
+      super(value.to_s)
+      prepend_outputs(outputs)
+    end
+  end
+
+  # Halting terminates the whole jq program. It is deliberately not a
+  # RuntimeError: try/catch and the optional operator only catch ordinary
+  # filter failures.
+  class HaltError < Error
+    attr_reader :value, :status
+
+    def initialize(value = nil, status = 0)
+      @value = value
+      @status = status
+      super(value.nil? ? 'halt_error' : value.to_s)
+    end
+  end
+
+  class JSONParseError < Error; end
+
+  class BreakSignal < RuntimeError
+    attr_reader :label, :value, :outputs
+
+    def initialize(label, value = nil, outputs: nil)
+      @label = label
+      @value = value
+      @outputs = outputs
+      super("break #{label}")
+    end
+  end
+end

data/lib/rjq/json/dumper.rb

@@ -0,0 +1,191 @@
+# frozen_string_literal: true
+
+module Rjq
+  module JSON
+    module Dumper
+      module_function
+
+      def dump(value, indent: 2, sort_keys: false, ascii: false, tab: false, io: nil)
+        destination = io || +''
+        step = tab ? "\t" : (' ' * Integer(indent || 0))
+        write(value, destination, indent.nil? ? nil : step, sort_keys, ascii)
+        io || destination
+      end
+
+      def write(value, destination, indent, sort_keys, ascii)
+        active = {}
+        stack = [[:value, value, 0]]
+        until stack.empty?
+          type, item, depth = stack.pop
+          case type
+          when :text
+            destination << item
+          when :leave
+            active.delete(item)
+          when :array_items
+            write_array_item(item, depth, destination, stack, indent)
+          when :object_items
+            write_object_item(item, depth, destination, stack, indent, ascii)
+          when :value
+            write_value(item, depth, destination, stack, active, indent, sort_keys, ascii)
+          end
+        end
+      end
+      private_class_method :write
+
+      def write_value(value, depth, destination, stack, active, indent, sort_keys, ascii)
+        case value
+        when NilClass
+          destination << 'null'
+        when TrueClass
+          destination << 'true'
+        when FalseClass
+          destination << 'false'
+        when Number
+          destination << value.dump
+        when Integer
+          destination << value.to_s
+        when Float
+          destination << dump_float(value)
+        when String
+          destination << dump_string(value, ascii)
+        when Array
+          write_array(value, depth, destination, stack, active, indent)
+        when Hash
+          write_object(value, depth, destination, stack, active, indent, sort_keys, ascii)
+        else
+          raise TypeError, "unsupported value type: #{value.class}"
+        end
+      end
+      private_class_method :write_value
+
+      def write_array(value, depth, destination, stack, active, indent)
+        return destination << '[]' if value.empty?
+
+        enter_container(value, active)
+        destination << (indent ? "[\n" : '[')
+        stack << [:leave, value.object_id, nil]
+        stack << [:text, indent ? "\n#{indent * depth}]" : ']', nil]
+        stack << [:array_items, [value, 0], depth]
+      end
+      private_class_method :write_array
+
+      def write_array_item(payload, depth, destination, stack, indent)
+        value, index = payload
+        return if index >= value.length
+
+        destination << if index.zero?
+                         indent ? indent * (depth + 1) : ''
+                       else
+                         indent ? ",\n#{indent * (depth + 1)}" : ','
+                       end
+        stack << [:array_items, [value, index + 1], depth]
+        stack << [:value, value[index], depth + 1]
+      end
+      private_class_method :write_array_item
+
+      def write_object(value, depth, destination, stack, active, indent, sort_keys, ascii)
+        return destination << '{}' if value.empty?
+
+        validate_object_keys!(value)
+        enter_container(value, active)
+        destination << (indent ? "{\n" : '{')
+        keys = sort_keys ? value.keys.sort : value.keys
+        stack << [:leave, value.object_id, nil]
+        stack << [:text, indent ? "\n#{indent * depth}}" : '}', nil]
+        stack << [:object_items, [value, keys, 0], depth]
+      end
+      private_class_method :write_object
+
+      def write_object_item(payload, depth, destination, stack, indent, ascii)
+        value, keys, index = payload
+        return if index >= keys.length
+
+        destination << if index.zero?
+                         indent ? indent * (depth + 1) : ''
+                       else
+                         indent ? ",\n#{indent * (depth + 1)}" : ','
+                       end
+        key = keys[index]
+        destination << dump_string(key, ascii)
+        destination << (indent ? ': ' : ':')
+        stack << [:object_items, [value, keys, index + 1], depth]
+        stack << [:value, value.fetch(key), depth + 1]
+      end
+      private_class_method :write_object_item
+
+      def enter_container(value, active)
+        raise TypeError, 'cannot dump a cyclic JSON value' if active[value.object_id]
+
+        active[value.object_id] = true
+      end
+      private_class_method :enter_container
+
+      def validate_object_keys!(value)
+        invalid = value.keys.find { |key| !key.is_a?(String) }
+        raise TypeError, "object key must be a string, got #{invalid.class}" if invalid
+      end
+      private_class_method :validate_object_keys!
+
+      def dump_float(value)
+        return 'null' if value.nan?
+        return Float::MAX.to_s if value.infinite? == 1
+        return "-#{Float::MAX}" if value.infinite? == -1
+        return '-0' if value.zero? && (1.0 / value).infinite? == -1
+
+        text = value.to_s.sub(/\.0(?=e)/, '')
+        if value == value.to_i
+          integer_text = value.to_i.to_s
+          return integer_text if value.abs < 1e16 || integer_text.length <= text.length
+        end
+        text.include?('.') || text.match?(/[eE]/) ? text : "#{text}.0"
+      end
+      private_class_method :dump_float
+
+      def dump_string(value, ascii)
+        string = value.encode(Encoding::UTF_8)
+        raise TypeError, 'invalid UTF-8 string' unless string.valid_encoding?
+
+        escaped = string.each_codepoint.map { |codepoint| escape_codepoint(codepoint, ascii) }.join
+        "\"#{escaped}\""
+      rescue EncodingError
+        raise TypeError, 'invalid UTF-8 string'
+      end
+      private_class_method :dump_string
+
+      def escape_codepoint(codepoint, ascii)
+        case codepoint
+        when 0x22
+          '\"'
+        when 0x5C
+          '\\\\'
+        when 0x08
+          '\\b'
+        when 0x0C
+          '\\f'
+        when 0x0A
+          '\\n'
+        when 0x0D
+          '\\r'
+        when 0x09
+          '\\t'
+        when 0x00..0x1F
+          '\\u%04x' % codepoint
+        else
+          ascii && codepoint > 0x7F ? unicode_escape(codepoint) : [codepoint].pack('U')
+        end
+      end
+      private_class_method :escape_codepoint
+
+      def unicode_escape(codepoint)
+        return '\\u%04x' % codepoint if codepoint <= 0xFFFF
+
+        n = codepoint - 0x10000
+        high = 0xD800 + (n >> 10)
+        low = 0xDC00 + (n & 0x3FF)
+        format('\\u%04x\\u%04x', high, low)
+      end
+      private_class_method :unicode_escape
+    end
+  end
+end