rjq 0.1.0

data/lib/rjq/lexer.rb

@@ -0,0 +1,344 @@
+# frozen_string_literal: true
+
+module Rjq
+  Token = Struct.new(:type, :value, :line, :column, :start_offset, :end_offset, :filename, keyword_init: true)
+  SourceFragment = Struct.new(:source, :filename, :line, :column, :start_offset, keyword_init: true)
+
+  class Lexer
+    KEYWORDS = %w[
+      if then elif else end as def reduce foreach try catch label import include module
+      null true false and or not break
+    ].freeze
+
+    TWO_CHAR_OPERATORS = ['//', '==', '!=', '<=', '>=', '|=', '+=', '-=', '*=', '/=', '%=', '..'].freeze
+    THREE_CHAR_OPERATORS = ['//='].freeze
+    SINGLE_CHAR_TOKENS = {
+      '.' => :dot,
+      '|' => :pipe,
+      ',' => :comma,
+      '(' => :lparen,
+      ')' => :rparen,
+      '[' => :lbracket,
+      ']' => :rbracket,
+      '{' => :lbrace,
+      '}' => :rbrace,
+      ':' => :colon,
+      ';' => :semicolon,
+      '?' => :question,
+      '+' => :operator,
+      '-' => :operator,
+      '*' => :operator,
+      '/' => :operator,
+      '%' => :operator,
+      '=' => :operator,
+      '<' => :operator,
+      '>' => :operator
+    }.freeze
+
+    attr_reader :tokens
+
+    def initialize(source, allow_comments: true, source_name: '<top-level>', initial_line: 1, initial_column: 1,
+                   start_offset: 0)
+      @source = source.to_s
+      @allow_comments = allow_comments
+      @source_name = source_name
+      @offset_base = start_offset
+      @index = 0
+      @line = initial_line
+      @column = initial_column
+      @tokens = []
+    end
+
+    def tokenize
+      until eof?
+        skip_space
+        break if eof?
+
+        start_line = @line
+        start_column = @column
+        start_offset = @index
+        char = current
+        token =
+          if char == '#'
+            unless @allow_comments
+              raise ParseError,
+                    "comments are disabled at line #{start_line}, column #{start_column}"
+            end
+
+            skip_comment
+            next
+          elsif char == '"'
+            Token.new(type: :string, value: read_string, line: start_line, column: start_column)
+          elsif char == '$'
+            advance
+            Token.new(type: :variable, value: read_identifier, line: start_line, column: start_column)
+          elsif char == '@'
+            advance
+            Token.new(type: :format, value: read_identifier, line: start_line, column: start_column)
+          elsif number_start?(char) || (char == '.' && digit?(@source[@index + 1]))
+            Token.new(type: :number, value: read_number, line: start_line, column: start_column)
+          elsif identifier_start?(char)
+            identifier = read_identifier
+            type = KEYWORDS.include?(identifier) ? :keyword : :identifier
+            Token.new(type: type, value: identifier, line: start_line, column: start_column)
+          else
+            read_operator_or_punctuation(start_line, start_column)
+          end
+        if token
+          token.start_offset = @offset_base + start_offset
+          token.end_offset = @offset_base + @index
+          token.filename = @source_name
+          @tokens << token
+        end
+      end
+      offset = @offset_base + @index
+      @tokens << Token.new(type: :eof, value: nil, line: @line, column: @column, start_offset: offset,
+                           end_offset: offset, filename: @source_name)
+      @tokens
+    end
+
+    private
+
+    def read_operator_or_punctuation(line, column)
+      three = @source[@index, 3]
+      if THREE_CHAR_OPERATORS.include?(three)
+        3.times { advance }
+        return Token.new(type: :operator, value: three, line: line, column: column)
+      end
+
+      two = @source[@index, 2]
+      if TWO_CHAR_OPERATORS.include?(two)
+        2.times { advance }
+        return Token.new(type: :operator, value: two, line: line, column: column)
+      end
+
+      type = SINGLE_CHAR_TOKENS[current]
+      raise ParseError, "unexpected character #{current.inspect} at line #{line}, column #{column}" unless type
+
+      value = current
+      advance
+      Token.new(type: type, value: value, line: line, column: column)
+    end
+
+    def read_string
+      expect('"')
+      segments = []
+      buffer = +''
+      until eof?
+        char = advance
+        if char == '"'
+          return segments.empty? ? buffer : segments_with_buffer(segments, buffer)
+        end
+
+        if char == '\\'
+          if current == '('
+            advance
+            segments << [:text, buffer] unless buffer.empty?
+            buffer = +''
+            fragment_line = @line
+            fragment_column = @column
+            fragment_offset = @offset_base + @index
+            fragment = SourceFragment.new(
+              source: read_interpolation,
+              filename: @source_name,
+              line: fragment_line,
+              column: fragment_column,
+              start_offset: fragment_offset
+            )
+            segments << [:expr, fragment]
+          else
+            buffer << read_escape
+          end
+        else
+          raise parse_error('unescaped control character in string') if char.ord < 0x20
+
+          buffer << char
+        end
+      end
+      raise parse_error('unterminated string')
+    end
+
+    def segments_with_buffer(segments, buffer)
+      segments << [:text, buffer] unless buffer.empty?
+      segments
+    end
+
+    def read_interpolation
+      frames = [{ type: :expression, depth: 1 }]
+      start = @index
+
+      until eof?
+        char = advance
+        frame = frames.last
+        case frame.fetch(:type)
+        when :comment
+          frames.pop if char == "\n"
+        when :string
+          if char == '\\'
+            if current == '('
+              advance
+              frames << { type: :expression, depth: 1 }
+            else
+              advance unless eof?
+            end
+          elsif char == frame.fetch(:quote)
+            frames.pop
+          end
+        when :expression
+          case char
+          when '#'
+            frames << { type: :comment }
+          when '"', "'"
+            frames << { type: :string, quote: char }
+          when '('
+            frame[:depth] += 1
+          when ')'
+            frame[:depth] -= 1
+            if frame.fetch(:depth).zero?
+              frames.pop
+              return @source[start...(@index - 1)] if frames.empty?
+            end
+          end
+        end
+      end
+
+      raise parse_error('unterminated interpolation')
+    end
+
+    def read_escape
+      raise parse_error('unterminated escape') if eof?
+
+      char = advance
+      case char
+      when '"', '\\', '/'
+        char
+      when 'b'
+        "\b"
+      when 'f'
+        "\f"
+      when 'n'
+        "\n"
+      when 'r'
+        "\r"
+      when 't'
+        "\t"
+      when 'u'
+        read_unicode_escape
+      else
+        raise parse_error("invalid escape \\#{char}")
+      end
+    end
+
+    def read_unicode_escape
+      codepoint = read_hex4
+      if codepoint.between?(0xD800, 0xDBFF)
+        raise parse_error('missing low surrogate') unless @source[@index, 2] == '\\u'
+
+        2.times { advance }
+        low = read_hex4
+        raise parse_error('invalid low surrogate') unless low.between?(0xDC00, 0xDFFF)
+
+        codepoint = 0x10000 + ((codepoint - 0xD800) << 10) + (low - 0xDC00)
+      elsif codepoint.between?(0xDC00, 0xDFFF)
+        raise parse_error('unexpected low surrogate')
+      end
+      [codepoint].pack('U')
+    end
+
+    def read_hex4
+      text = @source[@index, 4]
+      raise parse_error('invalid unicode escape') unless text&.match?(/\A[0-9a-fA-F]{4}\z/)
+
+      4.times { advance }
+      text.to_i(16)
+    end
+
+    def read_number
+      start = @index
+      leading_decimal = current == '.'
+      advance if leading_decimal
+      advance while digit?(current)
+      if !leading_decimal && current == '.'
+        advance
+        advance while digit?(current)
+      end
+      if %w[e E].include?(current)
+        advance
+        advance if ['+', '-'].include?(current)
+        advance while digit?(current)
+      end
+      text = @source[start...@index]
+      text = "0#{text}" if leading_decimal
+      Number.parse(text)
+    rescue ArgumentError
+      raise parse_error('invalid number')
+    end
+
+    def read_identifier
+      start = @index
+      raise parse_error('expected identifier') unless identifier_start?(current)
+
+      advance while identifier_part?(current)
+      @source[start...@index]
+    end
+
+    def skip_space
+      loop do
+        advance while current&.match?(/[ \t\r\n]/)
+        break unless @allow_comments && current == '#'
+
+        skip_comment
+      end
+    end
+
+    def skip_comment
+      advance until eof? || current == "\n"
+    end
+
+    def number_start?(char)
+      digit?(char)
+    end
+
+    def identifier_start?(char)
+      !char.nil? && char.match?(/[A-Za-z_]/)
+    end
+
+    def identifier_part?(char)
+      !char.nil? && char.match?(/[A-Za-z0-9_]/)
+    end
+
+    def digit?(char)
+      !char.nil? && char >= '0' && char <= '9'
+    end
+
+    def expect(char)
+      raise parse_error("expected #{char}") unless current == char
+
+      advance
+    end
+
+    def current
+      @source[@index]
+    end
+
+    def eof?
+      @index >= @source.length
+    end
+
+    def advance
+      char = @source[@index]
+      @index += char.length
+      if char == "\n"
+        @line += 1
+        @column = 1
+      else
+        @column += 1
+      end
+      char
+    end
+
+    def parse_error(message)
+      ParseError.new("#{message} at #{@source_name}, line #{@line}, column #{@column}")
+    end
+  end
+end

data/lib/rjq/math_functions.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+require 'fiddle/import'
+
+module Rjq
+  module MathFunctions
+    CANDIDATE_LIBRARIES = [
+      '/usr/lib/libSystem.B.dylib',
+      'libm.so.6',
+      'libm.so',
+      'libm.dylib'
+    ].freeze
+    NATIVE_SIGNATURES = {
+      fma: 'double fma(double, double, double)',
+      remainder: 'double remainder(double, double)',
+      scalb: 'double scalb(double, double)',
+      scalbln: 'double scalbln(double, long)'
+    }.freeze
+
+    module_function
+
+    def bessel(name, *values)
+      if %w[jn yn].include?(name)
+        bessel_library.public_send(name, values.fetch(0).to_i, values.fetch(1).to_f)
+      else
+        bessel_library.public_send(name, values.fetch(0).to_f)
+      end
+    end
+
+    def fma(left, right, addend)
+      library = native_library(:fma)
+      return library.fma(left.to_f, right.to_f, addend.to_f) if library
+
+      raise Rjq::RuntimeError, 'native fused multiply-add is not available on this platform'
+    end
+
+    def remainder(left, right)
+      library = native_library(:remainder)
+      return library.remainder(left.to_f, right.to_f) if library
+
+      raise Rjq::RuntimeError, 'native IEEE remainder is not available on this platform'
+    end
+
+    def scalb(value, exponent)
+      library = native_library(:scalb)
+      if library
+        result = library.scalb(value.to_f, exponent.to_f)
+        return signed_max(value.to_f) if result.infinite?
+
+        return result
+      end
+
+      exponent = exponent.to_f
+      return Float::NAN if exponent.nan?
+      return portable_scale(value.to_f, exponent.positive? ? 10_000 : -10_000) if exponent.infinite?
+
+      portable_scale(value.to_f, exponent.to_i)
+    end
+
+    def scalbln(value, exponent)
+      exponent = exponent.to_f
+      if RUBY_PLATFORM.include?('linux') && !exponent.finite?
+        # jq's Linux build passes non-finite values through a C long
+        # conversion, which yields zero on the supported libc targets.
+        return 0.0
+      end
+
+      integral_exponent = c_long_exponent(exponent)
+      library = native_library(:scalbln)
+      if library
+        result = library.scalbln(value.to_f, integral_exponent)
+        return signed_max(value.to_f) if result.infinite?
+
+        return result
+      end
+
+      portable_scale(value.to_f, integral_exponent)
+    end
+
+    def native_available?(name)
+      !native_library(name).nil?
+    end
+
+    def native_library(name)
+      @native_libraries ||= {}
+      return @native_libraries[name] if @native_libraries.key?(name)
+
+      signature = NATIVE_SIGNATURES.fetch(name)
+      @native_libraries[name] = CANDIDATE_LIBRARIES.lazy.filter_map do |library|
+        build_native_library(library, signature)
+      rescue Fiddle::DLError
+        nil
+      end.first
+    end
+
+    def build_native_library(library, signature)
+      Module.new do
+        extend Fiddle::Importer
+
+        dlload library
+        extern signature
+      end
+    end
+
+    def c_long_exponent(value)
+      bits = Fiddle::SIZEOF_LONG * 8
+      minimum = -(1 << (bits - 1))
+      maximum = (1 << (bits - 1)) - 1
+      return 0 if value.nan?
+      return value.positive? ? maximum : minimum if value.infinite?
+
+      [[value.to_i, minimum].max, maximum].min
+    end
+
+    def portable_scale(value, exponent)
+      return value if value.zero?
+      return signed_max(value) if value.infinite?
+
+      _, value_exponent = Math.frexp(value.abs)
+      target_exponent = value_exponent + exponent
+      return signed_max(value) if target_exponent > 1024
+      return value.negative? ? -0.0 : 0.0 if target_exponent < -1074
+
+      result = Math.ldexp(value, exponent)
+      result.infinite? ? signed_max(value) : result
+    end
+
+    def signed_max(value)
+      value.negative? ? -Float::MAX : Float::MAX
+    end
+
+    def bessel_library
+      @bessel_library ||= load_bessel_library
+    end
+
+    def bessel_available?
+      bessel_library
+      true
+    rescue Rjq::RuntimeError
+      false
+    end
+
+    def load_bessel_library
+      errors = []
+      CANDIDATE_LIBRARIES.each do |library|
+        return build_bessel_library(library)
+      rescue Fiddle::DLError => e
+        errors << "#{library}: #{e.message}"
+      end
+
+      raise Rjq::RuntimeError, "C math library with Bessel functions is not available (#{errors.join('; ')})"
+    end
+
+    def build_bessel_library(library)
+      Module.new do
+        extend Fiddle::Importer
+
+        dlload library
+        extern 'double j0(double)'
+        extern 'double j1(double)'
+        extern 'double y0(double)'
+        extern 'double y1(double)'
+        extern 'double jn(int, double)'
+        extern 'double yn(int, double)'
+      end
+    end
+  end
+end

data/lib/rjq/module_loader.rb

@@ -0,0 +1,178 @@
+# frozen_string_literal: true
+
+module Rjq
+  class ModuleLoader
+    Result = Struct.new(:program, :metadata, :variables, keyword_init: true)
+    MAX_DEPTH = 64
+
+    def initialize(resolver, allow_comments: true, max_filter_depth: Parser::DEFAULT_MAX_FILTER_DEPTH)
+      @resolver = resolver
+      @cache = {}
+      @allow_comments = allow_comments
+      @max_filter_depth = max_filter_depth
+    end
+
+    def load(program, source_path: nil, stack: [])
+      raise CompileError, "module import depth exceeds #{MAX_DEPTH}" if stack.length >= MAX_DEPTH
+      program.directives.select { |directive| directive.type == :module }.each do |directive|
+        ConstantEvaluator.evaluate_object(directive.metadata)
+      end
+
+      definitions = []
+      variables = {}
+      metadata = @resolver.initial_metadata.dup
+      program.directives.each do |directive|
+        next if directive.type == :module
+
+        options = directive.metadata ? ConstantEvaluator.evaluate_object(directive.metadata) : {}
+        data = directive.alias_name&.start_with?('$') || false
+        source = @resolver.resolve(directive.name, from: source_path, metadata: options, data: data)
+        raise CompileError, "circular module import: #{directive.name}" if stack.include?(source.path)
+
+        if data
+          value = JSON::Parser.parse_one(source.content)
+          name = directive.alias_name.delete_prefix('$')
+          variables[name] = value
+          definitions << AST::FunctionDefinition.new("#{name}::#{name}", [], AST::Literal.new(value))
+          metadata[directive.name] ||= data_metadata
+          next
+        end
+
+        loaded, parsed = @cache[source.path]
+        unless loaded
+          parsed = Parser.new(source.content, source_name: source.path, allow_comments: @allow_comments,
+                                              max_filter_depth: @max_filter_depth).parse
+          loaded = load(parsed, source_path: source.path, stack: stack + [source.path])
+          @cache[source.path] = [loaded, parsed]
+        end
+        metadata.merge!(loaded.metadata)
+        metadata[directive.name] = metadata_for(parsed)
+        variables.merge!(loaded.variables)
+        imported = loaded.program.definitions
+        imported = namespace_definitions(imported, directive.alias_name) if directive.type == :import
+        definitions.concat(imported)
+      end
+      definitions.concat(program.definitions)
+      Result.new(
+        program: AST::Program.new(program.body, definitions, []),
+        metadata: metadata,
+        variables: variables
+      )
+    end
+
+    private
+
+    def metadata_for(program)
+      declaration = program.directives.find { |directive| directive.type == :module }
+      object = declaration ? ConstantEvaluator.evaluate_object(declaration.metadata) : { 'whatever' => nil }
+      object.merge(
+        'deps' => program.directives.filter_map { |directive| dependency_metadata(directive) },
+        'defs' => program.definitions.map { |definition| "#{definition.name}/#{definition.params.length}" }
+      )
+    end
+
+    def dependency_metadata(directive)
+      return if directive.type == :module
+
+      data = directive.alias_name&.start_with?('$') || false
+      metadata = { 'is_data' => data, 'relpath' => directive.name }
+      metadata.merge!(ConstantEvaluator.evaluate_object(directive.metadata)) if directive.metadata
+      metadata['as'] = directive.alias_name.delete_prefix('$') if directive.alias_name
+      metadata['as'] ||= File.basename(directive.name) if directive.type == :include
+      metadata
+    end
+
+    def data_metadata
+      { 'whatever' => nil, 'deps' => [], 'defs' => [] }
+    end
+
+    def namespace_definitions(definitions, namespace)
+      cloned = Marshal.load(Marshal.dump(definitions))
+      signatures = cloned.to_h { |definition| [[definition.name, definition.params.length], true] }
+      cloned.each do |definition|
+        namespace_calls(definition.body, namespace, signatures)
+        definition.instance_variable_set(:@name, "#{namespace}::#{definition.name}")
+      end
+      cloned
+    end
+
+    def namespace_calls(value, namespace, signatures)
+      if value.is_a?(AST::FunctionCall)
+        name = value.name
+        value.instance_variable_set(:@name, "#{namespace}::#{name}") if signatures[[name, value.args.length]]
+      end
+      nested_values(value).each { |child| namespace_calls(child, namespace, signatures) }
+    end
+
+    def nested_values(value)
+      case value
+      when AST::Node
+        value.instance_variables.map { |name| value.instance_variable_get(name) }
+      when Array
+        value
+      when Hash
+        value.values
+      when Struct
+        value.to_a
+      else
+        []
+      end
+    end
+  end
+
+  class ConstantEvaluator
+    class << self
+      def evaluate_object(node)
+        value = evaluate(node)
+        raise CompileError, 'Module metadata must be an object' unless value.is_a?(Hash)
+
+        value
+      end
+
+      def evaluate(node)
+        case node
+        when AST::Literal
+          Value.deep_copy(node.value)
+        when AST::StringLiteral
+          raise CompileError, 'Module metadata must be constant' unless node.value.is_a?(String)
+
+          node.value
+        when AST::ArrayLiteral
+          expression = node.instance_variable_get(:@expression)
+          expression ? evaluate_sequence(expression) : []
+        when AST::ObjectLiteral
+          evaluate_object_literal(node)
+        when AST::UnaryOp
+          evaluate_unary(node)
+        else
+          raise CompileError, 'Module metadata must be constant'
+        end
+      end
+
+      private
+
+      def evaluate_sequence(node)
+        return evaluate_sequence(node.left) + evaluate_sequence(node.right) if node.is_a?(AST::Comma)
+
+        [evaluate(node)]
+      end
+
+      def evaluate_object_literal(node)
+        node.instance_variable_get(:@pairs).to_h do |pair|
+          key = pair.key
+          raise CompileError, 'Module metadata must be constant' unless key.is_a?(String)
+
+          [key, evaluate(pair.value)]
+        end
+      end
+
+      def evaluate_unary(node)
+        op = node.instance_variable_get(:@op)
+        value = evaluate(node.instance_variable_get(:@expression))
+        raise CompileError, 'Module metadata must be constant' unless op == '-' && value.is_a?(Numeric)
+
+        -value
+      end
+    end
+  end
+end