jrf 0.1.0

data/lib/jrf/reducers.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Jrf
+  module Reducers
+    module_function
+
+    class Reduce
+      def initialize(initial, finish_fn: nil, &step_fn)
+        @acc = initial
+        @step_fn = step_fn
+        @finish_fn = finish_fn || ->(acc) { acc }
+      end
+
+      def step(value)
+        @acc = @step_fn.call(@acc, value)
+      end
+
+      def finish
+        @finish_fn.call(@acc)
+      end
+    end
+
+    def reduce(initial, finish: nil, &step_fn)
+      Reduce.new(initial, finish_fn: finish, &step_fn)
+    end
+  end
+end

data/lib/jrf/row_context.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+require_relative "control"
+require_relative "reducers"
+
+module Jrf
+  class RowContext
+    MISSING = Object.new
+    ReducerToken = Struct.new(:index)
+
+    class << self
+      def define_reducer(name, &definition)
+        define_method(name) do |*args, **kwargs, &block|
+          spec = definition.call(self, *args, **kwargs, block: block)
+          create_reducer(
+            spec.fetch(:value),
+            initial: reducer_initial_value(spec.fetch(:initial)),
+            finish: spec[:finish],
+            emit_many: spec.fetch(:emit_many, false),
+            &spec.fetch(:step)
+          )
+        end
+      end
+    end
+
+    def initialize(obj = nil)
+      @obj = obj
+      @__jrf_stage = nil
+    end
+
+    def reset(obj)
+      @obj = obj
+      self
+    end
+
+    def _
+      @obj
+    end
+
+    def flat
+      Control::Flat.new(@obj)
+    end
+
+    define_reducer(:sum) do |_ctx, value, initial: 0, block: nil|
+      { value: value, initial: initial, step: ->(acc, v) { acc + v } }
+    end
+
+    define_reducer(:min) do |_ctx, value, block: nil|
+      { value: value, initial: nil, step: ->(acc, v) { acc.nil? || v < acc ? v : acc } }
+    end
+
+    define_reducer(:max) do |_ctx, value, block: nil|
+      { value: value, initial: nil, step: ->(acc, v) { acc.nil? || v > acc ? v : acc } }
+    end
+
+    define_reducer(:average) do |_ctx, value, block: nil|
+      {
+        value: value,
+        initial: -> { [0.0, 0] },
+        finish: ->((sum, count)) { count.zero? ? nil : (sum / count) },
+        step: ->(acc, v) {
+          acc[0] += v
+          acc[1] += 1
+          acc
+        }
+      }
+    end
+
+    define_reducer(:stdev) do |_ctx, value, sample: false, block: nil|
+      {
+        value: value,
+        initial: [0, 0.0, 0.0],
+        finish: ->((count, mean, m2)) {
+          return nil if count.zero?
+          return nil if sample && count < 2
+
+          denom = sample ? (count - 1) : count
+          Math.sqrt(m2 / denom)
+        },
+        step: ->(acc, x) {
+          count, mean, m2 = acc
+          count += 1
+          delta = x - mean
+          mean += delta / count
+          delta2 = x - mean
+          m2 += delta * delta2
+          acc[0] = count
+          acc[1] = mean
+          acc[2] = m2
+          acc
+        }
+      }
+    end
+
+    define_reducer(:sort) do |ctx, key = MISSING, block: nil|
+      if block
+        {
+          value: ctx._,
+          initial: -> { [] },
+          emit_many: true,
+          finish: ->(rows) { rows.sort(&block) },
+          step: ->(rows, row) { rows << row }
+        }
+      else
+        resolved_key = key.equal?(MISSING) ? ctx._ : key
+        {
+          value: [resolved_key, ctx._],
+          initial: -> { [] },
+          emit_many: true,
+          finish: ->(pairs) { pairs.sort_by(&:first).map(&:last) },
+          step: ->(pairs, pair) { pairs << pair }
+        }
+      end
+    end
+
+    define_reducer(:group) do |ctx, value = MISSING, block: nil|
+      resolved_value = value.equal?(MISSING) ? ctx._ : value
+      { value: resolved_value, initial: -> { [] }, step: ->(acc, v) { acc << v } }
+    end
+
+    define_reducer(:percentile) do |ctx, value, percentage, block: nil|
+      percentages = percentage.is_a?(Array) ? percentage : [percentage]
+      percentages.each { |p| ctx.send(:validate_percentile!, p) }
+      scalar = !percentage.is_a?(Array)
+
+      finish =
+        if scalar
+          ->(values) { ctx.send(:percentile_value, values.sort, percentages.first) }
+        else
+          ->(values) {
+            sorted = values.sort
+            percentages.map do |p|
+              { "percentile" => p, "value" => ctx.send(:percentile_value, sorted, p) }
+            end
+          }
+        end
+
+      {
+        value: value,
+        initial: -> { [] },
+        emit_many: !scalar,
+        finish: finish,
+        step: ->(acc, v) { acc << v }
+      }
+    end
+
+    def reduce(initial, &block)
+      raise ArgumentError, "reduce requires a block" unless block
+
+      create_reducer(@obj, initial: initial, &block)
+    end
+
+    def __jrf_begin_stage__(stage, probing: false)
+      @__jrf_stage = stage
+      stage[:reducer_cursor] = 0
+      stage[:reducer_called] = false
+      stage[:reducer_probing] = probing
+    end
+
+    def __jrf_reducer_called?
+      @__jrf_stage && @__jrf_stage[:reducer_called]
+    end
+
+  private
+
+    def create_reducer(value, initial:, emit_many: false, finish: nil, &step_fn)
+      raise "internal error: reducer used outside stage context" unless @__jrf_stage
+
+      reducers = (@__jrf_stage[:reducers] ||= [])
+      idx = @__jrf_stage[:reducer_cursor] || 0
+      reducers[idx] ||= Reducers.reduce(initial, finish: finish, &step_fn)
+      reducers[idx].step(value) unless @__jrf_stage[:reducer_probing]
+      @__jrf_stage[:reducer_cursor] = idx + 1
+      @__jrf_stage[:reducer_called] = true
+      @__jrf_stage[:reducer_emit_many] = emit_many if @__jrf_stage[:reducer_emit_many].nil?
+      ReducerToken.new(idx)
+    end
+
+    def reducer_initial_value(initial)
+      return initial.call if initial.respond_to?(:call)
+      return initial.dup if initial.is_a?(Array) || initial.is_a?(Hash)
+
+      initial
+    end
+
+    def validate_percentile!(value)
+      unless value.is_a?(Numeric) && value >= 0 && value <= 1
+        raise ArgumentError, "percentile must be numeric in [0, 1]"
+      end
+    end
+
+    def percentile_value(sorted_values, percentile)
+      return nil if sorted_values.empty?
+
+      idx = (percentile.to_f * sorted_values.length).ceil - 1
+      idx = 0 if idx.negative?
+      idx = sorted_values.length - 1 if idx >= sorted_values.length
+      sorted_values[idx]
+    end
+  end
+end

data/lib/jrf/runner.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "control"
+require_relative "pipeline_parser"
+require_relative "reducers"
+require_relative "row_context"
+
+module Jrf
+  class Runner
+    class ProbeValue
+      def [](key)
+        self
+      end
+
+      def method_missing(name, *args, &block)
+        self
+      end
+
+      def respond_to_missing?(name, include_private = false)
+        true
+      end
+    end
+
+    PROBE_VALUE = ProbeValue.new
+
+    def initialize(input: ARGF, out: $stdout, err: $stderr)
+      @input = input
+      @out = out
+      @err = err
+    end
+
+    def run(expression, verbose: false)
+      parsed = PipelineParser.new(expression).parse
+      stages = parsed[:stages]
+      dump_stages(stages) if verbose
+
+      ctx = RowContext.new
+      compiled = compile_stages(stages, ctx)
+      initialize_reducers(compiled, ctx)
+      error = nil
+
+      begin
+        @input.each_line do |line|
+          line = line.strip
+          next if line.empty?
+
+          process_value(JSON.parse(line), compiled, ctx)
+        end
+      rescue StandardError => e
+        error = e
+      ensure
+        flush_reducers(compiled, ctx)
+      end
+
+      raise error if error
+    end
+
+    private
+
+    def process_value(input, stages, ctx)
+      current_values = [input]
+
+      stages.each do |stage|
+        next_values = []
+
+        current_values.each do |value|
+          out = apply_stage(stage, value, ctx)
+          if out.equal?(Control::DROPPED)
+            next
+          elsif flat_event?(out)
+            unless out.value.is_a?(Array)
+              raise TypeError, "flat expects Array, got #{out.value.class}"
+            end
+            next_values.concat(out.value)
+          else
+            next_values << out
+          end
+        end
+
+        return if next_values.empty?
+        current_values = next_values
+      end
+
+      current_values.each { |value| @out.puts JSON.generate(value) }
+    end
+
+    def apply_stage(stage, input, ctx)
+      value = eval_stage(stage, input, ctx)
+      if value.equal?(Control::DROPPED)
+        Control::DROPPED
+      elsif ctx.__jrf_reducer_called?
+        stage[:reducer_template] ||= value
+        Control::DROPPED
+      else
+        value
+      end
+    end
+
+    def eval_stage(stage, input, ctx)
+      ctx.reset(input)
+      ctx.__jrf_begin_stage__(stage, probing: input.equal?(PROBE_VALUE))
+      ctx.public_send(stage[:method_name])
+    end
+
+    def flat_event?(value)
+      value.is_a?(Control::Flat)
+    end
+
+    def flush_reducers(stages, ctx)
+      tail = stages
+      loop do
+        tail = tail.drop_while { |stage| !reducer_stage?(stage) }
+        break if tail.empty?
+
+        stage = tail.first
+        reducers = stage[:reducers]
+        break unless reducers&.any?
+
+        out = finish_reducer_template(stage[:reducer_template], reducers)
+        if stage[:reducer_emit_many]
+          out.each { |value| process_value(value, tail.drop(1), ctx) }
+        else
+          process_value(out, tail.drop(1), ctx)
+        end
+        tail = tail.drop(1)
+      end
+    end
+
+    def compile_stages(stages, ctx)
+      mod = Module.new
+      compiled = []
+
+      stages.each_with_index do |stage, i|
+        method_name = :"__jrf_stage_#{i}"
+        mod.module_eval("def #{method_name}; #{stage[:src]}; end", "(jrf stage #{i})", 1)
+        compiled << stage.merge(method_name: method_name)
+      end
+
+      ctx.extend(mod)
+      compiled
+    end
+
+    def dump_stages(stages)
+      stages.each_with_index do |stage, i|
+        @err.puts "stage[#{i}] kind=#{stage[:kind]}"
+        @err.puts "  original: #{stage[:original]}"
+        @err.puts "  ruby: #{stage[:src]}"
+      end
+    end
+
+    def initialize_reducers(stages, ctx)
+      stages.each do |stage|
+        begin
+          value = eval_stage(stage, PROBE_VALUE, ctx)
+          stage[:reducer_template] ||= value if ctx.__jrf_reducer_called?
+        rescue StandardError
+          # Ignore probe-time errors; reducer will be created on first runtime event.
+        end
+      end
+    end
+
+    def reducer_stage?(stage)
+      stage[:reducers]&.any?
+    end
+
+    def finish_reducer_template(template, reducers)
+      if template.is_a?(RowContext::ReducerToken)
+        reducers.fetch(template.index).finish
+      elsif template.is_a?(Array)
+        template.map { |v| finish_reducer_template(v, reducers) }
+      elsif template.is_a?(Hash)
+        template.transform_values { |v| finish_reducer_template(v, reducers) }
+      else
+        template
+      end
+    end
+  end
+end

data/lib/jrf/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Jrf
+  VERSION = "0.1.0"
+end

data/lib/jrf.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require_relative "jrf/version"
+require_relative "jrf/cli"