jrf 0.1.4 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9ce648c2afbfe10dc161b08badb05acdb411baf839dde77433927380b6bb7439
-  data.tar.gz: 6be5a0851eecd3cfcbe93aff1cb8fdd163a84dd96a7b12e440fc514db03f67a0
+  metadata.gz: e9bb2a3a16d2bbe8cfb463267ff74d7d582511d4b4891e56ad3dfa6eee75fceb
+  data.tar.gz: a13b2e9c8517c3da997452166556505b24fc4d5f898765ad33495eafd57c3081
 SHA512:
-  metadata.gz: aa4dfead95dbe09453ec720cdbcf77ba4c7e3f1047c60f51d4ff54724dfa540bb1dbd5630ecb07d09d745e1e61e4c236f50f4407ff6d4c17dd5431b385679f57
-  data.tar.gz: 03c3f5dd3f36675a2bc31981effc506bb1822bb170e754785ccffe077becdd5af13421b4cbfd18fea1c1262f06feef61561be3e3243ca0379e1e6af21ad003c5
+  metadata.gz: 54b400cdaba584896f2511acfe9a41ef10af25033bf88cfc6e0386eaa840df9395fb0d008c320b3193d55a9c3fad444a7f54bd29f52c34f69bc9a9cf392a7809
+  data.tar.gz: 80c72675e179da483316bfeaee7114da6edb49dc66ae179aa072d48907c4c9caf74113c6681b2f4a83f4b97da6faac436f5d6af5bd31e82605b122d85892cede

data/lib/jrf/cli/runner.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "../pipeline"
+require_relative "../pipeline_parser"
+
+module Jrf
+  class CLI
+    class Runner
+      RS_CHAR = "\x1e"
+      DEFAULT_OUTPUT_BUFFER_LIMIT = 4096
+
+      class RsNormalizer
+        def initialize(input)
+          @input = input
+        end
+
+        def read(length = nil, outbuf = nil)
+          chunk = @input.read(length)
+          return nil if chunk.nil?
+
+          chunk = chunk.tr(RS_CHAR, "\n")
+          if outbuf
+            outbuf.replace(chunk)
+          else
+            chunk
+          end
+        end
+      end
+
+      def initialize(inputs:, out: $stdout, err: $stderr, lax: false, pretty: false, atomic_write_bytes: DEFAULT_OUTPUT_BUFFER_LIMIT)
+        @inputs = inputs
+        @out = out
+        @err = err
+        @lax = lax
+        @pretty = pretty
+        @atomic_write_bytes = atomic_write_bytes
+        @output_buffer = +""
+      end
+
+      def run(expression, verbose: false)
+        parsed = PipelineParser.new(expression).parse
+        stages = parsed[:stages]
+        dump_stages(stages) if verbose
+
+        blocks = stages.map { |stage|
+          eval("proc { #{stage[:src]} }", nil, "(jrf stage)", 1) # rubocop:disable Security/Eval
+        }
+        pipeline = Pipeline.new(*blocks)
+
+        input_enum = Enumerator.new { |y| each_input_value { |v| y << v } }
+        pipeline.call(input_enum) do |value|
+          emit_output(value)
+        end
+      ensure
+        write_output(@output_buffer)
+      end
+
+      private
+
+      def each_input_value
+        return each_input_value_lax { |value| yield value } if @lax
+
+        each_input_value_ndjson { |value| yield value }
+      end
+
+      def each_input_value_ndjson
+        each_input do |source|
+          source.each_line do |raw_line|
+            line = raw_line.strip
+            next if line.empty?
+
+            yield JSON.parse(line)
+          end
+        end
+      end
+
+      def each_input_value_lax
+        require "oj"
+        handler = Class.new(Oj::ScHandler) do
+          def initialize(&emit)
+            @emit = emit
+          end
+
+          def hash_start = {}
+          def hash_key(key) = key
+          def hash_set(hash, key, value) = hash[key] = value
+          def array_start = []
+          def array_append(array, value) = array << value
+          def add_value(value) = @emit.call(value)
+        end
+        each_input do |source|
+          Oj.sc_parse(handler.new { |value| yield value }, RsNormalizer.new(source))
+        end
+      rescue LoadError
+        raise "oj is required for --lax mode (gem install oj)"
+      rescue Oj::ParseError => e
+        raise JSON::ParserError, e.message
+      end
+
+      def dump_stages(stages)
+        stages.each_with_index do |stage, i|
+          @err.puts "stage[#{i}]: #{stage[:src]}"
+        end
+      end
+
+      def each_input
+        @inputs.each { |source| yield source }
+      end
+
+      def emit_output(value)
+        record = (@pretty ? JSON.pretty_generate(value) : JSON.generate(value)) << "\n"
+        if @output_buffer.bytesize + record.bytesize <= @atomic_write_bytes
+          @output_buffer << record
+        else
+          write_output(@output_buffer)
+          @output_buffer = record
+        end
+      end
+
+      def write_output(str)
+        @out.syswrite(str)
+      end
+    end
+  end
+end

data/lib/jrf/cli.rb

@@ -1,13 +1,13 @@
 # frozen_string_literal: true
 
-require_relative "runner"
+require_relative "cli/runner"
 
 module Jrf
   class CLI
-    USAGE = "usage: jrf [-v] [--lax] [--pretty] [--help] 'STAGE >> STAGE >> ...'"
+    USAGE = "usage: jrf [options] 'STAGE >> STAGE >> ...'"
 
     HELP_TEXT = <<~'TEXT'
-      usage: jrf [-v] [--lax] [--pretty] [--help] 'STAGE >> STAGE >> ...'
+      usage: jrf [options] 'STAGE >> STAGE >> ...'
 
       JSON filter with the power and speed of Ruby.
 
@@ -15,6 +15,8 @@ module Jrf
         -v, --verbose  print parsed stage expressions
         --lax          allow multiline JSON texts; split inputs by whitespace (also detects JSON-SEQ RS 0x1e)
         -p, --pretty   pretty-print JSON output instead of compact NDJSON
+        --atomic-write-bytes N
+                       group short outputs into atomic writes of up to N bytes
         -h, --help     show this help and exit
 
       Pipeline:
@@ -36,6 +38,7 @@ module Jrf
       verbose = false
       lax = false
       pretty = false
+      atomic_write_bytes = Runner::DEFAULT_OUTPUT_BUFFER_LIMIT
 
       while argv.first&.start_with?("-")
         case argv.first
@@ -48,6 +51,14 @@ module Jrf
         when "-p", "--pretty"
           pretty = true
           argv.shift
+        when /\A--atomic-write-bytes=(.+)\z/
+          atomic_write_bytes = parse_atomic_write_bytes(Regexp.last_match(1), err)
+          return 1 unless atomic_write_bytes
+          argv.shift
+        when "--atomic-write-bytes"
+          argv.shift
+          atomic_write_bytes = parse_atomic_write_bytes(argv.shift, err)
+          return 1 unless atomic_write_bytes
         when "-h", "--help"
           out.puts HELP_TEXT
           return 0
@@ -64,8 +75,43 @@ module Jrf
       end
 
       expression = argv.shift
-      Runner.new(input: input, out: out, err: err, lax: lax, pretty: pretty).run(expression, verbose: verbose)
+      inputs = Enumerator.new do |y|
+        if argv.empty?
+          y << input
+        else
+          argv.each do |path|
+            if path == "-"
+              y << input
+            elsif path.end_with?(".gz")
+              require "zlib"
+              Zlib::GzipReader.open(path) do |source|
+                y << source
+              end
+            else
+              File.open(path, "rb") do |source|
+                y << source
+              end
+            end
+          end
+        end
+      end
+      Runner.new(
+        inputs: inputs,
+        out: out,
+        err: err,
+        lax: lax,
+        pretty: pretty,
+        atomic_write_bytes: atomic_write_bytes
+      ).run(expression, verbose: verbose)
       0
     end
+
+    def self.parse_atomic_write_bytes(value, err)
+      bytes = Integer(value, exception: false)
+      return bytes if bytes && bytes.positive?
+
+      err.puts "--atomic-write-bytes requires a positive integer"
+      nil
+    end
   end
 end

data/lib/jrf/row_context.rb

@@ -26,10 +26,12 @@ module Jrf
     def initialize(obj = nil)
       @obj = obj
       @__jrf_current_stage = nil
+      @__jrf_current_input = obj
     end
 
     def reset(obj)
       @obj = obj
+      @__jrf_current_input = obj
       self
     end
 
@@ -38,11 +40,11 @@ module Jrf
     end
 
     def flat
-      Control::Flat.new(@obj)
+      Control::Flat.new(current_input)
     end
 
     def select(predicate)
-      predicate ? @obj : Control::DROPPED
+      predicate ? current_input : Control::DROPPED
     end
 
     define_reducer(:sum) do |_ctx, value, initial: 0, block: nil|
@@ -111,15 +113,16 @@ module Jrf
     define_reducer(:sort) do |ctx, key = MISSING, block: nil|
       if block
         {
-          value: ctx._,
+          value: ctx.send(:current_input),
           initial: -> { [] },
           finish: ->(rows) { rows.sort(&block) },
           step: ->(rows, row) { rows << row }
         }
       else
-        resolved_key = key.equal?(MISSING) ? ctx._ : key
+        current = ctx.send(:current_input)
+        resolved_key = key.equal?(MISSING) ? current : key
         {
-          value: [resolved_key, ctx._],
+          value: [resolved_key, current],
           initial: -> { [] },
           finish: ->(pairs) { pairs.sort_by(&:first).map(&:last) },
           step: ->(pairs, pair) { pairs << pair }
@@ -128,7 +131,7 @@ module Jrf
     end
 
     define_reducer(:group) do |ctx, value = MISSING, block: nil|
-      resolved_value = value.equal?(MISSING) ? ctx._ : value
+      resolved_value = value.equal?(MISSING) ? ctx.send(:current_input) : value
       { value: resolved_value, initial: -> { [] }, step: ->(acc, v) { acc << v } }
     end
 
@@ -158,7 +161,7 @@ module Jrf
     def reduce(initial, &block)
       raise ArgumentError, "reduce requires a block" unless block
 
-      @__jrf_current_stage.allocate_reducer(@obj, initial: initial, &block)
+      @__jrf_current_stage.allocate_reducer(current_input, initial: initial, &block)
     end
 
     def map(&block)
@@ -180,6 +183,18 @@ module Jrf
 
     private
 
+    def current_input
+      @__jrf_current_input
+    end
+
+    def __jrf_with_current_input(value)
+      saved_input = current_input
+      @__jrf_current_input = value
+      yield
+    ensure
+      @__jrf_current_input = saved_input
+    end
+
     def reducer_initial_value(initial)
       return initial.call if initial.respond_to?(:call)
       return initial.dup if initial.is_a?(Array) || initial.is_a?(Hash)

data/lib/jrf/stage.rb

@@ -64,10 +64,7 @@ module Jrf
 
       # Transformation mode (detected on first call)
       if @map_transforms[idx]
-        case type
-        when :array then return collection.map(&block)
-        when :hash then return collection.transform_values(&block)
-        end
+        return transform_collection(type, collection, &block)
       end
 
       map_reducer = (@reducers[idx] ||= MapReducer.new(type))
@@ -78,7 +75,7 @@ module Jrf
         collection.each_with_index do |v, i|
           slot = map_reducer.slot(i)
           with_scoped_reducers(slot.reducers) do
-            result = block.call(v)
+            result = @ctx.send(:__jrf_with_current_input, v) { block.call(v) }
             slot.template ||= result
           end
         end
@@ -87,7 +84,7 @@ module Jrf
         collection.each do |k, v|
           slot = map_reducer.slot(k)
           with_scoped_reducers(slot.reducers) do
-            result = block.call(v)
+            result = @ctx.send(:__jrf_with_current_input, v) { block.call(v) }
             slot.template ||= result
           end
         end
@@ -97,12 +94,7 @@ module Jrf
       if @mode.nil? && map_reducer.slots.values.all? { |s| s.reducers.empty? }
         @map_transforms[idx] = true
         @reducers[idx] = nil
-        case type
-        when :array
-          return map_reducer.slots.sort_by { |k, _| k }.map { |_, s| s.template }
-        when :hash
-          return map_reducer.slots.transform_values(&:template)
-        end
+        return transformed_slots(type, map_reducer)
       end
 
       ReducerToken.new(idx)
@@ -115,7 +107,7 @@ module Jrf
       row = @ctx._
       slot = map_reducer.slot(key)
       with_scoped_reducers(slot.reducers) do
-        result = block.call(row)
+        result = @ctx.send(:__jrf_with_current_input, row) { block.call(row) }
         slot.template ||= result
       end
 
@@ -146,6 +138,60 @@ module Jrf
       @cursor = saved_cursor
     end
 
+    def transform_collection(type, collection, &block)
+      case type
+      when :array
+        raise TypeError, "map expects Array, got #{collection.class}" unless collection.is_a?(Array)
+
+        collection.each_with_object([]) do |value, result|
+          mapped = @ctx.send(:__jrf_with_current_input, value) { block.call(value) }
+          append_map_result(result, mapped)
+        end
+      when :hash
+        raise TypeError, "map_values expects Hash, got #{collection.class}" unless collection.is_a?(Hash)
+
+        collection.each_with_object({}) do |(key, value), result|
+          mapped = @ctx.send(:__jrf_with_current_input, value) { block.call(value) }
+          next if mapped.equal?(Control::DROPPED)
+          raise TypeError, "flat is not supported inside map_values" if mapped.is_a?(Control::Flat)
+
+          result[key] = mapped
+        end
+      end
+    end
+
+    def transformed_slots(type, map_reducer)
+      case type
+      when :array
+        map_reducer.slots
+          .sort_by { |k, _| k }
+          .each_with_object([]) do |(_, slot), result|
+            append_map_result(result, slot.template)
+          end
+      when :hash
+        map_reducer.slots.each_with_object({}) do |(key, slot), result|
+          next if slot.template.equal?(Control::DROPPED)
+          raise TypeError, "flat is not supported inside map_values" if slot.template.is_a?(Control::Flat)
+
+          result[key] = slot.template
+        end
+      end
+    end
+
+    def append_map_result(result, mapped)
+      return if mapped.equal?(Control::DROPPED)
+
+      if mapped.is_a?(Control::Flat)
+        unless mapped.value.is_a?(Array)
+          raise TypeError, "flat expects Array, got #{mapped.value.class}"
+        end
+
+        result.concat(mapped.value)
+      else
+        result << mapped
+      end
+    end
+
     class MapReducer
       attr_reader :slots
 

data/lib/jrf/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Jrf
-  VERSION = "0.1.4"
+  VERSION = "0.1.6"
 end

data/test/jrf_test.rb

@@ -1,7 +1,17 @@
 # frozen_string_literal: true
 
+begin
+  require "bundler/setup"
+rescue LoadError
+  # Allow running tests in plain Ruby environments with globally installed gems.
+end
+
 require "json"
 require "open3"
+require "stringio"
+require "tmpdir"
+require "zlib"
+require_relative "../lib/jrf/cli/runner"
 
 def run_jrf(expr, input, *opts)
   Open3.capture3("./exe/jrf", *opts, expr, stdin_data: input)
@@ -41,6 +51,45 @@ def lines(str)
   str.lines.map(&:strip).reject(&:empty?)
 end
 
+class RecordingRunner < Jrf::CLI::Runner
+  attr_reader :writes
+
+  def initialize(**kwargs)
+    super
+    @writes = []
+  end
+
+  private
+
+  def write_output(str)
+    return if str.empty?
+
+    @writes << str
+  end
+end
+
+class ChunkedSource
+  def initialize(str, chunk_size: 5)
+    @str = str
+    @chunk_size = chunk_size
+    @offset = 0
+  end
+
+  def read(length = nil, outbuf = nil)
+    raise "expected chunked reads" if length.nil?
+
+    chunk = @str.byteslice(@offset, [length, @chunk_size].min)
+    return nil unless chunk
+
+    @offset += chunk.bytesize
+    if outbuf
+      outbuf.replace(chunk)
+    else
+      chunk
+    end
+  end
+end
+
 File.chmod(0o755, "./exe/jrf")
 
 input = <<~NDJSON
@@ -92,10 +141,11 @@ assert_includes(stderr, 'stage[1]: _["hello"]')
 
 stdout, stderr, status = Open3.capture3("./exe/jrf", "--help")
 assert_success(status, stderr, "help option")
-assert_includes(stdout, "usage: jrf [-v] [--lax] [--pretty] [--help] 'STAGE >> STAGE >> ...'")
+assert_includes(stdout, "usage: jrf [options] 'STAGE >> STAGE >> ...'")
 assert_includes(stdout, "JSON filter with the power and speed of Ruby.")
 assert_includes(stdout, "--lax")
 assert_includes(stdout, "--pretty")
+assert_includes(stdout, "--atomic-write-bytes N")
 assert_includes(stdout, "Pipeline:")
 assert_includes(stdout, "Connect stages with top-level >>.")
 assert_includes(stdout, "The current value in each stage is available as _.")
@@ -103,11 +153,72 @@ assert_includes(stdout, "See Also:")
 assert_includes(stdout, "https://github.com/kazuho/jrf#readme")
 assert_equal([], lines(stderr), "help stderr output")
 
+threshold_input = StringIO.new((1..4).map { |i| "{\"foo\":\"#{'x' * 1020}\",\"i\":#{i}}\n" }.join)
+buffered_runner = RecordingRunner.new(inputs: [threshold_input], out: StringIO.new, err: StringIO.new)
+buffered_runner.run('_')
+expected_line = JSON.generate({"foo" => "x" * 1020, "i" => 1}) + "\n"
+assert_equal(2, buffered_runner.writes.length, "default atomic write limit buffers records until the configured threshold")
+assert_equal(expected_line.bytesize * 3, buffered_runner.writes.first.bytesize, "default atomic write limit flushes before the next record would exceed the threshold")
+assert_equal(expected_line.bytesize, buffered_runner.writes.last.bytesize, "final buffer flush emits the remaining record")
+
+small_limit_runner = RecordingRunner.new(inputs: [StringIO.new("{\"foo\":1}\n{\"foo\":2}\n")], out: StringIO.new, err: StringIO.new, atomic_write_bytes: 1)
+small_limit_runner.run('_["foo"]')
+assert_equal(["1\n", "2\n"], small_limit_runner.writes, "small atomic write limit emits oversized records directly")
+
+error_runner = RecordingRunner.new(inputs: [StringIO.new("{\"foo\":1}\n{\"foo\":")], out: StringIO.new, err: StringIO.new)
+begin
+  error_runner.run('_["foo"]')
+  raise "expected parse error for buffered flush test"
+rescue JSON::ParserError
+  assert_equal(["1\n"], error_runner.writes, "buffer flushes pending output before parse errors escape")
+end
+
 stdout, stderr, status = run_jrf('select(_["hello"] == 123) >> _["hello"]', input_hello, "--verbose")
 assert_success(status, stderr, "dump stages verbose alias")
 assert_equal(%w[123], lines(stdout), "dump stages verbose alias output")
 assert_includes(stderr, 'stage[0]: select(_["hello"] == 123)')
 
+stdout, stderr, status = run_jrf('_["hello"]', input_hello, "--atomic-write-bytes", "512")
+assert_success(status, stderr, "atomic write bytes option")
+assert_equal(%w[123 456], lines(stdout), "atomic write bytes option output")
+
+stdout, stderr, status = run_jrf('_["hello"]', input_hello, "--atomic-write-bytes=512")
+assert_success(status, stderr, "atomic write bytes equals form")
+assert_equal(%w[123 456], lines(stdout), "atomic write bytes equals form output")
+
+stdout, stderr, status = Open3.capture3("./exe/jrf", "--atomic-write-bytes", "0", '_["hello"]', stdin_data: input_hello)
+assert_failure(status, "atomic write bytes rejects zero")
+assert_includes(stderr, "--atomic-write-bytes requires a positive integer")
+
+Dir.mktmpdir do |dir|
+  gz_path = File.join(dir, "input.ndjson.gz")
+  Zlib::GzipWriter.open(gz_path) do |io|
+    io.write("{\"foo\":10}\n{\"foo\":20}\n")
+  end
+
+  stdout, stderr, status = Open3.capture3("./exe/jrf", '_["foo"]', gz_path)
+  assert_success(status, stderr, "compressed input by suffix")
+  assert_equal(%w[10 20], lines(stdout), "compressed input output")
+
+  lax_gz_path = File.join(dir, "input-lax.json.gz")
+  Zlib::GzipWriter.open(lax_gz_path) do |io|
+    io.write("{\"foo\":30}\n\x1e{\"foo\":40}\n")
+  end
+
+  stdout, stderr, status = Open3.capture3("./exe/jrf", "--lax", '_["foo"]', lax_gz_path)
+  assert_success(status, stderr, "compressed lax input by suffix")
+  assert_equal(%w[30 40], lines(stdout), "compressed lax input output")
+
+  second_gz_path = File.join(dir, "input2.ndjson.gz")
+  Zlib::GzipWriter.open(second_gz_path) do |io|
+    io.write("{\"foo\":50}\n")
+  end
+
+  stdout, stderr, status = Open3.capture3("./exe/jrf", '_["foo"]', gz_path, second_gz_path)
+  assert_success(status, stderr, "multiple compressed inputs by suffix")
+  assert_equal(%w[10 20 50], lines(stdout), "multiple compressed input output")
+end
+
 stdout, stderr, status = run_jrf('_', input_hello, "--pretty")
 assert_success(status, stderr, "pretty output")
 assert_equal(
@@ -174,6 +285,14 @@ stdout, stderr, status = run_jrf('_["items"] >> flat >> group', input_flat)
 assert_success(status, stderr, "flat then group")
 assert_equal(['[1,2,3]'], lines(stdout), "flat then group output")
 
+stdout, stderr, status = run_jrf('map { |x| flat }', "[[1,2],[3],[4,5,6]]\n")
+assert_success(status, stderr, "flat inside map")
+assert_equal(['[1,2,3,4,5,6]'], lines(stdout), "flat inside map output")
+
+stdout, stderr, status = run_jrf('map_values { |v| flat }', "{\"a\":[1,2],\"b\":[3]}\n")
+assert_failure(status, "flat inside map_values")
+assert_includes(stderr, "flat is not supported inside map_values")
+
 stdout, stderr, status = run_jrf('_["foo"] >> flat', input)
 assert_failure(status, "flat requires array")
 assert_includes(stderr, "flat expects Array")
@@ -485,6 +604,26 @@ stdout, stderr, status = run_jrf('_["foo"]', input_lax_trailing_rs, "--lax")
 assert_success(status, stderr, "lax ignores trailing separator")
 assert_equal(%w[9], lines(stdout), "lax trailing separator output")
 
+chunked_lax_out = RecordingRunner.new(
+  inputs: [ChunkedSource.new("{\"foo\":1}\n\x1e{\"foo\":2}\n\t{\"foo\":3}\n")],
+  out: StringIO.new,
+  err: StringIO.new,
+  lax: true
+)
+chunked_lax_out.run('_["foo"]')
+assert_equal(%w[1 2 3], lines(chunked_lax_out.writes.join), "lax mode streams chunked input without whole-input reads")
+
+Dir.mktmpdir do |dir|
+  one = File.join(dir, "one.json")
+  two = File.join(dir, "two.json")
+  File.write(one, "1")
+  File.write(two, "2")
+
+  stdout, stderr, status = Open3.capture3("./exe/jrf", "--lax", "_", one, two)
+  assert_success(status, stderr, "lax keeps file boundaries")
+  assert_equal(%w[1 2], lines(stdout), "lax does not merge JSON across file boundaries")
+end
+
 stdout, stderr, status = run_jrf('select(_["x"] > ) >> _["foo"]', "")
 assert_failure(status, "syntax error should fail before row loop")
 assert_includes(stderr, "syntax error")
@@ -540,6 +679,10 @@ stdout, stderr, status = run_jrf('_["values"] >> map { |x| sum(_[0] + x) }', inp
 assert_success(status, stderr, "map keeps ambient _")
 assert_equal(['[12,66,606]'], lines(stdout), "map ambient _ output")
 
+stdout, stderr, status = run_jrf('_["values"] >> map { |x| reduce(0) { |acc, v| acc + v } }', input_map)
+assert_success(status, stderr, "map with reduce")
+assert_equal(['[6,60,600]'], lines(stdout), "map with reduce output")
+
 input_map_varying = <<~NDJSON
   [1,10]
   [2,20,200]
@@ -550,6 +693,20 @@ stdout, stderr, status = run_jrf('map { |x| sum(x) }', input_map_varying)
 assert_success(status, stderr, "map varying lengths")
 assert_equal(['[6,30,200]'], lines(stdout), "map varying lengths output")
 
+input_map_unsorted = <<~NDJSON
+  {"values":[3,30]}
+  {"values":[1,10]}
+  {"values":[2,20]}
+NDJSON
+
+stdout, stderr, status = run_jrf('_["values"] >> map { |x| group }', input_map)
+assert_success(status, stderr, "map with group")
+assert_equal(['[[1,2,3],[10,20,30],[100,200,300]]'], lines(stdout), "map with group output")
+
+stdout, stderr, status = run_jrf('_["values"] >> map { |x| sort }', input_map_unsorted)
+assert_success(status, stderr, "map with sort default key")
+assert_equal(['[[1,2,3],[10,20,30]]'], lines(stdout), "map with sort default key output")
+
 input_map_values = <<~NDJSON
   {"a":1,"b":10}
   {"a":2,"b":20}
@@ -578,10 +735,18 @@ stdout, stderr, status = run_jrf('map_values { |v| count(v) }', input_map_values
 assert_success(status, stderr, "map_values with count")
 assert_equal(['{"a":3,"b":3}'], lines(stdout), "map_values with count output")
 
+stdout, stderr, status = run_jrf('map_values { |v| group }', input_map_values)
+assert_success(status, stderr, "map_values with group")
+assert_equal(['{"a":[1,2,3],"b":[10,20,30]}'], lines(stdout), "map_values with group output")
+
 stdout, stderr, status = run_jrf('map_values { |v| sum(_["a"] + v) }', input_map_values)
 assert_success(status, stderr, "map_values keeps ambient _")
 assert_equal(['{"a":12,"b":66}'], lines(stdout), "map_values ambient _ output")
 
+stdout, stderr, status = run_jrf('map_values { |v| reduce(0) { |acc, x| acc + x } }', input_map_values)
+assert_success(status, stderr, "map_values with reduce")
+assert_equal(['{"a":6,"b":60}'], lines(stdout), "map_values with reduce output")
+
 stdout, stderr, status = run_jrf('select(false) >> map { |x| sum(x) }', input_map)
 assert_success(status, stderr, "map no matches")
 assert_equal([], lines(stdout), "map no matches output")
@@ -599,10 +764,18 @@ stdout, stderr, status = run_jrf('_["values"] >> map { |x| x + 1 }', input_map)
 assert_success(status, stderr, "map transform")
 assert_equal(['[2,11,101]', '[3,21,201]', '[4,31,301]'], lines(stdout), "map transform output")
 
+stdout, stderr, status = run_jrf('_["values"] >> map { |x| select(x >= 20) }', input_map)
+assert_success(status, stderr, "map transform with select")
+assert_equal(['[100]', '[20,200]', '[30,300]'], lines(stdout), "map transform with select output")
+
 stdout, stderr, status = run_jrf('map_values { |v| v * 2 }', input_map_values)
 assert_success(status, stderr, "map_values transform")
 assert_equal(['{"a":2,"b":20}', '{"a":4,"b":40}', '{"a":6,"b":60}'], lines(stdout), "map_values transform output")
 
+stdout, stderr, status = run_jrf('map_values { |v| select(v >= 10) }', input_map_values)
+assert_success(status, stderr, "map_values transform with select")
+assert_equal(['{"b":10}', '{"b":20}', '{"b":30}'], lines(stdout), "map_values transform with select output")
+
 stdout, stderr, status = run_jrf('_["values"] >> map { |x| x + 1 } >> map { |x| x * 10 }', input_map)
 assert_success(status, stderr, "chained map transforms")
 assert_equal(['[20,110,1010]', '[30,210,2010]', '[40,310,3010]'], lines(stdout), "chained map transforms output")
@@ -639,6 +812,12 @@ stdout, stderr, status = run_jrf('group_by(_["status"]) { |row| group(row["path"
 assert_success(status, stderr, "group_by with group(expr)")
 assert_equal(['{"200":["/a","/c","/d"],"404":["/b"]}'], lines(stdout), "group_by with group(expr) output")
 
+stdout, stderr, status = run_jrf('group_by(_["status"]) { group }', input_gb)
+assert_success(status, stderr, "group_by with implicit group")
+result = JSON.parse(lines(stdout).first)
+assert_equal(3, result["200"].length, "group_by implicit group 200 count")
+assert_equal("/a", result["200"][0]["path"], "group_by implicit group first row")
+
 stdout, stderr, status = run_jrf('group_by(_["status"]) { |row| min(row["latency"]) }', input_gb)
 assert_success(status, stderr, "group_by with min")
 assert_equal(['{"200":10,"404":50}'], lines(stdout), "group_by with min output")
@@ -647,6 +826,10 @@ stdout, stderr, status = run_jrf('group_by(_["status"]) { |row| {total: sum(row[
 assert_success(status, stderr, "group_by with multi-reducer")
 assert_equal(['{"200":{"total":60,"n":3},"404":{"total":50,"n":1}}'], lines(stdout), "group_by multi-reducer output")
 
+stdout, stderr, status = run_jrf('group_by(_["status"]) { reduce(0) { |acc, row| acc + row["latency"] } }', input_gb)
+assert_success(status, stderr, "group_by with reduce")
+assert_equal(['{"200":60,"404":50}'], lines(stdout), "group_by with reduce output")
+
 stdout, stderr, status = run_jrf('select(false) >> group_by(_["status"]) { count() }', input_gb)
 assert_success(status, stderr, "group_by no matches")
 assert_equal([], lines(stdout), "group_by no matches output")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: jrf
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.6
 platform: ruby
 authors:
 - kazuho
@@ -40,12 +40,12 @@ files:
 - jrf.gemspec
 - lib/jrf.rb
 - lib/jrf/cli.rb
+- lib/jrf/cli/runner.rb
 - lib/jrf/control.rb
 - lib/jrf/pipeline.rb
 - lib/jrf/pipeline_parser.rb
 - lib/jrf/reducers.rb
 - lib/jrf/row_context.rb
-- lib/jrf/runner.rb
 - lib/jrf/stage.rb
 - lib/jrf/version.rb
 - test/jrf_test.rb

data/lib/jrf/runner.rb

@@ -1,81 +0,0 @@
-# frozen_string_literal: true
-
-require "json"
-require_relative "pipeline"
-require_relative "pipeline_parser"
-
-module Jrf
-  class Runner
-    RS_CHAR = "\x1e"
-
-    def initialize(input: ARGF, out: $stdout, err: $stderr, lax: false, pretty: false)
-      @input = input
-      @out = out
-      @err = err
-      @lax = lax
-      @pretty = pretty
-    end
-
-    def run(expression, verbose: false)
-      parsed = PipelineParser.new(expression).parse
-      stages = parsed[:stages]
-      dump_stages(stages) if verbose
-
-      blocks = stages.map { |stage|
-        eval("proc { #{stage[:src]} }", nil, "(jrf stage)", 1) # rubocop:disable Security/Eval
-      }
-      pipeline = Pipeline.new(*blocks)
-
-      input_enum = Enumerator.new { |y| each_input_value { |v| y << v } }
-      pipeline.call(input_enum) do |value|
-        @out.puts(@pretty ? JSON.pretty_generate(value) : JSON.generate(value))
-      end
-    end
-
-    private
-
-    def each_input_value
-      return each_input_value_lax { |value| yield value } if @lax
-
-      each_input_value_ndjson { |value| yield value }
-    end
-
-    def each_input_value_ndjson
-      @input.each_line do |raw_line|
-        line = raw_line.strip
-        next if line.empty?
-
-        yield JSON.parse(line)
-      end
-    end
-
-    def each_input_value_lax
-      require "oj"
-      source = @input.read.to_s
-      source = source.include?(RS_CHAR) ? source.tr(RS_CHAR, "\n") : source
-      handler = Class.new(Oj::ScHandler) do
-        def initialize(&emit)
-          @emit = emit
-        end
-
-        def hash_start = {}
-        def hash_key(key) = key
-        def hash_set(hash, key, value) = hash[key] = value
-        def array_start = []
-        def array_append(array, value) = array << value
-        def add_value(value) = @emit.call(value)
-      end.new { |value| yield value }
-      Oj.sc_parse(handler, source)
-    rescue LoadError
-      raise "oj is required for --lax mode (gem install oj)"
-    rescue Oj::ParseError => e
-      raise JSON::ParserError, e.message
-    end
-
-    def dump_stages(stages)
-      stages.each_with_index do |stage, i|
-        @err.puts "stage[#{i}]: #{stage[:src]}"
-      end
-    end
-  end
-end