jrf 0.1.12 → 0.1.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,951 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "test_helper"
4
+
5
+ class CliRunnerTest < JrfTestCase
6
+ def test_extract_and_select
7
+ input = <<~NDJSON
8
+ {"foo":1,"x":5}
9
+ {"foo":2,"x":11}
10
+ {"foo":{"bar":"ok"},"x":50}
11
+ {"x":70}
12
+ NDJSON
13
+
14
+ stdout, stderr, status = run_jrf('_["foo"]', input)
15
+ assert_success(status, stderr, "simple extract")
16
+ assert_equal(%w[1 2 {"bar":"ok"} null], lines(stdout), "extract output")
17
+
18
+ input_nested = <<~NDJSON
19
+ {"foo":{"bar":"a"}}
20
+ {"foo":{"bar":"b"}}
21
+ NDJSON
22
+
23
+ stdout, stderr, status = run_jrf('_["foo"]["bar"]', input_nested)
24
+ assert_success(status, stderr, "nested extract")
25
+ assert_equal(%w["a" "b"], lines(stdout), "nested output")
26
+
27
+ stdout, stderr, status = run_jrf('select(_["x"] > 10) >> _["foo"]', input)
28
+ assert_success(status, stderr, "select + extract")
29
+ assert_equal(%w[2 {"bar":"ok"} null], lines(stdout), "filtered output")
30
+
31
+ stdout, stderr, status = run_jrf('select(_["x"] > 10)', input)
32
+ assert_success(status, stderr, "select only")
33
+ assert_equal(
34
+ ['{"foo":2,"x":11}', '{"foo":{"bar":"ok"},"x":50}', '{"x":70}'],
35
+ lines(stdout),
36
+ "select-only output"
37
+ )
38
+
39
+ input_hello = <<~NDJSON
40
+ {"hello":123}
41
+ {"hello":456}
42
+ NDJSON
43
+
44
+ stdout, stderr, status = run_jrf('select(_["hello"] == 123)', input_hello)
45
+ assert_success(status, stderr, "select-only hello")
46
+ assert_equal(['{"hello":123}'], lines(stdout), "select-only hello output")
47
+
48
+ stdout, stderr, status = run_jrf('select(_["hello"] == 123) >> _["hello"]', input_hello, "-v")
49
+ assert_success(status, stderr, "dump stages")
50
+ assert_equal(%w[123], lines(stdout), "dump stages output")
51
+ assert_includes(stderr, 'stage[0]: select(_["hello"] == 123)')
52
+ assert_includes(stderr, 'stage[1]: _["hello"]')
53
+ end
54
+
55
+ def test_help_version_and_atomic_write_options
56
+ input_hello = <<~NDJSON
57
+ {"hello":123}
58
+ {"hello":456}
59
+ NDJSON
60
+
61
+ stdout, stderr, status = Open3.capture3("./exe/jrf", "--help")
62
+ assert_success(status, stderr, "help option")
63
+ assert_includes(stdout, "usage: jrf [options] 'STAGE >> STAGE >> ...'")
64
+ assert_includes(stdout, "JSON filter with the power and speed of Ruby.")
65
+ assert_includes(stdout, "--lax")
66
+ assert_includes(stdout, "--output")
67
+ assert_includes(stdout, "--require LIBRARY")
68
+ assert_includes(stdout, "--no-jit")
69
+ assert_includes(stdout, "-V")
70
+ assert_includes(stdout, "--version")
71
+ assert_includes(stdout, "--atomic-write-bytes N")
72
+ assert_includes(stdout, "Pipeline:")
73
+ assert_includes(stdout, "Connect stages with top-level >>.")
74
+ assert_includes(stdout, "The current value in each stage is available as _.")
75
+ assert_includes(stdout, "See Also:")
76
+ assert_includes(stdout, "https://github.com/kazuho/jrf#readme")
77
+ assert_equal([], lines(stderr), "help stderr output")
78
+
79
+ stdout, stderr, status = Open3.capture3("./exe/jrf", "--version")
80
+ assert_success(status, stderr, "version long option")
81
+ assert_equal([Jrf::VERSION], lines(stdout), "version long option output")
82
+ assert_equal([], lines(stderr), "version long option stderr")
83
+
84
+ stdout, stderr, status = Open3.capture3("./exe/jrf", "-V")
85
+ assert_success(status, stderr, "version short option")
86
+ assert_equal([Jrf::VERSION], lines(stdout), "version short option output")
87
+ assert_equal([], lines(stderr), "version short option stderr")
88
+
89
+ stdout, stderr, status = run_jrf('select(_["hello"] == 123) >> _["hello"]', input_hello, "--verbose")
90
+ assert_success(status, stderr, "dump stages verbose alias")
91
+ assert_equal(%w[123], lines(stdout), "dump stages verbose alias output")
92
+ assert_includes(stderr, 'stage[0]: select(_["hello"] == 123)')
93
+
94
+ stdout, stderr, status = run_jrf('_["hello"]', input_hello, "--atomic-write-bytes", "512")
95
+ assert_success(status, stderr, "atomic write bytes option")
96
+ assert_equal(%w[123 456], lines(stdout), "atomic write bytes option output")
97
+
98
+ stdout, stderr, status = run_jrf('_["hello"]', input_hello, "--atomic-write-bytes=512")
99
+ assert_success(status, stderr, "atomic write bytes equals form")
100
+ assert_equal(%w[123 456], lines(stdout), "atomic write bytes equals form output")
101
+
102
+ stdout, stderr, status = Open3.capture3("./exe/jrf", "--atomic-write-bytes", "0", '_["hello"]', stdin_data: input_hello)
103
+ assert_failure(status, "atomic write bytes rejects zero")
104
+ assert_includes(stderr, "--atomic-write-bytes requires a positive integer")
105
+ end
106
+
107
+ def test_runner_buffering_and_require_option
108
+ threshold_input = StringIO.new((1..4).map { |i| "{\"foo\":\"#{'x' * 1020}\",\"i\":#{i}}\n" }.join)
109
+ buffered_runner = RecordingRunner.new(input: threshold_input, out: StringIO.new, err: StringIO.new)
110
+ buffered_runner.run('_')
111
+ expected_line = JSON.generate({"foo" => "x" * 1020, "i" => 1}) + "\n"
112
+ assert_equal(2, buffered_runner.writes.length, "default atomic write limit buffers records until the configured threshold")
113
+ assert_equal(expected_line.bytesize * 3, buffered_runner.writes.first.bytesize, "default atomic write limit flushes before the next record would exceed the threshold")
114
+ assert_equal(expected_line.bytesize, buffered_runner.writes.last.bytesize, "final buffer flush emits the remaining record")
115
+
116
+ small_limit_runner = RecordingRunner.new(input: StringIO.new("{\"foo\":1}\n{\"foo\":2}\n"), out: StringIO.new, err: StringIO.new, atomic_write_bytes: 1)
117
+ small_limit_runner.run('_["foo"]')
118
+ assert_equal(["1\n", "2\n"], small_limit_runner.writes, "small atomic write limit emits oversized records directly")
119
+
120
+ err_io = StringIO.new
121
+ error_runner = RecordingRunner.new(input: StringIO.new("{\"foo\":1}\n{\"foo\":"), out: StringIO.new, err: err_io)
122
+ error_runner.run('_["foo"]')
123
+ assert_equal(["1\n"], error_runner.writes, "buffer flushes pending output before parse errors")
124
+ assert_includes(err_io.string, "JSON::ParserError", "parse error reported to stderr")
125
+ assert(error_runner.input_errors?, "input_errors? is true after parse error")
126
+
127
+ input_hello = <<~NDJSON
128
+ {"hello":123}
129
+ {"hello":456}
130
+ NDJSON
131
+
132
+ Dir.mktmpdir do |dir|
133
+ helper = File.join(dir, "helpers.rb")
134
+ File.write(helper, <<~RUBY)
135
+ def double(value)
136
+ value * 2
137
+ end
138
+ RUBY
139
+
140
+ stdout, stderr, status = Open3.capture3("./exe/jrf", "-r", helper, 'double(_["hello"])', stdin_data: input_hello)
141
+ assert_success(status, stderr, "require helper option")
142
+ assert_equal(%w[246 912], lines(stdout), "require helper option output")
143
+ end
144
+ end
145
+
146
+ def test_yjit_option
147
+ if defined?(RubyVM::YJIT) && RubyVM::YJIT.respond_to?(:enabled?)
148
+ yjit_probe = "{\"probe\":1}\n"
149
+
150
+ stdout, stderr, status = run_jrf('RubyVM::YJIT.enabled?', yjit_probe)
151
+ assert_success(status, stderr, "default jit enablement")
152
+ assert_equal(%w[true], lines(stdout), "default jit enablement output")
153
+
154
+ stdout, stderr, status = run_jrf('RubyVM::YJIT.enabled?', yjit_probe, "--no-jit")
155
+ assert_success(status, stderr, "no-jit option")
156
+ assert_equal(%w[false], lines(stdout), "no-jit option output")
157
+ end
158
+ end
159
+
160
+ def test_compressed_inputs
161
+ Dir.mktmpdir do |dir|
162
+ gz_path = File.join(dir, "input.ndjson.gz")
163
+ Zlib::GzipWriter.open(gz_path) do |io|
164
+ io.write("{\"foo\":10}\n{\"foo\":20}\n")
165
+ end
166
+
167
+ stdout, stderr, status = Open3.capture3("./exe/jrf", '_["foo"]', gz_path)
168
+ assert_success(status, stderr, "compressed input by suffix")
169
+ assert_equal(%w[10 20], lines(stdout), "compressed input output")
170
+
171
+ lax_gz_path = File.join(dir, "input-lax.json.gz")
172
+ Zlib::GzipWriter.open(lax_gz_path) do |io|
173
+ io.write("{\"foo\":30}\n\x1e{\"foo\":40}\n")
174
+ end
175
+
176
+ stdout, stderr, status = Open3.capture3("./exe/jrf", "--lax", '_["foo"]', lax_gz_path)
177
+ assert_success(status, stderr, "compressed lax input by suffix")
178
+ assert_equal(%w[30 40], lines(stdout), "compressed lax input output")
179
+
180
+ second_gz_path = File.join(dir, "input2.ndjson.gz")
181
+ Zlib::GzipWriter.open(second_gz_path) do |io|
182
+ io.write("{\"foo\":50}\n")
183
+ end
184
+
185
+ stdout, stderr, status = Open3.capture3("./exe/jrf", '_["foo"]', gz_path, second_gz_path)
186
+ assert_success(status, stderr, "multiple compressed inputs by suffix")
187
+ assert_equal(%w[10 20 50], lines(stdout), "multiple compressed input output")
188
+ end
189
+ end
190
+
191
+ def test_output_formats
192
+ input_hello = <<~NDJSON
193
+ {"hello":123}
194
+ {"hello":456}
195
+ NDJSON
196
+
197
+ stdout, stderr, status = run_jrf('_', input_hello, "-o", "pretty")
198
+ assert_success(status, stderr, "pretty output")
199
+ assert_equal(
200
+ [
201
+ "{",
202
+ "\"hello\": 123",
203
+ "}",
204
+ "{",
205
+ "\"hello\": 456",
206
+ "}"
207
+ ],
208
+ lines(stdout),
209
+ "pretty output lines"
210
+ )
211
+
212
+ input_table_hash = '{"a":[1,2],"b":[3,4]}'
213
+ stdout, stderr, status = run_jrf('_', input_table_hash, "-o", "tsv")
214
+ assert_success(status, stderr, "tsv output hash of arrays")
215
+ assert_equal(["a\t1\t2", "b\t3\t4"], lines(stdout), "tsv output hash of arrays")
216
+
217
+ input_table_array = '[[1,"hello",true],[2,"world",false]]'
218
+ stdout, stderr, status = run_jrf('_', input_table_array, "-o", "tsv")
219
+ assert_success(status, stderr, "tsv output array of arrays")
220
+ assert_equal(["1\thello\ttrue", "2\tworld\tfalse"], lines(stdout), "tsv output array of arrays")
221
+
222
+ input_table_scalar = '{"foo":"bar","baz":42}'
223
+ stdout, stderr, status = run_jrf('_', input_table_scalar, "-o", "tsv")
224
+ assert_success(status, stderr, "tsv output hash of scalars")
225
+ assert_equal(["foo\tbar", "baz\t42"], lines(stdout), "tsv output hash of scalars")
226
+
227
+ input_table_nested = '{"a":[[1,2],[3,4]],"b":[[5,6],[7,8]]}'
228
+ stdout, stderr, status = run_jrf('_', input_table_nested, "-o", "tsv")
229
+ assert_success(status, stderr, "tsv output nested arrays as JSON")
230
+ assert_equal(["a\t[1,2]\t[3,4]", "b\t[5,6]\t[7,8]"], lines(stdout), "tsv output nested arrays as JSON")
231
+ end
232
+
233
+ def test_regex_and_parser_boundaries
234
+ input_regex = <<~NDJSON
235
+ {"foo":{"bar":"ok"},"x":50}
236
+ {"foo":{"bar":"ng"},"x":70}
237
+ NDJSON
238
+
239
+ stdout, stderr, status = run_jrf('select(/ok/.match(_["foo"]["bar"])) >> _["x"]', input_regex)
240
+ assert_success(status, stderr, "regex in select")
241
+ assert_equal(%w[50], lines(stdout), "regex filter output")
242
+
243
+ input_split = <<~NDJSON
244
+ {"x":1}
245
+ NDJSON
246
+
247
+ stdout, stderr, status = run_jrf('[1 >> 2] >> _', input_split)
248
+ assert_success(status, stderr, "no split inside []")
249
+ assert_equal(['[0]'], lines(stdout), "no split inside [] output")
250
+
251
+ stdout, stderr, status = run_jrf('{a: 1 >> 2} >> _[:a]', input_split)
252
+ assert_success(status, stderr, "no split inside {}")
253
+ assert_equal(%w[0], lines(stdout), "no split inside {} output")
254
+
255
+ stdout, stderr, status = run_jrf('(-> { 1 >> 2 }).call >> _ + 1', input_split)
256
+ assert_success(status, stderr, "no split inside block")
257
+ assert_equal(%w[1], lines(stdout), "no split inside block output")
258
+ end
259
+
260
+ def test_flat
261
+ input_flat = <<~NDJSON
262
+ {"items":[1,2]}
263
+ {"items":[3]}
264
+ {"items":[]}
265
+ NDJSON
266
+
267
+ stdout, stderr, status = run_jrf('_["items"] >> flat', input_flat)
268
+ assert_success(status, stderr, "flat basic")
269
+ assert_equal(%w[1 2 3], lines(stdout), "flat basic output")
270
+
271
+ input_flat_hash = <<~NDJSON
272
+ {"items":[{"x":1},{"x":2}]}
273
+ NDJSON
274
+
275
+ stdout, stderr, status = run_jrf('_["items"] >> flat >> _["x"]', input_flat_hash)
276
+ assert_success(status, stderr, "flat then extract")
277
+ assert_equal(%w[1 2], lines(stdout), "flat then extract output")
278
+
279
+ stdout, stderr, status = run_jrf('_["items"] >> flat >> sum(_)', input_flat)
280
+ assert_success(status, stderr, "flat then sum")
281
+ assert_equal(%w[6], lines(stdout), "flat then sum output")
282
+
283
+ stdout, stderr, status = run_jrf('_["items"] >> flat >> group', input_flat)
284
+ assert_success(status, stderr, "flat then group")
285
+ assert_equal(['[1,2,3]'], lines(stdout), "flat then group output")
286
+
287
+ stdout, stderr, status = run_jrf('map { |x| flat }', "[[1,2],[3],[4,5,6]]\n")
288
+ assert_success(status, stderr, "flat inside map")
289
+ assert_equal(['[1,2,3,4,5,6]'], lines(stdout), "flat inside map output")
290
+
291
+ stdout, stderr, status = run_jrf('map_values { |v| flat }', "{\"a\":[1,2],\"b\":[3]}\n")
292
+ assert_failure(status, "flat inside map_values")
293
+ assert_includes(stderr, "flat is not supported inside map_values")
294
+
295
+ stdout, stderr, status = run_jrf('_["foo"] >> flat', "{\"foo\":1}\n")
296
+ assert_failure(status, "flat requires array")
297
+ assert_includes(stderr, "flat expects Array")
298
+ end
299
+
300
+ def test_reducers
301
+ input = <<~NDJSON
302
+ {"foo":1,"x":5}
303
+ {"foo":2,"x":11}
304
+ {"foo":3,"x":50}
305
+ {"foo":4,"x":70}
306
+ NDJSON
307
+
308
+ stdout, stderr, status = run_jrf('sum(_["foo"])', input)
309
+ assert_success(status, stderr, "sum only")
310
+ assert_equal(%w[10], lines(stdout), "sum output")
311
+
312
+ stdout, stderr, status = run_jrf('count()', input)
313
+ assert_success(status, stderr, "count only")
314
+ assert_equal(%w[4], lines(stdout), "count output")
315
+
316
+ stdout, stderr, status = run_jrf('count(_["foo"])', input)
317
+ assert_success(status, stderr, "count(expr) only")
318
+ assert_equal(%w[4], lines(stdout), "count(expr) output")
319
+
320
+ stdout, stderr, status = run_jrf('min(_["foo"])', input)
321
+ assert_success(status, stderr, "min only")
322
+ assert_equal(%w[1], lines(stdout), "min output")
323
+
324
+ stdout, stderr, status = run_jrf('max(_["foo"])', input)
325
+ assert_success(status, stderr, "max only")
326
+ assert_equal(%w[4], lines(stdout), "max output")
327
+
328
+ stdout, stderr, status = run_jrf('select(_["x"] > 10) >> sum(_["foo"])', input)
329
+ assert_success(status, stderr, "select + sum")
330
+ assert_equal(%w[9], lines(stdout), "select + sum output")
331
+
332
+ stdout, stderr, status = run_jrf('{total: sum(_["foo"]), n: count()}', input)
333
+ assert_success(status, stderr, "structured reducer result")
334
+ assert_equal(['{"total":10,"n":4}'], lines(stdout), "structured reducer result output")
335
+
336
+ stdout, stderr, status = run_jrf('average(_["foo"])', input)
337
+ assert_success(status, stderr, "average")
338
+ assert_float_close(2.5, lines(stdout).first.to_f, 1e-12, "average output")
339
+
340
+ stdout, stderr, status = run_jrf('stdev(_["foo"])', input)
341
+ assert_success(status, stderr, "stdev")
342
+ assert_float_close(1.118033988749895, lines(stdout).first.to_f, 1e-12, "stdev output")
343
+
344
+ stdout, stderr, status = run_jrf('_["foo"] >> sum(_ * 2)', input)
345
+ assert_success(status, stderr, "extract + sum")
346
+ assert_equal(%w[20], lines(stdout), "extract + sum output")
347
+
348
+ stdout, stderr, status = run_jrf('sum(2 * _["foo"])', input)
349
+ assert_success(status, stderr, "sum with literal on left")
350
+ assert_equal(%w[20], lines(stdout), "sum with literal on left output")
351
+
352
+ stdout, stderr, status = run_jrf('select(_["x"] > 1000) >> sum(_["foo"])', input)
353
+ assert_success(status, stderr, "sum no matches")
354
+ assert_equal([], lines(stdout), "sum no matches output")
355
+
356
+ stdout, stderr, status = run_jrf('select(_["x"] > 1000) >> count()', input)
357
+ assert_success(status, stderr, "count no matches")
358
+ assert_equal([], lines(stdout), "count no matches output")
359
+
360
+ stdout, stderr, status = run_jrf('select(_["x"] > 1000) >> count(_["foo"])', input)
361
+ assert_success(status, stderr, "count(expr) no matches")
362
+ assert_equal([], lines(stdout), "count(expr) no matches output")
363
+
364
+ stdout, stderr, status = run_jrf('select(_["x"] > 1000) >> average(_["foo"])', input)
365
+ assert_success(status, stderr, "average no matches")
366
+ assert_equal([], lines(stdout), "average no matches output")
367
+
368
+ stdout, stderr, status = run_jrf('select(_["x"] > 1000) >> stdev(_["foo"])', input)
369
+ assert_success(status, stderr, "stdev no matches")
370
+ assert_equal([], lines(stdout), "stdev no matches output")
371
+
372
+ stdout, stderr, status = run_jrf('select(_["x"] > 1000) >> min(_["foo"])', input)
373
+ assert_success(status, stderr, "min no matches")
374
+ assert_equal([], lines(stdout), "min no matches output")
375
+
376
+ stdout, stderr, status = run_jrf('select(_["x"] > 1000) >> max(_["foo"])', input)
377
+ assert_success(status, stderr, "max no matches")
378
+ assert_equal([], lines(stdout), "max no matches output")
379
+
380
+ stdout, stderr, status = run_jrf('sum(_["foo"]) >> _ + 1', input)
381
+ assert_success(status, stderr, "reduce in middle")
382
+ assert_equal(%w[11], lines(stdout), "reduce in middle output")
383
+
384
+ stdout, stderr, status = run_jrf('select(_["x"] > 10) >> _["foo"] >> sum(_ * 2) >> select(_ > 10) >> _ + 1', input)
385
+ assert_success(status, stderr, "reduce mixed with select/extract")
386
+ assert_equal(%w[19], lines(stdout), "reduce mixed output")
387
+
388
+ stdout, stderr, status = run_jrf('_["foo"] >> sum(_) >> _ * 10 >> sum(_)', input)
389
+ assert_success(status, stderr, "multiple reducers")
390
+ assert_equal(%w[100], lines(stdout), "multiple reducers output")
391
+
392
+ stdout, stderr, status = run_jrf('_["foo"] >> min(_) >> _ * 10 >> max(_)', input)
393
+ assert_success(status, stderr, "min/max mixed reducers")
394
+ assert_equal(%w[10], lines(stdout), "min/max mixed reducers output")
395
+ end
396
+
397
+ def test_sort
398
+ input_sum = <<~NDJSON
399
+ {"foo":1,"x":5}
400
+ {"foo":2,"x":11}
401
+ {"foo":3,"x":50}
402
+ {"foo":4,"x":70}
403
+ NDJSON
404
+
405
+ input_sort_rows = <<~NDJSON
406
+ {"foo":"b","at":2}
407
+ {"foo":"c","at":3}
408
+ {"foo":"a","at":1}
409
+ NDJSON
410
+
411
+ stdout, stderr, status = run_jrf('sort(_["at"]) >> _["foo"]', input_sort_rows)
412
+ assert_success(status, stderr, "sort rows by field")
413
+ assert_equal(%w["a" "b" "c"], lines(stdout), "sort rows by field output")
414
+
415
+ stdout, stderr, status = run_jrf('sort { |a, b| b["at"] <=> a["at"] } >> _["foo"]', input_sort_rows)
416
+ assert_success(status, stderr, "sort rows by comparator")
417
+ assert_equal(%w["c" "b" "a"], lines(stdout), "sort rows by comparator output")
418
+
419
+ stdout, stderr, status = run_jrf('sort(_["at"]) >> _["foo"] >> group', input_sort_rows)
420
+ assert_success(status, stderr, "sort then group")
421
+ assert_equal(['["a","b","c"]'], lines(stdout), "sort then group output")
422
+
423
+ stdout, stderr, status = run_jrf('select(_["x"] > 1000) >> sort(_["x"]) >> _["foo"]', input_sum)
424
+ assert_success(status, stderr, "sort no matches")
425
+ assert_equal([], lines(stdout), "sort no matches output")
426
+
427
+ stdout, stderr, status = run_jrf('select(_["x"] > 1000) >> _["foo"] >> group', input_sum)
428
+ assert_success(status, stderr, "group no matches")
429
+ assert_equal([], lines(stdout), "group no matches output")
430
+ end
431
+
432
+ def test_group
433
+ input_group_multi = <<~NDJSON
434
+ {"x":1,"y":"a"}
435
+ {"x":2,"y":"b"}
436
+ {"x":3,"y":"c"}
437
+ NDJSON
438
+
439
+ stdout, stderr, status = run_jrf('{a: group(_["x"]), b: group(_["y"])}', input_group_multi)
440
+ assert_success(status, stderr, "group in hash")
441
+ assert_equal(['{"a":[1,2,3],"b":["a","b","c"]}'], lines(stdout), "group in hash output")
442
+
443
+ stdout, stderr, status = run_jrf('select(_["x"] > 1000) >> {a: group(_["x"]), b: group(_["y"])}', input_group_multi)
444
+ assert_success(status, stderr, "group in hash no matches")
445
+ assert_equal([], lines(stdout), "group in hash no-match output")
446
+ end
447
+
448
+ def test_percentile
449
+ input_sum = <<~NDJSON
450
+ {"foo":1,"x":5}
451
+ {"foo":2,"x":11}
452
+ {"foo":3,"x":50}
453
+ {"foo":4,"x":70}
454
+ NDJSON
455
+
456
+ stdout, stderr, status = run_jrf('percentile(_["foo"], 0.50)', input_sum)
457
+ assert_success(status, stderr, "single percentile")
458
+ assert_equal(%w[2], lines(stdout), "single percentile output")
459
+
460
+ stdout, stderr, status = run_jrf('percentile(_["foo"], [0.25, 0.50, 1.0])', input_sum)
461
+ assert_success(status, stderr, "array percentile")
462
+ assert_equal(['[1,2,4]'], lines(stdout), "array percentile output")
463
+
464
+ stdout, stderr, status = run_jrf('percentile(_["foo"], 0.25.step(1.0, 0.25))', input_sum)
465
+ assert_success(status, stderr, "enumerable percentile")
466
+ assert_equal(['[1,2,3,4]'], lines(stdout), "enumerable percentile output")
467
+ end
468
+
469
+ def test_nil_handling_for_aggregates
470
+ input_with_nil = <<~NDJSON
471
+ {"foo":1}
472
+ {"foo":null}
473
+ {"bar":999}
474
+ {"foo":3}
475
+ NDJSON
476
+
477
+ stdout, stderr, status = run_jrf('sum(_["foo"])', input_with_nil)
478
+ assert_success(status, stderr, "sum ignores nil")
479
+ assert_equal(%w[4], lines(stdout), "sum ignores nil output")
480
+
481
+ stdout, stderr, status = run_jrf('min(_["foo"])', input_with_nil)
482
+ assert_success(status, stderr, "min ignores nil")
483
+ assert_equal(%w[1], lines(stdout), "min ignores nil output")
484
+
485
+ stdout, stderr, status = run_jrf('max(_["foo"])', input_with_nil)
486
+ assert_success(status, stderr, "max ignores nil")
487
+ assert_equal(%w[3], lines(stdout), "max ignores nil output")
488
+
489
+ stdout, stderr, status = run_jrf('average(_["foo"])', input_with_nil)
490
+ assert_success(status, stderr, "average ignores nil")
491
+ assert_float_close(2.0, lines(stdout).first.to_f, 1e-12, "average ignores nil output")
492
+
493
+ stdout, stderr, status = run_jrf('stdev(_["foo"])', input_with_nil)
494
+ assert_success(status, stderr, "stdev ignores nil")
495
+ assert_float_close(1.0, lines(stdout).first.to_f, 1e-12, "stdev ignores nil output")
496
+
497
+ stdout, stderr, status = run_jrf('percentile(_["foo"], [0.5, 1.0])', input_with_nil)
498
+ assert_success(status, stderr, "percentile ignores nil")
499
+ assert_equal(['[1,3]'], lines(stdout), "percentile ignores nil output")
500
+
501
+ stdout, stderr, status = run_jrf('count()', input_with_nil)
502
+ assert_success(status, stderr, "count with nil rows")
503
+ assert_equal(%w[4], lines(stdout), "count with nil rows output")
504
+
505
+ stdout, stderr, status = run_jrf('count(_["foo"])', input_with_nil)
506
+ assert_success(status, stderr, "count(expr) ignores nil")
507
+ assert_equal(%w[2], lines(stdout), "count(expr) ignores nil output")
508
+
509
+ input_count_if = <<~NDJSON
510
+ {"x":1}
511
+ {"x":-2}
512
+ {"x":3}
513
+ {"x":-4}
514
+ {"x":5}
515
+ NDJSON
516
+
517
+ stdout, stderr, status = run_jrf('count_if(_["x"] > 0)', input_count_if)
518
+ assert_success(status, stderr, "count_if")
519
+ assert_equal(%w[3], lines(stdout), "count_if output")
520
+
521
+ stdout, stderr, status = run_jrf('[count_if(_["x"] > 0), count_if(_["x"] < 0)]', input_count_if)
522
+ assert_success(status, stderr, "count_if multiple")
523
+ assert_equal(["[3,2]"], lines(stdout), "count_if multiple output")
524
+
525
+ input_all_nil = <<~NDJSON
526
+ {"foo":null}
527
+ {"bar":1}
528
+ NDJSON
529
+
530
+ stdout, stderr, status = run_jrf('sum(_["foo"])', input_all_nil)
531
+ assert_success(status, stderr, "sum all nil")
532
+ assert_equal(%w[0], lines(stdout), "sum all nil output")
533
+
534
+ stdout, stderr, status = run_jrf('min(_["foo"])', input_all_nil)
535
+ assert_success(status, stderr, "min all nil")
536
+ assert_equal(%w[null], lines(stdout), "min all nil output")
537
+
538
+ stdout, stderr, status = run_jrf('max(_["foo"])', input_all_nil)
539
+ assert_success(status, stderr, "max all nil")
540
+ assert_equal(%w[null], lines(stdout), "max all nil output")
541
+
542
+ stdout, stderr, status = run_jrf('average(_["foo"])', input_all_nil)
543
+ assert_success(status, stderr, "average all nil")
544
+ assert_equal(%w[null], lines(stdout), "average all nil output")
545
+
546
+ stdout, stderr, status = run_jrf('stdev(_["foo"])', input_all_nil)
547
+ assert_success(status, stderr, "stdev all nil")
548
+ assert_equal(%w[null], lines(stdout), "stdev all nil output")
549
+
550
+ stdout, stderr, status = run_jrf('percentile(_["foo"], 0.5)', input_all_nil)
551
+ assert_success(status, stderr, "percentile all nil")
552
+ assert_equal(%w[null], lines(stdout), "percentile all nil output")
553
+
554
+ stdout, stderr, status = run_jrf('count(_["foo"])', input_all_nil)
555
+ assert_success(status, stderr, "count(expr) all nil")
556
+ assert_equal(%w[0], lines(stdout), "count(expr) all nil output")
557
+ end
558
+
559
+ def test_reduce
560
+ input_multi_cols = <<~NDJSON
561
+ {"a":1,"b":10}
562
+ {"a":2,"b":20}
563
+ {"a":3,"b":30}
564
+ {"a":4,"b":40}
565
+ NDJSON
566
+
567
+ stdout, stderr, status = run_jrf('{a: percentile(_["a"], [0.25, 0.50, 1.0]), b: percentile(_["b"], [0.25, 0.50, 1.0])}', input_multi_cols)
568
+ assert_success(status, stderr, "nested array percentile for multiple columns")
569
+ assert_equal(['{"a":[1,2,4],"b":[10,20,40]}'], lines(stdout), "nested array percentile output")
570
+
571
+ input_reduce = <<~NDJSON
572
+ {"s":"hello"}
573
+ {"s":"world"}
574
+ {"s":"jrf"}
575
+ NDJSON
576
+
577
+ stdout, stderr, status = run_jrf('_["s"] >> reduce("") { |acc, v| acc.empty? ? v : "#{acc} #{v}" }', input_reduce)
578
+ assert_success(status, stderr, "reduce with implicit value")
579
+ assert_equal(['"hello world jrf"'], lines(stdout), "reduce implicit value output")
580
+
581
+ stdout, stderr, status = run_jrf('_["s"] >> reduce("") { |acc, v| acc.empty? ? v : "#{acc} #{v}" }', input_reduce)
582
+ assert_success(status, stderr, "reduce in two-stage form")
583
+ assert_equal(['"hello world jrf"'], lines(stdout), "reduce in two-stage form output")
584
+
585
+ input_sum = <<~NDJSON
586
+ {"foo":1,"x":5}
587
+ {"foo":2,"x":11}
588
+ {"foo":3,"x":50}
589
+ {"foo":4,"x":70}
590
+ NDJSON
591
+
592
+ stdout, stderr, status = run_jrf('sum(_["foo"]) >> select(_ > 100)', input_sum)
593
+ assert_success(status, stderr, "post-reduce select drop")
594
+ assert_equal([], lines(stdout), "post-reduce select drop output")
595
+ end
596
+
597
+ def test_lax_input_mode
598
+ input_whitespace_stream = "{\"foo\":1} {\"foo\":2}\n\t{\"foo\":3}\n"
599
+ stdout, stderr, status = run_jrf('_["foo"]', input_whitespace_stream)
600
+ assert_failure(status, "default NDJSON should reject same-line multi-values")
601
+ assert_includes(stderr, "JSON::ParserError")
602
+
603
+ stdout, stderr, status = run_jrf('_["foo"]', input_whitespace_stream, "--lax")
604
+ assert_success(status, stderr, "whitespace-separated JSON stream with --lax")
605
+ assert_equal(%w[1 2 3], lines(stdout), "whitespace-separated stream output")
606
+
607
+ input_json_seq = "\x1e{\"foo\":10}\n\x1e{\"foo\":20}\n"
608
+ stdout, stderr, status = run_jrf('_["foo"]', input_json_seq)
609
+ assert_failure(status, "RS framing requires --lax")
610
+ assert_includes(stderr, "JSON::ParserError")
611
+
612
+ stdout, stderr, status = run_jrf('_["foo"]', input_json_seq, "--lax")
613
+ assert_success(status, stderr, "json-seq style RS framing with --lax")
614
+ assert_equal(%w[10 20], lines(stdout), "json-seq style output")
615
+
616
+ input_lax_multiline = <<~JSONS
617
+ {
618
+ "foo": 101,
619
+ "bar": {"x": 1}
620
+ }
621
+ {
622
+ "foo": 202,
623
+ "bar": {"x": 2}
624
+ }
625
+ JSONS
626
+ stdout, stderr, status = run_jrf('_["foo"]', input_lax_multiline)
627
+ assert_failure(status, "default NDJSON rejects multiline objects")
628
+ assert_includes(stderr, "JSON::ParserError")
629
+
630
+ stdout, stderr, status = run_jrf('_["bar"]["x"]', input_lax_multiline, "--lax")
631
+ assert_success(status, stderr, "lax accepts multiline objects")
632
+ assert_equal(%w[1 2], lines(stdout), "lax multiline object output")
633
+
634
+ input_lax_mixed_separators = "{\"foo\":1}\n\x1e{\"foo\":2}\t{\"foo\":3}\n"
635
+ stdout, stderr, status = run_jrf('_["foo"]', input_lax_mixed_separators, "--lax")
636
+ assert_success(status, stderr, "lax accepts mixed whitespace and RS separators")
637
+ assert_equal(%w[1 2 3], lines(stdout), "lax mixed separators output")
638
+
639
+ input_lax_with_escaped_newline = "{\"s\":\"line1\\nline2\"}\n{\"s\":\"ok\"}\n"
640
+ stdout, stderr, status = run_jrf('_["s"]', input_lax_with_escaped_newline, "--lax")
641
+ assert_success(status, stderr, "lax handles escaped newlines in strings")
642
+ assert_equal(['"line1\nline2"', '"ok"'], lines(stdout), "lax escaped newline string output")
643
+
644
+ input_lax_trailing_rs = "\x1e{\"foo\":9}\n\x1e"
645
+ stdout, stderr, status = run_jrf('_["foo"]', input_lax_trailing_rs, "--lax")
646
+ assert_success(status, stderr, "lax ignores trailing separator")
647
+ assert_equal(%w[9], lines(stdout), "lax trailing separator output")
648
+
649
+ chunked_lax_out = RecordingRunner.new(
650
+ input: ChunkedSource.new("{\"foo\":1}\n\x1e{\"foo\":2}\n\t{\"foo\":3}\n"),
651
+ out: StringIO.new,
652
+ err: StringIO.new,
653
+ lax: true
654
+ )
655
+ chunked_lax_out.run('_["foo"]')
656
+ assert_equal(%w[1 2 3], lines(chunked_lax_out.writes.join), "lax mode streams chunked input without whole-input reads")
657
+
658
+ Dir.mktmpdir do |dir|
659
+ one = File.join(dir, "one.json")
660
+ two = File.join(dir, "two.json")
661
+ File.write(one, "1")
662
+ File.write(two, "2")
663
+
664
+ stdout, stderr, status = Open3.capture3("./exe/jrf", "--lax", "_", one, two)
665
+ assert_success(status, stderr, "lax keeps file boundaries")
666
+ assert_equal(%w[1 2], lines(stdout), "lax does not merge JSON across file boundaries")
667
+ end
668
+ end
669
+
670
+ def test_parse_errors
671
+ stdout, stderr, status = run_jrf('select(_["x"] > ) >> _["foo"]', "")
672
+ assert_failure(status, "syntax error should fail before row loop")
673
+ assert_includes(stderr, "syntax error")
674
+
675
+ stdout, stderr, status = run_jrf('([)] >> _', "")
676
+ assert_failure(status, "mismatched delimiter should fail")
677
+ assert_includes(stderr, "mismatched delimiter")
678
+
679
+ stdout, stderr, status = run_jrf('(_["x"] >> _["y"]', "")
680
+ assert_failure(status, "unclosed delimiter should fail")
681
+ assert_includes(stderr, "unclosed delimiter")
682
+
683
+ input_broken_tail = <<~NDJSON
684
+ {"foo":1}
685
+ {"foo":2}
686
+ {"foo":
687
+ NDJSON
688
+
689
+ stdout, stderr, status = run_jrf('sum(_["foo"])', input_broken_tail)
690
+ assert_failure(status, "broken input should fail")
691
+ assert_equal(%w[3], lines(stdout), "reducers flush before parse error")
692
+ assert_includes(stderr, "JSON::ParserError")
693
+ refute_includes(stderr, "from ", "no stacktrace for parse errors")
694
+ end
695
+
696
+ def test_map
697
+ input_chain = <<~NDJSON
698
+ {"foo":{"bar":{"z":1},"keep":true}}
699
+ {"foo":{"bar":{"z":2},"keep":false}}
700
+ {"foo":{"bar":{"z":3},"keep":true}}
701
+ NDJSON
702
+
703
+ stdout, stderr, status = run_jrf('_["foo"] >> select(_["keep"]) >> _["bar"] >> select(_["z"] > 1) >> _["z"]', input_chain)
704
+ assert_success(status, stderr, "select/extract chain")
705
+ assert_equal(%w[3], lines(stdout), "chain output")
706
+
707
+ input_map = <<~NDJSON
708
+ {"values":[1,10,100]}
709
+ {"values":[2,20,200]}
710
+ {"values":[3,30,300]}
711
+ NDJSON
712
+
713
+ stdout, stderr, status = run_jrf('_["values"] >> map { |x| sum(x) }', input_map)
714
+ assert_success(status, stderr, "map with sum")
715
+ assert_equal(['[6,60,600]'], lines(stdout), "map with sum output")
716
+
717
+ stdout, stderr, status = run_jrf('_["values"] >> map { |x| min(x) }', input_map)
718
+ assert_success(status, stderr, "map with min")
719
+ assert_equal(['[1,10,100]'], lines(stdout), "map with min output")
720
+
721
+ stdout, stderr, status = run_jrf('_["values"] >> map { |x| max(x) }', input_map)
722
+ assert_success(status, stderr, "map with max")
723
+ assert_equal(['[3,30,300]'], lines(stdout), "map with max output")
724
+
725
+ stdout, stderr, status = run_jrf('_["values"] >> map { |x| sum(_[0] + x) }', input_map)
726
+ assert_success(status, stderr, "map keeps ambient _")
727
+ assert_equal(['[12,66,606]'], lines(stdout), "map ambient _ output")
728
+
729
+ stdout, stderr, status = run_jrf('_["values"] >> map { |x| reduce(0) { |acc, v| acc + v } }', input_map)
730
+ assert_success(status, stderr, "map with reduce")
731
+ assert_equal(['[6,60,600]'], lines(stdout), "map with reduce output")
732
+
733
+ input_map_varying = <<~NDJSON
734
+ [1,10]
735
+ [2,20,200]
736
+ [3]
737
+ NDJSON
738
+
739
+ stdout, stderr, status = run_jrf('map { |x| sum(x) }', input_map_varying)
740
+ assert_success(status, stderr, "map varying lengths")
741
+ assert_equal(['[6,30,200]'], lines(stdout), "map varying lengths output")
742
+
743
+ input_map_unsorted = <<~NDJSON
744
+ {"values":[3,30]}
745
+ {"values":[1,10]}
746
+ {"values":[2,20]}
747
+ NDJSON
748
+
749
+ stdout, stderr, status = run_jrf('_["values"] >> map { |x| group }', input_map)
750
+ assert_success(status, stderr, "map with group")
751
+ assert_equal(['[[1,2,3],[10,20,30],[100,200,300]]'], lines(stdout), "map with group output")
752
+
753
+ stdout, stderr, status = run_jrf('_["values"] >> map { |x| sort }', input_map_unsorted)
754
+ assert_success(status, stderr, "map with sort default key")
755
+ assert_equal(['[[1,2,3],[10,20,30]]'], lines(stdout), "map with sort default key output")
756
+
757
+ stdout, stderr, status = run_jrf('select(false) >> map { |x| sum(x) }', input_map)
758
+ assert_success(status, stderr, "map no matches")
759
+ assert_equal([], lines(stdout), "map no matches output")
760
+
761
+ stdout, stderr, status = run_jrf('_["values"] >> map { |x| x + 1 }', input_map)
762
+ assert_success(status, stderr, "map transform")
763
+ assert_equal(['[2,11,101]', '[3,21,201]', '[4,31,301]'], lines(stdout), "map transform output")
764
+
765
+ stdout, stderr, status = run_jrf('_["values"] >> map { |x| select(x >= 20) }', input_map)
766
+ assert_success(status, stderr, "map transform with select")
767
+ assert_equal(['[100]', '[20,200]', '[30,300]'], lines(stdout), "map transform with select output")
768
+ end
769
+
770
+ def test_map_values
771
+ input_map_values = <<~NDJSON
772
+ {"a":1,"b":10}
773
+ {"a":2,"b":20}
774
+ {"a":3,"b":30}
775
+ NDJSON
776
+
777
+ stdout, stderr, status = run_jrf('map_values { |v| sum(v) }', input_map_values)
778
+ assert_success(status, stderr, "map_values with sum")
779
+ assert_equal(['{"a":6,"b":60}'], lines(stdout), "map_values with sum output")
780
+
781
+ stdout, stderr, status = run_jrf('map_values { |v| min(v) }', input_map_values)
782
+ assert_success(status, stderr, "map_values with min")
783
+ assert_equal(['{"a":1,"b":10}'], lines(stdout), "map_values with min output")
784
+
785
+ input_map_values_varying = <<~NDJSON
786
+ {"a":1}
787
+ {"a":2,"b":20}
788
+ {"a":3,"b":30}
789
+ NDJSON
790
+
791
+ stdout, stderr, status = run_jrf('map_values { |v| sum(v) }', input_map_values_varying)
792
+ assert_success(status, stderr, "map_values varying keys")
793
+ assert_equal(['{"a":6,"b":50}'], lines(stdout), "map_values varying keys output")
794
+
795
+ stdout, stderr, status = run_jrf('map_values { |v| count(v) }', input_map_values)
796
+ assert_success(status, stderr, "map_values with count")
797
+ assert_equal(['{"a":3,"b":3}'], lines(stdout), "map_values with count output")
798
+
799
+ stdout, stderr, status = run_jrf('map_values { |v| group }', input_map_values)
800
+ assert_success(status, stderr, "map_values with group")
801
+ assert_equal(['{"a":[1,2,3],"b":[10,20,30]}'], lines(stdout), "map_values with group output")
802
+
803
+ stdout, stderr, status = run_jrf('map_values { |v| sum(_["a"] + v) }', input_map_values)
804
+ assert_success(status, stderr, "map_values keeps ambient _")
805
+ assert_equal(['{"a":12,"b":66}'], lines(stdout), "map_values ambient _ output")
806
+
807
+ stdout, stderr, status = run_jrf('map_values { |v| reduce(0) { |acc, x| acc + x } }', input_map_values)
808
+ assert_success(status, stderr, "map_values with reduce")
809
+ assert_equal(['{"a":6,"b":60}'], lines(stdout), "map_values with reduce output")
810
+
811
+ stdout, stderr, status = run_jrf('map { |k, v| "#{k}:#{v}" }', input_map_values)
812
+ assert_success(status, stderr, "map over hash transform")
813
+ assert_equal(['["a:1","b:10"]', '["a:2","b:20"]', '["a:3","b:30"]'], lines(stdout), "map over hash transform output")
814
+
815
+ stdout, stderr, status = run_jrf('map { |pair| pair }', input_map_values)
816
+ assert_success(status, stderr, "map over hash single block arg")
817
+ assert_equal(['[["a",1],["b",10]]', '[["a",2],["b",20]]', '[["a",3],["b",30]]'], lines(stdout), "map over hash single block arg output")
818
+
819
+ stdout, stderr, status = run_jrf('map { |k, v| select(v >= 10 && k != "a") }', input_map_values)
820
+ assert_success(status, stderr, "map over hash transform with select")
821
+ assert_equal(['[10]', '[20]', '[30]'], lines(stdout), "map over hash transform with select output")
822
+
823
+ stdout, stderr, status = run_jrf('map { |k, v| sum(v + k.length) }', input_map_values)
824
+ assert_success(status, stderr, "map over hash with sum")
825
+ assert_equal(['[9,63]'], lines(stdout), "map over hash with sum output")
826
+
827
+ stdout, stderr, status = run_jrf('map { |k, v| sum(_["a"] + v + k.length) }', input_map_values)
828
+ assert_success(status, stderr, "map over hash keeps ambient _")
829
+ assert_equal(['[15,69]'], lines(stdout), "map over hash ambient _ output")
830
+
831
+ stdout, stderr, status = run_jrf('select(false) >> map_values { |v| sum(v) }', input_map_values)
832
+ assert_success(status, stderr, "map_values no matches")
833
+ assert_equal([], lines(stdout), "map_values no matches output")
834
+
835
+ stdout, stderr, status = run_jrf('map_values { |v| sum(v) } >> map_values { |v| v * 10 }', input_map_values)
836
+ assert_success(status, stderr, "map_values piped to map_values passthrough")
837
+ assert_equal(['{"a":60,"b":600}'], lines(stdout), "map_values piped output")
838
+
839
+ stdout, stderr, status = run_jrf('map_values { |v| v * 2 }', input_map_values)
840
+ assert_success(status, stderr, "map_values transform")
841
+ assert_equal(['{"a":2,"b":20}', '{"a":4,"b":40}', '{"a":6,"b":60}'], lines(stdout), "map_values transform output")
842
+
843
+ stdout, stderr, status = run_jrf('map_values { |v| select(v >= 10) }', input_map_values)
844
+ assert_success(status, stderr, "map_values transform with select")
845
+ assert_equal(['{"b":10}', '{"b":20}', '{"b":30}'], lines(stdout), "map_values transform with select output")
846
+ end
847
+
848
+ def test_apply
849
+ input_map = <<~NDJSON
850
+ {"values":[1,10,100]}
851
+ {"values":[2,20,200]}
852
+ {"values":[3,30,300]}
853
+ NDJSON
854
+
855
+ stdout, stderr, status = run_jrf('_["values"] >> map { |x| x + 1 } >> map { |x| x * 10 }', input_map)
856
+ assert_success(status, stderr, "chained map transforms")
857
+ assert_equal(['[20,110,1010]', '[30,210,2010]', '[40,310,3010]'], lines(stdout), "chained map transforms output")
858
+
859
+ stdout, stderr, status = run_jrf('map { map { |y| [ sum(y[0]), sum(y[1]) ] } }', "[[[1,2]]]\n[[[3,4]]]\n")
860
+ assert_success(status, stderr, "nested map reducer binds to current target")
861
+ assert_equal(['[[[4,6]]]'], lines(stdout), "nested map reducer output")
862
+
863
+ stdout, stderr, status = run_jrf('map_values { |obj| map_values { |v| sum(v) } }', "{\"a\":{\"x\":1,\"y\":2},\"b\":{\"x\":10,\"y\":20}}\n{\"a\":{\"x\":3,\"y\":4},\"b\":{\"x\":30,\"y\":40}}\n")
864
+ assert_success(status, stderr, "nested map_values reducer binds to current target")
865
+ assert_equal(['{"a":{"x":4,"y":6},"b":{"x":40,"y":60}}'], lines(stdout), "nested map_values reducer output")
866
+
867
+ stdout, stderr, status = run_jrf('[apply { |x| sum(x["foo"]) }, _.length]', '[{"foo":1},{"foo":2}]' + "\n" + '[{"foo":10}]' + "\n")
868
+ assert_success(status, stderr, "apply with sum")
869
+ assert_equal(["[3,2]", "[10,1]"], lines(stdout), "apply with sum output")
870
+
871
+ stdout, stderr, status = run_jrf('apply { |x| x["foo"] }', '[{"foo":1},{"foo":2}]' + "\n")
872
+ assert_success(status, stderr, "apply passthrough")
873
+ assert_equal(["[1,2]"], lines(stdout), "apply passthrough output")
874
+
875
+ stdout, stderr, status = run_jrf('apply { |x| percentile(x, 0.5) }', '[10,20,30]' + "\n")
876
+ assert_success(status, stderr, "apply with percentile")
877
+ assert_equal(["20"], lines(stdout), "apply with percentile output")
878
+
879
+ stdout, stderr, status = run_jrf('map { |o| [apply(o["vals"]) { |x| sum(x) }, o["name"]] }', '[{"name":"a","vals":[1,2]},{"name":"b","vals":[10,20]}]' + "\n")
880
+ assert_success(status, stderr, "apply with explicit collection")
881
+ assert_equal(['[[3,"a"],[30,"b"]]'], lines(stdout), "apply with explicit collection output")
882
+
883
+ stdout, stderr, status = run_jrf('map(_["items"]) { |x| x * 2 }', '{"items":[1,2,3]}' + "\n")
884
+ assert_success(status, stderr, "map with explicit collection")
885
+ assert_equal(["[2,4,6]"], lines(stdout), "map with explicit collection output")
886
+
887
+ stdout, stderr, status = run_jrf('map_values(_["data"]) { |v| v * 10 }', '{"data":{"a":1,"b":2}}' + "\n")
888
+ assert_success(status, stderr, "map_values with explicit collection")
889
+ assert_equal(['{"a":10,"b":20}'], lines(stdout), "map_values with explicit collection output")
890
+ end
891
+
892
+ def test_group_by
893
+ input_gb = <<~NDJSON
894
+ {"status":200,"path":"/a","latency":10}
895
+ {"status":404,"path":"/b","latency":50}
896
+ {"status":200,"path":"/c","latency":30}
897
+ {"status":200,"path":"/d","latency":20}
898
+ NDJSON
899
+
900
+ stdout, stderr, status = run_jrf('group_by(_["status"]) { count() }', input_gb)
901
+ assert_success(status, stderr, "group_by with count")
902
+ assert_equal(['{"200":3,"404":1}'], lines(stdout), "group_by with count output")
903
+
904
+ stdout, stderr, status = run_jrf('group_by(_["status"]) { |row| sum(row["latency"]) }', input_gb)
905
+ assert_success(status, stderr, "group_by with sum")
906
+ assert_equal(['{"200":60,"404":50}'], lines(stdout), "group_by with sum output")
907
+
908
+ stdout, stderr, status = run_jrf('group_by(_["status"]) { |row| average(row["latency"]) }', input_gb)
909
+ assert_success(status, stderr, "group_by with average")
910
+ result = JSON.parse(lines(stdout).first)
911
+ assert_float_close(20.0, result["200"], 1e-12, "group_by average 200")
912
+ assert_float_close(50.0, result["404"], 1e-12, "group_by average 404")
913
+
914
+ stdout, stderr, status = run_jrf('group_by(_["status"])', input_gb)
915
+ assert_success(status, stderr, "group_by default (collect rows)")
916
+ result = JSON.parse(lines(stdout).first)
917
+ assert_equal(3, result["200"].length, "group_by default 200 count")
918
+ assert_equal(1, result["404"].length, "group_by default 404 count")
919
+ assert_equal("/a", result["200"][0]["path"], "group_by default first row")
920
+
921
+ stdout, stderr, status = run_jrf('group_by(_["status"]) { |row| group(row["path"]) }', input_gb)
922
+ assert_success(status, stderr, "group_by with group(expr)")
923
+ assert_equal(['{"200":["/a","/c","/d"],"404":["/b"]}'], lines(stdout), "group_by with group(expr) output")
924
+
925
+ stdout, stderr, status = run_jrf('group_by(_["status"]) { group }', input_gb)
926
+ assert_success(status, stderr, "group_by with implicit group")
927
+ result = JSON.parse(lines(stdout).first)
928
+ assert_equal(3, result["200"].length, "group_by implicit group 200 count")
929
+ assert_equal("/a", result["200"][0]["path"], "group_by implicit group first row")
930
+
931
+ stdout, stderr, status = run_jrf('group_by(_["status"]) { |row| min(row["latency"]) }', input_gb)
932
+ assert_success(status, stderr, "group_by with min")
933
+ assert_equal(['{"200":10,"404":50}'], lines(stdout), "group_by with min output")
934
+
935
+ stdout, stderr, status = run_jrf('group_by(_["status"]) { |row| {total: sum(row["latency"]), n: count()} }', input_gb)
936
+ assert_success(status, stderr, "group_by with multi-reducer")
937
+ assert_equal(['{"200":{"total":60,"n":3},"404":{"total":50,"n":1}}'], lines(stdout), "group_by multi-reducer output")
938
+
939
+ stdout, stderr, status = run_jrf('group_by(_["status"]) { reduce(0) { |acc, row| acc + row["latency"] } }', input_gb)
940
+ assert_success(status, stderr, "group_by with reduce")
941
+ assert_equal(['{"200":60,"404":50}'], lines(stdout), "group_by with reduce output")
942
+
943
+ stdout, stderr, status = run_jrf('select(false) >> group_by(_["status"]) { count() }', input_gb)
944
+ assert_success(status, stderr, "group_by no matches")
945
+ assert_equal([], lines(stdout), "group_by no matches output")
946
+
947
+ stdout, stderr, status = run_jrf('group_by(_["status"]) { count() } >> _[200]', input_gb)
948
+ assert_success(status, stderr, "group_by then extract")
949
+ assert_equal(%w[3], lines(stdout), "group_by then extract output")
950
+ end
951
+ end