jrf 0.1.12 → 0.1.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,951 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "test_helper"
4
+
5
+ class CliRunnerTest < JrfTestCase
6
+ def test_extract_and_select
7
+ input = <<~NDJSON
8
+ {"foo":1,"x":5}
9
+ {"foo":2,"x":11}
10
+ {"foo":{"bar":"ok"},"x":50}
11
+ {"x":70}
12
+ NDJSON
13
+
14
+ stdout, stderr, status = run_jrf('_["foo"]', input)
15
+ assert_success(status, stderr, "simple extract")
16
+ assert_equal(%w[1 2 {"bar":"ok"} null], lines(stdout), "extract output")
17
+
18
+ input_nested = <<~NDJSON
19
+ {"foo":{"bar":"a"}}
20
+ {"foo":{"bar":"b"}}
21
+ NDJSON
22
+
23
+ stdout, stderr, status = run_jrf('_["foo"]["bar"]', input_nested)
24
+ assert_success(status, stderr, "nested extract")
25
+ assert_equal(%w["a" "b"], lines(stdout), "nested output")
26
+
27
+ stdout, stderr, status = run_jrf('select(_["x"] > 10) >> _["foo"]', input)
28
+ assert_success(status, stderr, "select + extract")
29
+ assert_equal(%w[2 {"bar":"ok"} null], lines(stdout), "filtered output")
30
+
31
+ stdout, stderr, status = run_jrf('select(_["x"] > 10)', input)
32
+ assert_success(status, stderr, "select only")
33
+ assert_equal(
34
+ ['{"foo":2,"x":11}', '{"foo":{"bar":"ok"},"x":50}', '{"x":70}'],
35
+ lines(stdout),
36
+ "select-only output"
37
+ )
38
+
39
+ input_hello = <<~NDJSON
40
+ {"hello":123}
41
+ {"hello":456}
42
+ NDJSON
43
+
44
+ stdout, stderr, status = run_jrf('select(_["hello"] == 123)', input_hello)
45
+ assert_success(status, stderr, "select-only hello")
46
+ assert_equal(['{"hello":123}'], lines(stdout), "select-only hello output")
47
+
48
+ stdout, stderr, status = run_jrf('select(_["hello"] == 123) >> _["hello"]', input_hello, "-v")
49
+ assert_success(status, stderr, "dump stages")
50
+ assert_equal(%w[123], lines(stdout), "dump stages output")
51
+ assert_includes(stderr, 'stage[0]: select(_["hello"] == 123)')
52
+ assert_includes(stderr, 'stage[1]: _["hello"]')
53
+ end
54
+
55
+ def test_help_version_and_atomic_write_options
56
+ input_hello = <<~NDJSON
57
+ {"hello":123}
58
+ {"hello":456}
59
+ NDJSON
60
+
61
+ stdout, stderr, status = Open3.capture3("./exe/jrf", "--help")
62
+ assert_success(status, stderr, "help option")
63
+ assert_includes(stdout, "usage: jrf [options] 'STAGE >> STAGE >> ...'")
64
+ assert_includes(stdout, "JSON filter with the power and speed of Ruby.")
65
+ assert_includes(stdout, "--lax")
66
+ assert_includes(stdout, "--output")
67
+ assert_includes(stdout, "--require LIBRARY")
68
+ assert_includes(stdout, "--no-jit")
69
+ assert_includes(stdout, "-V")
70
+ assert_includes(stdout, "--version")
71
+ assert_includes(stdout, "--atomic-write-bytes N")
72
+ assert_includes(stdout, "Pipeline:")
73
+ assert_includes(stdout, "Connect stages with top-level >>.")
74
+ assert_includes(stdout, "The current value in each stage is available as _.")
75
+ assert_includes(stdout, "See Also:")
76
+ assert_includes(stdout, "https://github.com/kazuho/jrf#readme")
77
+ assert_equal([], lines(stderr), "help stderr output")
78
+
79
+ stdout, stderr, status = Open3.capture3("./exe/jrf", "--version")
80
+ assert_success(status, stderr, "version long option")
81
+ assert_equal([Jrf::VERSION], lines(stdout), "version long option output")
82
+ assert_equal([], lines(stderr), "version long option stderr")
83
+
84
+ stdout, stderr, status = Open3.capture3("./exe/jrf", "-V")
85
+ assert_success(status, stderr, "version short option")
86
+ assert_equal([Jrf::VERSION], lines(stdout), "version short option output")
87
+ assert_equal([], lines(stderr), "version short option stderr")
88
+
89
+ stdout, stderr, status = run_jrf('select(_["hello"] == 123) >> _["hello"]', input_hello, "--verbose")
90
+ assert_success(status, stderr, "dump stages verbose alias")
91
+ assert_equal(%w[123], lines(stdout), "dump stages verbose alias output")
92
+ assert_includes(stderr, 'stage[0]: select(_["hello"] == 123)')
93
+
94
+ stdout, stderr, status = run_jrf('_["hello"]', input_hello, "--atomic-write-bytes", "512")
95
+ assert_success(status, stderr, "atomic write bytes option")
96
+ assert_equal(%w[123 456], lines(stdout), "atomic write bytes option output")
97
+
98
+ stdout, stderr, status = run_jrf('_["hello"]', input_hello, "--atomic-write-bytes=512")
99
+ assert_success(status, stderr, "atomic write bytes equals form")
100
+ assert_equal(%w[123 456], lines(stdout), "atomic write bytes equals form output")
101
+
102
+ stdout, stderr, status = Open3.capture3("./exe/jrf", "--atomic-write-bytes", "0", '_["hello"]', stdin_data: input_hello)
103
+ assert_failure(status, "atomic write bytes rejects zero")
104
+ assert_includes(stderr, "--atomic-write-bytes requires a positive integer")
105
+ end
106
+
107
+ def test_runner_buffering_and_require_option
108
+ threshold_input = StringIO.new((1..4).map { |i| "{\"foo\":\"#{'x' * 1020}\",\"i\":#{i}}\n" }.join)
109
+ buffered_runner = RecordingRunner.new(inputs: [threshold_input], out: StringIO.new, err: StringIO.new)
110
+ buffered_runner.run('_')
111
+ expected_line = JSON.generate({"foo" => "x" * 1020, "i" => 1}) + "\n"
112
+ assert_equal(2, buffered_runner.writes.length, "default atomic write limit buffers records until the configured threshold")
113
+ assert_equal(expected_line.bytesize * 3, buffered_runner.writes.first.bytesize, "default atomic write limit flushes before the next record would exceed the threshold")
114
+ assert_equal(expected_line.bytesize, buffered_runner.writes.last.bytesize, "final buffer flush emits the remaining record")
115
+
116
+ small_limit_runner = RecordingRunner.new(inputs: [StringIO.new("{\"foo\":1}\n{\"foo\":2}\n")], out: StringIO.new, err: StringIO.new, atomic_write_bytes: 1)
117
+ small_limit_runner.run('_["foo"]')
118
+ assert_equal(["1\n", "2\n"], small_limit_runner.writes, "small atomic write limit emits oversized records directly")
119
+
120
+ error_runner = RecordingRunner.new(inputs: [StringIO.new("{\"foo\":1}\n{\"foo\":")], out: StringIO.new, err: StringIO.new)
121
+ begin
122
+ error_runner.run('_["foo"]')
123
+ flunk("expected parse error for buffered flush test")
124
+ rescue JSON::ParserError
125
+ assert_equal(["1\n"], error_runner.writes, "buffer flushes pending output before parse errors escape")
126
+ end
127
+
128
+ input_hello = <<~NDJSON
129
+ {"hello":123}
130
+ {"hello":456}
131
+ NDJSON
132
+
133
+ Dir.mktmpdir do |dir|
134
+ helper = File.join(dir, "helpers.rb")
135
+ File.write(helper, <<~RUBY)
136
+ def double(value)
137
+ value * 2
138
+ end
139
+ RUBY
140
+
141
+ stdout, stderr, status = Open3.capture3("./exe/jrf", "-r", helper, 'double(_["hello"])', stdin_data: input_hello)
142
+ assert_success(status, stderr, "require helper option")
143
+ assert_equal(%w[246 912], lines(stdout), "require helper option output")
144
+ end
145
+ end
146
+
147
+ def test_yjit_option
148
+ if defined?(RubyVM::YJIT) && RubyVM::YJIT.respond_to?(:enabled?)
149
+ yjit_probe = "{\"probe\":1}\n"
150
+
151
+ stdout, stderr, status = run_jrf('RubyVM::YJIT.enabled?', yjit_probe)
152
+ assert_success(status, stderr, "default jit enablement")
153
+ assert_equal(%w[true], lines(stdout), "default jit enablement output")
154
+
155
+ stdout, stderr, status = run_jrf('RubyVM::YJIT.enabled?', yjit_probe, "--no-jit")
156
+ assert_success(status, stderr, "no-jit option")
157
+ assert_equal(%w[false], lines(stdout), "no-jit option output")
158
+ end
159
+ end
160
+
161
+ def test_compressed_inputs
162
+ Dir.mktmpdir do |dir|
163
+ gz_path = File.join(dir, "input.ndjson.gz")
164
+ Zlib::GzipWriter.open(gz_path) do |io|
165
+ io.write("{\"foo\":10}\n{\"foo\":20}\n")
166
+ end
167
+
168
+ stdout, stderr, status = Open3.capture3("./exe/jrf", '_["foo"]', gz_path)
169
+ assert_success(status, stderr, "compressed input by suffix")
170
+ assert_equal(%w[10 20], lines(stdout), "compressed input output")
171
+
172
+ lax_gz_path = File.join(dir, "input-lax.json.gz")
173
+ Zlib::GzipWriter.open(lax_gz_path) do |io|
174
+ io.write("{\"foo\":30}\n\x1e{\"foo\":40}\n")
175
+ end
176
+
177
+ stdout, stderr, status = Open3.capture3("./exe/jrf", "--lax", '_["foo"]', lax_gz_path)
178
+ assert_success(status, stderr, "compressed lax input by suffix")
179
+ assert_equal(%w[30 40], lines(stdout), "compressed lax input output")
180
+
181
+ second_gz_path = File.join(dir, "input2.ndjson.gz")
182
+ Zlib::GzipWriter.open(second_gz_path) do |io|
183
+ io.write("{\"foo\":50}\n")
184
+ end
185
+
186
+ stdout, stderr, status = Open3.capture3("./exe/jrf", '_["foo"]', gz_path, second_gz_path)
187
+ assert_success(status, stderr, "multiple compressed inputs by suffix")
188
+ assert_equal(%w[10 20 50], lines(stdout), "multiple compressed input output")
189
+ end
190
+ end
191
+
192
+ def test_output_formats
193
+ input_hello = <<~NDJSON
194
+ {"hello":123}
195
+ {"hello":456}
196
+ NDJSON
197
+
198
+ stdout, stderr, status = run_jrf('_', input_hello, "-o", "pretty")
199
+ assert_success(status, stderr, "pretty output")
200
+ assert_equal(
201
+ [
202
+ "{",
203
+ "\"hello\": 123",
204
+ "}",
205
+ "{",
206
+ "\"hello\": 456",
207
+ "}"
208
+ ],
209
+ lines(stdout),
210
+ "pretty output lines"
211
+ )
212
+
213
+ input_table_hash = '{"a":[1,2],"b":[3,4]}'
214
+ stdout, stderr, status = run_jrf('_', input_table_hash, "-o", "tsv")
215
+ assert_success(status, stderr, "tsv output hash of arrays")
216
+ assert_equal(["a\t1\t2", "b\t3\t4"], lines(stdout), "tsv output hash of arrays")
217
+
218
+ input_table_array = '[[1,"hello",true],[2,"world",false]]'
219
+ stdout, stderr, status = run_jrf('_', input_table_array, "-o", "tsv")
220
+ assert_success(status, stderr, "tsv output array of arrays")
221
+ assert_equal(["1\thello\ttrue", "2\tworld\tfalse"], lines(stdout), "tsv output array of arrays")
222
+
223
+ input_table_scalar = '{"foo":"bar","baz":42}'
224
+ stdout, stderr, status = run_jrf('_', input_table_scalar, "-o", "tsv")
225
+ assert_success(status, stderr, "tsv output hash of scalars")
226
+ assert_equal(["foo\tbar", "baz\t42"], lines(stdout), "tsv output hash of scalars")
227
+
228
+ input_table_nested = '{"a":[[1,2],[3,4]],"b":[[5,6],[7,8]]}'
229
+ stdout, stderr, status = run_jrf('_', input_table_nested, "-o", "tsv")
230
+ assert_success(status, stderr, "tsv output nested arrays as JSON")
231
+ assert_equal(["a\t[1,2]\t[3,4]", "b\t[5,6]\t[7,8]"], lines(stdout), "tsv output nested arrays as JSON")
232
+ end
233
+
234
+ def test_regex_and_parser_boundaries
235
+ input_regex = <<~NDJSON
236
+ {"foo":{"bar":"ok"},"x":50}
237
+ {"foo":{"bar":"ng"},"x":70}
238
+ NDJSON
239
+
240
+ stdout, stderr, status = run_jrf('select(/ok/.match(_["foo"]["bar"])) >> _["x"]', input_regex)
241
+ assert_success(status, stderr, "regex in select")
242
+ assert_equal(%w[50], lines(stdout), "regex filter output")
243
+
244
+ input_split = <<~NDJSON
245
+ {"x":1}
246
+ NDJSON
247
+
248
+ stdout, stderr, status = run_jrf('[1 >> 2] >> _', input_split)
249
+ assert_success(status, stderr, "no split inside []")
250
+ assert_equal(['[0]'], lines(stdout), "no split inside [] output")
251
+
252
+ stdout, stderr, status = run_jrf('{a: 1 >> 2} >> _[:a]', input_split)
253
+ assert_success(status, stderr, "no split inside {}")
254
+ assert_equal(%w[0], lines(stdout), "no split inside {} output")
255
+
256
+ stdout, stderr, status = run_jrf('(-> { 1 >> 2 }).call >> _ + 1', input_split)
257
+ assert_success(status, stderr, "no split inside block")
258
+ assert_equal(%w[1], lines(stdout), "no split inside block output")
259
+ end
260
+
261
+ def test_flat
262
+ input_flat = <<~NDJSON
263
+ {"items":[1,2]}
264
+ {"items":[3]}
265
+ {"items":[]}
266
+ NDJSON
267
+
268
+ stdout, stderr, status = run_jrf('_["items"] >> flat', input_flat)
269
+ assert_success(status, stderr, "flat basic")
270
+ assert_equal(%w[1 2 3], lines(stdout), "flat basic output")
271
+
272
+ input_flat_hash = <<~NDJSON
273
+ {"items":[{"x":1},{"x":2}]}
274
+ NDJSON
275
+
276
+ stdout, stderr, status = run_jrf('_["items"] >> flat >> _["x"]', input_flat_hash)
277
+ assert_success(status, stderr, "flat then extract")
278
+ assert_equal(%w[1 2], lines(stdout), "flat then extract output")
279
+
280
+ stdout, stderr, status = run_jrf('_["items"] >> flat >> sum(_)', input_flat)
281
+ assert_success(status, stderr, "flat then sum")
282
+ assert_equal(%w[6], lines(stdout), "flat then sum output")
283
+
284
+ stdout, stderr, status = run_jrf('_["items"] >> flat >> group', input_flat)
285
+ assert_success(status, stderr, "flat then group")
286
+ assert_equal(['[1,2,3]'], lines(stdout), "flat then group output")
287
+
288
+ stdout, stderr, status = run_jrf('map { |x| flat }', "[[1,2],[3],[4,5,6]]\n")
289
+ assert_success(status, stderr, "flat inside map")
290
+ assert_equal(['[1,2,3,4,5,6]'], lines(stdout), "flat inside map output")
291
+
292
+ stdout, stderr, status = run_jrf('map_values { |v| flat }', "{\"a\":[1,2],\"b\":[3]}\n")
293
+ assert_failure(status, "flat inside map_values")
294
+ assert_includes(stderr, "flat is not supported inside map_values")
295
+
296
+ stdout, stderr, status = run_jrf('_["foo"] >> flat', "{\"foo\":1}\n")
297
+ assert_failure(status, "flat requires array")
298
+ assert_includes(stderr, "flat expects Array")
299
+ end
300
+
301
+ def test_reducers
302
+ input = <<~NDJSON
303
+ {"foo":1,"x":5}
304
+ {"foo":2,"x":11}
305
+ {"foo":3,"x":50}
306
+ {"foo":4,"x":70}
307
+ NDJSON
308
+
309
+ stdout, stderr, status = run_jrf('sum(_["foo"])', input)
310
+ assert_success(status, stderr, "sum only")
311
+ assert_equal(%w[10], lines(stdout), "sum output")
312
+
313
+ stdout, stderr, status = run_jrf('count()', input)
314
+ assert_success(status, stderr, "count only")
315
+ assert_equal(%w[4], lines(stdout), "count output")
316
+
317
+ stdout, stderr, status = run_jrf('count(_["foo"])', input)
318
+ assert_success(status, stderr, "count(expr) only")
319
+ assert_equal(%w[4], lines(stdout), "count(expr) output")
320
+
321
+ stdout, stderr, status = run_jrf('min(_["foo"])', input)
322
+ assert_success(status, stderr, "min only")
323
+ assert_equal(%w[1], lines(stdout), "min output")
324
+
325
+ stdout, stderr, status = run_jrf('max(_["foo"])', input)
326
+ assert_success(status, stderr, "max only")
327
+ assert_equal(%w[4], lines(stdout), "max output")
328
+
329
+ stdout, stderr, status = run_jrf('select(_["x"] > 10) >> sum(_["foo"])', input)
330
+ assert_success(status, stderr, "select + sum")
331
+ assert_equal(%w[9], lines(stdout), "select + sum output")
332
+
333
+ stdout, stderr, status = run_jrf('{total: sum(_["foo"]), n: count()}', input)
334
+ assert_success(status, stderr, "structured reducer result")
335
+ assert_equal(['{"total":10,"n":4}'], lines(stdout), "structured reducer result output")
336
+
337
+ stdout, stderr, status = run_jrf('average(_["foo"])', input)
338
+ assert_success(status, stderr, "average")
339
+ assert_float_close(2.5, lines(stdout).first.to_f, 1e-12, "average output")
340
+
341
+ stdout, stderr, status = run_jrf('stdev(_["foo"])', input)
342
+ assert_success(status, stderr, "stdev")
343
+ assert_float_close(1.118033988749895, lines(stdout).first.to_f, 1e-12, "stdev output")
344
+
345
+ stdout, stderr, status = run_jrf('_["foo"] >> sum(_ * 2)', input)
346
+ assert_success(status, stderr, "extract + sum")
347
+ assert_equal(%w[20], lines(stdout), "extract + sum output")
348
+
349
+ stdout, stderr, status = run_jrf('sum(2 * _["foo"])', input)
350
+ assert_success(status, stderr, "sum with literal on left")
351
+ assert_equal(%w[20], lines(stdout), "sum with literal on left output")
352
+
353
+ stdout, stderr, status = run_jrf('select(_["x"] > 1000) >> sum(_["foo"])', input)
354
+ assert_success(status, stderr, "sum no matches")
355
+ assert_equal([], lines(stdout), "sum no matches output")
356
+
357
+ stdout, stderr, status = run_jrf('select(_["x"] > 1000) >> count()', input)
358
+ assert_success(status, stderr, "count no matches")
359
+ assert_equal([], lines(stdout), "count no matches output")
360
+
361
+ stdout, stderr, status = run_jrf('select(_["x"] > 1000) >> count(_["foo"])', input)
362
+ assert_success(status, stderr, "count(expr) no matches")
363
+ assert_equal([], lines(stdout), "count(expr) no matches output")
364
+
365
+ stdout, stderr, status = run_jrf('select(_["x"] > 1000) >> average(_["foo"])', input)
366
+ assert_success(status, stderr, "average no matches")
367
+ assert_equal([], lines(stdout), "average no matches output")
368
+
369
+ stdout, stderr, status = run_jrf('select(_["x"] > 1000) >> stdev(_["foo"])', input)
370
+ assert_success(status, stderr, "stdev no matches")
371
+ assert_equal([], lines(stdout), "stdev no matches output")
372
+
373
+ stdout, stderr, status = run_jrf('select(_["x"] > 1000) >> min(_["foo"])', input)
374
+ assert_success(status, stderr, "min no matches")
375
+ assert_equal([], lines(stdout), "min no matches output")
376
+
377
+ stdout, stderr, status = run_jrf('select(_["x"] > 1000) >> max(_["foo"])', input)
378
+ assert_success(status, stderr, "max no matches")
379
+ assert_equal([], lines(stdout), "max no matches output")
380
+
381
+ stdout, stderr, status = run_jrf('sum(_["foo"]) >> _ + 1', input)
382
+ assert_success(status, stderr, "reduce in middle")
383
+ assert_equal(%w[11], lines(stdout), "reduce in middle output")
384
+
385
+ stdout, stderr, status = run_jrf('select(_["x"] > 10) >> _["foo"] >> sum(_ * 2) >> select(_ > 10) >> _ + 1', input)
386
+ assert_success(status, stderr, "reduce mixed with select/extract")
387
+ assert_equal(%w[19], lines(stdout), "reduce mixed output")
388
+
389
+ stdout, stderr, status = run_jrf('_["foo"] >> sum(_) >> _ * 10 >> sum(_)', input)
390
+ assert_success(status, stderr, "multiple reducers")
391
+ assert_equal(%w[100], lines(stdout), "multiple reducers output")
392
+
393
+ stdout, stderr, status = run_jrf('_["foo"] >> min(_) >> _ * 10 >> max(_)', input)
394
+ assert_success(status, stderr, "min/max mixed reducers")
395
+ assert_equal(%w[10], lines(stdout), "min/max mixed reducers output")
396
+ end
397
+
398
+ def test_sort
399
+ input_sum = <<~NDJSON
400
+ {"foo":1,"x":5}
401
+ {"foo":2,"x":11}
402
+ {"foo":3,"x":50}
403
+ {"foo":4,"x":70}
404
+ NDJSON
405
+
406
+ input_sort_rows = <<~NDJSON
407
+ {"foo":"b","at":2}
408
+ {"foo":"c","at":3}
409
+ {"foo":"a","at":1}
410
+ NDJSON
411
+
412
+ stdout, stderr, status = run_jrf('sort(_["at"]) >> _["foo"]', input_sort_rows)
413
+ assert_success(status, stderr, "sort rows by field")
414
+ assert_equal(%w["a" "b" "c"], lines(stdout), "sort rows by field output")
415
+
416
+ stdout, stderr, status = run_jrf('sort { |a, b| b["at"] <=> a["at"] } >> _["foo"]', input_sort_rows)
417
+ assert_success(status, stderr, "sort rows by comparator")
418
+ assert_equal(%w["c" "b" "a"], lines(stdout), "sort rows by comparator output")
419
+
420
+ stdout, stderr, status = run_jrf('sort(_["at"]) >> _["foo"] >> group', input_sort_rows)
421
+ assert_success(status, stderr, "sort then group")
422
+ assert_equal(['["a","b","c"]'], lines(stdout), "sort then group output")
423
+
424
+ stdout, stderr, status = run_jrf('select(_["x"] > 1000) >> sort(_["x"]) >> _["foo"]', input_sum)
425
+ assert_success(status, stderr, "sort no matches")
426
+ assert_equal([], lines(stdout), "sort no matches output")
427
+
428
+ stdout, stderr, status = run_jrf('select(_["x"] > 1000) >> _["foo"] >> group', input_sum)
429
+ assert_success(status, stderr, "group no matches")
430
+ assert_equal([], lines(stdout), "group no matches output")
431
+ end
432
+
433
+ def test_group
434
+ input_group_multi = <<~NDJSON
435
+ {"x":1,"y":"a"}
436
+ {"x":2,"y":"b"}
437
+ {"x":3,"y":"c"}
438
+ NDJSON
439
+
440
+ stdout, stderr, status = run_jrf('{a: group(_["x"]), b: group(_["y"])}', input_group_multi)
441
+ assert_success(status, stderr, "group in hash")
442
+ assert_equal(['{"a":[1,2,3],"b":["a","b","c"]}'], lines(stdout), "group in hash output")
443
+
444
+ stdout, stderr, status = run_jrf('select(_["x"] > 1000) >> {a: group(_["x"]), b: group(_["y"])}', input_group_multi)
445
+ assert_success(status, stderr, "group in hash no matches")
446
+ assert_equal([], lines(stdout), "group in hash no-match output")
447
+ end
448
+
449
+ def test_percentile
450
+ input_sum = <<~NDJSON
451
+ {"foo":1,"x":5}
452
+ {"foo":2,"x":11}
453
+ {"foo":3,"x":50}
454
+ {"foo":4,"x":70}
455
+ NDJSON
456
+
457
+ stdout, stderr, status = run_jrf('percentile(_["foo"], 0.50)', input_sum)
458
+ assert_success(status, stderr, "single percentile")
459
+ assert_equal(%w[2], lines(stdout), "single percentile output")
460
+
461
+ stdout, stderr, status = run_jrf('percentile(_["foo"], [0.25, 0.50, 1.0])', input_sum)
462
+ assert_success(status, stderr, "array percentile")
463
+ assert_equal(['[1,2,4]'], lines(stdout), "array percentile output")
464
+
465
+ stdout, stderr, status = run_jrf('percentile(_["foo"], 0.25.step(1.0, 0.25))', input_sum)
466
+ assert_success(status, stderr, "enumerable percentile")
467
+ assert_equal(['[1,2,3,4]'], lines(stdout), "enumerable percentile output")
468
+ end
469
+
470
+ def test_nil_handling_for_aggregates
471
+ input_with_nil = <<~NDJSON
472
+ {"foo":1}
473
+ {"foo":null}
474
+ {"bar":999}
475
+ {"foo":3}
476
+ NDJSON
477
+
478
+ stdout, stderr, status = run_jrf('sum(_["foo"])', input_with_nil)
479
+ assert_success(status, stderr, "sum ignores nil")
480
+ assert_equal(%w[4], lines(stdout), "sum ignores nil output")
481
+
482
+ stdout, stderr, status = run_jrf('min(_["foo"])', input_with_nil)
483
+ assert_success(status, stderr, "min ignores nil")
484
+ assert_equal(%w[1], lines(stdout), "min ignores nil output")
485
+
486
+ stdout, stderr, status = run_jrf('max(_["foo"])', input_with_nil)
487
+ assert_success(status, stderr, "max ignores nil")
488
+ assert_equal(%w[3], lines(stdout), "max ignores nil output")
489
+
490
+ stdout, stderr, status = run_jrf('average(_["foo"])', input_with_nil)
491
+ assert_success(status, stderr, "average ignores nil")
492
+ assert_float_close(2.0, lines(stdout).first.to_f, 1e-12, "average ignores nil output")
493
+
494
+ stdout, stderr, status = run_jrf('stdev(_["foo"])', input_with_nil)
495
+ assert_success(status, stderr, "stdev ignores nil")
496
+ assert_float_close(1.0, lines(stdout).first.to_f, 1e-12, "stdev ignores nil output")
497
+
498
+ stdout, stderr, status = run_jrf('percentile(_["foo"], [0.5, 1.0])', input_with_nil)
499
+ assert_success(status, stderr, "percentile ignores nil")
500
+ assert_equal(['[1,3]'], lines(stdout), "percentile ignores nil output")
501
+
502
+ stdout, stderr, status = run_jrf('count()', input_with_nil)
503
+ assert_success(status, stderr, "count with nil rows")
504
+ assert_equal(%w[4], lines(stdout), "count with nil rows output")
505
+
506
+ stdout, stderr, status = run_jrf('count(_["foo"])', input_with_nil)
507
+ assert_success(status, stderr, "count(expr) ignores nil")
508
+ assert_equal(%w[2], lines(stdout), "count(expr) ignores nil output")
509
+
510
+ input_count_if = <<~NDJSON
511
+ {"x":1}
512
+ {"x":-2}
513
+ {"x":3}
514
+ {"x":-4}
515
+ {"x":5}
516
+ NDJSON
517
+
518
+ stdout, stderr, status = run_jrf('count_if(_["x"] > 0)', input_count_if)
519
+ assert_success(status, stderr, "count_if")
520
+ assert_equal(%w[3], lines(stdout), "count_if output")
521
+
522
+ stdout, stderr, status = run_jrf('[count_if(_["x"] > 0), count_if(_["x"] < 0)]', input_count_if)
523
+ assert_success(status, stderr, "count_if multiple")
524
+ assert_equal(["[3,2]"], lines(stdout), "count_if multiple output")
525
+
526
+ input_all_nil = <<~NDJSON
527
+ {"foo":null}
528
+ {"bar":1}
529
+ NDJSON
530
+
531
+ stdout, stderr, status = run_jrf('sum(_["foo"])', input_all_nil)
532
+ assert_success(status, stderr, "sum all nil")
533
+ assert_equal(%w[0], lines(stdout), "sum all nil output")
534
+
535
+ stdout, stderr, status = run_jrf('min(_["foo"])', input_all_nil)
536
+ assert_success(status, stderr, "min all nil")
537
+ assert_equal(%w[null], lines(stdout), "min all nil output")
538
+
539
+ stdout, stderr, status = run_jrf('max(_["foo"])', input_all_nil)
540
+ assert_success(status, stderr, "max all nil")
541
+ assert_equal(%w[null], lines(stdout), "max all nil output")
542
+
543
+ stdout, stderr, status = run_jrf('average(_["foo"])', input_all_nil)
544
+ assert_success(status, stderr, "average all nil")
545
+ assert_equal(%w[null], lines(stdout), "average all nil output")
546
+
547
+ stdout, stderr, status = run_jrf('stdev(_["foo"])', input_all_nil)
548
+ assert_success(status, stderr, "stdev all nil")
549
+ assert_equal(%w[null], lines(stdout), "stdev all nil output")
550
+
551
+ stdout, stderr, status = run_jrf('percentile(_["foo"], 0.5)', input_all_nil)
552
+ assert_success(status, stderr, "percentile all nil")
553
+ assert_equal(%w[null], lines(stdout), "percentile all nil output")
554
+
555
+ stdout, stderr, status = run_jrf('count(_["foo"])', input_all_nil)
556
+ assert_success(status, stderr, "count(expr) all nil")
557
+ assert_equal(%w[0], lines(stdout), "count(expr) all nil output")
558
+ end
559
+
560
+ def test_reduce
561
+ input_multi_cols = <<~NDJSON
562
+ {"a":1,"b":10}
563
+ {"a":2,"b":20}
564
+ {"a":3,"b":30}
565
+ {"a":4,"b":40}
566
+ NDJSON
567
+
568
+ stdout, stderr, status = run_jrf('{a: percentile(_["a"], [0.25, 0.50, 1.0]), b: percentile(_["b"], [0.25, 0.50, 1.0])}', input_multi_cols)
569
+ assert_success(status, stderr, "nested array percentile for multiple columns")
570
+ assert_equal(['{"a":[1,2,4],"b":[10,20,40]}'], lines(stdout), "nested array percentile output")
571
+
572
+ input_reduce = <<~NDJSON
573
+ {"s":"hello"}
574
+ {"s":"world"}
575
+ {"s":"jrf"}
576
+ NDJSON
577
+
578
+ stdout, stderr, status = run_jrf('_["s"] >> reduce("") { |acc, v| acc.empty? ? v : "#{acc} #{v}" }', input_reduce)
579
+ assert_success(status, stderr, "reduce with implicit value")
580
+ assert_equal(['"hello world jrf"'], lines(stdout), "reduce implicit value output")
581
+
582
+ stdout, stderr, status = run_jrf('_["s"] >> reduce("") { |acc, v| acc.empty? ? v : "#{acc} #{v}" }', input_reduce)
583
+ assert_success(status, stderr, "reduce in two-stage form")
584
+ assert_equal(['"hello world jrf"'], lines(stdout), "reduce in two-stage form output")
585
+
586
+ input_sum = <<~NDJSON
587
+ {"foo":1,"x":5}
588
+ {"foo":2,"x":11}
589
+ {"foo":3,"x":50}
590
+ {"foo":4,"x":70}
591
+ NDJSON
592
+
593
+ stdout, stderr, status = run_jrf('sum(_["foo"]) >> select(_ > 100)', input_sum)
594
+ assert_success(status, stderr, "post-reduce select drop")
595
+ assert_equal([], lines(stdout), "post-reduce select drop output")
596
+ end
597
+
598
+ def test_lax_input_mode
599
+ input_whitespace_stream = "{\"foo\":1} {\"foo\":2}\n\t{\"foo\":3}\n"
600
+ stdout, stderr, status = run_jrf('_["foo"]', input_whitespace_stream)
601
+ assert_failure(status, "default NDJSON should reject same-line multi-values")
602
+ assert_includes(stderr, "JSON::ParserError")
603
+
604
+ stdout, stderr, status = run_jrf('_["foo"]', input_whitespace_stream, "--lax")
605
+ assert_success(status, stderr, "whitespace-separated JSON stream with --lax")
606
+ assert_equal(%w[1 2 3], lines(stdout), "whitespace-separated stream output")
607
+
608
+ input_json_seq = "\x1e{\"foo\":10}\n\x1e{\"foo\":20}\n"
609
+ stdout, stderr, status = run_jrf('_["foo"]', input_json_seq)
610
+ assert_failure(status, "RS framing requires --lax")
611
+ assert_includes(stderr, "JSON::ParserError")
612
+
613
+ stdout, stderr, status = run_jrf('_["foo"]', input_json_seq, "--lax")
614
+ assert_success(status, stderr, "json-seq style RS framing with --lax")
615
+ assert_equal(%w[10 20], lines(stdout), "json-seq style output")
616
+
617
+ input_lax_multiline = <<~JSONS
618
+ {
619
+ "foo": 101,
620
+ "bar": {"x": 1}
621
+ }
622
+ {
623
+ "foo": 202,
624
+ "bar": {"x": 2}
625
+ }
626
+ JSONS
627
+ stdout, stderr, status = run_jrf('_["foo"]', input_lax_multiline)
628
+ assert_failure(status, "default NDJSON rejects multiline objects")
629
+ assert_includes(stderr, "JSON::ParserError")
630
+
631
+ stdout, stderr, status = run_jrf('_["bar"]["x"]', input_lax_multiline, "--lax")
632
+ assert_success(status, stderr, "lax accepts multiline objects")
633
+ assert_equal(%w[1 2], lines(stdout), "lax multiline object output")
634
+
635
+ input_lax_mixed_separators = "{\"foo\":1}\n\x1e{\"foo\":2}\t{\"foo\":3}\n"
636
+ stdout, stderr, status = run_jrf('_["foo"]', input_lax_mixed_separators, "--lax")
637
+ assert_success(status, stderr, "lax accepts mixed whitespace and RS separators")
638
+ assert_equal(%w[1 2 3], lines(stdout), "lax mixed separators output")
639
+
640
+ input_lax_with_escaped_newline = "{\"s\":\"line1\\nline2\"}\n{\"s\":\"ok\"}\n"
641
+ stdout, stderr, status = run_jrf('_["s"]', input_lax_with_escaped_newline, "--lax")
642
+ assert_success(status, stderr, "lax handles escaped newlines in strings")
643
+ assert_equal(['"line1\nline2"', '"ok"'], lines(stdout), "lax escaped newline string output")
644
+
645
+ input_lax_trailing_rs = "\x1e{\"foo\":9}\n\x1e"
646
+ stdout, stderr, status = run_jrf('_["foo"]', input_lax_trailing_rs, "--lax")
647
+ assert_success(status, stderr, "lax ignores trailing separator")
648
+ assert_equal(%w[9], lines(stdout), "lax trailing separator output")
649
+
650
+ chunked_lax_out = RecordingRunner.new(
651
+ inputs: [ChunkedSource.new("{\"foo\":1}\n\x1e{\"foo\":2}\n\t{\"foo\":3}\n")],
652
+ out: StringIO.new,
653
+ err: StringIO.new,
654
+ lax: true
655
+ )
656
+ chunked_lax_out.run('_["foo"]')
657
+ assert_equal(%w[1 2 3], lines(chunked_lax_out.writes.join), "lax mode streams chunked input without whole-input reads")
658
+
659
+ Dir.mktmpdir do |dir|
660
+ one = File.join(dir, "one.json")
661
+ two = File.join(dir, "two.json")
662
+ File.write(one, "1")
663
+ File.write(two, "2")
664
+
665
+ stdout, stderr, status = Open3.capture3("./exe/jrf", "--lax", "_", one, two)
666
+ assert_success(status, stderr, "lax keeps file boundaries")
667
+ assert_equal(%w[1 2], lines(stdout), "lax does not merge JSON across file boundaries")
668
+ end
669
+ end
670
+
671
+ def test_parse_errors
672
+ stdout, stderr, status = run_jrf('select(_["x"] > ) >> _["foo"]', "")
673
+ assert_failure(status, "syntax error should fail before row loop")
674
+ assert_includes(stderr, "syntax error")
675
+
676
+ stdout, stderr, status = run_jrf('([)] >> _', "")
677
+ assert_failure(status, "mismatched delimiter should fail")
678
+ assert_includes(stderr, "mismatched delimiter")
679
+
680
+ stdout, stderr, status = run_jrf('(_["x"] >> _["y"]', "")
681
+ assert_failure(status, "unclosed delimiter should fail")
682
+ assert_includes(stderr, "unclosed delimiter")
683
+
684
+ input_broken_tail = <<~NDJSON
685
+ {"foo":1}
686
+ {"foo":2}
687
+ {"foo":
688
+ NDJSON
689
+
690
+ stdout, stderr, status = run_jrf('sum(_["foo"])', input_broken_tail)
691
+ assert_failure(status, "broken input should fail")
692
+ assert_equal(%w[3], lines(stdout), "reducers flush before parse error")
693
+ assert_includes(stderr, "JSON::ParserError")
694
+ end
695
+
696
+ def test_map
697
+ input_chain = <<~NDJSON
698
+ {"foo":{"bar":{"z":1},"keep":true}}
699
+ {"foo":{"bar":{"z":2},"keep":false}}
700
+ {"foo":{"bar":{"z":3},"keep":true}}
701
+ NDJSON
702
+
703
+ stdout, stderr, status = run_jrf('_["foo"] >> select(_["keep"]) >> _["bar"] >> select(_["z"] > 1) >> _["z"]', input_chain)
704
+ assert_success(status, stderr, "select/extract chain")
705
+ assert_equal(%w[3], lines(stdout), "chain output")
706
+
707
+ input_map = <<~NDJSON
708
+ {"values":[1,10,100]}
709
+ {"values":[2,20,200]}
710
+ {"values":[3,30,300]}
711
+ NDJSON
712
+
713
+ stdout, stderr, status = run_jrf('_["values"] >> map { |x| sum(x) }', input_map)
714
+ assert_success(status, stderr, "map with sum")
715
+ assert_equal(['[6,60,600]'], lines(stdout), "map with sum output")
716
+
717
+ stdout, stderr, status = run_jrf('_["values"] >> map { |x| min(x) }', input_map)
718
+ assert_success(status, stderr, "map with min")
719
+ assert_equal(['[1,10,100]'], lines(stdout), "map with min output")
720
+
721
+ stdout, stderr, status = run_jrf('_["values"] >> map { |x| max(x) }', input_map)
722
+ assert_success(status, stderr, "map with max")
723
+ assert_equal(['[3,30,300]'], lines(stdout), "map with max output")
724
+
725
+ stdout, stderr, status = run_jrf('_["values"] >> map { |x| sum(_[0] + x) }', input_map)
726
+ assert_success(status, stderr, "map keeps ambient _")
727
+ assert_equal(['[12,66,606]'], lines(stdout), "map ambient _ output")
728
+
729
+ stdout, stderr, status = run_jrf('_["values"] >> map { |x| reduce(0) { |acc, v| acc + v } }', input_map)
730
+ assert_success(status, stderr, "map with reduce")
731
+ assert_equal(['[6,60,600]'], lines(stdout), "map with reduce output")
732
+
733
+ input_map_varying = <<~NDJSON
734
+ [1,10]
735
+ [2,20,200]
736
+ [3]
737
+ NDJSON
738
+
739
+ stdout, stderr, status = run_jrf('map { |x| sum(x) }', input_map_varying)
740
+ assert_success(status, stderr, "map varying lengths")
741
+ assert_equal(['[6,30,200]'], lines(stdout), "map varying lengths output")
742
+
743
+ input_map_unsorted = <<~NDJSON
744
+ {"values":[3,30]}
745
+ {"values":[1,10]}
746
+ {"values":[2,20]}
747
+ NDJSON
748
+
749
+ stdout, stderr, status = run_jrf('_["values"] >> map { |x| group }', input_map)
750
+ assert_success(status, stderr, "map with group")
751
+ assert_equal(['[[1,2,3],[10,20,30],[100,200,300]]'], lines(stdout), "map with group output")
752
+
753
+ stdout, stderr, status = run_jrf('_["values"] >> map { |x| sort }', input_map_unsorted)
754
+ assert_success(status, stderr, "map with sort default key")
755
+ assert_equal(['[[1,2,3],[10,20,30]]'], lines(stdout), "map with sort default key output")
756
+
757
+ stdout, stderr, status = run_jrf('select(false) >> map { |x| sum(x) }', input_map)
758
+ assert_success(status, stderr, "map no matches")
759
+ assert_equal([], lines(stdout), "map no matches output")
760
+
761
+ stdout, stderr, status = run_jrf('_["values"] >> map { |x| x + 1 }', input_map)
762
+ assert_success(status, stderr, "map transform")
763
+ assert_equal(['[2,11,101]', '[3,21,201]', '[4,31,301]'], lines(stdout), "map transform output")
764
+
765
+ stdout, stderr, status = run_jrf('_["values"] >> map { |x| select(x >= 20) }', input_map)
766
+ assert_success(status, stderr, "map transform with select")
767
+ assert_equal(['[100]', '[20,200]', '[30,300]'], lines(stdout), "map transform with select output")
768
+ end
769
+
770
+ def test_map_values
771
+ input_map_values = <<~NDJSON
772
+ {"a":1,"b":10}
773
+ {"a":2,"b":20}
774
+ {"a":3,"b":30}
775
+ NDJSON
776
+
777
+ stdout, stderr, status = run_jrf('map_values { |v| sum(v) }', input_map_values)
778
+ assert_success(status, stderr, "map_values with sum")
779
+ assert_equal(['{"a":6,"b":60}'], lines(stdout), "map_values with sum output")
780
+
781
+ stdout, stderr, status = run_jrf('map_values { |v| min(v) }', input_map_values)
782
+ assert_success(status, stderr, "map_values with min")
783
+ assert_equal(['{"a":1,"b":10}'], lines(stdout), "map_values with min output")
784
+
785
+ input_map_values_varying = <<~NDJSON
786
+ {"a":1}
787
+ {"a":2,"b":20}
788
+ {"a":3,"b":30}
789
+ NDJSON
790
+
791
+ stdout, stderr, status = run_jrf('map_values { |v| sum(v) }', input_map_values_varying)
792
+ assert_success(status, stderr, "map_values varying keys")
793
+ assert_equal(['{"a":6,"b":50}'], lines(stdout), "map_values varying keys output")
794
+
795
+ stdout, stderr, status = run_jrf('map_values { |v| count(v) }', input_map_values)
796
+ assert_success(status, stderr, "map_values with count")
797
+ assert_equal(['{"a":3,"b":3}'], lines(stdout), "map_values with count output")
798
+
799
+ stdout, stderr, status = run_jrf('map_values { |v| group }', input_map_values)
800
+ assert_success(status, stderr, "map_values with group")
801
+ assert_equal(['{"a":[1,2,3],"b":[10,20,30]}'], lines(stdout), "map_values with group output")
802
+
803
+ stdout, stderr, status = run_jrf('map_values { |v| sum(_["a"] + v) }', input_map_values)
804
+ assert_success(status, stderr, "map_values keeps ambient _")
805
+ assert_equal(['{"a":12,"b":66}'], lines(stdout), "map_values ambient _ output")
806
+
807
+ stdout, stderr, status = run_jrf('map_values { |v| reduce(0) { |acc, x| acc + x } }', input_map_values)
808
+ assert_success(status, stderr, "map_values with reduce")
809
+ assert_equal(['{"a":6,"b":60}'], lines(stdout), "map_values with reduce output")
810
+
811
+ stdout, stderr, status = run_jrf('map { |k, v| "#{k}:#{v}" }', input_map_values)
812
+ assert_success(status, stderr, "map over hash transform")
813
+ assert_equal(['["a:1","b:10"]', '["a:2","b:20"]', '["a:3","b:30"]'], lines(stdout), "map over hash transform output")
814
+
815
+ stdout, stderr, status = run_jrf('map { |pair| pair }', input_map_values)
816
+ assert_success(status, stderr, "map over hash single block arg")
817
+ assert_equal(['[["a",1],["b",10]]', '[["a",2],["b",20]]', '[["a",3],["b",30]]'], lines(stdout), "map over hash single block arg output")
818
+
819
+ stdout, stderr, status = run_jrf('map { |k, v| select(v >= 10 && k != "a") }', input_map_values)
820
+ assert_success(status, stderr, "map over hash transform with select")
821
+ assert_equal(['[10]', '[20]', '[30]'], lines(stdout), "map over hash transform with select output")
822
+
823
+ stdout, stderr, status = run_jrf('map { |k, v| sum(v + k.length) }', input_map_values)
824
+ assert_success(status, stderr, "map over hash with sum")
825
+ assert_equal(['[9,63]'], lines(stdout), "map over hash with sum output")
826
+
827
+ stdout, stderr, status = run_jrf('map { |k, v| sum(_["a"] + v + k.length) }', input_map_values)
828
+ assert_success(status, stderr, "map over hash keeps ambient _")
829
+ assert_equal(['[15,69]'], lines(stdout), "map over hash ambient _ output")
830
+
831
+ stdout, stderr, status = run_jrf('select(false) >> map_values { |v| sum(v) }', input_map_values)
832
+ assert_success(status, stderr, "map_values no matches")
833
+ assert_equal([], lines(stdout), "map_values no matches output")
834
+
835
+ stdout, stderr, status = run_jrf('map_values { |v| sum(v) } >> map_values { |v| v * 10 }', input_map_values)
836
+ assert_success(status, stderr, "map_values piped to map_values passthrough")
837
+ assert_equal(['{"a":60,"b":600}'], lines(stdout), "map_values piped output")
838
+
839
+ stdout, stderr, status = run_jrf('map_values { |v| v * 2 }', input_map_values)
840
+ assert_success(status, stderr, "map_values transform")
841
+ assert_equal(['{"a":2,"b":20}', '{"a":4,"b":40}', '{"a":6,"b":60}'], lines(stdout), "map_values transform output")
842
+
843
+ stdout, stderr, status = run_jrf('map_values { |v| select(v >= 10) }', input_map_values)
844
+ assert_success(status, stderr, "map_values transform with select")
845
+ assert_equal(['{"b":10}', '{"b":20}', '{"b":30}'], lines(stdout), "map_values transform with select output")
846
+ end
847
+
848
+ def test_apply
849
+ input_map = <<~NDJSON
850
+ {"values":[1,10,100]}
851
+ {"values":[2,20,200]}
852
+ {"values":[3,30,300]}
853
+ NDJSON
854
+
855
+ stdout, stderr, status = run_jrf('_["values"] >> map { |x| x + 1 } >> map { |x| x * 10 }', input_map)
856
+ assert_success(status, stderr, "chained map transforms")
857
+ assert_equal(['[20,110,1010]', '[30,210,2010]', '[40,310,3010]'], lines(stdout), "chained map transforms output")
858
+
859
+ stdout, stderr, status = run_jrf('map { map { |y| [ sum(y[0]), sum(y[1]) ] } }', "[[[1,2]]]\n[[[3,4]]]\n")
860
+ assert_success(status, stderr, "nested map reducer binds to current target")
861
+ assert_equal(['[[[4,6]]]'], lines(stdout), "nested map reducer output")
862
+
863
+ stdout, stderr, status = run_jrf('map_values { |obj| map_values { |v| sum(v) } }', "{\"a\":{\"x\":1,\"y\":2},\"b\":{\"x\":10,\"y\":20}}\n{\"a\":{\"x\":3,\"y\":4},\"b\":{\"x\":30,\"y\":40}}\n")
864
+ assert_success(status, stderr, "nested map_values reducer binds to current target")
865
+ assert_equal(['{"a":{"x":4,"y":6},"b":{"x":40,"y":60}}'], lines(stdout), "nested map_values reducer output")
866
+
867
+ stdout, stderr, status = run_jrf('[apply { |x| sum(x["foo"]) }, _.length]', '[{"foo":1},{"foo":2}]' + "\n" + '[{"foo":10}]' + "\n")
868
+ assert_success(status, stderr, "apply with sum")
869
+ assert_equal(["[3,2]", "[10,1]"], lines(stdout), "apply with sum output")
870
+
871
+ stdout, stderr, status = run_jrf('apply { |x| x["foo"] }', '[{"foo":1},{"foo":2}]' + "\n")
872
+ assert_success(status, stderr, "apply passthrough")
873
+ assert_equal(["[1,2]"], lines(stdout), "apply passthrough output")
874
+
875
+ stdout, stderr, status = run_jrf('apply { |x| percentile(x, 0.5) }', '[10,20,30]' + "\n")
876
+ assert_success(status, stderr, "apply with percentile")
877
+ assert_equal(["20"], lines(stdout), "apply with percentile output")
878
+
879
+ stdout, stderr, status = run_jrf('map { |o| [apply(o["vals"]) { |x| sum(x) }, o["name"]] }', '[{"name":"a","vals":[1,2]},{"name":"b","vals":[10,20]}]' + "\n")
880
+ assert_success(status, stderr, "apply with explicit collection")
881
+ assert_equal(['[[3,"a"],[30,"b"]]'], lines(stdout), "apply with explicit collection output")
882
+
883
+ stdout, stderr, status = run_jrf('map(_["items"]) { |x| x * 2 }', '{"items":[1,2,3]}' + "\n")
884
+ assert_success(status, stderr, "map with explicit collection")
885
+ assert_equal(["[2,4,6]"], lines(stdout), "map with explicit collection output")
886
+
887
+ stdout, stderr, status = run_jrf('map_values(_["data"]) { |v| v * 10 }', '{"data":{"a":1,"b":2}}' + "\n")
888
+ assert_success(status, stderr, "map_values with explicit collection")
889
+ assert_equal(['{"a":10,"b":20}'], lines(stdout), "map_values with explicit collection output")
890
+ end
891
+
892
+ def test_group_by
893
+ input_gb = <<~NDJSON
894
+ {"status":200,"path":"/a","latency":10}
895
+ {"status":404,"path":"/b","latency":50}
896
+ {"status":200,"path":"/c","latency":30}
897
+ {"status":200,"path":"/d","latency":20}
898
+ NDJSON
899
+
900
+ stdout, stderr, status = run_jrf('group_by(_["status"]) { count() }', input_gb)
901
+ assert_success(status, stderr, "group_by with count")
902
+ assert_equal(['{"200":3,"404":1}'], lines(stdout), "group_by with count output")
903
+
904
+ stdout, stderr, status = run_jrf('group_by(_["status"]) { |row| sum(row["latency"]) }', input_gb)
905
+ assert_success(status, stderr, "group_by with sum")
906
+ assert_equal(['{"200":60,"404":50}'], lines(stdout), "group_by with sum output")
907
+
908
+ stdout, stderr, status = run_jrf('group_by(_["status"]) { |row| average(row["latency"]) }', input_gb)
909
+ assert_success(status, stderr, "group_by with average")
910
+ result = JSON.parse(lines(stdout).first)
911
+ assert_float_close(20.0, result["200"], 1e-12, "group_by average 200")
912
+ assert_float_close(50.0, result["404"], 1e-12, "group_by average 404")
913
+
914
+ stdout, stderr, status = run_jrf('group_by(_["status"])', input_gb)
915
+ assert_success(status, stderr, "group_by default (collect rows)")
916
+ result = JSON.parse(lines(stdout).first)
917
+ assert_equal(3, result["200"].length, "group_by default 200 count")
918
+ assert_equal(1, result["404"].length, "group_by default 404 count")
919
+ assert_equal("/a", result["200"][0]["path"], "group_by default first row")
920
+
921
+ stdout, stderr, status = run_jrf('group_by(_["status"]) { |row| group(row["path"]) }', input_gb)
922
+ assert_success(status, stderr, "group_by with group(expr)")
923
+ assert_equal(['{"200":["/a","/c","/d"],"404":["/b"]}'], lines(stdout), "group_by with group(expr) output")
924
+
925
+ stdout, stderr, status = run_jrf('group_by(_["status"]) { group }', input_gb)
926
+ assert_success(status, stderr, "group_by with implicit group")
927
+ result = JSON.parse(lines(stdout).first)
928
+ assert_equal(3, result["200"].length, "group_by implicit group 200 count")
929
+ assert_equal("/a", result["200"][0]["path"], "group_by implicit group first row")
930
+
931
+ stdout, stderr, status = run_jrf('group_by(_["status"]) { |row| min(row["latency"]) }', input_gb)
932
+ assert_success(status, stderr, "group_by with min")
933
+ assert_equal(['{"200":10,"404":50}'], lines(stdout), "group_by with min output")
934
+
935
+ stdout, stderr, status = run_jrf('group_by(_["status"]) { |row| {total: sum(row["latency"]), n: count()} }', input_gb)
936
+ assert_success(status, stderr, "group_by with multi-reducer")
937
+ assert_equal(['{"200":{"total":60,"n":3},"404":{"total":50,"n":1}}'], lines(stdout), "group_by multi-reducer output")
938
+
939
+ stdout, stderr, status = run_jrf('group_by(_["status"]) { reduce(0) { |acc, row| acc + row["latency"] } }', input_gb)
940
+ assert_success(status, stderr, "group_by with reduce")
941
+ assert_equal(['{"200":60,"404":50}'], lines(stdout), "group_by with reduce output")
942
+
943
+ stdout, stderr, status = run_jrf('select(false) >> group_by(_["status"]) { count() }', input_gb)
944
+ assert_success(status, stderr, "group_by no matches")
945
+ assert_equal([], lines(stdout), "group_by no matches output")
946
+
947
+ stdout, stderr, status = run_jrf('group_by(_["status"]) { count() } >> _[200]', input_gb)
948
+ assert_success(status, stderr, "group_by then extract")
949
+ assert_equal(%w[3], lines(stdout), "group_by then extract output")
950
+ end
951
+ end