rbjq 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,584 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rbjq
4
+ # Recursive-descent parser producing an array-based AST.
5
+ #
6
+ # Precedence (lowest to highest):
7
+ # def / as / label < | < , < // < = |= += ... < or < and
8
+ # < == != < <= > >= < + - < * / % < unary - < postfix (?, [], .foo)
9
+ class Parser
10
+ ASSIGN_OPS = %i[= |= += -= *= /= %= //=].freeze # rubocop:disable Lint/NestedPercentLiteral
11
+ COMPARE_OPS = %i[== != < <= > >=].freeze
12
+
13
+ def self.parse(src)
14
+ src = src.dup.force_encoding(Encoding::UTF_8) unless src.encoding == Encoding::UTF_8
15
+ src = src.scrub("\uFFFD") unless src.valid_encoding?
16
+ tokens = Lexer.new(src).tokenize
17
+ parser = new(tokens)
18
+ parser.parse_program
19
+ end
20
+
21
+ def initialize(tokens)
22
+ @tokens = tokens
23
+ @pos = 0
24
+ end
25
+
26
+ def parse_program
27
+ skip_module_directives
28
+ ast = if peek.type == :eof
29
+ [:identity]
30
+ else
31
+ parse_pipe
32
+ end
33
+ expect(:eof)
34
+ ast
35
+ end
36
+
37
+ # Parses only function definitions (used for library / prelude code).
38
+ def parse_defs
39
+ defs = []
40
+ defs << parse_funcdef while peek.type == :def
41
+ expect(:eof)
42
+ defs
43
+ end
44
+
45
+ private
46
+
47
+ def peek(offset = 0)
48
+ @tokens[@pos + offset]
49
+ end
50
+
51
+ def advance
52
+ t = @tokens[@pos]
53
+ @pos += 1
54
+ t
55
+ end
56
+
57
+ def accept(type)
58
+ return nil unless peek.type == type
59
+
60
+ advance
61
+ end
62
+
63
+ def expect(type)
64
+ t = peek
65
+ raise CompileError, "syntax error, unexpected #{t} (expecting #{type}) at line #{t.line}" unless t.type == type
66
+
67
+ advance
68
+ end
69
+
70
+ def unexpected!
71
+ t = peek
72
+ raise CompileError, "syntax error, unexpected #{t} at line #{t.line}"
73
+ end
74
+
75
+ def skip_module_directives
76
+ raise CompileError, 'import/include is not supported' if %i[import include].include?(peek.type)
77
+ return unless peek.type == :ident && peek.value == 'module'
78
+
79
+ raise CompileError, 'module directive is not supported'
80
+ end
81
+
82
+ def parse_pipe
83
+ if peek.type == :def
84
+ fd = parse_funcdef
85
+ body = parse_pipe
86
+ return [:func_def, fd[1], fd[2], fd[3], body]
87
+ end
88
+
89
+ lhs = parse_comma
90
+ if accept(:|)
91
+ [:pipe, lhs, parse_pipe]
92
+ else
93
+ lhs
94
+ end
95
+ end
96
+
97
+ def parse_funcdef
98
+ expect(:def)
99
+ name_tok = advance
100
+ unless name_tok.type == :ident || Lexer::KEYWORDS.include?(name_tok.value.to_s)
101
+ raise CompileError, "syntax error, unexpected #{name_tok} in def at line #{name_tok.line}"
102
+ end
103
+
104
+ params = []
105
+ if accept(:'(')
106
+ loop do
107
+ t = advance
108
+ case t.type
109
+ when :var then params << "$#{t.value}"
110
+ when :ident then params << t.value
111
+ else
112
+ raise CompileError, "syntax error, unexpected #{t} in parameter list at line #{t.line}"
113
+ end
114
+ break unless accept(:';')
115
+ end
116
+ expect(:')')
117
+ end
118
+ expect(:':')
119
+ body = parse_pipe
120
+ expect(:';')
121
+ [:def, name_tok.value, params, body]
122
+ end
123
+
124
+ def parse_comma
125
+ lhs = parse_alt_as
126
+ lhs = [:comma, lhs, parse_alt_as] while accept(:',')
127
+ lhs
128
+ end
129
+
130
+ # jq 1.8: the source of "as" is everything to its left down to "," or "|",
131
+ # e.g. `1 + 2 as $x | ...` binds 3. The body extends to the end of the pipe.
132
+ def parse_alt_as
133
+ src = parse_alt
134
+ return src unless accept(:as)
135
+
136
+ patterns = parse_patterns
137
+ expect(:|)
138
+ [:as, src, patterns, parse_pipe]
139
+ end
140
+
141
+ def parse_alt
142
+ lhs = parse_assign
143
+ if accept(:'//')
144
+ [:alt, lhs, parse_alt]
145
+ else
146
+ lhs
147
+ end
148
+ end
149
+
150
+ def parse_assign
151
+ lhs = parse_or
152
+ if ASSIGN_OPS.include?(peek.type)
153
+ op = advance.type
154
+ rhs = parse_alt
155
+ [:assign, op, lhs, rhs]
156
+ else
157
+ lhs
158
+ end
159
+ end
160
+
161
+ def parse_or
162
+ lhs = parse_and
163
+ lhs = [:or, lhs, parse_and] while accept(:or)
164
+ lhs
165
+ end
166
+
167
+ def parse_and
168
+ lhs = parse_compare
169
+ lhs = [:and, lhs, parse_compare] while accept(:and)
170
+ lhs
171
+ end
172
+
173
+ def parse_compare
174
+ lhs = parse_additive
175
+ if COMPARE_OPS.include?(peek.type)
176
+ op = advance.type
177
+ rhs = parse_additive
178
+ lhs = [:binop, op, lhs, rhs]
179
+ raise CompileError, "syntax error, unexpected #{peek} at line #{peek.line}" if COMPARE_OPS.include?(peek.type)
180
+ end
181
+ lhs
182
+ end
183
+
184
+ def parse_additive
185
+ lhs = parse_multiplicative
186
+ while %i[+ -].include?(peek.type)
187
+ op = advance.type
188
+ lhs = [:binop, op, lhs, parse_multiplicative]
189
+ end
190
+ lhs
191
+ end
192
+
193
+ def parse_multiplicative
194
+ lhs = parse_unary
195
+ while %i[* / %].include?(peek.type)
196
+ op = advance.type
197
+ lhs = [:binop, op, lhs, parse_unary]
198
+ end
199
+ lhs
200
+ end
201
+
202
+ def parse_unary
203
+ if accept(:-)
204
+ [:neg, parse_multiplicative_operand]
205
+ else
206
+ parse_postfix
207
+ end
208
+ end
209
+
210
+ def parse_multiplicative_operand
211
+ lhs = parse_unary
212
+ while %i[* / %].include?(peek.type)
213
+ op = advance.type
214
+ lhs = [:binop, op, lhs, parse_unary]
215
+ end
216
+ lhs
217
+ end
218
+
219
+ # Postfix expressions: Term followed by .foo, [..], ?, and optional "as" binding.
220
+ def parse_postfix
221
+ term = parse_primary
222
+ loop do
223
+ t = peek
224
+ case t.type
225
+ when :field
226
+ advance
227
+ term = [:index, term, [:literal, t.value]]
228
+ when :'.'
229
+ if peek(1).type == :str
230
+ advance
231
+ str = parse_string(advance, nil)
232
+ term = [:index, term, str]
233
+ elsif peek(1).type == :'['
234
+ advance
235
+ term = parse_bracket_suffix(term)
236
+ else
237
+ break
238
+ end
239
+ when :'['
240
+ term = parse_bracket_suffix(term)
241
+ when :'?'
242
+ advance
243
+ term = [:try, term, nil]
244
+ else
245
+ break
246
+ end
247
+ end
248
+ term
249
+ end
250
+
251
+ def parse_bracket_suffix(term)
252
+ expect(:'[')
253
+ return [:iterate, term] if accept(:']')
254
+
255
+ if accept(:':')
256
+ to = parse_pipe
257
+ expect(:']')
258
+ return [:slice, term, nil, to]
259
+ end
260
+
261
+ idx = parse_pipe
262
+ if accept(:':')
263
+ return [:slice, term, idx, nil] if accept(:']')
264
+
265
+ to = parse_pipe
266
+ expect(:']')
267
+ return [:slice, term, idx, to]
268
+ end
269
+ expect(:']')
270
+ [:index, term, idx]
271
+ end
272
+
273
+ def parse_primary
274
+ t = peek
275
+ case t.type
276
+ when :num
277
+ advance
278
+ [:literal, t.value]
279
+ when :str
280
+ advance
281
+ parse_string(t, nil)
282
+ when :format
283
+ advance
284
+ if peek.type == :str
285
+ parse_string(advance, t.value)
286
+ else
287
+ [:format, t.value]
288
+ end
289
+ when :'.'
290
+ advance
291
+ if peek.type == :str
292
+ [:index, [:identity], parse_string(advance, nil)]
293
+ else
294
+ [:identity]
295
+ end
296
+ when :'..'
297
+ advance
298
+ [:call, 'recurse', []]
299
+ when :field
300
+ advance
301
+ [:index, [:identity], [:literal, t.value]]
302
+ when :var
303
+ advance
304
+ [:var, t.value]
305
+ when :loc
306
+ advance
307
+ [:loc, t.value]
308
+ when :'('
309
+ advance
310
+ e = parse_pipe
311
+ expect(:')')
312
+ e
313
+ when :'['
314
+ advance
315
+ if accept(:']')
316
+ [:array, nil]
317
+ else
318
+ e = parse_pipe
319
+ expect(:']')
320
+ [:array, e]
321
+ end
322
+ when :'{'
323
+ parse_object
324
+ when :-
325
+ advance
326
+ [:neg, parse_postfix]
327
+ when :if
328
+ parse_if
329
+ when :try
330
+ advance
331
+ body = parse_postfix
332
+ handler = accept(:catch) ? parse_postfix : nil
333
+ [:try, body, handler]
334
+ when :reduce
335
+ advance
336
+ src = parse_alt
337
+ expect(:as)
338
+ patterns = parse_patterns
339
+ expect(:'(')
340
+ init = parse_pipe
341
+ expect(:';')
342
+ update = parse_pipe
343
+ expect(:')')
344
+ [:reduce, src, patterns, init, update]
345
+ when :foreach
346
+ advance
347
+ src = parse_alt
348
+ expect(:as)
349
+ patterns = parse_patterns
350
+ expect(:'(')
351
+ init = parse_pipe
352
+ expect(:';')
353
+ update = parse_pipe
354
+ extract = nil
355
+ extract = parse_pipe if accept(:';')
356
+ expect(:')')
357
+ [:foreach, src, patterns, init, update, extract]
358
+ when :label
359
+ advance
360
+ name = expect(:var).value
361
+ expect(:|)
362
+ body = parse_pipe
363
+ [:label, name, body]
364
+ when :def
365
+ fd = parse_funcdef
366
+ body = parse_pipe
367
+ [:func_def, fd[1], fd[2], fd[3], body]
368
+ when :ident
369
+ advance
370
+ if t.value == 'break'
371
+ name = expect(:var).value
372
+ return [:break, name]
373
+ end
374
+ if peek.type != :'('
375
+ case t.value
376
+ when 'true' then return [:literal, true]
377
+ when 'false' then return [:literal, false]
378
+ when 'null' then return [:literal, nil]
379
+ end
380
+ end
381
+ args = []
382
+ if accept(:'(')
383
+ loop do
384
+ args << parse_pipe
385
+ break unless accept(:';')
386
+ end
387
+ expect(:')')
388
+ end
389
+ [:call, t.value, args]
390
+ else
391
+ unexpected!
392
+ end
393
+ end
394
+
395
+ def parse_if
396
+ expect(:if)
397
+ cond = parse_pipe
398
+ expect(:then)
399
+ then_branch = parse_pipe
400
+ if accept(:elif)
401
+ @pos -= 1
402
+ @tokens[@pos] = Token.new(:if, 'if', @tokens[@pos].line)
403
+ else_branch = parse_if_chain
404
+ return [:if, cond, then_branch, else_branch]
405
+ end
406
+ else_branch = nil
407
+ else_branch = parse_pipe if accept(:else)
408
+ expect(:end)
409
+ [:if, cond, then_branch, else_branch]
410
+ end
411
+
412
+ # parses "if ... end" where the leading token was an elif (shares the final "end")
413
+ def parse_if_chain
414
+ expect(:if)
415
+ cond = parse_pipe
416
+ expect(:then)
417
+ then_branch = parse_pipe
418
+ if accept(:elif)
419
+ @pos -= 1
420
+ @tokens[@pos] = Token.new(:if, 'if', @tokens[@pos].line)
421
+ return [:if, cond, then_branch, parse_if_chain]
422
+ end
423
+ else_branch = nil
424
+ else_branch = parse_pipe if accept(:else)
425
+ expect(:end)
426
+ [:if, cond, then_branch, else_branch]
427
+ end
428
+
429
+ def parse_string(tok, format)
430
+ parts = tok.value.map do |part|
431
+ if part.is_a?(String)
432
+ part
433
+ else
434
+ sub = Parser.new(part)
435
+ e = sub.send(:parse_pipe)
436
+ sub.send(:expect, :eof)
437
+ e
438
+ end
439
+ end
440
+ parts = parts.reject { |p| p.is_a?(String) && p.empty? }
441
+ if parts.all?(String) && format.nil?
442
+ [:literal, parts.join]
443
+ elsif parts.empty?
444
+ [:literal, '']
445
+ else
446
+ [:string, parts, format]
447
+ end
448
+ end
449
+
450
+ def parse_object
451
+ expect(:'{')
452
+ entries = []
453
+ until accept(:'}')
454
+ t = peek
455
+ key = nil
456
+ value = nil
457
+ case t.type
458
+ when :var
459
+ advance
460
+ if accept(:':')
461
+ key = [:var, t.value]
462
+ value = parse_objval
463
+ else
464
+ key = [:literal, t.value]
465
+ value = [:var, t.value]
466
+ end
467
+ when :loc
468
+ advance
469
+ key = [:literal, '__loc__']
470
+ value = [:loc, t.value]
471
+ when :num
472
+ advance
473
+ key = [:literal, t.value]
474
+ expect(:':')
475
+ value = parse_objval
476
+ when :ident
477
+ advance
478
+ key = [:literal, t.value]
479
+ value = accept(:':') ? parse_objval : [:index, [:identity], key]
480
+ when :str
481
+ advance
482
+ key = parse_string(t, nil)
483
+ value = accept(:':') ? parse_objval : [:index, [:identity], key]
484
+ when :format
485
+ advance
486
+ raise CompileError, "syntax error, unexpected ':' at line #{t.line}" unless peek.type == :str
487
+
488
+ key = parse_string(advance, t.value)
489
+ value = accept(:':') ? parse_objval : [:index, [:identity], key]
490
+ when :'('
491
+ advance
492
+ key = parse_pipe
493
+ expect(:')')
494
+ expect(:':')
495
+ value = parse_objval
496
+ else
497
+ if Lexer::KEYWORDS.include?(t.value.to_s) && t.type != :str
498
+ advance
499
+ key = [:literal, t.value]
500
+ value = accept(:':') ? parse_objval : [:index, [:identity], key]
501
+ else
502
+ unexpected!
503
+ end
504
+ end
505
+ entries << [key, value]
506
+ break if accept(:'}')
507
+
508
+ expect(:',')
509
+ end
510
+ [:object, entries]
511
+ end
512
+
513
+ # Object values: no comma allowed, pipes allowed.
514
+ def parse_objval
515
+ lhs = parse_alt
516
+ lhs = [:pipe, lhs, parse_alt] while accept(:|)
517
+ lhs
518
+ end
519
+
520
+ def parse_patterns
521
+ pats = [parse_pattern]
522
+ pats << parse_pattern while accept(:'?//')
523
+ pats
524
+ end
525
+
526
+ def parse_pattern
527
+ t = peek
528
+ case t.type
529
+ when :var
530
+ advance
531
+ [:pvar, t.value]
532
+ when :'['
533
+ advance
534
+ elems = []
535
+ unexpected! if peek.type == :']'
536
+ loop do
537
+ elems << parse_pattern
538
+ break unless accept(:',')
539
+ end
540
+ expect(:']')
541
+ [:parray, elems]
542
+ when :'{'
543
+ advance
544
+ entries = []
545
+ loop do
546
+ k = peek
547
+ case k.type
548
+ when :var
549
+ advance
550
+ sub = accept(:':') ? parse_pattern : nil
551
+ entries << [[:literal, k.value], k.value, sub]
552
+ when :ident
553
+ advance
554
+ expect(:':')
555
+ entries << [[:literal, k.value], nil, parse_pattern]
556
+ when :str
557
+ advance
558
+ expect(:':')
559
+ entries << [parse_string(k, nil), nil, parse_pattern]
560
+ when :'('
561
+ advance
562
+ e = parse_pipe
563
+ expect(:')')
564
+ expect(:':')
565
+ entries << [e, nil, parse_pattern]
566
+ else
567
+ if Lexer::KEYWORDS.include?(k.value.to_s)
568
+ advance
569
+ expect(:':')
570
+ entries << [[:literal, k.value], nil, parse_pattern]
571
+ else
572
+ unexpected!
573
+ end
574
+ end
575
+ break unless accept(:',')
576
+ end
577
+ expect(:'}')
578
+ [:pobject, entries]
579
+ else
580
+ unexpected!
581
+ end
582
+ end
583
+ end
584
+ end
@@ -0,0 +1,104 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rbjq
4
+ # Builtins defined in the jq language itself.
5
+ #
6
+ # Most definitions follow jq 1.8.2's src/builtin.jq, which is distributed
7
+ # under the MIT license (Copyright (C) 2012 Stephen Dolan and jq
8
+ # contributors; see LICENSE-jq). The rest are implemented in Ruby
9
+ # (lib/rbjq/builtins.rb).
10
+ PRELUDE = <<~'JQ'
11
+ def values: select(. != null);
12
+ def nulls: select(. == null);
13
+ def booleans: select(type == "boolean");
14
+ def numbers: select(type == "number");
15
+ def strings: select(type == "string");
16
+ def arrays: select(type == "array");
17
+ def objects: select(type == "object");
18
+ def iterables: select(type|. == "array" or . == "object");
19
+ def scalars: select(type|. != "array" and . != "object");
20
+ def isfinite: type == "number" and (isinfinite | not);
21
+ def finites: select(isfinite);
22
+ def normals: select(isnormal);
23
+ def recurse(f; cond): def r: ., (f | select(cond) | r); r;
24
+ def map(f): [.[] | f];
25
+ def map_values(f): .[] |= f;
26
+ def with_entries(f): to_entries | map(f) | from_entries;
27
+ def add(f): reduce f as $x (null; . + $x);
28
+ def del(f): delpaths([path(f)]);
29
+ def abs: if . < 0 then - . else . end;
30
+ def paths: path(..) | select(length > 0);
31
+ def paths(node_filter): path(.. | select(node_filter)) | select(length > 0);
32
+ def any(generator; condition): isempty(first(generator | condition or empty)) | not;
33
+ def all(generator; condition): isempty(first(generator | condition and empty));
34
+ def any(condition): any(.[]; condition);
35
+ def all(condition): all(.[]; condition);
36
+ def any: any(.[]; .);
37
+ def all: all(.[]; .);
38
+ def in(xs): . as $x | xs | has($x);
39
+ def inside(xs): . as $x | xs | contains($x);
40
+ def nth($n): .[$n];
41
+ def first: .[0];
42
+ def last: .[-1];
43
+ def reverse: [.[length - 1 - range(0; length)]];
44
+ def index($i): indices($i) | .[0];
45
+ def rindex($i): indices($i) | .[-1:][0];
46
+ def sort_by(f): _sort_by_impl(map([f]));
47
+ def group_by(f): _group_by_impl(map([f]));
48
+ def unique_by(f): _unique_by_impl(map([f]));
49
+ def min_by(f): _min_by_impl(map([f]));
50
+ def max_by(f): _max_by_impl(map([f]));
51
+ def combinations: if length == 0 then [] else .[0][] as $x | (.[1:] | combinations) as $y | [$x] + $y end;
52
+ def combinations(n): . as $dot | [range(n) | $dot] | combinations;
53
+ def walk(f): def w: if type == "object" then map_values(w) elif type == "array" then map(w) else . end | f; w;
54
+ def env: $ENV;
55
+ def halt_error: halt_error(5);
56
+ def todateiso8601: strftime("%Y-%m-%dT%H:%M:%SZ");
57
+ def fromdateiso8601: strptime("%Y-%m-%dT%H:%M:%SZ") | mktime;
58
+ def todate: todateiso8601;
59
+ def fromdate: fromdateiso8601;
60
+ def match(re; mode): _match_impl(re; mode; false) | .[];
61
+ def match($val): ($val | type) as $vt
62
+ | if $vt == "string" then match($val; null)
63
+ elif $vt == "array" and ($val | length) > 1 then match($val[0]; $val[1])
64
+ elif $vt == "array" and ($val | length) > 0 then match($val[0]; null)
65
+ else error($vt + " not a string or array") end;
66
+ def test(re; mode): _match_impl(re; mode; true);
67
+ def test($val): ($val | type) as $vt
68
+ | if $vt == "string" then test($val; null)
69
+ elif $vt == "array" and ($val | length) > 1 then test($val[0]; $val[1])
70
+ elif $vt == "array" and ($val | length) > 0 then test($val[0]; null)
71
+ else error($vt + " not a string or array") end;
72
+ def capture(re; mods): match(re; mods) | reduce (.captures | .[] | select(.name != null) | {(.name): .string}) as $pair ({}; . + $pair);
73
+ def capture($val): ($val | type) as $vt
74
+ | if $vt == "string" then capture($val; null)
75
+ elif $vt == "array" and ($val | length) > 1 then capture($val[0]; $val[1])
76
+ elif $vt == "array" and ($val | length) > 0 then capture($val[0]; null)
77
+ else error($vt + " not a string or array") end;
78
+ def scan($re; $flags): match($re; "g" + $flags) | if (.captures | length) > 0 then [.captures | .[] | .string] else .string end;
79
+ def scan($re): scan($re; null);
80
+ def splits($re; flags): split($re; flags) | .[];
81
+ def splits($re): splits($re; null);
82
+ def tostream: path(def r: (.[]? | r), .; r) as $p | getpath($p) | reduce path(.[]?) as $q ([$p, .]; [$p + $q]);
83
+ def fromstream(f): {x: null, e: false} as $init
84
+ | foreach f as $i ($init;
85
+ if .e then $init else . end
86
+ | if $i | length == 2
87
+ then setpath(["e"]; $i[0] | length == 0) | setpath(["x"] + $i[0]; $i[1])
88
+ else setpath(["e"]; $i[0] | length == 1) end;
89
+ if .e then .x else empty end);
90
+ def truncate_stream(stream): . as $n | null | stream | . as $input | if (.[0] | length) > $n then setpath([0]; $input[0][$n:]) else empty end;
91
+ def INDEX(stream; idx_expr): reduce stream as $row ({}; .[$row | idx_expr | tostring] = $row);
92
+ def INDEX(idx_expr): INDEX(.[]; idx_expr);
93
+ def JOIN($idx; idx_expr): [.[] | [., $idx[idx_expr]]];
94
+ def JOIN($idx; stream; idx_expr): stream | [., $idx[idx_expr]];
95
+ def JOIN($idx; stream; idx_expr; join_expr): stream | [., $idx[idx_expr]] | join_expr;
96
+ def IN(s): any(s == .; .);
97
+ def IN(src; s): any(src == s; .);
98
+ def transpose: [range(0; map(length) | max // 0) as $i | [.[][$i]]];
99
+ def pick(pathexps): . as $in | reduce path(pathexps) as $a (null; setpath($a; $in | getpath($a)));
100
+ def debug(msgs): (msgs | debug | empty), .;
101
+ def get_search_list: ["~/.jq", "$ORIGIN/../lib/jq", "$ORIGIN/../lib"];
102
+ def modulemeta: if type == "string" then error("module not found: \(.)") else error("modulemeta input module name must be a string") end;
103
+ JQ
104
+ end