rbjq 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +121 -0
- data/LICENSE-jq +176 -0
- data/README.md +155 -0
- data/exe/rbjq +25 -0
- data/lib/rbjq/bessel.rb +139 -0
- data/lib/rbjq/builtins.rb +1357 -0
- data/lib/rbjq/checker.rb +140 -0
- data/lib/rbjq/cli.rb +519 -0
- data/lib/rbjq/env.rb +39 -0
- data/lib/rbjq/formats.rb +98 -0
- data/lib/rbjq/functions.rb +137 -0
- data/lib/rbjq/interpreter.rb +726 -0
- data/lib/rbjq/json.rb +518 -0
- data/lib/rbjq/lexer.rb +183 -0
- data/lib/rbjq/ops.rb +449 -0
- data/lib/rbjq/parser.rb +584 -0
- data/lib/rbjq/prelude.rb +104 -0
- data/lib/rbjq/program.rb +106 -0
- data/lib/rbjq/value.rb +188 -0
- data/lib/rbjq/version.rb +5 -0
- data/lib/rbjq.rb +90 -0
- metadata +66 -0
data/lib/rbjq/json.rb
ADDED
|
@@ -0,0 +1,518 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'strscan'
|
|
4
|
+
|
|
5
|
+
module Rbjq
|
|
6
|
+
# Streaming JSON parser: parses a sequence of whitespace-separated JSON texts.
|
|
7
|
+
class JSONParser
|
|
8
|
+
WS = /[ \t\r\n]*/
|
|
9
|
+
NUMBER = /-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?/
|
|
10
|
+
STR_CHUNK = /[^"\\\x00-\x1f]+/
|
|
11
|
+
ESCAPES = { '"' => '"', '\\' => '\\', '/' => '/', 'b' => "\b", 'f' => "\f", 'n' => "\n", 'r' => "\r",
|
|
12
|
+
't' => "\t" }.freeze
|
|
13
|
+
|
|
14
|
+
def self.parse(src)
|
|
15
|
+
p = new(src)
|
|
16
|
+
v = p.next_value
|
|
17
|
+
raise ParseError, "Expected JSON value (while parsing '#{src}')" if v.equal?(EOF)
|
|
18
|
+
|
|
19
|
+
raise ParseError, "Unexpected extra JSON values (while parsing '#{src}')" unless p.next_value.equal?(EOF)
|
|
20
|
+
|
|
21
|
+
v
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def self.parse_all(src)
|
|
25
|
+
p = new(src)
|
|
26
|
+
out = []
|
|
27
|
+
loop do
|
|
28
|
+
v = p.next_value
|
|
29
|
+
break if v.equal?(EOF)
|
|
30
|
+
|
|
31
|
+
out << v
|
|
32
|
+
end
|
|
33
|
+
out
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
EOF = Object.new.freeze
|
|
37
|
+
|
|
38
|
+
attr_reader :scanner
|
|
39
|
+
|
|
40
|
+
def initialize(src)
|
|
41
|
+
src = src.dup.force_encoding(Encoding::UTF_8)
|
|
42
|
+
src = Value.scrub_utf8(src) unless src.valid_encoding?
|
|
43
|
+
src = src.delete_prefix('') # byte order mark
|
|
44
|
+
@src = src
|
|
45
|
+
@s = StringScanner.new(src)
|
|
46
|
+
@counted_pos = 0
|
|
47
|
+
@counted_lines = 0
|
|
48
|
+
@line_number = 0
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# Line number as reported by jq's input_line_number: jq reads its input
|
|
52
|
+
# line by line, so this is the number of newlines up to and including the
|
|
53
|
+
# end of the line on which the last parsed value ended.
|
|
54
|
+
attr_reader :line_number
|
|
55
|
+
|
|
56
|
+
# Returns the next value or EOF
|
|
57
|
+
def next_value
|
|
58
|
+
@s.skip(WS)
|
|
59
|
+
return EOF if @s.eos?
|
|
60
|
+
|
|
61
|
+
v = parse_value
|
|
62
|
+
nl = @src.byteindex("\n", @s.pos)
|
|
63
|
+
@line_number = count_lines_upto(nl ? nl + 1 : @src.bytesize)
|
|
64
|
+
v
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def line
|
|
68
|
+
@src.byteslice(0, @s.pos).count("\n") + 1
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
private
|
|
72
|
+
|
|
73
|
+
def count_lines_upto(pos)
|
|
74
|
+
if pos > @counted_pos
|
|
75
|
+
@counted_lines += @src.byteslice(@counted_pos, pos - @counted_pos).count("\n")
|
|
76
|
+
@counted_pos = pos
|
|
77
|
+
end
|
|
78
|
+
@counted_lines
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def error(msg)
|
|
82
|
+
raise ParseError, "#{msg} at line #{line}, column #{column}"
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def column
|
|
86
|
+
before = @src.byteslice(0, @s.pos)
|
|
87
|
+
nl = before.rindex("\n")
|
|
88
|
+
nl ? before.length - nl - 1 : before.length
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def parse_value
|
|
92
|
+
@s.skip(WS)
|
|
93
|
+
c = @s.peek(1)
|
|
94
|
+
case c
|
|
95
|
+
when '{' then parse_object
|
|
96
|
+
when '[' then parse_array
|
|
97
|
+
when '"'
|
|
98
|
+
@s.pos += 1
|
|
99
|
+
parse_string
|
|
100
|
+
when ''
|
|
101
|
+
error('Unfinished JSON term at EOF')
|
|
102
|
+
when ']' then fail_at_char("Unmatched ']'")
|
|
103
|
+
when '}' then fail_at_char("Unmatched '}'")
|
|
104
|
+
when ':' then fail_at_char("Expected string key before ':'")
|
|
105
|
+
when ',' then fail_at_char("Expected value before ','")
|
|
106
|
+
when "'"
|
|
107
|
+
@s.skip(LITERAL)
|
|
108
|
+
error_after_token(%(Invalid string literal; expected ", but got '))
|
|
109
|
+
else
|
|
110
|
+
parse_literal
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# Characters of a bare token (anything up to a delimiter), like jq's tokenizer
|
|
115
|
+
LITERAL = /[^ \t\r\n\[\]{},:"]+/
|
|
116
|
+
NUMBER_LITERAL = /\A[+-]?(?:[0-9]+\.?[0-9]*|\.[0-9]+)(?:[eE][+-]?[0-9]+)?\z/
|
|
117
|
+
|
|
118
|
+
def parse_literal
|
|
119
|
+
tok = @s.scan(LITERAL) or error('Unfinished JSON term at EOF')
|
|
120
|
+
case tok
|
|
121
|
+
when 'true' then true
|
|
122
|
+
when 'false' then false
|
|
123
|
+
when 'null' then nil
|
|
124
|
+
when 'nan', 'NaN', '-nan', '-NaN' then Float::NAN
|
|
125
|
+
when 'Infinity', 'infinity' then Float::MAX
|
|
126
|
+
when '-Infinity', '-infinity' then -Float::MAX
|
|
127
|
+
else
|
|
128
|
+
return parse_number(tok) if tok.match?(NUMBER_LITERAL)
|
|
129
|
+
|
|
130
|
+
error_after_token(%w[t f n].include?(tok[0]) ? 'Invalid literal' : 'Invalid numeric literal')
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def parse_number(n)
|
|
135
|
+
n = n.delete_prefix('+')
|
|
136
|
+
if n.match?(/[.eE]/)
|
|
137
|
+
f = Value.parse_float(n.sub(/\A(-?)\./, '\10.').sub(/\.(?![0-9])/, '.0'))
|
|
138
|
+
f = Float::MAX if f == Float::INFINITY
|
|
139
|
+
f = -Float::MAX if f == -Float::INFINITY
|
|
140
|
+
Value.normalize_number(f)
|
|
141
|
+
else
|
|
142
|
+
# like jq without decNumber, numbers are doubles
|
|
143
|
+
return -0.0 if n.match?(/\A-0+\z/)
|
|
144
|
+
|
|
145
|
+
Value.normalize_number(n.to_i)
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# jq reports errors in a bare token after reading the delimiter that ends it
|
|
150
|
+
def error_after_token(msg)
|
|
151
|
+
raise ParseError, "#{msg} at line #{line}, column #{column + 1}" unless @s.eos?
|
|
152
|
+
|
|
153
|
+
error("#{msg} at EOF")
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
# jq reports a misplaced token after reading it (and its delimiter)
|
|
157
|
+
def unexpected_token(msg)
|
|
158
|
+
if @s.skip(LITERAL)
|
|
159
|
+
error_after_token(msg)
|
|
160
|
+
elsif @s.peek(1) == '"'
|
|
161
|
+
@s.pos += 1
|
|
162
|
+
parse_string
|
|
163
|
+
raise ParseError, "#{msg} at line #{line}, column #{column}"
|
|
164
|
+
else
|
|
165
|
+
fail_at_char(msg)
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# report an error after consuming the current (structural) character
|
|
170
|
+
def fail_at_char(msg)
|
|
171
|
+
@s.getch
|
|
172
|
+
raise ParseError, "#{msg} at line #{line}, column #{column}"
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
MAX_PARSING_DEPTH = 10_000
|
|
176
|
+
|
|
177
|
+
def enter
|
|
178
|
+
@depth = (@depth || 0) + 1
|
|
179
|
+
error('Exceeds depth limit for parsing') if @depth > MAX_PARSING_DEPTH
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
def leave
|
|
183
|
+
@depth -= 1
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
def parse_object
|
|
187
|
+
@s.pos += 1
|
|
188
|
+
enter
|
|
189
|
+
h = {}
|
|
190
|
+
@s.skip(WS)
|
|
191
|
+
if @s.skip(/\}/)
|
|
192
|
+
leave
|
|
193
|
+
return h
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
loop do
|
|
197
|
+
@s.skip(WS)
|
|
198
|
+
error('Unfinished JSON term at EOF') if @s.eos?
|
|
199
|
+
if @s.peek(1) == "'"
|
|
200
|
+
@s.skip(LITERAL)
|
|
201
|
+
error_after_token(%(Invalid string literal; expected ", but got '))
|
|
202
|
+
end
|
|
203
|
+
unless @s.skip(/"/)
|
|
204
|
+
case @s.peek(1)
|
|
205
|
+
when '}' then fail_at_char('Expected another key-value pair')
|
|
206
|
+
when ',' then fail_at_char("Expected value before ','")
|
|
207
|
+
when ']' then fail_at_char("Unmatched ']'")
|
|
208
|
+
end
|
|
209
|
+
error_after_token('Object keys must be strings') if @s.skip(LITERAL)
|
|
210
|
+
error('Object keys must be strings')
|
|
211
|
+
end
|
|
212
|
+
k = parse_string
|
|
213
|
+
@s.skip(WS)
|
|
214
|
+
error('Unfinished JSON term at EOF') if @s.eos?
|
|
215
|
+
unless @s.skip(/:/)
|
|
216
|
+
fail_at_char('Objects must consist of key:value pairs') if @s.peek(1) == ','
|
|
217
|
+
unexpected_token('Expected separator between values')
|
|
218
|
+
end
|
|
219
|
+
h[k] = parse_value
|
|
220
|
+
@s.skip(WS)
|
|
221
|
+
next if @s.skip(/,/)
|
|
222
|
+
break if @s.skip(/\}/)
|
|
223
|
+
|
|
224
|
+
error('Unfinished JSON term at EOF') if @s.eos?
|
|
225
|
+
fail_at_char("Unmatched ']'") if @s.peek(1) == ']'
|
|
226
|
+
unexpected_token('Expected separator between values')
|
|
227
|
+
end
|
|
228
|
+
leave
|
|
229
|
+
h
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
def parse_array
|
|
233
|
+
@s.pos += 1
|
|
234
|
+
enter
|
|
235
|
+
a = []
|
|
236
|
+
@s.skip(WS)
|
|
237
|
+
if @s.skip(/\]/)
|
|
238
|
+
leave
|
|
239
|
+
return a
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
loop do
|
|
243
|
+
@s.skip(WS)
|
|
244
|
+
case @s.peek(1)
|
|
245
|
+
when ']' then fail_at_char('Expected another array element')
|
|
246
|
+
when ',' then fail_at_char("Expected value before ','")
|
|
247
|
+
end
|
|
248
|
+
a << parse_value
|
|
249
|
+
@s.skip(WS)
|
|
250
|
+
next if @s.skip(/,/)
|
|
251
|
+
break if @s.skip(/\]/)
|
|
252
|
+
|
|
253
|
+
error('Unfinished JSON term at EOF') if @s.eos?
|
|
254
|
+
fail_at_char('Objects must consist of key:value pairs') if @s.peek(1) == '}'
|
|
255
|
+
fail_at_char("':' not as part of an object") if @s.peek(1) == ':'
|
|
256
|
+
unexpected_token('Expected separator between values')
|
|
257
|
+
end
|
|
258
|
+
leave
|
|
259
|
+
a
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
# Like jq, the whole string is read first, then validated; errors are
|
|
263
|
+
# reported at the closing quote.
|
|
264
|
+
def parse_string
|
|
265
|
+
buf = +''
|
|
266
|
+
failure = nil
|
|
267
|
+
loop do
|
|
268
|
+
if (chunk = @s.scan(STR_CHUNK))
|
|
269
|
+
buf << chunk
|
|
270
|
+
end
|
|
271
|
+
c = @s.getch
|
|
272
|
+
case c
|
|
273
|
+
when '"'
|
|
274
|
+
error(failure) if failure
|
|
275
|
+
return buf
|
|
276
|
+
when '\\'
|
|
277
|
+
e = @s.getch
|
|
278
|
+
if (r = ESCAPES[e])
|
|
279
|
+
buf << r
|
|
280
|
+
elsif e == 'u'
|
|
281
|
+
hex = @s.scan(/[0-9a-fA-F]{4}/)
|
|
282
|
+
unless hex
|
|
283
|
+
failure ||= 'Invalid \\uXXXX escape'
|
|
284
|
+
next
|
|
285
|
+
end
|
|
286
|
+
cp = hex.to_i(16)
|
|
287
|
+
if cp.between?(0xD800, 0xDBFF)
|
|
288
|
+
if @s.match?(/\\u[dD][c-fC-F][0-9a-fA-F]{2}/)
|
|
289
|
+
lo = @s.scan(/\\u..../)[2..].to_i(16)
|
|
290
|
+
cp = 0x10000 + ((cp - 0xD800) << 10) + (lo - 0xDC00)
|
|
291
|
+
else
|
|
292
|
+
failure ||= 'Invalid \\uXXXX\\uXXXX surrogate pair escape'
|
|
293
|
+
next
|
|
294
|
+
end
|
|
295
|
+
elsif cp.between?(0xDC00, 0xDFFF)
|
|
296
|
+
cp = 0xFFFD
|
|
297
|
+
end
|
|
298
|
+
buf << cp.chr(Encoding::UTF_8)
|
|
299
|
+
elsif e.nil?
|
|
300
|
+
error('Unfinished string at EOF')
|
|
301
|
+
else
|
|
302
|
+
failure ||= 'Invalid escape'
|
|
303
|
+
end
|
|
304
|
+
when nil
|
|
305
|
+
error('Unfinished string at EOF')
|
|
306
|
+
else
|
|
307
|
+
failure ||= 'Invalid string: control characters from U+0000 through U+001F must be escaped'
|
|
308
|
+
buf << c
|
|
309
|
+
end
|
|
310
|
+
end
|
|
311
|
+
end
|
|
312
|
+
end
|
|
313
|
+
|
|
314
|
+
# JSON encoder with jq-compatible number formatting and pretty-printing.
|
|
315
|
+
class Encoder
|
|
316
|
+
DEFAULT_COLORS = {
|
|
317
|
+
null: '0;90', false: '0;39', true: '0;39', number: '0;39', string: '0;32',
|
|
318
|
+
array: '1;39', object: '1;39', key: '1;34'
|
|
319
|
+
}.freeze
|
|
320
|
+
COLOR_KEYS = %i[null false true number string array object key].freeze
|
|
321
|
+
|
|
322
|
+
def self.dump(v)
|
|
323
|
+
new.encode(v)
|
|
324
|
+
end
|
|
325
|
+
|
|
326
|
+
def self.parse_colors(spec)
|
|
327
|
+
colors = DEFAULT_COLORS.dup
|
|
328
|
+
return colors if spec.nil? || spec.empty?
|
|
329
|
+
|
|
330
|
+
spec.split(':').each_with_index do |c, i|
|
|
331
|
+
break if i >= COLOR_KEYS.size
|
|
332
|
+
|
|
333
|
+
colors[COLOR_KEYS[i]] = c
|
|
334
|
+
end
|
|
335
|
+
colors
|
|
336
|
+
end
|
|
337
|
+
|
|
338
|
+
# pretty: nil means pretty-print when indenting (indent > 0 or tab)
|
|
339
|
+
def initialize(indent: 0, tab: false, sort_keys: false, ascii: false, color: false, colors: nil, pretty: nil)
|
|
340
|
+
@indent = indent
|
|
341
|
+
@tab = tab
|
|
342
|
+
@sort_keys = sort_keys
|
|
343
|
+
@ascii = ascii
|
|
344
|
+
@color = color
|
|
345
|
+
@colors = colors || DEFAULT_COLORS
|
|
346
|
+
@pretty = pretty.nil? ? (tab || indent.positive?) : pretty
|
|
347
|
+
end
|
|
348
|
+
|
|
349
|
+
def encode(v)
|
|
350
|
+
buf = +''
|
|
351
|
+
write(buf, v, 0)
|
|
352
|
+
buf
|
|
353
|
+
end
|
|
354
|
+
|
|
355
|
+
def self.format_number(n)
|
|
356
|
+
case n
|
|
357
|
+
when Integer
|
|
358
|
+
n.to_s
|
|
359
|
+
when Float
|
|
360
|
+
return 'null' if n.nan?
|
|
361
|
+
return '1.7976931348623157e+308' if n == Float::INFINITY
|
|
362
|
+
return '-1.7976931348623157e+308' if n == -Float::INFINITY
|
|
363
|
+
|
|
364
|
+
format_float(n)
|
|
365
|
+
end
|
|
366
|
+
end
|
|
367
|
+
|
|
368
|
+
# Port of jq's jvp_dtoa_fmt (shortest round-trip digits, %.17g-like layout)
|
|
369
|
+
def self.format_float(f)
|
|
370
|
+
return (1.0 / f).negative? ? '-0' : '0' if f.zero?
|
|
371
|
+
|
|
372
|
+
s = f.abs.to_s # shortest repr, e.g. "1.0e+20", "0.1", "123.456", "1.0e-05"
|
|
373
|
+
mant, exp = s.split('e')
|
|
374
|
+
exp = exp.to_i
|
|
375
|
+
int, frac = mant.split('.')
|
|
376
|
+
frac = '' if frac == '0'
|
|
377
|
+
digits = (int + frac)
|
|
378
|
+
lead = digits.length - digits.sub(/\A0+/, '').length
|
|
379
|
+
digits = digits.sub(/\A0+/, '')
|
|
380
|
+
decpt = int.length + exp - lead
|
|
381
|
+
digits = digits.sub(/0+\z/, '')
|
|
382
|
+
digits = '0' if digits.empty?
|
|
383
|
+
nd = digits.length
|
|
384
|
+
sign = f.negative? ? '-' : ''
|
|
385
|
+
|
|
386
|
+
if decpt <= -4 || decpt > nd + 15
|
|
387
|
+
e = decpt - 1
|
|
388
|
+
m = digits[0]
|
|
389
|
+
m += ".#{digits[1..]}" if nd > 1
|
|
390
|
+
es = e.negative? ? '-' : '+'
|
|
391
|
+
ea = e.abs
|
|
392
|
+
"#{sign}#{m}e#{es}#{ea < 10 ? "0#{ea}" : ea}"
|
|
393
|
+
elsif decpt <= 0
|
|
394
|
+
"#{sign}0.#{'0' * -decpt}#{digits}"
|
|
395
|
+
elsif decpt >= nd
|
|
396
|
+
"#{sign}#{digits}#{'0' * (decpt - nd)}"
|
|
397
|
+
else
|
|
398
|
+
"#{sign}#{digits[0, decpt]}.#{digits[decpt..]}"
|
|
399
|
+
end
|
|
400
|
+
end
|
|
401
|
+
|
|
402
|
+
ESCAPE_MAP = {
|
|
403
|
+
'"' => '\\"', '\\' => '\\\\', "\n" => '\\n', "\t" => '\\t', "\r" => '\\r', "\b" => '\\b', "\f" => '\\f'
|
|
404
|
+
}.freeze
|
|
405
|
+
|
|
406
|
+
def self.escape_string(s, ascii = false) # rubocop:disable Style/OptionalBooleanParameter
|
|
407
|
+
s = s.scrub('�') unless s.valid_encoding?
|
|
408
|
+
out = s.gsub(/["\\\x00-\x1f\x7f]/) do |c|
|
|
409
|
+
ESCAPE_MAP[c] || format('\\u%04x', c.ord)
|
|
410
|
+
end
|
|
411
|
+
if ascii
|
|
412
|
+
out = out.gsub(/[^\x00-\x7f]/) do |c|
|
|
413
|
+
cp = c.ord
|
|
414
|
+
if cp >= 0x10000
|
|
415
|
+
cp -= 0x10000
|
|
416
|
+
format('\u%<hi>04x\u%<lo>04x', hi: 0xD800 + (cp >> 10), lo: 0xDC00 + (cp & 0x3FF))
|
|
417
|
+
else
|
|
418
|
+
format('\\u%04x', cp)
|
|
419
|
+
end
|
|
420
|
+
end
|
|
421
|
+
end
|
|
422
|
+
"\"#{out}\""
|
|
423
|
+
end
|
|
424
|
+
|
|
425
|
+
private
|
|
426
|
+
|
|
427
|
+
def col(kind)
|
|
428
|
+
"\e[#{@colors[kind]}m"
|
|
429
|
+
end
|
|
430
|
+
|
|
431
|
+
RESET = "\e[0m"
|
|
432
|
+
|
|
433
|
+
def newline_indent(buf, level)
|
|
434
|
+
return unless @pretty
|
|
435
|
+
|
|
436
|
+
buf << "\n"
|
|
437
|
+
buf << (@tab ? "\t" * level : ' ' * (@indent * level))
|
|
438
|
+
end
|
|
439
|
+
|
|
440
|
+
MAX_PRINT_DEPTH = 10_000
|
|
441
|
+
|
|
442
|
+
def write(buf, v, level)
|
|
443
|
+
if level > MAX_PRINT_DEPTH
|
|
444
|
+
buf << '<skipped: too deep>'
|
|
445
|
+
return
|
|
446
|
+
end
|
|
447
|
+
|
|
448
|
+
case v
|
|
449
|
+
when nil
|
|
450
|
+
scalar(buf, :null, 'null')
|
|
451
|
+
when true
|
|
452
|
+
scalar(buf, :true, 'true')
|
|
453
|
+
when false
|
|
454
|
+
scalar(buf, :false, 'false')
|
|
455
|
+
when Integer, Float
|
|
456
|
+
scalar(buf, :number, Encoder.format_number(v))
|
|
457
|
+
when String
|
|
458
|
+
scalar(buf, :string, Encoder.escape_string(v, @ascii))
|
|
459
|
+
when Array
|
|
460
|
+
write_array(buf, v, level)
|
|
461
|
+
when Hash
|
|
462
|
+
write_object(buf, v, level)
|
|
463
|
+
else
|
|
464
|
+
raise TypeError, "cannot encode #{v.class}"
|
|
465
|
+
end
|
|
466
|
+
end
|
|
467
|
+
|
|
468
|
+
def scalar(buf, kind, text)
|
|
469
|
+
if @color
|
|
470
|
+
buf << col(kind) << text << RESET
|
|
471
|
+
else
|
|
472
|
+
buf << text
|
|
473
|
+
end
|
|
474
|
+
end
|
|
475
|
+
|
|
476
|
+
# Colored output follows jq 1.8: every bracket and separator is colored
|
|
477
|
+
# and immediately reset.
|
|
478
|
+
def punct(buf, c, text)
|
|
479
|
+
if c
|
|
480
|
+
buf << c << text << RESET
|
|
481
|
+
else
|
|
482
|
+
buf << text
|
|
483
|
+
end
|
|
484
|
+
end
|
|
485
|
+
|
|
486
|
+
def write_array(buf, v, level)
|
|
487
|
+
c = @color ? col(:array) : nil
|
|
488
|
+
return punct(buf, c, '[]') if v.empty?
|
|
489
|
+
|
|
490
|
+
punct(buf, c, '[')
|
|
491
|
+
v.each_with_index do |e, i|
|
|
492
|
+
punct(buf, c, ',') if i.positive?
|
|
493
|
+
newline_indent(buf, level + 1)
|
|
494
|
+
write(buf, e, level + 1)
|
|
495
|
+
end
|
|
496
|
+
newline_indent(buf, level)
|
|
497
|
+
punct(buf, c, ']')
|
|
498
|
+
end
|
|
499
|
+
|
|
500
|
+
def write_object(buf, v, level)
|
|
501
|
+
c = @color ? col(:object) : nil
|
|
502
|
+
return punct(buf, c, '{}') if v.empty?
|
|
503
|
+
|
|
504
|
+
punct(buf, c, '{')
|
|
505
|
+
keys = @sort_keys ? v.keys.sort_by(&:b) : v.keys
|
|
506
|
+
keys.each_with_index do |k, i|
|
|
507
|
+
punct(buf, c, ',') if i.positive?
|
|
508
|
+
newline_indent(buf, level + 1)
|
|
509
|
+
punct(buf, c && col(:key), Encoder.escape_string(k, @ascii))
|
|
510
|
+
punct(buf, c, ':')
|
|
511
|
+
buf << ' ' if @pretty
|
|
512
|
+
write(buf, v[k], level + 1)
|
|
513
|
+
end
|
|
514
|
+
newline_indent(buf, level)
|
|
515
|
+
punct(buf, c, '}')
|
|
516
|
+
end
|
|
517
|
+
end
|
|
518
|
+
end
|
data/lib/rbjq/lexer.rb
ADDED
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Rbjq
|
|
4
|
+
Token = Struct.new(:type, :value, :line) do
|
|
5
|
+
def to_s
|
|
6
|
+
case type
|
|
7
|
+
when :ident, :field, :var, :format, :num then value.to_s
|
|
8
|
+
when :str then 'string'
|
|
9
|
+
when :eof then 'end of file'
|
|
10
|
+
else type.to_s
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
class Lexer
|
|
16
|
+
KEYWORDS = %w[
|
|
17
|
+
def if then elif else end as reduce foreach try catch label import include and or __loc__
|
|
18
|
+
].freeze
|
|
19
|
+
|
|
20
|
+
PUNCTS = [
|
|
21
|
+
'?//', '|=', '+=', '-=', '*=', '/=', '%=', '//=', '==', '!=', '<=', '>=', '//', '..',
|
|
22
|
+
'|', ',', '+', '-', '*', '/', '%', '=', '<', '>', '.', '[', ']', '{', '}', '(', ')', ':', ';', '?'
|
|
23
|
+
].freeze
|
|
24
|
+
|
|
25
|
+
IDENT_RE = /[a-zA-Z_][a-zA-Z_0-9]*(?:::[a-zA-Z_][a-zA-Z_0-9]*)*/
|
|
26
|
+
NUM_RE = /(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?/
|
|
27
|
+
|
|
28
|
+
def initialize(src)
|
|
29
|
+
@src = src
|
|
30
|
+
@pos = 0
|
|
31
|
+
@line = 1
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def tokenize
|
|
35
|
+
tokens = lex(false)
|
|
36
|
+
tokens << Token.new(:eof, nil, @line)
|
|
37
|
+
tokens
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
def error(msg)
|
|
43
|
+
raise CompileError, "syntax error: #{msg} at line #{@line}"
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# Lex until EOF, or (when in_interp) until the closing paren of a string interpolation.
|
|
47
|
+
def lex(in_interp)
|
|
48
|
+
tokens = []
|
|
49
|
+
depth = 0
|
|
50
|
+
loop do
|
|
51
|
+
skip_space
|
|
52
|
+
if @pos >= @src.length
|
|
53
|
+
error('unterminated string interpolation') if in_interp
|
|
54
|
+
return tokens
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
c = @src[@pos]
|
|
58
|
+
if c == '"'
|
|
59
|
+
@pos += 1
|
|
60
|
+
tokens << Token.new(:str, lex_string, @line)
|
|
61
|
+
elsif (m = match(NUM_RE)) && !(c == '.' && @src[@pos + 1] !~ /[0-9]/)
|
|
62
|
+
@pos += m.length
|
|
63
|
+
tokens << Token.new(:num, parse_number(m), @line)
|
|
64
|
+
elsif c == '.' && (m = match(/\.([a-zA-Z_][a-zA-Z_0-9]*)/))
|
|
65
|
+
@pos += m.length
|
|
66
|
+
tokens << Token.new(:field, m[1..], @line)
|
|
67
|
+
elsif c == '$' && (m = match(/\$(#{IDENT_RE})/o))
|
|
68
|
+
@pos += m.length
|
|
69
|
+
name = m[1..]
|
|
70
|
+
tokens << (name == '__loc__' ? Token.new(:loc, @line, @line) : Token.new(:var, name, @line))
|
|
71
|
+
elsif c == '@' && (m = match(/@([a-zA-Z0-9_]+)/))
|
|
72
|
+
@pos += m.length
|
|
73
|
+
tokens << Token.new(:format, m[1..], @line)
|
|
74
|
+
elsif (m = match(IDENT_RE))
|
|
75
|
+
@pos += m.length
|
|
76
|
+
tokens << (KEYWORDS.include?(m) ? Token.new(m.to_sym, m, @line) : Token.new(:ident, m, @line))
|
|
77
|
+
else
|
|
78
|
+
p = PUNCTS.find { |s| @src[@pos, s.length] == s }
|
|
79
|
+
error("unexpected character #{c.inspect}") unless p
|
|
80
|
+
if in_interp
|
|
81
|
+
if p == '('
|
|
82
|
+
depth += 1
|
|
83
|
+
elsif p == ')'
|
|
84
|
+
if depth.zero?
|
|
85
|
+
@pos += 1
|
|
86
|
+
return tokens
|
|
87
|
+
end
|
|
88
|
+
depth -= 1
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
@pos += p.length
|
|
92
|
+
tokens << Token.new(p.to_sym, p, @line)
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def match(re)
|
|
98
|
+
m = re.match(@src, @pos)
|
|
99
|
+
m && m.begin(0) == @pos ? m[0] : nil
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def skip_space
|
|
103
|
+
while @pos < @src.length
|
|
104
|
+
c = @src[@pos]
|
|
105
|
+
if c == "\n"
|
|
106
|
+
@line += 1
|
|
107
|
+
@pos += 1
|
|
108
|
+
elsif [' ', "\t", "\r"].include?(c)
|
|
109
|
+
@pos += 1
|
|
110
|
+
elsif c == '#'
|
|
111
|
+
@pos += 1 while @pos < @src.length && @src[@pos] != "\n"
|
|
112
|
+
else
|
|
113
|
+
break
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def parse_number(s)
|
|
119
|
+
if s.match?(/[.eE]/)
|
|
120
|
+
f = Value.parse_float(s.start_with?('.') ? "0#{s}" : s.sub(/\.(?![0-9])/, '.0'))
|
|
121
|
+
Value.normalize_number(f)
|
|
122
|
+
else
|
|
123
|
+
Value.normalize_number(Integer(s, 10))
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# Returns an array of parts: String or Array (tokens of an interpolation)
|
|
128
|
+
def lex_string
|
|
129
|
+
parts = []
|
|
130
|
+
buf = +''
|
|
131
|
+
loop do
|
|
132
|
+
error('unterminated string') if @pos >= @src.length
|
|
133
|
+
c = @src[@pos]
|
|
134
|
+
if c == '"'
|
|
135
|
+
@pos += 1
|
|
136
|
+
parts << buf unless buf.empty? && !parts.empty?
|
|
137
|
+
return parts
|
|
138
|
+
elsif c == '\\'
|
|
139
|
+
e = @src[@pos + 1]
|
|
140
|
+
@pos += 2
|
|
141
|
+
case e
|
|
142
|
+
when 'n' then buf << "\n"
|
|
143
|
+
when 't' then buf << "\t"
|
|
144
|
+
when 'r' then buf << "\r"
|
|
145
|
+
when 'b' then buf << "\b"
|
|
146
|
+
when 'f' then buf << "\f"
|
|
147
|
+
when '"' then buf << '"'
|
|
148
|
+
when '\\' then buf << '\\'
|
|
149
|
+
when '/' then buf << '/'
|
|
150
|
+
when 'u'
|
|
151
|
+
hex = @src[@pos, 4]
|
|
152
|
+
error('invalid \\u escape') unless hex&.match?(/\A[0-9a-fA-F]{4}\z/)
|
|
153
|
+
@pos += 4
|
|
154
|
+
cp = hex.to_i(16)
|
|
155
|
+
lo = @src[@pos + 2, 4] if @src[@pos, 2] == '\\u'
|
|
156
|
+
if cp.between?(0xD800, 0xDBFF) && lo&.match?(/\A[dD][c-fC-F][0-9a-fA-F]{2}\z/)
|
|
157
|
+
@pos += 6
|
|
158
|
+
cp = 0x10000 + ((cp - 0xD800) << 10) + (lo.to_i(16) - 0xDC00)
|
|
159
|
+
elsif cp.between?(0xD800, 0xDBFF)
|
|
160
|
+
# like jq: a high surrogate must be followed by a low one
|
|
161
|
+
error('Invalid \\uXXXX\\uXXXX surrogate pair escape')
|
|
162
|
+
elsif cp.between?(0xDC00, 0xDFFF)
|
|
163
|
+
cp = 0xFFFD
|
|
164
|
+
end
|
|
165
|
+
buf << cp.chr(Encoding::UTF_8)
|
|
166
|
+
when '('
|
|
167
|
+
parts << buf
|
|
168
|
+
buf = +''
|
|
169
|
+
toks = lex(true)
|
|
170
|
+
toks << Token.new(:eof, nil, @line)
|
|
171
|
+
parts << toks
|
|
172
|
+
else
|
|
173
|
+
error("invalid escape \\#{e}")
|
|
174
|
+
end
|
|
175
|
+
else
|
|
176
|
+
@line += 1 if c == "\n"
|
|
177
|
+
buf << c
|
|
178
|
+
@pos += 1
|
|
179
|
+
end
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
end
|
|
183
|
+
end
|