rbjq 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +121 -0
- data/LICENSE-jq +176 -0
- data/README.md +155 -0
- data/exe/rbjq +25 -0
- data/lib/rbjq/bessel.rb +139 -0
- data/lib/rbjq/builtins.rb +1357 -0
- data/lib/rbjq/checker.rb +140 -0
- data/lib/rbjq/cli.rb +519 -0
- data/lib/rbjq/env.rb +39 -0
- data/lib/rbjq/formats.rb +98 -0
- data/lib/rbjq/functions.rb +137 -0
- data/lib/rbjq/interpreter.rb +726 -0
- data/lib/rbjq/json.rb +518 -0
- data/lib/rbjq/lexer.rb +183 -0
- data/lib/rbjq/ops.rb +449 -0
- data/lib/rbjq/parser.rb +584 -0
- data/lib/rbjq/prelude.rb +104 -0
- data/lib/rbjq/program.rb +106 -0
- data/lib/rbjq/value.rb +188 -0
- data/lib/rbjq/version.rb +5 -0
- data/lib/rbjq.rb +90 -0
- metadata +66 -0
data/lib/rbjq/checker.rb
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Rbjq
|
|
4
|
+
# Static check that every referenced function, variable and label is defined.
|
|
5
|
+
class Checker
|
|
6
|
+
Scope = Struct.new(:funcs, :vars, :labels) do
|
|
7
|
+
def with_func(key)
|
|
8
|
+
Scope.new(funcs.dup.add(key), vars, labels)
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def with_vars(names)
|
|
12
|
+
Scope.new(funcs, vars | names, labels)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def with_label(name)
|
|
16
|
+
Scope.new(funcs, vars, labels.dup.add(name))
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
GLOBAL_VARS = %w[ENV __loc__].freeze
|
|
21
|
+
|
|
22
|
+
def initialize(known_funcs)
|
|
23
|
+
@known = known_funcs
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def check(ast, vars)
|
|
27
|
+
walk(ast, Scope.new(Set.new, Set.new(vars + GLOBAL_VARS), Set.new))
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
private
|
|
31
|
+
|
|
32
|
+
def walk(node, s)
|
|
33
|
+
return if node.nil?
|
|
34
|
+
|
|
35
|
+
case node[0]
|
|
36
|
+
when :identity, :literal, :format, :loc
|
|
37
|
+
nil
|
|
38
|
+
when :index, :pipe, :comma, :and, :or, :alt
|
|
39
|
+
walk(node[1], s)
|
|
40
|
+
walk(node[2], s)
|
|
41
|
+
when :binop
|
|
42
|
+
walk(node[2], s)
|
|
43
|
+
walk(node[3], s)
|
|
44
|
+
when :assign
|
|
45
|
+
walk(node[2], s)
|
|
46
|
+
walk(node[3], s)
|
|
47
|
+
when :iterate, :neg, :array
|
|
48
|
+
walk(node[1], s)
|
|
49
|
+
when :slice
|
|
50
|
+
walk(node[1], s)
|
|
51
|
+
walk(node[2], s)
|
|
52
|
+
walk(node[3], s)
|
|
53
|
+
when :if
|
|
54
|
+
walk(node[1], s)
|
|
55
|
+
walk(node[2], s)
|
|
56
|
+
walk(node[3], s)
|
|
57
|
+
when :try
|
|
58
|
+
walk(node[1], s)
|
|
59
|
+
walk(node[2], s)
|
|
60
|
+
when :var
|
|
61
|
+
raise CompileError, "$#{node[1]} is not defined" unless s.vars.include?(node[1])
|
|
62
|
+
when :object
|
|
63
|
+
node[1].each do |k, v|
|
|
64
|
+
raise CompileError, "Cannot use #{Value.desc(k[1])} as object key" if k[0] == :literal && !k[1].is_a?(String)
|
|
65
|
+
|
|
66
|
+
walk(k, s)
|
|
67
|
+
walk(v, s)
|
|
68
|
+
end
|
|
69
|
+
when :string
|
|
70
|
+
node[1].each { |p| walk(p, s) unless p.is_a?(String) }
|
|
71
|
+
when :call
|
|
72
|
+
key = "#{node[1]}/#{node[2].size}"
|
|
73
|
+
raise CompileError, "#{key} is not defined" unless s.funcs.include?(key) || @known.include?(key)
|
|
74
|
+
|
|
75
|
+
node[2].each { |a| walk(a, s) }
|
|
76
|
+
when :func_def
|
|
77
|
+
_, name, params, body, rest = node
|
|
78
|
+
key = "#{name}/#{params.size}"
|
|
79
|
+
fs = s.with_func(key)
|
|
80
|
+
bs = fs
|
|
81
|
+
params.each do |p|
|
|
82
|
+
bs = if p.start_with?('$')
|
|
83
|
+
bs.with_func("#{p[1..]}/0").with_vars([p[1..]])
|
|
84
|
+
else
|
|
85
|
+
bs.with_func("#{p}/0")
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
walk(body, bs)
|
|
89
|
+
walk(rest, fs)
|
|
90
|
+
when :as
|
|
91
|
+
_, src, patterns, body = node
|
|
92
|
+
walk(src, s)
|
|
93
|
+
walk(body, s.with_vars(check_patterns(patterns, s)))
|
|
94
|
+
when :reduce
|
|
95
|
+
_, src, patterns, init, update = node
|
|
96
|
+
walk(src, s)
|
|
97
|
+
walk(init, s)
|
|
98
|
+
walk(update, s.with_vars(check_patterns(patterns, s)))
|
|
99
|
+
when :foreach
|
|
100
|
+
_, src, patterns, init, update, extract = node
|
|
101
|
+
walk(src, s)
|
|
102
|
+
walk(init, s)
|
|
103
|
+
ps = s.with_vars(check_patterns(patterns, s))
|
|
104
|
+
walk(update, ps)
|
|
105
|
+
walk(extract, ps)
|
|
106
|
+
when :label
|
|
107
|
+
walk(node[2], s.with_label(node[1]))
|
|
108
|
+
when :break
|
|
109
|
+
raise CompileError, "$*label-#{node[1]} is not defined" unless s.labels.include?(node[1])
|
|
110
|
+
else
|
|
111
|
+
raise CompileError, "unknown node #{node[0]}"
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def check_patterns(patterns, s)
|
|
116
|
+
names = []
|
|
117
|
+
patterns.each { |p| collect_pattern(p, s, names) }
|
|
118
|
+
names
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def collect_pattern(p, s, names)
|
|
122
|
+
case p[0]
|
|
123
|
+
when :pvar
|
|
124
|
+
names << p[1]
|
|
125
|
+
when :parray
|
|
126
|
+
p[1].each { |e| collect_pattern(e, s, names) }
|
|
127
|
+
when :pobject
|
|
128
|
+
p[1].each do |key, var, sub|
|
|
129
|
+
if key[0] == :literal && !key[1].is_a?(String)
|
|
130
|
+
raise CompileError, "Cannot use #{Value.desc(key[1])} as object key"
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
names << var if var
|
|
134
|
+
walk(key, s.with_vars(names))
|
|
135
|
+
collect_pattern(sub, s, names) if sub
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
end
|
|
140
|
+
end
|
data/lib/rbjq/cli.rb
ADDED
|
@@ -0,0 +1,519 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'rbjq'
|
|
4
|
+
|
|
5
|
+
module Rbjq
|
|
6
|
+
# jq-compatible command line interface.
|
|
7
|
+
class CLI
|
|
8
|
+
USAGE = <<~TXT
|
|
9
|
+
Usage:\trbjq [OPTIONS] FILTER [FILES...]
|
|
10
|
+
\trbjq [OPTIONS] --args FILTER [ARGUMENTS...]
|
|
11
|
+
\trbjq [OPTIONS] --jsonargs FILTER [JSON_VALUES...]
|
|
12
|
+
|
|
13
|
+
rbjq is a pure Ruby implementation of jq, a tool for processing JSON inputs.
|
|
14
|
+
|
|
15
|
+
Example:
|
|
16
|
+
|
|
17
|
+
\t$ echo '{"foo": 0}' | rbjq .
|
|
18
|
+
\t{
|
|
19
|
+
\t "foo": 0
|
|
20
|
+
\t}
|
|
21
|
+
|
|
22
|
+
Command options:
|
|
23
|
+
-n, --null-input use `null` as the single input value;
|
|
24
|
+
-R, --raw-input read each line as string instead of JSON;
|
|
25
|
+
-s, --slurp read all inputs into an array and use it as
|
|
26
|
+
the single input value;
|
|
27
|
+
-c, --compact-output compact instead of pretty-printed output;
|
|
28
|
+
-r, --raw-output output strings without escapes and quotes;
|
|
29
|
+
--raw-output0 implies -r and output NUL after each output;
|
|
30
|
+
-j, --join-output implies -r and output without newline after
|
|
31
|
+
each output;
|
|
32
|
+
-a, --ascii-output output strings by only ASCII characters
|
|
33
|
+
using escape sequences;
|
|
34
|
+
-S, --sort-keys sort keys of each object on output;
|
|
35
|
+
-C, --color-output colorize JSON output;
|
|
36
|
+
-M, --monochrome-output disable colored output;
|
|
37
|
+
--tab use tabs for indentation;
|
|
38
|
+
--indent n use n spaces for indentation (max 7 spaces);
|
|
39
|
+
--unbuffered flush output stream after each output;
|
|
40
|
+
--stream parse the input value in streaming fashion;
|
|
41
|
+
--seq parse input/output as application/json-seq;
|
|
42
|
+
-f, --from-file file load filter from the file;
|
|
43
|
+
--arg name value set $name to the string value;
|
|
44
|
+
--argjson name value set $name to the JSON value;
|
|
45
|
+
--slurpfile name file set $name to an array of JSON values read
|
|
46
|
+
from the file;
|
|
47
|
+
--rawfile name file set $name to string contents of file;
|
|
48
|
+
--args consume remaining arguments as positional
|
|
49
|
+
string values;
|
|
50
|
+
--jsonargs consume remaining arguments as positional
|
|
51
|
+
JSON values;
|
|
52
|
+
-e, --exit-status set exit status code based on the output;
|
|
53
|
+
-V, --version show the version;
|
|
54
|
+
-h, --help show the help;
|
|
55
|
+
-- terminates argument processing;
|
|
56
|
+
TXT
|
|
57
|
+
|
|
58
|
+
def self.start(argv = ARGV, stdin: $stdin, stdout: $stdout, stderr: $stderr)
|
|
59
|
+
new(argv, stdin: stdin, stdout: stdout, stderr: stderr).run
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def initialize(argv, stdin:, stdout:, stderr:)
|
|
63
|
+
@argv = argv.dup
|
|
64
|
+
@stdin = stdin
|
|
65
|
+
@stdout = stdout
|
|
66
|
+
@stderr = stderr
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def run
|
|
70
|
+
parse_options
|
|
71
|
+
return 0 if @done
|
|
72
|
+
|
|
73
|
+
prog = compile
|
|
74
|
+
return (@no_filter ? 2 : 3) unless prog
|
|
75
|
+
|
|
76
|
+
process(prog)
|
|
77
|
+
rescue UsageError => e
|
|
78
|
+
@stderr.puts "rbjq: #{e.message}"
|
|
79
|
+
@stderr.puts 'Use rbjq --help for help with command-line options,'
|
|
80
|
+
@stderr.puts 'or see the jq manpage, or online docs at https://jqlang.org'
|
|
81
|
+
2
|
|
82
|
+
rescue Errno::EPIPE # must come before SystemCallError, its superclass
|
|
83
|
+
0
|
|
84
|
+
rescue SystemCallError => e
|
|
85
|
+
@stderr.puts "rbjq: error: Could not open #{e.message}"
|
|
86
|
+
2
|
|
87
|
+
rescue Interrupt
|
|
88
|
+
130
|
|
89
|
+
rescue SystemStackError
|
|
90
|
+
@stderr.puts 'rbjq: error: stack overflow (the program recursed too deeply)'
|
|
91
|
+
5
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
class UsageError < StandardError; end
|
|
95
|
+
|
|
96
|
+
private
|
|
97
|
+
|
|
98
|
+
def parse_options
|
|
99
|
+
@opts = {
|
|
100
|
+
null_input: false, raw_input: false, slurp: false, compact: false, raw: false, raw0: false,
|
|
101
|
+
join: false, ascii: false, sort_keys: false, color: nil, tab: false, indent: 2,
|
|
102
|
+
exit_status: false, seq: false, stream: false, unbuffered: false
|
|
103
|
+
}
|
|
104
|
+
@named = {}
|
|
105
|
+
@positional = []
|
|
106
|
+
@filter = nil
|
|
107
|
+
@files = []
|
|
108
|
+
@from_file = nil
|
|
109
|
+
@rest_kind = nil
|
|
110
|
+
args = @argv
|
|
111
|
+
until args.empty?
|
|
112
|
+
a = args.shift
|
|
113
|
+
if a == '--'
|
|
114
|
+
rest = args.dup
|
|
115
|
+
args.clear
|
|
116
|
+
rest.each { |r| positional_arg(r) }
|
|
117
|
+
next
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
if !a.match?(/\A-(?:-|[A-Za-z]|\z)/)
|
|
121
|
+
positional_arg(a)
|
|
122
|
+
elsif a.start_with?('--')
|
|
123
|
+
case a
|
|
124
|
+
when '--null-input' then @opts[:null_input] = true
|
|
125
|
+
when '--raw-input' then @opts[:raw_input] = true
|
|
126
|
+
when '--slurp' then @opts[:slurp] = true
|
|
127
|
+
when '--compact-output' then @opts[:compact] = true
|
|
128
|
+
when '--raw-output' then @opts[:raw] = true
|
|
129
|
+
when '--raw-output0' then @opts[:raw] = @opts[:raw0] = true
|
|
130
|
+
when '--join-output' then @opts[:raw] = @opts[:join] = true
|
|
131
|
+
when '--ascii-output' then @opts[:ascii] = true
|
|
132
|
+
when '--sort-keys' then @opts[:sort_keys] = true
|
|
133
|
+
when '--color-output' then @opts[:color] = true
|
|
134
|
+
when '--monochrome-output' then @opts[:color] = false
|
|
135
|
+
when '--tab' then @opts[:tab] = true
|
|
136
|
+
when '--indent'
|
|
137
|
+
n = args.shift
|
|
138
|
+
unless n&.match?(/\A-?\d+\z/) && n.to_i.between?(-1, 7)
|
|
139
|
+
raise UsageError, '--indent takes a number between -1 and 7'
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
@opts[:indent] = n.to_i
|
|
143
|
+
when '--unbuffered' then @opts[:unbuffered] = true
|
|
144
|
+
when '--stream' then @opts[:stream] = true
|
|
145
|
+
when '--seq' then @opts[:seq] = true
|
|
146
|
+
when '--exit-status' then @opts[:exit_status] = true
|
|
147
|
+
when '--from-file' then @from_file = args.shift or raise UsageError, '-f takes a parameter'
|
|
148
|
+
when '--arg'
|
|
149
|
+
name, value = take2(args, '--arg')
|
|
150
|
+
@named[name] = value
|
|
151
|
+
when '--argjson'
|
|
152
|
+
name, text = take2(args, '--argjson')
|
|
153
|
+
begin
|
|
154
|
+
@named[name] = JSONParser.parse(text)
|
|
155
|
+
rescue ParseError
|
|
156
|
+
raise UsageError, 'invalid JSON text passed to --argjson'
|
|
157
|
+
end
|
|
158
|
+
when '--slurpfile', '--rawfile'
|
|
159
|
+
name, file = take2(args, a)
|
|
160
|
+
data = read_utf8(file)
|
|
161
|
+
@named[name] = a == '--rawfile' ? data : JSONParser.parse_all(data)
|
|
162
|
+
when '--args' then @rest_kind = :string
|
|
163
|
+
when '--jsonargs' then @rest_kind = :json
|
|
164
|
+
when '--help'
|
|
165
|
+
@stdout.puts USAGE
|
|
166
|
+
@done = true
|
|
167
|
+
return
|
|
168
|
+
when '--version'
|
|
169
|
+
@stdout.puts "rbjq-#{VERSION}"
|
|
170
|
+
@done = true
|
|
171
|
+
return
|
|
172
|
+
when '--debug-dump-disasm', '--debug-trace' then nil
|
|
173
|
+
else
|
|
174
|
+
raise UsageError, "Unknown option #{a}"
|
|
175
|
+
end
|
|
176
|
+
elsif a.length > 1
|
|
177
|
+
shorts = a[1..].chars
|
|
178
|
+
shorts.each do |c|
|
|
179
|
+
case c
|
|
180
|
+
when 'n' then @opts[:null_input] = true
|
|
181
|
+
when 'R' then @opts[:raw_input] = true
|
|
182
|
+
when 's' then @opts[:slurp] = true
|
|
183
|
+
when 'c' then @opts[:compact] = true
|
|
184
|
+
when 'r' then @opts[:raw] = true
|
|
185
|
+
when 'j' then @opts[:raw] = @opts[:join] = true
|
|
186
|
+
when 'a' then @opts[:ascii] = true
|
|
187
|
+
when 'S' then @opts[:sort_keys] = true
|
|
188
|
+
when 'C' then @opts[:color] = true
|
|
189
|
+
when 'M' then @opts[:color] = false
|
|
190
|
+
when 'e' then @opts[:exit_status] = true
|
|
191
|
+
when 'f' then @from_file = args.shift or raise UsageError, '-f takes a parameter'
|
|
192
|
+
when 'L' then args.shift
|
|
193
|
+
when 'h'
|
|
194
|
+
@stdout.puts USAGE
|
|
195
|
+
@done = true
|
|
196
|
+
return # rubocop:disable Lint/NonLocalExitFromIterator
|
|
197
|
+
when 'V'
|
|
198
|
+
@stdout.puts "rbjq-#{VERSION}"
|
|
199
|
+
@done = true
|
|
200
|
+
return # rubocop:disable Lint/NonLocalExitFromIterator
|
|
201
|
+
else
|
|
202
|
+
raise UsageError, "Unknown option #{a}"
|
|
203
|
+
end
|
|
204
|
+
end
|
|
205
|
+
else
|
|
206
|
+
positional_arg(a)
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
def take2(args, opt)
|
|
212
|
+
raise UsageError, "#{opt} takes two parameters (e.g. #{opt} varname value)" if args.size < 2
|
|
213
|
+
|
|
214
|
+
[args.shift, args.shift]
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
def positional_arg(a)
|
|
218
|
+
if @filter.nil? && @from_file.nil?
|
|
219
|
+
@filter = a
|
|
220
|
+
elsif @rest_kind
|
|
221
|
+
add_positional(a, @rest_kind)
|
|
222
|
+
else
|
|
223
|
+
@files << a
|
|
224
|
+
end
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
def add_positional(a, kind)
|
|
228
|
+
if kind == :json
|
|
229
|
+
begin
|
|
230
|
+
@positional << JSONParser.parse(a)
|
|
231
|
+
rescue ParseError
|
|
232
|
+
raise UsageError, 'invalid JSON text passed to --jsonargs'
|
|
233
|
+
end
|
|
234
|
+
else
|
|
235
|
+
@positional << a
|
|
236
|
+
end
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
def compile
|
|
240
|
+
if @from_file
|
|
241
|
+
src = read_utf8(@from_file)
|
|
242
|
+
# with -f, the first positional (if any) is an input file
|
|
243
|
+
@files.unshift(@filter) if @filter
|
|
244
|
+
@filter = src
|
|
245
|
+
end
|
|
246
|
+
# like jq, the filter defaults to "." unless both ends are terminals
|
|
247
|
+
@filter = '.' if @filter.nil? && !(tty?(@stdin) && tty?(@stdout))
|
|
248
|
+
if @filter.nil?
|
|
249
|
+
@stderr.puts USAGE
|
|
250
|
+
@no_filter = true
|
|
251
|
+
return nil
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
vars = @named.dup
|
|
255
|
+
vars['ARGS'] = { 'positional' => @positional, 'named' => @named.dup }
|
|
256
|
+
@current_file = nil
|
|
257
|
+
@current_line = 0
|
|
258
|
+
Program.new(@filter, vars: vars, stderr: @stderr,
|
|
259
|
+
filename_proc: -> { @read_any ? (@current_file || '<stdin>') : nil },
|
|
260
|
+
line_proc: -> { @current_line },
|
|
261
|
+
jq_origin: File.expand_path('../../exe', __dir__),
|
|
262
|
+
prog_origin: @from_file ? File.dirname(File.expand_path(@from_file)) : Dir.pwd)
|
|
263
|
+
rescue CompileError => e
|
|
264
|
+
@stderr.puts "rbjq: error: #{e.message}"
|
|
265
|
+
@stderr.puts 'rbjq: 1 compile error'
|
|
266
|
+
nil
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
# ------------------------------------------------------------ inputs
|
|
270
|
+
|
|
271
|
+
def sources
|
|
272
|
+
if @files.empty?
|
|
273
|
+
[[nil, @stdin]]
|
|
274
|
+
else
|
|
275
|
+
@files.map { |f| [f, nil] }
|
|
276
|
+
end
|
|
277
|
+
end
|
|
278
|
+
|
|
279
|
+
def read_utf8(path)
|
|
280
|
+
Value.scrub_utf8(File.binread(path))
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
def read_source(name, io)
|
|
284
|
+
if io
|
|
285
|
+
io.binmode if io.respond_to?(:binmode)
|
|
286
|
+
io.read
|
|
287
|
+
else
|
|
288
|
+
File.binread(name)
|
|
289
|
+
end
|
|
290
|
+
rescue SystemCallError => e
|
|
291
|
+
@stderr.puts "rbjq: error: Could not open file #{name}: #{SystemCallError.new(nil, e.errno).message}"
|
|
292
|
+
@input_failed = true
|
|
293
|
+
''
|
|
294
|
+
end
|
|
295
|
+
|
|
296
|
+
# Returns an Enumerator of input values
|
|
297
|
+
def input_values
|
|
298
|
+
Enumerator.new do |y|
|
|
299
|
+
if @opts[:raw_input]
|
|
300
|
+
if @opts[:slurp]
|
|
301
|
+
all = +''
|
|
302
|
+
sources.each do |name, io|
|
|
303
|
+
@current_file = name
|
|
304
|
+
@read_any = true
|
|
305
|
+
all << read_source(name, io)
|
|
306
|
+
@current_line = all.count("\n")
|
|
307
|
+
end
|
|
308
|
+
y << Value.scrub_utf8(all)
|
|
309
|
+
else
|
|
310
|
+
carry = +''
|
|
311
|
+
sources.each do |name, io|
|
|
312
|
+
@current_file = name
|
|
313
|
+
data = carry + read_source(name, io).force_encoding(Encoding::UTF_8)
|
|
314
|
+
lines = data.split("\n", -1)
|
|
315
|
+
carry = lines.pop || +''
|
|
316
|
+
@read_any = true
|
|
317
|
+
@current_line = 0
|
|
318
|
+
lines.each do |l|
|
|
319
|
+
@current_line += 1
|
|
320
|
+
y << Value.scrub_utf8(l)
|
|
321
|
+
end
|
|
322
|
+
end
|
|
323
|
+
y << Value.scrub_utf8(carry) unless carry.empty?
|
|
324
|
+
end
|
|
325
|
+
elsif @opts[:slurp]
|
|
326
|
+
all = []
|
|
327
|
+
sources.each do |name, io|
|
|
328
|
+
@current_file = name
|
|
329
|
+
each_json(read_source(name, io)) { |v| all << v }
|
|
330
|
+
end
|
|
331
|
+
y << all
|
|
332
|
+
else
|
|
333
|
+
sources.each do |name, io|
|
|
334
|
+
@current_file = name
|
|
335
|
+
each_json(read_source(name, io)) { |v| y << v }
|
|
336
|
+
end
|
|
337
|
+
end
|
|
338
|
+
end
|
|
339
|
+
end
|
|
340
|
+
|
|
341
|
+
def each_json(text, &blk)
|
|
342
|
+
return each_json_seq(text, &blk) if @opts[:seq]
|
|
343
|
+
|
|
344
|
+
parser = JSONParser.new(text)
|
|
345
|
+
@read_any = true
|
|
346
|
+
@current_line = 0
|
|
347
|
+
loop do
|
|
348
|
+
v = parser.next_value
|
|
349
|
+
@current_line = parser.line_number
|
|
350
|
+
break if v.equal?(JSONParser::EOF)
|
|
351
|
+
|
|
352
|
+
if @opts[:stream]
|
|
353
|
+
stream_events(v, [], true, &blk)
|
|
354
|
+
else
|
|
355
|
+
yield v
|
|
356
|
+
end
|
|
357
|
+
end
|
|
358
|
+
end
|
|
359
|
+
|
|
360
|
+
# application/json-seq: each text is preceded by RS; text before the
|
|
361
|
+
# first RS is abandoned and a broken text is skipped, like jq.
|
|
362
|
+
def each_json_seq(text, &blk)
|
|
363
|
+
parts = text.split("\x1e", -1)
|
|
364
|
+
head = parts.shift
|
|
365
|
+
unless head.strip.empty?
|
|
366
|
+
lines = head.count("\n")
|
|
367
|
+
col = head.length - (head.rindex("\n")&.+(1) || 0)
|
|
368
|
+
@stderr.puts "rbjq: ignoring parse error: Unfinished abandoned text at EOF at line #{lines + 1}, column #{col}"
|
|
369
|
+
end
|
|
370
|
+
parts.each do |part|
|
|
371
|
+
parser = JSONParser.new(part)
|
|
372
|
+
@read_any = true
|
|
373
|
+
loop do
|
|
374
|
+
v = parser.next_value
|
|
375
|
+
break if v.equal?(JSONParser::EOF)
|
|
376
|
+
|
|
377
|
+
@opts[:stream] ? stream_events(v, [], true, &blk) : yield(v)
|
|
378
|
+
end
|
|
379
|
+
rescue ParseError => e
|
|
380
|
+
@stderr.puts "rbjq: ignoring parse error: #{e.message}"
|
|
381
|
+
end
|
|
382
|
+
end
|
|
383
|
+
|
|
384
|
+
def stream_events(v, path, _top, &blk)
|
|
385
|
+
case v
|
|
386
|
+
when Array
|
|
387
|
+
if v.empty?
|
|
388
|
+
yield [path, []]
|
|
389
|
+
else
|
|
390
|
+
v.each_with_index { |e, i| stream_events(e, path + [i], false, &blk) }
|
|
391
|
+
yield [path + [v.length - 1]]
|
|
392
|
+
end
|
|
393
|
+
when Hash
|
|
394
|
+
if v.empty?
|
|
395
|
+
yield [path, {}]
|
|
396
|
+
else
|
|
397
|
+
v.each { |k, e| stream_events(e, path + [k], false, &blk) }
|
|
398
|
+
yield [path + [v.keys.last]]
|
|
399
|
+
end
|
|
400
|
+
else
|
|
401
|
+
yield [path, v]
|
|
402
|
+
end
|
|
403
|
+
nil
|
|
404
|
+
end
|
|
405
|
+
|
|
406
|
+
# ------------------------------------------------------------ output
|
|
407
|
+
|
|
408
|
+
def tty?(io)
|
|
409
|
+
io.respond_to?(:tty?) && io.tty?
|
|
410
|
+
end
|
|
411
|
+
|
|
412
|
+
def color?
|
|
413
|
+
return @opts[:color] unless @opts[:color].nil?
|
|
414
|
+
return false if ENV['NO_COLOR'] && !ENV['NO_COLOR'].empty?
|
|
415
|
+
|
|
416
|
+
@stdout.respond_to?(:tty?) && @stdout.tty?
|
|
417
|
+
end
|
|
418
|
+
|
|
419
|
+
# --indent -1 means tabs; 0 means newlines without indentation
|
|
420
|
+
def encoder
|
|
421
|
+
tab = !@opts[:compact] && (@opts[:tab] || @opts[:indent] == -1)
|
|
422
|
+
indent = @opts[:indent].clamp(0, 7)
|
|
423
|
+
Encoder.new(indent: indent, tab: tab, sort_keys: @opts[:sort_keys], ascii: @opts[:ascii],
|
|
424
|
+
color: color?, colors: Encoder.parse_colors(ENV.fetch('JQ_COLORS', nil)), pretty: !@opts[:compact])
|
|
425
|
+
end
|
|
426
|
+
|
|
427
|
+
def process(prog)
|
|
428
|
+
enc = encoder
|
|
429
|
+
values = input_values
|
|
430
|
+
provider = lambda do
|
|
431
|
+
values.next
|
|
432
|
+
rescue StopIteration
|
|
433
|
+
JSONParser::EOF
|
|
434
|
+
end
|
|
435
|
+
|
|
436
|
+
status = 0
|
|
437
|
+
last_output = :none
|
|
438
|
+
error_status = nil
|
|
439
|
+
emit = lambda do |v|
|
|
440
|
+
@stdout.write("\x1e") if @opts[:seq]
|
|
441
|
+
if @opts[:raw] && v.is_a?(String)
|
|
442
|
+
# with --ascii-output, jq prints strings as JSON even with -r
|
|
443
|
+
@stdout.write(@opts[:ascii] ? Encoder.escape_string(v, true) : v)
|
|
444
|
+
else
|
|
445
|
+
@stdout.write(enc.encode(v))
|
|
446
|
+
end
|
|
447
|
+
if @opts[:raw0]
|
|
448
|
+
@stdout.write("\0")
|
|
449
|
+
elsif !@opts[:join]
|
|
450
|
+
@stdout.write("\n")
|
|
451
|
+
end
|
|
452
|
+
@stdout.flush if @opts[:unbuffered]
|
|
453
|
+
last_output = v
|
|
454
|
+
end
|
|
455
|
+
|
|
456
|
+
run_one = lambda do |input|
|
|
457
|
+
error_status = nil
|
|
458
|
+
prog.each(input, inputs: provider) { |v| emit.call(v) }
|
|
459
|
+
rescue JQError => e
|
|
460
|
+
print_error(e)
|
|
461
|
+
error_status = 5
|
|
462
|
+
end
|
|
463
|
+
|
|
464
|
+
if @opts[:null_input]
|
|
465
|
+
run_one.call(nil)
|
|
466
|
+
else
|
|
467
|
+
loop do
|
|
468
|
+
begin
|
|
469
|
+
v = provider.call
|
|
470
|
+
rescue ParseError => e
|
|
471
|
+
@stderr.puts "rbjq: parse error: #{e.message}"
|
|
472
|
+
return 5
|
|
473
|
+
end
|
|
474
|
+
break if v.equal?(JSONParser::EOF)
|
|
475
|
+
|
|
476
|
+
run_one.call(v)
|
|
477
|
+
end
|
|
478
|
+
end
|
|
479
|
+
|
|
480
|
+
return 2 if @input_failed
|
|
481
|
+
|
|
482
|
+
status = error_status if error_status
|
|
483
|
+
if @opts[:exit_status] && status.zero?
|
|
484
|
+
status = if last_output == :none
|
|
485
|
+
4
|
|
486
|
+
elsif last_output.nil? || last_output == false
|
|
487
|
+
1
|
|
488
|
+
else
|
|
489
|
+
0
|
|
490
|
+
end
|
|
491
|
+
end
|
|
492
|
+
status
|
|
493
|
+
rescue HaltError => e
|
|
494
|
+
if e.value? && !e.value.nil?
|
|
495
|
+
if e.value.is_a?(String)
|
|
496
|
+
@stderr.write(e.value)
|
|
497
|
+
else
|
|
498
|
+
@stderr.puts(Encoder.dump(e.value))
|
|
499
|
+
end
|
|
500
|
+
end
|
|
501
|
+
e.code
|
|
502
|
+
end
|
|
503
|
+
|
|
504
|
+
def location
|
|
505
|
+
return '<unknown>' unless @read_any
|
|
506
|
+
|
|
507
|
+
"#{@current_file || '<stdin>'}:#{@current_line}"
|
|
508
|
+
end
|
|
509
|
+
|
|
510
|
+
def print_error(e)
|
|
511
|
+
v = e.value
|
|
512
|
+
if v.is_a?(String)
|
|
513
|
+
@stderr.puts "rbjq: error (at #{location}): #{v}"
|
|
514
|
+
else
|
|
515
|
+
@stderr.puts "rbjq: error (at #{location}) (not a string): #{Encoder.dump(v)}"
|
|
516
|
+
end
|
|
517
|
+
end
|
|
518
|
+
end
|
|
519
|
+
end
|
data/lib/rbjq/env.rb
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Rbjq
|
|
4
|
+
# Linked-list lexical environment holding variables, functions and labels.
|
|
5
|
+
class Env
|
|
6
|
+
attr_reader :kind, :name, :parent
|
|
7
|
+
attr_accessor :value
|
|
8
|
+
|
|
9
|
+
def initialize(kind, name, value, parent)
|
|
10
|
+
@kind = kind
|
|
11
|
+
@name = name
|
|
12
|
+
@value = value
|
|
13
|
+
@parent = parent
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def self.root
|
|
17
|
+
nil
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def self.bind_var(env, name, value)
|
|
21
|
+
Env.new(:var, name, value, env)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def self.lookup(env, kind, name)
|
|
25
|
+
e = env
|
|
26
|
+
while e
|
|
27
|
+
return e if e.kind == kind && e.name == name
|
|
28
|
+
|
|
29
|
+
e = e.parent
|
|
30
|
+
end
|
|
31
|
+
nil
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# A user-defined (or prelude) function
|
|
36
|
+
FuncDef = Struct.new(:name, :params, :body, :env)
|
|
37
|
+
# A closure argument passed to a function parameter
|
|
38
|
+
Closure = Struct.new(:node, :env, :value_param)
|
|
39
|
+
end
|