interscript 3.0.1 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/interscript/ml/byt5_onnx.rb +16 -16
- data/lib/interscript/ml/imf.rb +13 -6
- data/lib/interscript/ml/model.rb +13 -13
- data/lib/interscript/ml/plane.rb +183 -0
- data/lib/interscript/ml/provisioning.rb +19 -19
- data/lib/interscript/ml/translator.rb +5 -5
- data/lib/interscript/ml/vocab.rb +1 -2
- data/lib/interscript/ml.rb +3 -2
- data/lib/interscript/stdlib/functions/secryst_adapter.rb +0 -2
- data/lib/interscript/version.rb +1 -1
- metadata +4 -7
- data/lib/interscript/isc/grammar/isc.artifact.json +0 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 1d538b79aebc29b3d440b857ebf621c568660e6bddb3ae08234c2bb73392f35d
|
|
4
|
+
data.tar.gz: 59cfb7c5200fbe20dd1def58b5de0fda7a71642e45beb4754f986763f47152e9
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 269077f6aa5a46641dd013ea2c197b262298cc14be8e3f23648a54f64b385529e68bb838b0a5533275f22735ac8c495ae2fddd10a2cf8dfbdad882c0e84a121a
|
|
7
|
+
data.tar.gz: 650a1ba60637f312c23bc015a6aa439ee73b63858121958c500c99c95655a9323eba8ee16a72c58c7a2e2786b421a40257e74864ed6c8a4ad74c3c9042bdd082
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
require
|
|
1
|
+
require "tmpdir"
|
|
2
2
|
|
|
3
3
|
module Interscript::ML
|
|
4
4
|
# Byte-level seq2seq (ByT5 family) inference over ONNX sessions from an
|
|
@@ -11,24 +11,24 @@ module Interscript::ML
|
|
|
11
11
|
Interscript::ML.require_optional!("onnxruntime")
|
|
12
12
|
@manifest = IMF.manifest(zip_path)
|
|
13
13
|
graphs = IMF.verify_and_read(zip_path)
|
|
14
|
-
@tmpdir = Dir.mktmpdir(
|
|
15
|
-
@encoder = build_session(graphs,
|
|
16
|
-
@kv = @manifest[
|
|
17
|
-
@decoder = build_session(graphs, @kv ?
|
|
14
|
+
@tmpdir = Dir.mktmpdir("secryst-imf")
|
|
15
|
+
@encoder = build_session(graphs, "encoder.onnx")
|
|
16
|
+
@kv = @manifest["decoder"] == "kv" && graphs.key?("decoder-kv.onnx")
|
|
17
|
+
@decoder = build_session(graphs, @kv ? "decoder-kv.onnx" : "decoder.onnx")
|
|
18
18
|
@pasts = @kv ? zero_pasts : {}
|
|
19
19
|
end
|
|
20
20
|
|
|
21
21
|
def translate(text, max_seq_length: 256)
|
|
22
22
|
ids = IMF.encode(text)
|
|
23
|
-
return
|
|
23
|
+
return "" if ids.length == 1 # only the trailing EOS: empty input
|
|
24
24
|
|
|
25
|
-
hidden = @encoder.predict({
|
|
25
|
+
hidden = @encoder.predict({input_ids: [ids]})["last_hidden_state"]
|
|
26
26
|
tokens = @kv ? greedy_kv(hidden, max_seq_length) : greedy_plain(hidden, max_seq_length)
|
|
27
|
-
IMF.decode(tokens)
|
|
27
|
+
IMF.first_alternates(IMF.decode(tokens))
|
|
28
28
|
end
|
|
29
29
|
|
|
30
30
|
def id
|
|
31
|
-
@manifest[
|
|
31
|
+
@manifest["id"]
|
|
32
32
|
end
|
|
33
33
|
|
|
34
34
|
private
|
|
@@ -43,7 +43,7 @@ module Interscript::ML
|
|
|
43
43
|
|
|
44
44
|
def zero_pasts
|
|
45
45
|
@decoder.inputs.each_with_object({}) do |meta, pasts|
|
|
46
|
-
next unless meta[:name].start_with?(
|
|
46
|
+
next unless meta[:name].start_with?("past_")
|
|
47
47
|
shape = meta[:shape] # [batch, heads, past_seq, d_kv]; dynamic dims are Strings
|
|
48
48
|
heads = shape[1].is_a?(Integer) ? shape[1] : 4
|
|
49
49
|
d_kv = shape[3].is_a?(Integer) ? shape[3] : 8
|
|
@@ -71,13 +71,13 @@ module Interscript::ML
|
|
|
71
71
|
generated = []
|
|
72
72
|
max_seq_length.times do
|
|
73
73
|
results = @decoder.predict(
|
|
74
|
-
{
|
|
74
|
+
{input_ids: [current], encoder_hidden_states: hidden}.merge(pasts)
|
|
75
75
|
)
|
|
76
|
-
token = argmax(results[
|
|
76
|
+
token = argmax(results["logits"].first.last)
|
|
77
77
|
break if token == IMF::EOS_ID
|
|
78
78
|
|
|
79
79
|
generated << token
|
|
80
|
-
pasts = pasts.keys.to_h { |name| [name, results[name.sub(
|
|
80
|
+
pasts = pasts.keys.to_h { |name| [name, results[name.sub("past_", "present_")]] }
|
|
81
81
|
current = [token]
|
|
82
82
|
end
|
|
83
83
|
generated
|
|
@@ -88,13 +88,13 @@ module Interscript::ML
|
|
|
88
88
|
generated = []
|
|
89
89
|
max_seq_length.times do
|
|
90
90
|
logits = @decoder.predict(
|
|
91
|
-
{
|
|
92
|
-
)[
|
|
91
|
+
{input_ids: [decoder_ids], encoder_hidden_states: hidden}
|
|
92
|
+
)["logits"]
|
|
93
93
|
token = argmax(logits.first.last)
|
|
94
94
|
break if token == IMF::EOS_ID
|
|
95
95
|
|
|
96
96
|
generated << token
|
|
97
|
-
decoder_ids
|
|
97
|
+
decoder_ids += [token]
|
|
98
98
|
end
|
|
99
99
|
generated
|
|
100
100
|
end
|
data/lib/interscript/ml/imf.rb
CHANGED
|
@@ -14,7 +14,7 @@ module Interscript::ML
|
|
|
14
14
|
EOS_ID = 1
|
|
15
15
|
UNK_ID = 2
|
|
16
16
|
|
|
17
|
-
DEFAULT_INDEX_URL = "https://github.com/interscript/interscript-models/releases/download/index-
|
|
17
|
+
DEFAULT_INDEX_URL = "https://github.com/interscript/interscript-models/releases/download/index-v10/models-index.yaml"
|
|
18
18
|
|
|
19
19
|
class FormatError < StandardError; end
|
|
20
20
|
class RegistryError < StandardError; end
|
|
@@ -28,6 +28,13 @@ module Interscript::ML
|
|
|
28
28
|
text.bytes.map { |b| b + BYTE_OFFSET } + [EOS_ID]
|
|
29
29
|
end
|
|
30
30
|
|
|
31
|
+
# Seq2seq output contract: models trained on multi-reference
|
|
32
|
+
# corpora emit '/'-separated alternates per token; the runtime
|
|
33
|
+
# returns the primary choice only.
|
|
34
|
+
def first_alternates(text)
|
|
35
|
+
text.split.map { |tok| tok.split("/", 2).first }.join(" ")
|
|
36
|
+
end
|
|
37
|
+
|
|
31
38
|
def decode(token_ids)
|
|
32
39
|
out = +""
|
|
33
40
|
token_ids.each do |token|
|
|
@@ -105,7 +112,7 @@ module Interscript::ML
|
|
|
105
112
|
if channel.start_with?("file://")
|
|
106
113
|
FileUtils.cp(channel.sub(%r{\Afile://}, ""), tmp)
|
|
107
114
|
else
|
|
108
|
-
URI.
|
|
115
|
+
URI.parse(channel).open { |remote| IO.copy_stream(remote, tmp) }
|
|
109
116
|
end
|
|
110
117
|
end
|
|
111
118
|
actual = Digest::SHA256.file(tmp).hexdigest
|
|
@@ -133,14 +140,14 @@ module Interscript::ML
|
|
|
133
140
|
if part["url"].start_with?("file://")
|
|
134
141
|
File.open(source, "rb", &open_stream)
|
|
135
142
|
else
|
|
136
|
-
URI.open(
|
|
143
|
+
URI.parse(source).open("rb", &open_stream)
|
|
137
144
|
end
|
|
138
145
|
unless digest.hexdigest == part["sha256"]
|
|
139
146
|
raise RegistryError, "part #{index} of #{entry["filename"]} sha256 mismatch: got #{digest.hexdigest}, index says #{part["sha256"]}"
|
|
140
147
|
end
|
|
141
148
|
end
|
|
142
149
|
end
|
|
143
|
-
rescue
|
|
150
|
+
rescue
|
|
144
151
|
File.delete(tmp) if File.file?(tmp)
|
|
145
152
|
raise
|
|
146
153
|
end
|
|
@@ -160,10 +167,10 @@ module Interscript::ML
|
|
|
160
167
|
# HTTP sources must ship a sibling .sha256 sidecar; verify the
|
|
161
168
|
# body against it before parsing (GitHub Releases index assets do).
|
|
162
169
|
def fetch_index_with_sidecar(source)
|
|
163
|
-
body = URI.open(
|
|
170
|
+
body = URI.parse(source).open("rb") { |remote| remote.read }
|
|
164
171
|
sidecar =
|
|
165
172
|
begin
|
|
166
|
-
URI.
|
|
173
|
+
URI.parse("#{source}.sha256").open { |remote| remote.read }
|
|
167
174
|
rescue OpenURI::HTTPError => e
|
|
168
175
|
raise RegistryError, "index sha256 sidecar missing: #{source}.sha256 (#{e.message})"
|
|
169
176
|
end
|
data/lib/interscript/ml/model.rb
CHANGED
|
@@ -5,30 +5,30 @@ module Interscript::ML
|
|
|
5
5
|
def self.from_file(model_file)
|
|
6
6
|
# A models.yaml model id resolves (download -> verify -> cache)
|
|
7
7
|
# through the same contract as the Python/TS runtimes.
|
|
8
|
-
model_file = IMF.resolve(model_file) unless model_file.to_s.end_with?(
|
|
8
|
+
model_file = IMF.resolve(model_file) unless model_file.to_s.end_with?(".zip") || File.file?(model_file.to_s)
|
|
9
9
|
model_file = Provisioning.locate(model_file)
|
|
10
10
|
|
|
11
11
|
Zip::File.open(model_file) do |zip_file|
|
|
12
|
-
metadata = zip_file.glob(
|
|
12
|
+
metadata = zip_file.glob("metadata.yaml").first
|
|
13
13
|
metadata = YAML.safe_load(metadata.get_input_stream.read) if metadata
|
|
14
14
|
|
|
15
15
|
# IMF v1: the Interscript Model Format zip.
|
|
16
|
-
return Byt5Onnx.new(model_file) if metadata && metadata[
|
|
16
|
+
return Byt5Onnx.new(model_file) if metadata && metadata["format"] == "imf-v1"
|
|
17
17
|
|
|
18
|
-
name = metadata && metadata[
|
|
18
|
+
name = metadata && metadata["name"]
|
|
19
19
|
|
|
20
20
|
# Modern byte-level seq2seq (ByT5 family): encoder.onnx + decoder.onnx.
|
|
21
|
-
return Byt5Onnx.new(model_file) if name ==
|
|
21
|
+
return Byt5Onnx.new(model_file) if name == "byt5"
|
|
22
22
|
|
|
23
23
|
# Legacy single-file ONNX transformer zips (vocabs.yaml based).
|
|
24
|
-
vocabs = zip_file.glob(
|
|
25
|
-
raise
|
|
24
|
+
vocabs = zip_file.glob("vocabs.yaml").first
|
|
25
|
+
raise "vocabs.yaml is missing in model zip!" unless vocabs
|
|
26
26
|
vocabs = YAML.safe_load(vocabs.get_input_stream.read)
|
|
27
|
-
input_vocab = Vocab.new(vocabs[
|
|
28
|
-
target_vocab = Vocab.new(vocabs[
|
|
27
|
+
input_vocab = Vocab.new(vocabs["input"], specials: [])
|
|
28
|
+
target_vocab = Vocab.new(vocabs["target"], specials: [])
|
|
29
29
|
|
|
30
|
-
onnx = zip_file.glob(
|
|
31
|
-
raise
|
|
30
|
+
onnx = zip_file.glob("*.onnx").first
|
|
31
|
+
raise "onnx model file is missing in model zip!" unless onnx
|
|
32
32
|
Onnx.new(onnx.get_input_stream.read, input_vocab, target_vocab)
|
|
33
33
|
end
|
|
34
34
|
end
|
|
@@ -41,11 +41,11 @@ module Interscript::ML
|
|
|
41
41
|
end
|
|
42
42
|
|
|
43
43
|
def call(input, output, opts)
|
|
44
|
-
@model.predict({
|
|
44
|
+
@model.predict({src: input, tgt: output}.merge(opts))["output"]
|
|
45
45
|
end
|
|
46
46
|
|
|
47
47
|
def argmax(*args)
|
|
48
|
-
|
|
48
|
+
call(*args).map { |i| i.flatten.each_with_index.max[1] }
|
|
49
49
|
end
|
|
50
50
|
end
|
|
51
51
|
end
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "tmpdir"
|
|
4
|
+
require "json"
|
|
5
|
+
|
|
6
|
+
module Interscript
|
|
7
|
+
module ML
|
|
8
|
+
# Plane-factorized runtime (kind=plane artifacts): one ONNX graph
|
|
9
|
+
# maps (input_ids, plane_ids) -> per-position class logits over the
|
|
10
|
+
# diacritic classes; decoding is K mask-predict passes feeding the
|
|
11
|
+
# argmax plane back. Per-character class = majority vote of the
|
|
12
|
+
# char's byte-token predictions; render splices one mark after each
|
|
13
|
+
# base character. Token ids follow the canonical ByT5 table (byte+3,
|
|
14
|
+
# trailing EOS) — see Interscript::ML::IMF.
|
|
15
|
+
class PlaneModel
|
|
16
|
+
attr_reader :classes, :k_passes
|
|
17
|
+
|
|
18
|
+
def initialize(graph:, classes:, k_passes: 2)
|
|
19
|
+
Interscript::ML.require_optional!("onnxruntime")
|
|
20
|
+
@classes = classes.freeze
|
|
21
|
+
@n_classes = classes.length
|
|
22
|
+
@mask_id = @n_classes
|
|
23
|
+
@k_passes = k_passes
|
|
24
|
+
# a character is a diacritic iff it occurs inside some class the
|
|
25
|
+
# artifact knows — no per-language mark tables needed
|
|
26
|
+
@mark_chars = classes.flat_map { |c| c.chars }.reject(&:empty?).uniq.freeze
|
|
27
|
+
@class_to_id = classes.each_with_index.to_h.freeze
|
|
28
|
+
# the onnxruntime gem loads from paths, not bytes (as byt5_onnx):
|
|
29
|
+
# verified bytes go to a tmpfile owned by this instance
|
|
30
|
+
@graph_tmpdir = Dir.mktmpdir("isx-plane")
|
|
31
|
+
path = File.join(@graph_tmpdir, "plane.onnx")
|
|
32
|
+
File.binwrite(path, graph)
|
|
33
|
+
@sess = OnnxRuntime::Model.new(path)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Split text into (skeleton, per-base classes). Marks FOLLOW their
|
|
37
|
+
# base letter; leading marks bind to a "\x00" anchor (render drops
|
|
38
|
+
# it). Cluster order is preserved as written — render-exactness
|
|
39
|
+
# beats normalization (the Hebrew canon lesson).
|
|
40
|
+
def split_planes(text)
|
|
41
|
+
skeleton = []
|
|
42
|
+
classes = []
|
|
43
|
+
current = []
|
|
44
|
+
close = lambda do
|
|
45
|
+
next if current.empty?
|
|
46
|
+
|
|
47
|
+
if skeleton.empty?
|
|
48
|
+
skeleton << "\x00"
|
|
49
|
+
classes << current.join
|
|
50
|
+
else
|
|
51
|
+
classes[-1] = classes[-1] + current.join
|
|
52
|
+
end
|
|
53
|
+
current = []
|
|
54
|
+
end
|
|
55
|
+
text.each_char do |ch|
|
|
56
|
+
if @mark_chars.include?(ch)
|
|
57
|
+
current << ch
|
|
58
|
+
else
|
|
59
|
+
close.call
|
|
60
|
+
skeleton << ch
|
|
61
|
+
classes << ""
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
close.call
|
|
65
|
+
[skeleton.join, classes]
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def translate(text, preserve_diacritics: false)
|
|
69
|
+
return translate_plain(text) unless preserve_diacritics
|
|
70
|
+
|
|
71
|
+
skeleton, char_classes = split_planes(text)
|
|
72
|
+
tok_preds = decode_pinned(skeleton, char_classes)
|
|
73
|
+
pos = 0
|
|
74
|
+
final = char_classes.each_with_index.map do |user_cls, i|
|
|
75
|
+
ch = skeleton[i]
|
|
76
|
+
n = ch.bytesize
|
|
77
|
+
cls = if user_cls.empty?
|
|
78
|
+
votes = tok_preds[pos, n] || []
|
|
79
|
+
cid = votes.tally.max_by { |_, c| c }&.first || @n_classes
|
|
80
|
+
(cid < @n_classes) ? @classes[cid] : ""
|
|
81
|
+
else
|
|
82
|
+
user_cls # user diacritics round-trip byte-exactly
|
|
83
|
+
end
|
|
84
|
+
pos += n
|
|
85
|
+
cls
|
|
86
|
+
end
|
|
87
|
+
out = +""
|
|
88
|
+
skeleton.each_char.with_index do |ch, i|
|
|
89
|
+
out << ((ch == "\x00") ? final[i] : ch + final[i])
|
|
90
|
+
end
|
|
91
|
+
out
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
private
|
|
95
|
+
|
|
96
|
+
def translate_plain(text)
|
|
97
|
+
ids = IMF.encode(text)[0..-2] # strip EOS: one token per byte
|
|
98
|
+
preds = k_pass_decode(ids, Array.new(ids.length, @mask_id))
|
|
99
|
+
pos = 0
|
|
100
|
+
char_preds = text.each_char.map do |ch|
|
|
101
|
+
nbytes = ch.bytesize
|
|
102
|
+
votes = preds[pos, nbytes] || []
|
|
103
|
+
cid = votes.tally.max_by { |_, c| c }&.first || @n_classes
|
|
104
|
+
pos += nbytes
|
|
105
|
+
(cid < @n_classes) ? @classes[cid] : ""
|
|
106
|
+
end
|
|
107
|
+
PlaneModel.render_plane(text, char_preds)
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def k_pass_decode(ids, plane0)
|
|
111
|
+
plane = plane0
|
|
112
|
+
preds = nil
|
|
113
|
+
@k_passes.times do
|
|
114
|
+
logits = @sess.predict({"input_ids" => [ids], "plane_ids" => [plane]})["class_logits"][0]
|
|
115
|
+
preds = logits.each_with_index.map { |row, i| row.index(row.max) }
|
|
116
|
+
plane = preds
|
|
117
|
+
end
|
|
118
|
+
preds
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# K-pass decode with user classes pinned: known classes are
|
|
122
|
+
# conditioned AND never overwritten; unknown non-empty clusters
|
|
123
|
+
# stay masked during decode but are restored at render.
|
|
124
|
+
def decode_pinned(skeleton, char_classes)
|
|
125
|
+
ids = IMF.encode(skeleton)[0..-2]
|
|
126
|
+
per_token_pin = []
|
|
127
|
+
skeleton.each_char.with_index do |ch, i|
|
|
128
|
+
cls = char_classes[i]
|
|
129
|
+
pid = @class_to_id.fetch(cls, @mask_id)
|
|
130
|
+
ch.bytesize.times { per_token_pin << pid }
|
|
131
|
+
end
|
|
132
|
+
plane0 = per_token_pin.map { |p| (p == @mask_id) ? @mask_id : p }
|
|
133
|
+
k_pass_decode(ids, plane0)
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def self.render_plane(skeleton, char_classes) # rubocop:disable Lint/IneffectiveAccessModifier
|
|
137
|
+
skeleton.each_char.with_index.map do |ch, i|
|
|
138
|
+
ch + char_classes[i].to_s
|
|
139
|
+
end.join
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# The zip contract: metadata.yaml (kind=plane, k_passes, member
|
|
143
|
+
# sha256s) + plane.onnx + classes.json, every member sha-verified
|
|
144
|
+
# before the session is created.
|
|
145
|
+
# rubocop:disable Lint/IneffectiveAccessModifier
|
|
146
|
+
def self.from_zip(data)
|
|
147
|
+
require "zip"
|
|
148
|
+
zf = Zip::File.open_buffer(data)
|
|
149
|
+
names = zf.entries.map(&:name)
|
|
150
|
+
meta = YAML.safe_load(zf.read("metadata.yaml"), permitted_classes: [], aliases: false)
|
|
151
|
+
raise IMF::FormatError, "plane zip: kind=#{meta["kind"].inspect}, expected 'plane'" if meta["kind"] != "plane"
|
|
152
|
+
|
|
153
|
+
members = meta["members"] || {}
|
|
154
|
+
plane_graph = nil
|
|
155
|
+
classes_bytes = nil
|
|
156
|
+
%w[plane.onnx classes.json].each do |member|
|
|
157
|
+
raise IMF::FormatError, "plane zip: #{member} missing" unless names.include?(member)
|
|
158
|
+
|
|
159
|
+
bytes = zf.read(member)
|
|
160
|
+
want = members[member]
|
|
161
|
+
raise IMF::FormatError, "plane zip: #{member} has no sha256 in metadata" if want.nil?
|
|
162
|
+
got = Digest::SHA256.hexdigest(bytes)
|
|
163
|
+
raise IMF::FormatError, "plane zip: #{member} sha256 mismatch (#{got})" if want != got
|
|
164
|
+
|
|
165
|
+
plane_graph = bytes if member == "plane.onnx"
|
|
166
|
+
classes_bytes = bytes if member == "classes.json"
|
|
167
|
+
end
|
|
168
|
+
classes = JSON.parse(classes_bytes)
|
|
169
|
+
new(graph: plane_graph, classes: classes, k_passes: meta.fetch("k_passes", 2))
|
|
170
|
+
ensure
|
|
171
|
+
zf&.close
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
module_function
|
|
176
|
+
|
|
177
|
+
# rubocop:enable Lint/IneffectiveAccessModifier
|
|
178
|
+
|
|
179
|
+
def render_plane(skeleton, char_classes)
|
|
180
|
+
PlaneModel.render_plane(skeleton, char_classes)
|
|
181
|
+
end
|
|
182
|
+
end
|
|
183
|
+
end
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
require
|
|
2
|
-
require
|
|
3
|
-
require
|
|
4
|
-
require
|
|
5
|
-
require
|
|
1
|
+
require "open-uri"
|
|
2
|
+
require "uri"
|
|
3
|
+
require "fileutils"
|
|
4
|
+
require "digest/sha2"
|
|
5
|
+
require "yaml"
|
|
6
6
|
|
|
7
7
|
module Interscript::ML
|
|
8
8
|
# Module Interscript::ML::Provisioning is to provision remote models locally and to
|
|
@@ -37,7 +37,7 @@ module Interscript::ML
|
|
|
37
37
|
|
|
38
38
|
([ENV["INTERSCRIPT_ML_DATA"] || ENV["SECRYST_DATA"]] + possible_paths).compact.each do |path|
|
|
39
39
|
FileUtils.mkdir_p(path)
|
|
40
|
-
@write_path
|
|
40
|
+
@write_path ||= path
|
|
41
41
|
rescue
|
|
42
42
|
ensure
|
|
43
43
|
@read_paths << path if File.readable?(path)
|
|
@@ -48,11 +48,11 @@ module Interscript::ML
|
|
|
48
48
|
# Now, let's locate the first Secrystfile to be found
|
|
49
49
|
path = Dir.pwd
|
|
50
50
|
secrystfilepath = loop do
|
|
51
|
-
break unless
|
|
51
|
+
break unless %r{[/\\]}.match?(path)
|
|
52
52
|
if File.readable?(path + "/Secrystfile")
|
|
53
53
|
break path + "/Secrystfile"
|
|
54
54
|
end
|
|
55
|
-
path = path.sub(%r{[/\\][^/\\]*?\z},
|
|
55
|
+
path = path.sub(%r{[/\\][^/\\]*?\z}, "")
|
|
56
56
|
end
|
|
57
57
|
|
|
58
58
|
# It's found, so let's parse it.
|
|
@@ -70,12 +70,12 @@ module Interscript::ML
|
|
|
70
70
|
@loaded_remotes = @remotes.map do |uri|
|
|
71
71
|
cache_path = "#{@write_path}/remotes/#{Digest::SHA256.hexdigest(uri)}.yaml"
|
|
72
72
|
if !File.exist?(cache_path)
|
|
73
|
-
data = URI.
|
|
73
|
+
data = URI.parse(uri).open.read
|
|
74
74
|
File.write(cache_path, data)
|
|
75
75
|
elsif File.mtime(cache_path) + 60 < Time.now
|
|
76
76
|
begin
|
|
77
77
|
# Just *try* to download it.
|
|
78
|
-
data = URI.
|
|
78
|
+
data = URI.parse(uri).open.read
|
|
79
79
|
File.write(cache_path, data)
|
|
80
80
|
rescue
|
|
81
81
|
end
|
|
@@ -99,7 +99,7 @@ module Interscript::ML
|
|
|
99
99
|
|
|
100
100
|
def locate(name)
|
|
101
101
|
# Shortcut this if user gave a filename.
|
|
102
|
-
return name if
|
|
102
|
+
return name if %r{[.\\/]}.match?(name)
|
|
103
103
|
|
|
104
104
|
prepare_environment
|
|
105
105
|
|
|
@@ -130,24 +130,24 @@ module Interscript::ML
|
|
|
130
130
|
|
|
131
131
|
def download(model, remote:)
|
|
132
132
|
uri = model.uri
|
|
133
|
-
if uri.start_with?("./"
|
|
133
|
+
if uri.start_with?("./", "../")
|
|
134
134
|
remote_uri = remote.uri
|
|
135
|
-
if
|
|
136
|
-
remote_uri = "file://"+remote_uri
|
|
135
|
+
if %r{\A/|\A\w:[\\/]}.match?(remote_uri)
|
|
136
|
+
remote_uri = "file://" + remote_uri
|
|
137
137
|
end
|
|
138
138
|
|
|
139
139
|
uri = URI(remote_uri).merge(uri).to_s
|
|
140
|
-
uri = uri.sub(%r{\Afile://},
|
|
140
|
+
uri = uri.sub(%r{\Afile://}, "")
|
|
141
141
|
end
|
|
142
142
|
|
|
143
143
|
warn "* Downloading a Secryst model #{model.name}:#{model.version} from #{uri}..."
|
|
144
144
|
|
|
145
|
-
data = URI.
|
|
145
|
+
data = URI.parse(uri).open.read
|
|
146
146
|
path = @write_path + "/models/" + model.name
|
|
147
147
|
FileUtils.mkdir_p path
|
|
148
148
|
File.write(path + "/version", model.version)
|
|
149
149
|
File.write(path + "/model.zip", data)
|
|
150
|
-
|
|
150
|
+
path + "/model.zip"
|
|
151
151
|
end
|
|
152
152
|
|
|
153
153
|
class Remotefile
|
|
@@ -166,7 +166,7 @@ module Interscript::ML
|
|
|
166
166
|
end
|
|
167
167
|
end
|
|
168
168
|
|
|
169
|
-
class Model < Struct.new(:name, :uri, :version
|
|
169
|
+
class Model < Struct.new(:name, :uri, :version)
|
|
170
170
|
end
|
|
171
171
|
end
|
|
172
172
|
|
|
@@ -174,7 +174,7 @@ module Interscript::ML
|
|
|
174
174
|
def initialize(path)
|
|
175
175
|
@path = path
|
|
176
176
|
@models, @remotes = [], []
|
|
177
|
-
|
|
177
|
+
instance_eval(File.read(path), path)
|
|
178
178
|
end
|
|
179
179
|
|
|
180
180
|
def model(name)
|
|
@@ -6,7 +6,7 @@ module Interscript::ML
|
|
|
6
6
|
attr_accessor :model
|
|
7
7
|
|
|
8
8
|
def initialize(model_file:)
|
|
9
|
-
@device =
|
|
9
|
+
@device = "cpu"
|
|
10
10
|
@model = Model.from_file(model_file)
|
|
11
11
|
end
|
|
12
12
|
|
|
@@ -14,9 +14,9 @@ module Interscript::ML
|
|
|
14
14
|
return @model.translate(phrase, max_seq_length: max_seq_length) if @model.is_a?(Byt5Onnx)
|
|
15
15
|
|
|
16
16
|
Interscript::ML.require_optional!("numo-narray", feature: "numo/narray")
|
|
17
|
-
input = [
|
|
17
|
+
input = ["<sos>"] + phrase.chars + ["<eos>"]
|
|
18
18
|
input = Numo::NArray[input.map { |i| @model.input_vocab.stoi[i] }].transpose
|
|
19
|
-
output = Numo::NArray[[@model.target_vocab.stoi[
|
|
19
|
+
output = Numo::NArray[[@model.target_vocab.stoi["<sos>"]]]
|
|
20
20
|
src_key_padding_mask = input.transpose.eq(1)
|
|
21
21
|
|
|
22
22
|
max_seq_length.times do |i|
|
|
@@ -27,12 +27,12 @@ module Interscript::ML
|
|
|
27
27
|
src_key_padding_mask: src_key_padding_mask,
|
|
28
28
|
tgt_key_padding_mask: tgt_key_padding_mask,
|
|
29
29
|
memory_key_padding_mask: src_key_padding_mask)
|
|
30
|
-
break if @model.target_vocab.itos[prediction[i]] ==
|
|
30
|
+
break if @model.target_vocab.itos[prediction[i]] == "<eos>"
|
|
31
31
|
|
|
32
32
|
output = Numo::NArray.concatenate([output, Numo::NArray[[prediction[i]]]])
|
|
33
33
|
end
|
|
34
34
|
|
|
35
|
-
output[1
|
|
35
|
+
output[1..].to_a.flatten.map { |i| @model.target_vocab.itos[i] }.join("")
|
|
36
36
|
end
|
|
37
37
|
end
|
|
38
38
|
end
|
data/lib/interscript/ml/vocab.rb
CHANGED
data/lib/interscript/ml.rb
CHANGED
|
@@ -14,11 +14,12 @@ module Interscript::ML
|
|
|
14
14
|
autoload :Model, "interscript/ml/model"
|
|
15
15
|
autoload :Vocab, "interscript/ml/vocab"
|
|
16
16
|
autoload :Byt5Onnx, "interscript/ml/byt5_onnx"
|
|
17
|
+
autoload :PlaneModel, "interscript/ml/plane"
|
|
17
18
|
|
|
18
|
-
# Expected to be loaded before IMF/Byt5Onnx entry points.
|
|
19
|
+
# Expected to be loaded before IMF/Byt5Onnx/PlaneModel entry points.
|
|
19
20
|
def self.require_optional!(gem_name, feature: gem_name)
|
|
20
21
|
require feature
|
|
21
|
-
rescue LoadError
|
|
22
|
+
rescue LoadError
|
|
22
23
|
raise Interscript::ExternalUtilError,
|
|
23
24
|
"the neural layer needs the #{gem_name} gem: gem install #{gem_name} (or use a bundle including it)"
|
|
24
25
|
end
|
data/lib/interscript/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: interscript
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 3.0
|
|
4
|
+
version: 3.1.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ribose Inc.
|
|
8
|
-
autorequire:
|
|
9
8
|
bindir: exe
|
|
10
9
|
cert_chain: []
|
|
11
|
-
date:
|
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
12
11
|
dependencies:
|
|
13
12
|
- !ruby/object:Gem::Dependency
|
|
14
13
|
name: thor
|
|
@@ -140,7 +139,6 @@ files:
|
|
|
140
139
|
- lib/interscript/isc/grammar/concerns/system.rb
|
|
141
140
|
- lib/interscript/isc/grammar/concerns/tests.rb
|
|
142
141
|
- lib/interscript/isc/grammar/core.rb
|
|
143
|
-
- lib/interscript/isc/grammar/isc.artifact.json
|
|
144
142
|
- lib/interscript/isc/grammar/isc.parg
|
|
145
143
|
- lib/interscript/isc/items.rb
|
|
146
144
|
- lib/interscript/isc/model.rb
|
|
@@ -163,6 +161,7 @@ files:
|
|
|
163
161
|
- lib/interscript/ml/byt5_onnx.rb
|
|
164
162
|
- lib/interscript/ml/imf.rb
|
|
165
163
|
- lib/interscript/ml/model.rb
|
|
164
|
+
- lib/interscript/ml/plane.rb
|
|
166
165
|
- lib/interscript/ml/provisioning.rb
|
|
167
166
|
- lib/interscript/ml/translator.rb
|
|
168
167
|
- lib/interscript/ml/vocab.rb
|
|
@@ -209,7 +208,6 @@ metadata:
|
|
|
209
208
|
changelog_uri: https://github.com/interscript/interscript-ruby/releases
|
|
210
209
|
bug_tracker_uri: https://github.com/interscript/interscript-ruby/issues
|
|
211
210
|
rubygems_mfa_required: 'true'
|
|
212
|
-
post_install_message:
|
|
213
211
|
rdoc_options: []
|
|
214
212
|
require_paths:
|
|
215
213
|
- lib
|
|
@@ -224,8 +222,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
224
222
|
- !ruby/object:Gem::Version
|
|
225
223
|
version: '0'
|
|
226
224
|
requirements: []
|
|
227
|
-
rubygems_version:
|
|
228
|
-
signing_key:
|
|
225
|
+
rubygems_version: 4.0.16
|
|
229
226
|
specification_version: 4
|
|
230
227
|
summary: Interoperable script conversion systems
|
|
231
228
|
test_files: []
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":"0.1.0","grammar":"InterscriptIsc","shape":"parsanol-tree/v2","binding_version":1,"entries":{"isc":{"root":"isc_source","grammar":{"atoms":[{"Entity":{"atom":25}},{"Entity":{"atom":8}},{"Re":{"pattern":"[ ]"}},{"Re":{"pattern":"[\\x09]"}},{"Re":{"pattern":"[\\x0a]"}},{"Re":{"pattern":"[\\x0b]"}},{"Re":{"pattern":"[\\x0c]"}},{"Re":{"pattern":"[\\x0d]"}},{"Alternative":{"atoms":[2,3,4,5,6,7]}},{"Entity":{"atom":23}},{"Str":{"pattern":"#"}},{"Entity":{"atom":17}},{"Re":{"pattern":"[\\x0d]"}},{"Re":{"pattern":"[\\x0a]"}},{"Sequence":{"atoms":[12,13]}},{"Re":{"pattern":"[\\x0a]"}},{"Re":{"pattern":"[\\x0d]"}},{"Alternative":{"atoms":[14,15,16]}},{"Lookahead":{"atom":11,"positive":false}},{"Entity":{"atom":20}},{"Re":{"pattern":"[\\x00-\\u{10ffff}]"}},{"Sequence":{"atoms":[18,19]}},{"Repetition":{"atom":21,"min":0,"max":null,"tag":"Repetition"}},{"Sequence":{"atoms":[10,22]}},{"Alternative":{"atoms":[1,9]}},{"Repetition":{"atom":24,"min":1,"max":null,"tag":"Repetition"}},{"Repetition":{"atom":0,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":868}},{"Str":{"pattern":"system"}},{"Entity":{"atom":25}},{"Entity":{"atom":97}},{"Entity":{"atom":85}},{"Str":{"pattern":"\""}},{"Entity":{"atom":82}},{"Entity":{"atom":71}},{"Str":{"pattern":"\\"}},{"Entity":{"atom":38}},{"Str":{"pattern":"n"}},{"Named":{"name":"newline","atom":37}},{"Entity":{"atom":41}},{"Str":{"pattern":"r"}},{"Named":{"name":"carriage_return","atom":40}},{"Entity":{"atom":44}},{"Str":{"pattern":"t"}},{"Named":{"name":"tab","atom":43}},{"Entity":{"atom":47}},{"Str":{"pattern":"\""}},{"Named":{"name":"dquote","atom":46}},{"Entity":{"atom":50}},{"Str":{"pattern":"\\"}},{"Named":{"name":"backslash","atom":49}},{"Entity":{"atom":60}},{"Str":{"pattern":"u"}},{"Entity":{"atom":57}},{"Re":{"pattern":"[\\x30-\\x39]"}},{"Re":{"pattern":"[\\x41-\\x46]"}},{"Re":{"pattern":"[\\x61-\\x66]"}},{"Alternative":{"atoms":[54,55,56]}},{"Repetition":{"atom":53,"min":4,"max":4,"tag":"Repetition"}},{"Named":{"name":"unicode","atom":58}},{"Sequence":{"atoms":[52,59]}},{"Entity":{"atom":66}},{"Str":{"pattern":"U"}},{"Entity":{"atom":57}},{"Repetition":{"atom":63,"min":4,"max":8,"tag":"Repetition"}},{"Named":{"name":"unicode","atom":64}},{"Sequence":{"atoms":[62,65]}},{"Entity":{"atom":69}},{"Str":{"pattern":"U"}},{"Named":{"name":"u_lone","atom":68}},{"Alternative":{"atoms":[36,39,42,45,48,51,61,67]}},{"Sequence":{"atoms":[35,70]}},{"Entity":{"atom":79}},{"Str":{"pattern":"\\"}},{"Str":{"pattern":"\""}},{"Alternative":{"atoms":[73,74]}},{"Lookahead":{"atom":75,"positive":false}},{"Entity":{"atom":20}},{"Sequence":{"atoms":[76,77]}},{"Repetition":{"atom":78,"min":1,"max":null,"tag":"Repetition"}},{"Named":{"name":"run","atom":72}},{"Alternative":{"atoms":[34,80]}},{"Repetition":{"atom":81,"min":0,"max":null,"tag":"Repetition"}},{"Named":{"name":"string","atom":33}},{"Str":{"pattern":"\""}},{"Sequence":{"atoms":[32,83,84]}},{"Entity":{"atom":96}},{"Str":{"pattern":"'"}},{"Entity":{"atom":93}},{"Str":{"pattern":"'"}},{"Lookahead":{"atom":89,"positive":false}},{"Entity":{"atom":20}},{"Sequence":{"atoms":[90,91]}},{"Repetition":{"atom":92,"min":0,"max":null,"tag":"Repetition"}},{"Named":{"name":"string","atom":88}},{"Str":{"pattern":"'"}},{"Sequence":{"atoms":[87,94,95]}},{"Alternative":{"atoms":[31,86]}},{"Named":{"name":"system_code","atom":30}},{"Entity":{"atom":25}},{"Repetition":{"atom":99,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"{"}},{"Entity":{"atom":106}},{"Entity":{"atom":8}},{"Entity":{"atom":23}},{"Alternative":{"atoms":[103,104]}},{"Repetition":{"atom":105,"min":0,"max":null,"tag":"Repetition"}},{"Repetition":{"atom":102,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":860}},{"Entity":{"atom":25}},{"Repetition":{"atom":109,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":322}},{"Str":{"pattern":"metadata"}},{"Entity":{"atom":25}},{"Repetition":{"atom":113,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"{"}},{"Entity":{"atom":106}},{"Repetition":{"atom":116,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":317}},{"Entity":{"atom":316}},{"Entity":{"atom":25}},{"Repetition":{"atom":120,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":143}},{"Str":{"pattern":"description"}},{"Entity":{"atom":25}},{"Repetition":{"atom":124,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"{"}},{"Entity":{"atom":106}},{"Repetition":{"atom":127,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":138}},{"Str":{"pattern":"\\"}},{"Entity":{"atom":20}},{"Sequence":{"atoms":[130,131]}},{"Str":{"pattern":"}"}},{"Lookahead":{"atom":133,"positive":false}},{"Entity":{"atom":20}},{"Sequence":{"atoms":[134,135]}},{"Alternative":{"atoms":[132,136]}},{"Repetition":{"atom":137,"min":0,"max":null,"tag":"Repetition"}},{"Named":{"name":"description","atom":129}},{"Entity":{"atom":106}},{"Repetition":{"atom":140,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"}"}},{"Sequence":{"atoms":[123,125,126,128,139,141,142]}},{"Entity":{"atom":183}},{"Str":{"pattern":"relations"}},{"Entity":{"atom":25}},{"Repetition":{"atom":146,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"{"}},{"Entity":{"atom":106}},{"Repetition":{"atom":149,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":177}},{"Entity":{"atom":25}},{"Repetition":{"atom":152,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":162}},{"Str":{"pattern":"supersedes"}},{"Str":{"pattern":"superseded_by"}},{"Str":{"pattern":"based_on"}},{"Str":{"pattern":"basis_for"}},{"Str":{"pattern":"alias_of"}},{"Str":{"pattern":"adopted_from"}},{"Str":{"pattern":"related_to"}},{"Alternative":{"atoms":[155,156,157,158,159,160,161]}},{"Named":{"name":"type","atom":154}},{"Entity":{"atom":25}},{"Entity":{"atom":97}},{"Named":{"name":"system","atom":165}},{"Entity":{"atom":173}},{"Entity":{"atom":25}},{"Str":{"pattern":"note"}},{"Entity":{"atom":25}},{"Entity":{"atom":97}},{"Named":{"name":"note","atom":171}},{"Sequence":{"atoms":[168,169,170,172]}},{"Repetition":{"atom":167,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":25}},{"Repetition":{"atom":175,"min":0,"max":1,"tag":"Maybe"}},{"Sequence":{"atoms":[153,163,164,166,174,176]}},{"Repetition":{"atom":151,"min":0,"max":null,"tag":"Repetition"}},{"Named":{"name":"relations","atom":178}},{"Entity":{"atom":106}},{"Repetition":{"atom":180,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"}"}},{"Sequence":{"atoms":[145,147,148,150,179,181,182]}},{"Entity":{"atom":192}},{"Str":{"pattern":"system_status"}},{"Entity":{"atom":25}},{"Str":{"pattern":"current"}},{"Str":{"pattern":"former"}},{"Str":{"pattern":"inactive"}},{"Alternative":{"atoms":[187,188,189]}},{"Named":{"name":"system_status","atom":190}},{"Sequence":{"atoms":[185,186,191]}},{"Entity":{"atom":201}},{"Str":{"pattern":"code_status"}},{"Entity":{"atom":25}},{"Str":{"pattern":"preferred"}},{"Str":{"pattern":"proposed"}},{"Str":{"pattern":"deprecated"}},{"Alternative":{"atoms":[196,197,198]}},{"Named":{"name":"code_status","atom":199}},{"Sequence":{"atoms":[194,195,200]}},{"Entity":{"atom":218}},{"Str":{"pattern":"specification"}},{"Entity":{"atom":25}},{"Entity":{"atom":207}},{"Entity":{"atom":97}},{"Named":{"name":"specification","atom":206}},{"Entity":{"atom":212}},{"Str":{"pattern":","}},{"Entity":{"atom":25}},{"Repetition":{"atom":210,"min":0,"max":1,"tag":"Maybe"}},{"Sequence":{"atoms":[209,211]}},{"Entity":{"atom":207}},{"Sequence":{"atoms":[208,213]}},{"Repetition":{"atom":214,"min":0,"max":null,"tag":"Repetition"}},{"Sequence":{"atoms":[205,215]}},{"Named":{"name":"specification","atom":216}},{"Sequence":{"atoms":[203,204,217]}},{"Entity":{"atom":242}},{"Str":{"pattern":"notes"}},{"Entity":{"atom":25}},{"Repetition":{"atom":221,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"{"}},{"Entity":{"atom":106}},{"Repetition":{"atom":224,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":236}},{"Entity":{"atom":25}},{"Repetition":{"atom":227,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"note"}},{"Entity":{"atom":25}},{"Repetition":{"atom":230,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":97}},{"Named":{"name":"note","atom":232}},{"Entity":{"atom":25}},{"Repetition":{"atom":234,"min":0,"max":1,"tag":"Maybe"}},{"Sequence":{"atoms":[228,229,231,233,235]}},{"Repetition":{"atom":226,"min":0,"max":null,"tag":"Repetition"}},{"Named":{"name":"notes","atom":237}},{"Entity":{"atom":106}},{"Repetition":{"atom":239,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"}"}},{"Sequence":{"atoms":[220,222,223,225,238,240,241]}},{"Entity":{"atom":256}},{"Str":{"pattern":"provenance"}},{"Entity":{"atom":25}},{"Repetition":{"atom":245,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":249}},{"Entity":{"atom":97}},{"Named":{"name":"provenance","atom":248}},{"Entity":{"atom":212}},{"Entity":{"atom":249}},{"Sequence":{"atoms":[250,251]}},{"Repetition":{"atom":252,"min":0,"max":null,"tag":"Repetition"}},{"Sequence":{"atoms":[247,253]}},{"Named":{"name":"provenance","atom":254}},{"Sequence":{"atoms":[244,246,255]}},{"Entity":{"atom":312}},{"Entity":{"atom":272}},{"Entity":{"atom":263}},{"Re":{"pattern":"[\\x41-\\x5a]"}},{"Re":{"pattern":"[\\x61-\\x7a]"}},{"Re":{"pattern":"[_]"}},{"Alternative":{"atoms":[260,261,262]}},{"Entity":{"atom":269}},{"Re":{"pattern":"[\\x41-\\x5a]"}},{"Re":{"pattern":"[\\x61-\\x7a]"}},{"Re":{"pattern":"[\\x30-\\x39]"}},{"Re":{"pattern":"[_]"}},{"Alternative":{"atoms":[265,266,267,268]}},{"Repetition":{"atom":264,"min":0,"max":null,"tag":"Repetition"}},{"Sequence":{"atoms":[259,270]}},{"Named":{"name":"identifier","atom":271}},{"Named":{"name":"field_name","atom":258}},{"Entity":{"atom":282}},{"Entity":{"atom":279}},{"Re":{"pattern":"[ ]"}},{"Re":{"pattern":"[\\x09]"}},{"Alternative":{"atoms":[276,277]}},{"Repetition":{"atom":278,"min":1,"max":null,"tag":"Repetition"}},{"Entity":{"atom":23}},{"Alternative":{"atoms":[275,280]}},{"Repetition":{"atom":281,"min":0,"max":null,"tag":"Repetition"}},{"Entity":{"atom":296}},{"Entity":{"atom":97}},{"Entity":{"atom":294}},{"Re":{"pattern":"[\\x0a]"}},{"Lookahead":{"atom":286,"positive":false}},{"Str":{"pattern":"}"}},{"Lookahead":{"atom":288,"positive":false}},{"Str":{"pattern":"{"}},{"Lookahead":{"atom":290,"positive":false}},{"Entity":{"atom":20}},{"Sequence":{"atoms":[287,289,291,292]}},{"Repetition":{"atom":293,"min":1,"max":null,"tag":"Repetition"}},{"Named":{"name":"raw","atom":285}},{"Alternative":{"atoms":[284,295]}},{"Named":{"name":"field_value","atom":283}},{"Entity":{"atom":307}},{"Str":{"pattern":"{"}},{"Entity":{"atom":106}},{"Repetition":{"atom":300,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":138}},{"Named":{"name":"field_block","atom":302}},{"Entity":{"atom":106}},{"Repetition":{"atom":304,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"}"}},{"Sequence":{"atoms":[299,301,303,305,306]}},{"Entity":{"atom":310}},{"Entity":{"atom":17}},{"Lookahead":{"atom":309,"positive":true}},{"Alternative":{"atoms":[297,298,308]}},{"Sequence":{"atoms":[273,274,311]}},{"Alternative":{"atoms":[122,144,184,193,202,219,243,257]}},{"Entity":{"atom":25}},{"Repetition":{"atom":314,"min":0,"max":1,"tag":"Maybe"}},{"Sequence":{"atoms":[121,313,315]}},{"Repetition":{"atom":119,"min":0,"max":null,"tag":"Repetition"}},{"Named":{"name":"metadata","atom":118}},{"Entity":{"atom":106}},{"Repetition":{"atom":319,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"}"}},{"Sequence":{"atoms":[112,114,115,117,318,320,321]}},{"Entity":{"atom":569}},{"Str":{"pattern":"aliases"}},{"Entity":{"atom":25}},{"Repetition":{"atom":325,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"{"}},{"Entity":{"atom":106}},{"Repetition":{"atom":328,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":564}},{"Entity":{"atom":563}},{"Entity":{"atom":25}},{"Repetition":{"atom":332,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":272}},{"Named":{"name":"name","atom":334}},{"Entity":{"atom":25}},{"Repetition":{"atom":336,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"="}},{"Entity":{"atom":25}},{"Repetition":{"atom":339,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":559}},{"Entity":{"atom":557}},{"Entity":{"atom":97}},{"Entity":{"atom":346}},{"Str":{"pattern":"none"}},{"Named":{"name":"none","atom":345}},{"Entity":{"atom":355}},{"Str":{"pattern":"boundary"}},{"Str":{"pattern":"line_start"}},{"Str":{"pattern":"line_end"}},{"Str":{"pattern":"word_boundary"}},{"Str":{"pattern":"space"}},{"Str":{"pattern":"non_boundary"}},{"Alternative":{"atoms":[348,349,350,351,352,353]}},{"Named":{"name":"primitive","atom":354}},{"Entity":{"atom":358}},{"Str":{"pattern":"any_character"}},{"Named":{"name":"any_char","atom":357}},{"Entity":{"atom":556}},{"Str":{"pattern":"any"}},{"Str":{"pattern":"("}},{"Entity":{"atom":106}},{"Repetition":{"atom":362,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":551}},{"Entity":{"atom":375}},{"Entity":{"atom":97}},{"Named":{"name":"lo","atom":366}},{"Entity":{"atom":106}},{"Repetition":{"atom":368,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":".."}},{"Entity":{"atom":106}},{"Repetition":{"atom":371,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":97}},{"Named":{"name":"hi","atom":373}},{"Sequence":{"atoms":[367,369,370,372,374]}},{"Entity":{"atom":542}},{"Entity":{"atom":97}},{"Named":{"name":"single","atom":377}},{"Str":{"pattern":"["}},{"Entity":{"atom":106}},{"Repetition":{"atom":380,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":383}},{"Entity":{"atom":529}},{"Entity":{"atom":392}},{"Entity":{"atom":106}},{"Repetition":{"atom":385,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"+"}},{"Entity":{"atom":106}},{"Repetition":{"atom":388,"min":0,"max":1,"tag":"Maybe"}},{"Sequence":{"atoms":[386,387,389]}},{"Entity":{"atom":25}},{"Alternative":{"atoms":[390,391]}},{"Entity":{"atom":400}},{"Str":{"pattern":"to"}},{"Str":{"pattern":"before"}},{"Str":{"pattern":"after"}},{"Str":{"pattern":"not_before"}},{"Str":{"pattern":"not_after"}},{"Str":{"pattern":"}"}},{"Alternative":{"atoms":[394,395,396,397,398,399]}},{"Lookahead":{"atom":393,"positive":false}},{"Entity":{"atom":407}},{"Entity":{"atom":272}},{"Entity":{"atom":106}},{"Repetition":{"atom":404,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"="}},{"Sequence":{"atoms":[403,405,406]}},{"Lookahead":{"atom":402,"positive":false}},{"Entity":{"atom":413}},{"Str":{"pattern":"\""}},{"Str":{"pattern":"'"}},{"Entity":{"atom":263}},{"Alternative":{"atoms":[410,411,412]}},{"Lookahead":{"atom":409,"positive":true}},{"Entity":{"atom":525}},{"Entity":{"atom":430}},{"Str":{"pattern":"capture"}},{"Str":{"pattern":"("}},{"Entity":{"atom":106}},{"Repetition":{"atom":419,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":425}},{"Sequence":{"atoms":[384,401,408,414,415]}},{"Repetition":{"atom":422,"min":0,"max":null,"tag":"Repetition"}},{"Sequence":{"atoms":[342,423]}},{"Named":{"name":"concatenation","atom":424}},{"Named":{"name":"capture_inner","atom":421}},{"Entity":{"atom":106}},{"Repetition":{"atom":427,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":")"}},{"Sequence":{"atoms":[417,418,420,426,428,429]}},{"Entity":{"atom":441}},{"Str":{"pattern":"maybe"}},{"Str":{"pattern":"("}},{"Entity":{"atom":106}},{"Repetition":{"atom":434,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":425}},{"Named":{"name":"maybe_inner","atom":436}},{"Entity":{"atom":106}},{"Repetition":{"atom":438,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":")"}},{"Sequence":{"atoms":[432,433,435,437,439,440]}},{"Entity":{"atom":452}},{"Str":{"pattern":"some"}},{"Str":{"pattern":"("}},{"Entity":{"atom":106}},{"Repetition":{"atom":445,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":425}},{"Named":{"name":"some_inner","atom":447}},{"Entity":{"atom":106}},{"Repetition":{"atom":449,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":")"}},{"Sequence":{"atoms":[443,444,446,448,450,451]}},{"Entity":{"atom":461}},{"Str":{"pattern":"upcase"}},{"Str":{"pattern":"downcase"}},{"Str":{"pattern":"title_case"}},{"Str":{"pattern":"reverse"}},{"Str":{"pattern":"strip"}},{"Str":{"pattern":"swapcase"}},{"Alternative":{"atoms":[454,455,456,457,458,459]}},{"Named":{"name":"function","atom":460}},{"Entity":{"atom":476}},{"Entity":{"atom":475}},{"Str":{"pattern":"ref"}},{"Str":{"pattern":"("}},{"Entity":{"atom":106}},{"Repetition":{"atom":466,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":471}},{"Entity":{"atom":470}},{"Re":{"pattern":"[\\x30-\\x39]"}},{"Named":{"name":"digit","atom":469}},{"Entity":{"atom":106}},{"Repetition":{"atom":472,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":")"}},{"Sequence":{"atoms":[464,465,467,468,473,474]}},{"Named":{"name":"ref","atom":463}},{"Entity":{"atom":524}},{"Entity":{"atom":513}},{"Str":{"pattern":"parallel"}},{"Str":{"pattern":"sequence"}},{"Str":{"pattern":"stage"}},{"Str":{"pattern":"compose"}},{"Str":{"pattern":"separate"}},{"Str":{"pattern":"system"}},{"Str":{"pattern":"metadata"}},{"Str":{"pattern":"aliases"}},{"Str":{"pattern":"tests"}},{"Str":{"pattern":"notes"}},{"Str":{"pattern":"description"}},{"Str":{"pattern":"any_character"}},{"Str":{"pattern":"authority"}},{"Str":{"pattern":"dependency"}},{"Str":{"pattern":"run"}},{"Str":{"pattern":"sub"}},{"Str":{"pattern":"before"}},{"Str":{"pattern":"after"}},{"Str":{"pattern":"not_before"}},{"Str":{"pattern":"not_after"}},{"Str":{"pattern":"any"}},{"Str":{"pattern":"none"}},{"Str":{"pattern":"boundary"}},{"Str":{"pattern":"line_start"}},{"Str":{"pattern":"line_end"}},{"Str":{"pattern":"word_boundary"}},{"Str":{"pattern":"downcase"}},{"Str":{"pattern":"upcase"}},{"Str":{"pattern":"title_case"}},{"Str":{"pattern":"capture"}},{"Str":{"pattern":"maybe"}},{"Str":{"pattern":"some"}},{"Str":{"pattern":"ref"}},{"Str":{"pattern":"name"}},{"Alternative":{"atoms":[479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512]}},{"Lookahead":{"atom":478,"positive":false}},{"Entity":{"atom":272}},{"Str":{"pattern":"."}},{"Entity":{"atom":272}},{"Named":{"name":"qualified_name","atom":517}},{"Sequence":{"atoms":[516,518]}},{"Repetition":{"atom":519,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"{"}},{"Lookahead":{"atom":521,"positive":false}},{"Sequence":{"atoms":[514,515,520,522]}},{"Named":{"name":"alias","atom":523}},{"Alternative":{"atoms":[343,344,347,356,359,416,431,442,453,462,477]}},{"Sequence":{"atoms":[384,401,408,414,415]}},{"Repetition":{"atom":526,"min":0,"max":null,"tag":"Repetition"}},{"Sequence":{"atoms":[342,527]}},{"Named":{"name":"concatenation","atom":528}},{"Entity":{"atom":212}},{"Entity":{"atom":25}},{"Alternative":{"atoms":[530,531]}},{"Entity":{"atom":383}},{"Sequence":{"atoms":[532,533]}},{"Repetition":{"atom":534,"min":0,"max":null,"tag":"Repetition"}},{"Sequence":{"atoms":[382,535]}},{"Named":{"name":"list","atom":536}},{"Entity":{"atom":106}},{"Repetition":{"atom":538,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"]"}},{"Sequence":{"atoms":[379,381,537,539,540]}},{"Alternative":{"atoms":[378,541]}},{"Entity":{"atom":550}},{"Entity":{"atom":355}},{"Lookahead":{"atom":544,"positive":false}},{"Entity":{"atom":513}},{"Lookahead":{"atom":546,"positive":false}},{"Entity":{"atom":272}},{"Sequence":{"atoms":[545,547,548]}},{"Named":{"name":"alias_ref","atom":549}},{"Alternative":{"atoms":[365,376,543]}},{"Named":{"name":"any","atom":364}},{"Entity":{"atom":106}},{"Repetition":{"atom":553,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":")"}},{"Sequence":{"atoms":[360,361,363,552,554,555]}},{"Alternative":{"atoms":[343,344,347,356,359,416,431,442,453,462,477]}},{"Sequence":{"atoms":[342,527]}},{"Named":{"name":"concatenation","atom":558}},{"Named":{"name":"value","atom":341}},{"Entity":{"atom":25}},{"Repetition":{"atom":561,"min":0,"max":1,"tag":"Maybe"}},{"Sequence":{"atoms":[333,335,337,338,340,560,562]}},{"Repetition":{"atom":331,"min":0,"max":null,"tag":"Repetition"}},{"Named":{"name":"aliases","atom":330}},{"Entity":{"atom":106}},{"Repetition":{"atom":566,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"}"}},{"Sequence":{"atoms":[324,326,327,329,565,567,568]}},{"Entity":{"atom":608}},{"Str":{"pattern":"tests"}},{"Entity":{"atom":25}},{"Repetition":{"atom":572,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"{"}},{"Entity":{"atom":106}},{"Repetition":{"atom":575,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":603}},{"Entity":{"atom":602}},{"Entity":{"atom":25}},{"Repetition":{"atom":579,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":97}},{"Named":{"name":"input","atom":581}},{"Entity":{"atom":589}},{"Entity":{"atom":25}},{"Repetition":{"atom":584,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"->"}},{"Entity":{"atom":25}},{"Repetition":{"atom":587,"min":0,"max":1,"tag":"Maybe"}},{"Sequence":{"atoms":[585,586,588]}},{"Entity":{"atom":97}},{"Named":{"name":"expected","atom":590}},{"Entity":{"atom":598}},{"Entity":{"atom":25}},{"Str":{"pattern":"note"}},{"Entity":{"atom":25}},{"Entity":{"atom":97}},{"Named":{"name":"note","atom":596}},{"Sequence":{"atoms":[593,594,595,597]}},{"Repetition":{"atom":592,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":25}},{"Repetition":{"atom":600,"min":0,"max":1,"tag":"Maybe"}},{"Sequence":{"atoms":[580,582,583,591,599,601]}},{"Repetition":{"atom":578,"min":0,"max":null,"tag":"Repetition"}},{"Named":{"name":"tests","atom":577}},{"Entity":{"atom":106}},{"Repetition":{"atom":605,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"}"}},{"Sequence":{"atoms":[571,573,574,576,604,606,607]}},{"Entity":{"atom":842}},{"Str":{"pattern":"stage"}},{"Entity":{"atom":616}},{"Str":{"pattern":"("}},{"Entity":{"atom":272}},{"Named":{"name":"stage_name","atom":613}},{"Str":{"pattern":")"}},{"Sequence":{"atoms":[612,614,615]}},{"Entity":{"atom":621}},{"Entity":{"atom":25}},{"Entity":{"atom":272}},{"Named":{"name":"stage_name","atom":619}},{"Sequence":{"atoms":[618,620]}},{"Alternative":{"atoms":[611,617]}},{"Entity":{"atom":106}},{"Repetition":{"atom":623,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"{"}},{"Entity":{"atom":106}},{"Repetition":{"atom":626,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":837}},{"Entity":{"atom":836}},{"Entity":{"atom":106}},{"Repetition":{"atom":630,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":734}},{"Str":{"pattern":"sequence"}},{"Entity":{"atom":106}},{"Repetition":{"atom":634,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"{"}},{"Entity":{"atom":106}},{"Repetition":{"atom":637,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":729}},{"Entity":{"atom":728}},{"Entity":{"atom":106}},{"Repetition":{"atom":641,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":709}},{"Entity":{"atom":694}},{"Str":{"pattern":"sub"}},{"Entity":{"atom":106}},{"Repetition":{"atom":646,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"{"}},{"Entity":{"atom":106}},{"Repetition":{"atom":649,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"from"}},{"Entity":{"atom":25}},{"Entity":{"atom":559}},{"Named":{"name":"from","atom":653}},{"Entity":{"atom":106}},{"Repetition":{"atom":655,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"to"}},{"Entity":{"atom":25}},{"Entity":{"atom":559}},{"Named":{"name":"to","atom":659}},{"Entity":{"atom":25}},{"Entity":{"atom":687}},{"Entity":{"atom":668}},{"Str":{"pattern":"after"}},{"Entity":{"atom":25}},{"Entity":{"atom":559}},{"Named":{"name":"after","atom":666}},{"Sequence":{"atoms":[664,665,667]}},{"Entity":{"atom":674}},{"Str":{"pattern":"before"}},{"Entity":{"atom":25}},{"Entity":{"atom":559}},{"Named":{"name":"before","atom":672}},{"Sequence":{"atoms":[670,671,673]}},{"Entity":{"atom":680}},{"Str":{"pattern":"not_before"}},{"Entity":{"atom":25}},{"Entity":{"atom":559}},{"Named":{"name":"not_before","atom":678}},{"Sequence":{"atoms":[676,677,679]}},{"Entity":{"atom":686}},{"Str":{"pattern":"not_after"}},{"Entity":{"atom":25}},{"Entity":{"atom":559}},{"Named":{"name":"not_after","atom":684}},{"Sequence":{"atoms":[682,683,685]}},{"Alternative":{"atoms":[663,669,675,681]}},{"Sequence":{"atoms":[661,662]}},{"Repetition":{"atom":688,"min":0,"max":null,"tag":"Repetition"}},{"Named":{"name":"constraints","atom":689}},{"Entity":{"atom":106}},{"Repetition":{"atom":691,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"}"}},{"Sequence":{"atoms":[645,647,648,650,651,652,654,656,657,658,660,690,692,693]}},{"Entity":{"atom":708}},{"Str":{"pattern":"sub"}},{"Entity":{"atom":25}},{"Entity":{"atom":557}},{"Named":{"name":"from","atom":698}},{"Entity":{"atom":25}},{"Entity":{"atom":557}},{"Named":{"name":"to","atom":701}},{"Entity":{"atom":25}},{"Entity":{"atom":687}},{"Sequence":{"atoms":[703,704]}},{"Repetition":{"atom":705,"min":0,"max":null,"tag":"Repetition"}},{"Named":{"name":"constraints","atom":706}},{"Sequence":{"atoms":[696,697,699,700,702,707]}},{"Alternative":{"atoms":[644,695]}},{"Entity":{"atom":724}},{"Str":{"pattern":"#"}},{"Re":{"pattern":"[\\x0a]"}},{"Lookahead":{"atom":712,"positive":false}},{"Entity":{"atom":20}},{"Sequence":{"atoms":[713,714]}},{"Repetition":{"atom":715,"min":0,"max":null,"tag":"Repetition"}},{"Sequence":{"atoms":[711,716]}},{"Named":{"name":"comment","atom":717}},{"Entity":{"atom":513}},{"Lookahead":{"atom":719,"positive":false}},{"Entity":{"atom":272}},{"Sequence":{"atoms":[720,721]}},{"Named":{"name":"noop","atom":722}},{"Alternative":{"atoms":[718,723]}},{"Alternative":{"atoms":[643,710]}},{"Entity":{"atom":106}},{"Repetition":{"atom":726,"min":0,"max":1,"tag":"Maybe"}},{"Sequence":{"atoms":[642,725,727]}},{"Repetition":{"atom":640,"min":0,"max":null,"tag":"Repetition"}},{"Named":{"name":"sequence","atom":639}},{"Entity":{"atom":106}},{"Repetition":{"atom":731,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"}"}},{"Sequence":{"atoms":[633,635,636,638,730,732,733]}},{"Entity":{"atom":747}},{"Str":{"pattern":"parallel"}},{"Entity":{"atom":106}},{"Repetition":{"atom":737,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"{"}},{"Entity":{"atom":106}},{"Repetition":{"atom":740,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":729}},{"Named":{"name":"parallel","atom":742}},{"Entity":{"atom":106}},{"Repetition":{"atom":744,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"}"}},{"Sequence":{"atoms":[736,738,739,741,743,745,746]}},{"Entity":{"atom":766}},{"Str":{"pattern":"run"}},{"Entity":{"atom":25}},{"Entity":{"atom":758}},{"Str":{"pattern":"map."}},{"Entity":{"atom":272}},{"Named":{"name":"dep","atom":753}},{"Str":{"pattern":".stage."}},{"Entity":{"atom":272}},{"Named":{"name":"stage","atom":756}},{"Sequence":{"atoms":[752,754,755,757]}},{"Entity":{"atom":764}},{"Str":{"pattern":"stage."}},{"Entity":{"atom":272}},{"Named":{"name":"stage","atom":761}},{"Sequence":{"atoms":[760,762]}},{"Named":{"name":"run_stage_only","atom":763}},{"Alternative":{"atoms":[751,759]}},{"Sequence":{"atoms":[749,750,765]}},{"Entity":{"atom":778}},{"Str":{"pattern":"separate"}},{"Named":{"name":"separate","atom":768}},{"Entity":{"atom":776}},{"Entity":{"atom":25}},{"Str":{"pattern":"separator"}},{"Entity":{"atom":25}},{"Entity":{"atom":557}},{"Named":{"name":"separator","atom":774}},{"Sequence":{"atoms":[771,772,773,775]}},{"Repetition":{"atom":770,"min":0,"max":1,"tag":"Maybe"}},{"Sequence":{"atoms":[769,777]}},{"Entity":{"atom":813}},{"Entity":{"atom":784}},{"Str":{"pattern":"downcase"}},{"Str":{"pattern":"upcase"}},{"Alternative":{"atoms":[781,782]}},{"Named":{"name":"case","atom":783}},{"Entity":{"atom":812}},{"Str":{"pattern":"title_case"}},{"Named":{"name":"case","atom":786}},{"Entity":{"atom":810}},{"Entity":{"atom":25}},{"Entity":{"atom":808}},{"Entity":{"atom":803}},{"Entity":{"atom":802}},{"Entity":{"atom":272}},{"Named":{"name":"kwarg_name","atom":793}},{"Entity":{"atom":106}},{"Repetition":{"atom":795,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":":"}},{"Entity":{"atom":106}},{"Repetition":{"atom":798,"min":0,"max":1,"tag":"Maybe"}},{"Entity":{"atom":97}},{"Named":{"name":"kwarg_value","atom":800}},{"Sequence":{"atoms":[794,796,797,799,801]}},{"Named":{"name":"kwarg","atom":792}},{"Entity":{"atom":212}},{"Entity":{"atom":803}},{"Sequence":{"atoms":[804,805]}},{"Repetition":{"atom":806,"min":0,"max":null,"tag":"Repetition"}},{"Sequence":{"atoms":[791,807]}},{"Named":{"name":"case_kwargs","atom":790}},{"Sequence":{"atoms":[789,809]}},{"Repetition":{"atom":788,"min":0,"max":1,"tag":"Maybe"}},{"Sequence":{"atoms":[787,811]}},{"Alternative":{"atoms":[780,785]}},{"Entity":{"atom":818}},{"Str":{"pattern":"compose"}},{"Str":{"pattern":"decompose"}},{"Alternative":{"atoms":[815,816]}},{"Named":{"name":"compose","atom":817}},{"Entity":{"atom":828}},{"Str":{"pattern":"rababa"}},{"Named":{"name":"funcall_name","atom":820}},{"Entity":{"atom":826}},{"Entity":{"atom":25}},{"Entity":{"atom":808}},{"Named":{"name":"funcall_kwargs","atom":824}},{"Sequence":{"atoms":[823,825]}},{"Repetition":{"atom":822,"min":0,"max":1,"tag":"Maybe"}},{"Sequence":{"atoms":[821,827]}},{"Entity":{"atom":831}},{"Entity":{"atom":709}},{"Named":{"name":"bare_rule","atom":830}},{"Entity":{"atom":724}},{"Alternative":{"atoms":[632,735,748,767,779,814,819,829,832]}},{"Entity":{"atom":106}},{"Repetition":{"atom":834,"min":0,"max":1,"tag":"Maybe"}},{"Sequence":{"atoms":[631,833,835]}},{"Repetition":{"atom":629,"min":0,"max":null,"tag":"Repetition"}},{"Named":{"name":"stage","atom":628}},{"Entity":{"atom":106}},{"Repetition":{"atom":839,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"}"}},{"Sequence":{"atoms":[610,622,624,625,627,838,840,841]}},{"Entity":{"atom":856}},{"Str":{"pattern":"dependency"}},{"Entity":{"atom":25}},{"Entity":{"atom":97}},{"Named":{"name":"target","atom":846}},{"Entity":{"atom":854}},{"Entity":{"atom":25}},{"Str":{"pattern":"as"}},{"Entity":{"atom":25}},{"Entity":{"atom":272}},{"Named":{"name":"alias","atom":852}},{"Sequence":{"atoms":[849,850,851,853]}},{"Repetition":{"atom":848,"min":0,"max":1,"tag":"Maybe"}},{"Sequence":{"atoms":[844,845,847,855]}},{"Alternative":{"atoms":[111,323,570,609,843]}},{"Entity":{"atom":25}},{"Repetition":{"atom":858,"min":0,"max":1,"tag":"Maybe"}},{"Sequence":{"atoms":[110,857,859]}},{"Repetition":{"atom":108,"min":0,"max":null,"tag":"Repetition"}},{"Named":{"name":"body","atom":861}},{"Entity":{"atom":106}},{"Repetition":{"atom":863,"min":0,"max":1,"tag":"Maybe"}},{"Str":{"pattern":"}"}},{"Entity":{"atom":25}},{"Repetition":{"atom":866,"min":0,"max":1,"tag":"Maybe"}},{"Sequence":{"atoms":[28,29,98,100,101,107,862,864,865,867]}},{"Named":{"name":"system","atom":27}},{"Entity":{"atom":25}},{"Repetition":{"atom":870,"min":0,"max":1,"tag":"Maybe"}},{"Sequence":{"atoms":[26,869,871]}}],"root":872},"bindings":[]}},"preprocess":{},"tables":{},"lint":{"order_warnings":["rule newline: branches 1 and 3 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 2 and 3 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 2 and 5 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 2 and 6 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 2 and 16 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 2 and 32 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 3 and 5 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 3 and 6 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 3 and 16 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 3 and 32 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 4 and 30 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 5 and 6 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 5 and 16 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 5 and 32 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 6 and 16 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 6 and 32 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 7 and 31 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 8 and 12 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 8 and 13 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 8 and 18 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 8 and 21 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 9 and 29 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 10 and 19 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 10 and 20 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 10 and 22 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 10 and 34 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 11 and 14 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 11 and 27 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 12 and 13 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 12 and 18 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 12 and 21 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 13 and 18 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 13 and 21 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 14 and 27 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 15 and 33 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 16 and 32 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 17 and 23 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 18 and 21 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 19 and 20 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 19 and 22 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 19 and 34 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 20 and 22 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 20 and 34 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 22 and 34 are order-dependent (shared first bytes; ordered choice is decisive)","rule keyword: branches 24 and 25 are order-dependent (shared first bytes; ordered choice is decisive)","rule relation_type: branches 1 and 2 are order-dependent (shared first bytes; ordered choice is decisive)","rule relation_type: branches 3 and 4 are order-dependent (shared first bytes; ordered choice is decisive)","rule relation_type: branches 5 and 6 are order-dependent (shared first bytes; ordered choice is decisive)","rule md_field_value: branches 1 and 2 are order-dependent (first set not statically known)","rule item_sep: branches 1 and 2 are order-dependent (shared first bytes; ordered choice is decisive)","rule any_arg: branches 1 and 2 are order-dependent (shared first bytes; ordered choice is decisive)","rule any_arg: branches 1 and 3 are order-dependent (first set not statically known)","rule any_arg: branches 2 and 3 are order-dependent (first set not statically known)","rule cont_stop: branches 4 and 5 are order-dependent (shared first bytes; ordered choice is decisive)","rule item_atom: branches 1 and 11 are order-dependent (first set not statically known)","rule item_atom: branches 2 and 3 are order-dependent (shared first bytes; ordered choice is decisive)","rule item_atom: branches 2 and 11 are order-dependent (first set not statically known)","rule item_atom: branches 3 and 8 are order-dependent (shared first bytes; ordered choice is decisive)","rule item_atom: branches 3 and 9 are order-dependent (shared first bytes; ordered choice is decisive)","rule item_atom: branches 3 and 11 are order-dependent (first set not statically known)","rule item_atom: branches 4 and 5 are order-dependent (shared first bytes; ordered choice is decisive)","rule item_atom: branches 4 and 11 are order-dependent (first set not statically known)","rule item_atom: branches 5 and 11 are order-dependent (first set not statically known)","rule item_atom: branches 6 and 11 are order-dependent (first set not statically known)","rule item_atom: branches 7 and 11 are order-dependent (first set not statically known)","rule item_atom: branches 8 and 9 are order-dependent (shared first bytes; ordered choice is decisive)","rule item_atom: branches 8 and 11 are order-dependent (first set not statically known)","rule item_atom: branches 9 and 10 are order-dependent (shared first bytes; ordered choice is decisive)","rule item_atom: branches 9 and 11 are order-dependent (first set not statically known)","rule item_atom: branches 10 and 11 are order-dependent (first set not statically known)","rule constraint: branches 3 and 4 are order-dependent (shared first bytes; ordered choice is decisive)","rule rule: branches 1 and 2 are order-dependent (shared first bytes; ordered choice is decisive)","rule comment_item: branches 1 and 2 are order-dependent (first set not statically known)"]},"default_entry":"isc","render":{},"derive":{},"tests":[],"docs":{},"source":"# Interscript ISC grammar — parsanol grammar language (PG).\n#\n# Line-oriented port of lib/interscript/isc/grammar/concerns/*.rb.\n# The capture tree must reproduce the Ruby-DSL parser's tree exactly:\n# - repetitions are ABNF prefixes (1*x, *x)\n# - negation is !x, positive lookahead is &x\n# - captures at use sites (rule_ref as name) nest inner captures\n# the same way parslet .as does\n#\n# Consumed by the interscript gem (artifact parse) and parsanol npm.\n\ngrammar InterscriptIsc version \"0.1.0\" {\n\n # -- lexical primitives -------------------------------------------------\n\n chr = %x00-10FFFF\n space_char = %x20 / %x09 / %x0A / %x0B / %x0C / %x0D\n newline = %x0D.0A / %x0A / %x0D\n line_comment = \"#\" *(!(newline) chr)\n whitespace = 1*(space_char / line_comment)\n ws = *(space_char / line_comment)\n inline_space = 1*(%x20 / %x09)\n inline_ws = *(inline_space / line_comment)\n hexd = %x30-39 / %x41-46 / %x61-66\n digit = %x30-39\n\n ident_first = %x41-5A / %x61-7A / %x5F\n ident_rest = %x41-5A / %x61-7A / %x30-39 / %x5F\n identifier = (ident_first 0*ident_rest) as identifier\n\n keyword = \"parallel\" / \"sequence\" / \"stage\" / \"compose\" / \"separate\" / \"system\" / \"metadata\" / \"aliases\" / \"tests\" / \"notes\" / \"description\" / \"any_character\" / \"authority\" / \"dependency\" / \"run\" / \"sub\" / \"before\" / \"after\" / \"not_before\" / \"not_after\" / \"any\" / \"none\" / \"boundary\" / \"line_start\" / \"line_end\" / \"word_boundary\" / \"downcase\" / \"upcase\" / \"title_case\" / \"capture\" / \"maybe\" / \"some\" / \"ref\" / \"name\"\n\n # -- string literals ----------------------------------------------------\n\n esc_nl = (\"n\") as newline\n esc_cr = (\"r\") as carriage_return\n esc_tab = (\"t\") as tab\n esc_dq = \"\\\"\" as dquote\n esc_bs = \"\\\\\" as backslash\n esc_uni4 = \"u\" (4hexd) as unicode\n esc_uni8 = \"U\" (4*8hexd) as unicode\n esc_u_lone = \"U\" as u_lone\n escape_sequence = \"\\\\\" (esc_nl / esc_cr / esc_tab / esc_dq / esc_bs / esc_uni4 / esc_uni8 / esc_u_lone)\n\n raw_run = 1*(!(\"\\\\\" / \"\\\"\") chr)\n dq_parts = *(escape_sequence / (raw_run as run))\n double_quoted = \"\\\"\" (dq_parts as string) \"\\\"\"\n sq_body = *(!\"'\" chr)\n single_quoted = \"'\" (sq_body as string) \"'\"\n quoted_string = double_quoted / single_quoted\n\n arrow = [whitespace] \"->\" [whitespace]\n comma_ws = \",\" [whitespace]\n\n # -- metadata -----------------------------------------------------------\n\n md_authority = \"authority\" whitespace ((quoted_string) as authority)\n md_source_spelling = \"source_spelling\" whitespace ((quoted_string) as source_spelling)\n md_target_spelling = \"target_spelling\" whitespace ((quoted_string) as target_spelling)\n md_identifying = \"identifying\" whitespace ((quoted_string) as identifying)\n md_name = \"name\" whitespace ((quoted_string) as name)\n md_spec_value = quoted_string as specification\n md_specification = \"specification\" whitespace ((md_spec_value *(comma_ws md_spec_value)) as specification)\n md_description = \"description\" [whitespace] \"{\" [ws] (raw_text as description) [ws] \"}\"\n md_system_status = \"system_status\" whitespace ((\"current\" / \"former\" / \"inactive\") as system_status)\n md_code_status = \"code_status\" whitespace ((\"preferred\" / \"proposed\" / \"deprecated\") as code_status)\n\n relation_type = \"supersedes\" / \"superseded_by\" / \"based_on\" / \"basis_for\" / \"alias_of\" / \"adopted_from\" / \"related_to\"\n relation_note = whitespace \"note\" whitespace (quoted_string as note)\n relation_item = [whitespace] (relation_type as type) whitespace (quoted_string as system) [(relation_note)] [whitespace]\n md_relations = \"relations\" [whitespace] \"{\" [ws] ((*(relation_item)) as relations) [ws] \"}\"\n\n note_item = [whitespace] \"note\" [whitespace] (quoted_string as note) [whitespace]\n md_notes = \"notes\" [whitespace] \"{\" [ws] ((*(note_item)) as notes) [ws] \"}\"\n\n md_provenance_value = quoted_string as provenance\n md_provenance = \"provenance\" [whitespace] ((md_provenance_value *(comma_ws md_provenance_value)) as provenance)\n\n md_empty_field = &(newline)\n md_raw_field_value = 1*(!%x0A !\"}\" !\"{\" chr)\n md_field_value = quoted_string / (md_raw_field_value as raw)\n md_field_block = \"{\" [ws] (raw_text as field_block) [ws] \"}\"\n md_generic_field = (identifier as field_name) inline_ws ((md_field_value as field_value) / md_field_block / md_empty_field)\n\n metadata_field = [whitespace] (md_description / md_relations / md_system_status / md_code_status / md_specification / md_notes / md_provenance / md_generic_field) [whitespace]\n md_items = *(metadata_field)\n metadata_block = \"metadata\" [whitespace] \"{\" [ws] (md_items as metadata) [ws] \"}\"\n\n raw_text = *(\"\\\\\" chr / (!\"}\" chr))\n\n # -- aliases ------------------------------------------------------------\n\n alias_decl = [whitespace] (identifier as name) [whitespace] \"=\" [whitespace] (item as value) [whitespace]\n alias_items = *(alias_decl)\n aliases_block = \"aliases\" [whitespace] \"{\" [ws] (alias_items as aliases) [ws] \"}\"\n\n # -- dependencies -------------------------------------------------------\n\n dep_alias = whitespace \"as\" whitespace (identifier as alias)\n dependency_decl = \"dependency\" whitespace (quoted_string as target) [(dep_alias)]\n\n # -- item expressions ---------------------------------------------------\n\n zw_primitive = ((\"boundary\") / (\"line_start\") / (\"line_end\") / (\"word_boundary\") / (\"space\") / (\"non_boundary\")) as primitive\n fn_lit = ((\"upcase\") / (\"downcase\") / (\"title_case\") / (\"reverse\") / (\"strip\") / (\"swapcase\")) as function\n any_char_lit = (\"any_character\") as any_char\n none_lit = (\"none\") as none\n\n digit_cap = digit as digit\n capture_reference_body = \"ref\" \"(\" [ws] digit_cap [ws] \")\"\n capture_reference = capture_reference_body as ref\n\n item_ref = identifier \".\" (identifier as qualified_name)\n alias_reference = (!(keyword) identifier [(\".\" (identifier as qualified_name))] !\"{\" ) as alias\n\n item_seq = item_atom\n item_sep = ([ws] \"+\" [ws]) / whitespace\n item_cont = &item_atom\n\n range_arg = (quoted_string as lo) [ws] \"..\" [ws] (quoted_string as hi)\n list_item = item\n list_items = (list_item *((comma_ws / whitespace) list_item))\n set_arg = ((quoted_string as single)) / (\"[\" [ws] ((list_item *((comma_ws / whitespace) list_item)) as list) [ws] \"]\")\n alias_arg = (!zw_primitive !(keyword) identifier) as alias_ref\n any_arg = range_arg / set_arg / alias_arg\n any_ctor = \"any\" \"(\" [ws] (any_arg as any) [ws] \")\"\n capture_ctor = \"capture\" \"(\" [ws] (item as capture_inner) [ws] \")\"\n maybe_ctor = \"maybe\" \"(\" [ws] (item as maybe_inner) [ws] \")\"\n some_ctor = \"some\" \"(\" [ws] (item as some_inner) [ws] \")\"\n\n item_atom_start = \"\\\"\" / \"'\" / ident_first\n cont_stop = \"to\" / \"before\" / \"after\" / \"not_before\" / \"not_after\" / \"}\"\n alias_decl_stop = identifier [ws] \"=\"\n item_atom = quoted_string / none_lit / zw_primitive / any_char_lit / any_ctor / capture_ctor / maybe_ctor / some_ctor / fn_lit / capture_reference / alias_reference\n item = (item_atom 0*(item_sep !cont_stop !(alias_decl_stop) &item_atom_start item_atom)) as concatenation\n\n # -- rules and stages ---------------------------------------------------\n\n constraint_after = \"after\" whitespace (item as after)\n constraint_before = \"before\" whitespace (item as before)\n constraint_not_before = \"not_before\" whitespace (item as not_before)\n constraint_not_after = \"not_after\" whitespace (item as not_after)\n constraint = constraint_after / constraint_before / constraint_not_before / constraint_not_after\n constraints = *(whitespace constraint)\n\n compact_rule = \"sub\" whitespace (item_atom as from) whitespace (item_atom as to) ((*(whitespace constraint)) as constraints)\n\n block_rule = \"sub\" [ws] \"{\" [ws] \"from\" whitespace (item as from) [ws] \"to\" whitespace (item as to) ((*(whitespace constraint)) as constraints) [ws] \"}\"\n\n rule = block_rule / compact_rule\n\n comment_item = ((\"#\" *(!%x0A chr)) as comment) / ((!(keyword) identifier) as noop)\n rule_line = [ws] (rule / comment_item) [ws]\n rule_items = *(rule_line)\n sequence_block = \"sequence\" [ws] \"{\" [ws] (rule_items as sequence) [ws] \"}\"\n parallel_block = \"parallel\" [ws] \"{\" [ws] (rule_items as parallel) [ws] \"}\"\n\n separator_part = whitespace \"separator\" whitespace (item_atom as separator)\n separate_directive = (\"separate\") as separate [(separator_part)]\n case_directive = ((\"downcase\") / (\"upcase\")) as case\n case_kwargs_part = whitespace (kwarg_list as case_kwargs)\n title_case_directive = (\"title_case\") as case [(case_kwargs_part)]\n string_case_directive = case_directive / title_case_directive\n compose_directive = ((\"compose\") / (\"decompose\")) as compose\n kwarg = (identifier as kwarg_name) [ws] \":\" [ws] ((quoted_string) as kwarg_value)\n kwarg_item = (kwarg) as kwarg\n kwarg_list = (kwarg_item *(comma_ws kwarg_item))\n funcall_kwargs_part = whitespace (kwarg_list as funcall_kwargs)\n rababa_directive = (\"rababa\") as funcall_name [(funcall_kwargs_part)]\n run_dep = \"map.\" (identifier as dep) \".stage.\" (identifier as stage)\n run_stage_only = (\"stage.\" (identifier as stage)) as run_stage_only\n run_rule = \"run\" whitespace ((run_dep) / run_stage_only)\n\n bare_rule = (rule) as bare_rule\n stage_item = [ws] (sequence_block / parallel_block / run_rule / separate_directive / string_case_directive / compose_directive / rababa_directive / bare_rule / comment_item) [ws]\n stage_items = *(stage_item)\n stage_name_paren = \"(\" (identifier as stage_name) \")\"\n stage_name_ws = whitespace (identifier as stage_name)\n stage_block = \"stage\" (stage_name_paren / stage_name_ws) [ws] \"{\" [ws] (stage_items as stage) [ws] \"}\"\n\n # -- tests ---------------------------------------------------------------\n\n test_note = whitespace \"note\" whitespace (quoted_string as note)\n test_line = [whitespace] (quoted_string as input) arrow (quoted_string as expected) [(test_note)] [whitespace]\n test_items = *(test_line)\n tests_block = \"tests\" [whitespace] \"{\" [ws] (test_items as tests) [ws] \"}\"\n\n # -- system --------------------------------------------------------------\n\n block_item = [whitespace] (metadata_block / aliases_block / tests_block / stage_block / dependency_decl) [whitespace]\n body_items = *(block_item)\n system_block = \"system\" whitespace ((quoted_string) as system_code) [whitespace] \"{\" [ws] ((*(block_item)) as body) [ws] \"}\" [whitespace]\n isc_source = [whitespace] ((system_block) as system) [whitespace]\n\n entry isc: isc_source\n}\n","checksum":"sha256:ae411976ee55f727a1ebbf22820cff8000c1d858eb1d4008305ef0b7e48403ad"}
|