interscript 2.4.5 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +15 -0
- data/docs/demo/20191118-interscript-demo-cast.gif +0 -0
- data/exe/codemod-imp-to-isc +5 -0
- data/exe/diagnose_parse_failures +47 -0
- data/exe/interscript +2 -1
- data/exe/verify_isc_deep +211 -0
- data/exe/verify_isc_equivalence +128 -0
- data/interscript.gemspec +27 -20
- data/lib/interscript/command.rb +21 -12
- data/lib/interscript/compiler/javascript.rb +34 -35
- data/lib/interscript/compiler/json_ir.rb +214 -0
- data/lib/interscript/compiler/python.rb +354 -0
- data/lib/interscript/compiler/ruby.rb +23 -22
- data/lib/interscript/compiler.rb +20 -3
- data/lib/interscript/detector.rb +14 -7
- data/lib/interscript/dsl/aliases.rb +1 -1
- data/lib/interscript/dsl/document.rb +6 -3
- data/lib/interscript/dsl/group/parallel.rb +1 -1
- data/lib/interscript/dsl/group.rb +18 -9
- data/lib/interscript/dsl/items.rb +28 -16
- data/lib/interscript/dsl/metadata.rb +19 -18
- data/lib/interscript/dsl/stage.rb +1 -1
- data/lib/interscript/dsl/symbol_mm.rb +4 -2
- data/lib/interscript/dsl/tests.rb +1 -1
- data/lib/interscript/dsl.rb +21 -13
- data/lib/interscript/interpreter.rb +32 -21
- data/lib/interscript/isc/codemod.rb +791 -0
- data/lib/interscript/isc/document_builder.rb +354 -0
- data/lib/interscript/isc/generator.rb +191 -0
- data/lib/interscript/isc/grammar/concerns/aliases.rb +29 -0
- data/lib/interscript/isc/grammar/concerns/dependencies.rb +23 -0
- data/lib/interscript/isc/grammar/concerns/items.rb +176 -0
- data/lib/interscript/isc/grammar/concerns/metadata.rb +142 -0
- data/lib/interscript/isc/grammar/concerns/primitives.rb +112 -0
- data/lib/interscript/isc/grammar/concerns/stages.rb +129 -0
- data/lib/interscript/isc/grammar/concerns/system.rb +37 -0
- data/lib/interscript/isc/grammar/concerns/tests.rb +31 -0
- data/lib/interscript/isc/grammar/concerns.rb +18 -0
- data/lib/interscript/isc/grammar/core.rb +23 -0
- data/lib/interscript/isc/grammar/isc.artifact.json +1 -0
- data/lib/interscript/isc/grammar/isc.parg +196 -0
- data/lib/interscript/isc/grammar.rb +12 -0
- data/lib/interscript/isc/items.rb +185 -0
- data/lib/interscript/isc/model/alias.rb +20 -0
- data/lib/interscript/isc/model/constraint.rb +20 -0
- data/lib/interscript/isc/model/dependency.rb +19 -0
- data/lib/interscript/isc/model/document.rb +31 -0
- data/lib/interscript/isc/model/item.rb +50 -0
- data/lib/interscript/isc/model/rule.rb +23 -0
- data/lib/interscript/isc/model/stage.rb +20 -0
- data/lib/interscript/isc/model/stage_item.rb +33 -0
- data/lib/interscript/isc/model/test.rb +21 -0
- data/lib/interscript/isc/model.rb +19 -0
- data/lib/interscript/isc/node_adapter.rb +310 -0
- data/lib/interscript/isc/normalizer.rb +127 -0
- data/lib/interscript/isc/parser.rb +78 -0
- data/lib/interscript/isc/serializer.rb +248 -0
- data/lib/interscript/isc/transform.rb +138 -0
- data/lib/interscript/isc/yaml_bridge.rb +236 -0
- data/lib/interscript/isc.rb +30 -0
- data/lib/interscript/ml/byt5_onnx.rb +102 -0
- data/lib/interscript/ml/imf.rb +182 -0
- data/lib/interscript/ml/model.rb +52 -0
- data/lib/interscript/ml/provisioning.rb +189 -0
- data/lib/interscript/ml/translator.rb +38 -0
- data/lib/interscript/ml/vocab.rb +48 -0
- data/lib/interscript/ml.rb +25 -0
- data/lib/interscript/node/alias_def.rb +5 -5
- data/lib/interscript/node/dependency.rb +7 -7
- data/lib/interscript/node/document.rb +10 -10
- data/lib/interscript/node/group.rb +11 -9
- data/lib/interscript/node/item/alias.rb +14 -10
- data/lib/interscript/node/item/any.rb +14 -7
- data/lib/interscript/node/item/capture.rb +14 -9
- data/lib/interscript/node/item/group.rb +22 -17
- data/lib/interscript/node/item/repeat.rb +3 -3
- data/lib/interscript/node/item/stage.rb +4 -5
- data/lib/interscript/node/item/string.rb +18 -13
- data/lib/interscript/node/item.rb +27 -17
- data/lib/interscript/node/metadata.rb +6 -5
- data/lib/interscript/node/rule/funcall.rb +4 -5
- data/lib/interscript/node/rule/run.rb +4 -5
- data/lib/interscript/node/rule/sub.rb +55 -55
- data/lib/interscript/node/rule.rb +6 -5
- data/lib/interscript/node/stage.rb +11 -14
- data/lib/interscript/node/tests.rb +6 -6
- data/lib/interscript/node.rb +14 -17
- data/lib/interscript/stdlib/functions/rababa_adapter.rb +56 -0
- data/lib/interscript/stdlib/functions/secryst_adapter.rb +37 -0
- data/lib/interscript/stdlib/functions.rb +49 -0
- data/lib/interscript/stdlib.rb +25 -95
- data/lib/interscript/utils/helpers.rb +8 -5
- data/lib/interscript/utils/regexp_converter.rb +143 -144
- data/lib/interscript/version.rb +1 -1
- data/lib/interscript/visualize/json.rb +12 -13
- data/lib/interscript/visualize/nodes.rb +13 -13
- data/lib/interscript/visualize.rb +18 -18
- data/lib/interscript.rb +59 -45
- metadata +88 -31
- data/.github/workflows/assets.yml +0 -91
- data/.github/workflows/rake.yml +0 -52
- data/.github/workflows/release.yml +0 -68
- data/.gitignore +0 -79
- data/.rspec +0 -3
- data/Gemfile +0 -38
- data/Rakefile +0 -139
- data/bin/console +0 -10
- data/bin/interscript +0 -5
- data/bin/maps_analyze_staging +0 -168
- data/bin/maps_debug_compilers +0 -58
- data/bin/maps_debug_ordering +0 -88
- data/bin/maps_debug_ruby_compile +0 -24
- data/bin/maps_debug_step_by_step +0 -44
- data/bin/maps_optimize_order +0 -112
- data/bin/maps_v1_analyze_regexps +0 -45
- data/bin/maps_v1_to_v2 +0 -426
- data/bin/set_version +0 -16
- data/bin/setup +0 -8
- data/requirements.txt +0 -1
|
@@ -0,0 +1,354 @@
|
|
|
1
|
+
require "pycall"
|
|
2
|
+
|
|
3
|
+
class Interscript::Compiler::Python < Interscript::Compiler
|
|
4
|
+
def escape(val)
|
|
5
|
+
case val
|
|
6
|
+
when String, Integer
|
|
7
|
+
val.inspect
|
|
8
|
+
when Symbol
|
|
9
|
+
val.to_s.inspect
|
|
10
|
+
when Hash
|
|
11
|
+
"{" +
|
|
12
|
+
val.map { |k, v| "#{escape k}:#{escape v}" }.join(",") +
|
|
13
|
+
"}"
|
|
14
|
+
when Array
|
|
15
|
+
"[" + val.map { |i| escape i }.join(",") + "]"
|
|
16
|
+
when nil
|
|
17
|
+
"None"
|
|
18
|
+
else
|
|
19
|
+
pp [:error, val]
|
|
20
|
+
exit!
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def re_escape(val)
|
|
25
|
+
@pycall_regex ||= PyCall.import_module("regex")
|
|
26
|
+
@pycall_regex.escape(val).gsub("\\", "\\\\\\\\").gsub('"', "\\\\\"")
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def new_regexp(str)
|
|
30
|
+
"re.compile(\"#{str}\", re.MULTILINE)"
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def indent
|
|
34
|
+
@indent += 4
|
|
35
|
+
yield
|
|
36
|
+
@indent -= 4
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def emit(code)
|
|
40
|
+
@code << (" " * @indent) << code << "\n"
|
|
41
|
+
code
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def compile(map, debug: false)
|
|
45
|
+
@indent = 0
|
|
46
|
+
@map = map
|
|
47
|
+
@debug = debug
|
|
48
|
+
@parallel_trees = {}
|
|
49
|
+
@parallel_regexps = {}
|
|
50
|
+
@code = ""
|
|
51
|
+
emit "import interscript"
|
|
52
|
+
emit "import regex as re"
|
|
53
|
+
map.dependencies.map(&:full_name).each do |dep|
|
|
54
|
+
emit "interscript.load_map(#{escape dep})"
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
emit "interscript.stdlib.define_map(#{escape map.name})"
|
|
58
|
+
|
|
59
|
+
map.aliases.each do |name, value|
|
|
60
|
+
val = compile_item(value.data, map, :str)
|
|
61
|
+
emit "interscript.stdlib.add_map_alias(#{escape map.name}, #{escape name}, #{val})"
|
|
62
|
+
val = "\"" + compile_item(value.data, map, :re) + "\""
|
|
63
|
+
emit "interscript.stdlib.add_map_alias_re(#{escape map.name}, #{escape name}, #{val})"
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
map.stages.each do |_, stage|
|
|
67
|
+
compile_rule(stage, @map, true)
|
|
68
|
+
end
|
|
69
|
+
@parallel_trees.each do |k, v|
|
|
70
|
+
emit "_PTREE_#{k} = #{escape v}"
|
|
71
|
+
end
|
|
72
|
+
@parallel_regexps.each do |k, v|
|
|
73
|
+
v = %(["#{v[0]}", #{escape v[1]}])
|
|
74
|
+
emit "_PRE_#{k} = #{v}"
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def parallel_regexp_compile(subs_hash)
|
|
79
|
+
# puts subs_hash.inspect
|
|
80
|
+
subs_hash.each_with_index.map do |p, i|
|
|
81
|
+
"(?P<_%d>%s)" % [i, p[0]]
|
|
82
|
+
end.join("|")
|
|
83
|
+
|
|
84
|
+
# puts subs_regexp.inspect
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def compile_rule(r, map = @map, wrapper = false)
|
|
88
|
+
return if r.reverse_run == true
|
|
89
|
+
case r
|
|
90
|
+
when Interscript::Node::Stage
|
|
91
|
+
if @debug
|
|
92
|
+
emit "if not hasattr(interscript, 'map_debug'):"
|
|
93
|
+
indent { emit "interscript.map_debug = []" }
|
|
94
|
+
end
|
|
95
|
+
emit "def _stage_#{r.name}(s):"
|
|
96
|
+
indent do
|
|
97
|
+
r.children.each do |t|
|
|
98
|
+
comp = compile_rule(t, map)
|
|
99
|
+
emit %{interscript.map_debug.append([s, #{escape @map.name.to_s}, #{escape r.name.to_s}, #{escape t.inspect}, #{escape comp}])} if @debug
|
|
100
|
+
end
|
|
101
|
+
emit "return s\n"
|
|
102
|
+
end
|
|
103
|
+
emit "interscript.stdlib.add_map_stage(#{escape @map.name}, #{escape r.name}, _stage_#{r.name})"
|
|
104
|
+
when Interscript::Node::Group::Parallel
|
|
105
|
+
begin
|
|
106
|
+
# Try to build a tree
|
|
107
|
+
a = []
|
|
108
|
+
r.children.each do |i|
|
|
109
|
+
raise Interscript::SystemConversionError, "Can't parallelize #{i.class}" unless Interscript::Node::Rule::Sub === i
|
|
110
|
+
raise Interscript::SystemConversionError, "Can't parallelize rules with :before" if i.before
|
|
111
|
+
raise Interscript::SystemConversionError, "Can't parallelize rules with :after" if i.after
|
|
112
|
+
raise Interscript::SystemConversionError, "Can't parallelize rules with :not_before" if i.not_before
|
|
113
|
+
raise Interscript::SystemConversionError, "Can't parallelize rules with :not_after" if i.not_after
|
|
114
|
+
|
|
115
|
+
next if i.reverse_run == true
|
|
116
|
+
a << [compile_item(i.from, map, :par), compile_item(i.to, map, :parstr)]
|
|
117
|
+
end
|
|
118
|
+
ah = a.hash.abs
|
|
119
|
+
unless @parallel_trees.include? ah
|
|
120
|
+
tree = Interscript::Stdlib.parallel_replace_compile_tree(a)
|
|
121
|
+
@parallel_trees[ah] = tree
|
|
122
|
+
end
|
|
123
|
+
emit "s = interscript.stdlib.parallel_replace_tree(s, _PTREE_#{ah})"
|
|
124
|
+
rescue
|
|
125
|
+
# Otherwise let's build a megaregexp
|
|
126
|
+
a = []
|
|
127
|
+
Interscript::Stdlib.deterministic_sort_by_max_length(r.children).each do |i|
|
|
128
|
+
raise Interscript::SystemConversionError, "Can't parallelize #{i.class}" unless Interscript::Node::Rule::Sub === i
|
|
129
|
+
|
|
130
|
+
next if i.reverse_run == true
|
|
131
|
+
a << [build_regexp(i, map), compile_item(i.to, map, :parstr)]
|
|
132
|
+
end
|
|
133
|
+
ah = a.hash.abs
|
|
134
|
+
unless @parallel_regexps.include? ah
|
|
135
|
+
re = parallel_regexp_compile(a)
|
|
136
|
+
@parallel_regexps[ah] = [re, a.map(&:last)]
|
|
137
|
+
end
|
|
138
|
+
emit "s = interscript.stdlib.parallel_regexp_gsub(s, *_PRE_#{ah})"
|
|
139
|
+
end
|
|
140
|
+
when Interscript::Node::Rule::Sub
|
|
141
|
+
from = new_regexp build_regexp(r, map)
|
|
142
|
+
to = if r.to == :upcase
|
|
143
|
+
"interscript.stdlib.upper"
|
|
144
|
+
elsif r.to == :downcase
|
|
145
|
+
"interscript.stdlib.lower"
|
|
146
|
+
else
|
|
147
|
+
compile_item(r.to, map, :str)
|
|
148
|
+
end
|
|
149
|
+
emit "s = #{from}.sub(#{to}, s)"
|
|
150
|
+
when Interscript::Node::Rule::Funcall
|
|
151
|
+
emit "s = interscript.functions.#{r.name}(s, #{escape r.kwargs})"
|
|
152
|
+
when Interscript::Node::Rule::Run
|
|
153
|
+
if r.stage.map
|
|
154
|
+
doc = map.dep_aliases[r.stage.map].document
|
|
155
|
+
stage = doc.imported_stages[r.stage.name]
|
|
156
|
+
else
|
|
157
|
+
stage = map.imported_stages[r.stage.name]
|
|
158
|
+
end
|
|
159
|
+
emit "s = interscript.transliterate(#{escape stage.doc_name}, s, #{escape stage.name})"
|
|
160
|
+
else
|
|
161
|
+
raise Interscript::SystemConversionError, "Can't compile unhandled #{r.class}"
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def build_regexp(r, map = @map)
|
|
166
|
+
from = compile_item(r.from, map, :re)
|
|
167
|
+
before = compile_item(r.before, map, :re) if r.before
|
|
168
|
+
after = compile_item(r.after, map, :re) if r.after
|
|
169
|
+
not_before = compile_item(r.not_before, map, :re) if r.not_before
|
|
170
|
+
not_after = compile_item(r.not_after, map, :re) if r.not_after
|
|
171
|
+
|
|
172
|
+
re = ""
|
|
173
|
+
re += "(?<=#{before})" if before
|
|
174
|
+
re += "(?<!#{not_before})" if not_before
|
|
175
|
+
re += from
|
|
176
|
+
re += "(?!#{not_after})" if not_after
|
|
177
|
+
re += "(?=#{after})" if after
|
|
178
|
+
re
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
def compile_item i, doc = @map, target = nil
|
|
182
|
+
i = i.first_string if %i[str parstr].include? target
|
|
183
|
+
i = Interscript::Node::Item.try_convert(i)
|
|
184
|
+
if target == :parstr
|
|
185
|
+
parstr = true
|
|
186
|
+
target = :par
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
case i
|
|
190
|
+
when Interscript::Node::Item::Alias
|
|
191
|
+
astr = if i.map
|
|
192
|
+
d = doc.dep_aliases[i.map].document
|
|
193
|
+
a = d.imported_aliases[i.name]
|
|
194
|
+
raise Interscript::SystemConversionError, "Alias #{i.name} of #{i.stage.map} not found" unless a
|
|
195
|
+
"interscript.stdlib.get_alias_ALIASTYPE(#{escape a.doc_name}, #{escape a.name})"
|
|
196
|
+
elsif Interscript::Stdlib::ALIASES.include?(i.name)
|
|
197
|
+
if target != :re && Interscript::Stdlib.re_only_alias?(i.name)
|
|
198
|
+
raise Interscript::SystemConversionError, "Can't use #{i.name} in a #{target} context"
|
|
199
|
+
end
|
|
200
|
+
stdlib_alias = true
|
|
201
|
+
"interscript.stdlib.aliases[#{escape i.name}]"
|
|
202
|
+
else
|
|
203
|
+
a = doc.imported_aliases[i.name]
|
|
204
|
+
raise Interscript::SystemConversionError, "Alias #{i.name} not found" unless a
|
|
205
|
+
|
|
206
|
+
"interscript.stdlib.get_alias_ALIASTYPE(#{escape a.doc_name}, #{escape a.name})"
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
if target == :str
|
|
210
|
+
astr.sub("_ALIASTYPE(", "(")
|
|
211
|
+
elsif target == :re
|
|
212
|
+
"\"+#{astr.sub("_ALIASTYPE(", "_re(")}+\""
|
|
213
|
+
elsif parstr && stdlib_alias
|
|
214
|
+
Interscript::Stdlib::ALIASES[i.name]
|
|
215
|
+
elsif target == :par
|
|
216
|
+
# raise NotImplementedError, "Can't use aliases in parallel mode yet"
|
|
217
|
+
Interscript::Stdlib::ALIASES[i.name]
|
|
218
|
+
end
|
|
219
|
+
when Interscript::Node::Item::String
|
|
220
|
+
if target == :str
|
|
221
|
+
# Replace \1 with \\1, this is weird, but it works!
|
|
222
|
+
i.data.gsub("\\", "\\\\\\\\").inspect
|
|
223
|
+
elsif target == :par
|
|
224
|
+
i.data
|
|
225
|
+
elsif target == :re
|
|
226
|
+
re_escape(i.data)
|
|
227
|
+
end
|
|
228
|
+
when Interscript::Node::Item::Group
|
|
229
|
+
if target == :par
|
|
230
|
+
i.children.map do |j|
|
|
231
|
+
compile_item(j, doc, target)
|
|
232
|
+
end.reduce([""]) do |j, k|
|
|
233
|
+
Array(j).product(Array(k)).map(&:join)
|
|
234
|
+
end
|
|
235
|
+
elsif target == :str
|
|
236
|
+
i.children.map { |j| compile_item(j, doc, target) }.join("+")
|
|
237
|
+
elsif target == :re
|
|
238
|
+
i.children.map { |j| compile_item(j, doc, target) }.join
|
|
239
|
+
end
|
|
240
|
+
when Interscript::Node::Item::CaptureGroup
|
|
241
|
+
if target != :re
|
|
242
|
+
raise Interscript::SystemConversionError, "Can't use a CaptureGroup in a #{target} context"
|
|
243
|
+
end
|
|
244
|
+
"(" + compile_item(i.data, doc, target) + ")"
|
|
245
|
+
when Interscript::Node::Item::Maybe,
|
|
246
|
+
Interscript::Node::Item::MaybeSome,
|
|
247
|
+
Interscript::Node::Item::Some
|
|
248
|
+
|
|
249
|
+
resuffix = {Interscript::Node::Item::Maybe => "?",
|
|
250
|
+
Interscript::Node::Item::Some => "+",
|
|
251
|
+
Interscript::Node::Item::MaybeSome => "*"}[i.class]
|
|
252
|
+
|
|
253
|
+
if target == :par
|
|
254
|
+
raise Interscript::SystemConversionError, "Can't use a Maybe in a #{target} context"
|
|
255
|
+
end
|
|
256
|
+
if Interscript::Node::Item::String === i.data && i.data.data.length != 1
|
|
257
|
+
"(?:" + compile_item(i.data, doc, target) + ")" + resuffix
|
|
258
|
+
else
|
|
259
|
+
compile_item(i.data, doc, target) + resuffix
|
|
260
|
+
end
|
|
261
|
+
when Interscript::Node::Item::CaptureRef
|
|
262
|
+
if target == :par
|
|
263
|
+
raise Interscript::SystemConversionError, "Can't use CaptureRef in parallel mode"
|
|
264
|
+
elsif target == :re
|
|
265
|
+
"\\\\#{i.id}"
|
|
266
|
+
elsif target == :str
|
|
267
|
+
"\"\\\\#{i.id}\""
|
|
268
|
+
end
|
|
269
|
+
when Interscript::Node::Item::Any
|
|
270
|
+
if target == :str
|
|
271
|
+
raise Interscript::SystemConversionError, "Can't use Any in a string context" # A linter could find this!
|
|
272
|
+
elsif target == :par
|
|
273
|
+
i.data.map(&:data)
|
|
274
|
+
elsif target == :re
|
|
275
|
+
case i.value
|
|
276
|
+
when Array
|
|
277
|
+
data = i.data.map { |j| compile_item(j, doc, target) }
|
|
278
|
+
"(?:" + data.join("|") + ")"
|
|
279
|
+
when String
|
|
280
|
+
"[#{re_escape(i.value)}]"
|
|
281
|
+
when Range
|
|
282
|
+
"[#{re_escape(i.value.first)}-#{re_escape(i.value.last)}]"
|
|
283
|
+
end
|
|
284
|
+
end
|
|
285
|
+
end
|
|
286
|
+
end
|
|
287
|
+
|
|
288
|
+
@maps_loaded = {}
|
|
289
|
+
@ctx = nil
|
|
290
|
+
class << self
|
|
291
|
+
attr_accessor :maps_loaded
|
|
292
|
+
attr_accessor :ctx
|
|
293
|
+
attr_accessor :maps_dir
|
|
294
|
+
end
|
|
295
|
+
|
|
296
|
+
def load
|
|
297
|
+
# Keyed by document identity: synthetic spec documents reuse names
|
|
298
|
+
# (example-1, example-2, ...), and a stale engine under a reused
|
|
299
|
+
# name would silently serve the previous document.
|
|
300
|
+
if self.class.maps_loaded[@map.name] != @map.object_id
|
|
301
|
+
@map.dependencies.each do |dep|
|
|
302
|
+
dep = dep.full_name
|
|
303
|
+
if !self.class.maps_loaded[dep]
|
|
304
|
+
Interscript.load(dep, compiler: self.class).load
|
|
305
|
+
end
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
ctx = self.class.ctx
|
|
309
|
+
python_src_path = ENV["INTERSCRIPT_PYTHON_SRC"] ||
|
|
310
|
+
File.join(__dir__, "..", "..", "..", "..", "python", "src")
|
|
311
|
+
unless ctx
|
|
312
|
+
# Insert at 0: an editable install of interscript-python may
|
|
313
|
+
# shadow the monorepo checkout on sys.path.
|
|
314
|
+
PyCall.sys.path.insert(0, python_src_path)
|
|
315
|
+
self.class.ctx = PyCall.import_module("interscript")
|
|
316
|
+
ctx = self.class.ctx
|
|
317
|
+
end
|
|
318
|
+
# The rewritten python runtime executes ISC sources from its load
|
|
319
|
+
# paths, so in-memory documents cross the boundary as generated
|
|
320
|
+
# ISC text in a per-process temp dir — never inside the python
|
|
321
|
+
# package tree.
|
|
322
|
+
unless self.class.maps_dir
|
|
323
|
+
require "tmpdir" unless defined?(Dir.mktmpdir)
|
|
324
|
+
self.class.maps_dir = Dir.mktmpdir("interscript-python")
|
|
325
|
+
ctx.add_load_path(self.class.maps_dir)
|
|
326
|
+
end
|
|
327
|
+
File.write(File.join(self.class.maps_dir, "#{@map.name}.isc"),
|
|
328
|
+
Interscript::Isc::Generator.generate(@map))
|
|
329
|
+
self.class.ctx.load_map(@map.name)
|
|
330
|
+
|
|
331
|
+
if self.class.maps_loaded.key?(@map.name)
|
|
332
|
+
self.class.ctx.unload_map(@map.name)
|
|
333
|
+
end
|
|
334
|
+
self.class.maps_loaded[@map.name] = @map.object_id
|
|
335
|
+
end
|
|
336
|
+
end
|
|
337
|
+
|
|
338
|
+
def call(str, stage = :main)
|
|
339
|
+
load
|
|
340
|
+
unless stage.to_s == "main"
|
|
341
|
+
raise Interscript::MapLogicError,
|
|
342
|
+
"the python runtime transliterates the whole map; stage #{stage} can't be selected"
|
|
343
|
+
end
|
|
344
|
+
self.class.ctx.transliterate(@map.name, str)
|
|
345
|
+
end
|
|
346
|
+
|
|
347
|
+
def self.read_debug_data
|
|
348
|
+
(ctx["map_debug"] || []).map(&:to_a).to_a
|
|
349
|
+
end
|
|
350
|
+
|
|
351
|
+
def self.reset_debug_data
|
|
352
|
+
ctx["map_debug"].clear
|
|
353
|
+
end
|
|
354
|
+
end
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# rubocop:disable Style/GlobalVars, Security/Eval -- map-evaluation context globals; eval runs generated map code
|
|
1
2
|
$main_binding = binding
|
|
2
3
|
|
|
3
4
|
class Interscript::Compiler::Ruby < Interscript::Compiler
|
|
@@ -19,22 +20,21 @@ class Interscript::Compiler::Ruby < Interscript::Compiler
|
|
|
19
20
|
c << "def self.get_alias_re(map,name); @maps[map].aliases_re[name]; end\n"
|
|
20
21
|
c << "def self.transliterate(map,string,stage=:main); @maps[map].stages[stage].(string); end\n"
|
|
21
22
|
c << "end; end; end\n"
|
|
22
|
-
c
|
|
23
23
|
|
|
24
24
|
map.aliases.each do |name, value|
|
|
25
25
|
val = compile_item(value.data, map, :str)
|
|
26
26
|
c << "Interscript::Maps.add_map_alias(#{map.name.inspect}, #{name.inspect}, #{val})\n"
|
|
27
|
-
val =
|
|
27
|
+
val = "/" + compile_item(value.data, map, :re).gsub("/", '\\\\/') + "/"
|
|
28
28
|
c << "Interscript::Maps.add_map_alias_re(#{map.name.inspect}, #{name.inspect}, #{val})\n"
|
|
29
29
|
end
|
|
30
30
|
|
|
31
31
|
map.stages.each do |_, stage|
|
|
32
32
|
c << compile_rule(stage, @map, true)
|
|
33
33
|
end
|
|
34
|
-
@parallel_trees.each do |k,v|
|
|
34
|
+
@parallel_trees.each do |k, v|
|
|
35
35
|
c << "Interscript::Maps::Cache::PTREE_#{k} ||= #{v.inspect}\n"
|
|
36
36
|
end
|
|
37
|
-
@parallel_regexps.each do |k,v|
|
|
37
|
+
@parallel_regexps.each do |k, v|
|
|
38
38
|
c << "Interscript::Maps::Cache::PRE_#{k} ||= #{v.inspect}\n"
|
|
39
39
|
end
|
|
40
40
|
@code = c
|
|
@@ -51,7 +51,7 @@ class Interscript::Compiler::Ruby < Interscript::Compiler
|
|
|
51
51
|
r.children.each do |t|
|
|
52
52
|
comp = compile_rule(t, map)
|
|
53
53
|
c += comp
|
|
54
|
-
c += %
|
|
54
|
+
c += %($map_debug << [s.dup, #{@map.name.to_s.inspect}, #{r.name.to_s.inspect}, #{t.inspect.inspect}, #{comp.inspect}]\n) if @debug
|
|
55
55
|
end
|
|
56
56
|
c += "s\n"
|
|
57
57
|
c += "end\n"
|
|
@@ -93,12 +93,12 @@ class Interscript::Compiler::Ruby < Interscript::Compiler
|
|
|
93
93
|
end
|
|
94
94
|
when Interscript::Node::Rule::Sub
|
|
95
95
|
from = "/#{build_regexp(r, map).gsub("/", "\\\\/")}/"
|
|
96
|
-
if r.to == :upcase
|
|
97
|
-
|
|
96
|
+
to = if r.to == :upcase
|
|
97
|
+
"&:upcase"
|
|
98
98
|
elsif r.to == :downcase
|
|
99
|
-
|
|
99
|
+
"&:downcase"
|
|
100
100
|
else
|
|
101
|
-
|
|
101
|
+
compile_item(r.to, map, :str)
|
|
102
102
|
end
|
|
103
103
|
c += "s.gsub!(#{from}, #{to})\n"
|
|
104
104
|
when Interscript::Node::Rule::Funcall
|
|
@@ -117,7 +117,7 @@ class Interscript::Compiler::Ruby < Interscript::Compiler
|
|
|
117
117
|
c
|
|
118
118
|
end
|
|
119
119
|
|
|
120
|
-
def build_regexp(r, map
|
|
120
|
+
def build_regexp(r, map = @map)
|
|
121
121
|
from = compile_item(r.from, map, :re)
|
|
122
122
|
before = compile_item(r.before, map, :re) if r.before
|
|
123
123
|
after = compile_item(r.after, map, :re) if r.after
|
|
@@ -133,7 +133,7 @@ class Interscript::Compiler::Ruby < Interscript::Compiler
|
|
|
133
133
|
re
|
|
134
134
|
end
|
|
135
135
|
|
|
136
|
-
def compile_item i, doc
|
|
136
|
+
def compile_item i, doc = @map, target = nil
|
|
137
137
|
i = i.first_string if %i[str parstr].include? target
|
|
138
138
|
i = Interscript::Node::Item.try_convert(i)
|
|
139
139
|
if target == :parstr
|
|
@@ -141,7 +141,7 @@ class Interscript::Compiler::Ruby < Interscript::Compiler
|
|
|
141
141
|
target = :par
|
|
142
142
|
end
|
|
143
143
|
|
|
144
|
-
|
|
144
|
+
case i
|
|
145
145
|
when Interscript::Node::Item::Alias
|
|
146
146
|
astr = if i.map
|
|
147
147
|
d = doc.dep_aliases[i.map].document
|
|
@@ -162,14 +162,14 @@ class Interscript::Compiler::Ruby < Interscript::Compiler
|
|
|
162
162
|
end
|
|
163
163
|
|
|
164
164
|
if target == :str
|
|
165
|
-
astr
|
|
165
|
+
astr.sub("_ALIASTYPE(", "(")
|
|
166
166
|
elsif target == :re
|
|
167
|
-
|
|
167
|
+
"\#{#{astr.sub("_ALIASTYPE(", "_re(")}}"
|
|
168
168
|
elsif parstr && stdlib_alias
|
|
169
|
-
|
|
169
|
+
Interscript::Stdlib::ALIASES[i.name]
|
|
170
170
|
elsif target == :par
|
|
171
171
|
# raise NotImplementedError, "Can't use aliases in parallel mode yet"
|
|
172
|
-
|
|
172
|
+
Interscript::Stdlib::ALIASES[i.name]
|
|
173
173
|
end
|
|
174
174
|
when Interscript::Node::Item::String
|
|
175
175
|
if target == :str
|
|
@@ -184,7 +184,7 @@ class Interscript::Compiler::Ruby < Interscript::Compiler
|
|
|
184
184
|
if target == :par
|
|
185
185
|
i.children.map do |j|
|
|
186
186
|
compile_item(j, doc, target)
|
|
187
|
-
end.reduce([""]) do |j,k|
|
|
187
|
+
end.reduce([""]) do |j, k|
|
|
188
188
|
Array(j).product(Array(k)).map(&:join)
|
|
189
189
|
end
|
|
190
190
|
elsif target == :str
|
|
@@ -201,9 +201,9 @@ class Interscript::Compiler::Ruby < Interscript::Compiler
|
|
|
201
201
|
Interscript::Node::Item::MaybeSome,
|
|
202
202
|
Interscript::Node::Item::Some
|
|
203
203
|
|
|
204
|
-
resuffix = {
|
|
205
|
-
|
|
206
|
-
|
|
204
|
+
resuffix = {Interscript::Node::Item::Maybe => "?",
|
|
205
|
+
Interscript::Node::Item::Some => "+",
|
|
206
|
+
Interscript::Node::Item::MaybeSome => "*"}[i.class]
|
|
207
207
|
|
|
208
208
|
if target == :par
|
|
209
209
|
raise Interscript::SystemConversionError, "Can't use a Maybe in a #{target} context"
|
|
@@ -230,7 +230,7 @@ class Interscript::Compiler::Ruby < Interscript::Compiler
|
|
|
230
230
|
case i.value
|
|
231
231
|
when Array
|
|
232
232
|
data = i.data.map { |j| compile_item(j, doc, target) }
|
|
233
|
-
"(?:"+data.join("|")+")"
|
|
233
|
+
"(?:" + data.join("|") + ")"
|
|
234
234
|
when String
|
|
235
235
|
"[#{Regexp.escape(i.value)}]"
|
|
236
236
|
when Range
|
|
@@ -252,7 +252,7 @@ class Interscript::Compiler::Ruby < Interscript::Compiler
|
|
|
252
252
|
end
|
|
253
253
|
end
|
|
254
254
|
|
|
255
|
-
def call(str, stage
|
|
255
|
+
def call(str, stage = :main)
|
|
256
256
|
load
|
|
257
257
|
Interscript::Maps.transliterate(@map.name, str, stage)
|
|
258
258
|
end
|
|
@@ -265,3 +265,4 @@ class Interscript::Compiler::Ruby < Interscript::Compiler
|
|
|
265
265
|
$map_debug = []
|
|
266
266
|
end
|
|
267
267
|
end
|
|
268
|
+
# rubocop:enable Style/GlobalVars, Security/Eval
|
data/lib/interscript/compiler.rb
CHANGED
|
@@ -1,16 +1,33 @@
|
|
|
1
1
|
# An Interscript compiler interface
|
|
2
2
|
class Interscript::Compiler
|
|
3
|
+
# Autoload all compiler variants (OCP: new compiler = one autoload).
|
|
4
|
+
autoload :Javascript, "interscript/compiler/javascript"
|
|
5
|
+
autoload :Python, "interscript/compiler/python"
|
|
6
|
+
autoload :Ruby, "interscript/compiler/ruby"
|
|
7
|
+
autoload :JsonIR, "interscript/compiler/json_ir"
|
|
8
|
+
|
|
3
9
|
attr_accessor :code
|
|
4
10
|
|
|
5
11
|
def self.call(map, **kwargs)
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
12
|
+
# DSL.parse locates, dispatches .isc and caches the parsed document —
|
|
13
|
+
# calling parse_isc here re-ran a full grammar parse on every
|
|
14
|
+
# compiler instantiation.
|
|
15
|
+
map = Interscript::DSL.parse(map) if String === map
|
|
9
16
|
compiler = new
|
|
10
17
|
compiler.compile(map, **kwargs)
|
|
11
18
|
compiler
|
|
12
19
|
end
|
|
13
20
|
|
|
21
|
+
# Parse an ISC source file into an Interscript::Node::Document.
|
|
22
|
+
# This bridges the new ISC format to the existing runtime.
|
|
23
|
+
def self.parse_isc(path)
|
|
24
|
+
source = File.read(path, encoding: "UTF-8")
|
|
25
|
+
filename = File.basename(path)
|
|
26
|
+
tree = Interscript::Isc::Parser.parse(source, filename: filename)
|
|
27
|
+
doc = Interscript::Isc::DocumentBuilder.build(tree, filename: filename)
|
|
28
|
+
Interscript::Isc::NodeAdapter.to_interscript_node(doc)
|
|
29
|
+
end
|
|
30
|
+
|
|
14
31
|
def compile(map)
|
|
15
32
|
raise NotImplementedError, "Compile method on #{self.class} is not implemented"
|
|
16
33
|
end
|
data/lib/interscript/detector.rb
CHANGED
|
@@ -26,8 +26,8 @@ class Interscript::Detector
|
|
|
26
26
|
end
|
|
27
27
|
|
|
28
28
|
def set_from_kwargs(**kwargs)
|
|
29
|
-
kwargs.each do |k,v|
|
|
30
|
-
|
|
29
|
+
kwargs.each do |k, v|
|
|
30
|
+
public_send(:"#{k}=", v)
|
|
31
31
|
end
|
|
32
32
|
end
|
|
33
33
|
|
|
@@ -36,13 +36,20 @@ class Interscript::Detector
|
|
|
36
36
|
maps = Interscript.exclude_maps(maps, compiler: self.class)
|
|
37
37
|
maps = Interscript.exclude_maps(maps, compiler: @compiler)
|
|
38
38
|
|
|
39
|
-
summary = maps.
|
|
40
|
-
|
|
39
|
+
summary = maps.filter_map do |map|
|
|
40
|
+
# A map that fails to load or execute (unsupported constructs,
|
|
41
|
+
# missing dependencies) is skipped, mirroring the TypeScript
|
|
42
|
+
# detector — one bad map must not kill the whole ranking.
|
|
43
|
+
begin
|
|
44
|
+
try_dest = Interscript.transliterate(map, source, compiler: @compiler)
|
|
45
|
+
rescue
|
|
46
|
+
next nil
|
|
47
|
+
end
|
|
41
48
|
|
|
42
49
|
[map, try_dest]
|
|
43
50
|
end.map do |map, try_dest|
|
|
44
|
-
dist = @distance_computer.(try_dest, destination)
|
|
45
|
-
|
|
51
|
+
dist = @distance_computer.call(try_dest, destination)
|
|
52
|
+
|
|
46
53
|
[map, dist]
|
|
47
54
|
end.sort_by(&:last).to_h
|
|
48
55
|
|
|
@@ -59,4 +66,4 @@ class Interscript::Detector
|
|
|
59
66
|
module DistanceComputer
|
|
60
67
|
Levenshtein = Text::Levenshtein.method(:distance)
|
|
61
68
|
end
|
|
62
|
-
end
|
|
69
|
+
end
|
|
@@ -6,7 +6,7 @@ class Interscript::DSL::Document
|
|
|
6
6
|
def initialize(name = nil, &block)
|
|
7
7
|
@node = Interscript::Node::Document.new
|
|
8
8
|
@node.name = name if name
|
|
9
|
-
|
|
9
|
+
instance_exec(&block) if block_given?
|
|
10
10
|
end
|
|
11
11
|
|
|
12
12
|
def metadata(&block)
|
|
@@ -22,7 +22,10 @@ class Interscript::DSL::Document
|
|
|
22
22
|
def aliases(&block)
|
|
23
23
|
aliases = Interscript::DSL::Aliases.new(&block)
|
|
24
24
|
@node.aliases = aliases.node
|
|
25
|
-
@node.aliases.transform_values { |v|
|
|
25
|
+
@node.aliases.transform_values { |v|
|
|
26
|
+
v.doc_name = @node.name
|
|
27
|
+
v
|
|
28
|
+
}
|
|
26
29
|
end
|
|
27
30
|
|
|
28
31
|
def dependency(full_name, **kargs)
|
|
@@ -38,7 +41,7 @@ class Interscript::DSL::Document
|
|
|
38
41
|
end
|
|
39
42
|
|
|
40
43
|
def stage(name = :main, dont_reverse: false, &block)
|
|
41
|
-
puts "stage(#{name}) from #{
|
|
44
|
+
puts "stage(#{name}) from #{inspect}" if $DEBUG
|
|
42
45
|
stage = Interscript::DSL::Stage.new(name, &block)
|
|
43
46
|
stage.node.doc_name = @node.name
|
|
44
47
|
stage.node.dont_reverse = dont_reverse
|
|
@@ -5,7 +5,7 @@ class Interscript::DSL::Group
|
|
|
5
5
|
|
|
6
6
|
def initialize(&block)
|
|
7
7
|
@node = Interscript::Node::Group.new
|
|
8
|
-
|
|
8
|
+
instance_exec(&block)
|
|
9
9
|
end
|
|
10
10
|
|
|
11
11
|
def run(stage, **kwargs)
|
|
@@ -16,20 +16,27 @@ class Interscript::DSL::Group
|
|
|
16
16
|
end
|
|
17
17
|
|
|
18
18
|
def sub(from, to, **kwargs, &block)
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
19
|
+
if $DEBUG
|
|
20
|
+
puts "sub(#{from.inspect},#{to}, kwargs = #{
|
|
21
|
+
kwargs.inspect
|
|
22
|
+
}) from #{inspect}"
|
|
23
|
+
end
|
|
22
24
|
|
|
23
25
|
rule = Interscript::Node::Rule::Sub.new(from, to, **kwargs)
|
|
24
26
|
@node.children << rule
|
|
25
27
|
end
|
|
26
28
|
|
|
27
|
-
def upcase
|
|
28
|
-
|
|
29
|
+
def upcase
|
|
30
|
+
:upcase
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def downcase
|
|
34
|
+
:downcase
|
|
35
|
+
end
|
|
29
36
|
|
|
30
37
|
Interscript::Stdlib.available_functions.each do |fun|
|
|
31
38
|
define_method fun do |**kwargs|
|
|
32
|
-
puts "funcall(#{fun}, #{kwargs.inspect}) from #{
|
|
39
|
+
puts "funcall(#{fun}, #{kwargs.inspect}) from #{inspect}" if $DEBUG
|
|
33
40
|
|
|
34
41
|
rule = Interscript::Node::Rule::Funcall.new(fun, **kwargs)
|
|
35
42
|
@node.children << rule
|
|
@@ -37,10 +44,12 @@ class Interscript::DSL::Group
|
|
|
37
44
|
end
|
|
38
45
|
|
|
39
46
|
def parallel(**kwargs, &block)
|
|
40
|
-
puts "parallel(#{chars.inspect}) from #{
|
|
47
|
+
puts "parallel(#{chars.inspect}) from #{inspect}" if $DEBUG
|
|
41
48
|
group = Interscript::DSL::Group::Parallel.new(**kwargs, &block)
|
|
42
49
|
@node.children << group.node
|
|
43
50
|
end
|
|
44
51
|
end
|
|
45
52
|
|
|
46
|
-
|
|
53
|
+
class Interscript::DSL::Group
|
|
54
|
+
autoload :Parallel, "interscript/dsl/group/parallel"
|
|
55
|
+
end
|