herb 0.11.1 → 0.11.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (247) hide show
  1. checksums.yaml +4 -4
  2. data/Rakefile +4 -2
  3. data/config.yml +6 -0
  4. data/ext/herb/extconf.rb +8 -31
  5. data/ext/herb/nodes.c +75 -75
  6. data/ext/herb/parser_options_helpers.c +26 -0
  7. data/lib/herb/analysis/partial_index.rb +1 -1
  8. data/lib/herb/analysis/partial_resolution.rb +27 -0
  9. data/lib/herb/ast/helpers.rb +1 -1
  10. data/lib/herb/configuration.rb +11 -1
  11. data/lib/herb/dev/classifier.rb +1 -1
  12. data/lib/herb/engine/compiler.rb +5 -8
  13. data/lib/herb/engine/slots/dynamics_compiler.rb +53 -14
  14. data/lib/herb/engine/slots/subtree_compiler.rb +1 -1
  15. data/lib/herb/engine/slots/visitor.rb +32 -4
  16. data/lib/herb/engine/visitors/instrumentation_visitor.rb +18 -6
  17. data/lib/herb/parser_options.rb +6 -3
  18. data/lib/herb/project.rb +19 -8
  19. data/lib/herb/version.rb +1 -1
  20. data/lib/herb.rb +1 -1
  21. data/sig/herb/analysis/partial_resolution.rbs +8 -0
  22. data/sig/herb/configuration.rbs +3 -0
  23. data/sig/herb/engine/slots/dynamics_compiler.rbs +24 -7
  24. data/sig/herb/engine/slots/visitor.rbs +14 -0
  25. data/sig/herb/engine/visitors/instrumentation_visitor.rbs +5 -1
  26. data/sig/herb/parser_options.rbs +6 -4
  27. data/sig/herb.rbs +2 -2
  28. data/src/analyze/action_view/attribute_extraction_helpers.c +40 -21
  29. data/src/analyze/action_view/generated_handlers.c +36 -36
  30. data/src/analyze/action_view/image_tag.c +7 -3
  31. data/src/analyze/action_view/javascript_include_tag.c +6 -3
  32. data/src/analyze/action_view/link_to.c +23 -23
  33. data/src/analyze/action_view/registry.c +7 -3
  34. data/src/analyze/action_view/tag_helper_node_builders.c +6 -13
  35. data/src/analyze/action_view/tag_helpers.c +139 -90
  36. data/src/analyze/action_view/turbo_frame_tag.c +5 -4
  37. data/src/analyze/analyze.c +35 -19
  38. data/src/analyze/analyze_helpers.c +161 -188
  39. data/src/analyze/analyzed_ruby.c +8 -7
  40. data/src/analyze/control_type.c +29 -22
  41. data/src/analyze/herb_directives.c +16 -20
  42. data/src/analyze/iteration_nodes.c +69 -24
  43. data/src/analyze/parse_errors.c +32 -22
  44. data/src/analyze/postfix_conditionals.c +23 -13
  45. data/src/analyze/prism_annotate.c +29 -28
  46. data/src/analyze/render_nodes.c +51 -41
  47. data/src/analyze/strict_locals.c +14 -15
  48. data/src/analyze/ternary_conditionals.c +14 -10
  49. data/src/herb.c +10 -8
  50. data/src/include/analyze/action_view/tag_helper_handler.h +1 -1
  51. data/src/include/analyze/action_view/tag_helper_node_builders.h +1 -0
  52. data/src/include/analyze/action_view/tag_helpers.h +1 -1
  53. data/src/include/analyze/analyze.h +3 -2
  54. data/src/include/analyze/analyzed_ruby.h +4 -2
  55. data/src/include/analyze/helpers.h +6 -6
  56. data/src/include/herb.h +3 -2
  57. data/src/include/parser/parser_options.h +5 -0
  58. data/src/include/prism/prism_context.h +12 -12
  59. data/src/include/prism/prism_helpers.h +30 -0
  60. data/src/include/prism_version.h +11 -0
  61. data/src/include/version.h +1 -1
  62. data/src/parser/parser_helpers.c +1 -2
  63. data/src/parser.c +6 -7
  64. data/src/parser_options_defaults.c +1 -0
  65. data/src/prism/prism_helpers.c +84 -50
  66. data/src/prism/ruby_parser.c +20 -20
  67. data/templates/ext/herb/nodes.c.erb +5 -5
  68. data/templates/ext/herb/parser_options_helpers.c.erb +33 -0
  69. data/templates/java/nodes.c.erb +6 -6
  70. data/templates/java/parser_options_helpers.c.erb +41 -2
  71. data/templates/java/parser_options_helpers.h.erb +2 -1
  72. data/templates/javascript/packages/node/extension/nodes.cpp.erb +5 -5
  73. data/templates/javascript/packages/node/extension/parser_options_helpers.cpp.erb +34 -2
  74. data/templates/javascript/packages/node/extension/parser_options_helpers.h.erb +1 -1
  75. data/templates/lib/herb/parser_options.rb.erb +1 -1
  76. data/templates/rust/src/prism/generated_deserialize.rs.erb +90 -44
  77. data/templates/rust/src/prism/generated_prettyprint.rs.erb +18 -27
  78. data/templates/src/analyze/action_view/generated_handlers.c.erb +23 -22
  79. data/templates/src/include/parser/parser_options.h.erb +10 -0
  80. data/templates/src/include/prism_version.h.erb +17 -0
  81. data/templates/src/parser_options_defaults.c.erb +1 -1
  82. data/templates/template.rb +17 -2
  83. data/templates/wasm/nodes.cpp.erb +5 -5
  84. data/templates/wasm/parser_options_helpers.cpp.erb +24 -0
  85. data/vendor/prism/Rakefile +76 -7
  86. data/vendor/prism/config.yml +174 -161
  87. data/vendor/prism/include/prism/arena.h +45 -0
  88. data/vendor/prism/include/prism/arena.hpp +42 -0
  89. data/vendor/prism/include/prism/ast.h +2738 -611
  90. data/vendor/prism/include/prism/ast.hpp +641 -0
  91. data/vendor/prism/include/prism/buffer.h +60 -0
  92. data/vendor/prism/include/prism/buffer.hpp +55 -0
  93. data/vendor/prism/include/prism/comments.h +72 -0
  94. data/vendor/prism/include/prism/compiler/accel.h +26 -0
  95. data/vendor/prism/include/prism/compiler/align.h +36 -0
  96. data/vendor/prism/include/prism/compiler/assume.h +25 -0
  97. data/vendor/prism/include/prism/compiler/branch.h +26 -0
  98. data/vendor/prism/include/prism/compiler/bswap.h +46 -0
  99. data/vendor/prism/include/prism/compiler/cplusplus.hpp +16 -0
  100. data/vendor/prism/include/prism/compiler/endianness.h +34 -0
  101. data/vendor/prism/include/prism/compiler/exceptions.hpp +17 -0
  102. data/vendor/prism/include/prism/compiler/exported.h +24 -0
  103. data/vendor/prism/include/prism/compiler/fallthrough.h +22 -0
  104. data/vendor/prism/include/prism/compiler/filesystem.h +32 -0
  105. data/vendor/prism/include/prism/compiler/flex_array.h +19 -0
  106. data/vendor/prism/include/prism/compiler/force_inline.h +21 -0
  107. data/vendor/prism/include/prism/compiler/format.h +27 -0
  108. data/vendor/prism/include/prism/compiler/inline.h +17 -0
  109. data/vendor/prism/include/prism/compiler/lifetimebound.hpp +25 -0
  110. data/vendor/prism/include/prism/compiler/nodiscard.h +22 -0
  111. data/vendor/prism/include/prism/compiler/nonnull.h +18 -0
  112. data/vendor/prism/include/prism/compiler/unused.h +18 -0
  113. data/vendor/prism/include/prism/constant_pool.h +106 -0
  114. data/vendor/prism/include/prism/diagnostic.h +55 -402
  115. data/vendor/prism/include/prism/errors_format.h +61 -0
  116. data/vendor/prism/include/prism/excludes.h +29 -0
  117. data/vendor/prism/include/prism/integer.h +41 -0
  118. data/vendor/prism/include/prism/internal/allocator.h +68 -0
  119. data/vendor/prism/include/prism/internal/allocator_debug.h +88 -0
  120. data/vendor/prism/include/prism/internal/arena.h +108 -0
  121. data/vendor/prism/include/prism/internal/bit.h +42 -0
  122. data/vendor/prism/include/prism/internal/buffer.h +94 -0
  123. data/vendor/prism/include/prism/internal/char.h +145 -0
  124. data/vendor/prism/include/prism/internal/comments.h +54 -0
  125. data/vendor/prism/include/prism/internal/constant_pool.h +152 -0
  126. data/vendor/prism/include/prism/internal/diagnostic.h +389 -0
  127. data/vendor/prism/include/prism/{encoding.h → internal/encoding.h} +27 -68
  128. data/vendor/prism/include/prism/internal/integer.h +68 -0
  129. data/vendor/prism/include/prism/internal/isinf.h +16 -0
  130. data/vendor/prism/include/prism/internal/line_offset_list.h +34 -0
  131. data/vendor/prism/include/prism/{util/pm_list.h → internal/list.h} +11 -52
  132. data/vendor/prism/include/prism/internal/magic_comments.h +23 -0
  133. data/vendor/prism/include/prism/internal/memchr.h +15 -0
  134. data/vendor/prism/include/prism/internal/node.h +42 -0
  135. data/vendor/prism/include/prism/internal/options.h +232 -0
  136. data/vendor/prism/include/prism/internal/parser.h +1015 -0
  137. data/vendor/prism/include/prism/internal/regexp.h +26 -0
  138. data/vendor/prism/include/prism/internal/serialize.h +39 -0
  139. data/vendor/prism/include/prism/internal/source.h +72 -0
  140. data/vendor/prism/include/prism/internal/static_literals.h +60 -0
  141. data/vendor/prism/include/prism/internal/stringy.h +30 -0
  142. data/vendor/prism/include/prism/{util/pm_strncasecmp.h → internal/strncasecmp.h} +3 -17
  143. data/vendor/prism/include/prism/{util/pm_strpbrk.h → internal/strpbrk.h} +7 -20
  144. data/vendor/prism/include/prism/internal/tokens.h +11 -0
  145. data/vendor/prism/include/prism/internal/xxh3.h +16 -0
  146. data/vendor/prism/include/prism/json.h +40 -0
  147. data/vendor/prism/include/prism/line_offset_list.h +69 -0
  148. data/vendor/prism/include/prism/magic_comments.h +53 -0
  149. data/vendor/prism/include/prism/node.h +59 -49
  150. data/vendor/prism/include/prism/node.hpp +272 -0
  151. data/vendor/prism/include/prism/node_type.hpp +38 -0
  152. data/vendor/prism/include/prism/options.h +154 -279
  153. data/vendor/prism/include/prism/options.hpp +43 -0
  154. data/vendor/prism/include/prism/parser.h +367 -870
  155. data/vendor/prism/include/prism/parser.hpp +274 -0
  156. data/vendor/prism/include/prism/prettyprint.h +14 -9
  157. data/vendor/prism/include/prism/serialize.h +123 -0
  158. data/vendor/prism/include/prism/source.h +163 -0
  159. data/vendor/prism/include/prism/stream.h +36 -0
  160. data/vendor/prism/include/prism/string_query.h +82 -0
  161. data/vendor/prism/include/prism/stringy.h +61 -0
  162. data/vendor/prism/include/prism/stringy.hpp +31 -0
  163. data/vendor/prism/include/prism/version.h +20 -3
  164. data/vendor/prism/include/prism.h +35 -320
  165. data/vendor/prism/include/prism.hpp +24 -0
  166. data/vendor/prism/src/arena.c +122 -0
  167. data/vendor/prism/src/{util/pm_buffer.c → buffer.c} +53 -21
  168. data/vendor/prism/src/{util/pm_char.c → char.c} +28 -57
  169. data/vendor/prism/src/comments.c +455 -0
  170. data/vendor/prism/src/constant_pool.c +406 -0
  171. data/vendor/prism/src/diagnostic.c +120 -68
  172. data/vendor/prism/src/encoding.c +8 -3
  173. data/vendor/prism/src/errors_format.c +519 -0
  174. data/vendor/prism/src/{util/pm_integer.c → integer.c} +38 -22
  175. data/vendor/prism/src/json.c +5733 -0
  176. data/vendor/prism/src/line_offset_list.c +100 -0
  177. data/vendor/prism/src/list.c +24 -0
  178. data/vendor/prism/src/{util/pm_memchr.c → memchr.c} +8 -6
  179. data/vendor/prism/src/node.c +3465 -7787
  180. data/vendor/prism/src/options.c +161 -54
  181. data/vendor/prism/src/parser.c +371 -0
  182. data/vendor/prism/src/prettyprint.c +421 -354
  183. data/vendor/prism/src/prism.c +8522 -7280
  184. data/vendor/prism/src/regexp.c +960 -45
  185. data/vendor/prism/src/serialize.c +2434 -2048
  186. data/vendor/prism/src/source.c +524 -0
  187. data/vendor/prism/src/static_literals.c +303 -360
  188. data/vendor/prism/src/string_query.c +180 -0
  189. data/vendor/prism/src/stringy.c +61 -0
  190. data/vendor/prism/src/{util/pm_strncasecmp.c → strncasecmp.c} +5 -4
  191. data/vendor/prism/src/strpbrk.c +447 -0
  192. data/vendor/prism/src/{token_type.c → tokens.c} +39 -18
  193. data/vendor/prism/src/xxh3.c +292 -0
  194. data/vendor/prism/templates/ext/prism/api_node.c.erb +58 -44
  195. data/vendor/prism/templates/include/prism/ast.h.erb +67 -17
  196. data/vendor/prism/templates/include/prism/ast.hpp.erb +31 -0
  197. data/vendor/prism/templates/include/prism/internal/diagnostic.h.erb +60 -0
  198. data/vendor/prism/templates/java/{org → api/src/main/java-templates/org/ruby_lang}/prism/AbstractNodeVisitor.java.erb +1 -1
  199. data/vendor/prism/templates/java/{org → api/src/main/java-templates/org/ruby_lang}/prism/Loader.java.erb +232 -133
  200. data/vendor/prism/templates/java/{org → api/src/main/java-templates/org/ruby_lang}/prism/Nodes.java.erb +62 -23
  201. data/vendor/prism/templates/javascript/src/deserialize.js.erb +153 -146
  202. data/vendor/prism/templates/javascript/src/nodes.js.erb +4 -9
  203. data/vendor/prism/templates/lib/prism/compiler.rb.erb +14 -5
  204. data/vendor/prism/templates/lib/prism/dispatcher.rb.erb +20 -12
  205. data/vendor/prism/templates/lib/prism/dot_visitor.rb.erb +37 -11
  206. data/vendor/prism/templates/lib/prism/dsl.rb.erb +43 -4
  207. data/vendor/prism/templates/lib/prism/inspect_visitor.rb.erb +26 -10
  208. data/vendor/prism/templates/lib/prism/mutation_compiler.rb.erb +5 -2
  209. data/vendor/prism/templates/lib/prism/node.rb.erb +283 -105
  210. data/vendor/prism/templates/lib/prism/reflection.rb.erb +11 -2
  211. data/vendor/prism/templates/lib/prism/serialize.rb.erb +291 -134
  212. data/vendor/prism/templates/lib/prism/visitor.rb.erb +18 -0
  213. data/vendor/prism/templates/src/diagnostic.c.erb +114 -63
  214. data/vendor/prism/templates/src/json.c.erb +130 -0
  215. data/vendor/prism/templates/src/node.c.erb +79 -236
  216. data/vendor/prism/templates/src/prettyprint.c.erb +24 -13
  217. data/vendor/prism/templates/src/serialize.c.erb +369 -183
  218. data/vendor/prism/templates/src/{token_type.c.erb → tokens.c.erb} +22 -11
  219. data/vendor/prism/templates/template.rb +193 -91
  220. metadata +100 -42
  221. data/vendor/prism/include/prism/defines.h +0 -260
  222. data/vendor/prism/include/prism/pack.h +0 -163
  223. data/vendor/prism/include/prism/regexp.h +0 -43
  224. data/vendor/prism/include/prism/static_literals.h +0 -121
  225. data/vendor/prism/include/prism/util/pm_buffer.h +0 -236
  226. data/vendor/prism/include/prism/util/pm_char.h +0 -204
  227. data/vendor/prism/include/prism/util/pm_constant_pool.h +0 -218
  228. data/vendor/prism/include/prism/util/pm_integer.h +0 -130
  229. data/vendor/prism/include/prism/util/pm_memchr.h +0 -29
  230. data/vendor/prism/include/prism/util/pm_newline_list.h +0 -113
  231. data/vendor/prism/include/prism/util/pm_string.h +0 -200
  232. data/vendor/prism/src/pack.c +0 -509
  233. data/vendor/prism/src/util/pm_constant_pool.c +0 -342
  234. data/vendor/prism/src/util/pm_list.c +0 -49
  235. data/vendor/prism/src/util/pm_newline_list.c +0 -125
  236. data/vendor/prism/src/util/pm_string.c +0 -381
  237. data/vendor/prism/src/util/pm_strpbrk.c +0 -206
  238. data/vendor/prism/templates/include/prism/diagnostic.h.erb +0 -130
  239. data/vendor/prism/templates/rbi/prism/dsl.rbi.erb +0 -68
  240. data/vendor/prism/templates/rbi/prism/node.rbi.erb +0 -167
  241. data/vendor/prism/templates/rbi/prism/visitor.rbi.erb +0 -18
  242. data/vendor/prism/templates/sig/prism/_private/dot_visitor.rbs.erb +0 -45
  243. data/vendor/prism/templates/sig/prism/dsl.rbs.erb +0 -31
  244. data/vendor/prism/templates/sig/prism/mutation_compiler.rbs.erb +0 -7
  245. data/vendor/prism/templates/sig/prism/node.rbs.erb +0 -136
  246. data/vendor/prism/templates/sig/prism/visitor.rbs.erb +0 -17
  247. data/vendor/prism/templates/sig/prism.rbs.erb +0 -92
@@ -1,5 +1,5 @@
1
- <%- string_type = Prism::Template::JAVA_STRING_TYPE -%>
2
- package org.prism;
1
+ <%- id_type = Prism::Template::JAVA_IDENTIFIER_TYPE -%>
2
+ package org.ruby_lang.prism;
3
3
 
4
4
  import java.lang.Override;
5
5
  import java.lang.String;
@@ -8,7 +8,6 @@ import java.lang.annotation.ElementType;
8
8
  import java.lang.annotation.Retention;
9
9
  import java.lang.annotation.RetentionPolicy;
10
10
  import java.lang.annotation.Target;
11
- import java.nio.charset.StandardCharsets;
12
11
  import java.util.ArrayList;
13
12
  import java.util.Arrays;
14
13
 
@@ -16,7 +15,7 @@ import java.util.Arrays;
16
15
  // @formatter:off
17
16
  public abstract class Nodes {
18
17
 
19
- public static final <%= string_type %>[] EMPTY_STRING_ARRAY = {};
18
+ public static final <%= id_type %>[] EMPTY_IDENTIFIER_ARRAY = {};
20
19
 
21
20
  @Target(ElementType.FIELD)
22
21
  @Retention(RetentionPolicy.SOURCE)
@@ -47,12 +46,10 @@ public abstract class Nodes {
47
46
  }
48
47
 
49
48
  public static final class Source {
50
- public final byte[] bytes;
51
49
  private int startLine = 1;
52
50
  private int[] lineOffsets = null;
53
51
 
54
- Source(byte[] bytes) {
55
- this.bytes = bytes;
52
+ Source() {
56
53
  }
57
54
 
58
55
  void setStartLine(int startLine) {
@@ -70,7 +67,6 @@ public abstract class Nodes {
70
67
 
71
68
  // 0-based
72
69
  public int findLine(int byteOffset) {
73
- if (byteOffset >= bytes.length) byteOffset = bytes.length - 1;
74
70
  assert byteOffset >= 0 : byteOffset;
75
71
  int index = Arrays.binarySearch(lineOffsets, byteOffset);
76
72
  int line;
@@ -142,7 +138,23 @@ public abstract class Nodes {
142
138
 
143
139
  protected abstract String toString(String indent);
144
140
  }
145
- <%# FLAGS -%>
141
+
142
+ protected static String asString(Object value) {
143
+ return value.toString();
144
+ }
145
+
146
+ protected static String asString(byte[] value) {
147
+ StringBuilder buf = new StringBuilder(value.length);
148
+ for (byte b : value) {
149
+ if (b >= 0x20 && b <= 0x7e) {
150
+ buf.append((char) b);
151
+ } else {
152
+ buf.append(String.format("\\x%02x", Byte.toUnsignedInt(b)));
153
+ }
154
+ }
155
+ return buf.toString();
156
+ }
157
+ <%-# FLAGS -%>
146
158
  <%- flags.each do |flag| -%>
147
159
 
148
160
  /**
@@ -194,7 +206,7 @@ public abstract class Nodes {
194
206
  <%- end -%>
195
207
  }
196
208
  <%- end -%>
197
- <%# NODES -%>
209
+ <%-# NODES -%>
198
210
  <%- nodes.each do |node| -%>
199
211
 
200
212
  /**
@@ -205,8 +217,11 @@ public abstract class Nodes {
205
217
  * </pre>
206
218
  */
207
219
  public static final class <%= node.name -%> extends Node {
208
- <%- if node.needs_serialized_length? -%>
209
- public final int serializedLength;
220
+ <%- if node.lazy_field -%>
221
+ /** The position of the lazy marker in the serialized output, if this node is lazy. */
222
+ public final int lazyPosition;
223
+ /** The loader that loaded this node, if this node is lazy. */
224
+ public final Loader loader;
210
225
  <%- end -%>
211
226
  <%- if node.flags -%>
212
227
  public final short flags;
@@ -234,7 +249,10 @@ public abstract class Nodes {
234
249
  params = []
235
250
  params << "int nodeId" if Prism::Template::INCLUDE_NODE_ID
236
251
  params << "int startOffset" << "int length"
237
- params << "int serializedLength" if node.needs_serialized_length?
252
+ if node.lazy_field
253
+ params << "int lazyPosition"
254
+ params << "Loader loader"
255
+ end
238
256
  params << "short flags" if node.flags
239
257
  params.concat(node.semantic_fields.map { |field| "#{field.java_type} #{field.name}" })
240
258
  -%>
@@ -244,8 +262,9 @@ public abstract class Nodes {
244
262
  <%- else -%>
245
263
  super(startOffset, length);
246
264
  <%- end -%>
247
- <%- if node.needs_serialized_length? -%>
248
- this.serializedLength = serializedLength;
265
+ <%- if node.lazy_field -%>
266
+ this.lazyPosition = lazyPosition;
267
+ this.loader = loader;
249
268
  <%- end -%>
250
269
  <%- if node.flags -%>
251
270
  this.flags = flags;
@@ -254,7 +273,22 @@ public abstract class Nodes {
254
273
  this.<%= field.name %> = <%= field.name %>;
255
274
  <%- end -%>
256
275
  }
257
- <%# methods for flags -%>
276
+ <%-# extra methods for DefNode -%>
277
+ <%- if node.lazy_field -%>
278
+
279
+ public boolean isLazy() {
280
+ return loader != null;
281
+ }
282
+
283
+ public <%= node.name -%> getNonLazy() {
284
+ if (isLazy()) {
285
+ return loader.loadNonLazy<%= node.name %>(lazyPosition);
286
+ } else {
287
+ return this;
288
+ }
289
+ }
290
+ <%- end -%>
291
+ <%-# methods for flags -%>
258
292
  <%- if (node_flags = node.flags) -%>
259
293
  <%- node_flags.values.each do |value| -%>
260
294
 
@@ -263,7 +297,7 @@ public abstract class Nodes {
263
297
  }
264
298
  <%- end -%>
265
299
  <%- end -%>
266
- <%# potential override of setNewLineFlag() -%>
300
+ <%-# potential override of setNewLineFlag() -%>
267
301
  <%- if node.newline == false -%>
268
302
 
269
303
  @Override
@@ -288,6 +322,7 @@ public abstract class Nodes {
288
322
  }
289
323
  <%- end -%>
290
324
 
325
+ @Override
291
326
  public <T> void visitChildNodes(AbstractNodeVisitor<T> visitor) {
292
327
  <%- node.semantic_fields.each do |field| -%>
293
328
  <%- case field -%>
@@ -305,6 +340,7 @@ public abstract class Nodes {
305
340
  <%- end -%>
306
341
  }
307
342
 
343
+ @Override
308
344
  public Node[] childNodes() {
309
345
  <%- if node.semantic_fields.none?(Prism::Template::NodeListField) and node.semantic_fields.none?(Prism::Template::NodeKindField) -%>
310
346
  return EMPTY_ARRAY;
@@ -326,6 +362,7 @@ public abstract class Nodes {
326
362
  <%- end -%>
327
363
  }
328
364
 
365
+ @Override
329
366
  public <T> T accept(AbstractNodeVisitor<T> visitor) {
330
367
  return visitor.visit<%= node.name -%>(this);
331
368
  }
@@ -338,11 +375,13 @@ public abstract class Nodes {
338
375
  builder.append("[Li]");
339
376
  }
340
377
  builder.append('\n');
378
+ <%- unless [*node.flags, *node.semantic_fields].empty? -%>
341
379
  String nextIndent = indent + " ";
380
+ <%- end -%>
342
381
  <%- if node.fields.any?(Prism::Template::NodeListField) or node.fields.any?(Prism::Template::ConstantListField) -%>
343
382
  String nextNextIndent = nextIndent + " ";
344
383
  <%- end -%>
345
- <%- [*node.flags, *node.fields.grep_v(Prism::Template::LocationField).grep_v(Prism::Template::OptionalLocationField)].each do |field| -%>
384
+ <%- [*node.flags, *node.semantic_fields].each do |field| -%>
346
385
  builder.append(nextIndent);
347
386
  builder.append("<%= field.name %>: ");
348
387
  <%- case field -%>
@@ -356,18 +395,18 @@ public abstract class Nodes {
356
395
  builder.append(nextNextIndent).append(child.toString(nextNextIndent));
357
396
  }
358
397
  <%- when Prism::Template::StringField -%>
359
- builder.append('"' + new String(this.<%= field.name %>, StandardCharsets.UTF_8) + '"');
398
+ builder.append('"' + asString(this.<%= field.name %>) + '"');
360
399
  builder.append('\n');
361
400
  <%- when Prism::Template::ConstantField -%>
362
- builder.append('"').append(this.<%= field.name %>).append('"');
401
+ builder.append('"').append(asString(this.<%= field.name %>)).append('"');
363
402
  builder.append('\n');
364
403
  <%- when Prism::Template::OptionalConstantField -%>
365
- builder.append(this.<%= field.name %> == null ? "null" : "\"" + this.<%= field.name %> + "\"");
404
+ builder.append(this.<%= field.name %> == null ? "null" : "\"" + asString(this.<%= field.name %>) + "\"");
366
405
  builder.append('\n');
367
406
  <%- when Prism::Template::ConstantListField -%>
368
407
  builder.append('\n');
369
- for (<%= string_type %> constant : this.<%= field.name %>) {
370
- builder.append(nextNextIndent).append('"').append(constant).append('"').append('\n');
408
+ for (<%= id_type %> constant : this.<%= field.name %>) {
409
+ builder.append(nextNextIndent).append('"').append(asString(constant)).append('"').append('\n');
371
410
  }
372
411
  <%- when Prism::Template::Flags -%>
373
412
  builder.append(flags);
@@ -1,20 +1,23 @@
1
1
  import * as nodes from "./nodes.js";
2
+ import { getDecoder } from "./decoding.js";
3
+ import { Location } from "./location.js";
4
+ import { Source } from "./source.js";
2
5
 
3
- const MAJOR_VERSION = 1;
4
- const MINOR_VERSION = 9;
6
+ const MAJOR_VERSION = 2;
7
+ const MINOR_VERSION = 1;
5
8
  const PATCH_VERSION = 0;
6
9
 
7
- // The DataView getFloat64 function takes an optional second argument that
8
- // specifies whether the number is little-endian or big-endian. It does not
9
- // appear to have a native endian mode, so we need to determine the endianness
10
- // of the system at runtime.
10
+ /* The DataView getFloat64 function takes an optional second argument that
11
+ * specifies whether the number is little-endian or big-endian. It does not
12
+ * appear to have a native endian mode, so we need to determine the endianness
13
+ * of the system at runtime. */
11
14
  const LITTLE_ENDIAN = (() => {
12
15
  let uint32 = new Uint32Array([0x11223344]);
13
16
  let uint8 = new Uint8Array(uint32.buffer);
14
17
 
15
18
  if (uint8[0] === 0x44) {
16
19
  return true;
17
- } else if (uInt8[0] === 0x11) {
20
+ } else if (uint8[0] === 0x11) {
18
21
  return false;
19
22
  } else {
20
23
  throw new Error("Mixed endianness");
@@ -22,19 +25,11 @@ const LITTLE_ENDIAN = (() => {
22
25
  })();
23
26
 
24
27
  class SerializationBuffer {
25
- FORCED_UTF8_ENCODING_FLAG = 1 << 2;
26
- FORCED_BINARY_ENCODING_FLAG = 1 << 3;
27
-
28
- DECODER_MAP = new Map([
29
- ["ascii-8bit", "ascii"]
30
- ]);
31
-
32
- constructor(source, array) {
33
- this.source = source;
28
+ constructor(array) {
34
29
  this.array = array;
35
30
  this.index = 0;
36
- this.fileEncoding = "utf-8";
37
- this.decoders = new Map();
31
+ this.encoding = "utf-8";
32
+ this.source = null;
38
33
  }
39
34
 
40
35
  readByte() {
@@ -53,7 +48,7 @@ class SerializationBuffer {
53
48
  return this.decodeString(this.readBytes(length), flags).value;
54
49
  }
55
50
 
56
- // Read a 32-bit unsigned integer in little-endian format.
51
+ /* Read a 32-bit unsigned integer in little-endian format. */
57
52
  readUint32() {
58
53
  const result = this.scanUint32(this.index);
59
54
  this.index += 4;
@@ -65,6 +60,17 @@ class SerializationBuffer {
65
60
  return bytes[0] | (bytes[1] << 8) | (bytes[2] << 16) | (bytes[3] << 24);
66
61
  }
67
62
 
63
+ /**
64
+ * Read a signed integer that was written in zigzag encoding, where the low
65
+ * bit carries the sign. Both operators here are 32-bit, which is what makes
66
+ * this correct for zigzag values above 2^31, where readVarInt overflows and
67
+ * returns the bit pattern as a negative number.
68
+ */
69
+ readVarSInt() {
70
+ const value = this.readVarInt();
71
+ return (value >>> 1) ^ -(value & 1);
72
+ }
73
+
68
74
  readVarInt() {
69
75
  let result = 0;
70
76
  let shift = 0;
@@ -83,7 +89,9 @@ class SerializationBuffer {
83
89
  }
84
90
 
85
91
  readLocation() {
86
- return { startOffset: this.readVarInt(), length: this.readVarInt() };
92
+ const startOffset = this.readVarInt();
93
+ const length = this.readVarInt();
94
+ return new Location(this.source, startOffset, length);
87
95
  }
88
96
 
89
97
  readOptionalLocation() {
@@ -96,32 +104,14 @@ class SerializationBuffer {
96
104
 
97
105
  readStringField(flags) {
98
106
  if (flags === undefined) flags = 0;
99
- const type = this.readByte();
100
-
101
- switch (type) {
102
- case 1: {
103
- const startOffset = this.readVarInt();
104
- const length = this.readVarInt();
105
- return this.decodeString(this.source.slice(startOffset, startOffset + length), flags);
106
- }
107
- case 2:
108
- return this.decodeString(this.readBytes(this.readVarInt()), flags);
109
- default:
110
- throw new Error(`Unknown serialized string type: ${type}`);
111
- }
107
+ return this.decodeString(this.readBytes(this.readVarInt()), flags);
112
108
  }
113
109
 
114
110
  scanConstant(constantPoolOffset, constantIndex) {
115
111
  const offset = constantPoolOffset + constantIndex * 8;
116
- let startOffset = this.scanUint32(offset);
112
+ const startOffset = this.scanUint32(offset);
117
113
  const length = this.scanUint32(offset + 4);
118
-
119
- if (startOffset & (1 << 31)) {
120
- startOffset &= (1 << 31) - 1;
121
- return new TextDecoder().decode(this.array.slice(startOffset, startOffset + length));
122
- } else {
123
- return new TextDecoder().decode(this.source.slice(startOffset, startOffset + length));
124
- }
114
+ return getDecoder(this.encoding, { fatal: true }).decode(this.array.slice(startOffset, startOffset + length));
125
115
  }
126
116
 
127
117
  readDouble() {
@@ -134,67 +124,37 @@ class SerializationBuffer {
134
124
  }
135
125
 
136
126
  decodeString(bytes, flags) {
137
- const forcedBin = (flags & this.FORCED_BINARY_ENCODING_FLAG) !== 0;
138
- const forcedUtf8 = (flags & this.FORCED_UTF8_ENCODING_FLAG) !== 0;
139
-
140
- if (forcedBin) {
141
- // just use raw bytes
142
- return {
143
- encoding: "ascii",
144
- validEncoding: true,
145
- value: this.asciiDecoder.decode(bytes)
146
- };
147
- } else {
148
- const encoding = forcedUtf8 ? "utf-8" : this.fileEncoding.toLowerCase();
149
- const decoder = this.getDecoder(encoding);
150
-
151
- try {
152
- // decode with encoding
153
- return {
154
- encoding,
155
- validEncoding: true,
156
- value: decoder.decode(bytes)
157
- };
158
- } catch(e) {
159
- // just use raw bytes, capture what the encoding should be, set flag saying encoding is invalid
160
- if (e.code === "ERR_ENCODING_INVALID_ENCODED_DATA") {
161
- return {
162
- encoding,
163
- validEncoding: false,
164
- value: this.asciiDecoder.decode(bytes)
165
- };
166
- }
167
-
168
- throw e;
169
- }
170
- }
171
- }
172
-
173
- getDecoder(encoding) {
174
- encoding = this.DECODER_MAP.get(encoding) || encoding;
127
+ let value, encoding, validEncoding;
175
128
 
176
- if (!this.decoders.has(encoding)) {
177
- this.decoders.set(encoding, new TextDecoder(encoding, {fatal: true}));
129
+ if ((flags & nodes.StringFlags.FORCED_BINARY_ENCODING) !== 0) {
130
+ encoding = "ascii-8bit";
131
+ } else if ((flags & nodes.StringFlags.FORCED_UTF8_ENCODING) !== 0) {
132
+ encoding = "utf-8";
133
+ } else {
134
+ encoding = this.encoding.toLowerCase();
178
135
  }
179
136
 
180
- return this.decoders.get(encoding);
181
- }
182
-
183
- get asciiDecoder() {
184
- if (!this._asciiDecoder) {
185
- this._asciiDecoder = new TextDecoder("ascii");
137
+ const decoder = getDecoder(encoding, { fatal: true });
138
+
139
+ try {
140
+ value = decoder.decode(bytes);
141
+ validEncoding = true;
142
+ } catch (error) {
143
+ /* A fatal decoder reports bytes that are not valid in its encoding by
144
+ * throwing a TypeError, and that is the whole of what it throws, so the
145
+ * string is read back as binary and flagged. */
146
+ if (error instanceof TypeError) {
147
+ value = getDecoder("ascii-8bit").decode(bytes);
148
+ validEncoding = false;
149
+ } else {
150
+ throw error;
151
+ }
186
152
  }
187
153
 
188
- return this._asciiDecoder;
154
+ return { value, encoding, validEncoding };
189
155
  }
190
156
  }
191
157
 
192
- /**
193
- * A location in the source code.
194
- *
195
- * @typedef {{ startOffset: number, length: number }} Location
196
- */
197
-
198
158
  /**
199
159
  * A comment in the source code.
200
160
  *
@@ -247,6 +207,7 @@ export class ParseResult {
247
207
  /**
248
208
  * @type {Location | null}
249
209
  */
210
+ dataLoc;
250
211
 
251
212
  /**
252
213
  * @type {ParseError[]}
@@ -258,20 +219,28 @@ export class ParseResult {
258
219
  */
259
220
  warnings;
260
221
 
222
+ /**
223
+ * @type {boolean}
224
+ */
225
+ continuable;
226
+
261
227
  /**
262
228
  * @param {nodes.ProgramNode} value
263
229
  * @param {Comment[]} comments
264
230
  * @param {MagicComment[]} magicComments
231
+ * @param {Location | null} dataLoc
265
232
  * @param {ParseError[]} errors
266
233
  * @param {ParseWarning[]} warnings
234
+ * @param {boolean} continuable
267
235
  */
268
- constructor(value, comments, magicComments, dataLoc, errors, warnings) {
236
+ constructor(value, comments, magicComments, dataLoc, errors, warnings, continuable) {
269
237
  this.value = value;
270
238
  this.comments = comments;
271
239
  this.magicComments = magicComments;
272
240
  this.dataLoc = dataLoc;
273
241
  this.errors = errors;
274
242
  this.warnings = warnings;
243
+ this.continuable = continuable;
275
244
  }
276
245
  }
277
246
 
@@ -299,7 +268,7 @@ const warningTypes = [
299
268
  * @throws {Error}
300
269
  */
301
270
  export function deserialize(source, array) {
302
- const buffer = new SerializationBuffer(source, array);
271
+ const buffer = new SerializationBuffer(array);
303
272
 
304
273
  if (buffer.readString(5) !== "PRISM") {
305
274
  throw new Error("Invalid serialization");
@@ -314,24 +283,34 @@ export function deserialize(source, array) {
314
283
  }
315
284
 
316
285
  // Read the file's encoding.
317
- buffer.fileEncoding = buffer.readString(buffer.readVarInt());
286
+ buffer.encoding = buffer.readString(buffer.readVarInt());
318
287
 
319
- // Skip past the start line, as we don't support that option yet in
320
- // JavaScript.
321
- buffer.readVarInt();
288
+ // Read the line that the source starts on along with the byte offset of the
289
+ // start of each line, which together let locations resolve lines and columns.
290
+ const startLine = buffer.readVarSInt();
322
291
 
323
- // Skip past the line offsets, as there is no Source object yet in JavaScript.
324
- // const lineOffsets = Array.from({ length: buffer.readVarInt() }, () => buffer.readVarInt());
325
- const lineOffsetsCount = buffer.readVarInt();
326
- for (let i = 0; i < lineOffsetsCount; i ++) {
327
- buffer.readVarInt();
328
- }
292
+ // Skip the 8-byte hash of the source, which this library does not expose.
293
+ buffer.readBytes(8);
294
+
295
+ const lineOffsets = Array.from({ length: buffer.readVarInt() }, () => buffer.readVarInt());
296
+
297
+ buffer.source = new Source(source, buffer.encoding, startLine, lineOffsets);
329
298
 
330
299
  const comments = Array.from({ length: buffer.readVarInt() }, () => ({
331
300
  type: buffer.readVarInt(),
332
301
  location: buffer.readLocation()
333
302
  }));
334
303
 
304
+ // Skip the comments attached to each node, which this library does not
305
+ // expose. Each entry is a node id followed by the indices of its comments.
306
+ for (let count = buffer.readVarInt(); count > 0; count--) {
307
+ buffer.readVarInt();
308
+
309
+ for (let length = buffer.readVarInt(); length > 0; length--) {
310
+ buffer.readVarInt();
311
+ }
312
+ }
313
+
335
314
  const magicComments = Array.from({ length: buffer.readVarInt() }, () => ({
336
315
  startLocation: buffer.readLocation(),
337
316
  endLocation: buffer.readLocation()
@@ -353,53 +332,81 @@ export function deserialize(source, array) {
353
332
  level: warningLevels[buffer.readByte()]
354
333
  }));
355
334
 
335
+ const continuable = buffer.readByte() !== 0;
336
+
356
337
  const constantPoolOffset = buffer.readUint32();
357
338
  const constants = Array.from({ length: buffer.readVarInt() }, () => null);
358
339
 
359
- return new ParseResult(readRequiredNode(), comments, magicComments, dataLoc, errors, warnings);
360
-
361
- function readRequiredNode() {
362
- const type = buffer.readByte();
363
- const nodeID = buffer.readVarInt();
364
- const location = buffer.readLocation();
365
- let flags;
366
-
367
- switch (type) {
368
- <%- nodes.each.with_index(1) do |node, index| -%>
369
- case <%= index %>:
370
- <%- if node.needs_serialized_length? -%>
371
- buffer.readUint32();
340
+ return new ParseResult(readNodes(constantPoolOffset), comments, magicComments, dataLoc, errors, warnings, continuable);
341
+
342
+ /* Read the nodes that are serialized before the given offset and return the
343
+ * root. Nodes are serialized in post-order, so the children of each node are
344
+ * the topmost values on the stack, in field order, by the time its record is
345
+ * read. */
346
+ function readNodes(finish) {
347
+ const stack = [];
348
+
349
+ while (buffer.index < finish) {
350
+ const type = buffer.readByte();
351
+
352
+ switch (type) {
353
+ case 0:
354
+ stack.push(null);
355
+ break;
356
+ case <%= Prism::Template::LAZY_MARKER %>:
357
+ buffer.readUint32();
358
+ buffer.readUint32();
359
+ break;
360
+ <%- local = ->(field) { field.name == "arguments" ? "arguments_" : field.name.gsub(/_([a-z])/) { $1.upcase } } -%>
361
+ <%- nodes.each.with_index(1) do |node, index| -%>
362
+ case <%= index %>: {
363
+ const nodeID = buffer.readVarInt();
364
+ const location = buffer.readLocation();
365
+ const flags = buffer.readVarInt();
366
+ <%- node.fields.each do |field| -%>
367
+ <%- case field -%>
368
+ <%- when Prism::Template::NodeField, Prism::Template::OptionalNodeField -%>
369
+ <%- when Prism::Template::NodeListField -%>
370
+ const <%= local[field] %>Size = buffer.readVarInt();
371
+ <%- else -%>
372
+ const <%= local[field] %> = <%=
373
+ case field
374
+ when Prism::Template::StringField then "buffer.readStringField(flags)"
375
+ when Prism::Template::ConstantField then "readRequiredConstant()"
376
+ when Prism::Template::OptionalConstantField then "readOptionalConstant()"
377
+ when Prism::Template::ConstantListField then "Array.from({ length: buffer.readVarInt() }, readRequiredConstant)"
378
+ when Prism::Template::LocationField then "buffer.readLocation()"
379
+ when Prism::Template::OptionalLocationField then "buffer.readOptionalLocation()"
380
+ when Prism::Template::UInt8Field then "buffer.readByte()"
381
+ when Prism::Template::UInt32Field then "buffer.readVarInt()"
382
+ when Prism::Template::IntegerField then "readInteger()"
383
+ when Prism::Template::DoubleField then "buffer.readDouble()"
384
+ else raise
385
+ end
386
+ %>;
387
+ <%- end -%>
388
+ <%- end -%>
389
+ <%- node.child_fields.reverse_each do |field| -%>
390
+ <%- if field.is_a?(Prism::Template::NodeListField) -%>
391
+ const <%= local[field] %> = stack.splice(stack.length - <%= local[field] %>Size, <%= local[field] %>Size);
392
+ <%- else -%>
393
+ const <%= local[field] %> = stack.pop();
394
+ <%- end -%>
395
+ <%- end -%>
396
+ stack.push(new nodes.<%= node.name %>(<%= ["nodeID", "location", "flags", *node.fields.map { |field| local[field] }].join(", ") %>));
397
+ break;
398
+ }
372
399
  <%- end -%>
373
- return new nodes.<%= node.name %>(<%= ["nodeID", "location", "flags = buffer.readVarInt()", *node.fields.map { |field|
374
- case field
375
- when Prism::Template::NodeField then "readRequiredNode()"
376
- when Prism::Template::OptionalNodeField then "readOptionalNode()"
377
- when Prism::Template::StringField then "buffer.readStringField(flags)"
378
- when Prism::Template::NodeListField then "Array.from({ length: buffer.readVarInt() }, readRequiredNode)"
379
- when Prism::Template::ConstantField then "readRequiredConstant()"
380
- when Prism::Template::OptionalConstantField then "readOptionalConstant()"
381
- when Prism::Template::ConstantListField then "Array.from({ length: buffer.readVarInt() }, readRequiredConstant)"
382
- when Prism::Template::LocationField then "buffer.readLocation()"
383
- when Prism::Template::OptionalLocationField then "buffer.readOptionalLocation()"
384
- when Prism::Template::UInt8Field then "buffer.readByte()"
385
- when Prism::Template::UInt32Field then "buffer.readVarInt()"
386
- when Prism::Template::IntegerField then "readInteger()"
387
- when Prism::Template::DoubleField then "buffer.readDouble()"
388
- end
389
- }].join(", ") -%>);
390
- <%- end -%>
391
- default:
392
- throw new Error(`Unknown node type: ${type}`);
400
+ default:
401
+ throw new Error(`Unknown node type: ${type}`);
402
+ }
393
403
  }
394
- }
395
404
 
396
- function readOptionalNode() {
397
- if (buffer.readByte() != 0) {
398
- buffer.index -= 1;
399
- return readRequiredNode();
400
- } else {
401
- return null;
405
+ if (buffer.index !== finish || stack.length !== 1) {
406
+ throw new Error("Expected the serialized nodes to end at the constant pool with exactly one root");
402
407
  }
408
+
409
+ return stack[0];
403
410
  }
404
411
 
405
412
  function scanConstant(constantIndex) {
@@ -24,6 +24,7 @@ def jstype(field)
24
24
  end
25
25
  end
26
26
  -%>
27
+ import { Location } from "./location.js"
27
28
  import * as visitors from "./visitor.js"
28
29
 
29
30
  <%- flags.each do |flag| -%>
@@ -31,19 +32,13 @@ import * as visitors from "./visitor.js"
31
32
  /**
32
33
  * <%= flag.comment %>
33
34
  */
34
- const <%= flag.name %> = {
35
+ export const <%= flag.name %> = {
35
36
  <%- flag.values.each_with_index do |value, index| -%>
36
37
  <%= value.name %>: 1 << <%= index + Prism::Template::COMMON_FLAGS_COUNT %>,
37
38
  <%- end -%>
38
39
  };
39
40
  <%- end -%>
40
41
 
41
- /**
42
- * A location in the source code.
43
- *
44
- * @typedef {{ startOffset: number, length: number }} Location
45
- */
46
-
47
42
  /**
48
43
  * An encoded Ruby string.
49
44
  *
@@ -137,7 +132,7 @@ export class <%= node.name -%> {
137
132
  when Prism::Template::NodeField, Prism::Template::OptionalNodeField then "this.#{prop(field)}"
138
133
  when Prism::Template::NodeListField then "...this.#{prop(field)}"
139
134
  end
140
- }.compact.join(", ") %>]
135
+ }.compact.join(", ") %>];
141
136
  }
142
137
 
143
138
  /**
@@ -159,7 +154,7 @@ export class <%= node.name -%> {
159
154
  compact.push(this.<%= prop(field) %>);
160
155
  }
161
156
  <%- when Prism::Template::NodeListField -%>
162
- compact.concat(this.<%= prop(field) %>);
157
+ compact.push(...this.<%= prop(field) %>);
163
158
  <%- end -%>
164
159
  <%- end -%>
165
160