prism 0.15.1 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +35 -1
  3. data/Makefile +12 -0
  4. data/README.md +3 -1
  5. data/config.yml +66 -50
  6. data/docs/configuration.md +2 -0
  7. data/docs/fuzzing.md +1 -1
  8. data/docs/javascript.md +90 -0
  9. data/docs/releasing.md +27 -0
  10. data/docs/ruby_api.md +2 -0
  11. data/docs/serialization.md +28 -29
  12. data/ext/prism/api_node.c +856 -826
  13. data/ext/prism/api_pack.c +20 -9
  14. data/ext/prism/extension.c +494 -119
  15. data/ext/prism/extension.h +1 -1
  16. data/include/prism/ast.h +3157 -747
  17. data/include/prism/defines.h +40 -8
  18. data/include/prism/diagnostic.h +36 -3
  19. data/include/prism/enc/pm_encoding.h +119 -28
  20. data/include/prism/node.h +38 -30
  21. data/include/prism/options.h +204 -0
  22. data/include/prism/pack.h +44 -33
  23. data/include/prism/parser.h +445 -199
  24. data/include/prism/prettyprint.h +26 -0
  25. data/include/prism/regexp.h +16 -2
  26. data/include/prism/util/pm_buffer.h +102 -18
  27. data/include/prism/util/pm_char.h +162 -48
  28. data/include/prism/util/pm_constant_pool.h +128 -34
  29. data/include/prism/util/pm_list.h +68 -38
  30. data/include/prism/util/pm_memchr.h +18 -3
  31. data/include/prism/util/pm_newline_list.h +71 -28
  32. data/include/prism/util/pm_state_stack.h +25 -7
  33. data/include/prism/util/pm_string.h +115 -27
  34. data/include/prism/util/pm_string_list.h +25 -6
  35. data/include/prism/util/pm_strncasecmp.h +32 -0
  36. data/include/prism/util/pm_strpbrk.h +31 -17
  37. data/include/prism/version.h +28 -3
  38. data/include/prism.h +229 -36
  39. data/lib/prism/compiler.rb +5 -5
  40. data/lib/prism/debug.rb +43 -13
  41. data/lib/prism/desugar_compiler.rb +1 -1
  42. data/lib/prism/dispatcher.rb +27 -26
  43. data/lib/prism/dsl.rb +16 -16
  44. data/lib/prism/ffi.rb +138 -61
  45. data/lib/prism/lex_compat.rb +26 -16
  46. data/lib/prism/mutation_compiler.rb +11 -11
  47. data/lib/prism/node.rb +426 -227
  48. data/lib/prism/node_ext.rb +23 -16
  49. data/lib/prism/node_inspector.rb +1 -1
  50. data/lib/prism/pack.rb +79 -40
  51. data/lib/prism/parse_result/comments.rb +7 -2
  52. data/lib/prism/parse_result/newlines.rb +4 -0
  53. data/lib/prism/parse_result.rb +157 -21
  54. data/lib/prism/pattern.rb +14 -3
  55. data/lib/prism/ripper_compat.rb +28 -10
  56. data/lib/prism/serialize.rb +935 -307
  57. data/lib/prism/visitor.rb +9 -5
  58. data/lib/prism.rb +20 -2
  59. data/prism.gemspec +11 -2
  60. data/rbi/prism.rbi +7305 -0
  61. data/rbi/prism_static.rbi +196 -0
  62. data/sig/prism.rbs +4468 -0
  63. data/sig/prism_static.rbs +123 -0
  64. data/src/diagnostic.c +56 -53
  65. data/src/enc/pm_big5.c +1 -0
  66. data/src/enc/pm_euc_jp.c +1 -0
  67. data/src/enc/pm_gbk.c +1 -0
  68. data/src/enc/pm_shift_jis.c +1 -0
  69. data/src/enc/pm_tables.c +316 -80
  70. data/src/enc/pm_unicode.c +54 -9
  71. data/src/enc/pm_windows_31j.c +1 -0
  72. data/src/node.c +357 -345
  73. data/src/options.c +170 -0
  74. data/src/prettyprint.c +7697 -1643
  75. data/src/prism.c +1964 -1125
  76. data/src/regexp.c +153 -95
  77. data/src/serialize.c +432 -397
  78. data/src/token_type.c +3 -1
  79. data/src/util/pm_buffer.c +88 -23
  80. data/src/util/pm_char.c +103 -57
  81. data/src/util/pm_constant_pool.c +52 -22
  82. data/src/util/pm_list.c +12 -4
  83. data/src/util/pm_memchr.c +5 -3
  84. data/src/util/pm_newline_list.c +25 -63
  85. data/src/util/pm_state_stack.c +9 -3
  86. data/src/util/pm_string.c +95 -85
  87. data/src/util/pm_string_list.c +14 -15
  88. data/src/util/pm_strncasecmp.c +10 -3
  89. data/src/util/pm_strpbrk.c +25 -19
  90. metadata +12 -3
  91. data/docs/prism.png +0 -0
data/lib/prism/visitor.rb CHANGED
@@ -11,14 +11,18 @@ module Prism
11
11
  # implement each one that they need. For a default implementation that
12
12
  # continues walking the tree, see the Visitor class.
13
13
  class BasicVisitor
14
+ # Calls `accept` on the given node if it is not `nil`, which in turn should
15
+ # call back into this visitor by calling the appropriate `visit_*` method.
14
16
  def visit(node)
15
17
  node&.accept(self)
16
18
  end
17
19
 
20
+ # Visits each node in `nodes` by calling `accept` on each one.
18
21
  def visit_all(nodes)
19
22
  nodes.each { |node| node&.accept(self) }
20
23
  end
21
24
 
25
+ # Visits the child nodes of `node` by calling `accept` on each one.
22
26
  def visit_child_nodes(node)
23
27
  node.compact_child_nodes.each { |node| node.accept(self) }
24
28
  end
@@ -296,9 +300,6 @@ module Prism
296
300
  # Visit a KeywordHashNode node
297
301
  alias visit_keyword_hash_node visit_child_nodes
298
302
 
299
- # Visit a KeywordParameterNode node
300
- alias visit_keyword_parameter_node visit_child_nodes
301
-
302
303
  # Visit a KeywordRestParameterNode node
303
304
  alias visit_keyword_rest_parameter_node visit_child_nodes
304
305
 
@@ -359,6 +360,9 @@ module Prism
359
360
  # Visit a NumberedReferenceReadNode node
360
361
  alias visit_numbered_reference_read_node visit_child_nodes
361
362
 
363
+ # Visit a OptionalKeywordParameterNode node
364
+ alias visit_optional_keyword_parameter_node visit_child_nodes
365
+
362
366
  # Visit a OptionalParameterNode node
363
367
  alias visit_optional_parameter_node visit_child_nodes
364
368
 
@@ -398,8 +402,8 @@ module Prism
398
402
  # Visit a RegularExpressionNode node
399
403
  alias visit_regular_expression_node visit_child_nodes
400
404
 
401
- # Visit a RequiredDestructuredParameterNode node
402
- alias visit_required_destructured_parameter_node visit_child_nodes
405
+ # Visit a RequiredKeywordParameterNode node
406
+ alias visit_required_keyword_parameter_node visit_child_nodes
403
407
 
404
408
  # Visit a RequiredParameterNode node
405
409
  alias visit_required_parameter_node visit_child_nodes
data/lib/prism.rb CHANGED
@@ -1,10 +1,16 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # The Prism Ruby parser.
4
+ #
5
+ # "Parsing Ruby is suddenly manageable!"
6
+ # - You, hopefully
7
+ #
3
8
  module Prism
4
9
  # There are many files in prism that are templated to handle every node type,
5
10
  # which means the files can end up being quite large. We autoload them to make
6
11
  # our require speed faster since consuming libraries are unlikely to use all
7
12
  # of these features.
13
+
8
14
  autoload :BasicVisitor, "prism/visitor"
9
15
  autoload :Compiler, "prism/compiler"
10
16
  autoload :Debug, "prism/debug"
@@ -23,17 +29,26 @@ module Prism
23
29
 
24
30
  # Some of these constants are not meant to be exposed, so marking them as
25
31
  # private here.
32
+
26
33
  private_constant :Debug
27
34
  private_constant :LexCompat
28
35
  private_constant :LexRipper
29
36
 
37
+ # :call-seq:
38
+ # Prism::lex_compat(source, **options) -> Array
39
+ #
30
40
  # Returns an array of tokens that closely resembles that of the Ripper lexer.
31
41
  # The only difference is that since we don't keep track of lexer state in the
32
42
  # same way, it's going to always return the NONE state.
33
- def self.lex_compat(source, filepath = "")
34
- LexCompat.new(source, filepath).result
43
+ #
44
+ # For supported options, see Prism::parse.
45
+ def self.lex_compat(source, **options)
46
+ LexCompat.new(source, **options).result
35
47
  end
36
48
 
49
+ # :call-seq:
50
+ # Prism::lex_ripper(source) -> Array
51
+ #
37
52
  # This lexes with the Ripper lex. It drops any space events but otherwise
38
53
  # returns the same tokens. Raises SyntaxError if the syntax in source is
39
54
  # invalid.
@@ -41,6 +56,9 @@ module Prism
41
56
  LexRipper.new(source).result
42
57
  end
43
58
 
59
+ # :call-seq:
60
+ # Prism::load(source, serialized) -> ParseResult
61
+ #
44
62
  # Load the serialized AST using the source as a reference into a tree.
45
63
  def self.load(source, serialized)
46
64
  Serialize.load(source, serialized)
data/prism.gemspec CHANGED
@@ -2,7 +2,7 @@
2
2
 
3
3
  Gem::Specification.new do |spec|
4
4
  spec.name = "prism"
5
- spec.version = "0.15.1"
5
+ spec.version = "0.17.0"
6
6
  spec.authors = ["Shopify"]
7
7
  spec.email = ["ruby@shopify.com"]
8
8
 
@@ -28,8 +28,9 @@ Gem::Specification.new do |spec|
28
28
  "docs/encoding.md",
29
29
  "docs/fuzzing.md",
30
30
  "docs/heredocs.md",
31
+ "docs/javascript.md",
31
32
  "docs/mapping.md",
32
- "docs/prism.png",
33
+ "docs/releasing.md",
33
34
  "docs/ripper.md",
34
35
  "docs/ruby_api.md",
35
36
  "docs/serialization.md",
@@ -44,8 +45,10 @@ Gem::Specification.new do |spec|
44
45
  "include/prism/diagnostic.h",
45
46
  "include/prism/enc/pm_encoding.h",
46
47
  "include/prism/node.h",
48
+ "include/prism/options.h",
47
49
  "include/prism/pack.h",
48
50
  "include/prism/parser.h",
51
+ "include/prism/prettyprint.h",
49
52
  "include/prism/regexp.h",
50
53
  "include/prism/util/pm_buffer.h",
51
54
  "include/prism/util/pm_char.h",
@@ -54,6 +57,7 @@ Gem::Specification.new do |spec|
54
57
  "include/prism/util/pm_memchr.h",
55
58
  "include/prism/util/pm_newline_list.h",
56
59
  "include/prism/util/pm_state_stack.h",
60
+ "include/prism/util/pm_strncasecmp.h",
57
61
  "include/prism/util/pm_string.h",
58
62
  "include/prism/util/pm_string_list.h",
59
63
  "include/prism/util/pm_strpbrk.h",
@@ -103,8 +107,13 @@ Gem::Specification.new do |spec|
103
107
  "src/util/pm_string_list.c",
104
108
  "src/util/pm_strncasecmp.c",
105
109
  "src/util/pm_strpbrk.c",
110
+ "src/options.c",
106
111
  "src/prism.c",
107
112
  "prism.gemspec",
113
+ "sig/prism.rbs",
114
+ "sig/prism_static.rbs",
115
+ "rbi/prism.rbi",
116
+ "rbi/prism_static.rbi"
108
117
  ]
109
118
 
110
119
  spec.extensions = ["ext/prism/extconf.rb"]