prism 0.15.1 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +35 -1
  3. data/Makefile +12 -0
  4. data/README.md +3 -1
  5. data/config.yml +66 -50
  6. data/docs/configuration.md +2 -0
  7. data/docs/fuzzing.md +1 -1
  8. data/docs/javascript.md +90 -0
  9. data/docs/releasing.md +27 -0
  10. data/docs/ruby_api.md +2 -0
  11. data/docs/serialization.md +28 -29
  12. data/ext/prism/api_node.c +856 -826
  13. data/ext/prism/api_pack.c +20 -9
  14. data/ext/prism/extension.c +494 -119
  15. data/ext/prism/extension.h +1 -1
  16. data/include/prism/ast.h +3157 -747
  17. data/include/prism/defines.h +40 -8
  18. data/include/prism/diagnostic.h +36 -3
  19. data/include/prism/enc/pm_encoding.h +119 -28
  20. data/include/prism/node.h +38 -30
  21. data/include/prism/options.h +204 -0
  22. data/include/prism/pack.h +44 -33
  23. data/include/prism/parser.h +445 -199
  24. data/include/prism/prettyprint.h +26 -0
  25. data/include/prism/regexp.h +16 -2
  26. data/include/prism/util/pm_buffer.h +102 -18
  27. data/include/prism/util/pm_char.h +162 -48
  28. data/include/prism/util/pm_constant_pool.h +128 -34
  29. data/include/prism/util/pm_list.h +68 -38
  30. data/include/prism/util/pm_memchr.h +18 -3
  31. data/include/prism/util/pm_newline_list.h +71 -28
  32. data/include/prism/util/pm_state_stack.h +25 -7
  33. data/include/prism/util/pm_string.h +115 -27
  34. data/include/prism/util/pm_string_list.h +25 -6
  35. data/include/prism/util/pm_strncasecmp.h +32 -0
  36. data/include/prism/util/pm_strpbrk.h +31 -17
  37. data/include/prism/version.h +28 -3
  38. data/include/prism.h +229 -36
  39. data/lib/prism/compiler.rb +5 -5
  40. data/lib/prism/debug.rb +43 -13
  41. data/lib/prism/desugar_compiler.rb +1 -1
  42. data/lib/prism/dispatcher.rb +27 -26
  43. data/lib/prism/dsl.rb +16 -16
  44. data/lib/prism/ffi.rb +138 -61
  45. data/lib/prism/lex_compat.rb +26 -16
  46. data/lib/prism/mutation_compiler.rb +11 -11
  47. data/lib/prism/node.rb +426 -227
  48. data/lib/prism/node_ext.rb +23 -16
  49. data/lib/prism/node_inspector.rb +1 -1
  50. data/lib/prism/pack.rb +79 -40
  51. data/lib/prism/parse_result/comments.rb +7 -2
  52. data/lib/prism/parse_result/newlines.rb +4 -0
  53. data/lib/prism/parse_result.rb +157 -21
  54. data/lib/prism/pattern.rb +14 -3
  55. data/lib/prism/ripper_compat.rb +28 -10
  56. data/lib/prism/serialize.rb +935 -307
  57. data/lib/prism/visitor.rb +9 -5
  58. data/lib/prism.rb +20 -2
  59. data/prism.gemspec +11 -2
  60. data/rbi/prism.rbi +7305 -0
  61. data/rbi/prism_static.rbi +196 -0
  62. data/sig/prism.rbs +4468 -0
  63. data/sig/prism_static.rbs +123 -0
  64. data/src/diagnostic.c +56 -53
  65. data/src/enc/pm_big5.c +1 -0
  66. data/src/enc/pm_euc_jp.c +1 -0
  67. data/src/enc/pm_gbk.c +1 -0
  68. data/src/enc/pm_shift_jis.c +1 -0
  69. data/src/enc/pm_tables.c +316 -80
  70. data/src/enc/pm_unicode.c +54 -9
  71. data/src/enc/pm_windows_31j.c +1 -0
  72. data/src/node.c +357 -345
  73. data/src/options.c +170 -0
  74. data/src/prettyprint.c +7697 -1643
  75. data/src/prism.c +1964 -1125
  76. data/src/regexp.c +153 -95
  77. data/src/serialize.c +432 -397
  78. data/src/token_type.c +3 -1
  79. data/src/util/pm_buffer.c +88 -23
  80. data/src/util/pm_char.c +103 -57
  81. data/src/util/pm_constant_pool.c +52 -22
  82. data/src/util/pm_list.c +12 -4
  83. data/src/util/pm_memchr.c +5 -3
  84. data/src/util/pm_newline_list.c +25 -63
  85. data/src/util/pm_state_stack.c +9 -3
  86. data/src/util/pm_string.c +95 -85
  87. data/src/util/pm_string_list.c +14 -15
  88. data/src/util/pm_strncasecmp.c +10 -3
  89. data/src/util/pm_strpbrk.c +25 -19
  90. metadata +12 -3
  91. data/docs/prism.png +0 -0
@@ -3,6 +3,19 @@
3
3
  # Here we are reopening the prism module to provide methods on nodes that aren't
4
4
  # templated and are meant as convenience methods.
5
5
  module Prism
6
+ module RegularExpressionOptions # :nodoc:
7
+ # Returns a numeric value that represents the flags that were used to create
8
+ # the regular expression.
9
+ def options
10
+ o = flags & (RegularExpressionFlags::IGNORE_CASE | RegularExpressionFlags::EXTENDED | RegularExpressionFlags::MULTI_LINE)
11
+ o |= Regexp::FIXEDENCODING if flags.anybits?(RegularExpressionFlags::EUC_JP | RegularExpressionFlags::WINDOWS_31J | RegularExpressionFlags::UTF_8)
12
+ o |= Regexp::NOENCODING if flags.anybits?(RegularExpressionFlags::ASCII_8BIT)
13
+ o
14
+ end
15
+ end
16
+
17
+ private_constant :RegularExpressionOptions
18
+
6
19
  class FloatNode < Node
7
20
  # Returns the value of the node as a Ruby Float.
8
21
  def value
@@ -24,15 +37,16 @@ module Prism
24
37
  end
25
38
  end
26
39
 
40
+ class InterpolatedMatchLastLineNode < Node
41
+ include RegularExpressionOptions
42
+ end
43
+
27
44
  class InterpolatedRegularExpressionNode < Node
28
- # Returns a numeric value that represents the flags that were used to create
29
- # the regular expression.
30
- def options
31
- o = flags & (RegularExpressionFlags::IGNORE_CASE | RegularExpressionFlags::EXTENDED | RegularExpressionFlags::MULTI_LINE)
32
- o |= Regexp::FIXEDENCODING if flags.anybits?(RegularExpressionFlags::EUC_JP | RegularExpressionFlags::WINDOWS_31J | RegularExpressionFlags::UTF_8)
33
- o |= Regexp::NOENCODING if flags.anybits?(RegularExpressionFlags::ASCII_8BIT)
34
- o
35
- end
45
+ include RegularExpressionOptions
46
+ end
47
+
48
+ class MatchLastLineNode < Node
49
+ include RegularExpressionOptions
36
50
  end
37
51
 
38
52
  class RationalNode < Node
@@ -43,14 +57,7 @@ module Prism
43
57
  end
44
58
 
45
59
  class RegularExpressionNode < Node
46
- # Returns a numeric value that represents the flags that were used to create
47
- # the regular expression.
48
- def options
49
- o = flags & (RegularExpressionFlags::IGNORE_CASE | RegularExpressionFlags::EXTENDED | RegularExpressionFlags::MULTI_LINE)
50
- o |= Regexp::FIXEDENCODING if flags.anybits?(RegularExpressionFlags::EUC_JP | RegularExpressionFlags::WINDOWS_31J | RegularExpressionFlags::UTF_8)
51
- o |= Regexp::NOENCODING if flags.anybits?(RegularExpressionFlags::ASCII_8BIT)
52
- o
53
- end
60
+ include RegularExpressionOptions
54
61
  end
55
62
 
56
63
  class ConstantReadNode < Node
@@ -3,7 +3,7 @@
3
3
  module Prism
4
4
  # This object is responsible for generating the output for the inspect method
5
5
  # implementations of child nodes.
6
- class NodeInspector
6
+ class NodeInspector # :nodoc:
7
7
  attr_reader :prefix, :output
8
8
 
9
9
  def initialize(prefix = "")
data/lib/prism/pack.rb CHANGED
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Prism
4
+ # A parser for the pack template language.
4
5
  module Pack
5
6
  %i[
6
7
  SPACE
@@ -54,9 +55,36 @@ module Prism
54
55
  const_set(const, const)
55
56
  end
56
57
 
58
+ # A directive in the pack template language.
57
59
  class Directive
58
- attr_reader :version, :variant, :source, :type, :signed, :endian, :size, :length_type, :length
60
+ # A symbol representing the version of Ruby.
61
+ attr_reader :version
59
62
 
63
+ # A symbol representing whether or not we are packing or unpacking.
64
+ attr_reader :variant
65
+
66
+ # A byteslice of the source string that this directive represents.
67
+ attr_reader :source
68
+
69
+ # The type of the directive.
70
+ attr_reader :type
71
+
72
+ # The type of signedness of the directive.
73
+ attr_reader :signed
74
+
75
+ # The type of endianness of the directive.
76
+ attr_reader :endian
77
+
78
+ # The size of the directive.
79
+ attr_reader :size
80
+
81
+ # The length type of this directive (used for integers).
82
+ attr_reader :length_type
83
+
84
+ # The length of this directive (used for integers).
85
+ attr_reader :length
86
+
87
+ # Initialize a new directive with the given values.
60
88
  def initialize(version, variant, source, type, signed, endian, size, length_type, length)
61
89
  @version = version
62
90
  @variant = variant
@@ -69,38 +97,42 @@ module Prism
69
97
  @length = length
70
98
  end
71
99
 
100
+ # The descriptions of the various types of endianness.
72
101
  ENDIAN_DESCRIPTIONS = {
73
- AGNOSTIC_ENDIAN: 'agnostic',
74
- LITTLE_ENDIAN: 'little-endian (VAX)',
75
- BIG_ENDIAN: 'big-endian (network)',
76
- NATIVE_ENDIAN: 'native-endian',
77
- ENDIAN_NA: 'n/a'
102
+ AGNOSTIC_ENDIAN: "agnostic",
103
+ LITTLE_ENDIAN: "little-endian (VAX)",
104
+ BIG_ENDIAN: "big-endian (network)",
105
+ NATIVE_ENDIAN: "native-endian",
106
+ ENDIAN_NA: "n/a"
78
107
  }
79
108
 
109
+ # The descriptions of the various types of signedness.
80
110
  SIGNED_DESCRIPTIONS = {
81
- UNSIGNED: 'unsigned',
82
- SIGNED: 'signed',
83
- SIGNED_NA: 'n/a'
111
+ UNSIGNED: "unsigned",
112
+ SIGNED: "signed",
113
+ SIGNED_NA: "n/a"
84
114
  }
85
115
 
116
+ # The descriptions of the various types of sizes.
86
117
  SIZE_DESCRIPTIONS = {
87
- SIZE_SHORT: 'short',
88
- SIZE_INT: 'int-width',
89
- SIZE_LONG: 'long',
90
- SIZE_LONG_LONG: 'long long',
91
- SIZE_8: '8-bit',
92
- SIZE_16: '16-bit',
93
- SIZE_32: '32-bit',
94
- SIZE_64: '64-bit',
95
- SIZE_P: 'pointer-width'
118
+ SIZE_SHORT: "short",
119
+ SIZE_INT: "int-width",
120
+ SIZE_LONG: "long",
121
+ SIZE_LONG_LONG: "long long",
122
+ SIZE_8: "8-bit",
123
+ SIZE_16: "16-bit",
124
+ SIZE_32: "32-bit",
125
+ SIZE_64: "64-bit",
126
+ SIZE_P: "pointer-width"
96
127
  }
97
128
 
129
+ # Provide a human-readable description of the directive.
98
130
  def describe
99
131
  case type
100
132
  when SPACE
101
- 'whitespace'
133
+ "whitespace"
102
134
  when COMMENT
103
- 'comment'
135
+ "comment"
104
136
  when INTEGER
105
137
  if size == SIZE_8
106
138
  base = "#{SIGNED_DESCRIPTIONS[signed]} #{SIZE_DESCRIPTIONS[size]} integer"
@@ -115,58 +147,65 @@ module Prism
115
147
  base
116
148
  end
117
149
  when LENGTH_MAX
118
- base + ', as many as possible'
150
+ base + ", as many as possible"
119
151
  end
120
152
  when UTF8
121
- 'UTF-8 character'
153
+ "UTF-8 character"
122
154
  when BER
123
- 'BER-compressed integer'
155
+ "BER-compressed integer"
124
156
  when FLOAT
125
157
  "#{SIZE_DESCRIPTIONS[size]} #{ENDIAN_DESCRIPTIONS[endian]} float"
126
158
  when STRING_SPACE_PADDED
127
- 'arbitrary binary string (space padded)'
159
+ "arbitrary binary string (space padded)"
128
160
  when STRING_NULL_PADDED
129
- 'arbitrary binary string (null padded, count is width)'
161
+ "arbitrary binary string (null padded, count is width)"
130
162
  when STRING_NULL_TERMINATED
131
- 'arbitrary binary string (null padded, count is width), except that null is added with *'
163
+ "arbitrary binary string (null padded, count is width), except that null is added with *"
132
164
  when STRING_MSB
133
- 'bit string (MSB first)'
165
+ "bit string (MSB first)"
134
166
  when STRING_LSB
135
- 'bit string (LSB first)'
167
+ "bit string (LSB first)"
136
168
  when STRING_HEX_HIGH
137
- 'hex string (high nibble first)'
169
+ "hex string (high nibble first)"
138
170
  when STRING_HEX_LOW
139
- 'hex string (low nibble first)'
171
+ "hex string (low nibble first)"
140
172
  when STRING_UU
141
- 'UU-encoded string'
173
+ "UU-encoded string"
142
174
  when STRING_MIME
143
- 'quoted printable, MIME encoding'
175
+ "quoted printable, MIME encoding"
144
176
  when STRING_BASE64
145
- 'base64 encoded string'
177
+ "base64 encoded string"
146
178
  when STRING_FIXED
147
- 'pointer to a structure (fixed-length string)'
179
+ "pointer to a structure (fixed-length string)"
148
180
  when STRING_POINTER
149
- 'pointer to a null-terminated string'
181
+ "pointer to a null-terminated string"
150
182
  when MOVE
151
- 'move to absolute position'
183
+ "move to absolute position"
152
184
  when BACK
153
- 'back up a byte'
185
+ "back up a byte"
154
186
  when NULL
155
- 'null byte'
187
+ "null byte"
156
188
  else
157
189
  raise
158
190
  end
159
191
  end
160
192
  end
161
193
 
194
+ # The result of parsing a pack template.
162
195
  class Format
163
- attr_reader :directives, :encoding
196
+ # A list of the directives in the template.
197
+ attr_reader :directives
198
+
199
+ # The encoding of the template.
200
+ attr_reader :encoding
164
201
 
202
+ # Create a new Format with the given directives and encoding.
165
203
  def initialize(directives, encoding)
166
204
  @directives = directives
167
205
  @encoding = encoding
168
206
  end
169
207
 
208
+ # Provide a human-readable description of the format.
170
209
  def describe
171
210
  source_width = directives.map { |d| d.source.inspect.length }.max
172
211
  directive_lines = directives.map do |directive|
@@ -178,7 +217,7 @@ module Prism
178
217
  " #{source.ljust(source_width)} #{directive.describe}"
179
218
  end
180
219
 
181
- (['Directives:'] + directive_lines + ['Encoding:', " #{encoding}"]).join("\n")
220
+ (["Directives:"] + directive_lines + ["Encoding:", " #{encoding}"]).join("\n")
182
221
  end
183
222
  end
184
223
  end
@@ -19,7 +19,7 @@ module Prism
19
19
  class Comments
20
20
  # A target for attaching comments that is based on a specific node's
21
21
  # location.
22
- class NodeTarget
22
+ class NodeTarget # :nodoc:
23
23
  attr_reader :node
24
24
 
25
25
  def initialize(node)
@@ -46,7 +46,7 @@ module Prism
46
46
 
47
47
  # A target for attaching comments that is based on a location field on a
48
48
  # node. For example, the `end` token of a ClassNode.
49
- class LocationTarget
49
+ class LocationTarget # :nodoc:
50
50
  attr_reader :location
51
51
 
52
52
  def initialize(location)
@@ -70,12 +70,17 @@ module Prism
70
70
  end
71
71
  end
72
72
 
73
+ # The parse result that we are attaching comments to.
73
74
  attr_reader :parse_result
74
75
 
76
+ # Create a new Comments object that will attach comments to the given
77
+ # parse result.
75
78
  def initialize(parse_result)
76
79
  @parse_result = parse_result
77
80
  end
78
81
 
82
+ # Attach the comments to their respective locations in the tree by
83
+ # mutating the parse result.
79
84
  def attach!
80
85
  parse_result.comments.each do |comment|
81
86
  preceding, enclosing, following = nearest_targets(parse_result.value, comment)
@@ -18,10 +18,12 @@ module Prism
18
18
  # MarkNewlinesVisitor, since that visitor is responsible for marking the
19
19
  # newlines for JRuby/TruffleRuby.
20
20
  class Newlines < Visitor
21
+ # Create a new Newlines visitor with the given newline offsets.
21
22
  def initialize(newline_marked)
22
23
  @newline_marked = newline_marked
23
24
  end
24
25
 
26
+ # Permit block/lambda nodes to mark newlines within themselves.
25
27
  def visit_block_node(node)
26
28
  old_newline_marked = @newline_marked
27
29
  @newline_marked = Array.new(old_newline_marked.size, false)
@@ -35,6 +37,7 @@ module Prism
35
37
 
36
38
  alias_method :visit_lambda_node, :visit_block_node
37
39
 
40
+ # Mark if/unless nodes as newlines.
38
41
  def visit_if_node(node)
39
42
  node.set_newline_flag(@newline_marked)
40
43
  super(node)
@@ -42,6 +45,7 @@ module Prism
42
45
 
43
46
  alias_method :visit_unless_node, :visit_if_node
44
47
 
48
+ # Permit statements lists to mark newlines within themselves.
45
49
  def visit_statements_node(node)
46
50
  node.body.each do |child|
47
51
  child.set_newline_flag(@newline_marked)