prism 0.15.1 → 0.17.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (91) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +35 -1
  3. data/Makefile +12 -0
  4. data/README.md +3 -1
  5. data/config.yml +66 -50
  6. data/docs/configuration.md +2 -0
  7. data/docs/fuzzing.md +1 -1
  8. data/docs/javascript.md +90 -0
  9. data/docs/releasing.md +27 -0
  10. data/docs/ruby_api.md +2 -0
  11. data/docs/serialization.md +28 -29
  12. data/ext/prism/api_node.c +856 -826
  13. data/ext/prism/api_pack.c +20 -9
  14. data/ext/prism/extension.c +494 -119
  15. data/ext/prism/extension.h +1 -1
  16. data/include/prism/ast.h +3157 -747
  17. data/include/prism/defines.h +40 -8
  18. data/include/prism/diagnostic.h +36 -3
  19. data/include/prism/enc/pm_encoding.h +119 -28
  20. data/include/prism/node.h +38 -30
  21. data/include/prism/options.h +204 -0
  22. data/include/prism/pack.h +44 -33
  23. data/include/prism/parser.h +445 -199
  24. data/include/prism/prettyprint.h +26 -0
  25. data/include/prism/regexp.h +16 -2
  26. data/include/prism/util/pm_buffer.h +102 -18
  27. data/include/prism/util/pm_char.h +162 -48
  28. data/include/prism/util/pm_constant_pool.h +128 -34
  29. data/include/prism/util/pm_list.h +68 -38
  30. data/include/prism/util/pm_memchr.h +18 -3
  31. data/include/prism/util/pm_newline_list.h +71 -28
  32. data/include/prism/util/pm_state_stack.h +25 -7
  33. data/include/prism/util/pm_string.h +115 -27
  34. data/include/prism/util/pm_string_list.h +25 -6
  35. data/include/prism/util/pm_strncasecmp.h +32 -0
  36. data/include/prism/util/pm_strpbrk.h +31 -17
  37. data/include/prism/version.h +28 -3
  38. data/include/prism.h +229 -36
  39. data/lib/prism/compiler.rb +5 -5
  40. data/lib/prism/debug.rb +43 -13
  41. data/lib/prism/desugar_compiler.rb +1 -1
  42. data/lib/prism/dispatcher.rb +27 -26
  43. data/lib/prism/dsl.rb +16 -16
  44. data/lib/prism/ffi.rb +138 -61
  45. data/lib/prism/lex_compat.rb +26 -16
  46. data/lib/prism/mutation_compiler.rb +11 -11
  47. data/lib/prism/node.rb +426 -227
  48. data/lib/prism/node_ext.rb +23 -16
  49. data/lib/prism/node_inspector.rb +1 -1
  50. data/lib/prism/pack.rb +79 -40
  51. data/lib/prism/parse_result/comments.rb +7 -2
  52. data/lib/prism/parse_result/newlines.rb +4 -0
  53. data/lib/prism/parse_result.rb +157 -21
  54. data/lib/prism/pattern.rb +14 -3
  55. data/lib/prism/ripper_compat.rb +28 -10
  56. data/lib/prism/serialize.rb +935 -307
  57. data/lib/prism/visitor.rb +9 -5
  58. data/lib/prism.rb +20 -2
  59. data/prism.gemspec +11 -2
  60. data/rbi/prism.rbi +7305 -0
  61. data/rbi/prism_static.rbi +196 -0
  62. data/sig/prism.rbs +4468 -0
  63. data/sig/prism_static.rbs +123 -0
  64. data/src/diagnostic.c +56 -53
  65. data/src/enc/pm_big5.c +1 -0
  66. data/src/enc/pm_euc_jp.c +1 -0
  67. data/src/enc/pm_gbk.c +1 -0
  68. data/src/enc/pm_shift_jis.c +1 -0
  69. data/src/enc/pm_tables.c +316 -80
  70. data/src/enc/pm_unicode.c +54 -9
  71. data/src/enc/pm_windows_31j.c +1 -0
  72. data/src/node.c +357 -345
  73. data/src/options.c +170 -0
  74. data/src/prettyprint.c +7697 -1643
  75. data/src/prism.c +1964 -1125
  76. data/src/regexp.c +153 -95
  77. data/src/serialize.c +432 -397
  78. data/src/token_type.c +3 -1
  79. data/src/util/pm_buffer.c +88 -23
  80. data/src/util/pm_char.c +103 -57
  81. data/src/util/pm_constant_pool.c +52 -22
  82. data/src/util/pm_list.c +12 -4
  83. data/src/util/pm_memchr.c +5 -3
  84. data/src/util/pm_newline_list.c +25 -63
  85. data/src/util/pm_state_stack.c +9 -3
  86. data/src/util/pm_string.c +95 -85
  87. data/src/util/pm_string_list.c +14 -15
  88. data/src/util/pm_strncasecmp.c +10 -3
  89. data/src/util/pm_strpbrk.c +25 -19
  90. metadata +12 -3
  91. data/docs/prism.png +0 -0
@@ -3,6 +3,19 @@
3
3
  # Here we are reopening the prism module to provide methods on nodes that aren't
4
4
  # templated and are meant as convenience methods.
5
5
  module Prism
6
+ module RegularExpressionOptions # :nodoc:
7
+ # Returns a numeric value that represents the flags that were used to create
8
+ # the regular expression.
9
+ def options
10
+ o = flags & (RegularExpressionFlags::IGNORE_CASE | RegularExpressionFlags::EXTENDED | RegularExpressionFlags::MULTI_LINE)
11
+ o |= Regexp::FIXEDENCODING if flags.anybits?(RegularExpressionFlags::EUC_JP | RegularExpressionFlags::WINDOWS_31J | RegularExpressionFlags::UTF_8)
12
+ o |= Regexp::NOENCODING if flags.anybits?(RegularExpressionFlags::ASCII_8BIT)
13
+ o
14
+ end
15
+ end
16
+
17
+ private_constant :RegularExpressionOptions
18
+
6
19
  class FloatNode < Node
7
20
  # Returns the value of the node as a Ruby Float.
8
21
  def value
@@ -24,15 +37,16 @@ module Prism
24
37
  end
25
38
  end
26
39
 
40
+ class InterpolatedMatchLastLineNode < Node
41
+ include RegularExpressionOptions
42
+ end
43
+
27
44
  class InterpolatedRegularExpressionNode < Node
28
- # Returns a numeric value that represents the flags that were used to create
29
- # the regular expression.
30
- def options
31
- o = flags & (RegularExpressionFlags::IGNORE_CASE | RegularExpressionFlags::EXTENDED | RegularExpressionFlags::MULTI_LINE)
32
- o |= Regexp::FIXEDENCODING if flags.anybits?(RegularExpressionFlags::EUC_JP | RegularExpressionFlags::WINDOWS_31J | RegularExpressionFlags::UTF_8)
33
- o |= Regexp::NOENCODING if flags.anybits?(RegularExpressionFlags::ASCII_8BIT)
34
- o
35
- end
45
+ include RegularExpressionOptions
46
+ end
47
+
48
+ class MatchLastLineNode < Node
49
+ include RegularExpressionOptions
36
50
  end
37
51
 
38
52
  class RationalNode < Node
@@ -43,14 +57,7 @@ module Prism
43
57
  end
44
58
 
45
59
  class RegularExpressionNode < Node
46
- # Returns a numeric value that represents the flags that were used to create
47
- # the regular expression.
48
- def options
49
- o = flags & (RegularExpressionFlags::IGNORE_CASE | RegularExpressionFlags::EXTENDED | RegularExpressionFlags::MULTI_LINE)
50
- o |= Regexp::FIXEDENCODING if flags.anybits?(RegularExpressionFlags::EUC_JP | RegularExpressionFlags::WINDOWS_31J | RegularExpressionFlags::UTF_8)
51
- o |= Regexp::NOENCODING if flags.anybits?(RegularExpressionFlags::ASCII_8BIT)
52
- o
53
- end
60
+ include RegularExpressionOptions
54
61
  end
55
62
 
56
63
  class ConstantReadNode < Node
@@ -3,7 +3,7 @@
3
3
  module Prism
4
4
  # This object is responsible for generating the output for the inspect method
5
5
  # implementations of child nodes.
6
- class NodeInspector
6
+ class NodeInspector # :nodoc:
7
7
  attr_reader :prefix, :output
8
8
 
9
9
  def initialize(prefix = "")
data/lib/prism/pack.rb CHANGED
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Prism
4
+ # A parser for the pack template language.
4
5
  module Pack
5
6
  %i[
6
7
  SPACE
@@ -54,9 +55,36 @@ module Prism
54
55
  const_set(const, const)
55
56
  end
56
57
 
58
+ # A directive in the pack template language.
57
59
  class Directive
58
- attr_reader :version, :variant, :source, :type, :signed, :endian, :size, :length_type, :length
60
+ # A symbol representing the version of Ruby.
61
+ attr_reader :version
59
62
 
63
+ # A symbol representing whether or not we are packing or unpacking.
64
+ attr_reader :variant
65
+
66
+ # A byteslice of the source string that this directive represents.
67
+ attr_reader :source
68
+
69
+ # The type of the directive.
70
+ attr_reader :type
71
+
72
+ # The type of signedness of the directive.
73
+ attr_reader :signed
74
+
75
+ # The type of endianness of the directive.
76
+ attr_reader :endian
77
+
78
+ # The size of the directive.
79
+ attr_reader :size
80
+
81
+ # The length type of this directive (used for integers).
82
+ attr_reader :length_type
83
+
84
+ # The length of this directive (used for integers).
85
+ attr_reader :length
86
+
87
+ # Initialize a new directive with the given values.
60
88
  def initialize(version, variant, source, type, signed, endian, size, length_type, length)
61
89
  @version = version
62
90
  @variant = variant
@@ -69,38 +97,42 @@ module Prism
69
97
  @length = length
70
98
  end
71
99
 
100
+ # The descriptions of the various types of endianness.
72
101
  ENDIAN_DESCRIPTIONS = {
73
- AGNOSTIC_ENDIAN: 'agnostic',
74
- LITTLE_ENDIAN: 'little-endian (VAX)',
75
- BIG_ENDIAN: 'big-endian (network)',
76
- NATIVE_ENDIAN: 'native-endian',
77
- ENDIAN_NA: 'n/a'
102
+ AGNOSTIC_ENDIAN: "agnostic",
103
+ LITTLE_ENDIAN: "little-endian (VAX)",
104
+ BIG_ENDIAN: "big-endian (network)",
105
+ NATIVE_ENDIAN: "native-endian",
106
+ ENDIAN_NA: "n/a"
78
107
  }
79
108
 
109
+ # The descriptions of the various types of signedness.
80
110
  SIGNED_DESCRIPTIONS = {
81
- UNSIGNED: 'unsigned',
82
- SIGNED: 'signed',
83
- SIGNED_NA: 'n/a'
111
+ UNSIGNED: "unsigned",
112
+ SIGNED: "signed",
113
+ SIGNED_NA: "n/a"
84
114
  }
85
115
 
116
+ # The descriptions of the various types of sizes.
86
117
  SIZE_DESCRIPTIONS = {
87
- SIZE_SHORT: 'short',
88
- SIZE_INT: 'int-width',
89
- SIZE_LONG: 'long',
90
- SIZE_LONG_LONG: 'long long',
91
- SIZE_8: '8-bit',
92
- SIZE_16: '16-bit',
93
- SIZE_32: '32-bit',
94
- SIZE_64: '64-bit',
95
- SIZE_P: 'pointer-width'
118
+ SIZE_SHORT: "short",
119
+ SIZE_INT: "int-width",
120
+ SIZE_LONG: "long",
121
+ SIZE_LONG_LONG: "long long",
122
+ SIZE_8: "8-bit",
123
+ SIZE_16: "16-bit",
124
+ SIZE_32: "32-bit",
125
+ SIZE_64: "64-bit",
126
+ SIZE_P: "pointer-width"
96
127
  }
97
128
 
129
+ # Provide a human-readable description of the directive.
98
130
  def describe
99
131
  case type
100
132
  when SPACE
101
- 'whitespace'
133
+ "whitespace"
102
134
  when COMMENT
103
- 'comment'
135
+ "comment"
104
136
  when INTEGER
105
137
  if size == SIZE_8
106
138
  base = "#{SIGNED_DESCRIPTIONS[signed]} #{SIZE_DESCRIPTIONS[size]} integer"
@@ -115,58 +147,65 @@ module Prism
115
147
  base
116
148
  end
117
149
  when LENGTH_MAX
118
- base + ', as many as possible'
150
+ base + ", as many as possible"
119
151
  end
120
152
  when UTF8
121
- 'UTF-8 character'
153
+ "UTF-8 character"
122
154
  when BER
123
- 'BER-compressed integer'
155
+ "BER-compressed integer"
124
156
  when FLOAT
125
157
  "#{SIZE_DESCRIPTIONS[size]} #{ENDIAN_DESCRIPTIONS[endian]} float"
126
158
  when STRING_SPACE_PADDED
127
- 'arbitrary binary string (space padded)'
159
+ "arbitrary binary string (space padded)"
128
160
  when STRING_NULL_PADDED
129
- 'arbitrary binary string (null padded, count is width)'
161
+ "arbitrary binary string (null padded, count is width)"
130
162
  when STRING_NULL_TERMINATED
131
- 'arbitrary binary string (null padded, count is width), except that null is added with *'
163
+ "arbitrary binary string (null padded, count is width), except that null is added with *"
132
164
  when STRING_MSB
133
- 'bit string (MSB first)'
165
+ "bit string (MSB first)"
134
166
  when STRING_LSB
135
- 'bit string (LSB first)'
167
+ "bit string (LSB first)"
136
168
  when STRING_HEX_HIGH
137
- 'hex string (high nibble first)'
169
+ "hex string (high nibble first)"
138
170
  when STRING_HEX_LOW
139
- 'hex string (low nibble first)'
171
+ "hex string (low nibble first)"
140
172
  when STRING_UU
141
- 'UU-encoded string'
173
+ "UU-encoded string"
142
174
  when STRING_MIME
143
- 'quoted printable, MIME encoding'
175
+ "quoted printable, MIME encoding"
144
176
  when STRING_BASE64
145
- 'base64 encoded string'
177
+ "base64 encoded string"
146
178
  when STRING_FIXED
147
- 'pointer to a structure (fixed-length string)'
179
+ "pointer to a structure (fixed-length string)"
148
180
  when STRING_POINTER
149
- 'pointer to a null-terminated string'
181
+ "pointer to a null-terminated string"
150
182
  when MOVE
151
- 'move to absolute position'
183
+ "move to absolute position"
152
184
  when BACK
153
- 'back up a byte'
185
+ "back up a byte"
154
186
  when NULL
155
- 'null byte'
187
+ "null byte"
156
188
  else
157
189
  raise
158
190
  end
159
191
  end
160
192
  end
161
193
 
194
+ # The result of parsing a pack template.
162
195
  class Format
163
- attr_reader :directives, :encoding
196
+ # A list of the directives in the template.
197
+ attr_reader :directives
198
+
199
+ # The encoding of the template.
200
+ attr_reader :encoding
164
201
 
202
+ # Create a new Format with the given directives and encoding.
165
203
  def initialize(directives, encoding)
166
204
  @directives = directives
167
205
  @encoding = encoding
168
206
  end
169
207
 
208
+ # Provide a human-readable description of the format.
170
209
  def describe
171
210
  source_width = directives.map { |d| d.source.inspect.length }.max
172
211
  directive_lines = directives.map do |directive|
@@ -178,7 +217,7 @@ module Prism
178
217
  " #{source.ljust(source_width)} #{directive.describe}"
179
218
  end
180
219
 
181
- (['Directives:'] + directive_lines + ['Encoding:', " #{encoding}"]).join("\n")
220
+ (["Directives:"] + directive_lines + ["Encoding:", " #{encoding}"]).join("\n")
182
221
  end
183
222
  end
184
223
  end
@@ -19,7 +19,7 @@ module Prism
19
19
  class Comments
20
20
  # A target for attaching comments that is based on a specific node's
21
21
  # location.
22
- class NodeTarget
22
+ class NodeTarget # :nodoc:
23
23
  attr_reader :node
24
24
 
25
25
  def initialize(node)
@@ -46,7 +46,7 @@ module Prism
46
46
 
47
47
  # A target for attaching comments that is based on a location field on a
48
48
  # node. For example, the `end` token of a ClassNode.
49
- class LocationTarget
49
+ class LocationTarget # :nodoc:
50
50
  attr_reader :location
51
51
 
52
52
  def initialize(location)
@@ -70,12 +70,17 @@ module Prism
70
70
  end
71
71
  end
72
72
 
73
+ # The parse result that we are attaching comments to.
73
74
  attr_reader :parse_result
74
75
 
76
+ # Create a new Comments object that will attach comments to the given
77
+ # parse result.
75
78
  def initialize(parse_result)
76
79
  @parse_result = parse_result
77
80
  end
78
81
 
82
+ # Attach the comments to their respective locations in the tree by
83
+ # mutating the parse result.
79
84
  def attach!
80
85
  parse_result.comments.each do |comment|
81
86
  preceding, enclosing, following = nearest_targets(parse_result.value, comment)
@@ -18,10 +18,12 @@ module Prism
18
18
  # MarkNewlinesVisitor, since that visitor is responsible for marking the
19
19
  # newlines for JRuby/TruffleRuby.
20
20
  class Newlines < Visitor
21
+ # Create a new Newlines visitor with the given newline offsets.
21
22
  def initialize(newline_marked)
22
23
  @newline_marked = newline_marked
23
24
  end
24
25
 
26
+ # Permit block/lambda nodes to mark newlines within themselves.
25
27
  def visit_block_node(node)
26
28
  old_newline_marked = @newline_marked
27
29
  @newline_marked = Array.new(old_newline_marked.size, false)
@@ -35,6 +37,7 @@ module Prism
35
37
 
36
38
  alias_method :visit_lambda_node, :visit_block_node
37
39
 
40
+ # Mark if/unless nodes as newlines.
38
41
  def visit_if_node(node)
39
42
  node.set_newline_flag(@newline_marked)
40
43
  super(node)
@@ -42,6 +45,7 @@ module Prism
42
45
 
43
46
  alias_method :visit_unless_node, :visit_if_node
44
47
 
48
+ # Permit statements lists to mark newlines within themselves.
45
49
  def visit_statements_node(node)
46
50
  node.body.each do |child|
47
51
  child.set_newline_flag(@newline_marked)