prism 0.15.1 → 0.17.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +35 -1
- data/Makefile +12 -0
- data/README.md +3 -1
- data/config.yml +66 -50
- data/docs/configuration.md +2 -0
- data/docs/fuzzing.md +1 -1
- data/docs/javascript.md +90 -0
- data/docs/releasing.md +27 -0
- data/docs/ruby_api.md +2 -0
- data/docs/serialization.md +28 -29
- data/ext/prism/api_node.c +856 -826
- data/ext/prism/api_pack.c +20 -9
- data/ext/prism/extension.c +494 -119
- data/ext/prism/extension.h +1 -1
- data/include/prism/ast.h +3157 -747
- data/include/prism/defines.h +40 -8
- data/include/prism/diagnostic.h +36 -3
- data/include/prism/enc/pm_encoding.h +119 -28
- data/include/prism/node.h +38 -30
- data/include/prism/options.h +204 -0
- data/include/prism/pack.h +44 -33
- data/include/prism/parser.h +445 -199
- data/include/prism/prettyprint.h +26 -0
- data/include/prism/regexp.h +16 -2
- data/include/prism/util/pm_buffer.h +102 -18
- data/include/prism/util/pm_char.h +162 -48
- data/include/prism/util/pm_constant_pool.h +128 -34
- data/include/prism/util/pm_list.h +68 -38
- data/include/prism/util/pm_memchr.h +18 -3
- data/include/prism/util/pm_newline_list.h +71 -28
- data/include/prism/util/pm_state_stack.h +25 -7
- data/include/prism/util/pm_string.h +115 -27
- data/include/prism/util/pm_string_list.h +25 -6
- data/include/prism/util/pm_strncasecmp.h +32 -0
- data/include/prism/util/pm_strpbrk.h +31 -17
- data/include/prism/version.h +28 -3
- data/include/prism.h +229 -36
- data/lib/prism/compiler.rb +5 -5
- data/lib/prism/debug.rb +43 -13
- data/lib/prism/desugar_compiler.rb +1 -1
- data/lib/prism/dispatcher.rb +27 -26
- data/lib/prism/dsl.rb +16 -16
- data/lib/prism/ffi.rb +138 -61
- data/lib/prism/lex_compat.rb +26 -16
- data/lib/prism/mutation_compiler.rb +11 -11
- data/lib/prism/node.rb +426 -227
- data/lib/prism/node_ext.rb +23 -16
- data/lib/prism/node_inspector.rb +1 -1
- data/lib/prism/pack.rb +79 -40
- data/lib/prism/parse_result/comments.rb +7 -2
- data/lib/prism/parse_result/newlines.rb +4 -0
- data/lib/prism/parse_result.rb +157 -21
- data/lib/prism/pattern.rb +14 -3
- data/lib/prism/ripper_compat.rb +28 -10
- data/lib/prism/serialize.rb +935 -307
- data/lib/prism/visitor.rb +9 -5
- data/lib/prism.rb +20 -2
- data/prism.gemspec +11 -2
- data/rbi/prism.rbi +7305 -0
- data/rbi/prism_static.rbi +196 -0
- data/sig/prism.rbs +4468 -0
- data/sig/prism_static.rbs +123 -0
- data/src/diagnostic.c +56 -53
- data/src/enc/pm_big5.c +1 -0
- data/src/enc/pm_euc_jp.c +1 -0
- data/src/enc/pm_gbk.c +1 -0
- data/src/enc/pm_shift_jis.c +1 -0
- data/src/enc/pm_tables.c +316 -80
- data/src/enc/pm_unicode.c +54 -9
- data/src/enc/pm_windows_31j.c +1 -0
- data/src/node.c +357 -345
- data/src/options.c +170 -0
- data/src/prettyprint.c +7697 -1643
- data/src/prism.c +1964 -1125
- data/src/regexp.c +153 -95
- data/src/serialize.c +432 -397
- data/src/token_type.c +3 -1
- data/src/util/pm_buffer.c +88 -23
- data/src/util/pm_char.c +103 -57
- data/src/util/pm_constant_pool.c +52 -22
- data/src/util/pm_list.c +12 -4
- data/src/util/pm_memchr.c +5 -3
- data/src/util/pm_newline_list.c +25 -63
- data/src/util/pm_state_stack.c +9 -3
- data/src/util/pm_string.c +95 -85
- data/src/util/pm_string_list.c +14 -15
- data/src/util/pm_strncasecmp.c +10 -3
- data/src/util/pm_strpbrk.c +25 -19
- metadata +12 -3
- data/docs/prism.png +0 -0
data/lib/prism/node_ext.rb
CHANGED
@@ -3,6 +3,19 @@
|
|
3
3
|
# Here we are reopening the prism module to provide methods on nodes that aren't
|
4
4
|
# templated and are meant as convenience methods.
|
5
5
|
module Prism
|
6
|
+
module RegularExpressionOptions # :nodoc:
|
7
|
+
# Returns a numeric value that represents the flags that were used to create
|
8
|
+
# the regular expression.
|
9
|
+
def options
|
10
|
+
o = flags & (RegularExpressionFlags::IGNORE_CASE | RegularExpressionFlags::EXTENDED | RegularExpressionFlags::MULTI_LINE)
|
11
|
+
o |= Regexp::FIXEDENCODING if flags.anybits?(RegularExpressionFlags::EUC_JP | RegularExpressionFlags::WINDOWS_31J | RegularExpressionFlags::UTF_8)
|
12
|
+
o |= Regexp::NOENCODING if flags.anybits?(RegularExpressionFlags::ASCII_8BIT)
|
13
|
+
o
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
private_constant :RegularExpressionOptions
|
18
|
+
|
6
19
|
class FloatNode < Node
|
7
20
|
# Returns the value of the node as a Ruby Float.
|
8
21
|
def value
|
@@ -24,15 +37,16 @@ module Prism
|
|
24
37
|
end
|
25
38
|
end
|
26
39
|
|
40
|
+
class InterpolatedMatchLastLineNode < Node
|
41
|
+
include RegularExpressionOptions
|
42
|
+
end
|
43
|
+
|
27
44
|
class InterpolatedRegularExpressionNode < Node
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
o |= Regexp::NOENCODING if flags.anybits?(RegularExpressionFlags::ASCII_8BIT)
|
34
|
-
o
|
35
|
-
end
|
45
|
+
include RegularExpressionOptions
|
46
|
+
end
|
47
|
+
|
48
|
+
class MatchLastLineNode < Node
|
49
|
+
include RegularExpressionOptions
|
36
50
|
end
|
37
51
|
|
38
52
|
class RationalNode < Node
|
@@ -43,14 +57,7 @@ module Prism
|
|
43
57
|
end
|
44
58
|
|
45
59
|
class RegularExpressionNode < Node
|
46
|
-
|
47
|
-
# the regular expression.
|
48
|
-
def options
|
49
|
-
o = flags & (RegularExpressionFlags::IGNORE_CASE | RegularExpressionFlags::EXTENDED | RegularExpressionFlags::MULTI_LINE)
|
50
|
-
o |= Regexp::FIXEDENCODING if flags.anybits?(RegularExpressionFlags::EUC_JP | RegularExpressionFlags::WINDOWS_31J | RegularExpressionFlags::UTF_8)
|
51
|
-
o |= Regexp::NOENCODING if flags.anybits?(RegularExpressionFlags::ASCII_8BIT)
|
52
|
-
o
|
53
|
-
end
|
60
|
+
include RegularExpressionOptions
|
54
61
|
end
|
55
62
|
|
56
63
|
class ConstantReadNode < Node
|
data/lib/prism/node_inspector.rb
CHANGED
data/lib/prism/pack.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module Prism
|
4
|
+
# A parser for the pack template language.
|
4
5
|
module Pack
|
5
6
|
%i[
|
6
7
|
SPACE
|
@@ -54,9 +55,36 @@ module Prism
|
|
54
55
|
const_set(const, const)
|
55
56
|
end
|
56
57
|
|
58
|
+
# A directive in the pack template language.
|
57
59
|
class Directive
|
58
|
-
|
60
|
+
# A symbol representing the version of Ruby.
|
61
|
+
attr_reader :version
|
59
62
|
|
63
|
+
# A symbol representing whether or not we are packing or unpacking.
|
64
|
+
attr_reader :variant
|
65
|
+
|
66
|
+
# A byteslice of the source string that this directive represents.
|
67
|
+
attr_reader :source
|
68
|
+
|
69
|
+
# The type of the directive.
|
70
|
+
attr_reader :type
|
71
|
+
|
72
|
+
# The type of signedness of the directive.
|
73
|
+
attr_reader :signed
|
74
|
+
|
75
|
+
# The type of endianness of the directive.
|
76
|
+
attr_reader :endian
|
77
|
+
|
78
|
+
# The size of the directive.
|
79
|
+
attr_reader :size
|
80
|
+
|
81
|
+
# The length type of this directive (used for integers).
|
82
|
+
attr_reader :length_type
|
83
|
+
|
84
|
+
# The length of this directive (used for integers).
|
85
|
+
attr_reader :length
|
86
|
+
|
87
|
+
# Initialize a new directive with the given values.
|
60
88
|
def initialize(version, variant, source, type, signed, endian, size, length_type, length)
|
61
89
|
@version = version
|
62
90
|
@variant = variant
|
@@ -69,38 +97,42 @@ module Prism
|
|
69
97
|
@length = length
|
70
98
|
end
|
71
99
|
|
100
|
+
# The descriptions of the various types of endianness.
|
72
101
|
ENDIAN_DESCRIPTIONS = {
|
73
|
-
AGNOSTIC_ENDIAN:
|
74
|
-
LITTLE_ENDIAN:
|
75
|
-
BIG_ENDIAN:
|
76
|
-
NATIVE_ENDIAN:
|
77
|
-
ENDIAN_NA:
|
102
|
+
AGNOSTIC_ENDIAN: "agnostic",
|
103
|
+
LITTLE_ENDIAN: "little-endian (VAX)",
|
104
|
+
BIG_ENDIAN: "big-endian (network)",
|
105
|
+
NATIVE_ENDIAN: "native-endian",
|
106
|
+
ENDIAN_NA: "n/a"
|
78
107
|
}
|
79
108
|
|
109
|
+
# The descriptions of the various types of signedness.
|
80
110
|
SIGNED_DESCRIPTIONS = {
|
81
|
-
UNSIGNED:
|
82
|
-
SIGNED:
|
83
|
-
SIGNED_NA:
|
111
|
+
UNSIGNED: "unsigned",
|
112
|
+
SIGNED: "signed",
|
113
|
+
SIGNED_NA: "n/a"
|
84
114
|
}
|
85
115
|
|
116
|
+
# The descriptions of the various types of sizes.
|
86
117
|
SIZE_DESCRIPTIONS = {
|
87
|
-
SIZE_SHORT:
|
88
|
-
SIZE_INT:
|
89
|
-
SIZE_LONG:
|
90
|
-
SIZE_LONG_LONG:
|
91
|
-
SIZE_8:
|
92
|
-
SIZE_16:
|
93
|
-
SIZE_32:
|
94
|
-
SIZE_64:
|
95
|
-
SIZE_P:
|
118
|
+
SIZE_SHORT: "short",
|
119
|
+
SIZE_INT: "int-width",
|
120
|
+
SIZE_LONG: "long",
|
121
|
+
SIZE_LONG_LONG: "long long",
|
122
|
+
SIZE_8: "8-bit",
|
123
|
+
SIZE_16: "16-bit",
|
124
|
+
SIZE_32: "32-bit",
|
125
|
+
SIZE_64: "64-bit",
|
126
|
+
SIZE_P: "pointer-width"
|
96
127
|
}
|
97
128
|
|
129
|
+
# Provide a human-readable description of the directive.
|
98
130
|
def describe
|
99
131
|
case type
|
100
132
|
when SPACE
|
101
|
-
|
133
|
+
"whitespace"
|
102
134
|
when COMMENT
|
103
|
-
|
135
|
+
"comment"
|
104
136
|
when INTEGER
|
105
137
|
if size == SIZE_8
|
106
138
|
base = "#{SIGNED_DESCRIPTIONS[signed]} #{SIZE_DESCRIPTIONS[size]} integer"
|
@@ -115,58 +147,65 @@ module Prism
|
|
115
147
|
base
|
116
148
|
end
|
117
149
|
when LENGTH_MAX
|
118
|
-
base +
|
150
|
+
base + ", as many as possible"
|
119
151
|
end
|
120
152
|
when UTF8
|
121
|
-
|
153
|
+
"UTF-8 character"
|
122
154
|
when BER
|
123
|
-
|
155
|
+
"BER-compressed integer"
|
124
156
|
when FLOAT
|
125
157
|
"#{SIZE_DESCRIPTIONS[size]} #{ENDIAN_DESCRIPTIONS[endian]} float"
|
126
158
|
when STRING_SPACE_PADDED
|
127
|
-
|
159
|
+
"arbitrary binary string (space padded)"
|
128
160
|
when STRING_NULL_PADDED
|
129
|
-
|
161
|
+
"arbitrary binary string (null padded, count is width)"
|
130
162
|
when STRING_NULL_TERMINATED
|
131
|
-
|
163
|
+
"arbitrary binary string (null padded, count is width), except that null is added with *"
|
132
164
|
when STRING_MSB
|
133
|
-
|
165
|
+
"bit string (MSB first)"
|
134
166
|
when STRING_LSB
|
135
|
-
|
167
|
+
"bit string (LSB first)"
|
136
168
|
when STRING_HEX_HIGH
|
137
|
-
|
169
|
+
"hex string (high nibble first)"
|
138
170
|
when STRING_HEX_LOW
|
139
|
-
|
171
|
+
"hex string (low nibble first)"
|
140
172
|
when STRING_UU
|
141
|
-
|
173
|
+
"UU-encoded string"
|
142
174
|
when STRING_MIME
|
143
|
-
|
175
|
+
"quoted printable, MIME encoding"
|
144
176
|
when STRING_BASE64
|
145
|
-
|
177
|
+
"base64 encoded string"
|
146
178
|
when STRING_FIXED
|
147
|
-
|
179
|
+
"pointer to a structure (fixed-length string)"
|
148
180
|
when STRING_POINTER
|
149
|
-
|
181
|
+
"pointer to a null-terminated string"
|
150
182
|
when MOVE
|
151
|
-
|
183
|
+
"move to absolute position"
|
152
184
|
when BACK
|
153
|
-
|
185
|
+
"back up a byte"
|
154
186
|
when NULL
|
155
|
-
|
187
|
+
"null byte"
|
156
188
|
else
|
157
189
|
raise
|
158
190
|
end
|
159
191
|
end
|
160
192
|
end
|
161
193
|
|
194
|
+
# The result of parsing a pack template.
|
162
195
|
class Format
|
163
|
-
|
196
|
+
# A list of the directives in the template.
|
197
|
+
attr_reader :directives
|
198
|
+
|
199
|
+
# The encoding of the template.
|
200
|
+
attr_reader :encoding
|
164
201
|
|
202
|
+
# Create a new Format with the given directives and encoding.
|
165
203
|
def initialize(directives, encoding)
|
166
204
|
@directives = directives
|
167
205
|
@encoding = encoding
|
168
206
|
end
|
169
207
|
|
208
|
+
# Provide a human-readable description of the format.
|
170
209
|
def describe
|
171
210
|
source_width = directives.map { |d| d.source.inspect.length }.max
|
172
211
|
directive_lines = directives.map do |directive|
|
@@ -178,7 +217,7 @@ module Prism
|
|
178
217
|
" #{source.ljust(source_width)} #{directive.describe}"
|
179
218
|
end
|
180
219
|
|
181
|
-
([
|
220
|
+
(["Directives:"] + directive_lines + ["Encoding:", " #{encoding}"]).join("\n")
|
182
221
|
end
|
183
222
|
end
|
184
223
|
end
|
@@ -19,7 +19,7 @@ module Prism
|
|
19
19
|
class Comments
|
20
20
|
# A target for attaching comments that is based on a specific node's
|
21
21
|
# location.
|
22
|
-
class NodeTarget
|
22
|
+
class NodeTarget # :nodoc:
|
23
23
|
attr_reader :node
|
24
24
|
|
25
25
|
def initialize(node)
|
@@ -46,7 +46,7 @@ module Prism
|
|
46
46
|
|
47
47
|
# A target for attaching comments that is based on a location field on a
|
48
48
|
# node. For example, the `end` token of a ClassNode.
|
49
|
-
class LocationTarget
|
49
|
+
class LocationTarget # :nodoc:
|
50
50
|
attr_reader :location
|
51
51
|
|
52
52
|
def initialize(location)
|
@@ -70,12 +70,17 @@ module Prism
|
|
70
70
|
end
|
71
71
|
end
|
72
72
|
|
73
|
+
# The parse result that we are attaching comments to.
|
73
74
|
attr_reader :parse_result
|
74
75
|
|
76
|
+
# Create a new Comments object that will attach comments to the given
|
77
|
+
# parse result.
|
75
78
|
def initialize(parse_result)
|
76
79
|
@parse_result = parse_result
|
77
80
|
end
|
78
81
|
|
82
|
+
# Attach the comments to their respective locations in the tree by
|
83
|
+
# mutating the parse result.
|
79
84
|
def attach!
|
80
85
|
parse_result.comments.each do |comment|
|
81
86
|
preceding, enclosing, following = nearest_targets(parse_result.value, comment)
|
@@ -18,10 +18,12 @@ module Prism
|
|
18
18
|
# MarkNewlinesVisitor, since that visitor is responsible for marking the
|
19
19
|
# newlines for JRuby/TruffleRuby.
|
20
20
|
class Newlines < Visitor
|
21
|
+
# Create a new Newlines visitor with the given newline offsets.
|
21
22
|
def initialize(newline_marked)
|
22
23
|
@newline_marked = newline_marked
|
23
24
|
end
|
24
25
|
|
26
|
+
# Permit block/lambda nodes to mark newlines within themselves.
|
25
27
|
def visit_block_node(node)
|
26
28
|
old_newline_marked = @newline_marked
|
27
29
|
@newline_marked = Array.new(old_newline_marked.size, false)
|
@@ -35,6 +37,7 @@ module Prism
|
|
35
37
|
|
36
38
|
alias_method :visit_lambda_node, :visit_block_node
|
37
39
|
|
40
|
+
# Mark if/unless nodes as newlines.
|
38
41
|
def visit_if_node(node)
|
39
42
|
node.set_newline_flag(@newline_marked)
|
40
43
|
super(node)
|
@@ -42,6 +45,7 @@ module Prism
|
|
42
45
|
|
43
46
|
alias_method :visit_unless_node, :visit_if_node
|
44
47
|
|
48
|
+
# Permit statements lists to mark newlines within themselves.
|
45
49
|
def visit_statements_node(node)
|
46
50
|
node.body.each do |child|
|
47
51
|
child.set_newline_flag(@newline_marked)
|