prism 0.15.1 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +35 -1
- data/Makefile +12 -0
- data/README.md +3 -1
- data/config.yml +66 -50
- data/docs/configuration.md +2 -0
- data/docs/fuzzing.md +1 -1
- data/docs/javascript.md +90 -0
- data/docs/releasing.md +27 -0
- data/docs/ruby_api.md +2 -0
- data/docs/serialization.md +28 -29
- data/ext/prism/api_node.c +856 -826
- data/ext/prism/api_pack.c +20 -9
- data/ext/prism/extension.c +494 -119
- data/ext/prism/extension.h +1 -1
- data/include/prism/ast.h +3157 -747
- data/include/prism/defines.h +40 -8
- data/include/prism/diagnostic.h +36 -3
- data/include/prism/enc/pm_encoding.h +119 -28
- data/include/prism/node.h +38 -30
- data/include/prism/options.h +204 -0
- data/include/prism/pack.h +44 -33
- data/include/prism/parser.h +445 -199
- data/include/prism/prettyprint.h +26 -0
- data/include/prism/regexp.h +16 -2
- data/include/prism/util/pm_buffer.h +102 -18
- data/include/prism/util/pm_char.h +162 -48
- data/include/prism/util/pm_constant_pool.h +128 -34
- data/include/prism/util/pm_list.h +68 -38
- data/include/prism/util/pm_memchr.h +18 -3
- data/include/prism/util/pm_newline_list.h +71 -28
- data/include/prism/util/pm_state_stack.h +25 -7
- data/include/prism/util/pm_string.h +115 -27
- data/include/prism/util/pm_string_list.h +25 -6
- data/include/prism/util/pm_strncasecmp.h +32 -0
- data/include/prism/util/pm_strpbrk.h +31 -17
- data/include/prism/version.h +28 -3
- data/include/prism.h +229 -36
- data/lib/prism/compiler.rb +5 -5
- data/lib/prism/debug.rb +43 -13
- data/lib/prism/desugar_compiler.rb +1 -1
- data/lib/prism/dispatcher.rb +27 -26
- data/lib/prism/dsl.rb +16 -16
- data/lib/prism/ffi.rb +138 -61
- data/lib/prism/lex_compat.rb +26 -16
- data/lib/prism/mutation_compiler.rb +11 -11
- data/lib/prism/node.rb +426 -227
- data/lib/prism/node_ext.rb +23 -16
- data/lib/prism/node_inspector.rb +1 -1
- data/lib/prism/pack.rb +79 -40
- data/lib/prism/parse_result/comments.rb +7 -2
- data/lib/prism/parse_result/newlines.rb +4 -0
- data/lib/prism/parse_result.rb +157 -21
- data/lib/prism/pattern.rb +14 -3
- data/lib/prism/ripper_compat.rb +28 -10
- data/lib/prism/serialize.rb +935 -307
- data/lib/prism/visitor.rb +9 -5
- data/lib/prism.rb +20 -2
- data/prism.gemspec +11 -2
- data/rbi/prism.rbi +7305 -0
- data/rbi/prism_static.rbi +196 -0
- data/sig/prism.rbs +4468 -0
- data/sig/prism_static.rbs +123 -0
- data/src/diagnostic.c +56 -53
- data/src/enc/pm_big5.c +1 -0
- data/src/enc/pm_euc_jp.c +1 -0
- data/src/enc/pm_gbk.c +1 -0
- data/src/enc/pm_shift_jis.c +1 -0
- data/src/enc/pm_tables.c +316 -80
- data/src/enc/pm_unicode.c +54 -9
- data/src/enc/pm_windows_31j.c +1 -0
- data/src/node.c +357 -345
- data/src/options.c +170 -0
- data/src/prettyprint.c +7697 -1643
- data/src/prism.c +1964 -1125
- data/src/regexp.c +153 -95
- data/src/serialize.c +432 -397
- data/src/token_type.c +3 -1
- data/src/util/pm_buffer.c +88 -23
- data/src/util/pm_char.c +103 -57
- data/src/util/pm_constant_pool.c +52 -22
- data/src/util/pm_list.c +12 -4
- data/src/util/pm_memchr.c +5 -3
- data/src/util/pm_newline_list.c +25 -63
- data/src/util/pm_state_stack.c +9 -3
- data/src/util/pm_string.c +95 -85
- data/src/util/pm_string_list.c +14 -15
- data/src/util/pm_strncasecmp.c +10 -3
- data/src/util/pm_strpbrk.c +25 -19
- metadata +12 -3
- data/docs/prism.png +0 -0
data/lib/prism/node_ext.rb
CHANGED
@@ -3,6 +3,19 @@
|
|
3
3
|
# Here we are reopening the prism module to provide methods on nodes that aren't
|
4
4
|
# templated and are meant as convenience methods.
|
5
5
|
module Prism
|
6
|
+
module RegularExpressionOptions # :nodoc:
|
7
|
+
# Returns a numeric value that represents the flags that were used to create
|
8
|
+
# the regular expression.
|
9
|
+
def options
|
10
|
+
o = flags & (RegularExpressionFlags::IGNORE_CASE | RegularExpressionFlags::EXTENDED | RegularExpressionFlags::MULTI_LINE)
|
11
|
+
o |= Regexp::FIXEDENCODING if flags.anybits?(RegularExpressionFlags::EUC_JP | RegularExpressionFlags::WINDOWS_31J | RegularExpressionFlags::UTF_8)
|
12
|
+
o |= Regexp::NOENCODING if flags.anybits?(RegularExpressionFlags::ASCII_8BIT)
|
13
|
+
o
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
private_constant :RegularExpressionOptions
|
18
|
+
|
6
19
|
class FloatNode < Node
|
7
20
|
# Returns the value of the node as a Ruby Float.
|
8
21
|
def value
|
@@ -24,15 +37,16 @@ module Prism
|
|
24
37
|
end
|
25
38
|
end
|
26
39
|
|
40
|
+
class InterpolatedMatchLastLineNode < Node
|
41
|
+
include RegularExpressionOptions
|
42
|
+
end
|
43
|
+
|
27
44
|
class InterpolatedRegularExpressionNode < Node
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
o |= Regexp::NOENCODING if flags.anybits?(RegularExpressionFlags::ASCII_8BIT)
|
34
|
-
o
|
35
|
-
end
|
45
|
+
include RegularExpressionOptions
|
46
|
+
end
|
47
|
+
|
48
|
+
class MatchLastLineNode < Node
|
49
|
+
include RegularExpressionOptions
|
36
50
|
end
|
37
51
|
|
38
52
|
class RationalNode < Node
|
@@ -43,14 +57,7 @@ module Prism
|
|
43
57
|
end
|
44
58
|
|
45
59
|
class RegularExpressionNode < Node
|
46
|
-
|
47
|
-
# the regular expression.
|
48
|
-
def options
|
49
|
-
o = flags & (RegularExpressionFlags::IGNORE_CASE | RegularExpressionFlags::EXTENDED | RegularExpressionFlags::MULTI_LINE)
|
50
|
-
o |= Regexp::FIXEDENCODING if flags.anybits?(RegularExpressionFlags::EUC_JP | RegularExpressionFlags::WINDOWS_31J | RegularExpressionFlags::UTF_8)
|
51
|
-
o |= Regexp::NOENCODING if flags.anybits?(RegularExpressionFlags::ASCII_8BIT)
|
52
|
-
o
|
53
|
-
end
|
60
|
+
include RegularExpressionOptions
|
54
61
|
end
|
55
62
|
|
56
63
|
class ConstantReadNode < Node
|
data/lib/prism/node_inspector.rb
CHANGED
data/lib/prism/pack.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module Prism
|
4
|
+
# A parser for the pack template language.
|
4
5
|
module Pack
|
5
6
|
%i[
|
6
7
|
SPACE
|
@@ -54,9 +55,36 @@ module Prism
|
|
54
55
|
const_set(const, const)
|
55
56
|
end
|
56
57
|
|
58
|
+
# A directive in the pack template language.
|
57
59
|
class Directive
|
58
|
-
|
60
|
+
# A symbol representing the version of Ruby.
|
61
|
+
attr_reader :version
|
59
62
|
|
63
|
+
# A symbol representing whether or not we are packing or unpacking.
|
64
|
+
attr_reader :variant
|
65
|
+
|
66
|
+
# A byteslice of the source string that this directive represents.
|
67
|
+
attr_reader :source
|
68
|
+
|
69
|
+
# The type of the directive.
|
70
|
+
attr_reader :type
|
71
|
+
|
72
|
+
# The type of signedness of the directive.
|
73
|
+
attr_reader :signed
|
74
|
+
|
75
|
+
# The type of endianness of the directive.
|
76
|
+
attr_reader :endian
|
77
|
+
|
78
|
+
# The size of the directive.
|
79
|
+
attr_reader :size
|
80
|
+
|
81
|
+
# The length type of this directive (used for integers).
|
82
|
+
attr_reader :length_type
|
83
|
+
|
84
|
+
# The length of this directive (used for integers).
|
85
|
+
attr_reader :length
|
86
|
+
|
87
|
+
# Initialize a new directive with the given values.
|
60
88
|
def initialize(version, variant, source, type, signed, endian, size, length_type, length)
|
61
89
|
@version = version
|
62
90
|
@variant = variant
|
@@ -69,38 +97,42 @@ module Prism
|
|
69
97
|
@length = length
|
70
98
|
end
|
71
99
|
|
100
|
+
# The descriptions of the various types of endianness.
|
72
101
|
ENDIAN_DESCRIPTIONS = {
|
73
|
-
AGNOSTIC_ENDIAN:
|
74
|
-
LITTLE_ENDIAN:
|
75
|
-
BIG_ENDIAN:
|
76
|
-
NATIVE_ENDIAN:
|
77
|
-
ENDIAN_NA:
|
102
|
+
AGNOSTIC_ENDIAN: "agnostic",
|
103
|
+
LITTLE_ENDIAN: "little-endian (VAX)",
|
104
|
+
BIG_ENDIAN: "big-endian (network)",
|
105
|
+
NATIVE_ENDIAN: "native-endian",
|
106
|
+
ENDIAN_NA: "n/a"
|
78
107
|
}
|
79
108
|
|
109
|
+
# The descriptions of the various types of signedness.
|
80
110
|
SIGNED_DESCRIPTIONS = {
|
81
|
-
UNSIGNED:
|
82
|
-
SIGNED:
|
83
|
-
SIGNED_NA:
|
111
|
+
UNSIGNED: "unsigned",
|
112
|
+
SIGNED: "signed",
|
113
|
+
SIGNED_NA: "n/a"
|
84
114
|
}
|
85
115
|
|
116
|
+
# The descriptions of the various types of sizes.
|
86
117
|
SIZE_DESCRIPTIONS = {
|
87
|
-
SIZE_SHORT:
|
88
|
-
SIZE_INT:
|
89
|
-
SIZE_LONG:
|
90
|
-
SIZE_LONG_LONG:
|
91
|
-
SIZE_8:
|
92
|
-
SIZE_16:
|
93
|
-
SIZE_32:
|
94
|
-
SIZE_64:
|
95
|
-
SIZE_P:
|
118
|
+
SIZE_SHORT: "short",
|
119
|
+
SIZE_INT: "int-width",
|
120
|
+
SIZE_LONG: "long",
|
121
|
+
SIZE_LONG_LONG: "long long",
|
122
|
+
SIZE_8: "8-bit",
|
123
|
+
SIZE_16: "16-bit",
|
124
|
+
SIZE_32: "32-bit",
|
125
|
+
SIZE_64: "64-bit",
|
126
|
+
SIZE_P: "pointer-width"
|
96
127
|
}
|
97
128
|
|
129
|
+
# Provide a human-readable description of the directive.
|
98
130
|
def describe
|
99
131
|
case type
|
100
132
|
when SPACE
|
101
|
-
|
133
|
+
"whitespace"
|
102
134
|
when COMMENT
|
103
|
-
|
135
|
+
"comment"
|
104
136
|
when INTEGER
|
105
137
|
if size == SIZE_8
|
106
138
|
base = "#{SIGNED_DESCRIPTIONS[signed]} #{SIZE_DESCRIPTIONS[size]} integer"
|
@@ -115,58 +147,65 @@ module Prism
|
|
115
147
|
base
|
116
148
|
end
|
117
149
|
when LENGTH_MAX
|
118
|
-
base +
|
150
|
+
base + ", as many as possible"
|
119
151
|
end
|
120
152
|
when UTF8
|
121
|
-
|
153
|
+
"UTF-8 character"
|
122
154
|
when BER
|
123
|
-
|
155
|
+
"BER-compressed integer"
|
124
156
|
when FLOAT
|
125
157
|
"#{SIZE_DESCRIPTIONS[size]} #{ENDIAN_DESCRIPTIONS[endian]} float"
|
126
158
|
when STRING_SPACE_PADDED
|
127
|
-
|
159
|
+
"arbitrary binary string (space padded)"
|
128
160
|
when STRING_NULL_PADDED
|
129
|
-
|
161
|
+
"arbitrary binary string (null padded, count is width)"
|
130
162
|
when STRING_NULL_TERMINATED
|
131
|
-
|
163
|
+
"arbitrary binary string (null padded, count is width), except that null is added with *"
|
132
164
|
when STRING_MSB
|
133
|
-
|
165
|
+
"bit string (MSB first)"
|
134
166
|
when STRING_LSB
|
135
|
-
|
167
|
+
"bit string (LSB first)"
|
136
168
|
when STRING_HEX_HIGH
|
137
|
-
|
169
|
+
"hex string (high nibble first)"
|
138
170
|
when STRING_HEX_LOW
|
139
|
-
|
171
|
+
"hex string (low nibble first)"
|
140
172
|
when STRING_UU
|
141
|
-
|
173
|
+
"UU-encoded string"
|
142
174
|
when STRING_MIME
|
143
|
-
|
175
|
+
"quoted printable, MIME encoding"
|
144
176
|
when STRING_BASE64
|
145
|
-
|
177
|
+
"base64 encoded string"
|
146
178
|
when STRING_FIXED
|
147
|
-
|
179
|
+
"pointer to a structure (fixed-length string)"
|
148
180
|
when STRING_POINTER
|
149
|
-
|
181
|
+
"pointer to a null-terminated string"
|
150
182
|
when MOVE
|
151
|
-
|
183
|
+
"move to absolute position"
|
152
184
|
when BACK
|
153
|
-
|
185
|
+
"back up a byte"
|
154
186
|
when NULL
|
155
|
-
|
187
|
+
"null byte"
|
156
188
|
else
|
157
189
|
raise
|
158
190
|
end
|
159
191
|
end
|
160
192
|
end
|
161
193
|
|
194
|
+
# The result of parsing a pack template.
|
162
195
|
class Format
|
163
|
-
|
196
|
+
# A list of the directives in the template.
|
197
|
+
attr_reader :directives
|
198
|
+
|
199
|
+
# The encoding of the template.
|
200
|
+
attr_reader :encoding
|
164
201
|
|
202
|
+
# Create a new Format with the given directives and encoding.
|
165
203
|
def initialize(directives, encoding)
|
166
204
|
@directives = directives
|
167
205
|
@encoding = encoding
|
168
206
|
end
|
169
207
|
|
208
|
+
# Provide a human-readable description of the format.
|
170
209
|
def describe
|
171
210
|
source_width = directives.map { |d| d.source.inspect.length }.max
|
172
211
|
directive_lines = directives.map do |directive|
|
@@ -178,7 +217,7 @@ module Prism
|
|
178
217
|
" #{source.ljust(source_width)} #{directive.describe}"
|
179
218
|
end
|
180
219
|
|
181
|
-
([
|
220
|
+
(["Directives:"] + directive_lines + ["Encoding:", " #{encoding}"]).join("\n")
|
182
221
|
end
|
183
222
|
end
|
184
223
|
end
|
@@ -19,7 +19,7 @@ module Prism
|
|
19
19
|
class Comments
|
20
20
|
# A target for attaching comments that is based on a specific node's
|
21
21
|
# location.
|
22
|
-
class NodeTarget
|
22
|
+
class NodeTarget # :nodoc:
|
23
23
|
attr_reader :node
|
24
24
|
|
25
25
|
def initialize(node)
|
@@ -46,7 +46,7 @@ module Prism
|
|
46
46
|
|
47
47
|
# A target for attaching comments that is based on a location field on a
|
48
48
|
# node. For example, the `end` token of a ClassNode.
|
49
|
-
class LocationTarget
|
49
|
+
class LocationTarget # :nodoc:
|
50
50
|
attr_reader :location
|
51
51
|
|
52
52
|
def initialize(location)
|
@@ -70,12 +70,17 @@ module Prism
|
|
70
70
|
end
|
71
71
|
end
|
72
72
|
|
73
|
+
# The parse result that we are attaching comments to.
|
73
74
|
attr_reader :parse_result
|
74
75
|
|
76
|
+
# Create a new Comments object that will attach comments to the given
|
77
|
+
# parse result.
|
75
78
|
def initialize(parse_result)
|
76
79
|
@parse_result = parse_result
|
77
80
|
end
|
78
81
|
|
82
|
+
# Attach the comments to their respective locations in the tree by
|
83
|
+
# mutating the parse result.
|
79
84
|
def attach!
|
80
85
|
parse_result.comments.each do |comment|
|
81
86
|
preceding, enclosing, following = nearest_targets(parse_result.value, comment)
|
@@ -18,10 +18,12 @@ module Prism
|
|
18
18
|
# MarkNewlinesVisitor, since that visitor is responsible for marking the
|
19
19
|
# newlines for JRuby/TruffleRuby.
|
20
20
|
class Newlines < Visitor
|
21
|
+
# Create a new Newlines visitor with the given newline offsets.
|
21
22
|
def initialize(newline_marked)
|
22
23
|
@newline_marked = newline_marked
|
23
24
|
end
|
24
25
|
|
26
|
+
# Permit block/lambda nodes to mark newlines within themselves.
|
25
27
|
def visit_block_node(node)
|
26
28
|
old_newline_marked = @newline_marked
|
27
29
|
@newline_marked = Array.new(old_newline_marked.size, false)
|
@@ -35,6 +37,7 @@ module Prism
|
|
35
37
|
|
36
38
|
alias_method :visit_lambda_node, :visit_block_node
|
37
39
|
|
40
|
+
# Mark if/unless nodes as newlines.
|
38
41
|
def visit_if_node(node)
|
39
42
|
node.set_newline_flag(@newline_marked)
|
40
43
|
super(node)
|
@@ -42,6 +45,7 @@ module Prism
|
|
42
45
|
|
43
46
|
alias_method :visit_unless_node, :visit_if_node
|
44
47
|
|
48
|
+
# Permit statements lists to mark newlines within themselves.
|
45
49
|
def visit_statements_node(node)
|
46
50
|
node.body.each do |child|
|
47
51
|
child.set_newline_flag(@newline_marked)
|