prism 0.17.1 → 0.19.0

data/config.yml

@@ -331,27 +331,46 @@ tokens:
 flags:
   - name: ArgumentsNodeFlags
     values:
-      - name: KEYWORD_SPLAT
+      - name: CONTAINS_KEYWORD_SPLAT
         comment: "if arguments contain keyword splat"
     comment: Flags for arguments nodes.
+  - name: ArrayNodeFlags
+    values:
+      - name: CONTAINS_SPLAT
+        comment: "if array contains splat nodes"
+    comment: Flags for array nodes.
   - name: CallNodeFlags
     values:
       - name: SAFE_NAVIGATION
         comment: "&. operator"
       - name: VARIABLE_CALL
         comment: "a call that could have been a local variable"
+      - name: ATTRIBUTE_WRITE
+        comment: "a call that is an attribute write, so the value being written should be returned"
     comment: Flags for call nodes.
+  - name: EncodingFlags
+    values:
+      - name: FORCED_UTF8_ENCODING
+        comment: "internal bytes forced the encoding to UTF-8"
+      - name: FORCED_BINARY_ENCODING
+        comment: "internal bytes forced the encoding to binary"
+    comment: Flags for nodes that have unescaped content.
   - name: IntegerBaseFlags
     values:
       - name: BINARY
         comment: "0b prefix"
-      - name: OCTAL
-        comment: "0o or 0 prefix"
       - name: DECIMAL
         comment: "0d or no prefix"
+      - name: OCTAL
+        comment: "0o or 0 prefix"
       - name: HEXADECIMAL
         comment: "0x prefix"
     comment: Flags for integer nodes that correspond to the base of the integer.
+  - name: KeywordHashNodeFlags
+    values:
+      - name: STATIC_KEYS
+        comment: "a keyword hash which only has `AssocNode` elements all with static literal keys, which means the elements can be treated as keyword arguments"
+    comment: Flags for keyword hash nodes.
   - name: LoopFlags
     values:
       - name: BEGIN_MODIFIER
@@ -380,12 +399,31 @@ flags:
         comment: "s - forces the Windows-31J encoding"
       - name: UTF_8
         comment: "u - forces the UTF-8 encoding"
+      - name: FORCED_UTF8_ENCODING
+        comment: "internal bytes forced the encoding to UTF-8"
+      - name: FORCED_BINARY_ENCODING
+        comment: "internal bytes forced the encoding to binary"
+      - name: FORCED_US_ASCII_ENCODING
+        comment: "internal bytes forced the encoding to US-ASCII"
     comment: Flags for regular expression and match last line nodes.
   - name: StringFlags
     values:
+      - name: FORCED_UTF8_ENCODING
+        comment: "internal bytes forced the encoding to UTF-8"
+      - name: FORCED_BINARY_ENCODING
+        comment: "internal bytes forced the encoding to binary"
       - name: FROZEN
         comment: "frozen by virtue of a `frozen_string_literal` comment"
     comment: Flags for string nodes.
+  - name: SymbolFlags
+    values:
+      - name: FORCED_UTF8_ENCODING
+        comment: "internal bytes forced the encoding to UTF-8"
+      - name: FORCED_BINARY_ENCODING
+        comment: "internal bytes forced the encoding to binary"
+      - name: FORCED_US_ASCII_ENCODING
+        comment: "internal bytes forced the encoding to US-ASCII"
+    comment: Flags for symbol nodes.
 nodes:
   - name: AliasGlobalVariableNode
     fields:
@@ -441,11 +479,11 @@ nodes:
           ^^^^^^^^^^^^^^
   - name: ArgumentsNode
     fields:
-      - name: arguments
-        type: node[]
       - name: flags
         type: flags
         kind: ArgumentsNodeFlags
+      - name: arguments
+        type: node[]
     comment: |
       Represents a set of arguments to a method or a keyword.
 
@@ -453,6 +491,9 @@ nodes:
                  ^^^^^^^^^^^^^
   - name: ArrayNode
     fields:
+      - name: flags
+        type: flags
+        kind: ArrayNodeFlags
       - name: elements
         type: node[]
       - name: opening_loc
@@ -579,9 +620,10 @@ nodes:
     fields:
       - name: locals
         type: constant[]
+      - name: locals_body_index
+        type: uint32
       - name: parameters
         type: node?
-        kind: BlockParametersNode
       - name: body
         type: node?
       - name: opening_loc
@@ -641,15 +683,15 @@ nodes:
           ^^^^^^^^^
   - name: CallAndWriteNode
     fields:
+      - name: flags
+        type: flags
+        kind: CallNodeFlags
       - name: receiver
         type: node?
       - name: call_operator_loc
         type: location?
       - name: message_loc
         type: location?
-      - name: flags
-        type: flags
-        kind: CallNodeFlags
       - name: read_name
         type: constant
       - name: write_name
@@ -665,10 +707,15 @@ nodes:
           ^^^^^^^^^^^^^^^^^
   - name: CallNode
     fields:
+      - name: flags
+        type: flags
+        kind: CallNodeFlags
       - name: receiver
         type: node?
       - name: call_operator_loc
         type: location?
+      - name: name
+        type: constant
       - name: message_loc
         type: location?
       - name: opening_loc
@@ -680,11 +727,6 @@ nodes:
         type: location?
       - name: block
         type: node?
-      - name: flags
-        type: flags
-        kind: CallNodeFlags
-      - name: name
-        type: constant
     comment: |
       Represents a method call, in all of the various forms that can take.
 
@@ -707,15 +749,15 @@ nodes:
           ^^^^^^^^
   - name: CallOperatorWriteNode
     fields:
+      - name: flags
+        type: flags
+        kind: CallNodeFlags
       - name: receiver
         type: node?
       - name: call_operator_loc
         type: location?
       - name: message_loc
         type: location?
-      - name: flags
-        type: flags
-        kind: CallNodeFlags
       - name: read_name
         type: constant
       - name: write_name
@@ -733,15 +775,15 @@ nodes:
           ^^^^^^^^^^^^^^
   - name: CallOrWriteNode
     fields:
+      - name: flags
+        type: flags
+        kind: CallNodeFlags
       - name: receiver
         type: node?
       - name: call_operator_loc
         type: location?
       - name: message_loc
         type: location?
-      - name: flags
-        type: flags
-        kind: CallNodeFlags
       - name: read_name
         type: constant
       - name: write_name
@@ -755,6 +797,32 @@ nodes:
 
           foo.bar ||= value
           ^^^^^^^^^^^^^^^^^
+  - name: CallTargetNode
+    fields:
+      - name: flags
+        type: flags
+        kind: CallNodeFlags
+      - name: receiver
+        type: node
+      - name: call_operator_loc
+        type: location
+      - name: name
+        type: constant
+      - name: message_loc
+        type: location
+    comment: |
+      Represents assigning to a method call.
+
+          foo.bar, = 1
+          ^^^^^^^
+
+          begin
+          rescue => foo.bar
+                    ^^^^^^^
+          end
+
+          for foo.bar in baz do end
+              ^^^^^^^
   - name: CapturePatternNode
     fields:
       - name: value
@@ -768,6 +836,26 @@ nodes:
 
           foo => [bar => baz]
                  ^^^^^^^^^^^^
+  - name: CaseMatchNode
+    fields:
+      - name: predicate
+        type: node?
+      - name: conditions
+        type: node[]
+      - name: consequent
+        type: node?
+        kind: ElseNode
+      - name: case_keyword_loc
+        type: location
+      - name: end_keyword_loc
+        type: location
+    comment: |
+      Represents the use of a case statement for pattern matching.
+
+          case true
+          in false
+          end
+          ^^^^^^^^^
   - name: CaseNode
     fields:
       - name: predicate
@@ -1076,6 +1164,8 @@ nodes:
         type: node?
       - name: locals
         type: constant[]
+      - name: locals_body_index
+        type: uint32
       - name: def_keyword_loc
         type: location
       - name: operator_loc
@@ -1199,15 +1289,15 @@ nodes:
                  ^^^^^^^^^^^^^^^^^^^^
   - name: FlipFlopNode
     fields:
+      - name: flags
+        type: flags
+        kind: RangeFlags
       - name: left
         type: node?
       - name: right
         type: node?
       - name: operator_loc
         type: location
-      - name: flags
-        type: flags
-        kind: RangeFlags
     comment: |
       Represents the use of the `..` or `...` operators to create flip flops.
 
@@ -1385,6 +1475,8 @@ nodes:
         type: location?
       - name: predicate
         type: node
+      - name: then_keyword_loc
+        type: location?
       - name: statements
         type: node?
         kind: StatementsNode
@@ -1423,6 +1515,21 @@ nodes:
 
           { Foo: }
             ^^^^
+  - name: ImplicitRestNode
+    comment: |
+      Represents using a trailing comma to indicate an implicit rest parameter.
+
+          foo { |bar,| }
+                    ^
+
+          foo in [bar,]
+                     ^
+
+          for foo, in bar do end
+                 ^
+
+          foo, = bar
+             ^
   - name: InNode
     fields:
       - name: pattern
@@ -1441,6 +1548,9 @@ nodes:
                   ^^^^^^^^^^^
   - name: IndexAndWriteNode
     fields:
+      - name: flags
+        type: flags
+        kind: CallNodeFlags
       - name: receiver
         type: node?
       - name: call_operator_loc
@@ -1454,9 +1564,6 @@ nodes:
         type: location
       - name: block
         type: node?
-      - name: flags
-        type: flags
-        kind: CallNodeFlags
       - name: operator_loc
         type: location
       - name: value
@@ -1468,6 +1575,9 @@ nodes:
           ^^^^^^^^^^^^^^^^^^^^^^
   - name: IndexOperatorWriteNode
     fields:
+      - name: flags
+        type: flags
+        kind: CallNodeFlags
       - name: receiver
         type: node?
       - name: call_operator_loc
@@ -1481,9 +1591,6 @@ nodes:
         type: location
       - name: block
         type: node?
-      - name: flags
-        type: flags
-        kind: CallNodeFlags
       - name: operator
         type: constant
       - name: operator_loc
@@ -1497,6 +1604,9 @@ nodes:
           ^^^^^^^^^^^^^^^^^^^^^
   - name: IndexOrWriteNode
     fields:
+      - name: flags
+        type: flags
+        kind: CallNodeFlags
       - name: receiver
         type: node?
       - name: call_operator_loc
@@ -1510,9 +1620,6 @@ nodes:
         type: location
       - name: block
         type: node?
-      - name: flags
-        type: flags
-        kind: CallNodeFlags
       - name: operator_loc
         type: location
       - name: value
@@ -1522,6 +1629,35 @@ nodes:
 
           foo.bar[baz] ||= value
           ^^^^^^^^^^^^^^^^^^^^^^
+  - name: IndexTargetNode
+    fields:
+      - name: flags
+        type: flags
+        kind: CallNodeFlags
+      - name: receiver
+        type: node
+      - name: opening_loc
+        type: location
+      - name: arguments
+        type: node?
+        kind: ArgumentsNode
+      - name: closing_loc
+        type: location
+      - name: block
+        type: node?
+    comment: |
+      Represents assigning to an index.
+
+          foo[bar], = 1
+          ^^^^^^^^
+
+          begin
+          rescue => foo[bar]
+                    ^^^^^^^^
+          end
+
+          for foo[bar] in baz do end
+              ^^^^^^^^
   - name: InstanceVariableAndWriteNode
     fields:
       - name: name
@@ -1614,15 +1750,15 @@ nodes:
           ^
   - name: InterpolatedMatchLastLineNode
     fields:
+      - name: flags
+        type: flags
+        kind: RegularExpressionFlags
       - name: opening_loc
         type: location
       - name: parts
         type: node[]
       - name: closing_loc
         type: location
-      - name: flags
-        type: flags
-        kind: RegularExpressionFlags
     newline: parts
     comment: |
       Represents a regular expression literal that contains interpolation that
@@ -1633,15 +1769,15 @@ nodes:
              ^^^^^^^^^^^^^^^^
   - name: InterpolatedRegularExpressionNode
     fields:
+      - name: flags
+        type: flags
+        kind: RegularExpressionFlags
       - name: opening_loc
         type: location
       - name: parts
         type: node[]
       - name: closing_loc
         type: location
-      - name: flags
-        type: flags
-        kind: RegularExpressionFlags
     newline: parts
     comment: |
       Represents a regular expression literal that contains interpolation.
@@ -1692,6 +1828,9 @@ nodes:
           ^^^^^^^^^^^^^^^^
   - name: KeywordHashNode
     fields:
+      - name: flags
+        type: flags
+        kind: KeywordHashNodeFlags
       - name: elements
         type: node[]
     comment: |
@@ -1717,6 +1856,8 @@ nodes:
     fields:
       - name: locals
         type: constant[]
+      - name: locals_body_index
+        type: uint32
       - name: operator_loc
         type: location
       - name: opening_loc
@@ -1725,7 +1866,6 @@ nodes:
         type: location
       - name: parameters
         type: node?
-        kind: BlockParametersNode
       - name: body
         type: node?
     comment: |
@@ -1829,6 +1969,9 @@ nodes:
           ^^^^^^^
   - name: MatchLastLineNode
     fields:
+      - name: flags
+        type: flags
+        kind: RegularExpressionFlags
       - name: opening_loc
         type: location
       - name: content_loc
@@ -1837,9 +1980,6 @@ nodes:
         type: location
       - name: unescaped
         type: string
-      - name: flags
-        type: flags
-        kind: RegularExpressionFlags
     comment: |
       Represents a regular expression literal used in the predicate of a
       conditional to implicitly match against the last line read by an IO
@@ -1878,8 +2018,8 @@ nodes:
       - name: call
         type: node
         kind: CallNode
-      - name: locals
-        type: constant[]
+      - name: targets
+        type: node[]
     comment: |
       Represents writing local variables using a regular expression match with
       named capture groups.
@@ -1977,6 +2117,16 @@ nodes:
           def a(**nil)
                 ^^^^^
           end
+  - name: NumberedParametersNode
+    fields:
+      - name: maximum
+        type: uint8
+    comment: |
+      Represents an implicit set of parameters through the use of numbered
+      parameters within a block or lambda.
+
+          -> { _1 + _2 }
+          ^^^^^^^^^^^^^^
   - name: NumberedReferenceReadNode
     fields:
       - name: number
@@ -2037,7 +2187,6 @@ nodes:
         type: node[]
       - name: rest
         type: node?
-        kind: RestParameterNode
       - name: posts
         type: node[]
       - name: keywords
@@ -2137,15 +2286,15 @@ nodes:
     comment: The top level node of any parse tree.
   - name: RangeNode
     fields:
+      - name: flags
+        type: flags
+        kind: RangeFlags
       - name: left
         type: node?
       - name: right
         type: node?
       - name: operator_loc
         type: location
-      - name: flags
-        type: flags
-        kind: RangeFlags
     comment: |
       Represents the use of the `..` or `...` operators.
 
@@ -2171,6 +2320,9 @@ nodes:
           ^^^^
   - name: RegularExpressionNode
     fields:
+      - name: flags
+        type: flags
+        kind: RegularExpressionFlags
       - name: opening_loc
         type: location
       - name: content_loc
@@ -2179,9 +2331,6 @@ nodes:
         type: location
       - name: unescaped
         type: string
-      - name: flags
-        type: flags
-        kind: RegularExpressionFlags
     comment: |
       Represents a regular expression literal with no interpolation.
 
@@ -2348,17 +2497,6 @@ nodes:
 
           foo; bar; baz
           ^^^^^^^^^^^^^
-  - name: StringConcatNode
-    fields:
-      - name: left
-        type: node
-      - name: right
-        type: node
-    comment: |
-      Represents the use of compile-time string concatenation.
-
-          "foo" "bar"
-          ^^^^^^^^^^^
   - name: StringNode
     fields:
       - name: flags
@@ -2407,6 +2545,9 @@ nodes:
           ^^^^^^^^^^^^^^
   - name: SymbolNode
     fields:
+      - name: flags
+        type: flags
+        kind: SymbolFlags
       - name: opening_loc
         type: location?
       - name: value_loc
@@ -2446,6 +2587,8 @@ nodes:
         type: location
       - name: predicate
         type: node
+      - name: then_keyword_loc
+        type: location?
       - name: statements
         type: node?
         kind: StatementsNode
@@ -2465,6 +2608,9 @@ nodes:
           ^^^^^^^^^^^^^^^^^^^^^^^
   - name: UntilNode
     fields:
+      - name: flags
+        type: flags
+        kind: LoopFlags
       - name: keyword_loc
         type: location
       - name: closing_loc
@@ -2474,9 +2620,6 @@ nodes:
       - name: statements
         type: node?
         kind: StatementsNode
-      - name: flags
-        type: flags
-        kind: LoopFlags
     newline: predicate
     comment: |
       Represents the use of the `until` keyword, either in the block form or the modifier form.
@@ -2504,6 +2647,9 @@ nodes:
           end
   - name: WhileNode
     fields:
+      - name: flags
+        type: flags
+        kind: LoopFlags
       - name: keyword_loc
         type: location
       - name: closing_loc
@@ -2513,9 +2659,6 @@ nodes:
       - name: statements
         type: node?
         kind: StatementsNode
-      - name: flags
-        type: flags
-        kind: LoopFlags
     newline: predicate
     comment: |
       Represents the use of the `while` keyword, either in the block form or the modifier form.
@@ -2527,6 +2670,9 @@ nodes:
           ^^^^^^^^^^^^^^^^^^^^
   - name: XStringNode
     fields:
+      - name: flags
+        type: flags
+        kind: EncodingFlags
       - name: opening_loc
         type: location
       - name: content_loc

data/docs/build_system.md

@@ -16,14 +16,14 @@ The main solution for the second point seems a Makefile, otherwise many of the u
 ## General Design
 
 1. Templates are generated by `templates/template.rb`
-4. The `Makefile` compiles both `librubyparser.a` and `librubyparser.{so,dylib,dll}` from the `src/**/*.c` and `include/**/*.h` files
+4. The `Makefile` compiles both `libprism.a` and `libprism.{so,dylib,dll}` from the `src/**/*.c` and `include/**/*.h` files
 5. The `Rakefile` `:compile` task ensures the above prerequisites are done, then calls `make`,
-  and uses `Rake::ExtensionTask` to compile the C extension (using its `extconf.rb`), which uses `librubyparser.a`
+  and uses `Rake::ExtensionTask` to compile the C extension (using its `extconf.rb`), which uses `libprism.a`
 
 This way there is minimal duplication, and each layer builds on the previous one and has its own responsibilities.
 
 The static library exports no symbols, to avoid any conflict.
-The shared library exports some symbols, and this is fine since there should only be one librubyparser shared library
+The shared library exports some symbols, and this is fine since there should only be one libprism shared library
 loaded per process (i.e., at most one version of the prism *gem* loaded in a process, only the gem uses the shared library).
 
 ## The various ways to build prism
@@ -36,11 +36,11 @@ loaded per process (i.e., at most one version of the prism *gem* loaded in a pro
 
 The gem contains the pre-generated templates.
 When installing the gem, `extconf.rb` is used and that:
-* runs `make build/librubyparser.a`
+* runs `make build/libprism.a`
 * compiles the C extension with mkmf
 
 When installing the gem on JRuby and TruffleRuby, no C extension is built, so instead of the last step,
-there is Ruby code using FFI which uses `librubyparser.{so,dylib,dll}`
+there is Ruby code using FFI which uses `libprism.{so,dylib,dll}`
 to implement the same methods as the C extension, but using serialization instead of many native calls/accesses
 (JRuby does not support C extensions, serialization is faster on TruffleRuby than the C extension).
 
@@ -67,7 +67,7 @@ The script generates the templates when importing.
 Then when `mx build` builds TruffleRuby and the `prism` mx project inside, it runs `make`.
 
 Then the `prism bindings` mx project is built, which contains the [bindings](https://github.com/oracle/truffleruby/blob/master/src/main/c/prism_bindings/src/prism_bindings.c)
-and links to `librubyparser.a` (to avoid exporting symbols, so no conflict when installing the prism gem).
+and links to `libprism.a` (to avoid exporting symbols, so no conflict when installing the prism gem).
 
 ### Building prism as part of JRuby
 

data/docs/building.md

@@ -1,7 +1,6 @@
 # Building
 
-The following describes how to build prism from source.
-This comes directly from the [Makefile](../Makefile).
+The following describes how to build prism from source. This comes directly from the [Makefile](../Makefile).
 
 ## Common
 
@@ -10,7 +9,7 @@ All of the source files match `src/**/*.c` and all of the headers match `include
 The following flags should be used to compile prism:
 
 * `-std=c99` - Use the C99 standard
-* `-Wall -Wconversion -Wextra -Wpedantic -Wundef` - Enable the warnings we care about
+* `-Wall -Wconversion -Wextra -Wpedantic -Wundef -Wno-missing-braces` - Enable the warnings we care about
 * `-Werror` - Treat warnings as errors
 * `-fvisibility=hidden` - Hide all symbols by default
 
@@ -20,3 +19,11 @@ If you want to build prism as a shared library and link against it, you should c
 
 * `-fPIC -shared` - Compile as a shared library
 * `-DPRISM_EXPORT_SYMBOLS` - Export the symbols (by default nothing is exported)
+
+## Flags
+
+`make` respects the `MAKEFLAGS` environment variable. As such, to speed up the build you can run:
+
+```
+MAKEFLAGS="-j10" bundle exec rake compile
+```