yarp 0.9.0 → 0.11.0

data/docs/configuration.md

@@ -51,7 +51,6 @@ The available values for `type` are:
 * `constant[]` - A child node that is an array of constants. This is a `yp_constant_id_list_t` in C.
 * `location` - A child node that is a location. This is a `yp_location_t` in C.
 * `location?` - A child node that is a location that is optionally present. This is a `yp_location_t` in C, but if the value is not present then the `start` and `end` fields will be `NULL`.
-* `location[]` - A child node that is an array of locations. This is a `yp_location_list_t` in C.
 * `uint32` - A child node that is a 32-bit unsigned integer. This is a `uint32_t` in C.
 
 If the type is `node` or `node?` then the value also accepts an optional `kind` key (a string). This key is expected to match to the name of another node type within `config.yml`. This changes a couple of places where code is templated out to use the more specific struct name instead of the generic `yp_node_t`. For example, with `kind: StatementsNode` the `yp_node_t *` in C becomes a `yp_statements_node_t *`.

data/docs/encoding.md

@@ -61,22 +61,22 @@ typedef struct {
     // Return the number of bytes that the next character takes if it is valid
     // in the encoding. Does not read more than n bytes. It is assumed that n is
     // at least 1.
-    size_t (*char_width)(const char *c, ptrdiff_t n);
+    size_t (*char_width)(const uint8_t *b, ptrdiff_t n);
 
     // Return the number of bytes that the next character takes if it is valid
     // in the encoding and is alphabetical. Does not read more than n bytes. It
     // is assumed that n is at least 1.
-    size_t (*alpha_char)(const char *c, ptrdiff_t n);
+    size_t (*alpha_char)(const uint8_t *b, ptrdiff_t n);
 
     // Return the number of bytes that the next character takes if it is valid
     // in the encoding and is alphanumeric. Does not read more than n bytes. It
     // is assumed that n is at least 1.
-    size_t (*alnum_char)(const char *c, ptrdiff_t n);
+    size_t (*alnum_char)(const uint8_t *b, ptrdiff_t n);
 
     // Return true if the next character is valid in the encoding and is an
     // uppercase character. Does not read more than n bytes. It is assumed that
     // n is at least 1.
-    bool (*isupper_char)(const char *c, ptrdiff_t n);
+    bool (*isupper_char)(const uint8_t *b, ptrdiff_t n);
 
     // The name of the encoding. This should correspond to a value that can be
     // passed to Encoding.find in Ruby.
@@ -90,7 +90,7 @@ typedef struct {
 // the ability here to call out to a user-defined function to get an encoding
 // struct. If the function returns something that isn't NULL, we set that to
 // our encoding and use it to parse identifiers.
-typedef yp_encoding_t *(*yp_encoding_decode_callback_t)(yp_parser_t *parser, const char *name, size_t width);
+typedef yp_encoding_t *(*yp_encoding_decode_callback_t)(yp_parser_t *parser, const uint8_t *name, size_t width);
 
 // Register a callback that will be called when YARP encounters a magic comment
 // with an encoding referenced that it doesn't understand. The callback should

data/docs/mapping.md

@@ -10,108 +10,108 @@ The following table shows how the various CRuby nodes are mapped to YARP nodes.
 | --- | --- |
 | `NODE_SCOPE` | |
 | `NODE_BLOCK` | |
-| `NODE_IF` | `YP_NODE_IF_NODE` |
-| `NODE_UNLESS` | `YP_NODE_UNLESS_NODE` |
-| `NODE_CASE` | `YP_NODE_CASE_NODE` |
-| `NODE_CASE2` | `YP_NODE_CASE_NODE` (with a null predicate) |
+| `NODE_IF` | `YP_IF_NODE` |
+| `NODE_UNLESS` | `YP_UNLESS_NODE` |
+| `NODE_CASE` | `YP_CASE_NODE` |
+| `NODE_CASE2` | `YP_CASE_NODE` (with a null predicate) |
 | `NODE_CASE3` | |
-| `NODE_WHEN` | `YP_NODE_WHEN_NODE` |
-| `NODE_IN` | `YP_NODE_IN_NODE` |
-| `NODE_WHILE` | `YP_NODE_WHILE_NODE` |
-| `NODE_UNTIL` | `YP_NODE_UNTIL_NODE` |
-| `NODE_ITER` | `YP_NODE_CALL_NODE` (with a non-null block) |
-| `NODE_FOR` | `YP_NODE_FOR_NODE` |
-| `NODE_FOR_MASGN` | `YP_NODE_FOR_NODE` (with a multi-write node as the index) |
-| `NODE_BREAK` | `YP_NODE_BREAK_NODE` |
-| `NODE_NEXT` | `YP_NODE_NEXT_NODE` |
-| `NODE_REDO` | `YP_NODE_REDO_NODE` |
-| `NODE_RETRY` | `YP_NODE_RETRY_NODE` |
-| `NODE_BEGIN` | `YP_NODE_BEGIN_NODE` |
-| `NODE_RESCUE` | `YP_NODE_RESCUE_NODE` |
+| `NODE_WHEN` | `YP_WHEN_NODE` |
+| `NODE_IN` | `YP_IN_NODE` |
+| `NODE_WHILE` | `YP_WHILE_NODE` |
+| `NODE_UNTIL` | `YP_UNTIL_NODE` |
+| `NODE_ITER` | `YP_CALL_NODE` (with a non-null block) |
+| `NODE_FOR` | `YP_FOR_NODE` |
+| `NODE_FOR_MASGN` | `YP_FOR_NODE` (with a multi-write node as the index) |
+| `NODE_BREAK` | `YP_BREAK_NODE` |
+| `NODE_NEXT` | `YP_NEXT_NODE` |
+| `NODE_REDO` | `YP_REDO_NODE` |
+| `NODE_RETRY` | `YP_RETRY_NODE` |
+| `NODE_BEGIN` | `YP_BEGIN_NODE` |
+| `NODE_RESCUE` | `YP_RESCUE_NODE` |
 | `NODE_RESBODY` | |
-| `NODE_ENSURE` | `YP_NODE_ENSURE_NODE` |
-| `NODE_AND` | `YP_NODE_AND_NODE` |
-| `NODE_OR` | `YP_NODE_OR_NODE` |
-| `NODE_MASGN` | `YP_NODE_MULTI_WRITE_NODE` |
-| `NODE_LASGN` | `YP_NODE_LOCAL_VARIABLE_WRITE_NODE` |
-| `NODE_DASGN` | `YP_NODE_LOCAL_VARIABLE_WRITE_NODE` |
-| `NODE_GASGN` | `YP_NODE_GLOBAL_VARIABLE_WRITE_NODE` |
-| `NODE_IASGN` | `YP_NODE_INSTANCE_VARIABLE_WRITE_NODE` |
-| `NODE_CDECL` | `YP_NODE_CONSTANT_PATH_WRITE_NODE` |
-| `NODE_CVASGN` | `YP_NODE_CLASS_VARIABLE_WRITE_NODE` |
+| `NODE_ENSURE` | `YP_ENSURE_NODE` |
+| `NODE_AND` | `YP_AND_NODE` |
+| `NODE_OR` | `YP_OR_NODE` |
+| `NODE_MASGN` | `YP_MULTI_WRITE_NODE` |
+| `NODE_LASGN` | `YP_LOCAL_VARIABLE_WRITE_NODE` |
+| `NODE_DASGN` | `YP_LOCAL_VARIABLE_WRITE_NODE` |
+| `NODE_GASGN` | `YP_GLOBAL_VARIABLE_WRITE_NODE` |
+| `NODE_IASGN` | `YP_INSTANCE_VARIABLE_WRITE_NODE` |
+| `NODE_CDECL` | `YP_CONSTANT_PATH_WRITE_NODE` |
+| `NODE_CVASGN` | `YP_CLASS_VARIABLE_WRITE_NODE` |
 | `NODE_OP_ASGN1` | |
 | `NODE_OP_ASGN2` | |
-| `NODE_OP_ASGN_AND` | `YP_NODE_OPERATOR_AND_ASSIGNMENT_NODE` |
-| `NODE_OP_ASGN_OR` | `YP_NODE_OPERATOR_OR_ASSIGNMENT_NODE` |
+| `NODE_OP_ASGN_AND` | `YP_OPERATOR_AND_ASSIGNMENT_NODE` |
+| `NODE_OP_ASGN_OR` | `YP_OPERATOR_OR_ASSIGNMENT_NODE` |
 | `NODE_OP_CDECL` | |
-| `NODE_CALL` | `YP_NODE_CALL_NODE` |
-| `NODE_OPCALL` | `YP_NODE_CALL_NODE` (with an operator as the method) |
-| `NODE_FCALL` | `YP_NODE_CALL_NODE` (with a null receiver and parentheses) |
-| `NODE_VCALL` | `YP_NODE_CALL_NODE` (with a null receiver and parentheses or arguments) |
-| `NODE_QCALL` | `YP_NODE_CALL_NODE` (with a &. operator) |
-| `NODE_SUPER` | `YP_NODE_SUPER_NODE` |
-| `NODE_ZSUPER` | `YP_NODE_FORWARDING_SUPER_NODE` |
-| `NODE_LIST` | `YP_NODE_ARRAY_NODE` |
-| `NODE_ZLIST` | `YP_NODE_ARRAY_NODE` (with no child elements) |
-| `NODE_VALUES` | `YP_NODE_ARGUMENTS_NODE` |
-| `NODE_HASH` | `YP_NODE_HASH_NODE` |
-| `NODE_RETURN` | `YP_NODE_RETURN_NODE` |
-| `NODE_YIELD` | `YP_NODE_YIELD_NODE` |
-| `NODE_LVAR` | `YP_NODE_LOCAL_VARIABLE_READ_NODE` |
-| `NODE_DVAR` | `YP_NODE_LOCAL_VARIABLE_READ_NODE` |
-| `NODE_GVAR` | `YP_NODE_GLOBAL_VARIABLE_READ_NODE` |
-| `NODE_IVAR` | `YP_NODE_INSTANCE_VARIABLE_READ_NODE` |
-| `NODE_CONST` | `YP_NODE_CONSTANT_PATH_READ_NODE` |
-| `NODE_CVAR` | `YP_NODE_CLASS_VARIABLE_READ_NODE` |
-| `NODE_NTH_REF` | `YP_NODE_NUMBERED_REFERENCE_READ_NODE` |
-| `NODE_BACK_REF` | `YP_NODE_BACK_REFERENCE_READ_NODE` |
+| `NODE_CALL` | `YP_CALL_NODE` |
+| `NODE_OPCALL` | `YP_CALL_NODE` (with an operator as the method) |
+| `NODE_FCALL` | `YP_CALL_NODE` (with a null receiver and parentheses) |
+| `NODE_VCALL` | `YP_CALL_NODE` (with a null receiver and parentheses or arguments) |
+| `NODE_QCALL` | `YP_CALL_NODE` (with a &. operator) |
+| `NODE_SUPER` | `YP_SUPER_NODE` |
+| `NODE_ZSUPER` | `YP_FORWARDING_SUPER_NODE` |
+| `NODE_LIST` | `YP_ARRAY_NODE` |
+| `NODE_ZLIST` | `YP_ARRAY_NODE` (with no child elements) |
+| `NODE_VALUES` | `YP_ARGUMENTS_NODE` |
+| `NODE_HASH` | `YP_HASH_NODE` |
+| `NODE_RETURN` | `YP_RETURN_NODE` |
+| `NODE_YIELD` | `YP_YIELD_NODE` |
+| `NODE_LVAR` | `YP_LOCAL_VARIABLE_READ_NODE` |
+| `NODE_DVAR` | `YP_LOCAL_VARIABLE_READ_NODE` |
+| `NODE_GVAR` | `YP_GLOBAL_VARIABLE_READ_NODE` |
+| `NODE_IVAR` | `YP_INSTANCE_VARIABLE_READ_NODE` |
+| `NODE_CONST` | `YP_CONSTANT_PATH_READ_NODE` |
+| `NODE_CVAR` | `YP_CLASS_VARIABLE_READ_NODE` |
+| `NODE_NTH_REF` | `YP_NUMBERED_REFERENCE_READ_NODE` |
+| `NODE_BACK_REF` | `YP_BACK_REFERENCE_READ_NODE` |
 | `NODE_MATCH` | |
-| `NODE_MATCH2` | `YP_NODE_CALL_NODE` (with regular expression as receiver) |
-| `NODE_MATCH3` | `YP_NODE_CALL_NODE` (with regular expression as only argument) |
+| `NODE_MATCH2` | `YP_CALL_NODE` (with regular expression as receiver) |
+| `NODE_MATCH3` | `YP_CALL_NODE` (with regular expression as only argument) |
 | `NODE_LIT` | |
-| `NODE_STR` | `YP_NODE_STRING_NODE` |
-| `NODE_DSTR` | `YP_NODE_INTERPOLATED_STRING_NODE` |
-| `NODE_XSTR` | `YP_NODE_X_STRING_NODE` |
-| `NODE_DXSTR` | `YP_NODE_INTERPOLATED_X_STRING_NODE` |
-| `NODE_EVSTR` | `YP_NODE_STRING_INTERPOLATED_NODE` |
-| `NODE_DREGX` | `YP_NODE_INTERPOLATED_REGULAR_EXPRESSION_NODE` |
+| `NODE_STR` | `YP_STRING_NODE` |
+| `NODE_DSTR` | `YP_INTERPOLATED_STRING_NODE` |
+| `NODE_XSTR` | `YP_X_STRING_NODE` |
+| `NODE_DXSTR` | `YP_INTERPOLATED_X_STRING_NODE` |
+| `NODE_EVSTR` | `YP_STRING_INTERPOLATED_NODE` |
+| `NODE_DREGX` | `YP_INTERPOLATED_REGULAR_EXPRESSION_NODE` |
 | `NODE_ONCE` | |
-| `NODE_ARGS` | `YP_NODE_PARAMETERS_NODE` |
+| `NODE_ARGS` | `YP_PARAMETERS_NODE` |
 | `NODE_ARGS_AUX` | |
-| `NODE_OPT_ARG` | `YP_NODE_OPTIONAL_PARAMETER_NODE` |
-| `NODE_KW_ARG` | `YP_NODE_KEYWORD_PARAMETER_NODE` |
-| `NODE_POSTARG` | `YP_NODE_REQUIRED_PARAMETER_NODE` |
+| `NODE_OPT_ARG` | `YP_OPTIONAL_PARAMETER_NODE` |
+| `NODE_KW_ARG` | `YP_KEYWORD_PARAMETER_NODE` |
+| `NODE_POSTARG` | `YP_REQUIRED_PARAMETER_NODE` |
 | `NODE_ARGSCAT` | |
 | `NODE_ARGSPUSH` | |
-| `NODE_SPLAT` | `YP_NODE_SPLAT_NODE` |
-| `NODE_BLOCK_PASS` | `YP_NODE_BLOCK_ARGUMENT_NODE` |
-| `NODE_DEFN` | `YP_NODE_DEF_NODE` (with a null receiver) |
-| `NODE_DEFS` | `YP_NODE_DEF_NODE` (with a non-null receiver) |
-| `NODE_ALIAS` | `YP_NODE_ALIAS_NODE` |
-| `NODE_VALIAS` | `YP_NODE_ALIAS_NODE` (with a global variable first argument) |
-| `NODE_UNDEF` | `YP_NODE_UNDEF_NODE` |
-| `NODE_CLASS` | `YP_NODE_CLASS_NODE` |
-| `NODE_MODULE` | `YP_NODE_MODULE_NODE` |
-| `NODE_SCLASS` | `YP_NODE_S_CLASS_NODE` |
-| `NODE_COLON2` | `YP_NODE_CONSTANT_PATH_NODE` |
-| `NODE_COLON3` | `YP_NODE_CONSTANT_PATH_NODE` (with a null receiver) |
-| `NODE_DOT2` | `YP_NODE_RANGE_NODE` (with a .. operator) |
-| `NODE_DOT3` | `YP_NODE_RANGE_NODE` (with a ... operator) |
-| `NODE_FLIP2` | `YP_NODE_RANGE_NODE` (with a .. operator) |
-| `NODE_FLIP3` | `YP_NODE_RANGE_NODE` (with a ... operator) |
-| `NODE_SELF` | `YP_NODE_SELF_NODE` |
-| `NODE_NIL` | `YP_NODE_NIL_NODE` |
-| `NODE_TRUE` | `YP_NODE_TRUE_NODE` |
-| `NODE_FALSE` | `YP_NODE_FALSE_NODE` |
+| `NODE_SPLAT` | `YP_SPLAT_NODE` |
+| `NODE_BLOCK_PASS` | `YP_BLOCK_ARGUMENT_NODE` |
+| `NODE_DEFN` | `YP_DEF_NODE` (with a null receiver) |
+| `NODE_DEFS` | `YP_DEF_NODE` (with a non-null receiver) |
+| `NODE_ALIAS` | `YP_ALIAS_NODE` |
+| `NODE_VALIAS` | `YP_ALIAS_NODE` (with a global variable first argument) |
+| `NODE_UNDEF` | `YP_UNDEF_NODE` |
+| `NODE_CLASS` | `YP_CLASS_NODE` |
+| `NODE_MODULE` | `YP_MODULE_NODE` |
+| `NODE_SCLASS` | `YP_S_CLASS_NODE` |
+| `NODE_COLON2` | `YP_CONSTANT_PATH_NODE` |
+| `NODE_COLON3` | `YP_CONSTANT_PATH_NODE` (with a null receiver) |
+| `NODE_DOT2` | `YP_RANGE_NODE` (with a .. operator) |
+| `NODE_DOT3` | `YP_RANGE_NODE` (with a ... operator) |
+| `NODE_FLIP2` | `YP_RANGE_NODE` (with a .. operator) |
+| `NODE_FLIP3` | `YP_RANGE_NODE` (with a ... operator) |
+| `NODE_SELF` | `YP_SELF_NODE` |
+| `NODE_NIL` | `YP_NIL_NODE` |
+| `NODE_TRUE` | `YP_TRUE_NODE` |
+| `NODE_FALSE` | `YP_FALSE_NODE` |
 | `NODE_ERRINFO` | |
-| `NODE_DEFINED` | `YP_NODE_DEFINED_NODE` |
-| `NODE_POSTEXE` | `YP_NODE_POST_EXECUTION_NODE` |
-| `NODE_DSYM` | `YP_NODE_INTERPOLATED_SYMBOL_NODE` |
-| `NODE_ATTRASGN` | `YP_NODE_CALL_NODE` (with a message that ends with =) |
-| `NODE_LAMBDA` | `YP_NODE_LAMBDA_NODE` |
-| `NODE_ARYPTN` | `YP_NODE_ARRAY_PATTERN_NODE` |
-| `NODE_HSHPTN` | `YP_NODE_HASH_PATTERN_NODE` |
-| `NODE_FNDPTN` | `YP_NODE_FIND_PATTERN_NODE` |
-| `NODE_ERROR` | `YP_NODE_MISSING_NODE` |
+| `NODE_DEFINED` | `YP_DEFINED_NODE` |
+| `NODE_POSTEXE` | `YP_POST_EXECUTION_NODE` |
+| `NODE_DSYM` | `YP_INTERPOLATED_SYMBOL_NODE` |
+| `NODE_ATTRASGN` | `YP_CALL_NODE` (with a message that ends with =) |
+| `NODE_LAMBDA` | `YP_LAMBDA_NODE` |
+| `NODE_ARYPTN` | `YP_ARRAY_PATTERN_NODE` |
+| `NODE_HSHPTN` | `YP_HASH_PATTERN_NODE` |
+| `NODE_FNDPTN` | `YP_FIND_PATTERN_NODE` |
+| `NODE_ERROR` | `YP_MISSING_NODE` |
 | `NODE_LAST` | |
 ```

data/docs/serialization.md

@@ -81,32 +81,35 @@ Each node is structured like the following table:
 | `1` | node type |
 | location | node location |
 
-Each node's child is then appended to the serialized string.
-The child node types can be determined by referencing `config.yml`.
-Depending on the type of child node, it could take a couple of different forms, described below:
-
-* `node` - A child node that is a node itself. This is structured just as like parent node.
-* `node?` - A child node that is optionally present. If the node is not present, then a single `0` byte will be written in its place. If it is present, then it will be structured just as like parent node.
-* `node[]` - A child node that is an array of nodes. This is structured as a variable-length integer length, followed by the child nodes themselves.
-* `string` - A child node that is a string. For example, this is used as the name of the method in a call node, since it cannot directly reference the source string (as in `@-` or `foo=`). This is structured as a variable-length integer byte length, followed by the string itself (_without_ a trailing null byte).
+Every field on the node is then appended to the serialized string. The fields can be determined by referencing `config.yml`. Depending on the type of field, it could take a couple of different forms, described below:
+
+* `node` - A field that is a node. This is structured just as like parent node.
+* `node?` - A field that is a node that is optionally present. If the node is not present, then a single `0` byte will be written in its place. If it is present, then it will be structured just as like parent node.
+* `node[]` - A field that is an array of nodes. This is structured as a variable-length integer length, followed by the child nodes themselves.
+* `string` - A field that is a string. For example, this is used as the name of the method in a call node, since it cannot directly reference the source string (as in `@-` or `foo=`). This is structured as a variable-length integer byte length, followed by the string itself (_without_ a trailing null byte).
 * `constant` - A variable-length integer that represents an index in the constant pool.
-* `constant[]` - A child node that is an array of constants. This is structured as a variable-length integer length, followed by the child constants themselves.
-* `location` - A child node that is a location. This is structured as a variable-length integer start followed by a variable-length integer length.
-* `location?` - A child node that is a location that is optionally present. If the location is not present, then a single `0` byte will be written in its place. If it is present, then it will be structured just like the `location` child node.
-* `location[]` - A child node that is an array of locations. This is structured as a `4` byte length, followed by the locations themselves.
-* `uint32` - A child node that is a 32-bit unsigned integer. This is structured as a variable-length integer.
+* `constant?` - An optional variable-length integer that represents an index in the constant pool. If it's not present, then a single `0` byte will be written in its place.
+* `location` - A field that is a location. This is structured as a variable-length integer start followed by a variable-length integer length.
+* `location?` - A field that is a location that is optionally present. If the location is not present, then a single `0` byte will be written in its place. If it is present, then it will be structured just like the `location` child node.
+* `uint32` - A field that is a 32-bit unsigned integer. This is structured as a variable-length integer.
+
+After the syntax tree, the content pool is serialized. This is a list of constants that were referenced from within the tree. The content pool begins at the offset specified in the header. Constants can be either "owned" (in which case their contents are embedded in the serialization) or "shared" (in which case their contents represent a slice of the source string). The most significant bit of the constant indicates whether it is owned or shared.
+
+In the case that it is owned, the constant is structured as follows:
+
+| # bytes | field |
+| --- | --- |
+| `4` | the byte offset in the serialization for the contents of the constant |
+| `4` | the byte length in the serialization |
 
-After the syntax tree, the content pool is serialized.
-This is a list of constants that were referenced from within the tree.
-The content pool begins at the offset specified in the header.
-Each constant is structured as:
+Note that you will need to mask off the most significant bit for the byte offset in the serialization. In the case that it is shared, the constant is structured as follows:
 
 | # bytes | field |
 | --- | --- |
-| `4` | the byte offset in the source |
-| `4` | the byte length in the source |
+| `4` | the byte offset in the source string for the contents of the constant |
+| `4` | the byte length in the source string |
 
-At the end of the serialization, the buffer is null terminated.
+After the constant pool, the contents of the owned constants are serialized. This is just a sequence of bytes that represent the contents of the constants. At the end of the serialization, the buffer is null terminated.
 
 ## APIs
 
@@ -130,14 +133,14 @@ void yp_buffer_free(yp_buffer_t *);
 
 // Parse and serialize the AST represented by the given source to the given
 // buffer.
-void yp_parse_serialize(const char *, size_t, yp_buffer_t *, const char *);
+void yp_parse_serialize(const uint8_t *source, size_t length, yp_buffer_t *buffer, const char *metadata);
 ```
 
 Typically you would use a stack-allocated `yp_buffer_t` and call `yp_parse_serialize`, as in:
 
 ```c
 void
-serialize(const char *source, size_t length) {
+serialize(const uint8_t *source, size_t length) {
   yp_buffer_t buffer;
   if (!yp_buffer_init(&buffer)) return;