json_scanner 0.2.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f34237d4ceab009f685b82a4e480247f23c804db96bded6d1bacea5ddd4a0946
-  data.tar.gz: 87484e4cbab84666b41ddb67553b0c985ef6dc29d8f1154a458173deade08587
+  metadata.gz: ca9d160b389a5b605a37baeebd8e9d704a9b42712b9c8de9d0667ffa7e6b3d35
+  data.tar.gz: f2c0362e6bb4484e47fd5dbdda4afd7332ce42d266b4d0ebe681f24e18c31b0a
 SHA512:
-  metadata.gz: 5f6ae7f8d7afc88fee60e88eb8efe776b7ae0dffe25ecffa5ffc61241629eb4c3ec86a9ab1b1e76d49b4bd12498ee15625677afbc997039f9295e65e510a32df
-  data.tar.gz: b5e95df2d53c0a224f6a089a55a6a0c502adb7e4dce276f2f85c48a063182f65f1425eff1f97cf409de46c815b45bbfa0626e54d76036a5c66c13eefa4146648
+  metadata.gz: 91d190291d0e2c16db38c523a58fbbff23cde136481b2c4a19446d25169a77390eee4792979cd3a22724bdf2c89132b5dc3ccc78c1553c17fe21227a70e1bf3c
+  data.tar.gz: dc704ac0ba609f209883d5af50f3f9cabcde04a0eb2c3f5ac04f964010bf866fd727c5c175dd40917526b693f13bc4b532f7a4fabce9859c0b44238cf3fa421b

data/README.md

@@ -2,7 +2,7 @@
 
 # JsonScanner
 
-Extract values from JSON without full parsing. This gem uses yajl lib to scan a json string and allows you to parse pieces of it.
+Extract values from JSON without full parsing. This gem uses the `yajl` library to scan a JSON string and allows you to parse pieces of it.
 
 ## Installation
 
@@ -16,6 +16,8 @@ If bundler is not being used to manage dependencies, install the gem by executin
 
 ## Usage
 
+Basic usage
+
 ```ruby
 require "json"
 require "json_scanner"
@@ -32,15 +34,123 @@ emoji_json = '{"grin": "😁", "heart": "😍", "rofl": "🤣"}'
 begin_pos, end_pos, = JsonScanner.scan(emoji_json, [["heart"]], false).first.first
 emoji_json.byteslice(begin_pos...end_pos)
 # => "\"😍\""
-# Note: most likely don't need `quirks_mode` option, unless you are using some old ruby
-# with stdlib version of json gem or its old version. In new versions `quirks_mode` is default
+# Note: You most likely don't need the `quirks_mode` option unless you are using an older version
+# of Ruby with the stdlib - or just also old - version of the json gem. In newer versions, `quirks_mode` is enabled by default.
 JSON.parse(emoji_json.byteslice(begin_pos...end_pos), quirks_mode: true)
 # => "😍"
 # You can also do this
 # emoji_json.force_encoding(Encoding::BINARY)[begin_pos...end_pos].force_encoding(Encoding::UTF_8)
 # => "\"😍\""
+
+# Ranges are supported as matchers for indexes with the following restrictions:
+# - the start of a range must be positive
+# - the end of a range must be positive or -1
+# - a range with -1 end must be closed, e.g. (0..-1) works, but (0...-1) is forbidden
+JsonScanner.scan('[0, 42, 0]', [[(1..-1)]])
+# => [[[4, 6, :number], [8, 9, :number]]]
+JsonScanner.scan('[0, 42, 0]', [[JsonScanner::ANY_INDEX]])
+# => [[[1, 2, :number], [4, 6, :number], [8, 9, :number]]]
+
+# Special matcher JsonScanner::ANY_KEY is supported for object keys
+JsonScanner.scan('{"a": 1, "b": 2}', [[JsonScanner::ANY_KEY]], with_path: true)
+# => [[[["a"], [6, 7, :number]], [["b"], [14, 15, :number]]]]
+# Regex mathers aren't supported yet, but you can simulate it using `with_path` option
+JsonScanner.scan(
+  '{"question1": 1, "answer": 42, "question2": 2}',
+  [[JsonScanner::ANY_KEY]], with_path: true,
+).map do |res|
+  res.map do |path, (begin_pos, end_pos, type)|
+    [begin_pos, end_pos, type] if path[0] =~ /\Aquestion/
+  end.compact
+end
+# => [[[14, 15, :number], [44, 45, :number]]]
+```
+
+## Options
+
+`JsonScanner` supports multiple options
+
+```ruby
+JsonScanner.scan('[0, 42, 0]', [[(1..-1)]], with_path: true)
+# => [[[[1], [4, 6, :number]], [[2], [8, 9, :number]]]]
+JsonScanner.scan('[0, 42],', [[(1..-1)]], verbose_error: true)
+# JsonScanner::ParseError (parse error: trailing garbage)
+#                                 [0, 42],
+#                      (right here) ------^
+# Note: the 'right here' pointer is wrong in case of a premature EOF error, it's a bug of the libyajl
+JsonScanner.scan('[0, 42,', [[(1..-1)]], verbose_error: true)
+# JsonScanner::ParseError (parse error: premature EOF)
+#                                        [0, 42,
+#                      (right here) ------^
+JsonScanner.scan('[0, /* answer */ 42, 0]', [[(1..-1)]], allow_comments: true)
+# => [[[17, 19, :number], [21, 22, :number]]]
+JsonScanner.scan("\"\x81\x83\"", [[]], dont_validate_strings: true)
+# => [[[0, 4, :string]]]
+JsonScanner.scan("{\"\x81\x83\": 42}", [[JsonScanner::ANY_KEY]], dont_validate_strings: true, with_path: true)
+# => [[[["\x81\x83"], [7, 9, :number]]]]
+JsonScanner.scan('[0, 42, 0]garbage', [[(1..-1)]], allow_trailing_garbage: true)
+# => [[[4, 6, :number], [8, 9, :number]]]
+JsonScanner.scan('[0, 42, 0]  [0, 34]', [[(1..-1)]], allow_multiple_values: true)
+# => [[[4, 6, :number], [8, 9, :number], [16, 18, :number]]]
+JsonScanner.scan('[0, 42, 0', [[(1..-1)]], allow_partial_values: true)
+# => [[[4, 6, :number], [8, 9, :number]]]
+JsonScanner.scan('{"a": 1}', [[JsonScanner::ANY_KEY]], with_path: true, symbolize_path_keys: true)
+# => [[[[:a], [6, 7, :number]]]]
 ```
 
+### Comments in the JSON
+
+Note that the standard `JSON` library supports comments, so you may want to enable it in the `JsonScanner` as well
+```ruby
+json_str = '{"answer": {"value": 42 /* the Ultimate Question of Life, the Universe, and Everything */ }}'
+JsonScanner.scan(json_str, [["answer"]], allow_comments: true).first.map do |begin_pos, end_pos, _type|
+  JSON.parse(json_str.byteslice(begin_pos...end_pos), quirks_mode: true)
+end
+# => [{"value"=>42}]
+```
+
+### Find the end of a JSON string
+
+`allow_trailing_garbage` option may come in handy if you want to extract a JSON string from a JS text
+```ruby
+script_text = <<~'JS'
+      <script>window.__APOLLO_STATE__={"ContentItem:0":{"__typename":"ContentItem","id":0, "configurationType":"NO_CONFIGURATION","replacementPartsUrl":null,"relatedCategories":[{"__ref":"Category:109450"},{"__ref":"Category:82044355"},{"__ref":"Category:109441"},{"__ref":"Category:109442"},{"__ref":"Category:109449"},{"__ref":"Category:109444"},{"__ref":"Category:82043730"}],"recommendedOptions":[]}};window.__APPVERSION__=7018;window.__CONFIG_ENV__={value: 'PRODUCTION'};</script>
+JS
+json_with_trailing_garbage = script_text[/__APOLLO_STATE__\s*=\s*({.+)/, 1]
+json_end_pos = JsonScanner.scan(json_with_trailing_garbage, [[]], allow_trailing_garbage: true).first.first[1]
+apollo_state = JSON.parse(json_with_trailing_garbage[0...json_end_pos])
+```
+
+## Reuse configuration
+
+You can create a `JsonScanner::Config` instance and reuse it between `JsonScanner.scan` calls
+
+```ruby
+require "json_scanner"
+
+config = JsonScanner::Config.new([[], ["key"], [(0..-1)]])
+# => #<JsonScanner::Config [[], ['key'], [(0..9223372036854775807)]]>
+JsonScanner.scan('{"key": "42"}', config)
+# => [[[0, 13, :object]], [[8, 12, :string]], []]
+JsonScanner.scan('{"key": "42"}', config, with_path: true)
+# => [[[[], [0, 13, :object]]], [[["key"], [8, 12, :string]]], []]
+JsonScanner.scan('[0, 42]', config)
+# => [[[0, 7, :array]], [], [[1, 2, :number], [4, 6, :number]]]
+JsonScanner.scan('[0, 42]', config, with_path: true)
+# => [[[[], [0, 7, :array]]], [], [[[0], [1, 2, :number]], [[1], [4, 6, :number]]]]
+```
+
+Options can be passed as a hash, even on Ruby 3
+```ruby
+options = { allow_trailing_garbage: true, allow_partial_values: true }
+JsonScanner.scan('[0, 42', [[1]], options) == JsonScanner.scan('[0, 42]_', [[1]], options)
+# => true
+```
+
+## Streaming mode
+
+Streaming mode isn't supported yet, as it's harder to implement and to use. I plan to add it in the future, its API is a subject to discussion. If you have suggestions, use cases, or preferences for how it should behave, I’d love to hear from you!
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/ext/json_scanner/json_scanner.c

@@ -1,8 +1,12 @@
 #include "json_scanner.h"
 
 VALUE rb_mJsonScanner;
+VALUE rb_cJsonScannerConfig;
 VALUE rb_eJsonScannerParseError;
-ID scan_kwargs_table[7];
+#define BYTES_CONSUMED "bytes_consumed"
+ID rb_iv_bytes_consumed;
+#define SCAN_KWARGS_SIZE 8
+ID scan_kwargs_table[SCAN_KWARGS_SIZE];
 
 VALUE null_sym;
 VALUE boolean_sym;
@@ -11,11 +15,13 @@ VALUE string_sym;
 VALUE object_sym;
 VALUE array_sym;
 
+VALUE any_key_sym;
+
 enum matcher_type
 {
   MATCHER_KEY,
   MATCHER_INDEX,
-  // MATCHER_ANY_KEY,
+  MATCHER_ANY_KEY,
   MATCHER_INDEX_RANGE,
   // MATCHER_KEYS_LIST,
   // MATCHER_KEY_REGEX,
@@ -70,30 +76,115 @@ typedef struct
 typedef struct
 {
   int with_path;
-  paths_t *paths;
+  int symbolize_path_keys;
   int paths_len;
-  path_elem_t *current_path;
+  paths_t *paths;
   int current_path_len;
   int max_path_len;
+  path_elem_t *current_path;
   // Easier to use a Ruby array for result than convert later
+  // must be supplied by the caller and RB_GC_GUARD-ed if it isn't on the stack
   VALUE points_list;
   // by depth
   size_t *starts;
   // VALUE rb_err;
   yajl_handle handle;
+  size_t yajl_bytes_consumed;
 } scan_ctx;
 
+inline size_t scan_ctx_get_bytes_consumed(scan_ctx *ctx)
+{
+  return ctx->yajl_bytes_consumed + yajl_get_bytes_consumed(ctx->handle);
+}
+
+inline void scan_ctx_save_bytes_consumed(scan_ctx *ctx)
+{
+  ctx->yajl_bytes_consumed += yajl_get_bytes_consumed(ctx->handle);
+}
+
+void scan_ctx_debug(scan_ctx *ctx)
+{
+  // actually might have been cleared by GC already, be careful, debug only when in valid state
+  VALUE points_list_inspect = ctx->points_list == Qundef ? rb_str_new_cstr("undef") : rb_sprintf("%" PRIsVALUE, rb_inspect(ctx->points_list));
+  fprintf(stderr, "\nscan_ctx {\n");
+  fprintf(stderr, "  with_path: %s,\n", ctx->with_path ? "true" : "false");
+  fprintf(stderr, "  symbolize_path_keys: %s,\n", ctx->symbolize_path_keys ? "true" : "false");
+  fprintf(stderr, "  paths_len: %d,\n", ctx->paths_len);
+
+  fprintf(stderr, "  paths: [\n");
+  for (int i = 0; ctx->paths && i < ctx->paths_len; i++)
+  {
+    fprintf(stderr, "    [");
+    for (int j = 0; j < ctx->paths[i].len; j++)
+    {
+      switch (ctx->paths[i].elems[j].type)
+      {
+      case MATCHER_KEY:
+        fprintf(stderr, "'%.*s'", (int)ctx->paths[i].elems[j].value.key.len, ctx->paths[i].elems[j].value.key.val);
+        break;
+      case MATCHER_INDEX:
+        fprintf(stderr, "%ld", ctx->paths[i].elems[j].value.index);
+        break;
+      case MATCHER_INDEX_RANGE:
+        fprintf(stderr, "(%ld..%ld)", ctx->paths[i].elems[j].value.range.start, ctx->paths[i].elems[j].value.range.end);
+        break;
+      case MATCHER_ANY_KEY:
+        fprintf(stderr, "('*'..'*')");
+        break;
+      }
+      if (j < ctx->paths[i].len - 1)
+        fprintf(stderr, ", ");
+    }
+    fprintf(stderr, "],\n");
+  }
+  fprintf(stderr, "  ],\n");
+
+  fprintf(stderr, "  current_path_len: %d,\n", ctx->current_path_len);
+  fprintf(stderr, "  max_path_len: %d,\n", ctx->max_path_len);
+  fprintf(stderr, "  current_path: [");
+  for (int i = 0; i < ctx->current_path_len; i++)
+  {
+    switch (ctx->current_path[i].type)
+    {
+    case PATH_KEY:
+      fprintf(stderr, "'%.*s'", (int)ctx->current_path[i].value.key.len, ctx->current_path[i].value.key.val);
+      break;
+    case PATH_INDEX:
+      fprintf(stderr, "%ld", ctx->current_path[i].value.index);
+      break;
+    }
+    if (i < ctx->current_path_len - 1)
+      fprintf(stderr, ", ");
+  }
+  fprintf(stderr, "],\n");
+
+  fprintf(stderr, "  points_list: %.*s,\n", RSTRING_LENINT(points_list_inspect), RSTRING_PTR(points_list_inspect));
+  fprintf(stderr, "  starts: [");
+  for (int i = 0; i <= ctx->max_path_len; i++)
+  {
+    fprintf(stderr, "%ld", ctx->starts[i]);
+    if (i < ctx->max_path_len)
+      fprintf(stderr, ", ");
+  }
+  fprintf(stderr, "],\n");
+
+  fprintf(stderr, "  handle: %p,\n", ctx->handle);
+  fprintf(stderr, "  yajl_bytes_consumed: %ld,\n", ctx->yajl_bytes_consumed);
+  fprintf(stderr, "}\n\n\n");
+}
+
 // FIXME: This will cause memory leak if ruby_xmalloc raises
-scan_ctx *scan_ctx_init(VALUE path_ary, int with_path)
+// path_ary must be RB_GC_GUARD-ed by the caller
+VALUE scan_ctx_init(scan_ctx *ctx, VALUE path_ary, VALUE string_keys)
 {
   int path_ary_len;
-  scan_ctx *ctx;
   paths_t *paths;
   // TODO: Allow to_ary and sized enumerables
   rb_check_type(path_ary, T_ARRAY);
   path_ary_len = rb_long2int(rb_array_len(path_ary));
   // Check types early before any allocations, so exception is ok
   // TODO: Fix this, just handle errors
+  // It's not possible that another Ruby thread changes path_ary items between these two loops, because C call holds GVL
   for (int i = 0; i < path_ary_len; i++)
   {
     int path_len;
@@ -103,38 +194,42 @@ scan_ctx *scan_ctx_init(VALUE path_ary, int with_path)
     for (int j = 0; j < path_len; j++)
     {
       VALUE entry = rb_ary_entry(path, j);
-      int type = TYPE(entry);
-      if (type == T_STRING)
+      switch (TYPE(entry))
       {
+      case T_SYMBOL:
+        entry = rb_sym2str(entry);
+        /* fall through */
+      case T_STRING:
 #if LONG_MAX > SIZE_MAX
         RSTRING_LENINT(entry);
 #endif
-      }
-      else if (type == T_FIXNUM || type == T_BIGNUM)
-      {
-        RB_NUM2LONG(entry);
-      }
-      else
+        break;
+      case T_FIXNUM:
+      case T_BIGNUM:
+        NUM2LONG(entry);
+        break;
+      default:
       {
         VALUE range_beg, range_end;
         long end_val;
         int open_ended;
         if (rb_range_values(entry, &range_beg, &range_end, &open_ended) != Qtrue)
-          rb_raise(rb_eArgError, "path elements must be strings, integers, or ranges");
-        if (RB_NUM2LONG(range_beg) < 0L)
-          rb_raise(rb_eArgError, "range start must be positive");
-        end_val = RB_NUM2LONG(range_end);
-        if (end_val < -1L)
-          rb_raise(rb_eArgError, "range end must be positive or -1");
-        if (end_val == -1L && open_ended)
-          rb_raise(rb_eArgError, "range with -1 end must be closed");
+          return rb_exc_new_cstr(rb_eArgError, "path elements must be strings, integers, or ranges");
+        if (range_beg != any_key_sym || range_end != any_key_sym)
+        {
+          if (NUM2LONG(range_beg) < 0L)
+            return rb_exc_new_cstr(rb_eArgError, "range start must be positive");
+          end_val = NUM2LONG(range_end);
+          if (end_val < -1L)
+            return rb_exc_new_cstr(rb_eArgError, "range end must be positive or -1");
+          if (end_val == -1L && open_ended)
+            return rb_exc_new_cstr(rb_eArgError, "range with -1 end must be closed");
+        }
+      }
       }
     }
   }
 
-  ctx = ruby_xmalloc(sizeof(scan_ctx));
-
-  ctx->with_path = with_path;
   ctx->max_path_len = 0;
 
   paths = ruby_xmalloc(sizeof(paths_t) * path_ary_len);
@@ -149,9 +244,21 @@ scan_ctx *scan_ctx_init(VALUE path_ary, int with_path)
     for (int j = 0; j < path_len; j++)
     {
       VALUE entry = rb_ary_entry(path, j);
-      int type = TYPE(entry);
-      if (type == T_STRING)
+      switch (TYPE(entry))
       {
+      case T_SYMBOL:
+        entry = rb_sym2str(entry);
+        /* fall through */
+      case T_STRING:
+      {
+        if (string_keys != Qundef)
+        {
+          // If string_keys is provided, we need to duplicate the string
+          // to avoid use-after-free issues and to add the newly created string to the string_keys array.
+          // In Ruby 2.2 and newer symbols can be GC-ed, so we need to duplicate them as well.
+          entry = rb_str_dup(entry);
+          rb_ary_push(string_keys, entry);
+        }
         paths[i].elems[j].type = MATCHER_KEY;
         paths[i].elems[j].value.key.val = RSTRING_PTR(entry);
 #if LONG_MAX > SIZE_MAX
@@ -160,25 +267,36 @@ scan_ctx *scan_ctx_init(VALUE path_ary, int with_path)
         paths[i].elems[j].value.key.len = RSTRING_LEN(entry);
 #endif
       }
-      else if (type == T_FIXNUM || type == T_BIGNUM)
+      break;
+      case T_FIXNUM:
+      case T_BIGNUM:
       {
         paths[i].elems[j].type = MATCHER_INDEX;
         paths[i].elems[j].value.index = FIX2LONG(entry);
       }
-      else
+      break;
+      default:
       {
         VALUE range_beg, range_end;
         int open_ended;
-        paths[i].elems[j].type = MATCHER_INDEX_RANGE;
         rb_range_values(entry, &range_beg, &range_end, &open_ended);
-        paths[i].elems[j].value.range.start = RB_NUM2LONG(range_beg);
-        paths[i].elems[j].value.range.end = RB_NUM2LONG(range_end);
-        // (value..-1) works as expected, (value...-1) is forbidden above
-        if (paths[i].elems[j].value.range.end == -1L)
-          paths[i].elems[j].value.range.end = LONG_MAX;
-        // -1 here is fine, so, (0...0) works just as expected - doesn't match anything
-        if (open_ended)
-          paths[i].elems[j].value.range.end--;
+        if (range_beg == any_key_sym && range_end == any_key_sym)
+        {
+          paths[i].elems[j].type = MATCHER_ANY_KEY;
+        }
+        else
+        {
+          paths[i].elems[j].type = MATCHER_INDEX_RANGE;
+          paths[i].elems[j].value.range.start = NUM2LONG(range_beg);
+          paths[i].elems[j].value.range.end = NUM2LONG(range_end);
+          // (value..-1) works as expected, (value...-1) is forbidden above
+          if (paths[i].elems[j].value.range.end == -1L)
+            paths[i].elems[j].value.range.end = LONG_MAX;
+          // -1 here is fine, so, (0...0) works just as expected - doesn't match anything
+          if (open_ended)
+            paths[i].elems[j].value.range.end--;
+        }
+      }
       }
     }
     paths[i].len = path_len;
@@ -189,32 +307,37 @@ scan_ctx *scan_ctx_init(VALUE path_ary, int with_path)
   ctx->paths_len = path_ary_len;
   ctx->current_path = ruby_xmalloc2(sizeof(path_elem_t), ctx->max_path_len);
 
-  ctx->current_path_len = 0;
-  ctx->points_list = rb_ary_new_capa(path_ary_len);
-  for (int i = 0; i < path_ary_len; i++)
-  {
-    rb_ary_push(ctx->points_list, rb_ary_new());
-  }
-
   ctx->starts = ruby_xmalloc2(sizeof(size_t), ctx->max_path_len + 1);
+  return Qundef; // no error
+}
+
+// resets temporary values in the config
+void scan_ctx_reset(scan_ctx *ctx, VALUE points_list, int with_path, int symbolize_path_keys)
+{
+  // TODO: reset matched_depth if implemented
+  ctx->current_path_len = 0;
   // ctx->rb_err = Qnil;
   ctx->handle = NULL;
-
-  return ctx;
+  ctx->yajl_bytes_consumed = 0;
+  ctx->points_list = points_list;
+  ctx->with_path = with_path;
+  ctx->symbolize_path_keys = symbolize_path_keys;
 }
 
 void scan_ctx_free(scan_ctx *ctx)
 {
+  // fprintf(stderr, "scan_ctx_free\n");
   if (!ctx)
     return;
   ruby_xfree(ctx->starts);
   ruby_xfree(ctx->current_path);
+  if (!ctx->paths)
+    return;
   for (int i = 0; i < ctx->paths_len; i++)
   {
     ruby_xfree(ctx->paths[i].elems);
   }
   ruby_xfree(ctx->paths);
-  ruby_xfree(ctx);
 }
 
 // noexcept
@@ -239,37 +362,38 @@ typedef enum
 } value_type;
 
 // noexcept
-VALUE create_point(scan_ctx *sctx, value_type type, size_t length, size_t curr_pos)
+VALUE create_point(scan_ctx *sctx, value_type type, size_t length)
 {
-  VALUE values[3];
-  VALUE point = rb_ary_new_capa(3);
+  VALUE values[3], point;
+  size_t curr_pos = scan_ctx_get_bytes_consumed(sctx);
+  point = rb_ary_new_capa(3);
   // noexcept
-  values[1] = RB_ULONG2NUM(curr_pos);
+  values[1] = ULL2NUM(curr_pos);
   switch (type)
   {
     // FIXME: size_t can be longer than ulong
   case null_value:
-    values[0] = RB_ULONG2NUM(curr_pos - length);
+    values[0] = ULL2NUM(curr_pos - length);
     values[2] = null_sym;
     break;
   case boolean_value:
-    values[0] = RB_ULONG2NUM(curr_pos - length);
+    values[0] = ULL2NUM(curr_pos - length);
     values[2] = boolean_sym;
     break;
   case number_value:
-    values[0] = RB_ULONG2NUM(curr_pos - length);
+    values[0] = ULL2NUM(curr_pos - length);
     values[2] = number_sym;
     break;
   case string_value:
-    values[0] = RB_ULONG2NUM(curr_pos - length);
+    values[0] = ULL2NUM(curr_pos - length);
     values[2] = string_sym;
     break;
   case object_value:
-    values[0] = RB_ULONG2NUM(sctx->starts[sctx->current_path_len]);
+    values[0] = ULL2NUM(sctx->starts[sctx->current_path_len]);
     values[2] = object_sym;
     break;
   case array_value:
-    values[0] = RB_ULONG2NUM(sctx->starts[sctx->current_path_len]);
+    values[0] = ULL2NUM(sctx->starts[sctx->current_path_len]);
     values[2] = array_sym;
     break;
   }
@@ -288,10 +412,13 @@ VALUE create_path(scan_ctx *sctx)
     switch (sctx->current_path[i].type)
     {
     case PATH_KEY:
-      entry = rb_str_new(sctx->current_path[i].value.key.val, sctx->current_path[i].value.key.len);
+      if (sctx->symbolize_path_keys)
+        entry = rb_id2sym(rb_intern2(sctx->current_path[i].value.key.val, sctx->current_path[i].value.key.len));
+      else
+        entry = rb_str_new(sctx->current_path[i].value.key.val, sctx->current_path[i].value.key.len);
       break;
     case PATH_INDEX:
-      entry = RB_ULONG2NUM(sctx->current_path[i].value.index);
+      entry = LONG2NUM(sctx->current_path[i].value.index);
       break;
     default:
       entry = Qnil;
@@ -307,7 +434,7 @@ void save_point(scan_ctx *sctx, value_type type, size_t length)
   // TODO: Abort parsing if all paths are matched and no more mathces are possible: only trivial key/index matchers at the current level
   // TODO: Don't re-compare already matched prefixes; hard to invalidate, though
   // TODO: Might fail in case of no memory
-  VALUE point = Qundef;
+  VALUE point = Qundef, path;
   int match;
   for (int i = 0; i < sctx->paths_len; i++)
   {
@@ -319,6 +446,10 @@ void save_point(scan_ctx *sctx, value_type type, size_t length)
     {
       switch (sctx->paths[i].elems[j].type)
       {
+      case MATCHER_ANY_KEY:
+        if (sctx->current_path[j].type != PATH_KEY)
+          match = false;
+        break;
       case MATCHER_KEY:
         if (sctx->current_path[j].type != PATH_KEY ||
             sctx->current_path[j].value.key.len != sctx->paths[i].elems[j].value.key.len ||
@@ -344,10 +475,11 @@ void save_point(scan_ctx *sctx, value_type type, size_t length)
     {
       if (point == Qundef)
       {
-        point = create_point(sctx, type, length, yajl_get_bytes_consumed(sctx->handle));
+        point = create_point(sctx, type, length);
         if (sctx->with_path)
         {
-          point = rb_ary_new_from_args(2, create_path(sctx), point);
+          path = create_path(sctx);
+          point = rb_ary_new_from_args(2, path, point);
         }
       }
       // rb_ary_push raises only in case of a frozen array, which is not the case
@@ -411,7 +543,7 @@ int scan_on_start_object(void *ctx)
     return true;
   }
   increment_arr_index(sctx);
-  sctx->starts[sctx->current_path_len] = yajl_get_bytes_consumed(sctx->handle) - 1;
+  sctx->starts[sctx->current_path_len] = scan_ctx_get_bytes_consumed(sctx) - 1;
   if (sctx->current_path_len < sctx->max_path_len)
     sctx->current_path[sctx->current_path_len].type = PATH_KEY;
   sctx->current_path_len++;
@@ -451,7 +583,7 @@ int scan_on_start_array(void *ctx)
     return true;
   }
   increment_arr_index(sctx);
-  sctx->starts[sctx->current_path_len] = yajl_get_bytes_consumed(sctx->handle) - 1;
+  sctx->starts[sctx->current_path_len] = scan_ctx_get_bytes_consumed(sctx) - 1;
   if (sctx->current_path_len < sctx->max_path_len)
   {
     sctx->current_path[sctx->current_path_len].type = PATH_INDEX;
@@ -471,6 +603,107 @@ int scan_on_end_array(void *ctx)
   return true;
 }
 
+void config_free(void *data)
+{
+  scan_ctx_free((scan_ctx *)data);
+  ruby_xfree(data);
+}
+
+size_t config_size(const void *data)
+{
+  // see ObjectSpace.memsize_of
+  scan_ctx *ctx = (scan_ctx *)data;
+  size_t res = sizeof(scan_ctx);
+  // current_path
+  if (ctx->current_path != NULL)
+    res += ctx->max_path_len * sizeof(path_elem_t);
+  // starts
+  if (ctx->starts != NULL)
+    res += ctx->max_path_len * sizeof(size_t);
+  if (ctx->paths != NULL)
+  {
+    res += ctx->paths_len * sizeof(paths_t);
+    for (int i = 0; i < ctx->paths_len; i++)
+    {
+      res += ctx->paths[i].len * sizeof(path_matcher_elem_t);
+    }
+  }
+  return res;
+}
+
+static const rb_data_type_t config_type = {
+    .wrap_struct_name = "json_scanner_config",
+    .function = {
+        .dfree = config_free,
+        .dsize = config_size,
+    },
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY,
+};
+
+VALUE config_alloc(VALUE self)
+{
+  scan_ctx *ctx = ruby_xmalloc(sizeof(scan_ctx));
+  ctx->paths = NULL;
+  ctx->paths_len = 0;
+  ctx->current_path = NULL;
+  ctx->max_path_len = 0;
+  ctx->starts = NULL;
+  scan_ctx_reset(ctx, Qundef, false, false);
+  return TypedData_Wrap_Struct(self, &config_type, ctx);
+}
+
+VALUE config_m_initialize(VALUE self, VALUE path_ary)
+{
+  scan_ctx *ctx;
+  VALUE scan_ctx_init_err, string_keys;
+  TypedData_Get_Struct(self, scan_ctx, &config_type, ctx);
+  string_keys = rb_ary_new();
+  scan_ctx_init_err = scan_ctx_init(ctx, path_ary, string_keys);
+  if (scan_ctx_init_err != Qundef)
+  {
+    rb_exc_raise(scan_ctx_init_err);
+  }
+  rb_iv_set(self, "string_keys", string_keys);
+  return self;
+}
+
+VALUE config_m_inspect(VALUE self)
+{
+  scan_ctx *ctx;
+  VALUE res;
+  TypedData_Get_Struct(self, scan_ctx, &config_type, ctx);
+  res = rb_sprintf("#<%" PRIsVALUE " [", rb_class_name(CLASS_OF(self)));
+  for (int i = 0; ctx->paths && i < ctx->paths_len; i++)
+  {
+    rb_str_cat_cstr(res, "[");
+    for (int j = 0; j < ctx->paths[i].len; j++)
+    {
+      switch (ctx->paths[i].elems[j].type)
+      {
+      case MATCHER_KEY:
+        rb_str_catf(res, "'%.*s'", (int)ctx->paths[i].elems[j].value.key.len, ctx->paths[i].elems[j].value.key.val);
+        break;
+      case MATCHER_INDEX:
+        rb_str_catf(res, "%ld", ctx->paths[i].elems[j].value.index);
+        break;
+      case MATCHER_INDEX_RANGE:
+        rb_str_catf(res, "(%ld..%ld)", ctx->paths[i].elems[j].value.range.start, ctx->paths[i].elems[j].value.range.end);
+        break;
+      case MATCHER_ANY_KEY:
+        rb_str_cat_cstr(res, "('*'..'*')");
+        break;
+      }
+      if (j < ctx->paths[i].len - 1)
+        rb_str_cat_cstr(res, ", ");
+    }
+    rb_str_cat_cstr(res, "]");
+    if (i < ctx->paths_len - 1)
+      rb_str_cat_cstr(res, ", ");
+  }
+  rb_str_cat_cstr(res, "]>");
+  return res;
+}
+
 static yajl_callbacks scan_callbacks = {
     scan_on_null,
     scan_on_boolean,
@@ -492,15 +725,16 @@ static yajl_callbacks scan_callbacks = {
 VALUE scan(int argc, VALUE *argv, VALUE self)
 {
   VALUE json_str, path_ary, with_path_flag, kwargs;
-  VALUE kwargs_values[7];
+  VALUE kwargs_values[SCAN_KWARGS_SIZE];
 
-  int with_path = false, verbose_error = false;
+  int with_path = false, verbose_error = false, symbolize_path_keys = false;
   char *json_text;
   size_t json_text_len;
   yajl_handle handle;
   yajl_status stat;
   scan_ctx *ctx;
-  VALUE err = Qnil, result;
+  int free_ctx = true;
+  VALUE err_msg = Qnil, bytes_consumed, result;
   // Turned out callbacks can't raise exceptions
   // VALUE callback_err;
 #if RUBY_API_VERSION_MAJOR > 2 || (RUBY_API_VERSION_MAJOR == 2 && RUBY_API_VERSION_MINOR >= 7)
@@ -512,11 +746,13 @@ VALUE scan(int argc, VALUE *argv, VALUE self)
   with_path = RTEST(with_path_flag);
   if (kwargs != Qnil)
   {
-    rb_get_kwargs(kwargs, scan_kwargs_table, 0, 7, kwargs_values);
+    rb_get_kwargs(kwargs, scan_kwargs_table, 0, SCAN_KWARGS_SIZE, kwargs_values);
     if (kwargs_values[0] != Qundef)
       with_path = RTEST(kwargs_values[0]);
     if (kwargs_values[1] != Qundef)
       verbose_error = RTEST(kwargs_values[1]);
+    if (kwargs_values[7] != Qundef)
+      symbolize_path_keys = RTEST(kwargs_values[7]);
   }
   rb_check_type(json_str, T_STRING);
   json_text = RSTRING_PTR(json_str);
@@ -525,7 +761,30 @@ VALUE scan(int argc, VALUE *argv, VALUE self)
 #else
   json_text_len = RSTRING_LEN(json_str);
 #endif
-  ctx = scan_ctx_init(path_ary, with_path);
+  if (rb_obj_is_kind_of(path_ary, rb_cJsonScannerConfig))
+  {
+    free_ctx = false;
+    TypedData_Get_Struct(path_ary, scan_ctx, &config_type, ctx);
+  }
+  else
+  {
+    VALUE scan_ctx_init_err;
+    ctx = ruby_xmalloc(sizeof(scan_ctx));
+    scan_ctx_init_err = scan_ctx_init(ctx, path_ary, Qundef);
+    if (scan_ctx_init_err != Qundef)
+    {
+      ruby_xfree(ctx);
+      rb_exc_raise(scan_ctx_init_err);
+    }
+  }
+  // Need to keep a ref to result array on the stack to prevent it from being GC-ed
+  result = rb_ary_new_capa(ctx->paths_len);
+  for (int i = 0; i < ctx->paths_len; i++)
+  {
+    rb_ary_push(result, rb_ary_new());
+  }
+  scan_ctx_reset(ctx, result, with_path, symbolize_path_keys);
+  // scan_ctx_debug(ctx);
 
   handle = yajl_alloc(&scan_callbacks, NULL, (void *)ctx);
   if (kwargs != Qnil) // it's safe to read kwargs_values only if rb_get_kwargs was called
@@ -544,23 +803,52 @@ VALUE scan(int argc, VALUE *argv, VALUE self)
   ctx->handle = handle;
   stat = yajl_parse(handle, (unsigned char *)json_text, json_text_len);
   if (stat == yajl_status_ok)
+  {
+    scan_ctx_save_bytes_consumed(ctx);
     stat = yajl_complete_parse(handle);
+  }
 
   if (stat != yajl_status_ok)
   {
     char *str = (char *)yajl_get_error(handle, verbose_error, (unsigned char *)json_text, json_text_len);
-    err = rb_utf8_str_new_cstr(str);
+    err_msg = rb_utf8_str_new_cstr(str);
+    bytes_consumed = ULL2NUM(scan_ctx_get_bytes_consumed(ctx));
     yajl_free_error(handle, (unsigned char *)str);
   }
+  // // Needed when yajl_allow_partial_values is set
+  // if (ctx->current_path_len > 0)
+  // {
+  //   if (ctx->current_path_len > ctx->max_path_len)
+  //     ctx->current_path_len = ctx->max_path_len;
+  //   for (int i = ctx->current_path_len - 1; i > 0; i--)
+  //   {
+  //     switch (ctx->current_path[i].type)
+  //     {
+  //     case PATH_KEY:
+  //       scan_on_end_object(ctx);
+  //       break;
+  //     case PATH_INDEX:
+  //       scan_on_end_array(ctx);
+  //       break;
+  //     }
+  //   }
+  // }
   // callback_err = ctx->rb_err;
-  result = ctx->points_list;
-  scan_ctx_free(ctx);
+  if (free_ctx)
+  {
+    // fprintf(stderr, "free_ctx\n");
+    scan_ctx_free(ctx);
+    ruby_xfree(ctx);
+  }
   yajl_free(handle);
-  if (err != Qnil)
-    rb_exc_raise(rb_exc_new_str(rb_eJsonScannerParseError, err));
+  if (err_msg != Qnil)
+  {
+    VALUE err = rb_exc_new_str(rb_eJsonScannerParseError, err_msg);
+    rb_ivar_set(err, rb_iv_bytes_consumed, bytes_consumed);
+    rb_exc_raise(err);
+  }
   // if (callback_err != Qnil)
   //   rb_exc_raise(callback_err);
-  // TODO: report yajl_get_bytes_consumed(handle)
   return result;
 }
 
@@ -568,8 +856,16 @@ RUBY_FUNC_EXPORTED void
 Init_json_scanner(void)
 {
   rb_mJsonScanner = rb_define_module("JsonScanner");
+  rb_cJsonScannerConfig = rb_define_class_under(rb_mJsonScanner, "Config", rb_cObject);
+  rb_define_alloc_func(rb_cJsonScannerConfig, config_alloc);
+  rb_define_method(rb_cJsonScannerConfig, "initialize", config_m_initialize, 1);
+  rb_define_method(rb_cJsonScannerConfig, "inspect", config_m_inspect, 0);
   rb_define_const(rb_mJsonScanner, "ANY_INDEX", rb_range_new(INT2FIX(0), INT2FIX(-1), false));
+  any_key_sym = rb_id2sym(rb_intern("*"));
+  rb_define_const(rb_mJsonScanner, "ANY_KEY", rb_range_new(any_key_sym, any_key_sym, false));
   rb_eJsonScannerParseError = rb_define_class_under(rb_mJsonScanner, "ParseError", rb_eRuntimeError);
+  rb_define_attr(rb_eJsonScannerParseError, BYTES_CONSUMED, true, false);
+  rb_iv_bytes_consumed = rb_intern("@" BYTES_CONSUMED);
   rb_define_module_function(rb_mJsonScanner, "scan", scan, -1);
   null_sym = rb_id2sym(rb_intern("null"));
   boolean_sym = rb_id2sym(rb_intern("boolean"));
@@ -584,4 +880,5 @@ Init_json_scanner(void)
   scan_kwargs_table[4] = rb_intern("allow_trailing_garbage");
   scan_kwargs_table[5] = rb_intern("allow_multiple_values");
   scan_kwargs_table[6] = rb_intern("allow_partial_values");
+  scan_kwargs_table[7] = rb_intern("symbolize_path_keys");
 }

data/lib/json_scanner/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JsonScanner
-  VERSION = "0.2.0"
+  VERSION = "0.3.1"
 end

data/spec/extensiontesttask.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require "rake/clean"
+require "rake/extensiontask"
+
+module Rake
+  class ExtensionTestTask < ExtensionTask
+    #
+    # The C files to compile.
+    #
+    attr_accessor :c_spec_files
+
+    #
+    # The folders where includes for the test files are.
+    #
+    # Default: %w{/usr/include /usr/include/google}
+    #
+    attr_accessor :test_includes
+
+    #
+    # The libraries to link against.
+    #
+    # Default: %w{cmockery}
+    #
+    attr_accessor :test_libraries
+
+    #
+    # The folders where the libraries are
+    #
+    # Default: %w{/usr/lib}
+    #
+    attr_accessor :test_lib_folders
+
+    def initialize(*args, &block)
+      super
+      @c_spec_files = []
+      @test_includes = %w[/usr/include /usr/include/google]
+      @test_libraries = %w[cmockery]
+      @test_lib_folders = %w[/usr/lib]
+      init_test_tasks(
+        "#{@tmp_dir}/test", "compile:#{@name}:test",
+        "spec:c:#{@name}", "spec:valgrind:#{@name}", "spec:gdb:#{@name}",
+      )
+    end
+
+    private
+
+    def includes
+      @includes ||= (@test_includes + [
+        ".",
+        "../../#{@ext_dir}",
+        "/usr/include/ruby-#{RUBY_VERSION}",
+        "/usr/include/ruby-#{RUBY_VERSION}/#{RUBY_PLATFORM}",
+      ]).map { |l| "-I#{l}" }.join(" ")
+    end
+
+    def libraries
+      @libraries ||= (@test_libraries + %w[ruby pthread crypto]).map { |l| "-l#{l}" }.join(" ")
+    end
+
+    def lib_folders
+      @lib_folders ||= (@test_lib_folders + %w[/usr/lib .]).map { |l| "-L#{l}" }.join(" ")
+    end
+
+    def compile_tests
+      # compile the test sources
+      FileList["*.c"].each do |cfile|
+        sh "gcc -g #{includes} -c #{cfile}"
+      end
+
+      source_objects = FileList["../#{RUBY_PLATFORM}/#{@name}/#{RUBY_VERSION}/*.o"]
+      # link the executables
+      FileList["*.o"].each do |ofile|
+        sh "gcc -g #{lib_folders} #{libraries} #{source_objects} #{ofile} -o #{ofile.ext}"
+      end
+    end
+
+    def init_compile_task(compile_dir, compile_task)
+      directory compile_dir
+      desc "Compile #{@name} tests"
+      task compile_task => ["compile:#{@name}", compile_dir] do
+        # copy the test files into the compilation folder
+        @c_spec_files.each { |file| cp file, compile_dir }
+
+        # start compilation
+        chdir(compile_dir) { compile_tests }
+      end
+    end
+
+    def init_valgrind_task(compile_dir, compile_task, valgrind_task)
+      desc "Execute valgrind for a #{@name} test"
+      task valgrind_task => [compile_task] do |_t, args|
+        sh "valgrind --num-callers=50 --error-limit=no --partial-loads-ok=yes --undef-value-errors=no " \
+           "--leak-check=full #{compile_dir}/#{args.test}"
+      end
+    end
+
+    def init_gdb_task(compile_dir, compile_task, gdb_task)
+      desc "Execute gdb for a #{@name} test"
+      task gdb_task => [compile_task] do |_t, args|
+        sh "gdb #{compile_dir}/#{args.test}"
+      end
+    end
+
+    def init_test_task(compile_dir, compile_task, test_task)
+      desc "Test #{@name}"
+      task test_task => [compile_task] do |_t, args|
+        if args.test
+          sh "#{compile_dir}/#{args.test}"
+        else
+          FileList["#{compile_dir}/*.o"].each do |ofile|
+            sh ofile.ext.to_s
+          end
+        end
+      end
+    end
+
+    def init_test_tasks(compile_dir, compile_task, test_task, valgrind_task, gdb_task)
+      init_compile_task(compile_dir, compile_task)
+      init_valgrind_task(compile_dir, compile_task, valgrind_task)
+      init_gdb_task(compile_dir, compile_task, gdb_task)
+      init_test_task(compile_dir, compile_task, test_task)
+
+      desc "Test all C extensions"
+      task "spec:c" => [test_task]
+    end
+  end
+end

data/spec/json_scanner_spec.rb

@@ -22,6 +22,34 @@ RSpec.describe JsonScanner do
     )
   end
 
+  it "supports 'symbolize_path_keys'" do
+    expect(
+      described_class.scan('{"a": {"b": 1}}', [[:a, "b"]], with_path: true),
+    ).to eq([[[%w[a b], [12, 13, :number]]]])
+    expect(
+      described_class.scan('{"a": {"b": 1}}', [[:a, "b"]], with_path: true, symbolize_path_keys: true),
+    ).to eq([[[%i[a b], [12, 13, :number]]]])
+  end
+
+  it "supports any key selector" do
+    expect(
+      described_class.scan(
+        '[{"a":1,"b":2},{"c":3,"d":4},[5]]',
+        [[described_class::ANY_INDEX, described_class::ANY_KEY]],
+      ),
+    ).to eq(
+      [[[6, 7, :number], [12, 13, :number], [20, 21, :number], [26, 27, :number]]],
+    )
+    expect(
+      described_class.scan(
+        '{"a":[1,2],"b":{"c":3}}',
+        [[described_class::ANY_KEY, described_class::ANY_INDEX]],
+      ),
+    ).to eq(
+      [[[6, 7, :number], [8, 9, :number]]],
+    )
+  end
+
   it "works with max path len correctly" do
     expect(
       described_class.scan('{"a": [1]}', [[], ["a"]]),
@@ -90,11 +118,32 @@ RSpec.describe JsonScanner do
     expect do
       described_class.scan "{1}", [], verbose_error: true
     end.to raise_error described_class::ParseError, /invalid object key(?=.*\(right here\))/m
+    expect do
+      described_class.scan("[0, 42,", [[(1..-1)]], verbose_error: true)
+    end.to raise_error described_class::ParseError, /parse error: premature EOF.*\[0, 42,.*\(right here\) ------\^/m
+  end
+
+  it "includes bytes consumed in the exception" do
+    expect do
+      described_class.scan("[[1,2],,[3,4]]", [])
+    end.to(
+      raise_error(described_class::ParseError) do |exc|
+        expect(exc.bytes_consumed).to eq(8)
+      end,
+    )
+    expect do
+      described_class.scan("[[1,2", [])
+    end.to(
+      raise_error(described_class::ParseError) do |exc|
+        # 6 because of the final " " chunk - that's how yajl works
+        expect(exc.bytes_consumed).to eq(6)
+      end,
+    )
   end
 
   it "allows to return an actual path to the element" do
     with_path_expected_res = [
-      # result for first mathcer, each element array of two items:
+      # result for first matcher, each element array of two items:
       # array of path elements and 3-element array start,end,type
       [[[0], [1, 6, :array]], [[1], [7, 12, :array]]],
       [
@@ -128,7 +177,7 @@ RSpec.describe JsonScanner do
       ),
     ).to eq(
       [
-        # result for first mathcer, each element 3-element array start,end,type
+        # result for first matcher, each element 3-element array start,end,type
         [[1, 6, :array], [7, 12, :array]],
         [
           [2, 3, :number], [4, 5, :number],
@@ -190,4 +239,122 @@ RSpec.describe JsonScanner do
       described_class.scan(json, [[]])
     end.to raise_error(described_class::ParseError)
   end
+
+  context "with yajl params" do
+    it "supports 'allow_comments'" do
+      params = ["[0, /* answer */ 42, 0]", [[(1..-1)]]]
+      expect(described_class.scan(*params, allow_comments: true)).to eq(
+        [[[17, 19, :number], [21, 22, :number]]],
+      )
+      expect do
+        described_class.scan(*params)
+      end.to raise_error(described_class::ParseError)
+    end
+
+    it "supports 'dont_validate_strings'" do
+      params = ["\"\x81\x83\"", [[]]]
+      expect(described_class.scan(*params, dont_validate_strings: true)).to eq(
+        [[[0, 4, :string]]],
+      )
+      expect do
+        described_class.scan(*params)
+      end.to raise_error(described_class::ParseError)
+      params = ["{\"\x81\x83\": 42}", [[JsonScanner::ANY_KEY]]]
+      expect(described_class.scan(*params, dont_validate_strings: true, with_path: true)).to eq(
+        [[[["\x81\x83".dup.force_encoding(Encoding::BINARY)], [7, 9, :number]]]],
+      )
+      expect do
+        described_class.scan(*params, with_path: true)
+      end.to raise_error(described_class::ParseError)
+    end
+
+    it "supports 'allow_trailing_garbage'" do
+      params = ["[0, 42, 0]garbage", [[(1..-1)]]]
+      expect(described_class.scan(*params, allow_trailing_garbage: true)).to eq(
+        [[[4, 6, :number], [8, 9, :number]]],
+      )
+      expect do
+        described_class.scan(*params)
+      end.to raise_error(described_class::ParseError)
+    end
+
+    it "supports 'allow_multiple_values'" do
+      params = ["[0, 42, 0]  [0, 34]", [[(1..-1)]]]
+      expect(described_class.scan(*params, allow_multiple_values: true)).to eq(
+        [[[4, 6, :number], [8, 9, :number], [16, 18, :number]]],
+      )
+      expect do
+        described_class.scan(*params)
+      end.to raise_error(described_class::ParseError)
+    end
+
+    it "handles multiple top-level values correctly with 'allow_multiple_values'" do
+      expect(described_class.scan("[0, 42, 0]  [0, 34]", [[]], allow_multiple_values: true)).to eq(
+        [[[0, 10, :array], [12, 19, :array]]],
+      )
+      expect(described_class.scan('{"42": 34}  [0, 34]', [[]], allow_multiple_values: true)).to eq(
+        [[[0, 10, :object], [12, 19, :array]]],
+      )
+      expect(described_class.scan('[0, 42, 0]  {"42": 34}', [[]], allow_multiple_values: true)).to eq(
+        [[[0, 10, :array], [12, 22, :object]]],
+      )
+      expect(described_class.scan('{"42": 34}  {"0": 34}', [[]], allow_multiple_values: true)).to eq(
+        [[[0, 10, :object], [12, 21, :object]]],
+      )
+    end
+
+    it "supports 'allow_partial_values'" do
+      params = ["[0, 42, 0,", [[(1..-1)]]]
+      expect(described_class.scan(*params, allow_partial_values: true)).to eq(
+        [[[4, 6, :number], [8, 9, :number]]],
+      )
+      expect do
+        described_class.scan(*params)
+      end.to raise_error(described_class::ParseError)
+      expect(described_class.scan("[0, 42, 0", [[(1..-1)]], allow_partial_values: true)).to eq(
+        [[[4, 6, :number], [8, 9, :number]]],
+      )
+      expect(described_class.scan("[0, 42, true", [[(1..-1)]], allow_partial_values: true)).to eq(
+        [[[4, 6, :number], [8, 12, :boolean]]],
+      )
+    end
+  end
+
+  describe described_class::Config do
+    it "saves state" do
+      key = "abracadabra".dup
+      conf = described_class.new [[], [key]]
+      key["cad"] = 0.chr
+      key = nil # rubocop:disable Lint/UselessAssignment
+      GC.start
+      expect(
+        10.times.map do
+          JsonScanner.scan '{"abracadabra": 10}', conf, with_path: true
+        end.uniq,
+      ).to eq([[[[[], [0, 19, :object]]], [[["abracadabra"], [16, 18, :number]]]]])
+      expect(
+        10.times.map do
+          JsonScanner.scan '{"abracadabra": 10}', conf
+        end.uniq,
+      ).to eq([[[[0, 19, :object]], [[16, 18, :number]]]])
+    end
+
+    it "re-raises exceptions" do
+      expect do
+        described_class.new [[(0...-1)]]
+      end.to raise_error ArgumentError
+      expect do
+        described_class.new [[(0..-2)]]
+      end.to raise_error ArgumentError
+      expect do
+        described_class.new [[(-42..1)]]
+      end.to raise_error ArgumentError
+    end
+
+    it "supports inspect" do
+      expect(
+        described_class.new([[], ["abracadabra", JsonScanner::ANY_INDEX], [42, JsonScanner::ANY_KEY]]).inspect,
+      ).to eq("#<JsonScanner::Config [[], ['abracadabra', (0..9223372036854775807)], [42, ('*'..'*')]]>")
+    end
+  end
 end

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: json_scanner
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.1
 platform: ruby
 authors:
 - uvlad7
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-12-27 00:00:00.000000000 Z
+date: 2025-08-14 00:00:00.000000000 Z
 dependencies: []
-description: This gem uses yajl lib to scan a json string and allows you to parse
+description: This gem uses the yajl lib to scan a JSON string and allows you to parse
   pieces of it
 email:
 - uvlad7@gmail.com
@@ -26,6 +26,8 @@ files:
 - lib/json_scanner.rb
 - lib/json_scanner/version.rb
 - sig/json_scanner.rbs
+- spec/extensiontesttask.rb
+- spec/json_scanner_spec.c
 - spec/json_scanner_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/uvlad7/json_scanner