json_scanner 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9c1ddff519827bc802cdcacb5b048402706544b0882c8ac91cd1aa414c4b57e0
-  data.tar.gz: d1c4f41dbd71ed08a488c2f9647194fd2692c91d522fee35d28d7060bf80321c
+  metadata.gz: ca9d160b389a5b605a37baeebd8e9d704a9b42712b9c8de9d0667ffa7e6b3d35
+  data.tar.gz: f2c0362e6bb4484e47fd5dbdda4afd7332ce42d266b4d0ebe681f24e18c31b0a
 SHA512:
-  metadata.gz: 57bf59cc9495f46675bb98d2fc7545bdc3b8392631c443ad2b89595b22be054c8f8bb268a798c5f104d1e38b73d577662f96637fd9311260c9b0a45b55044265
-  data.tar.gz: '055432559a23dbf34e679aac7be4967ea163684fda718b433e978c34feb73f298f1346b00629fada8279a9e49e1267b990f73d2d57c8403330faf42ce4086bb8'
+  metadata.gz: 91d190291d0e2c16db38c523a58fbbff23cde136481b2c4a19446d25169a77390eee4792979cd3a22724bdf2c89132b5dc3ccc78c1553c17fe21227a70e1bf3c
+  data.tar.gz: dc704ac0ba609f209883d5af50f3f9cabcde04a0eb2c3f5ac04f964010bf866fd727c5c175dd40917526b693f13bc4b532f7a4fabce9859c0b44238cf3fa421b

data/README.md

@@ -54,13 +54,30 @@ JsonScanner.scan('[0, 42, 0]', [[JsonScanner::ANY_INDEX]])
 # Special matcher JsonScanner::ANY_KEY is supported for object keys
 JsonScanner.scan('{"a": 1, "b": 2}', [[JsonScanner::ANY_KEY]], with_path: true)
 # => [[[["a"], [6, 7, :number]], [["b"], [14, 15, :number]]]]
+# Regex mathers aren't supported yet, but you can simulate it using `with_path` option
+JsonScanner.scan(
+  '{"question1": 1, "answer": 42, "question2": 2}',
+  [[JsonScanner::ANY_KEY]], with_path: true,
+).map do |res|
+  res.map do |path, (begin_pos, end_pos, type)|
+    [begin_pos, end_pos, type] if path[0] =~ /\Aquestion/
+  end.compact
+end
+# => [[[14, 15, :number], [44, 45, :number]]]
 ```
 
-It supports multiple options
+## Options
+
+`JsonScanner` supports multiple options
 
 ```ruby
 JsonScanner.scan('[0, 42, 0]', [[(1..-1)]], with_path: true)
 # => [[[[1], [4, 6, :number]], [[2], [8, 9, :number]]]]
+JsonScanner.scan('[0, 42],', [[(1..-1)]], verbose_error: true)
+# JsonScanner::ParseError (parse error: trailing garbage)
+#                                 [0, 42],
+#                      (right here) ------^
+# Note: the 'right here' pointer is wrong in case of a premature EOF error, it's a bug of the libyajl
 JsonScanner.scan('[0, 42,', [[(1..-1)]], verbose_error: true)
 # JsonScanner::ParseError (parse error: premature EOF)
 #                                        [0, 42,
@@ -81,6 +98,8 @@ JsonScanner.scan('{"a": 1}', [[JsonScanner::ANY_KEY]], with_path: true, symboliz
 # => [[[[:a], [6, 7, :number]]]]
 ```
 
+### Comments in the JSON
+
 Note that the standard `JSON` library supports comments, so you may want to enable it in the `JsonScanner` as well
 ```ruby
 json_str = '{"answer": {"value": 42 /* the Ultimate Question of Life, the Universe, and Everything */ }}'
@@ -90,7 +109,21 @@ end
 # => [{"value"=>42}]
 ```
 
-You can also create a config and reuse it
+### Find the end of a JSON string
+
+`allow_trailing_garbage` option may come in handy if you want to extract a JSON string from a JS text
+```ruby
+script_text = <<~'JS'
+      <script>window.__APOLLO_STATE__={"ContentItem:0":{"__typename":"ContentItem","id":0, "configurationType":"NO_CONFIGURATION","replacementPartsUrl":null,"relatedCategories":[{"__ref":"Category:109450"},{"__ref":"Category:82044355"},{"__ref":"Category:109441"},{"__ref":"Category:109442"},{"__ref":"Category:109449"},{"__ref":"Category:109444"},{"__ref":"Category:82043730"}],"recommendedOptions":[]}};window.__APPVERSION__=7018;window.__CONFIG_ENV__={value: 'PRODUCTION'};</script>
+JS
+json_with_trailing_garbage = script_text[/__APOLLO_STATE__\s*=\s*({.+)/, 1]
+json_end_pos = JsonScanner.scan(json_with_trailing_garbage, [[]], allow_trailing_garbage: true).first.first[1]
+apollo_state = JSON.parse(json_with_trailing_garbage[0...json_end_pos])
+```
+
+## Reuse configuration
+
+You can create a `JsonScanner::Config` instance and reuse it between `JsonScanner.scan` calls
 
 ```ruby
 require "json_scanner"
@@ -107,6 +140,17 @@ JsonScanner.scan('[0, 42]', config, with_path: true)
 # => [[[[], [0, 7, :array]]], [], [[[0], [1, 2, :number]], [[1], [4, 6, :number]]]]
 ```
 
+Options can be passed as a hash, even on Ruby 3
+```ruby
+options = { allow_trailing_garbage: true, allow_partial_values: true }
+JsonScanner.scan('[0, 42', [[1]], options) == JsonScanner.scan('[0, 42]_', [[1]], options)
+# => true
+```
+
+## Streaming mode
+
+Streaming mode isn't supported yet, as it's harder to implement and to use. I plan to add it in the future, its API is a subject to discussion. If you have suggestions, use cases, or preferences for how it should behave, I’d love to hear from you!
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/ext/json_scanner/json_scanner.c

@@ -97,7 +97,7 @@ inline size_t scan_ctx_get_bytes_consumed(scan_ctx *ctx)
   return ctx->yajl_bytes_consumed + yajl_get_bytes_consumed(ctx->handle);
 }
 
-inline void scan_ctx_update_bytes_consumed(scan_ctx *ctx)
+inline void scan_ctx_save_bytes_consumed(scan_ctx *ctx)
 {
   ctx->yajl_bytes_consumed += yajl_get_bytes_consumed(ctx->handle);
 }
@@ -254,7 +254,8 @@ VALUE scan_ctx_init(scan_ctx *ctx, VALUE path_ary, VALUE string_keys)
         if (string_keys != Qundef)
         {
           // If string_keys is provided, we need to duplicate the string
-          // to avoid use-after-free issues and to add the newly created string to the string_keys array
+          // to avoid use-after-free issues and to add the newly created string to the string_keys array.
+          // In Ruby 2.2 and newer symbols can be GC-ed, so we need to duplicate them as well.
           entry = rb_str_dup(entry);
           rb_ary_push(string_keys, entry);
         }
@@ -733,7 +734,7 @@ VALUE scan(int argc, VALUE *argv, VALUE self)
   yajl_status stat;
   scan_ctx *ctx;
   int free_ctx = true;
-  VALUE err_msg = Qnil, bytes_consumed, err, result;
+  VALUE err_msg = Qnil, bytes_consumed, result;
   // Turned out callbacks can't raise exceptions
   // VALUE callback_err;
 #if RUBY_API_VERSION_MAJOR > 2 || (RUBY_API_VERSION_MAJOR == 2 && RUBY_API_VERSION_MINOR >= 7)
@@ -801,19 +802,17 @@ VALUE scan(int argc, VALUE *argv, VALUE self)
   }
   ctx->handle = handle;
   stat = yajl_parse(handle, (unsigned char *)json_text, json_text_len);
-  scan_ctx_update_bytes_consumed(ctx);
   if (stat == yajl_status_ok)
   {
+    scan_ctx_save_bytes_consumed(ctx);
     stat = yajl_complete_parse(handle);
-    scan_ctx_update_bytes_consumed(ctx);
   }
 
   if (stat != yajl_status_ok)
   {
     char *str = (char *)yajl_get_error(handle, verbose_error, (unsigned char *)json_text, json_text_len);
     err_msg = rb_utf8_str_new_cstr(str);
-    // TODO: maybe use scan_ctx_get_bytes_consumed here too? But it makes difference in premature EOF
-    bytes_consumed = ULL2NUM(yajl_get_bytes_consumed(handle));
+    bytes_consumed = ULL2NUM(scan_ctx_get_bytes_consumed(ctx));
     yajl_free_error(handle, (unsigned char *)str);
   }
   // // Needed when yajl_allow_partial_values is set
@@ -844,7 +843,7 @@ VALUE scan(int argc, VALUE *argv, VALUE self)
   yajl_free(handle);
   if (err_msg != Qnil)
   {
-    err = rb_exc_new_str(rb_eJsonScannerParseError, err_msg);
+    VALUE err = rb_exc_new_str(rb_eJsonScannerParseError, err_msg);
     rb_ivar_set(err, rb_iv_bytes_consumed, bytes_consumed);
     rb_exc_raise(err);
   }

data/lib/json_scanner/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JsonScanner
-  VERSION = "0.3.0"
+  VERSION = "0.3.1"
 end

data/spec/json_scanner_spec.rb

@@ -131,6 +131,14 @@ RSpec.describe JsonScanner do
         expect(exc.bytes_consumed).to eq(8)
       end,
     )
+    expect do
+      described_class.scan("[[1,2", [])
+    end.to(
+      raise_error(described_class::ParseError) do |exc|
+        # 6 because of the final " " chunk - that's how yajl works
+        expect(exc.bytes_consumed).to eq(6)
+      end,
+    )
   end
 
   it "allows to return an actual path to the element" do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: json_scanner
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - uvlad7