fast_jsonparser 0.3.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cb4380ffb8ced606931028f66c05ddb3af40498a9aada56833be2f5ef0bc47b4
-  data.tar.gz: 3a3159926d6f1b1b0d90431171b7c97a93df4082bc1371313c4fe31e6c4de0c8
+  metadata.gz: d87b77e2dd63a557d8a32fdf47c962a52476aa921e981dd0120a3d57a5873453
+  data.tar.gz: b859d5d018cb9b7ce88d1bf6fe662ebf80dfbf57f5aed831e1fc22059dde0ceb
 SHA512:
-  metadata.gz: ffa4a69c6550db893fd93c1f13df5778004188f1f3b54944b2d20049366026bbf2720fb0d62dce124d279fab650ed4e2f12c763c48207323ec9a900da240b32a
-  data.tar.gz: fe75932c18f3cf0d896536ffc1d4f8f30067fe85046c7c1bf1806e819674f3f0a48816240cc634ac5dd530eb06e80f7b2a35f63a91e5c4ff1e432e8bf5310df5
+  metadata.gz: f3c9ef4836bfae90bb7cfce534a108cd102c621c90a23cd783bacbf65bfe830b863eebb89ee92abf26549b954c247857f32f3d00b44f1e0c37b327393b69161d
+  data.tar.gz: acc8fca8c8110f4f6c88b74cad2a73357d0664db214b8fed5ca3503f0c3290f528d394c9d7abedf6c677ec1f5608667358ec56f0995fa7aa7ce8ada060ddbfcd

data/.github/workflows/ruby.yml

@@ -0,0 +1,36 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
+# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
+
+name: Ruby
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+    
+    strategy:
+      matrix:
+        ruby-version: [3.0.1, 2.7.3, 2.6.7, 2.6.5]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby ${{ matrix.ruby-version }}
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+    - name: Install dependencies
+      run: bundle install
+    - name: Compile
+      run: bundle exec rake compile
+    - name: Run tests
+      run: bundle exec rake

data/.gitignore

@@ -6,3 +6,5 @@
 /pkg/
 /spec/reports/
 /tmp/
+*.so
+*.bundle

data/CHANGELOG.md

@@ -0,0 +1,9 @@
+# 0.6.0
+* Fix performance on Ruby 3.0 [Issue #20](https://github.com/anilmaurya/fast_jsonparser/issues/20), thanks to [Watson1978](https://github.com/Watson1978)
+# 0.5.0
+* Handle concurrent use of the parser in [Issue #15](https://github.com/anilmaurya/fast_jsonparser/pull/15), thanks to [casperisfine](https://github.com/casperisfine)
+
+# 0.4.0
+* load_many accept batch_size parameter to parse documents larger than 1 MB in [PR #5](https://github.com/anilmaurya/fast_jsonparser/pull/5), thanks to [casperisfine](https://github.com/casperisfine)
+* Add option for symbolize_keys, default to true in [PR #9](https://github.com/anilmaurya/fast_jsonparser/pull/9), thanks to [casperisfine](https://github.com/casperisfine)
+* Parse string values as UTF-8 in [PR #10](https://github.com/anilmaurya/fast_jsonparser/pull/10), thanks to [casperisfine](https://github.com/casperisfine)

data/Gemfile.lock

@@ -1,20 +1,21 @@
 PATH
   remote: .
   specs:
-    fast_jsonparser (0.2.0)
+    fast_jsonparser (0.5.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    minitest (5.14.1)
-    oj (3.10.6)
-    rake (13.0.1)
-    rake-compiler (1.1.0)
+    minitest (5.14.4)
+    oj (3.11.7)
+    rake (13.0.3)
+    rake-compiler (1.1.1)
       rake
     yajl-ruby (1.4.1)
 
 PLATFORMS
   ruby
+  x86_64-linux
 
 DEPENDENCIES
   bundler (~> 2.0)
@@ -26,4 +27,4 @@ DEPENDENCIES
   yajl-ruby
 
 BUNDLED WITH
-   2.0.1
+   2.2.3

data/README.md

@@ -107,14 +107,36 @@ Example: logs.json with following content
 "17/May/2015:08:05:23 +0000"
 "17/May/2015:08:05:24 +0000"
 ```
+If size of json batch is greater than 1 MB then use `batch_size` option
 
+```
+FastJsonparser.load_many(f.path, batch_size: 2_000) {}
+```
+
+4. Accept optional param :symbolize_keys (default symbolize_keys: true)
 
-4. Raise FastJsonparser::ParseError when invalid JSON provided for parsing
+If string key is expected in parsed result then use
+
+```
+FastJsonparser.parse('{"one": 1, "two": 2}', symbolize_keys: false)
+
+```
+
+5. Raise FastJsonparser::ParseError when invalid JSON provided for parsing
 
 ```
 FastJsonparser.parse("123: 1") # FastJsonparser::ParseError (parse error)
 ```
 
+### Known Incompatibilities with stdlib JSON
+
+`FastJsonparser` behaves mostly like stdlib's `JSON`, but there are a few corner cases:
+
+- `FastJsonparser` will use symbols for hash keys by default. You can pass `symbolize_keys: false` to have strings instead like `JSON`.
+- `FastJsonparser` will raise on integers outside of the 64bits range (`-9223372036854775808..18446744073709551615`), `JSON` will parse them fine.
+- `FastJsonparser` will raise on invalid string escapings (`"\x15"`), `JSON` will often handle some of them.
+- `FastJsonparser` will raise on `/**/` comments. `JSON` will sometimes ignore them, sometimes raise.
+
 ### Example
 
 ```
@@ -124,9 +146,9 @@ FastJsonparser.parse("123: 1") # FastJsonparser::ParseError (parse error)
 ```
 ## Development
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+After checking out the repo, run `rake compile` to install dependencies. Then, run `rake test` to run the tests.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+For more option, refer https://github.com/rake-compiler/rake-compiler
 
 ## Contributing
 

data/Rakefile

@@ -3,6 +3,7 @@ require "rake/testtask"
 require "rake/extensiontask"
 
 Rake::ExtensionTask.new("fast_jsonparser") do |ext|
+  ext.ext_dir = 'ext/fast_jsonparser'
   ext.lib_dir = "lib/fast_jsonparser"
 end
 

data/ext/fast_jsonparser/extconf.rb

@@ -1,5 +1,5 @@
 require 'mkmf'
-$CXXFLAGS += ' -std=c++1z -Wno-register '
+$CXXFLAGS += ' $(optflags) $(debugflags) -std=c++1z -Wno-register '
 
 
 create_makefile 'fast_jsonparser/fast_jsonparser'

data/ext/fast_jsonparser/fast_jsonparser.cpp

@@ -2,116 +2,153 @@
 
 #include "simdjson.h"
 
-VALUE rb_mFastJsonparser;
-
-VALUE rb_eFastJsonparserParseError;
+VALUE rb_eFastJsonparserUnknownError, rb_eFastJsonparserParseError;
 
 using namespace simdjson;
 
+typedef struct {
+    dom::parser *parser;
+} parser_t;
+
+static void Parser_delete(void *ptr) {
+    parser_t *data = (parser_t*) ptr;
+    delete data->parser;
+}
+
+static size_t Parser_memsize(const void *parser) {
+    return sizeof(dom::parser); // TODO: low priority, figure the real size, e.g. internal buffers etc.
+}
+
+static const rb_data_type_t parser_data_type = {
+    "Parser",
+    { 0, Parser_delete, Parser_memsize, },
+    0, 0, RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static VALUE parser_allocate(VALUE klass) {
+    parser_t *data;
+    VALUE obj = TypedData_Make_Struct(klass, parser_t, &parser_data_type, data);
+    data->parser = new dom::parser;
+    return obj;
+}
+
+static inline dom::parser * get_parser(VALUE self) {
+    parser_t *data;
+    TypedData_Get_Struct(self, parser_t, &parser_data_type, data);
+    return data->parser;
+}
+
 // Convert tape to Ruby's Object
-static VALUE make_ruby_object(dom::element element)
+static VALUE make_ruby_object(dom::element element, bool symbolize_keys)
 {
-    auto t = element.type();
-    if (t == dom::element_type::ARRAY)
+    switch (element.type())
+    {
+    case dom::element_type::ARRAY:
     {
         VALUE ary = rb_ary_new();
         for (dom::element x : element)
         {
-            VALUE e = make_ruby_object(x);
+            VALUE e = make_ruby_object(x, symbolize_keys);
             rb_ary_push(ary, e);
         }
         return ary;
     }
-    else if (t == dom::element_type::OBJECT)
+    case dom::element_type::OBJECT:
     {
         VALUE hash = rb_hash_new();
         for (dom::key_value_pair field : dom::object(element))
         {
             std::string_view view(field.key);
-            VALUE k = rb_intern(view.data());
-            VALUE v = make_ruby_object(field.value);
-            rb_hash_aset(hash, ID2SYM(k), v);
+            VALUE k = rb_utf8_str_new(view.data(), view.size());
+            if (symbolize_keys)
+            {
+                k = ID2SYM(rb_intern_str(k));
+            }
+            VALUE v = make_ruby_object(field.value, symbolize_keys);
+            rb_hash_aset(hash, k, v);
         }
         return hash;
     }
-    else if (t == dom::element_type::INT64)
+    case dom::element_type::INT64:
     {
         return LONG2NUM(element.get<int64_t>());
     }
-    else if (t == dom::element_type::UINT64)
+    case dom::element_type::UINT64:
     {
         return ULONG2NUM(element.get<uint64_t>());
     }
-    else if (t == dom::element_type::DOUBLE)
+    case dom::element_type::DOUBLE:
     {
         return DBL2NUM(double(element));
     }
-    else if (t == dom::element_type::STRING)
+    case dom::element_type::STRING:
     {
         std::string_view view(element);
-        return rb_str_new(view.data(), view.size());
+        return rb_utf8_str_new(view.data(), view.size());
     }
-    else if (t == dom::element_type::BOOL)
+    case dom::element_type::BOOL:
     {
         return bool(element) ? Qtrue : Qfalse;
     }
-    else if (t == dom::element_type::NULL_VALUE)
+    case dom::element_type::NULL_VALUE:
     {
         return Qnil;
     }
+    }
     // unknown case (bug)
     rb_raise(rb_eException, "[BUG] must not happen");
 }
 
-static VALUE rb_fast_jsonparser_parse(VALUE self, VALUE arg)
+static VALUE rb_fast_jsonparser_parse(VALUE self, VALUE arg, VALUE symbolize_keys)
 {
     Check_Type(arg, T_STRING);
+    dom::parser *parser = get_parser(self);
 
-    dom::parser parser;
-    auto [doc, error] = parser.parse(RSTRING_PTR(arg), RSTRING_LEN(arg));
-    if (error == SUCCESS)
+    auto [doc, error] = parser->parse(RSTRING_PTR(arg), RSTRING_LEN(arg));
+    if (error != SUCCESS)
     {
-        return make_ruby_object(doc);
+        rb_raise(rb_eFastJsonparserParseError, "%s", error_message(error));
     }
-    // TODO better error handling
-    rb_raise(rb_eFastJsonparserParseError, "parse error");
-    return Qnil;
+    return make_ruby_object(doc, RTEST(symbolize_keys));
 }
 
-static VALUE rb_fast_jsonparser_load(VALUE self, VALUE arg)
+static VALUE rb_fast_jsonparser_load(VALUE self, VALUE arg, VALUE symbolize_keys)
 {
     Check_Type(arg, T_STRING);
+    dom::parser *parser = get_parser(self);
 
-    dom::parser parser;
-    auto [doc, error] = parser.load(RSTRING_PTR(arg));
-    if (error == SUCCESS)
+    auto [doc, error] = parser->load(RSTRING_PTR(arg));
+    if (error != SUCCESS)
     {
-        return make_ruby_object(doc);
+        rb_raise(rb_eFastJsonparserParseError, "%s", error_message(error));
     }
-    // TODO better error handling
-    rb_raise(rb_eFastJsonparserParseError, "parse error");
-    return Qnil;
+    return make_ruby_object(doc, RTEST(symbolize_keys));
 }
 
-static VALUE rb_fast_jsonparser_load_many(VALUE self, VALUE arg)
+static VALUE rb_fast_jsonparser_load_many(VALUE self, VALUE arg, VALUE symbolize_keys, VALUE batch_size)
 {
     Check_Type(arg, T_STRING);
+    Check_Type(batch_size, T_FIXNUM);
+    dom::parser *parser = get_parser(self);
+
+    try {
+        auto [docs, error] = parser->load_many(RSTRING_PTR(arg), FIX2INT(batch_size));
+        if (error != SUCCESS)
+        {
+            rb_raise(rb_eFastJsonparserParseError, "%s", error_message(error));
+        }
 
-    dom::parser parser;
-    auto [docs, error] = parser.load_many(RSTRING_PTR(arg));
-    if (error == SUCCESS)
-    {
         for (dom::element doc : docs)
         {
-            if (rb_block_given_p())
-            {
-                rb_yield(make_ruby_object(doc));
-            }
+            rb_yield(make_ruby_object(doc, RTEST(symbolize_keys)));
         }
+
         return Qnil;
     }
-    rb_raise(rb_eFastJsonparserParseError, "parse error");
-    return Qnil;
+    catch (simdjson::simdjson_error error)
+    {
+        rb_raise(rb_eFastJsonparserUnknownError, "%s", error.what());
+    }
 }
 
 extern "C"
@@ -119,10 +156,17 @@ extern "C"
 
     void Init_fast_jsonparser(void)
     {
-        rb_mFastJsonparser = rb_define_module("FastJsonparser");
-        rb_eFastJsonparserParseError = rb_define_class_under(rb_mFastJsonparser, "ParseError", rb_eStandardError);
-        rb_define_module_function(rb_mFastJsonparser, "parse", reinterpret_cast<VALUE (*)(...)>(rb_fast_jsonparser_parse), 1);
-        rb_define_module_function(rb_mFastJsonparser, "load", reinterpret_cast<VALUE (*)(...)>(rb_fast_jsonparser_load), 1);
-        rb_define_module_function(rb_mFastJsonparser, "load_many", reinterpret_cast<VALUE (*)(...)>(rb_fast_jsonparser_load_many), 1);
+        VALUE rb_mFastJsonparser = rb_const_get(rb_cObject, rb_intern("FastJsonparser"));
+        VALUE rb_cFastJsonparserNative = rb_const_get(rb_mFastJsonparser, rb_intern("Native"));
+
+        rb_define_alloc_func(rb_cFastJsonparserNative, parser_allocate);
+        rb_define_method(rb_cFastJsonparserNative, "_parse", reinterpret_cast<VALUE (*)(...)>(rb_fast_jsonparser_parse), 2);
+        rb_define_method(rb_cFastJsonparserNative, "_load", reinterpret_cast<VALUE (*)(...)>(rb_fast_jsonparser_load), 2);
+        rb_define_method(rb_cFastJsonparserNative, "_load_many", reinterpret_cast<VALUE (*)(...)>(rb_fast_jsonparser_load_many), 3);
+
+        rb_eFastJsonparserParseError = rb_const_get(rb_mFastJsonparser, rb_intern("ParseError"));
+        rb_global_variable(&rb_eFastJsonparserParseError);
+        rb_eFastJsonparserUnknownError = rb_const_get(rb_mFastJsonparser, rb_intern("UnknownError"));
+        rb_global_variable(&rb_eFastJsonparserUnknownError);
     }
 }

data/ext/fast_jsonparser/simdjson.h

@@ -2308,7 +2308,7 @@ using ErrorValues [[deprecated("This is an alias and will be removed, use error_
  * @deprecated Error codes should be stored and returned as `error_code`, use `error_message()` instead.
  */
 [[deprecated("Error codes should be stored and returned as `error_code`, use `error_message()` instead.")]]
-inline const std::string &error_message(int error) noexcept;
+inline const std::string error_message(int error) noexcept;
 
 } // namespace simdjson
 
@@ -6367,7 +6367,7 @@ namespace internal {
   // We store the error code so we can validate the error message is associated with the right code
   struct error_code_info {
     error_code code;
-    std::string message;
+    const char* message;
   };
   // These MUST match the codes in error_code. We check this constraint in basictests.
   extern SIMDJSON_DLLIMPORTEXPORT const error_code_info error_codes[];
@@ -6376,10 +6376,10 @@ namespace internal {
 
 inline const char *error_message(error_code error) noexcept {
   // If you're using error_code, we're trusting you got it from the enum.
-  return internal::error_codes[int(error)].message.c_str();
+  return internal::error_codes[int(error)].message;
 }
 
-inline const std::string &error_message(int error) noexcept {
+inline const std::string error_message(int error) noexcept {
   if (error < 0 || error >= error_code::NUM_ERROR_CODES) {
     return internal::error_codes[UNEXPECTED_ERROR].message;
   }

data/lib/fast_jsonparser/version.rb

@@ -1,3 +1,3 @@
 module FastJsonparser
-  VERSION = "0.3.0"
+  VERSION = "0.6.0"
 end

data/lib/fast_jsonparser.rb

@@ -1,8 +1,46 @@
+# frozen_string_literal: true
+
 require "fast_jsonparser/version"
 
 module FastJsonparser
-  class Error < StandardError; end
-  # Your code goes here...
-end
+  Error = Class.new(StandardError)
+  ParseError = Class.new(Error)
+  UnknownError = Class.new(Error)
+  BatchSizeTooSmall = Class.new(Error)
+
+  DEFAULT_BATCH_SIZE = 1_000_000 # from include/simdjson/dom/parser.h
+
+  class << self
+    def parse(source, symbolize_keys: true)
+      parser._parse(source, symbolize_keys)
+    end
+
+    def load(source, symbolize_keys: true)
+      parser._load(source, symbolize_keys)
+    end
+
+    def load_many(source, symbolize_keys: true, batch_size: DEFAULT_BATCH_SIZE, &block)
+      Native.new._load_many(source, symbolize_keys, batch_size, &block)
+    rescue UnknownError => error
+      case error.message
+      when "This parser can't support a document that big"
+        raise BatchSizeTooSmall, "One of the documents was bigger than the batch size (#{batch_size}B), try increasing it."
+      else
+        raise
+      end
+    end
 
-require "fast_jsonparser/fast_jsonparser" # loads cpp extension
+    private
+
+    def parser
+      @parser ||= Native.new
+    end
+  end
+
+  class Native
+  end
+
+  require "fast_jsonparser/fast_jsonparser" # loads cpp extension
+
+  private_constant :Native
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fast_jsonparser
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Anil Maurya
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-07-14 00:00:00.000000000 Z
+date: 2022-07-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -102,8 +102,10 @@ extensions:
 - ext/fast_jsonparser/extconf.rb
 extra_rdoc_files: []
 files:
+- ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".travis.yml"
+- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
 - Gemfile.lock
@@ -119,7 +121,6 @@ files:
 - ext/fast_jsonparser/simdjson.h
 - fast_jsonparser.gemspec
 - lib/fast_jsonparser.rb
-- lib/fast_jsonparser/fast_jsonparser.bundle
 - lib/fast_jsonparser/version.rb
 homepage: https://github.com/anilmaurya/fast_jsonparser
 licenses:
@@ -140,7 +141,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.2.22
 signing_key:
 specification_version: 4
 summary: Fast Json Parser