fast_jsonparser 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cb4380ffb8ced606931028f66c05ddb3af40498a9aada56833be2f5ef0bc47b4
-  data.tar.gz: 3a3159926d6f1b1b0d90431171b7c97a93df4082bc1371313c4fe31e6c4de0c8
+  metadata.gz: 2b9a3639a83aa2f68468df13dcc8922b244c727c00b2e040a7dd94f9b3832698
+  data.tar.gz: 49e6c154239a25e2ecc695288166914ee60a1500d7e2773ef25d05e39eddfa4d
 SHA512:
-  metadata.gz: ffa4a69c6550db893fd93c1f13df5778004188f1f3b54944b2d20049366026bbf2720fb0d62dce124d279fab650ed4e2f12c763c48207323ec9a900da240b32a
-  data.tar.gz: fe75932c18f3cf0d896536ffc1d4f8f30067fe85046c7c1bf1806e819674f3f0a48816240cc634ac5dd530eb06e80f7b2a35f63a91e5c4ff1e432e8bf5310df5
+  metadata.gz: '08ebbd273a4180261ab78c50874e7a5cf28afa7c1718be05fbc8bec93bd6652097c88ad2b0ac7eba184f4755a209cc0e9256551d04e1350ac1139d4141a50412'
+  data.tar.gz: 6400ba1f5f0f0083b4c2a75f947888ab782695c66c676a1bb35011efde40b7ce2b1ab38d0fbbfc76eda50e80cef6194cc8bf495e3f8cb6b1aef0c8855bf2ed76

data/.gitignore

@@ -6,3 +6,5 @@
 /pkg/
 /spec/reports/
 /tmp/
+*.so
+*.bundle

data/CHANGELOG.md

@@ -0,0 +1,4 @@
+# 0.4.0
+* load_many accept batch_size parameter to parse documents larger than 1 MB in [PR #5](https://github.com/anilmaurya/fast_jsonparser/pull/5), thanks to [casperisfine](https://github.com/casperisfine)
+* Add option for symbolize_keys, default to true in [PR #9](https://github.com/anilmaurya/fast_jsonparser/pull/9), thanks to [casperisfine](https://github.com/casperisfine)
+* Parse string values as UTF-8 in [PR #10](https://github.com/anilmaurya/fast_jsonparser/pull/10), thanks to [casperisfine](https://github.com/casperisfine)

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fast_jsonparser (0.2.0)
+    fast_jsonparser (0.3.0)
 
 GEM
   remote: https://rubygems.org/

data/README.md

@@ -107,9 +107,22 @@ Example: logs.json with following content
 "17/May/2015:08:05:23 +0000"
 "17/May/2015:08:05:24 +0000"
 ```
+If size of json batch is greater than 1 MB then use `batch_size` option
 
+```
+FastJsonparser.load_many(f.path, batch_size: 2_000) {}
+```
+
+4. Accept optional param :symbolize_keys (default symbolize_keys: true)
+
+If string key is expected in parsed result then use
+
+```
+FastJsonparser.parse('{"one": 1, "two": 2}', symbolize_keys: false)
+
+```
 
-4. Raise FastJsonparser::ParseError when invalid JSON provided for parsing
+5. Raise FastJsonparser::ParseError when invalid JSON provided for parsing
 
 ```
 FastJsonparser.parse("123: 1") # FastJsonparser::ParseError (parse error)
@@ -124,9 +137,9 @@ FastJsonparser.parse("123: 1") # FastJsonparser::ParseError (parse error)
 ```
 ## Development
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+After checking out the repo, run `rake compile` to install dependencies. Then, run `rake test` to run the tests.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+For more option, refer https://github.com/rake-compiler/rake-compiler
 
 ## Contributing
 

data/Rakefile

@@ -3,6 +3,7 @@ require "rake/testtask"
 require "rake/extensiontask"
 
 Rake::ExtensionTask.new("fast_jsonparser") do |ext|
+  ext.ext_dir = 'ext/fast_jsonparser'
   ext.lib_dir = "lib/fast_jsonparser"
 end
 

data/ext/fast_jsonparser/fast_jsonparser.cpp

@@ -2,116 +2,122 @@
 
 #include "simdjson.h"
 
-VALUE rb_mFastJsonparser;
-
-VALUE rb_eFastJsonparserParseError;
+VALUE rb_eFastJsonparserUnknownError, rb_eFastJsonparserParseError;
 
 using namespace simdjson;
 
 // Convert tape to Ruby's Object
-static VALUE make_ruby_object(dom::element element)
+static VALUE make_ruby_object(dom::element element, bool symbolize_keys)
 {
-    auto t = element.type();
-    if (t == dom::element_type::ARRAY)
+    switch (element.type())
+    {
+    case dom::element_type::ARRAY:
     {
         VALUE ary = rb_ary_new();
         for (dom::element x : element)
         {
-            VALUE e = make_ruby_object(x);
+            VALUE e = make_ruby_object(x, symbolize_keys);
             rb_ary_push(ary, e);
         }
         return ary;
     }
-    else if (t == dom::element_type::OBJECT)
+    case dom::element_type::OBJECT:
     {
         VALUE hash = rb_hash_new();
         for (dom::key_value_pair field : dom::object(element))
         {
             std::string_view view(field.key);
-            VALUE k = rb_intern(view.data());
-            VALUE v = make_ruby_object(field.value);
-            rb_hash_aset(hash, ID2SYM(k), v);
+            VALUE k = rb_utf8_str_new(view.data(), view.size());
+            if (symbolize_keys)
+            {
+                k = ID2SYM(rb_intern_str(k));
+            }
+            VALUE v = make_ruby_object(field.value, symbolize_keys);
+            rb_hash_aset(hash, k, v);
         }
         return hash;
     }
-    else if (t == dom::element_type::INT64)
+    case dom::element_type::INT64:
     {
         return LONG2NUM(element.get<int64_t>());
     }
-    else if (t == dom::element_type::UINT64)
+    case dom::element_type::UINT64:
     {
         return ULONG2NUM(element.get<uint64_t>());
     }
-    else if (t == dom::element_type::DOUBLE)
+    case dom::element_type::DOUBLE:
     {
         return DBL2NUM(double(element));
     }
-    else if (t == dom::element_type::STRING)
+    case dom::element_type::STRING:
     {
         std::string_view view(element);
-        return rb_str_new(view.data(), view.size());
+        return rb_utf8_str_new(view.data(), view.size());
     }
-    else if (t == dom::element_type::BOOL)
+    case dom::element_type::BOOL:
     {
         return bool(element) ? Qtrue : Qfalse;
     }
-    else if (t == dom::element_type::NULL_VALUE)
+    case dom::element_type::NULL_VALUE:
     {
         return Qnil;
     }
+    }
     // unknown case (bug)
     rb_raise(rb_eException, "[BUG] must not happen");
 }
 
-static VALUE rb_fast_jsonparser_parse(VALUE self, VALUE arg)
+static VALUE rb_fast_jsonparser_parse(VALUE self, VALUE arg, VALUE symbolize_keys)
 {
     Check_Type(arg, T_STRING);
 
     dom::parser parser;
     auto [doc, error] = parser.parse(RSTRING_PTR(arg), RSTRING_LEN(arg));
-    if (error == SUCCESS)
+    if (error != SUCCESS)
     {
-        return make_ruby_object(doc);
+        rb_raise(rb_eFastJsonparserParseError, "%s", error_message(error));
     }
-    // TODO better error handling
-    rb_raise(rb_eFastJsonparserParseError, "parse error");
-    return Qnil;
+    return make_ruby_object(doc, RTEST(symbolize_keys));
 }
 
-static VALUE rb_fast_jsonparser_load(VALUE self, VALUE arg)
+static VALUE rb_fast_jsonparser_load(VALUE self, VALUE arg, VALUE symbolize_keys)
 {
     Check_Type(arg, T_STRING);
 
     dom::parser parser;
     auto [doc, error] = parser.load(RSTRING_PTR(arg));
-    if (error == SUCCESS)
+    if (error != SUCCESS)
     {
-        return make_ruby_object(doc);
+        rb_raise(rb_eFastJsonparserParseError, "%s", error_message(error));
     }
-    // TODO better error handling
-    rb_raise(rb_eFastJsonparserParseError, "parse error");
-    return Qnil;
+    return make_ruby_object(doc, RTEST(symbolize_keys));
 }
 
-static VALUE rb_fast_jsonparser_load_many(VALUE self, VALUE arg)
+static VALUE rb_fast_jsonparser_load_many(VALUE self, VALUE arg, VALUE symbolize_keys, VALUE batch_size)
 {
     Check_Type(arg, T_STRING);
+    Check_Type(batch_size, T_FIXNUM);
 
-    dom::parser parser;
-    auto [docs, error] = parser.load_many(RSTRING_PTR(arg));
-    if (error == SUCCESS)
+    try
     {
+        dom::parser parser;
+        auto [docs, error] = parser.load_many(RSTRING_PTR(arg), FIX2INT(batch_size));
+        if (error != SUCCESS)
+        {
+            rb_raise(rb_eFastJsonparserParseError, "%s", error_message(error));
+        }
+
         for (dom::element doc : docs)
         {
-            if (rb_block_given_p())
-            {
-                rb_yield(make_ruby_object(doc));
-            }
+            rb_yield(make_ruby_object(doc, RTEST(symbolize_keys)));
         }
+
         return Qnil;
     }
-    rb_raise(rb_eFastJsonparserParseError, "parse error");
-    return Qnil;
+    catch (simdjson::simdjson_error error)
+    {
+        rb_raise(rb_eFastJsonparserUnknownError, "%s", error.what());
+    }
 }
 
 extern "C"
@@ -119,10 +125,15 @@ extern "C"
 
     void Init_fast_jsonparser(void)
     {
-        rb_mFastJsonparser = rb_define_module("FastJsonparser");
-        rb_eFastJsonparserParseError = rb_define_class_under(rb_mFastJsonparser, "ParseError", rb_eStandardError);
-        rb_define_module_function(rb_mFastJsonparser, "parse", reinterpret_cast<VALUE (*)(...)>(rb_fast_jsonparser_parse), 1);
-        rb_define_module_function(rb_mFastJsonparser, "load", reinterpret_cast<VALUE (*)(...)>(rb_fast_jsonparser_load), 1);
-        rb_define_module_function(rb_mFastJsonparser, "load_many", reinterpret_cast<VALUE (*)(...)>(rb_fast_jsonparser_load_many), 1);
+        VALUE rb_mFastJsonparser = rb_const_get(rb_cObject, rb_intern("FastJsonparser"));
+
+        rb_define_module_function(rb_mFastJsonparser, "_parse", reinterpret_cast<VALUE (*)(...)>(rb_fast_jsonparser_parse), 2);
+        rb_define_module_function(rb_mFastJsonparser, "_load", reinterpret_cast<VALUE (*)(...)>(rb_fast_jsonparser_load), 2);
+        rb_define_module_function(rb_mFastJsonparser, "_load_many", reinterpret_cast<VALUE (*)(...)>(rb_fast_jsonparser_load_many), 3);
+
+        rb_eFastJsonparserParseError = rb_const_get(rb_mFastJsonparser, rb_intern("ParseError"));
+        rb_global_variable(&rb_eFastJsonparserParseError);
+        rb_eFastJsonparserUnknownError = rb_const_get(rb_mFastJsonparser, rb_intern("UnknownError"));
+        rb_global_variable(&rb_eFastJsonparserUnknownError);
     }
 }

data/lib/fast_jsonparser.rb

@@ -1,8 +1,36 @@
+# frozen_string_literal: true
+
 require "fast_jsonparser/version"
 
 module FastJsonparser
-  class Error < StandardError; end
-  # Your code goes here...
-end
+  Error = Class.new(StandardError)
+  ParseError = Class.new(Error)
+  UnknownError = Class.new(Error)
+  BatchSizeTooSmall = Class.new(Error)
+
+  DEFAULT_BATCH_SIZE = 1_000_000 # from include/simdjson/dom/parser.h
+
+  class << self
+    def parse(source, symbolize_keys: true)
+      _parse(source, symbolize_keys)
+    end
 
-require "fast_jsonparser/fast_jsonparser" # loads cpp extension
+    def load(source, symbolize_keys: true)
+      _load(source, symbolize_keys)
+    end
+
+    def load_many(source, symbolize_keys: true, batch_size: DEFAULT_BATCH_SIZE, &block)
+      _load_many(source, symbolize_keys, batch_size, &block)
+    rescue UnknownError => error
+      case error.message
+      when "This parser can't support a document that big"
+        raise BatchSizeTooSmall, "One of the documents was bigger than the batch size (#{batch_size}B), try increasing it."
+      else
+        raise
+      end
+    end
+
+    require "fast_jsonparser/fast_jsonparser" # loads cpp extension
+    private :_parse, :_load, :_load_many
+  end
+end

data/lib/fast_jsonparser/version.rb

@@ -1,3 +1,3 @@
 module FastJsonparser
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fast_jsonparser
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Anil Maurya
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-07-14 00:00:00.000000000 Z
+date: 2020-07-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -104,6 +104,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".travis.yml"
+- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
 - Gemfile.lock
@@ -119,7 +120,6 @@ files:
 - ext/fast_jsonparser/simdjson.h
 - fast_jsonparser.gemspec
 - lib/fast_jsonparser.rb
-- lib/fast_jsonparser/fast_jsonparser.bundle
 - lib/fast_jsonparser/version.rb
 homepage: https://github.com/anilmaurya/fast_jsonparser
 licenses: