json_scanner 0.1.1 → 0.3.0

data/ext/json_scanner/json_scanner.h

@@ -3,6 +3,7 @@
 
 #include "ruby.h"
 #include "ruby/intern.h"
+#include "ruby/version.h"
 #include <yajl/yajl_parse.h>
 #include <yajl/yajl_gen.h>
 

data/lib/json_scanner/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JsonScanner
-  VERSION = "0.1.1"
+  VERSION = "0.3.0"
 end

data/spec/extensiontesttask.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require "rake/clean"
+require "rake/extensiontask"
+
+module Rake
+  class ExtensionTestTask < ExtensionTask
+    #
+    # The C files to compile.
+    #
+    attr_accessor :c_spec_files
+
+    #
+    # The folders where includes for the test files are.
+    #
+    # Default: %w{/usr/include /usr/include/google}
+    #
+    attr_accessor :test_includes
+
+    #
+    # The libraries to link against.
+    #
+    # Default: %w{cmockery}
+    #
+    attr_accessor :test_libraries
+
+    #
+    # The folders where the libraries are
+    #
+    # Default: %w{/usr/lib}
+    #
+    attr_accessor :test_lib_folders
+
+    def initialize(*args, &block)
+      super
+      @c_spec_files = []
+      @test_includes = %w[/usr/include /usr/include/google]
+      @test_libraries = %w[cmockery]
+      @test_lib_folders = %w[/usr/lib]
+      init_test_tasks(
+        "#{@tmp_dir}/test", "compile:#{@name}:test",
+        "spec:c:#{@name}", "spec:valgrind:#{@name}", "spec:gdb:#{@name}",
+      )
+    end
+
+    private
+
+    def includes
+      @includes ||= (@test_includes + [
+        ".",
+        "../../#{@ext_dir}",
+        "/usr/include/ruby-#{RUBY_VERSION}",
+        "/usr/include/ruby-#{RUBY_VERSION}/#{RUBY_PLATFORM}",
+      ]).map { |l| "-I#{l}" }.join(" ")
+    end
+
+    def libraries
+      @libraries ||= (@test_libraries + %w[ruby pthread crypto]).map { |l| "-l#{l}" }.join(" ")
+    end
+
+    def lib_folders
+      @lib_folders ||= (@test_lib_folders + %w[/usr/lib .]).map { |l| "-L#{l}" }.join(" ")
+    end
+
+    def compile_tests
+      # compile the test sources
+      FileList["*.c"].each do |cfile|
+        sh "gcc -g #{includes} -c #{cfile}"
+      end
+
+      source_objects = FileList["../#{RUBY_PLATFORM}/#{@name}/#{RUBY_VERSION}/*.o"]
+      # link the executables
+      FileList["*.o"].each do |ofile|
+        sh "gcc -g #{lib_folders} #{libraries} #{source_objects} #{ofile} -o #{ofile.ext}"
+      end
+    end
+
+    def init_compile_task(compile_dir, compile_task)
+      directory compile_dir
+      desc "Compile #{@name} tests"
+      task compile_task => ["compile:#{@name}", compile_dir] do
+        # copy the test files into the compilation folder
+        @c_spec_files.each { |file| cp file, compile_dir }
+
+        # start compilation
+        chdir(compile_dir) { compile_tests }
+      end
+    end
+
+    def init_valgrind_task(compile_dir, compile_task, valgrind_task)
+      desc "Execute valgrind for a #{@name} test"
+      task valgrind_task => [compile_task] do |_t, args|
+        sh "valgrind --num-callers=50 --error-limit=no --partial-loads-ok=yes --undef-value-errors=no " \
+           "--leak-check=full #{compile_dir}/#{args.test}"
+      end
+    end
+
+    def init_gdb_task(compile_dir, compile_task, gdb_task)
+      desc "Execute gdb for a #{@name} test"
+      task gdb_task => [compile_task] do |_t, args|
+        sh "gdb #{compile_dir}/#{args.test}"
+      end
+    end
+
+    def init_test_task(compile_dir, compile_task, test_task)
+      desc "Test #{@name}"
+      task test_task => [compile_task] do |_t, args|
+        if args.test
+          sh "#{compile_dir}/#{args.test}"
+        else
+          FileList["#{compile_dir}/*.o"].each do |ofile|
+            sh ofile.ext.to_s
+          end
+        end
+      end
+    end
+
+    def init_test_tasks(compile_dir, compile_task, test_task, valgrind_task, gdb_task)
+      init_compile_task(compile_dir, compile_task)
+      init_valgrind_task(compile_dir, compile_task, valgrind_task)
+      init_gdb_task(compile_dir, compile_task, gdb_task)
+      init_test_task(compile_dir, compile_task, test_task)
+
+      desc "Test all C extensions"
+      task "spec:c" => [test_task]
+    end
+  end
+end

data/spec/json_scanner_spec.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "spec_helper"
+require "json"
 
 RSpec.describe JsonScanner do
   it "has a version number" do
@@ -8,19 +9,63 @@ RSpec.describe JsonScanner do
   end
 
   it "scans json" do
-    result = described_class.scan('["1", {"a": 2}]', [[0], [1, "a"], []], false)
+    result = described_class.scan('["1", {"a": 2}]', [[0], [1, "a"], []])
     expect(result).to eq([[[1, 4, :string]], [[12, 13, :number]], [[0, 15, :array]]])
-    expect(described_class.scan('"2"', [[]], false)).to eq([[[0, 3, :string]]])
+    expect(described_class.scan('"2"', [[]])).to eq([[[0, 3, :string]]])
     expect(
-      described_class.scan("[0,1,2,3,4,5,6,7]", [[(0..2)], [(4...6)]], false)
+      described_class.scan("[0,1,2,3,4,5,6,7]", [[(0..2)], [(4...6)]]),
     ).to eq(
-      [[[1, 2, :number], [3, 4, :number], [5, 6, :number]], [[9, 10, :number], [11, 12, :number]]]
+      [[[1, 2, :number], [3, 4, :number], [5, 6, :number]], [[9, 10, :number], [11, 12, :number]]],
     )
-    expect(described_class.scan('{"a": 1}', [["a"], []], false)).to eq(
-      [[[6, 7, :number]], [[0, 8, :object]]]
+    expect(described_class.scan('{"a": 1}', [["a"], []])).to eq(
+      [[[6, 7, :number]], [[0, 8, :object]]],
     )
   end
 
+  it "supports 'symbolize_path_keys'" do
+    expect(
+      described_class.scan('{"a": {"b": 1}}', [[:a, "b"]], with_path: true),
+    ).to eq([[[%w[a b], [12, 13, :number]]]])
+    expect(
+      described_class.scan('{"a": {"b": 1}}', [[:a, "b"]], with_path: true, symbolize_path_keys: true),
+    ).to eq([[[%i[a b], [12, 13, :number]]]])
+  end
+
+  it "supports any key selector" do
+    expect(
+      described_class.scan(
+        '[{"a":1,"b":2},{"c":3,"d":4},[5]]',
+        [[described_class::ANY_INDEX, described_class::ANY_KEY]],
+      ),
+    ).to eq(
+      [[[6, 7, :number], [12, 13, :number], [20, 21, :number], [26, 27, :number]]],
+    )
+    expect(
+      described_class.scan(
+        '{"a":[1,2],"b":{"c":3}}',
+        [[described_class::ANY_KEY, described_class::ANY_INDEX]],
+      ),
+    ).to eq(
+      [[[6, 7, :number], [8, 9, :number]]],
+    )
+  end
+
+  it "works with max path len correctly" do
+    expect(
+      described_class.scan('{"a": [1]}', [[], ["a"]]),
+    ).to eq(
+      [[[0, 10, :object]], [[6, 9, :array]]],
+    )
+    expect(
+      described_class.scan('{"a": {"b": 1}}', [[], ["a"]]),
+    ).to eq(
+      [[[0, 15, :object]], [[6, 14, :object]]],
+    )
+    expect(described_class.scan('{"a": 1}', [[]])).to eq([[[0, 8, :object]]])
+    expect(described_class.scan("[[1]]", [[]])).to eq([[[0, 5, :array]]])
+    expect(described_class.scan("[[1]]", [[0]])).to eq([[[1, 4, :array]]])
+  end
+
   it "raises on invalid json" do
     expect do
       begin
@@ -28,10 +73,10 @@ RSpec.describe JsonScanner do
         # TODO: investigate
         # got "munmap_chunk(): invalid pointer" in in console once after
         # JsonScanner.scan '[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]]]]]]]]', [[0,0,0,0,0,0,0]], true + Ctrl+D
-        # (last arg wasn't handled at the time)
-        # but I don't think it's a problem of tht extension or libyajl, it happened at exit and I free everything before
+        # (last arg wasn't handled at the time and was intended for with_path kwarg)
+        # but I don't think it's a problem of the extension or libyajl, it happened at exit and I free everything before
         # `JsonScanner.scan` returns
-        described_class.scan "[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]]]]]]]]", [[0, 0, 0, 0, 0, 0, 0]], false
+        described_class.scan "[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]]]]]]]]", [[0, 0, 0, 0, 0, 0, 0]]
       ensure
         GC.stress = false
       end
@@ -40,26 +85,268 @@ RSpec.describe JsonScanner do
 
   it "allows to select ranges" do
     expect(
-      described_class.scan("[[1,2],[3,4]]", [[described_class::ANY_INDEX, described_class::ANY_INDEX]], false)
+      described_class.scan("[[1,2],[3,4]]", [[described_class::ANY_INDEX, described_class::ANY_INDEX]]),
     ).to eq(
-      [[[2, 3, :number], [4, 5, :number], [8, 9, :number], [10, 11, :number]]]
+      [[[2, 3, :number], [4, 5, :number], [8, 9, :number], [10, 11, :number]]],
     )
     expect(
-      described_class.scan("[[1,2],[3,4]]", [[described_class::ANY_INDEX, (0...1)]], false)
+      described_class.scan("[[1,2],[3,4]]", [[described_class::ANY_INDEX, (0...1)]]),
     ).to eq(
-      [[[2, 3, :number], [8, 9, :number]]]
+      [[[2, 3, :number], [8, 9, :number]]],
     )
   end
 
   it "allows only positive or -1 values" do
     expect do
-      described_class.scan("[[1,2],[3,4]]", [[(0...-1)]], false)
+      described_class.scan("[[1,2],[3,4]]", [[(0...-1)]])
     end.to raise_error ArgumentError
     expect do
-      described_class.scan("[[1,2],[3,4]]", [[(0..-2)]], false)
+      described_class.scan("[[1,2],[3,4]]", [[(0..-2)]])
     end.to raise_error ArgumentError
     expect do
-      described_class.scan("[[1,2],[3,4]]", [[(-42..1)]], false)
+      described_class.scan("[[1,2],[3,4]]", [[(-42..1)]])
     end.to raise_error ArgumentError
   end
+
+  it "allows to configure error messages" do
+    expect do
+      described_class.scan "{1}", []
+    end.to raise_error described_class::ParseError, /invalid object key(?!.*\(right here\))/m
+    expect do
+      described_class.scan "{1}", [], verbose_error: false
+    end.to raise_error described_class::ParseError, /invalid object key(?!.*\(right here\))/m
+    expect do
+      described_class.scan "{1}", [], verbose_error: true
+    end.to raise_error described_class::ParseError, /invalid object key(?=.*\(right here\))/m
+    expect do
+      described_class.scan("[0, 42,", [[(1..-1)]], verbose_error: true)
+    end.to raise_error described_class::ParseError, /parse error: premature EOF.*\[0, 42,.*\(right here\) ------\^/m
+  end
+
+  it "includes bytes consumed in the exception" do
+    expect do
+      described_class.scan("[[1,2],,[3,4]]", [])
+    end.to(
+      raise_error(described_class::ParseError) do |exc|
+        expect(exc.bytes_consumed).to eq(8)
+      end,
+    )
+  end
+
+  it "allows to return an actual path to the element" do
+    with_path_expected_res = [
+      # result for first matcher, each element array of two items:
+      # array of path elements and 3-element array start,end,type
+      [[[0], [1, 6, :array]], [[1], [7, 12, :array]]],
+      [
+        [[0, 0], [2, 3, :number]], [[0, 1], [4, 5, :number]],
+        [[1, 0], [8, 9, :number]], [[1, 1], [10, 11, :number]],
+      ],
+    ]
+    params = [
+      "[[1,2],[3,4]]",
+      [
+        [described_class::ANY_INDEX],
+        [described_class::ANY_INDEX, described_class::ANY_INDEX],
+      ],
+    ]
+    expect(described_class.scan(*params, with_path: true)).to eq(with_path_expected_res)
+    expect(described_class.scan(*params, true)).to eq(with_path_expected_res)
+    expect(
+      described_class.scan(*params, false, with_path: true),
+    ).to eq(with_path_expected_res)
+  end
+
+  it "ignores reqular flag if kwarg is given" do
+    expect(
+      described_class.scan(
+        "[[1,2],[3,4]]",
+        [
+          [described_class::ANY_INDEX],
+          [described_class::ANY_INDEX, described_class::ANY_INDEX],
+        ],
+        true, with_path: false,
+      ),
+    ).to eq(
+      [
+        # result for first matcher, each element 3-element array start,end,type
+        [[1, 6, :array], [7, 12, :array]],
+        [
+          [2, 3, :number], [4, 5, :number],
+          [8, 9, :number], [10, 11, :number],
+        ],
+      ],
+    )
+  end
+
+  it "allows to pass config as a hash" do
+    expect(
+      described_class.scan("[1]", [[0]], { with_path: true }),
+    ).to eq(
+      [
+        [[[0], [1, 2, :number]]],
+      ],
+    )
+  end
+
+  it "allows to configure yajl" do
+    expect(
+      described_class.scan("[1]____________", [[0]], { allow_trailing_garbage: true }),
+    ).to eq([[[1, 2, :number]]])
+    expect(
+      described_class.scan(
+        '["1", {"a": /* comment */ 2}]____________', [[1, "a"]],
+        { allow_trailing_garbage: true, allow_comments: true },
+      ),
+    ).to eq([[[26, 27, :number]]])
+    expect(
+      described_class.scan(
+        '[{"a": /* comment */ 1}]_________', [[]],
+        { allow_comments: true, allow_trailing_garbage: true },
+      ),
+    ).to eq([[[0, 24, :array]]])
+  end
+
+  it "works with utf-8" do
+    json = '{"ルビー": ["Руби"]}'.encode(Encoding::UTF_8)
+    expect(described_class.scan(json, [[]])).to eq([[[0, json.bytesize, :object]]])
+    res = described_class.scan(json, [["ルビー", 0]])
+    expect(res).to eq([[[15, 25, :string]]])
+    elem = res.first.first
+    expect(JSON.parse(json.byteslice(elem[0]...elem[1]), quirks_mode: true)).to eq("Руби")
+  end
+
+  it "raises exceptions in utf-8" do
+    bad_json = '{"ルビー": ["Руби" 1]}'.encode(Encoding::UTF_8)
+    expect do
+      described_class.scan(bad_json, [[]], verbose_error: true)
+      # Checks encoding
+    end.to raise_error(described_class::ParseError, Regexp.new(Regexp.escape(bad_json)))
+  end
+
+  it "works with different encodings" do
+    # TODO: encoding validation
+    json = '{"a": 1}'.encode(Encoding::UTF_32LE)
+    expect do
+      described_class.scan(json, [[]])
+    end.to raise_error(described_class::ParseError)
+  end
+
+  context "with yajl params" do
+    it "supports 'allow_comments'" do
+      params = ["[0, /* answer */ 42, 0]", [[(1..-1)]]]
+      expect(described_class.scan(*params, allow_comments: true)).to eq(
+        [[[17, 19, :number], [21, 22, :number]]],
+      )
+      expect do
+        described_class.scan(*params)
+      end.to raise_error(described_class::ParseError)
+    end
+
+    it "supports 'dont_validate_strings'" do
+      params = ["\"\x81\x83\"", [[]]]
+      expect(described_class.scan(*params, dont_validate_strings: true)).to eq(
+        [[[0, 4, :string]]],
+      )
+      expect do
+        described_class.scan(*params)
+      end.to raise_error(described_class::ParseError)
+      params = ["{\"\x81\x83\": 42}", [[JsonScanner::ANY_KEY]]]
+      expect(described_class.scan(*params, dont_validate_strings: true, with_path: true)).to eq(
+        [[[["\x81\x83".dup.force_encoding(Encoding::BINARY)], [7, 9, :number]]]],
+      )
+      expect do
+        described_class.scan(*params, with_path: true)
+      end.to raise_error(described_class::ParseError)
+    end
+
+    it "supports 'allow_trailing_garbage'" do
+      params = ["[0, 42, 0]garbage", [[(1..-1)]]]
+      expect(described_class.scan(*params, allow_trailing_garbage: true)).to eq(
+        [[[4, 6, :number], [8, 9, :number]]],
+      )
+      expect do
+        described_class.scan(*params)
+      end.to raise_error(described_class::ParseError)
+    end
+
+    it "supports 'allow_multiple_values'" do
+      params = ["[0, 42, 0]  [0, 34]", [[(1..-1)]]]
+      expect(described_class.scan(*params, allow_multiple_values: true)).to eq(
+        [[[4, 6, :number], [8, 9, :number], [16, 18, :number]]],
+      )
+      expect do
+        described_class.scan(*params)
+      end.to raise_error(described_class::ParseError)
+    end
+
+    it "handles multiple top-level values correctly with 'allow_multiple_values'" do
+      expect(described_class.scan("[0, 42, 0]  [0, 34]", [[]], allow_multiple_values: true)).to eq(
+        [[[0, 10, :array], [12, 19, :array]]],
+      )
+      expect(described_class.scan('{"42": 34}  [0, 34]', [[]], allow_multiple_values: true)).to eq(
+        [[[0, 10, :object], [12, 19, :array]]],
+      )
+      expect(described_class.scan('[0, 42, 0]  {"42": 34}', [[]], allow_multiple_values: true)).to eq(
+        [[[0, 10, :array], [12, 22, :object]]],
+      )
+      expect(described_class.scan('{"42": 34}  {"0": 34}', [[]], allow_multiple_values: true)).to eq(
+        [[[0, 10, :object], [12, 21, :object]]],
+      )
+    end
+
+    it "supports 'allow_partial_values'" do
+      params = ["[0, 42, 0,", [[(1..-1)]]]
+      expect(described_class.scan(*params, allow_partial_values: true)).to eq(
+        [[[4, 6, :number], [8, 9, :number]]],
+      )
+      expect do
+        described_class.scan(*params)
+      end.to raise_error(described_class::ParseError)
+      expect(described_class.scan("[0, 42, 0", [[(1..-1)]], allow_partial_values: true)).to eq(
+        [[[4, 6, :number], [8, 9, :number]]],
+      )
+      expect(described_class.scan("[0, 42, true", [[(1..-1)]], allow_partial_values: true)).to eq(
+        [[[4, 6, :number], [8, 12, :boolean]]],
+      )
+    end
+  end
+
+  describe described_class::Config do
+    it "saves state" do
+      key = "abracadabra".dup
+      conf = described_class.new [[], [key]]
+      key["cad"] = 0.chr
+      key = nil # rubocop:disable Lint/UselessAssignment
+      GC.start
+      expect(
+        10.times.map do
+          JsonScanner.scan '{"abracadabra": 10}', conf, with_path: true
+        end.uniq,
+      ).to eq([[[[[], [0, 19, :object]]], [[["abracadabra"], [16, 18, :number]]]]])
+      expect(
+        10.times.map do
+          JsonScanner.scan '{"abracadabra": 10}', conf
+        end.uniq,
+      ).to eq([[[[0, 19, :object]], [[16, 18, :number]]]])
+    end
+
+    it "re-raises exceptions" do
+      expect do
+        described_class.new [[(0...-1)]]
+      end.to raise_error ArgumentError
+      expect do
+        described_class.new [[(0..-2)]]
+      end.to raise_error ArgumentError
+      expect do
+        described_class.new [[(-42..1)]]
+      end.to raise_error ArgumentError
+    end
+
+    it "supports inspect" do
+      expect(
+        described_class.new([[], ["abracadabra", JsonScanner::ANY_INDEX], [42, JsonScanner::ANY_KEY]]).inspect,
+      ).to eq("#<JsonScanner::Config [[], ['abracadabra', (0..9223372036854775807)], [42, ('*'..'*')]]>")
+    end
+  end
 end

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: json_scanner
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.3.0
 platform: ruby
 authors:
 - uvlad7
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-12-16 00:00:00.000000000 Z
+date: 2025-08-14 00:00:00.000000000 Z
 dependencies: []
-description: This gem uses yajl lib to scan a json string and allows you to parse
+description: This gem uses the yajl lib to scan a JSON string and allows you to parse
   pieces of it
 email:
 - uvlad7@gmail.com
@@ -26,6 +26,8 @@ files:
 - lib/json_scanner.rb
 - lib/json_scanner/version.rb
 - sig/json_scanner.rbs
+- spec/extensiontesttask.rb
+- spec/json_scanner_spec.c
 - spec/json_scanner_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/uvlad7/json_scanner