osv 0.3.13 → 0.3.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3e44a6bcd337573fcfd8586b5a6989940a8e0e5497f7376a85177eed59223d62
-  data.tar.gz: a577f31562aa3d5806222f2deb4b86f6594ace07bcb7079d80b812449437abbe
+  metadata.gz: 872cf06d1389f45f77b4eefc178cc8462ab165b833ab2c5bf4dc7f92e1c8308e
+  data.tar.gz: 84e6c5d0e03389966b8882a5a73f1698ddee3ed0edae24f2fd5b7f257935a98e
 SHA512:
-  metadata.gz: 346984e4dfd2a01943f874848920aaa0f63d2e59db72d0307e192e3bc2679a60bba1749020b357021ba4285cccba86480b6a1a13f8d2723803df765009126c10
-  data.tar.gz: 71b5a79edf20b0377748ff78ebc9f6d11b077f7f8cdc70248d917df108e4e57ef9e5e1f611524b533ec300db5009a02fde3631ab6eaa739bf45aa705e131ebde
+  metadata.gz: 445581447e8f5ec336da7843af715a5f5fbc298232a24f303a22eebb844f83f65ecc2e85d877a448119adae9e6a5529e377d87399a36e6f070562fa4ce0a11b7
+  data.tar.gz: '08f417b19b0549aa4a3db1538e4be413c5ec8faa3bd18e4c101a6fc3ea3e9496d04c30e39ea8eec9cc0cc3a38f8f83f7c2274e09c75259a26f3609620cf07a80'

data/Cargo.lock

@@ -1,6 +1,6 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
-version = 3
+version = 4
 
 [[package]]
 name = "adler2"
@@ -8,6 +8,19 @@ version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627"
 
+[[package]]
+name = "ahash"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
+dependencies = [
+ "cfg-if",
+ "getrandom",
+ "once_cell",
+ "version_check",
+ "zerocopy",
+]
+
 [[package]]
 name = "aho-corasick"
 version = "1.1.3"
@@ -49,6 +62,15 @@ version = "2.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
 
+[[package]]
+name = "cc"
+version = "1.2.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a012a0df96dd6d06ba9a1b29d6402d1a5d77c6befd2566afdc26e10603dc93d7"
+dependencies = [
+ "shlex",
+]
+
 [[package]]
 name = "cexpr"
 version = "0.6.0"
@@ -127,6 +149,17 @@ version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"
 
+[[package]]
+name = "getrandom"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi",
+]
+
 [[package]]
 name = "glob"
 version = "0.3.1"
@@ -148,6 +181,26 @@ version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"
 
+[[package]]
+name = "jemalloc-sys"
+version = "0.5.4+5.3.0-patched"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac6c1946e1cea1788cbfde01c993b52a10e2da07f4bac608228d1bed20bfebf2"
+dependencies = [
+ "cc",
+ "libc",
+]
+
+[[package]]
+name = "jemallocator"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a0de374a9f8e63150e6f5e8a60cc14c668226d7a347d8aee1a45766e3c4dd3bc"
+dependencies = [
+ "jemalloc-sys",
+ "libc",
+]
+
 [[package]]
 name = "kanal"
 version = "0.1.0-pre8"
@@ -186,6 +239,16 @@ dependencies = [
  "windows-targets",
 ]
 
+[[package]]
+name = "libmimalloc-sys"
+version = "0.1.39"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23aa6811d3bd4deb8a84dde645f943476d13b248d818edcf8ce0b2f37f036b44"
+dependencies = [
+ "cc",
+ "libc",
+]
+
 [[package]]
 name = "lock_api"
 version = "0.4.12"
@@ -237,6 +300,15 @@ version = "2.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
 
+[[package]]
+name = "mimalloc"
+version = "0.1.43"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68914350ae34959d83f732418d51e2427a794055d0b9529f48259ac07af65633"
+dependencies = [
+ "libmimalloc-sys",
+]
+
 [[package]]
 name = "minimal-lexical"
 version = "0.2.1"
@@ -262,19 +334,27 @@ dependencies = [
  "minimal-lexical",
 ]
 
+[[package]]
+name = "once_cell"
+version = "1.20.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"
+
 [[package]]
 name = "osv"
 version = "0.1.0"
 dependencies = [
+ "ahash",
  "csv",
  "flate2",
+ "jemallocator",
  "kanal",
  "magnus 0.7.1",
+ "mimalloc",
  "rb-sys",
  "serde",
  "serde_magnus",
  "thiserror",
- "xxhash-rust",
 ]
 
 [[package]]
@@ -464,6 +544,18 @@ version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83"
 
+[[package]]
+name = "version_check"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
+
+[[package]]
+name = "wasi"
+version = "0.11.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
+
 [[package]]
 name = "windows-targets"
 version = "0.52.6"
@@ -529,7 +621,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
 [[package]]
-name = "xxhash-rust"
-version = "0.8.14"
+name = "zerocopy"
+version = "0.7.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7d48f1b18be023c95e7b75f481cac649d74be7c507ff4a407c55cfb957f7934"
+checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
+dependencies = [
+ "zerocopy-derive",
+]
+
+[[package]]
+name = "zerocopy-derive"
+version = "0.7.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]

data/Gemfile

@@ -2,12 +2,12 @@ source "https://rubygems.org"
 
 gem "rb_sys", "~> 0.9.56"
 gem "rake"
-gem "csv"
 
 # Use local version of osv
 gemspec
 
 group :development, :test do
+  gem "csv"
   gem "minitest", "~> 5.0"
   gem "benchmark-ips", "~> 2.12"
   gem "fastcsv", "~> 0.0.7"

data/README.md

@@ -114,104 +114,62 @@ When `has_headers` is false, hash keys will be generated as `"c0"`, `"c1"`, etc.
 
 ## Performance
 
-This library is faster than the standard Ruby CSV library, and is comparable to the fastest CSV parser gems I've used.
+This library is faster than the standard Ruby CSV library. It's also faster than any other CSV gem I've been able to find.
 
 Here's some unscientific benchmarks. You can find the code in the [benchmark/comparison_benchmark.rb](benchmark/comparison_benchmark.rb) file.
 
-### 10,000 lines
+### 1,000,000 records
 
 ```
-Benchmarking with 100001 lines of data
+🏃  Running benchmarks...
+Benchmarking with 3000001 lines of data
 
-ruby 3.3.6 (2024-11-05 revision 75015d4c1f) [arm64-darwin24]
+ruby 3.3.6 (2024-11-05 revision 75015d4c1f) +YJIT [arm64-darwin24]
 Warming up --------------------------------------
-   OSV - Hash output     1.000 i/100ms
-   CSV - Hash output     1.000 i/100ms
-  OSV - Array output     1.000 i/100ms
-OSV - Direct Open Array output
-                        12.719M i/100ms
-  CSV - Array output     1.000 i/100ms
-FastCSV - Array output
-                         1.000 i/100ms
-      OSV - StringIO     1.000 i/100ms
       CSV - StringIO     1.000 i/100ms
   FastCSV - StringIO     1.000 i/100ms
-       OSV - Gzipped     1.000 i/100ms
-       CSV - Gzipped     1.000 i/100ms
-Calculating -------------------------------------
-   OSV - Hash output      6.722 (±14.9%) i/s  (148.77 ms/i) -     59.000 in  10.074753s
-   CSV - Hash output      1.223 (± 0.0%) i/s  (817.62 ms/i) -     13.000 in  10.788284s
-  OSV - Array output     17.284 (±11.6%) i/s   (57.86 ms/i) -    171.000 in  10.007321s
-OSV - Direct Open Array output
-                        213.629M (±13.5%) i/s    (4.68 ns/i) -      1.921B in  10.005506s
-  CSV - Array output      2.193 (± 0.0%) i/s  (455.93 ms/i) -     22.000 in  10.052607s
-FastCSV - Array output
-                          7.993 (± 0.0%) i/s  (125.11 ms/i) -     80.000 in  10.053729s
-      OSV - StringIO      6.626 (±15.1%) i/s  (150.91 ms/i) -     66.000 in  10.103646s
-      CSV - StringIO      1.478 (± 0.0%) i/s  (676.78 ms/i) -     15.000 in  10.158640s
-  FastCSV - StringIO     17.074 (± 5.9%) i/s   (58.57 ms/i) -    171.000 in  10.059266s
-       OSV - Gzipped      5.639 (± 0.0%) i/s  (177.32 ms/i) -     57.000 in  10.152487s
-       CSV - Gzipped      1.176 (± 0.0%) i/s  (850.19 ms/i) -     12.000 in  10.233398s
-
-Comparison:
-OSV - Direct Open Array output: 213629268.6 i/s
-  OSV - Array output:       17.3 i/s - 12360250.79x  slower
-  FastCSV - StringIO:       17.1 i/s - 12511956.50x  slower
-FastCSV - Array output:        8.0 i/s - 26727225.72x  slower
-   OSV - Hash output:        6.7 i/s - 31780615.83x  slower
-      OSV - StringIO:        6.6 i/s - 32239620.60x  slower
-       OSV - Gzipped:        5.6 i/s - 37881517.48x  slower
-  CSV - Array output:        2.2 i/s - 97400427.87x  slower
-      CSV - StringIO:        1.5 i/s - 144580048.04x  slower
-   CSV - Hash output:        1.2 i/s - 174666591.31x  slower
-       CSV - Gzipped:        1.2 i/s - 181626018.23x  slower
-```
-
-### 1,000,000 lines
-
-```
-Benchmarking with 1000001 lines of data
-
-ruby 3.3.6 (2024-11-05 revision 75015d4c1f) [arm64-darwin24]
-Warming up --------------------------------------
-   OSV - Hash output     1.000 i/100ms
+      OSV - StringIO     1.000 i/100ms
    CSV - Hash output     1.000 i/100ms
-  OSV - Array output     1.000 i/100ms
-OSV - Direct Open Array output
-                         1.000 i/100ms
+   OSV - Hash output     1.000 i/100ms
   CSV - Array output     1.000 i/100ms
+  OSV - Array output     1.000 i/100ms
 FastCSV - Array output
                          1.000 i/100ms
-      OSV - StringIO     1.000 i/100ms
-      CSV - StringIO     1.000 i/100ms
-  FastCSV - StringIO     1.000 i/100ms
+OSV - Direct Open Array output
+                         1.000 i/100ms
        OSV - Gzipped     1.000 i/100ms
+OSV - Gzipped Direct     1.000 i/100ms
+   FastCSV - Gzipped     1.000 i/100ms
        CSV - Gzipped     1.000 i/100ms
 Calculating -------------------------------------
-   OSV - Hash output      0.492 (± 0.0%) i/s     (2.03 s/i) -      5.000 in  10.463278s
-   CSV - Hash output      0.114 (± 0.0%) i/s     (8.75 s/i) -      2.000 in  17.573877s
-  OSV - Array output      1.502 (± 0.0%) i/s  (665.58 ms/i) -     14.000 in  10.217551s
-OSV - Direct Open Array output
-                          1.626 (± 0.0%) i/s  (614.90 ms/i) -     16.000 in  10.190323s
-  CSV - Array output      0.183 (± 0.0%) i/s     (5.46 s/i) -      2.000 in  10.951943s
+      CSV - StringIO      0.079 (± 0.0%) i/s    (12.69 s/i) -      3.000 in  38.139709s
+  FastCSV - StringIO      0.370 (± 0.0%) i/s     (2.71 s/i) -     12.000 in  32.474164s
+      OSV - StringIO      0.635 (± 0.0%) i/s     (1.58 s/i) -     19.000 in  30.772490s
+   CSV - Hash output      0.058 (± 0.0%) i/s    (17.11 s/i) -      2.000 in  34.212335s
+   OSV - Hash output      0.249 (± 0.0%) i/s     (4.01 s/i) -      8.000 in  32.124319s
+  CSV - Array output      0.066 (± 0.0%) i/s    (15.11 s/i) -      2.000 in  30.212137s
+  OSV - Array output      0.665 (± 0.0%) i/s     (1.50 s/i) -     20.000 in  30.813986s
 FastCSV - Array output
-                          0.326 (± 0.0%) i/s     (3.07 s/i) -      4.000 in  12.340605s
-      OSV - StringIO      0.567 (± 0.0%) i/s     (1.76 s/i) -      6.000 in  10.698027s
-      CSV - StringIO      0.141 (± 0.0%) i/s     (7.10 s/i) -      2.000 in  14.237144s
-  FastCSV - StringIO      0.923 (± 0.0%) i/s     (1.08 s/i) -     10.000 in  11.567775s
-       OSV - Gzipped      0.437 (± 0.0%) i/s     (2.29 s/i) -      5.000 in  11.452764s
-       CSV - Gzipped      0.104 (± 0.0%) i/s     (9.64 s/i) -      2.000 in  19.373423s
+                          0.351 (± 0.0%) i/s     (2.85 s/i) -     11.000 in  31.418786s
+OSV - Direct Open Array output
+                          0.713 (± 0.0%) i/s     (1.40 s/i) -     22.000 in  30.938525s
+       OSV - Gzipped      0.506 (± 0.0%) i/s     (1.98 s/i) -     16.000 in  31.709708s
+OSV - Gzipped Direct      0.685 (± 0.0%) i/s     (1.46 s/i) -     21.000 in  31.145435s
+   FastCSV - Gzipped      0.324 (± 0.0%) i/s     (3.09 s/i) -     10.000 in  30.983582s
+       CSV - Gzipped      0.057 (± 0.0%) i/s    (17.69 s/i) -      2.000 in  35.379009s
 
 Comparison:
-OSV - Direct Open Array output:        1.6 i/s
-  OSV - Array output:        1.5 i/s - 1.08x  slower
-  FastCSV - StringIO:        0.9 i/s - 1.76x  slower
-      OSV - StringIO:        0.6 i/s - 2.87x  slower
-   OSV - Hash output:        0.5 i/s - 3.30x  slower
-       OSV - Gzipped:        0.4 i/s - 3.72x  slower
-FastCSV - Array output:        0.3 i/s - 4.99x  slower
-  CSV - Array output:        0.2 i/s - 8.88x  slower
-      CSV - StringIO:        0.1 i/s - 11.55x  slower
-   CSV - Hash output:        0.1 i/s - 14.24x  slower
-       CSV - Gzipped:        0.1 i/s - 15.68x  slower
+OSV - Direct Open Array output:        0.7 i/s
+OSV - Gzipped Direct:        0.7 i/s - 1.04x  slower
+  OSV - Array output:        0.7 i/s - 1.07x  slower
+      OSV - StringIO:        0.6 i/s - 1.12x  slower
+       OSV - Gzipped:        0.5 i/s - 1.41x  slower
+  FastCSV - StringIO:        0.4 i/s - 1.93x  slower
+FastCSV - Array output:        0.4 i/s - 2.03x  slower
+   FastCSV - Gzipped:        0.3 i/s - 2.20x  slower
+   OSV - Hash output:        0.2 i/s - 2.86x  slower
+      CSV - StringIO:        0.1 i/s - 9.05x  slower
+  CSV - Array output:        0.1 i/s - 10.77x  slower
+   CSV - Hash output:        0.1 i/s - 12.20x  slower
+       CSV - Gzipped:        0.1 i/s - 12.61x  slower
 ```

data/Rakefile

@@ -1,21 +1,19 @@
 # frozen_string_literal: true
 
 require "rake/testtask"
-require "rake/extensiontask"
+require "rb_sys/extensiontask"
 
 task default: :test
 
-Rake::ExtensionTask.new("osv") do |c|
-  c.lib_dir = "lib/osv"
-  c.ext_dir = "ext/osv"
-end
+GEMSPEC = Gem::Specification.load("osv.gemspec")
 
-task :dev do
-  ENV["RB_SYS_CARGO_PROFILE"] = "release"
+RbSys::ExtensionTask.new("osv", GEMSPEC) do |ext|
+  ext.lib_dir = "lib/osv"
+  ext.ext_dir = "ext/osv"
 end
 
 Rake::TestTask.new do |t|
-  t.deps << :dev << :compile
+  t.deps << :compile
   t.test_files = FileList[File.expand_path("test/*_test.rb", __dir__)]
   t.libs << "lib"
   t.libs << "test"

data/ext/osv/Cargo.toml

@@ -7,6 +7,7 @@ edition = "2021"
 crate-type = ["cdylib"]
 
 [dependencies]
+ahash = "0.8"
 csv = "^1.3"
 flate2 = "1.0.35"
 kanal = "0.1.0-pre8"
@@ -15,4 +16,9 @@ rb-sys = "^0.9"
 serde = { version = "1.0", features = ["derive"] }
 serde_magnus = "0.8.1"
 thiserror = "2.0"
-xxhash-rust = { version = "0.8.12", features = ["xxh3"] }
+
+[target.'cfg(target_os = "linux")'.dependencies]
+jemallocator = { version = "0.5", features = ["disable_initial_exec_tls"] }
+
+[target.'cfg(not(any(target_os = "linux", target_os = "windows")))'.dependencies]
+mimalloc = { version = "0.1", default-features = false }

data/ext/osv/src/allocator.rs

@@ -0,0 +1,13 @@
+#[cfg(target_os = "linux")]
+use jemallocator::Jemalloc;
+
+#[cfg(not(any(target_os = "linux", target_os = "windows")))]
+use mimalloc::MiMalloc;
+
+#[global_allocator]
+#[cfg(target_os = "linux")]
+static ALLOC: Jemalloc = Jemalloc;
+
+#[global_allocator]
+#[cfg(not(any(target_os = "linux", target_os = "windows")))]
+static ALLOC: MiMalloc = MiMalloc;