kreuzberg 4.0.0.pre.rc.14 → 4.0.0.pre.rc.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1b0f873b7da0609856d3f396a8d43cc30bcd62f7919fa0ec7572f32f990f99f6
-  data.tar.gz: ea902df98f54a593113a999dc08c7497d38008d0df3881a677249453b4ca3886
+  metadata.gz: 1ac94696cb48598d98ae55f75c69c59e1d248577b965a3921e21998ee33d2352
+  data.tar.gz: 684e9f74a5f0d5c2c52677fec3cec493707b084dc77815396b237864dfeded90
 SHA512:
-  metadata.gz: 197bb0ad826ab4362efcff8dc5fded982360d9cb252d5150c786a762d28d3bf98c7d72b6e9334dd40ae1e65712a54c0261dbd71a037f098cafb599642269dae3
-  data.tar.gz: e206c9553e656a00ee7722e7f570475037689e874540b37fe5d9c6353e64ba0de05c82893b5d2ae057d68d3320d3e59c624553b8fff08f2ae3c896506c6c275b
+  metadata.gz: 6ed0b13217aad741e169850f155a28f921a37a41ffa95fb12a733798b49625f7a9db030eae90ddf00ee3e367b5a563a426fa301c8a16604c8ad5ca3ba78432fc
+  data.tar.gz: 6c3acf2fb24f573a65e81fdac91f3735a6e2335c340d79a453d73fb43b63b807a8b9e93bbbba38a8c55550be72f0f503513b142238a1c3965e279d8ed522b3ae

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    kreuzberg (4.0.0.pre.rc.14)
+    kreuzberg (4.0.0.pre.rc.15)
 
 GEM
   remote: https://rubygems.org/
@@ -136,7 +136,6 @@ GEM
     yard (0.9.38)
 
 PLATFORMS
-  arm64-darwin-23
   arm64-darwin-24
   x86_64-linux
 

data/ext/kreuzberg_rb/native/Cargo.lock

@@ -75,56 +75,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "anstream"
-version = "0.6.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a"
-dependencies = [
- "anstyle",
- "anstyle-parse",
- "anstyle-query",
- "anstyle-wincon",
- "colorchoice",
- "is_terminal_polyfill",
- "utf8parse",
-]
-
-[[package]]
-name = "anstyle"
-version = "1.0.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78"
-
-[[package]]
-name = "anstyle-parse"
-version = "0.2.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2"
-dependencies = [
- "utf8parse",
-]
-
-[[package]]
-name = "anstyle-query"
-version = "1.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
-dependencies = [
- "windows-sys 0.61.2",
-]
-
-[[package]]
-name = "anstyle-wincon"
-version = "3.0.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
-dependencies = [
- "anstyle",
- "once_cell_polyfill",
- "windows-sys 0.61.2",
-]
-
 [[package]]
 name = "anyhow"
 version = "1.0.100"
@@ -466,30 +416,15 @@ dependencies = [
  "syn",
 ]
 
-[[package]]
-name = "bit-set"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0481a0e032742109b1133a095184ee93d88f3dc9e0d28a5d033dc77a073f44f"
-dependencies = [
- "bit-vec 0.7.0",
-]
-
 [[package]]
 name = "bit-set"
 version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3"
 dependencies = [
- "bit-vec 0.8.0",
+ "bit-vec",
 ]
 
-[[package]]
-name = "bit-vec"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2c54ff287cfc0a34f38a6b832ea1bd8e448a330b3e40a50859e6488bee07f22"
-
 [[package]]
 name = "bit-vec"
 version = "0.8.0"
@@ -661,25 +596,6 @@ dependencies = [
  "cipher",
 ]
 
-[[package]]
-name = "cbindgen"
-version = "0.29.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "befbfd072a8e81c02f8c507aefce431fe5e7d051f83d48a23ffc9b9fe5a11799"
-dependencies = [
- "clap",
- "heck",
- "indexmap",
- "log",
- "proc-macro2",
- "quote",
- "serde",
- "serde_json",
- "syn",
- "tempfile",
- "toml 0.9.10+spec-1.1.0",
-]
-
 [[package]]
 name = "cc"
 version = "1.2.50"
@@ -780,33 +696,6 @@ dependencies = [
  "libloading 0.8.9",
 ]
 
-[[package]]
-name = "clap"
-version = "4.5.53"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c9e340e012a1bf4935f5282ed1436d1489548e8f72308207ea5df0e23d2d03f8"
-dependencies = [
- "clap_builder",
-]
-
-[[package]]
-name = "clap_builder"
-version = "4.5.53"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d76b5d13eaa18c901fd2f7fca939fefe3a0727a953561fefdf3b2922b8569d00"
-dependencies = [
- "anstream",
- "anstyle",
- "clap_lex",
- "strsim",
-]
-
-[[package]]
-name = "clap_lex"
-version = "0.7.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d"
-
 [[package]]
 name = "cmake"
 version = "0.1.57"
@@ -831,12 +720,6 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
 
-[[package]]
-name = "colorchoice"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
-
 [[package]]
 name = "compact_str"
 version = "0.9.0"
@@ -945,9 +828,9 @@ dependencies = [
 
 [[package]]
 name = "crc"
-version = "3.4.0"
+version = "3.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d"
+checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675"
 dependencies = [
  "crc-catalog",
 ]
@@ -1401,7 +1284,7 @@ version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298"
 dependencies = [
- "bit-set 0.8.0",
+ "bit-set",
  "regex-automata",
  "regex-syntax",
 ]
@@ -1497,17 +1380,6 @@ dependencies = [
  "windows-sys 0.60.2",
 ]
 
-[[package]]
-name = "filetime_creation"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c25b5d475550e559de5b0c0084761c65325444e3b6c9e298af9cefe7a9ef3a5f"
-dependencies = [
- "cfg-if",
- "filetime",
- "windows-sys 0.52.0",
-]
-
 [[package]]
 name = "find-msvc-tools"
 version = "0.1.5"
@@ -2312,12 +2184,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "is_terminal_polyfill"
-version = "1.70.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
-
 [[package]]
 name = "itertools"
 version = "0.12.1"
@@ -2414,7 +2280,7 @@ dependencies = [
 
 [[package]]
 name = "kreuzberg"
-version = "4.0.0-rc.13"
+version = "4.0.0-rc.14"
 dependencies = [
  "ahash",
  "async-trait",
@@ -2467,7 +2333,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_yaml_ng",
- "sevenz-rust",
+ "sevenz-rust2",
  "tar",
  "text-splitter",
  "thiserror 2.0.17",
@@ -2483,30 +2349,16 @@ dependencies = [
  "uuid",
  "whatlang",
  "yake-rust",
- "zip 6.0.0",
-]
-
-[[package]]
-name = "kreuzberg-ffi"
-version = "4.0.0-rc.13"
-dependencies = [
- "async-trait",
- "cbindgen",
- "html-to-markdown-rs",
- "kreuzberg",
- "serde",
- "serde_json",
- "tokio",
+ "zip 7.0.0",
 ]
 
 [[package]]
 name = "kreuzberg-rb"
-version = "4.0.0-rc.14"
+version = "4.0.0-rc.15"
 dependencies = [
  "async-trait",
  "html-to-markdown-rs",
  "kreuzberg",
- "kreuzberg-ffi",
  "magnus",
  "pretty_assertions",
  "rb-sys",
@@ -2516,14 +2368,14 @@ dependencies = [
 
 [[package]]
 name = "kreuzberg-tesseract"
-version = "4.0.0-rc.13"
+version = "4.0.0-rc.14"
 dependencies = [
  "cc",
  "cmake",
  "libc",
  "reqwest",
  "thiserror 2.0.17",
- "zip 6.0.0",
+ "zip 7.0.0",
 ]
 
 [[package]]
@@ -2726,20 +2578,11 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "lzma-rust"
-version = "0.1.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5baab2bbbd7d75a144d671e9ff79270e903957d92fb7386fd39034c709bd2661"
-dependencies = [
- "byteorder",
-]
-
 [[package]]
 name = "lzma-rust2"
-version = "0.13.0"
+version = "0.15.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c60a23ffb90d527e23192f1246b14746e2f7f071cb84476dd879071696c18a4a"
+checksum = "48172246aa7c3ea28e423295dd1ca2589a24617cc4e588bb8cfe177cb2c54d95"
 dependencies = [
  "crc",
  "sha2",
@@ -3076,16 +2919,6 @@ dependencies = [
  "chrono",
 ]
 
-[[package]]
-name = "nt-time"
-version = "0.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2de419e64947cd8830e66beb584acc3fb42ed411d103e3c794dda355d1b374b5"
-dependencies = [
- "chrono",
- "time",
-]
-
 [[package]]
 name = "num-bigint"
 version = "0.4.6"
@@ -3218,12 +3051,6 @@ version = "1.21.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
 
-[[package]]
-name = "once_cell_polyfill"
-version = "1.70.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
-
 [[package]]
 name = "onig"
 version = "6.5.1"
@@ -5042,18 +4869,19 @@ dependencies = [
 ]
 
 [[package]]
-name = "sevenz-rust"
-version = "0.6.1"
+name = "sevenz-rust2"
+version = "0.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26482cf1ecce4540dc782fc70019eba89ffc4d87b3717eb5ec524b5db6fdefef"
+checksum = "611081ec4fc67633b979fc0c24385de90fa60acd18126d796c8758a24294a950"
 dependencies = [
- "bit-set 0.6.0",
- "byteorder",
- "crc",
- "filetime_creation",
+ "aes",
+ "bzip2",
+ "cbc",
+ "crc32fast",
+ "getrandom 0.3.4",
  "js-sys",
- "lzma-rust",
- "nt-time",
+ "lzma-rust2",
+ "ppmd-rust",
  "sha2",
  "wasm-bindgen",
 ]
@@ -5401,19 +5229,6 @@ dependencies = [
  "xattr",
 ]
 
-[[package]]
-name = "tempfile"
-version = "3.23.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16"
-dependencies = [
- "fastrand",
- "getrandom 0.3.4",
- "once_cell",
- "rustix",
- "windows-sys 0.61.2",
-]
-
 [[package]]
 name = "tendril"
 version = "0.4.3"
@@ -6122,12 +5937,6 @@ version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
 
-[[package]]
-name = "utf8parse"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
-
 [[package]]
 name = "uuid"
 version = "1.19.0"
@@ -6829,9 +6638,9 @@ dependencies = [
 
 [[package]]
 name = "zip"
-version = "6.0.0"
+version = "7.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b"
+checksum = "bdd8a47718a4ee5fe78e07667cd36f3de80e7c2bfe727c7074245ffc7303c037"
 dependencies = [
  "aes",
  "arbitrary",
@@ -6840,6 +6649,7 @@ dependencies = [
  "crc32fast",
  "deflate64",
  "flate2",
+ "generic-array",
  "getrandom 0.3.4",
  "hmac",
  "indexmap",

data/ext/kreuzberg_rb/native/Cargo.toml

@@ -7,7 +7,7 @@ rb-sys = { path = "../../../vendor/rb-sys" }
 
 [package]
 name = "kreuzberg-rb"
-version = "4.0.0-rc.14"
+version = "4.0.0-rc.15"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]
@@ -30,7 +30,6 @@ default = []
 [dependencies]
 async-trait = "0.1.89"
 kreuzberg = { path = "../../../vendor/kreuzberg", features = ["full"] }
-kreuzberg-ffi = { path = "../../../vendor/kreuzberg-ffi", features = ["embeddings"] }
 magnus = { git = "https://github.com/matsadler/magnus", rev = "f6db11769efb517427bf7f121f9c32e18b059b38", features = [
     "rb-sys",
 ] }

data/ext/kreuzberg_rb/native/build.rs

@@ -1,5 +1,16 @@
 #[cfg(target_os = "macos")]
 fn main() {
+    if let Ok(cargo_manifest_dir) = std::env::var("CARGO_MANIFEST_DIR") {
+        let lib_path = std::path::Path::new(&cargo_manifest_dir)
+            .parent()
+            .and_then(|p| p.parent())
+            .and_then(|p| p.parent())
+            .and_then(|p| p.parent())
+            .and_then(|p| p.parent())
+            .map(|p| p.join("target/release"))
+            .expect("Failed to construct lib path");
+        println!("cargo:rustc-link-search={}", lib_path.display());
+    }
     println!("cargo:rustc-link-arg=-Wl,-undefined,dynamic_lookup");
     println!("cargo:rustc-link-arg=-Wl,-rpath,@loader_path");
     println!("cargo:rustc-link-arg=-Wl,-rpath,@loader_path/.");
@@ -7,9 +18,35 @@ fn main() {
 
 #[cfg(target_os = "linux")]
 fn main() {
+    if let Ok(cargo_manifest_dir) = std::env::var("CARGO_MANIFEST_DIR") {
+        let lib_path = std::path::Path::new(&cargo_manifest_dir)
+            .parent()
+            .and_then(|p| p.parent())
+            .and_then(|p| p.parent())
+            .and_then(|p| p.parent())
+            .and_then(|p| p.parent())
+            .map(|p| p.join("target/release"))
+            .expect("Failed to construct lib path");
+        println!("cargo:rustc-link-search={}", lib_path.display());
+    }
     println!("cargo:rustc-link-arg=-Wl,-rpath,$ORIGIN");
     println!("cargo:rustc-link-arg=-Wl,-rpath,$ORIGIN/.");
 }
 
-#[cfg(not(any(target_os = "macos", target_os = "linux")))]
+#[cfg(target_os = "windows")]
+fn main() {
+    if let Ok(cargo_manifest_dir) = std::env::var("CARGO_MANIFEST_DIR") {
+        let lib_path = std::path::Path::new(&cargo_manifest_dir)
+            .parent()
+            .and_then(|p| p.parent())
+            .and_then(|p| p.parent())
+            .and_then(|p| p.parent())
+            .and_then(|p| p.parent())
+            .map(|p| p.join("target/release"))
+            .expect("Failed to construct lib path");
+        println!("cargo:rustc-link-search={}", lib_path.display());
+    }
+}
+
+#[cfg(not(any(target_os = "macos", target_os = "linux", target_os = "windows")))]
 fn main() {}

data/lib/kreuzberg/result.rb

@@ -223,7 +223,7 @@ module Kreuzberg
     end
 
     def parse_chunks(chunks_data)
-      return nil if chunks_data.nil?
+      return [] if chunks_data.nil? || chunks_data.empty?
 
       chunks_data.map do |chunk_hash|
         Chunk.new(

data/lib/kreuzberg/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kreuzberg
-  VERSION = '4.0.0-rc.14'
+  VERSION = '4.0.0-rc.15'
 end

data/vendor/Cargo.toml

@@ -1,8 +1,8 @@
 [workspace]
-members = ["kreuzberg", "kreuzberg-ffi", "kreuzberg-tesseract"]
+members = ["kreuzberg", "kreuzberg-tesseract"]
 
 [workspace.package]
-version = "4.0.0-rc.14"
+version = "4.0.0-rc.15"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]

data/vendor/kreuzberg/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzberg"
-version = "4.0.0-rc.14"
+version = "4.0.0-rc.15"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]

data/vendor/kreuzberg/build.rs

@@ -228,7 +228,10 @@ fn ensure_windows_import_library(pdfium_dir: &Path) {
 /// Fetch the latest release version from a GitHub repository
 ///
 /// Uses curl to query the GitHub API and extract the tag_name from the
-/// latest release JSON response. Falls back to "7529" if API call fails.
+/// latest release JSON response. Uses improved JSON parsing with fallback logic.
+///
+/// For WASM (paulocoutinhox/pdfium-lib), falls back to known stable versions.
+/// For non-WASM (bblanchon/pdfium-binaries), uses a different fallback strategy.
 fn get_latest_version(repo: &str) -> String {
     let api_url = format!("https://api.github.com/repos/{}/releases/latest", repo);
 
@@ -238,19 +241,60 @@ fn get_latest_version(repo: &str) -> String {
         && output.status.success()
     {
         let json = String::from_utf8_lossy(&output.stdout);
-        if let Some(start) = json.find("\"tag_name\":") {
-            let after_colon = &json[start + "\"tag_name\":".len()..];
-            if let Some(opening_quote) = after_colon.find('"')
-                && let Some(closing_quote) = after_colon[opening_quote + 1..].find('"')
-            {
-                let tag_start = opening_quote + 1;
-                let tag = &after_colon[tag_start..tag_start + closing_quote];
-                return tag.split('/').next_back().unwrap_or(tag).to_string();
+
+        // Try to extract tag_name from JSON
+        if let Some(tag) = extract_tag_from_json(&json) {
+            return tag;
+        }
+    }
+
+    // Fallback versions based on repository
+    // These are stable versions known to have all required assets
+    if repo.contains("paulocoutinhox") {
+        eprintln!(
+            "cargo:warning=Failed to fetch latest PDFium WASM version from GitHub API, using fallback version 7442b"
+        );
+        "7442b".to_string()
+    } else if repo.contains("bblanchon") {
+        eprintln!(
+            "cargo:warning=Failed to fetch latest PDFium binaries version from GitHub API, using fallback version 7568"
+        );
+        "7568".to_string()
+    } else {
+        eprintln!(
+            "cargo:warning=Failed to fetch latest PDFium version from GitHub API (unknown repository: {})",
+            repo
+        );
+        String::new()
+    }
+}
+
+/// Extract tag_name from GitHub API JSON response
+///
+/// Parses JSON by finding the tag_name field and extracting the value between quotes.
+/// Handles various JSON formatting variations.
+fn extract_tag_from_json(json: &str) -> Option<String> {
+    // Look for "tag_name": "..." pattern
+    if let Some(start) = json.find("\"tag_name\"") {
+        let after_colon = &json[start + "\"tag_name\"".len()..];
+
+        // Skip whitespace and colon
+        let after_colon = after_colon.trim_start();
+        let after_colon = after_colon.strip_prefix(':')?;
+        let after_colon = after_colon.trim_start();
+
+        // Extract value between quotes
+        if let Some(opening_quote) = after_colon.find('"') {
+            let value_start = opening_quote + 1;
+            if let Some(closing_quote) = after_colon[value_start..].find('"') {
+                let tag = &after_colon[value_start..value_start + closing_quote];
+                // Handle releases with '/' in tag (e.g., "chromium/1234")
+                return Some(tag.split('/').next_back().unwrap_or(tag).to_string());
             }
         }
     }
 
-    "7529".to_string()
+    None
 }
 
 /// Get the download URL and library name for the target platform

data/vendor/kreuzberg/src/api/mod.rs

@@ -8,6 +8,8 @@
 //! - `POST /extract` - Extract text from uploaded files (multipart form data)
 //! - `GET /health` - Health check endpoint
 //! - `GET /info` - Server information
+//! - `GET /cache/stats` - Get cache statistics
+//! - `DELETE /cache/clear` - Clear all cached files
 //!
 //! # Examples
 //!
@@ -62,6 +64,12 @@
 //!
 //! # Server info
 //! curl http://localhost:8000/info
+//!
+//! # Cache statistics
+//! curl http://localhost:8000/cache/stats
+//!
+//! # Clear cache
+//! curl -X DELETE http://localhost:8000/cache/clear
 //! ```
 
 mod error;