kreuzberg 4.10.2 → 4.10.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ext/kreuzberg_rb/native/.cargo/config.toml +12 -12
- data/ext/kreuzberg_rb/native/Cargo.toml +7 -7
- data/lib/kreuzberg/version.rb +1 -1
- data/vendor/Cargo.toml +17 -17
- data/vendor/kreuzberg/Cargo.toml +28 -28
- data/vendor/kreuzberg/README.md +1 -1
- data/vendor/kreuzberg/src/core/config/extraction/loaders.rs +76 -3
- data/vendor/kreuzberg/src/core/config/tree_sitter.rs +1 -0
- data/vendor/kreuzberg/src/extraction/doc/mod.rs +41 -2
- data/vendor/kreuzberg/src/extraction/docx/parser.rs +5 -0
- data/vendor/kreuzberg/src/extractors/docbook.rs +5 -0
- data/vendor/kreuzberg/src/extractors/rst.rs +5 -0
- data/vendor/kreuzberg/src/llm/vlm_embeddings.rs +1 -4
- data/vendor/kreuzberg/tests/typst_behavioral_tests.rs +2 -0
- data/vendor/kreuzberg-ffi/Cargo.toml +7 -7
- data/vendor/kreuzberg-ffi/cbindgen.toml +5 -5
- data/vendor/kreuzberg-ffi/kreuzberg.h +2 -2
- data/vendor/kreuzberg-paddle-ocr/Cargo.toml +4 -4
- data/vendor/kreuzberg-pdfium-render/Cargo.toml +1 -1
- data/vendor/kreuzberg-tesseract/Cargo.toml +3 -3
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 264aa242b21a61576262606d663b454d3db6d03cc524853a4396a801b603d8c3
|
|
4
|
+
data.tar.gz: f706609c1d6ed2a6d6b130513e33c33778213587e9d7b8a318ff87d67900150e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: d9a9951a204651a8768ff6751615a6311e573342c43ef2a9825efdc07b2c797fab27dee06074f31603f0b087e8ba539dba93041192264ad2875be0cf41b7c420
|
|
7
|
+
data.tar.gz: 70a4e53e38bf114b92d0fbf6dc16547829dd1813aeb3269ebb0dea3ec436f29a7a687d330fc093a938f724327209c452de7d3444732602917af2e64a766413f8
|
|
@@ -3,18 +3,18 @@ rustflags = ["-A", "unpredictable-function-pointer-comparisons", "-A", "fn_ptr_e
|
|
|
3
3
|
|
|
4
4
|
[target.x86_64-pc-windows-gnu]
|
|
5
5
|
rustflags = [
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
6
|
+
"-A",
|
|
7
|
+
"unpredictable-function-pointer-comparisons",
|
|
8
|
+
"-A",
|
|
9
|
+
"fn_ptr_eq",
|
|
10
|
+
"-C",
|
|
11
|
+
"link-arg=-lbcrypt",
|
|
12
|
+
"-C",
|
|
13
|
+
"link-arg=-lcrypt32",
|
|
14
|
+
"-C",
|
|
15
|
+
"link-arg=-lws2_32",
|
|
16
|
+
"-C",
|
|
17
|
+
"link-arg=-luserenv",
|
|
18
18
|
]
|
|
19
19
|
|
|
20
20
|
[target.x86_64-pc-windows-msvc]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "kreuzberg-rb"
|
|
3
|
-
version = "4.10.
|
|
3
|
+
version = "4.10.3"
|
|
4
4
|
edition = "2024"
|
|
5
5
|
rust-version = "1.91"
|
|
6
6
|
authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]
|
|
@@ -22,7 +22,7 @@ default = ["embeddings"]
|
|
|
22
22
|
embeddings = ["kreuzberg/embeddings"]
|
|
23
23
|
|
|
24
24
|
[dependencies]
|
|
25
|
-
async-trait = "0.1.
|
|
25
|
+
async-trait = "0.1.92"
|
|
26
26
|
kreuzberg = { path = "../../../vendor/kreuzberg", default-features = false, features = [
|
|
27
27
|
"pdf",
|
|
28
28
|
"excel",
|
|
@@ -49,13 +49,13 @@ kreuzberg-ffi = { path = "../../../vendor/kreuzberg-ffi" }
|
|
|
49
49
|
magnus = { git = "https://github.com/matsadler/magnus", rev = "f6db11769efb517427bf7f121f9c32e18b059b38", features = [
|
|
50
50
|
"rb-sys",
|
|
51
51
|
] }
|
|
52
|
-
rb-sys = { version = "0.9.
|
|
52
|
+
rb-sys = { version = "0.9.130", default-features = false, features = [
|
|
53
53
|
"stable-api-compiled-fallback",
|
|
54
54
|
] }
|
|
55
|
-
serde_json = "1.0.
|
|
56
|
-
toml = "1.1.
|
|
55
|
+
serde_json = "1.0.151"
|
|
56
|
+
toml = "1.1.5"
|
|
57
57
|
serde_yaml_ng = "0.10"
|
|
58
|
-
tokio = { version = "1.
|
|
58
|
+
tokio = { version = "1.53.1", features = [
|
|
59
59
|
"rt",
|
|
60
60
|
"rt-multi-thread",
|
|
61
61
|
"macros",
|
|
@@ -65,7 +65,7 @@ tokio = { version = "1.52.3", features = [
|
|
|
65
65
|
"time",
|
|
66
66
|
"io-util",
|
|
67
67
|
] }
|
|
68
|
-
html-to-markdown-rs = { version = "3.
|
|
68
|
+
html-to-markdown-rs = { version = "3.12.0", default-features = false }
|
|
69
69
|
|
|
70
70
|
[dev-dependencies]
|
|
71
71
|
pretty_assertions = "1.4"
|
data/lib/kreuzberg/version.rb
CHANGED
data/vendor/Cargo.toml
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
members = ["kreuzberg", "kreuzberg-ffi", "kreuzberg-tesseract", "kreuzberg-paddle-ocr", "kreuzberg-pdfium-render"]
|
|
3
3
|
|
|
4
4
|
[workspace.package]
|
|
5
|
-
version = "4.10.
|
|
5
|
+
version = "4.10.3"
|
|
6
6
|
edition = "2024"
|
|
7
7
|
rust-version = "1.91"
|
|
8
8
|
authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]
|
|
@@ -12,9 +12,9 @@ homepage = "https://kreuzberg.dev"
|
|
|
12
12
|
|
|
13
13
|
[workspace.dependencies]
|
|
14
14
|
ahash = { version = "0.8.12", features = ["serde"] }
|
|
15
|
-
anyhow = "1.0.
|
|
16
|
-
async-trait = "0.1.
|
|
17
|
-
base64 = "0.
|
|
15
|
+
anyhow = "1.0.104"
|
|
16
|
+
async-trait = "0.1.92"
|
|
17
|
+
base64 = "0.23.1"
|
|
18
18
|
blake3 = "1"
|
|
19
19
|
bytes = { version = "1", features = ["serde"] }
|
|
20
20
|
cfb = "0.14"
|
|
@@ -28,35 +28,35 @@ dbase = "0.8"
|
|
|
28
28
|
futures = "0.3"
|
|
29
29
|
getrandom = { version = "0.4.3", features = ["wasm_js"] }
|
|
30
30
|
hex = "0.4.3"
|
|
31
|
-
html-to-markdown-rs = { version = "3.
|
|
31
|
+
html-to-markdown-rs = { version = "3.12.0", default-features = false }
|
|
32
32
|
image = { version = "0.25.10", default-features = false }
|
|
33
33
|
itertools = "0.15"
|
|
34
34
|
js-sys = "0.3"
|
|
35
|
-
kreuzberg = { path = "./crates/kreuzberg", version = "4.10.
|
|
36
|
-
kreuzberg-ffi = { path = "./crates/kreuzberg-ffi", version = "4.10.
|
|
35
|
+
kreuzberg = { path = "./crates/kreuzberg", version = "4.10.3", default-features = false }
|
|
36
|
+
kreuzberg-ffi = { path = "./crates/kreuzberg-ffi", version = "4.10.3" }
|
|
37
37
|
lazy_static = "1.5.0"
|
|
38
|
-
libc = "0.2.
|
|
39
|
-
liter-llm = { version = "1.
|
|
38
|
+
libc = "0.2.189"
|
|
39
|
+
liter-llm = { version = "1.19", features = ["native-http"], default-features = false }
|
|
40
40
|
log = "0.4"
|
|
41
|
-
lzma-rust2 = { version = "0.
|
|
41
|
+
lzma-rust2 = { version = "0.20.1" }
|
|
42
42
|
memmap2 = "0.9.11"
|
|
43
43
|
minijinja = "2"
|
|
44
44
|
num_cpus = "1.17.0"
|
|
45
45
|
once_cell = "1.21.4"
|
|
46
|
-
ort = { version = "2.0.0-rc.12", features = ["std", "api-18"], default-features = false }
|
|
46
|
+
ort = { version = "=2.0.0-rc.12", features = ["std", "api-18"], default-features = false }
|
|
47
47
|
parking_lot = "0.12.5"
|
|
48
48
|
pdfium-render = { package = "kreuzberg-pdfium-render", path = "crates/kreuzberg-pdfium-render", version = "4.3" }
|
|
49
49
|
rayon = "1.12.0"
|
|
50
50
|
reqwest = { version = "0.13.4", default-features = false }
|
|
51
|
-
serde = { version = "1.0.
|
|
52
|
-
serde_json = { version = "1.0.
|
|
51
|
+
serde = { version = "1.0.229", features = ["derive"] }
|
|
52
|
+
serde_json = { version = "1.0.151" }
|
|
53
53
|
serde_toon_format = "0.1"
|
|
54
54
|
tempfile = "3.27.0"
|
|
55
|
-
thiserror = "2.0.
|
|
56
|
-
tokio = { version = "1.
|
|
57
|
-
toml = "1.1.
|
|
55
|
+
thiserror = "2.0.20"
|
|
56
|
+
tokio = { version = "1.53.1", features = ["rt", "rt-multi-thread", "macros", "sync", "process", "fs", "time", "io-util"] }
|
|
57
|
+
toml = "1.1.5"
|
|
58
58
|
tracing = "0.1"
|
|
59
|
-
tree-sitter-language-pack = { version = "1.
|
|
59
|
+
tree-sitter-language-pack = { version = "1.16.1", features = ["serde"], default-features = false }
|
|
60
60
|
wasm-bindgen = { version = "0.2", features = ["enable-interning"] }
|
|
61
61
|
wasm-bindgen-futures = "0.4"
|
|
62
62
|
web-sys = { version = "0.3", features = ["Blob", "File", "FileReader", "console", "TextDecoder", "ImageData", "Window", "Response"] }
|
data/vendor/kreuzberg/Cargo.toml
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "kreuzberg"
|
|
3
|
-
version = "4.10.
|
|
3
|
+
version = "4.10.3"
|
|
4
4
|
edition = "2024"
|
|
5
5
|
rust-version = "1.91"
|
|
6
6
|
authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]
|
|
@@ -191,15 +191,15 @@ cli = [
|
|
|
191
191
|
|
|
192
192
|
[dependencies]
|
|
193
193
|
ahash = { version = "0.8.12", features = ["serde"] }
|
|
194
|
-
async-trait = "0.1.
|
|
194
|
+
async-trait = "0.1.92"
|
|
195
195
|
axum = { version = "0.8", features = ["macros", "json", "multipart"], optional = true }
|
|
196
|
-
base64 = "0.
|
|
196
|
+
base64 = "0.23.1"
|
|
197
197
|
biblatex = { version = "0.12", optional = true }
|
|
198
|
-
biblib = { version = "0.
|
|
198
|
+
biblib = { version = "0.8", default-features = false, features = ["ris", "pubmed", "xml"], optional = true }
|
|
199
199
|
bitvec = "1.1"
|
|
200
200
|
blake3 = "1"
|
|
201
201
|
bytes = { version = "1", features = ["serde"] }
|
|
202
|
-
calamine = { version = "0.36.
|
|
202
|
+
calamine = { version = "0.36.1", features = ["dates"], optional = true }
|
|
203
203
|
cfb = { version = "0.14", optional = true }
|
|
204
204
|
chardetng = { version = "1.0.0", optional = true }
|
|
205
205
|
chrono = { version = "0.4", optional = true }
|
|
@@ -208,12 +208,12 @@ dashmap = "6.2"
|
|
|
208
208
|
dbase = { version = "0.8", optional = true }
|
|
209
209
|
dirs = "6"
|
|
210
210
|
encoding_rs = { version = "0.8.35" }
|
|
211
|
-
fast_image_resize = { version = "6.
|
|
211
|
+
fast_image_resize = { version = "6.1.0", optional = true }
|
|
212
212
|
flate2 = { version = "1.1", optional = true }
|
|
213
213
|
hayro-jbig2 = { version = "0.3", default-features = false, features = ["std"], optional = true }
|
|
214
214
|
hayro-jpeg2000 = { version = "0.3", default-features = false, features = ["std", "simd"], optional = true }
|
|
215
215
|
hex = "0.4.3"
|
|
216
|
-
html-to-markdown-rs = { version = "3.
|
|
216
|
+
html-to-markdown-rs = { version = "3.12.0", default-features = false, features = ["inline-images", "metadata"], optional = true }
|
|
217
217
|
image = { version = "0.25.10", default-features = false, features = [
|
|
218
218
|
"png",
|
|
219
219
|
"jpeg",
|
|
@@ -224,17 +224,17 @@ image = { version = "0.25.10", default-features = false, features = [
|
|
|
224
224
|
"pnm",
|
|
225
225
|
"rayon",
|
|
226
226
|
], optional = true }
|
|
227
|
-
indexmap = "2.14.
|
|
228
|
-
infer = "0.
|
|
227
|
+
indexmap = "2.14.2"
|
|
228
|
+
infer = "0.22.0"
|
|
229
229
|
jotdown = "0.10"
|
|
230
230
|
kamadak-exif = { version = "0.6.1", optional = true }
|
|
231
231
|
|
|
232
232
|
kreuzberg-tesseract = { path = "../kreuzberg-tesseract", optional = true }
|
|
233
|
-
libc = "0.2.
|
|
234
|
-
liter-llm = { version = "1.
|
|
233
|
+
libc = "0.2.189"
|
|
234
|
+
liter-llm = { version = "1.19", features = ["native-http"], default-features = false, optional = true }
|
|
235
235
|
log = "0.4"
|
|
236
236
|
lopdf = { version = "0.44.0", default-features = false, features = ["chrono", "rayon"], optional = true }
|
|
237
|
-
mail-parser = { version = "0.11.
|
|
237
|
+
mail-parser = { version = "0.11.8", optional = true }
|
|
238
238
|
memchr = "2.8.3"
|
|
239
239
|
memmap2 = "0.9.11"
|
|
240
240
|
mime_guess = "2.0"
|
|
@@ -245,17 +245,17 @@ once_cell = "1.21.4"
|
|
|
245
245
|
opentelemetry = { version = "0.32", features = ["trace"], optional = true }
|
|
246
246
|
opentelemetry_sdk = { version = "0.32", features = ["rt-tokio"], optional = true }
|
|
247
247
|
org = { version = "0.3", optional = true }
|
|
248
|
-
ort = { version = "2.0.0-rc.12", default-features = false, features = ["std", "ndarray", "api-18"], optional = true }
|
|
248
|
+
ort = { version = "=2.0.0-rc.12", default-features = false, features = ["std", "ndarray", "api-18"], optional = true }
|
|
249
249
|
outlook-pst = { version = "1.2.0", optional = true }
|
|
250
250
|
parking_lot = "0.12.5"
|
|
251
251
|
pastey = "0.2"
|
|
252
252
|
pdfium-render = { package = "kreuzberg-pdfium-render", path = "../kreuzberg-pdfium-render", features = ["thread_safe", "image_latest"], optional = true }
|
|
253
253
|
pulldown-cmark = { version = "0.13" }
|
|
254
|
-
quick-xml = { version = "0.41.0", features = ["serialize"], optional = true }
|
|
254
|
+
quick-xml = { version = "=0.41.0", features = ["serialize"], optional = true }
|
|
255
255
|
rake = { version = "0.3.6", optional = true }
|
|
256
256
|
rayon = "1.12.0"
|
|
257
|
-
regex = "1.13.
|
|
258
|
-
rmcp = { version = "
|
|
257
|
+
regex = "1.13.1"
|
|
258
|
+
rmcp = { version = "3.2.0", features = [
|
|
259
259
|
"server",
|
|
260
260
|
"macros",
|
|
261
261
|
"base64",
|
|
@@ -266,8 +266,8 @@ rmcp = { version = "2.2.0", features = [
|
|
|
266
266
|
rmp-serde = "1.3"
|
|
267
267
|
|
|
268
268
|
roxmltree = { version = "0.21.1", optional = true }
|
|
269
|
-
serde = { version = "1.0.
|
|
270
|
-
serde_json = { version = "1.0.
|
|
269
|
+
serde = { version = "1.0.229", features = ["derive"] }
|
|
270
|
+
serde_json = { version = "1.0.151" }
|
|
271
271
|
serde_toon_format = "0.1"
|
|
272
272
|
serde_yaml_ng = "0.10.0"
|
|
273
273
|
sevenz-rust2 = { version = "0.20.2", optional = true }
|
|
@@ -277,13 +277,13 @@ snap = { version = "1.1", optional = true }
|
|
|
277
277
|
tar = { version = "0.4.46", optional = true }
|
|
278
278
|
tempfile = { version = "3.27.0", optional = true }
|
|
279
279
|
text-splitter = { version = "0.32.0", features = ["markdown"], optional = true }
|
|
280
|
-
thiserror = "2.0.
|
|
280
|
+
thiserror = "2.0.20"
|
|
281
281
|
tiff = { version = "0.11", optional = true }
|
|
282
282
|
# Keep aligned with text-splitter's optional tokenizers integration so ChunkSizer
|
|
283
283
|
# is implemented for the same Tokenizer type used by Kreuzberg.
|
|
284
|
-
tokenizers = { version = "0.23.
|
|
285
|
-
tokio = { version = "1.
|
|
286
|
-
toml = "1.1.
|
|
284
|
+
tokenizers = { version = "0.23.2", optional = true, default-features = false, features = ["http", "fancy-regex"] }
|
|
285
|
+
tokio = { version = "1.53.1", features = ["rt", "rt-multi-thread", "macros", "sync", "process", "fs", "time", "io-util"], optional = true }
|
|
286
|
+
toml = "1.1.5"
|
|
287
287
|
tower = { version = "0.5", features = ["timeout", "limit", "util"], optional = true }
|
|
288
288
|
tower-http = { version = "0.7", features = [
|
|
289
289
|
"cors",
|
|
@@ -310,13 +310,13 @@ hf-hub = { version = "1.0", default-features = false, features = ["blocking", "r
|
|
|
310
310
|
# PaddleOCR via ONNX Runtime - not available on WASM (vendored from paddle-ocr-rs)
|
|
311
311
|
kreuzberg-paddle-ocr = { path = "../kreuzberg-paddle-ocr", optional = true }
|
|
312
312
|
pprof = { version = "0.15.0", features = ["flamegraph"], optional = true }
|
|
313
|
-
ureq = { version = "3.
|
|
313
|
+
ureq = { version = "3.4", default-features = false, features = ["rustls", "json"], optional = true }
|
|
314
314
|
|
|
315
315
|
[target.'cfg(all(target_os = "windows", not(target_arch = "wasm32")))'.dependencies]
|
|
316
316
|
hf-hub = { version = "1.0", default-features = false, features = ["blocking", "rustls-tls"], optional = true }
|
|
317
317
|
# PaddleOCR via ONNX Runtime - not available on WASM (vendored from paddle-ocr-rs)
|
|
318
318
|
kreuzberg-paddle-ocr = { path = "../kreuzberg-paddle-ocr", optional = true }
|
|
319
|
-
ureq = { version = "3.
|
|
319
|
+
ureq = { version = "3.4", default-features = false, features = ["native-tls", "json"], optional = true }
|
|
320
320
|
|
|
321
321
|
[target.'cfg(not(target_arch = "wasm32"))'.dependencies.tree-sitter-language-pack]
|
|
322
322
|
workspace = true
|
|
@@ -327,7 +327,7 @@ optional = true
|
|
|
327
327
|
# Override getrandom to enable js feature for WASM targets
|
|
328
328
|
# This is needed because ring/rustls (via ureq) depend on getrandom without js feature
|
|
329
329
|
getrandom = { version = "0.4.3", features = ["wasm_js"] }
|
|
330
|
-
tree-sitter-language-pack = { version = "1.
|
|
330
|
+
tree-sitter-language-pack = { version = "1.16.1", features = ["serde"], default-features = false, optional = true }
|
|
331
331
|
wasm-bindgen-rayon = { version = "1.3", optional = true }
|
|
332
332
|
|
|
333
333
|
[build-dependencies]
|
|
@@ -335,16 +335,16 @@ pkg-config = "0.3"
|
|
|
335
335
|
tracing = "0.1"
|
|
336
336
|
|
|
337
337
|
[dev-dependencies]
|
|
338
|
-
anyhow = "1.0.
|
|
338
|
+
anyhow = "1.0.104"
|
|
339
339
|
criterion = { version = "0.8", features = ["html_reports"] }
|
|
340
340
|
dotenvy = "0.15"
|
|
341
341
|
filetime = "0.2"
|
|
342
342
|
image = { version = "0.25.10", default-features = false, features = ["png"] }
|
|
343
|
-
jsonschema = "0.
|
|
343
|
+
jsonschema = "0.53"
|
|
344
344
|
serial_test = "3.5.0"
|
|
345
345
|
tar = "0.4.46"
|
|
346
346
|
tempfile = "3.27.0"
|
|
347
|
-
tokio = { version = "1.
|
|
347
|
+
tokio = { version = "1.53.1", features = ["macros", "time"] }
|
|
348
348
|
tokio-test = "0.4"
|
|
349
349
|
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
|
350
350
|
zip = { version = ">=7.0.0, <8.6.0", default-features = false, features = ["deflate-flate2"] }
|
data/vendor/kreuzberg/README.md
CHANGED
|
@@ -60,7 +60,7 @@ High-performance document intelligence library for Rust. Extract text, metadata,
|
|
|
60
60
|
|
|
61
61
|
This is the core Rust library that powers the Python, TypeScript, and Ruby bindings.
|
|
62
62
|
|
|
63
|
-
> **🚀 Version 4.10.
|
|
63
|
+
> **🚀 Version 4.10.3 Release**
|
|
64
64
|
> This is a pre-release version. We invite you to test the library and [report any issues](https://github.com/kreuzberg-dev/kreuzberg-lts/issues) you encounter.
|
|
65
65
|
>
|
|
66
66
|
> **Note**: The Rust crate is not currently published to crates.io for this RC. Use git dependencies or language bindings (Python, TypeScript, Ruby) instead.
|
|
@@ -175,16 +175,36 @@ impl ExtractionConfig {
|
|
|
175
175
|
Ok((*config_arc).clone())
|
|
176
176
|
}
|
|
177
177
|
|
|
178
|
-
/// Discover configuration file
|
|
178
|
+
/// Discover a configuration file.
|
|
179
179
|
///
|
|
180
|
-
///
|
|
180
|
+
/// Resolution order:
|
|
181
|
+
/// 1. `kreuzberg.toml` in the current directory, then each parent up to the
|
|
182
|
+
/// filesystem root (project-local config wins).
|
|
183
|
+
/// 2. The user-level config directory as a fallback, resolved
|
|
184
|
+
/// platform-natively via [`dirs::config_dir`] (same approach as the cache
|
|
185
|
+
/// dir): `$XDG_CONFIG_HOME/kreuzberg/kreuzberg.toml` or
|
|
186
|
+
/// `~/.config/kreuzberg/kreuzberg.toml` on Linux,
|
|
187
|
+
/// `~/Library/Application Support/kreuzberg/kreuzberg.toml` on macOS,
|
|
188
|
+
/// `%APPDATA%\kreuzberg\kreuzberg.toml` on Windows.
|
|
181
189
|
///
|
|
182
190
|
/// # Returns
|
|
183
191
|
///
|
|
184
192
|
/// - `Some(config)` if found
|
|
185
193
|
/// - `None` if no config file found
|
|
186
194
|
pub fn discover() -> Result<Option<Self>> {
|
|
187
|
-
let
|
|
195
|
+
let cwd = std::env::current_dir().map_err(KreuzbergError::Io)?;
|
|
196
|
+
let global = dirs::config_dir().map(|dir| dir.join("kreuzberg").join("kreuzberg.toml"));
|
|
197
|
+
Self::discover_from(&cwd, global.as_deref())
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/// Walk up from `start` looking for `kreuzberg.toml`, then fall back to the
|
|
201
|
+
/// user-level `global` config path if provided.
|
|
202
|
+
///
|
|
203
|
+
/// Split out from [`discover`](Self::discover) so the resolution order is
|
|
204
|
+
/// unit-testable without mutating process-global state (the working
|
|
205
|
+
/// directory or `HOME`).
|
|
206
|
+
fn discover_from(start: &Path, global: Option<&Path>) -> Result<Option<Self>> {
|
|
207
|
+
let mut current = start.to_path_buf();
|
|
188
208
|
|
|
189
209
|
loop {
|
|
190
210
|
let kreuzberg_toml = current.join("kreuzberg.toml");
|
|
@@ -199,6 +219,59 @@ impl ExtractionConfig {
|
|
|
199
219
|
}
|
|
200
220
|
}
|
|
201
221
|
|
|
222
|
+
if let Some(global) = global
|
|
223
|
+
&& global.exists()
|
|
224
|
+
{
|
|
225
|
+
return Ok(Some(Self::from_toml_file(global)?));
|
|
226
|
+
}
|
|
227
|
+
|
|
202
228
|
Ok(None)
|
|
203
229
|
}
|
|
204
230
|
}
|
|
231
|
+
|
|
232
|
+
#[cfg(test)]
|
|
233
|
+
mod discover_tests {
|
|
234
|
+
use super::*;
|
|
235
|
+
use std::fs;
|
|
236
|
+
|
|
237
|
+
const TOML: &str = "[ocr]\nenabled = true\n";
|
|
238
|
+
|
|
239
|
+
#[test]
|
|
240
|
+
fn project_config_takes_precedence_over_global() {
|
|
241
|
+
// Project config has an [ocr] section; the global fallback does not, so
|
|
242
|
+
// `ocr.is_some()` proves the project-local file was the one loaded.
|
|
243
|
+
let project = tempfile::tempdir().expect("tempdir");
|
|
244
|
+
fs::write(project.path().join("kreuzberg.toml"), TOML).expect("write project");
|
|
245
|
+
let global = tempfile::tempdir().expect("tempdir");
|
|
246
|
+
let global_file = global.path().join("kreuzberg.toml");
|
|
247
|
+
fs::write(&global_file, "output_format = \"markdown\"\n").expect("write global");
|
|
248
|
+
|
|
249
|
+
let found = ExtractionConfig::discover_from(project.path(), Some(&global_file))
|
|
250
|
+
.expect("discover")
|
|
251
|
+
.expect("some config");
|
|
252
|
+
assert!(
|
|
253
|
+
found.ocr.is_some(),
|
|
254
|
+
"project-local config must win over the global fallback"
|
|
255
|
+
);
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
#[test]
|
|
259
|
+
fn falls_back_to_global_when_no_project_config() {
|
|
260
|
+
let empty = tempfile::tempdir().expect("tempdir");
|
|
261
|
+
let global = tempfile::tempdir().expect("tempdir");
|
|
262
|
+
let global_file = global.path().join("kreuzberg.toml");
|
|
263
|
+
fs::write(&global_file, TOML).expect("write global");
|
|
264
|
+
|
|
265
|
+
let found = ExtractionConfig::discover_from(empty.path(), Some(&global_file)).expect("discover");
|
|
266
|
+
assert!(found.is_some(), "should fall back to the global config file");
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
#[test]
|
|
270
|
+
fn returns_none_when_no_config_anywhere() {
|
|
271
|
+
let empty = tempfile::tempdir().expect("tempdir");
|
|
272
|
+
let missing = empty.path().join("nope").join("kreuzberg.toml");
|
|
273
|
+
|
|
274
|
+
let found = ExtractionConfig::discover_from(empty.path(), Some(&missing)).expect("discover");
|
|
275
|
+
assert!(found.is_none(), "no project or global config should yield None");
|
|
276
|
+
}
|
|
277
|
+
}
|
|
@@ -8,6 +8,12 @@
|
|
|
8
8
|
use crate::error::{KreuzbergError, Result};
|
|
9
9
|
use std::io::Cursor;
|
|
10
10
|
|
|
11
|
+
/// Zero-based pair index of fcClx/lcbClx in FibRgFcLcb97.
|
|
12
|
+
///
|
|
13
|
+
/// See [MS-DOC] section 2.5.6:
|
|
14
|
+
/// <https://learn.microsoft.com/en-us/openspecs/office_file_formats/ms-doc/0c9df81f-98d0-454e-ad84-b612cd05b1a4>
|
|
15
|
+
const CLX_PAIR_INDEX: usize = 33;
|
|
16
|
+
|
|
11
17
|
/// Result of DOC text extraction.
|
|
12
18
|
pub struct DocExtractionResult {
|
|
13
19
|
/// Extracted text content.
|
|
@@ -116,10 +122,14 @@ fn extract_text_word97(word_doc: &[u8], table_stream: &[u8]) -> Result<String> {
|
|
|
116
122
|
return Err(KreuzbergError::parsing("FIB too short for cbRgFcLcb"));
|
|
117
123
|
}
|
|
118
124
|
|
|
119
|
-
let
|
|
125
|
+
let cb_rg_fc_lcb = u16::from_le_bytes([word_doc[cbrgfclcb_offset], word_doc[cbrgfclcb_offset + 1]]) as usize;
|
|
120
126
|
let rg_fc_lcb_offset = cbrgfclcb_offset + 2;
|
|
121
127
|
|
|
122
|
-
|
|
128
|
+
if cb_rg_fc_lcb <= CLX_PAIR_INDEX {
|
|
129
|
+
return extract_text_contiguous(word_doc, ccp_text);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
let fc_clx_offset = rg_fc_lcb_offset + CLX_PAIR_INDEX * 8;
|
|
123
133
|
let lcb_clx_offset = fc_clx_offset + 4;
|
|
124
134
|
|
|
125
135
|
if word_doc.len() < lcb_clx_offset + 4 {
|
|
@@ -673,4 +683,33 @@ mod tests {
|
|
|
673
683
|
let result = extract_doc_text(b"not a doc file");
|
|
674
684
|
assert!(result.is_err());
|
|
675
685
|
}
|
|
686
|
+
|
|
687
|
+
#[test]
|
|
688
|
+
fn test_extract_word97_reads_clx_from_fib_rg_fc_lcb97_pair_33() {
|
|
689
|
+
let mut word_doc = vec![0_u8; 900];
|
|
690
|
+
word_doc[32..34].copy_from_slice(&14_u16.to_le_bytes());
|
|
691
|
+
word_doc[62..64].copy_from_slice(&22_u16.to_le_bytes());
|
|
692
|
+
word_doc[76..80].copy_from_slice(&1_u32.to_le_bytes());
|
|
693
|
+
|
|
694
|
+
let rg_fc_lcb_offset = 154;
|
|
695
|
+
word_doc[152..154].copy_from_slice(&34_u16.to_le_bytes());
|
|
696
|
+
let clx_pair_offset = rg_fc_lcb_offset + 33 * 8;
|
|
697
|
+
word_doc[clx_pair_offset..clx_pair_offset + 4].copy_from_slice(&8_u32.to_le_bytes());
|
|
698
|
+
word_doc[clx_pair_offset + 4..clx_pair_offset + 8].copy_from_slice(&21_u32.to_le_bytes());
|
|
699
|
+
|
|
700
|
+
let text_offset = 800_usize;
|
|
701
|
+
word_doc[text_offset] = 0xE9;
|
|
702
|
+
|
|
703
|
+
let mut table_stream = vec![0_u8; 29];
|
|
704
|
+
table_stream[8] = 0x02;
|
|
705
|
+
table_stream[9..13].copy_from_slice(&16_u32.to_le_bytes());
|
|
706
|
+
table_stream[13..17].copy_from_slice(&0_u32.to_le_bytes());
|
|
707
|
+
table_stream[17..21].copy_from_slice(&1_u32.to_le_bytes());
|
|
708
|
+
let compressed_fc = 0x4000_0000_u32 | (text_offset as u32 * 2);
|
|
709
|
+
table_stream[23..27].copy_from_slice(&compressed_fc.to_le_bytes());
|
|
710
|
+
|
|
711
|
+
let text = extract_text_word97(&word_doc, &table_stream).expect("failed to read synthetic Word 97 text");
|
|
712
|
+
|
|
713
|
+
assert_eq!(text, "é");
|
|
714
|
+
}
|
|
676
715
|
}
|
|
@@ -9,6 +9,11 @@
|
|
|
9
9
|
//! - Removed file-path based APIs (we only need bytes/reader)
|
|
10
10
|
//! - Added markdown rendering and formatting support (fixes #376)
|
|
11
11
|
|
|
12
|
+
// TODO(kreuzberg-dev/kreuzberg-lts#4): 4 cyclomatic-complexity and 46 size/complexity findings
|
|
13
|
+
// in this file, currently excluded via the quality-debt baseline in poly.toml. Splitting
|
|
14
|
+
// these needs compiler-in-the-loop verification, not a mechanical pass. Delete this
|
|
15
|
+
// note and the file's baseline entry together once it goes green. Help wanted.
|
|
16
|
+
|
|
12
17
|
use ahash::AHashMap;
|
|
13
18
|
use std::io::{Cursor, Read, Seek};
|
|
14
19
|
|
|
@@ -15,6 +15,11 @@
|
|
|
15
15
|
//! - Tables
|
|
16
16
|
//! - Cross-references and links
|
|
17
17
|
|
|
18
|
+
// TODO(kreuzberg-dev/kreuzberg-lts#4): 4 cyclomatic-complexity and 27 size/complexity findings
|
|
19
|
+
// in this file, currently excluded via the quality-debt baseline in poly.toml. Splitting
|
|
20
|
+
// these needs compiler-in-the-loop verification, not a mechanical pass. Delete this
|
|
21
|
+
// note and the file's baseline entry together once it goes green. Help wanted.
|
|
22
|
+
|
|
18
23
|
use crate::Result;
|
|
19
24
|
use crate::core::config::ExtractionConfig;
|
|
20
25
|
use crate::extraction::{cells_to_markdown, cells_to_text};
|
|
@@ -12,6 +12,11 @@
|
|
|
12
12
|
//! - Inline markup (emphasis, strong, code, links)
|
|
13
13
|
//! - Images and references
|
|
14
14
|
|
|
15
|
+
// TODO(kreuzberg-dev/kreuzberg-lts#4): 4 cyclomatic-complexity and 16 size/complexity findings
|
|
16
|
+
// in this file, currently excluded via the quality-debt baseline in poly.toml. Splitting
|
|
17
|
+
// these needs compiler-in-the-loop verification, not a mechanical pass. Delete this
|
|
18
|
+
// note and the file's baseline entry together once it goes green. Help wanted.
|
|
19
|
+
|
|
15
20
|
#[cfg(feature = "office")]
|
|
16
21
|
use crate::Result;
|
|
17
22
|
#[cfg(feature = "office")]
|
|
@@ -64,10 +64,7 @@ pub async fn embed_via_llm<T: AsRef<str>>(
|
|
|
64
64
|
let mut data = response.data;
|
|
65
65
|
data.sort_by_key(|obj| obj.index);
|
|
66
66
|
|
|
67
|
-
let mut embeddings: Vec<Vec<f32>> = data
|
|
68
|
-
.into_iter()
|
|
69
|
-
.map(|obj| obj.embedding.into_iter().map(|v| v as f32).collect())
|
|
70
|
-
.collect();
|
|
67
|
+
let mut embeddings: Vec<Vec<f32>> = data.into_iter().map(|obj| obj.embedding).collect();
|
|
71
68
|
|
|
72
69
|
if normalize {
|
|
73
70
|
for embedding in &mut embeddings {
|
|
@@ -851,6 +851,8 @@ async fn test_typst_empty_heading_edge_case() {
|
|
|
851
851
|
"Should extract regular content even if some headings are empty."
|
|
852
852
|
);
|
|
853
853
|
}
|
|
854
|
+
// An error here still proves the no-panic guarantee this test exists to check; the
|
|
855
|
+
// error value itself is not part of the assertion. ~keep
|
|
854
856
|
Err(_) => {}
|
|
855
857
|
}
|
|
856
858
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "kreuzberg-ffi"
|
|
3
|
-
version = "4.10.
|
|
3
|
+
version = "4.10.3"
|
|
4
4
|
edition = "2024"
|
|
5
5
|
rust-version = "1.91"
|
|
6
6
|
authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]
|
|
@@ -27,15 +27,15 @@ tree-sitter = ["kreuzberg/tree-sitter"]
|
|
|
27
27
|
|
|
28
28
|
[dependencies]
|
|
29
29
|
ahash = { version = "0.8.12", features = ["serde"] }
|
|
30
|
-
async-trait = "0.1.
|
|
30
|
+
async-trait = "0.1.92"
|
|
31
31
|
ctor = "1.0"
|
|
32
|
-
html-to-markdown-rs = { version = "3.
|
|
33
|
-
kreuzberg = { path = "../kreuzberg", version = "4.10.
|
|
32
|
+
html-to-markdown-rs = { version = "3.12.0", default-features = false }
|
|
33
|
+
kreuzberg = { path = "../kreuzberg", version = "4.10.3", default-features = false, features = ["bundled-pdfium", "full"] }
|
|
34
34
|
log = "0.4"
|
|
35
35
|
rayon = { version = "1.12.0", optional = true }
|
|
36
|
-
serde = { version = "1.0.
|
|
37
|
-
serde_json = { version = "1.0.
|
|
38
|
-
tokio = { version = "1.
|
|
36
|
+
serde = { version = "1.0.229", features = ["derive"] }
|
|
37
|
+
serde_json = { version = "1.0.151" }
|
|
38
|
+
tokio = { version = "1.53.1", features = ["rt", "rt-multi-thread", "macros", "sync", "process", "fs", "time", "io-util"] }
|
|
39
39
|
|
|
40
40
|
[build-dependencies]
|
|
41
41
|
cbindgen = "0.29"
|
|
@@ -45,11 +45,11 @@ ExtractionResult = "ExtractionResult"
|
|
|
45
45
|
|
|
46
46
|
[export]
|
|
47
47
|
include = [
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
48
|
+
"kreuzberg_extract_file_sync",
|
|
49
|
+
"kreuzberg_free_string",
|
|
50
|
+
"kreuzberg_free_result",
|
|
51
|
+
"kreuzberg_last_error",
|
|
52
|
+
"kreuzberg_version",
|
|
53
53
|
]
|
|
54
54
|
|
|
55
55
|
[parse]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "kreuzberg-paddle-ocr"
|
|
3
|
-
version = "4.10.
|
|
3
|
+
version = "4.10.3"
|
|
4
4
|
edition = "2024"
|
|
5
5
|
rust-version = "1.91"
|
|
6
6
|
authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]
|
|
@@ -35,7 +35,7 @@ image = { version = "0.25.10", default-features = false }
|
|
|
35
35
|
# Disable rayon - OCR parallelism is handled at higher level
|
|
36
36
|
imageproc = { version = "0.27", default-features = false }
|
|
37
37
|
ndarray = "0.17"
|
|
38
|
-
ort = { version = "2.0.0-rc.12", features = ["ndarray"], default-features = false }
|
|
38
|
+
ort = { version = "=2.0.0-rc.12", features = ["ndarray"], default-features = false }
|
|
39
39
|
# Workspace dependencies
|
|
40
|
-
serde = { version = "1.0.
|
|
41
|
-
thiserror = "2.0.
|
|
40
|
+
serde = { version = "1.0.229", features = ["derive"] }
|
|
41
|
+
thiserror = "2.0.20"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "kreuzberg-tesseract"
|
|
3
|
-
version = "4.10.
|
|
3
|
+
version = "4.10.3"
|
|
4
4
|
edition = "2024"
|
|
5
5
|
rust-version = "1.91"
|
|
6
6
|
authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]
|
|
@@ -35,10 +35,10 @@ static-linking = ["build-tesseract"]
|
|
|
35
35
|
dynamic-linking = []
|
|
36
36
|
|
|
37
37
|
[dependencies]
|
|
38
|
-
thiserror = "2.0.
|
|
38
|
+
thiserror = "2.0.20"
|
|
39
39
|
|
|
40
40
|
[build-dependencies]
|
|
41
|
-
cc = { version = "^1.
|
|
41
|
+
cc = { version = "^1.4.5", optional = true }
|
|
42
42
|
cmake = { version = "0.1.58", optional = true }
|
|
43
43
|
zip = { version = ">=7.0.0", optional = true, default-features = false, features = [
|
|
44
44
|
"deflate-flate2-zlib-rs",
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: kreuzberg
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 4.10.
|
|
4
|
+
version: 4.10.3
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Na'aman Hirschfeld
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-
|
|
11
|
+
date: 2026-09-05 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: rb_sys
|