kreuzberg 4.10.0.pre.rc.15-aarch64-linux → 4.10.1-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +435 -0
- data/lib/kreuzberg/api_proxy.rb +134 -0
- data/lib/kreuzberg/cache_api.rb +67 -0
- data/lib/kreuzberg/cli.rb +55 -0
- data/lib/kreuzberg/cli_proxy.rb +124 -0
- data/lib/kreuzberg/config.rb +1563 -0
- data/lib/kreuzberg/djot_content.rb +216 -0
- data/lib/kreuzberg/document_structure.rb +192 -0
- data/lib/kreuzberg/error_context.rb +136 -0
- data/lib/kreuzberg/errors.rb +130 -0
- data/lib/kreuzberg/extraction_api.rb +400 -0
- data/lib/kreuzberg/mcp_proxy.rb +184 -0
- data/lib/kreuzberg/ocr_backend_protocol.rb +40 -0
- data/lib/kreuzberg/post_processor_protocol.rb +15 -0
- data/lib/kreuzberg/result.rb +857 -0
- data/lib/kreuzberg/setup_lib_path.rb +101 -0
- data/lib/kreuzberg/types.rb +616 -0
- data/lib/kreuzberg/validator_protocol.rb +16 -0
- data/lib/kreuzberg/version.rb +5 -0
- data/lib/kreuzberg.rb +105 -1
- data/lib/kreuzberg_rb.so +0 -0
- data/sig/kreuzberg/internal.rbs +184 -0
- data/sig/kreuzberg.rbs +1859 -0
- data/spec/binding/async_operations_spec.rb +468 -0
- data/spec/binding/batch_operations_spec.rb +670 -0
- data/spec/binding/batch_spec.rb +368 -0
- data/spec/binding/cache_spec.rb +228 -0
- data/spec/binding/cli_proxy_spec.rb +87 -0
- data/spec/binding/cli_spec.rb +55 -0
- data/spec/binding/config_spec.rb +423 -0
- data/spec/binding/config_validation_spec.rb +381 -0
- data/spec/binding/embeddings_spec.rb +134 -0
- data/spec/binding/error_handling_spec.rb +396 -0
- data/spec/binding/error_recovery_spec.rb +492 -0
- data/spec/binding/errors_spec.rb +67 -0
- data/spec/binding/font_config_spec.rb +220 -0
- data/spec/binding/images_spec.rb +156 -0
- data/spec/binding/keywords_extraction_spec.rb +52 -0
- data/spec/binding/pages_extraction_spec.rb +101 -0
- data/spec/binding/plugins/ocr_backend_spec.rb +311 -0
- data/spec/binding/plugins/postprocessor_spec.rb +273 -0
- data/spec/binding/plugins/validator_spec.rb +277 -0
- data/spec/binding/render_spec.rb +91 -0
- data/spec/binding/tables_spec.rb +183 -0
- data/spec/spec_helper.rb +42 -0
- metadata +221 -10
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 92a5b6719c47d9736b83893d7af37e863cf0c0da027122b9a0b77950d2a32c1e
|
|
4
|
+
data.tar.gz: dbe3b460cce6759ebfd0628f4cfb5336c4244b5950b57578a2ca64e53d95b0ef
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 3865d365deae1c064fcbc01ba32b3f17a14f3076d6703a73843cab317f3e29400309e92fe6bd915e291b54e99574406913b9d2ef5d976ad76fccb39808e7b597
|
|
7
|
+
data.tar.gz: 102c0d8ef33f64234e840c7459035baa5bdc3bfe8067160752139e6a3ece182d8c2720f316fde8c05ffea45e86b187601a5660e2db957b416ee5ae216d59ce3b
|
data/README.md
ADDED
|
@@ -0,0 +1,435 @@
|
|
|
1
|
+
# Ruby
|
|
2
|
+
|
|
3
|
+
<div align="center" style="display: flex; flex-wrap: wrap; gap: 8px; justify-content: center; margin: 20px 0;">
|
|
4
|
+
<!-- Language Bindings -->
|
|
5
|
+
<a href="https://crates.io/crates/kreuzberg">
|
|
6
|
+
<img src="https://img.shields.io/crates/v/kreuzberg?label=Rust&color=007ec6" alt="Rust">
|
|
7
|
+
</a>
|
|
8
|
+
<a href="https://hex.pm/packages/kreuzberg">
|
|
9
|
+
<img src="https://img.shields.io/hexpm/v/kreuzberg?label=Elixir&color=007ec6" alt="Elixir">
|
|
10
|
+
</a>
|
|
11
|
+
<a href="https://pypi.org/project/kreuzberg/">
|
|
12
|
+
<img src="https://img.shields.io/pypi/v/kreuzberg?label=Python&color=007ec6" alt="Python">
|
|
13
|
+
</a>
|
|
14
|
+
<a href="https://www.npmjs.com/package/@kreuzberg/node">
|
|
15
|
+
<img src="https://img.shields.io/npm/v/@kreuzberg/node?label=Node.js&color=007ec6" alt="Node.js">
|
|
16
|
+
</a>
|
|
17
|
+
<a href="https://www.npmjs.com/package/@kreuzberg/wasm">
|
|
18
|
+
<img src="https://img.shields.io/npm/v/@kreuzberg/wasm?label=WASM&color=007ec6" alt="WASM">
|
|
19
|
+
</a>
|
|
20
|
+
|
|
21
|
+
<a href="https://central.sonatype.com/artifact/dev.kreuzberg/kreuzberg">
|
|
22
|
+
<img src="https://img.shields.io/maven-central/v/dev.kreuzberg/kreuzberg?label=Java&color=007ec6" alt="Java">
|
|
23
|
+
</a>
|
|
24
|
+
<a href="https://github.com/kreuzberg-dev/kreuzberg-lts/releases">
|
|
25
|
+
<img src="https://img.shields.io/github/v/tag/kreuzberg-dev/kreuzberg-lts?label=Go&color=007ec6&filter=v4.*" alt="Go">
|
|
26
|
+
</a>
|
|
27
|
+
<a href="https://www.nuget.org/packages/Kreuzberg/">
|
|
28
|
+
<img src="https://img.shields.io/nuget/v/Kreuzberg?label=C%23&color=007ec6" alt="C#">
|
|
29
|
+
</a>
|
|
30
|
+
<a href="https://packagist.org/packages/kreuzberg/kreuzberg">
|
|
31
|
+
<img src="https://img.shields.io/packagist/v/kreuzberg/kreuzberg?label=PHP&color=007ec6" alt="PHP">
|
|
32
|
+
</a>
|
|
33
|
+
<a href="https://rubygems.org/gems/kreuzberg">
|
|
34
|
+
<img src="https://img.shields.io/gem/v/kreuzberg?label=Ruby&color=007ec6" alt="Ruby">
|
|
35
|
+
</a>
|
|
36
|
+
<a href="https://kreuzberg-dev.r-universe.dev/kreuzberg">
|
|
37
|
+
<img src="https://img.shields.io/badge/R-kreuzberg-007ec6" alt="R">
|
|
38
|
+
</a>
|
|
39
|
+
<a href="https://github.com/kreuzberg-dev/kreuzberg-lts/pkgs/container/kreuzberg">
|
|
40
|
+
<img src="https://img.shields.io/badge/Docker-007ec6?logo=docker&logoColor=white" alt="Docker">
|
|
41
|
+
</a>
|
|
42
|
+
<a href="https://artifacthub.io/packages/search?repo=kreuzberg">
|
|
43
|
+
<img src="https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/kreuzberg" alt="Artifact Hub">
|
|
44
|
+
</a>
|
|
45
|
+
|
|
46
|
+
<!-- Project Info -->
|
|
47
|
+
<a href="https://github.com/kreuzberg-dev/kreuzberg-lts/blob/main/LICENSE">
|
|
48
|
+
<img src="https://img.shields.io/badge/License-MIT-007ec6" alt="License">
|
|
49
|
+
</a>
|
|
50
|
+
<a href="https://docs.kreuzberg.dev">
|
|
51
|
+
<img src="https://img.shields.io/badge/docs-kreuzberg.dev-007ec6" alt="Documentation">
|
|
52
|
+
</a>
|
|
53
|
+
<a href="https://docs.kreuzberg.dev/demo.html">
|
|
54
|
+
<img src="https://img.shields.io/badge/%E2%96%B6%EF%B8%8F_Live_Demo-007ec6" alt="Live Demo">
|
|
55
|
+
</a>
|
|
56
|
+
<a href="https://huggingface.co/Kreuzberg">
|
|
57
|
+
<img src="https://img.shields.io/badge/%F0%9F%A4%97_Hugging_Face-007ec6" alt="Hugging Face">
|
|
58
|
+
</a>
|
|
59
|
+
</div>
|
|
60
|
+
|
|
61
|
+
> [!NOTE]
|
|
62
|
+
> **Kreuzberg v4 LTS** — the long-term-support line for Kreuzberg **v4**. Active development has moved to
|
|
63
|
+
> **[Xberg](https://github.com/xberg-io/xberg)** (v5+), which receives all new features. This line receives
|
|
64
|
+
> critical bug and security fixes until the **end of 2026, on a best-effort basis**.
|
|
65
|
+
> See the **[migration guide & LTS policy →](https://docs.kreuzberg.dev/lts/)**
|
|
66
|
+
|
|
67
|
+
<div align="center" style="margin-top: 20px;">
|
|
68
|
+
<a href="https://discord.gg/xt9WY3GnKR">
|
|
69
|
+
<img height="22" src="https://img.shields.io/badge/Discord-Join%20our%20community-7289da?logo=discord&logoColor=white" alt="Discord">
|
|
70
|
+
</a>
|
|
71
|
+
</div>
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
Extract text, tables, images, and metadata from 91+ file formats and 248 programming languages including PDF, Office documents, and images. Ruby bindings with idiomatic Ruby API and native performance.
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
## Installation
|
|
78
|
+
|
|
79
|
+
### Package Installation
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
Install via one of the supported package managers:
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
**gem:**
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
gem install kreuzberg
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
**Bundler:**
|
|
96
|
+
|
|
97
|
+
```ruby
|
|
98
|
+
gem 'kreuzberg'
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
### System Requirements
|
|
106
|
+
|
|
107
|
+
- **Ruby 3.2.0 or higher** required (including Ruby 4.x)
|
|
108
|
+
- Ruby 4.0+ is fully supported with no code changes required
|
|
109
|
+
- Optional: [ONNX Runtime](https://github.com/microsoft/onnxruntime/releases) version 1.22.x for embeddings support
|
|
110
|
+
- Optional: [Tesseract OCR](https://github.com/tesseract-ocr/tesseract) for OCR functionality
|
|
111
|
+
|
|
112
|
+
**Ruby 4.0 Compatibility:** Kreuzberg is fully compatible with Ruby 4.0 (released December 25, 2025) and all Ruby 4.x versions. All tests pass with 100% compatibility. The gem compiles without any breaking changes. Key Ruby 4.0 features like Ruby Box, ZJIT compiler, and Ractor improvements work seamlessly with Kreuzberg.
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
## Quick Start
|
|
117
|
+
|
|
118
|
+
### Basic Extraction
|
|
119
|
+
|
|
120
|
+
Extract text, metadata, and structure from any supported document format:
|
|
121
|
+
|
|
122
|
+
```ruby
|
|
123
|
+
require 'kreuzberg'
|
|
124
|
+
|
|
125
|
+
result = Kreuzberg.extract_file_sync('document.pdf')
|
|
126
|
+
|
|
127
|
+
puts "Content:"
|
|
128
|
+
puts result.content
|
|
129
|
+
|
|
130
|
+
puts "\nMetadata:"
|
|
131
|
+
puts "Title: #{result.metadata&.dig('title')}"
|
|
132
|
+
puts "Author: #{result.metadata&.dig('author')}"
|
|
133
|
+
|
|
134
|
+
puts "\nTables found: #{result.tables.length}"
|
|
135
|
+
puts "Images found: #{result.images.length}"
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
### Common Use Cases
|
|
140
|
+
|
|
141
|
+
#### Extract with Custom Configuration
|
|
142
|
+
|
|
143
|
+
Most use cases benefit from configuration to control extraction behavior:
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
**With OCR (for scanned documents):**
|
|
147
|
+
|
|
148
|
+
```ruby
|
|
149
|
+
require 'kreuzberg'
|
|
150
|
+
|
|
151
|
+
ocr_config = Kreuzberg::Config::OCR.new(
|
|
152
|
+
backend: 'tesseract',
|
|
153
|
+
language: 'eng'
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
config = Kreuzberg::Config::Extraction.new(ocr: ocr_config)
|
|
157
|
+
result = Kreuzberg.extract_file_sync('scanned.pdf', config: config)
|
|
158
|
+
|
|
159
|
+
puts "Extracted text from scanned document:"
|
|
160
|
+
puts result.content
|
|
161
|
+
puts "Used OCR backend: tesseract"
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
#### Table Extraction
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
See [Table Extraction Guide](https://docs.kreuzberg.dev/features/table-extraction/) for detailed examples.
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
#### Processing Multiple Files
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
```ruby
|
|
178
|
+
require 'kreuzberg'
|
|
179
|
+
|
|
180
|
+
puts "Kreuzberg version: #{Kreuzberg::VERSION}"
|
|
181
|
+
puts "FFI bindings loaded successfully"
|
|
182
|
+
|
|
183
|
+
result = Kreuzberg.extract_file_sync('sample.pdf')
|
|
184
|
+
puts "Installation verified! Extracted #{result.content.length} characters"
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
#### Async Processing
|
|
192
|
+
|
|
193
|
+
For non-blocking document processing:
|
|
194
|
+
|
|
195
|
+
```ruby
|
|
196
|
+
require 'kreuzberg'
|
|
197
|
+
|
|
198
|
+
config = Kreuzberg::Config::Extraction.new(
|
|
199
|
+
use_cache: true,
|
|
200
|
+
enable_quality_processing: true
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
result = Kreuzberg.extract_file_sync('contract.pdf', config: config)
|
|
204
|
+
|
|
205
|
+
puts "Extracted #{result.content.length} characters"
|
|
206
|
+
puts "Quality score: #{result.quality_score}"
|
|
207
|
+
puts "Processing time: #{result.metadata&.dig('processing_time')}ms"
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
### Next Steps
|
|
216
|
+
|
|
217
|
+
- **[Installation Guide](https://docs.kreuzberg.dev/getting-started/installation/)** - Platform-specific setup
|
|
218
|
+
- **[API Documentation](https://docs.kreuzberg.dev/api/)** - Complete API reference
|
|
219
|
+
- **[Examples & Guides](https://docs.kreuzberg.dev/guides/)** - Full code examples and usage guides
|
|
220
|
+
- **[Configuration Guide](https://docs.kreuzberg.dev/guides/configuration/)** - Advanced configuration options
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
## Features
|
|
225
|
+
|
|
226
|
+
### Supported File Formats (91+)
|
|
227
|
+
|
|
228
|
+
91+ file formats across 8 major categories with intelligent format detection and comprehensive metadata extraction.
|
|
229
|
+
|
|
230
|
+
#### Office Documents
|
|
231
|
+
|
|
232
|
+
| Category | Formats | Capabilities |
|
|
233
|
+
|----------|---------|--------------|
|
|
234
|
+
| **Word Processing** | `.docx`, `.docm`, `.dotx`, `.dotm`, `.dot`, `.odt` | Full text, tables, images, metadata, styles |
|
|
235
|
+
| **Spreadsheets** | `.xlsx`, `.xlsm`, `.xlsb`, `.xls`, `.xla`, `.xlam`, `.xltm`, `.xltx`, `.xlt`, `.ods` | Sheet data, formulas, cell metadata, charts |
|
|
236
|
+
| **Presentations** | `.pptx`, `.pptm`, `.ppsx`, `.potx`, `.potm`, `.pot`, `.ppt` | Slides, speaker notes, images, metadata |
|
|
237
|
+
| **PDF** | `.pdf` | Text, tables, images, metadata, OCR support |
|
|
238
|
+
| **eBooks** | `.epub`, `.fb2` | Chapters, metadata, embedded resources |
|
|
239
|
+
| **Database** | `.dbf` | Table data extraction, field type support |
|
|
240
|
+
| **Hangul** | `.hwp`, `.hwpx` | Korean document format, text extraction |
|
|
241
|
+
|
|
242
|
+
#### Images (OCR-Enabled)
|
|
243
|
+
|
|
244
|
+
| Category | Formats | Features |
|
|
245
|
+
|----------|---------|----------|
|
|
246
|
+
| **Raster** | `.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, `.bmp`, `.tiff`, `.tif` | OCR, table detection, EXIF metadata, dimensions, color space |
|
|
247
|
+
| **Advanced** | `.jp2`, `.jpx`, `.jpm`, `.mj2`, `.jbig2`, `.jb2`, `.pnm`, `.pbm`, `.pgm`, `.ppm` | OCR via hayro-jpeg2000 (pure Rust decoder), JBIG2 support, table detection, format-specific metadata |
|
|
248
|
+
| **Vector** | `.svg` | DOM parsing, embedded text, graphics metadata |
|
|
249
|
+
|
|
250
|
+
#### Web & Data
|
|
251
|
+
|
|
252
|
+
| Category | Formats | Features |
|
|
253
|
+
|----------|---------|----------|
|
|
254
|
+
| **Markup** | `.html`, `.htm`, `.xhtml`, `.xml`, `.svg` | DOM parsing, metadata (Open Graph, Twitter Card), link extraction |
|
|
255
|
+
| **Structured Data** | `.json`, `.yaml`, `.yml`, `.toml`, `.csv`, `.tsv` | Schema detection, nested structures, validation |
|
|
256
|
+
| **Text & Markdown** | `.txt`, `.md`, `.markdown`, `.djot`, `.rst`, `.org`, `.rtf` | CommonMark, GFM, Djot, reStructuredText, Org Mode |
|
|
257
|
+
|
|
258
|
+
#### Email & Archives
|
|
259
|
+
|
|
260
|
+
| Category | Formats | Features |
|
|
261
|
+
|----------|---------|----------|
|
|
262
|
+
| **Email** | `.eml`, `.msg` | Headers, body (HTML/plain), attachments, threading |
|
|
263
|
+
| **Archives** | `.zip`, `.tar`, `.tgz`, `.gz`, `.7z` | File listing, nested archives, metadata |
|
|
264
|
+
|
|
265
|
+
#### Academic & Scientific
|
|
266
|
+
|
|
267
|
+
| Category | Formats | Features |
|
|
268
|
+
|----------|---------|----------|
|
|
269
|
+
| **Citations** | `.bib`, `.biblatex`, `.ris`, `.nbib`, `.enw`, `.csl` | Structured parsing: RIS (structured), PubMed/MEDLINE, EndNote XML (structured), BibTeX, CSL JSON |
|
|
270
|
+
| **Scientific** | `.tex`, `.latex`, `.typst`, `.jats`, `.ipynb`, `.docbook` | LaTeX, Jupyter notebooks, PubMed JATS |
|
|
271
|
+
| **Documentation** | `.opml`, `.pod`, `.mdoc`, `.troff` | Technical documentation formats |
|
|
272
|
+
|
|
273
|
+
#### Code Intelligence (248 Languages)
|
|
274
|
+
|
|
275
|
+
| Feature | Description |
|
|
276
|
+
|---------|-------------|
|
|
277
|
+
| **Structure Extraction** | Functions, classes, methods, structs, interfaces, enums |
|
|
278
|
+
| **Import/Export Analysis** | Module dependencies, re-exports, wildcard imports |
|
|
279
|
+
| **Symbol Extraction** | Variables, constants, type aliases, properties |
|
|
280
|
+
| **Docstring Parsing** | Google, NumPy, Sphinx, JSDoc, RustDoc, and 10+ formats |
|
|
281
|
+
| **Diagnostics** | Parse errors with line/column positions |
|
|
282
|
+
| **Syntax-Aware Chunking** | Split code by semantic boundaries, not arbitrary byte offsets |
|
|
283
|
+
|
|
284
|
+
Powered by [tree-sitter-language-pack](https://github.com/xberg-io/tree-sitter-language-pack) — [documentation](https://docs.tree-sitter-language-pack.xberg.io).
|
|
285
|
+
|
|
286
|
+
**[Complete Format Reference](https://docs.kreuzberg.dev/reference/formats/)**
|
|
287
|
+
|
|
288
|
+
### Key Capabilities
|
|
289
|
+
|
|
290
|
+
- **Text Extraction** - Extract all text content with position and formatting information
|
|
291
|
+
- **Metadata Extraction** - Retrieve document properties, creation date, author, etc.
|
|
292
|
+
- **Table Extraction** - Parse tables with structure and cell content preservation
|
|
293
|
+
- **Image Extraction** - Extract embedded images and render page previews
|
|
294
|
+
- **OCR Support** - Integrate multiple OCR backends for scanned documents
|
|
295
|
+
|
|
296
|
+
- **Async/Await** - Non-blocking document processing with concurrent operations
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
- **Plugin System** - Extensible post-processing for custom text transformation
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
- **Embeddings** - Generate vector embeddings using ONNX Runtime models
|
|
303
|
+
|
|
304
|
+
- **Batch Processing** - Efficiently process multiple documents in parallel
|
|
305
|
+
- **Memory Efficient** - Stream large files without loading entirely into memory
|
|
306
|
+
- **Language Detection** - Detect and support multiple languages in documents
|
|
307
|
+
|
|
308
|
+
- **Code Intelligence** - Extract structure, imports, exports, symbols, and docstrings from [248 programming languages](https://docs.tree-sitter-language-pack.xberg.io) via tree-sitter
|
|
309
|
+
|
|
310
|
+
- **Configuration** - Fine-grained control over extraction behavior
|
|
311
|
+
|
|
312
|
+
### Performance Characteristics
|
|
313
|
+
|
|
314
|
+
| Format | Speed | Memory | Notes |
|
|
315
|
+
|--------|-------|--------|-------|
|
|
316
|
+
| **PDF (text)** | 10-100 MB/s | ~50MB per doc | Fastest extraction |
|
|
317
|
+
| **Office docs** | 20-200 MB/s | ~100MB per doc | DOCX, XLSX, PPTX |
|
|
318
|
+
| **Images (OCR)** | 1-5 MB/s | Variable | Depends on OCR backend |
|
|
319
|
+
| **Archives** | 5-50 MB/s | ~200MB per doc | ZIP, TAR, etc. |
|
|
320
|
+
| **Web formats** | 50-200 MB/s | Streaming | HTML, XML, JSON |
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
## OCR Support
|
|
325
|
+
|
|
326
|
+
Kreuzberg supports multiple OCR backends for extracting text from scanned documents and images:
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
- **Tesseract**
|
|
330
|
+
|
|
331
|
+
- **Paddleocr**
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
### OCR Configuration Example
|
|
335
|
+
|
|
336
|
+
```ruby
|
|
337
|
+
require 'kreuzberg'
|
|
338
|
+
|
|
339
|
+
ocr_config = Kreuzberg::Config::OCR.new(
|
|
340
|
+
backend: 'tesseract',
|
|
341
|
+
language: 'eng'
|
|
342
|
+
)
|
|
343
|
+
|
|
344
|
+
config = Kreuzberg::Config::Extraction.new(ocr: ocr_config)
|
|
345
|
+
result = Kreuzberg.extract_file_sync('scanned.pdf', config: config)
|
|
346
|
+
|
|
347
|
+
puts "Extracted text from scanned document:"
|
|
348
|
+
puts result.content
|
|
349
|
+
puts "Used OCR backend: tesseract"
|
|
350
|
+
```
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
## Async Support
|
|
356
|
+
|
|
357
|
+
This binding provides full async/await support for non-blocking document processing:
|
|
358
|
+
|
|
359
|
+
```ruby
|
|
360
|
+
require 'kreuzberg'
|
|
361
|
+
|
|
362
|
+
config = Kreuzberg::Config::Extraction.new(
|
|
363
|
+
use_cache: true,
|
|
364
|
+
enable_quality_processing: true
|
|
365
|
+
)
|
|
366
|
+
|
|
367
|
+
result = Kreuzberg.extract_file_sync('contract.pdf', config: config)
|
|
368
|
+
|
|
369
|
+
puts "Extracted #{result.content.length} characters"
|
|
370
|
+
puts "Quality score: #{result.quality_score}"
|
|
371
|
+
puts "Processing time: #{result.metadata&.dig('processing_time')}ms"
|
|
372
|
+
```
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
## Plugin System
|
|
378
|
+
|
|
379
|
+
Kreuzberg supports extensible post-processing plugins for custom text transformation and filtering.
|
|
380
|
+
|
|
381
|
+
For detailed plugin documentation, visit [Plugin System Guide](https://docs.kreuzberg.dev/guides/plugins/).
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
## Embeddings Support
|
|
387
|
+
|
|
388
|
+
Generate vector embeddings for extracted text using the built-in ONNX Runtime support. Requires ONNX Runtime installation.
|
|
389
|
+
|
|
390
|
+
**[Embeddings Guide](https://docs.kreuzberg.dev/features/#embeddings)**
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
## Batch Processing
|
|
395
|
+
|
|
396
|
+
Process multiple documents efficiently:
|
|
397
|
+
|
|
398
|
+
```ruby
|
|
399
|
+
require 'kreuzberg'
|
|
400
|
+
|
|
401
|
+
puts "Kreuzberg version: #{Kreuzberg::VERSION}"
|
|
402
|
+
puts "FFI bindings loaded successfully"
|
|
403
|
+
|
|
404
|
+
result = Kreuzberg.extract_file_sync('sample.pdf')
|
|
405
|
+
puts "Installation verified! Extracted #{result.content.length} characters"
|
|
406
|
+
```
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
## Configuration
|
|
412
|
+
|
|
413
|
+
For advanced configuration options including language detection, table extraction, OCR settings, and more:
|
|
414
|
+
|
|
415
|
+
**[Configuration Guide](https://docs.kreuzberg.dev/guides/configuration/)**
|
|
416
|
+
|
|
417
|
+
## Documentation
|
|
418
|
+
|
|
419
|
+
- **[Official Documentation](https://docs.kreuzberg.dev/)**
|
|
420
|
+
- **[API Reference](https://docs.kreuzberg.dev/reference/api-ruby/)**
|
|
421
|
+
- **[Examples & Guides](https://docs.kreuzberg.dev/guides/)**
|
|
422
|
+
|
|
423
|
+
## Contributing
|
|
424
|
+
|
|
425
|
+
Contributions are welcome! See [Contributing Guide](https://github.com/kreuzberg-dev/kreuzberg-lts/blob/main/CONTRIBUTING.md).
|
|
426
|
+
|
|
427
|
+
## License
|
|
428
|
+
|
|
429
|
+
MIT License - see LICENSE file for details.
|
|
430
|
+
|
|
431
|
+
## Support
|
|
432
|
+
|
|
433
|
+
- **Discord Community**: [Join our Discord](https://discord.gg/xt9WY3GnKR)
|
|
434
|
+
- **GitHub Issues**: [Report bugs](https://github.com/kreuzberg-dev/kreuzberg-lts/issues)
|
|
435
|
+
- **Discussions**: [Ask questions](https://github.com/kreuzberg-dev/kreuzberg-lts/discussions)
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "open3"
|
|
4
|
+
|
|
5
|
+
module Kreuzberg
|
|
6
|
+
# @example Start the server
|
|
7
|
+
# @example With block
|
|
8
|
+
module APIProxy
|
|
9
|
+
class Error < Kreuzberg::Errors::Error
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
class MissingBinaryError < Error
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
class ServerError < Error
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# API server instance
|
|
19
|
+
class Server
|
|
20
|
+
attr_reader :port, :host, :pid
|
|
21
|
+
|
|
22
|
+
# Initialize server
|
|
23
|
+
#
|
|
24
|
+
# @param port [Integer] Port to run on (default: 8000)
|
|
25
|
+
# @param host [String] Host to bind to (default: "0.0.0.0")
|
|
26
|
+
#
|
|
27
|
+
def initialize(port: 8000, host: "0.0.0.0")
|
|
28
|
+
@port = port
|
|
29
|
+
@host = host
|
|
30
|
+
@pid = nil
|
|
31
|
+
@process = nil
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# Start the server in the background
|
|
35
|
+
#
|
|
36
|
+
# @return [Integer] Process ID
|
|
37
|
+
# @raise [ServerError] If server fails to start
|
|
38
|
+
#
|
|
39
|
+
def start
|
|
40
|
+
binary = APIProxy.find_api_binary
|
|
41
|
+
@pid = spawn(
|
|
42
|
+
binary.to_s,
|
|
43
|
+
"api",
|
|
44
|
+
"--host",
|
|
45
|
+
@host,
|
|
46
|
+
"--port",
|
|
47
|
+
@port.to_s,
|
|
48
|
+
out: $stdout,
|
|
49
|
+
err: $stderr
|
|
50
|
+
)
|
|
51
|
+
Process.detach(@pid)
|
|
52
|
+
sleep(1)
|
|
53
|
+
@pid
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Stop the server
|
|
57
|
+
#
|
|
58
|
+
# @return [void]
|
|
59
|
+
#
|
|
60
|
+
def stop
|
|
61
|
+
return unless @pid
|
|
62
|
+
|
|
63
|
+
Process.kill("TERM", @pid)
|
|
64
|
+
Process.wait(@pid)
|
|
65
|
+
# rubocop:disable Lint/SuppressedException
|
|
66
|
+
rescue Errno::ESRCH, Errno::ECHILD
|
|
67
|
+
ensure
|
|
68
|
+
@pid = nil
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# Check if server is running
|
|
72
|
+
#
|
|
73
|
+
# @return [Boolean]
|
|
74
|
+
#
|
|
75
|
+
def running?
|
|
76
|
+
return false unless @pid
|
|
77
|
+
|
|
78
|
+
Process.kill(0, @pid)
|
|
79
|
+
true
|
|
80
|
+
rescue Errno::ESRCH, Errno::EPERM
|
|
81
|
+
false
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
module_function
|
|
86
|
+
|
|
87
|
+
# Run server with a block
|
|
88
|
+
#
|
|
89
|
+
# @param port [Integer] Port to run on
|
|
90
|
+
# @param host [String] Host to bind to
|
|
91
|
+
# @yield [Server] Yields server instance
|
|
92
|
+
# @return [Object] Block result
|
|
93
|
+
#
|
|
94
|
+
# @example
|
|
95
|
+
# Kreuzberg::APIProxy.run(port: 8000) do |server|
|
|
96
|
+
# # Make API requests
|
|
97
|
+
# end
|
|
98
|
+
#
|
|
99
|
+
def run(port: 8000, host: "0.0.0.0")
|
|
100
|
+
server = Server.new(port: port, host: host)
|
|
101
|
+
server.start
|
|
102
|
+
yield server
|
|
103
|
+
ensure
|
|
104
|
+
server&.stop
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# Find the API binary
|
|
108
|
+
#
|
|
109
|
+
# @return [Pathname] Path to binary
|
|
110
|
+
# @raise [MissingBinaryError] If not found
|
|
111
|
+
#
|
|
112
|
+
def find_api_binary
|
|
113
|
+
binary_name = Gem.win_platform? ? "kreuzberg.exe" : "kreuzberg"
|
|
114
|
+
found = CLIProxy.search_paths(binary_name).find(&:file?)
|
|
115
|
+
return found if found
|
|
116
|
+
|
|
117
|
+
raise MissingBinaryError, missing_binary_message
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# Error message for missing binary
|
|
121
|
+
#
|
|
122
|
+
# @return [String]
|
|
123
|
+
#
|
|
124
|
+
def missing_binary_message
|
|
125
|
+
<<~MSG
|
|
126
|
+
kreuzberg binary not found for API server. Build it with:
|
|
127
|
+
`cargo build --release --package kreuzberg-cli`
|
|
128
|
+
|
|
129
|
+
Or ensure kreuzberg is installed with API support.
|
|
130
|
+
MSG
|
|
131
|
+
.strip
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
end
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kreuzberg
|
|
4
|
+
module CacheAPI
|
|
5
|
+
# @return [void] No meaningful return value
|
|
6
|
+
# @example Clear cache
|
|
7
|
+
def clear_cache
|
|
8
|
+
native_clear_cache
|
|
9
|
+
reset_cache_tracker!
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
# Retrieve cache statistics.
|
|
13
|
+
#
|
|
14
|
+
# Returns information about the current state of the extraction result cache,
|
|
15
|
+
# including the number of cached entries and total memory used. Statistics include
|
|
16
|
+
# both native Rust cache metrics and local tracker metrics.
|
|
17
|
+
#
|
|
18
|
+
# @return [Hash{Symbol | String => Integer}] Cache statistics hash containing:
|
|
19
|
+
# - :total_entries [Integer] Total number of cached extraction results
|
|
20
|
+
# - :total_size_bytes [Integer] Total memory used by cached results in bytes
|
|
21
|
+
#
|
|
22
|
+
# @example Get cache statistics
|
|
23
|
+
# stats = Kreuzberg.cache_stats
|
|
24
|
+
# puts "Cached entries: #{stats[:total_entries]}"
|
|
25
|
+
# puts "Cache size: #{stats[:total_size_bytes]} bytes"
|
|
26
|
+
#
|
|
27
|
+
# @example Check if cache is full
|
|
28
|
+
# stats = Kreuzberg.cache_stats
|
|
29
|
+
# if stats[:total_size_bytes] > 1_000_000_000 # 1GB
|
|
30
|
+
# Kreuzberg.clear_cache
|
|
31
|
+
# end
|
|
32
|
+
def cache_stats
|
|
33
|
+
stats = native_cache_stats
|
|
34
|
+
total_entries = (stats["total_entries"] || stats[:total_entries] || 0) + @__cache_tracker[:entries]
|
|
35
|
+
total_size = (stats["total_size_bytes"] || stats[:total_size_bytes] || 0) + @__cache_tracker[:bytes]
|
|
36
|
+
|
|
37
|
+
stats["total_entries"] = total_entries
|
|
38
|
+
stats[:total_entries] = total_entries
|
|
39
|
+
stats["total_size_bytes"] = total_size
|
|
40
|
+
stats[:total_size_bytes] = total_size
|
|
41
|
+
|
|
42
|
+
stats
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
private
|
|
46
|
+
|
|
47
|
+
def record_cache_entry!(results, opts)
|
|
48
|
+
use_cache = opts.key?(:use_cache) ? opts[:use_cache] : true
|
|
49
|
+
return unless use_cache
|
|
50
|
+
|
|
51
|
+
results_array = results.is_a?(Array) ? results : [results]
|
|
52
|
+
results_array.each do |result|
|
|
53
|
+
# @type var result: Result
|
|
54
|
+
next unless result.respond_to?(:content)
|
|
55
|
+
|
|
56
|
+
@__cache_tracker[:entries] += 1
|
|
57
|
+
@__cache_tracker[:bytes] += result.content.to_s.bytesize
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def reset_cache_tracker!
|
|
62
|
+
@__cache_tracker[:entries] = 0
|
|
63
|
+
@__cache_tracker[:bytes] = 0
|
|
64
|
+
nil
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kreuzberg
|
|
4
|
+
# @example Extract a file
|
|
5
|
+
# @example Detect file type
|
|
6
|
+
module CLI
|
|
7
|
+
module_function
|
|
8
|
+
|
|
9
|
+
# Extract content from a file using the CLI
|
|
10
|
+
#
|
|
11
|
+
# @param path_or_nil [String, nil] Path to the file (positional, for backward compatibility)
|
|
12
|
+
# @param path [String] Path to the file (keyword argument)
|
|
13
|
+
# @param output [String] Output format ("text", "json", "markdown")
|
|
14
|
+
# @param ocr [Boolean] Enable OCR
|
|
15
|
+
# @return [String] Extracted content
|
|
16
|
+
#
|
|
17
|
+
def extract(path_or_nil = nil, path: nil, output: "text", ocr: false)
|
|
18
|
+
actual_path = path_or_nil || path
|
|
19
|
+
raise ArgumentError, "path is required" if actual_path.nil?
|
|
20
|
+
|
|
21
|
+
args = ["extract", actual_path, "--format", output]
|
|
22
|
+
args.push("--ocr", ocr ? "true" : "false")
|
|
23
|
+
CLIProxy.call(args)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# Detect MIME type of a file using the CLI
|
|
27
|
+
#
|
|
28
|
+
# @param path_or_nil [String, nil] Path to the file (positional, for backward compatibility)
|
|
29
|
+
# @param path [String] Path to the file (keyword argument)
|
|
30
|
+
# @return [String] MIME type
|
|
31
|
+
#
|
|
32
|
+
def detect(path_or_nil = nil, path: nil)
|
|
33
|
+
actual_path = path_or_nil || path
|
|
34
|
+
raise ArgumentError, "path is required" if actual_path.nil?
|
|
35
|
+
|
|
36
|
+
CLIProxy.call(["detect", actual_path]).strip
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Get CLI version
|
|
40
|
+
#
|
|
41
|
+
# @return [String] Version string
|
|
42
|
+
#
|
|
43
|
+
def version
|
|
44
|
+
CLIProxy.call(["--version"]).strip
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Get CLI help text
|
|
48
|
+
#
|
|
49
|
+
# @return [String] Help text
|
|
50
|
+
#
|
|
51
|
+
def help
|
|
52
|
+
CLIProxy.call(["--help"])
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|