embedded-raptor 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +232 -0
- package/dist/binary-file-reader.d.ts +9 -0
- package/dist/binary-file-reader.d.ts.map +1 -0
- package/dist/binary-format.d.ts +22 -0
- package/dist/binary-format.d.ts.map +1 -0
- package/dist/candidate-set.d.ts +16 -0
- package/dist/candidate-set.d.ts.map +1 -0
- package/dist/cli.cjs +109 -0
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.mjs +105 -0
- package/dist/commands/flags.d.ts +23 -0
- package/dist/commands/flags.d.ts.map +1 -0
- package/dist/commands/get.d.ts +41 -0
- package/dist/commands/get.d.ts.map +1 -0
- package/dist/commands/index.d.ts +4 -0
- package/dist/commands/index.d.ts.map +1 -0
- package/dist/commands/search.d.ts +65 -0
- package/dist/commands/search.d.ts.map +1 -0
- package/dist/commands/store.d.ts +42 -0
- package/dist/commands/store.d.ts.map +1 -0
- package/dist/engine-Bl1yeWoe.cjs +374 -0
- package/dist/engine-D06Gh_gw.mjs +335 -0
- package/dist/engine-DISO9uFr.mjs +31601 -0
- package/dist/engine-KhnrAv7v.cjs +38642 -0
- package/dist/engine-r-qHfsLd.cjs +370 -0
- package/dist/engine.d.ts +64 -0
- package/dist/engine.d.ts.map +1 -0
- package/dist/index.cjs +3 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.mjs +3 -0
- package/dist/types.d.ts +22 -0
- package/dist/types.d.ts.map +1 -0
- package/package.json +82 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Christoffer Artmann <artgaard@gmail.com>
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
# Raptor
|
|
2
|
+
|
|
3
|
+
[](https://github.com/artmann/raptor/actions/workflows/ci.yml)
|
|
4
|
+
[](https://www.npmjs.com/package/raptor)
|
|
5
|
+
[](https://www.npmjs.com/package/raptor)
|
|
6
|
+
[](https://opensource.org/licenses/MIT)
|
|
7
|
+
[](https://www.typescriptlang.org/)
|
|
8
|
+
|
|
9
|
+
> A lightweight semantic search database with text embeddings for Node.js and
|
|
10
|
+
> Bun
|
|
11
|
+
|
|
12
|
+
Raptor lets you build semantic search into your applications with just a few
|
|
13
|
+
lines of code. Store text, search by meaning, and find similar content—perfect
|
|
14
|
+
for RAG systems, chatbots, and recommendation engines.
|
|
15
|
+
|
|
16
|
+
## What is Raptor?
|
|
17
|
+
|
|
18
|
+
Raptor is an embedding database that automatically converts text into vector
|
|
19
|
+
embeddings and stores them in an efficient binary format. Instead of searching
|
|
20
|
+
by exact keywords, you can search by semantic similarity—finding documents that
|
|
21
|
+
mean the same thing, even if they use different words.
|
|
22
|
+
|
|
23
|
+
**Example:** Search for "how to reset password" and find results like "forgot my
|
|
24
|
+
login credentials" or "change account password".
|
|
25
|
+
|
|
26
|
+
## Why Raptor?
|
|
27
|
+
|
|
28
|
+
- **Simple API** - No complex setup, just store and search
|
|
29
|
+
- **Semantic Search** - Find content by meaning, not just keywords
|
|
30
|
+
- **Fast & Efficient** - Binary storage format ~50% smaller than JSON
|
|
31
|
+
- **Zero Dependencies** - Embeddings generated locally, no API keys needed
|
|
32
|
+
- **Works Everywhere** - Compatible with Node.js and Bun
|
|
33
|
+
- **Built for RAG** - Perfect for Retrieval Augmented Generation systems
|
|
34
|
+
|
|
35
|
+
## Use Cases
|
|
36
|
+
|
|
37
|
+
- **FAQ Bots** - Match user questions to answers by meaning
|
|
38
|
+
- **Document Search** - Semantic search over documentation or knowledge bases
|
|
39
|
+
- **Code Search** - Find code snippets by describing what they do
|
|
40
|
+
- **Content Recommendations** - "More like this" functionality
|
|
41
|
+
- **RAG Systems** - Retrieve relevant context for LLM prompts
|
|
42
|
+
|
|
43
|
+
## Installation
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
# Using npm
|
|
47
|
+
npm install raptor
|
|
48
|
+
|
|
49
|
+
# Using bun
|
|
50
|
+
bun add raptor
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
## Quick Start
|
|
54
|
+
|
|
55
|
+
### Programmatic API
|
|
56
|
+
|
|
57
|
+
```typescript
|
|
58
|
+
import { EmbeddingEngine } from 'raptor'
|
|
59
|
+
|
|
60
|
+
const engine = new EmbeddingEngine({
|
|
61
|
+
storePath: './my-database.raptor'
|
|
62
|
+
})
|
|
63
|
+
|
|
64
|
+
// Store documents
|
|
65
|
+
await engine.storeMany([
|
|
66
|
+
{ key: 'doc1', text: 'How to reset your password' },
|
|
67
|
+
{ key: 'doc2', text: 'Machine learning basics' },
|
|
68
|
+
{ key: 'doc3', text: 'Getting started with Bun' }
|
|
69
|
+
])
|
|
70
|
+
|
|
71
|
+
// Search by meaning
|
|
72
|
+
const results = await engine.search('forgot my password', 5)
|
|
73
|
+
console.log(results[0].key) // 'doc1' - matched by meaning!
|
|
74
|
+
console.log(results[0].similarity) // 0.87 - high similarity score
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
### Command Line Interface
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
# Store documents
|
|
81
|
+
raptor store doc1 "How to reset your password"
|
|
82
|
+
raptor store doc2 "Machine learning basics"
|
|
83
|
+
|
|
84
|
+
# Search by meaning
|
|
85
|
+
raptor search "forgot my password" --limit 5
|
|
86
|
+
|
|
87
|
+
# Retrieve by key
|
|
88
|
+
raptor get doc1
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
## Examples
|
|
92
|
+
|
|
93
|
+
See the [examples/](examples/) directory for complete, runnable examples:
|
|
94
|
+
|
|
95
|
+
| Example | Description | Run |
|
|
96
|
+
| -------------------------- | ------------------------------------------------------- | ----------------------------------------------- |
|
|
97
|
+
| **Document Search / RAG** | Semantic search over documentation chunks | `bun run examples/01-document-search.ts` |
|
|
98
|
+
| **FAQ Bot** | Match user questions to FAQs with confidence thresholds | `bun run examples/02-faq-bot.ts` |
|
|
99
|
+
| **Code Snippet Library** | Search code by natural language descriptions | `bun run examples/03-code-snippets.ts` |
|
|
100
|
+
| **Content Recommendation** | "More like this" functionality | `bun run examples/04-content-recommendation.ts` |
|
|
101
|
+
|
|
102
|
+
## API Reference
|
|
103
|
+
|
|
104
|
+
### `EmbeddingEngine`
|
|
105
|
+
|
|
106
|
+
#### `constructor(options)`
|
|
107
|
+
|
|
108
|
+
Create a new embedding engine.
|
|
109
|
+
|
|
110
|
+
```typescript
|
|
111
|
+
const engine = new EmbeddingEngine({
|
|
112
|
+
storePath: './database.raptor' // Path to storage file
|
|
113
|
+
})
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
#### `store(key, text)`
|
|
117
|
+
|
|
118
|
+
Store a single text entry with auto-generated embedding.
|
|
119
|
+
|
|
120
|
+
```typescript
|
|
121
|
+
await engine.store('doc1', 'The quick brown fox')
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
#### `storeMany(items)`
|
|
125
|
+
|
|
126
|
+
Store multiple entries in batch (faster than multiple `store()` calls).
|
|
127
|
+
|
|
128
|
+
```typescript
|
|
129
|
+
await engine.storeMany([
|
|
130
|
+
{ key: 'doc1', text: 'First document' },
|
|
131
|
+
{ key: 'doc2', text: 'Second document' }
|
|
132
|
+
])
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
#### `search(query, limit?, minSimilarity?)`
|
|
136
|
+
|
|
137
|
+
Search for semantically similar entries.
|
|
138
|
+
|
|
139
|
+
```typescript
|
|
140
|
+
const results = await engine.search(
|
|
141
|
+
'artificial intelligence', // query text
|
|
142
|
+
10, // max results (default: 10)
|
|
143
|
+
0.7 // min similarity 0-1 (default: 0)
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
// Results are sorted by similarity (highest first)
|
|
147
|
+
results.forEach((result) => {
|
|
148
|
+
console.log(result.key) // Document key
|
|
149
|
+
console.log(result.similarity) // Similarity score 0-1
|
|
150
|
+
})
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
#### `get(key)`
|
|
154
|
+
|
|
155
|
+
Retrieve a specific entry by key.
|
|
156
|
+
|
|
157
|
+
```typescript
|
|
158
|
+
const entry = await engine.get('doc1')
|
|
159
|
+
if (entry) {
|
|
160
|
+
console.log(entry.key) // 'doc1'
|
|
161
|
+
console.log(entry.embedding) // [0.1, 0.2, ...] (768 dimensions)
|
|
162
|
+
}
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
## CLI Reference
|
|
166
|
+
|
|
167
|
+
### Commands
|
|
168
|
+
|
|
169
|
+
```bash
|
|
170
|
+
# Store text
|
|
171
|
+
raptor store <key> <text> [--storePath path]
|
|
172
|
+
|
|
173
|
+
# Search for similar text
|
|
174
|
+
raptor search <query> [--limit 10] [--minSimilarity 0] [--storePath path]
|
|
175
|
+
|
|
176
|
+
# Get by key
|
|
177
|
+
raptor get <key> [--storePath path]
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
### Options
|
|
181
|
+
|
|
182
|
+
- `-s, --storePath` - Path to database file (default: `./database.raptor`)
|
|
183
|
+
- `-l, --limit` - Maximum results to return (default: 10)
|
|
184
|
+
- `-m, --minSimilarity` - Minimum similarity threshold 0-1 (default: 0)
|
|
185
|
+
|
|
186
|
+
### Examples
|
|
187
|
+
|
|
188
|
+
```bash
|
|
189
|
+
# Store documents
|
|
190
|
+
raptor store doc1 "The quick brown fox jumps over the lazy dog"
|
|
191
|
+
raptor store doc2 "Machine learning is a subset of artificial intelligence"
|
|
192
|
+
raptor store doc3 "Bun is a fast JavaScript runtime"
|
|
193
|
+
|
|
194
|
+
# Search with default settings
|
|
195
|
+
raptor search "artificial intelligence"
|
|
196
|
+
|
|
197
|
+
# Search with custom limit and threshold
|
|
198
|
+
raptor search "AI and ML" --limit 3 --minSimilarity 0.7
|
|
199
|
+
|
|
200
|
+
# Use custom database path
|
|
201
|
+
raptor store key1 "Some text" --storePath ./data/custom.raptor
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
## How It Works
|
|
205
|
+
|
|
206
|
+
1. **Text → Embeddings**: Raptor uses the BGE-Base-EN model to convert text into
|
|
207
|
+
768-dimensional vector embeddings
|
|
208
|
+
2. **Storage**: Embeddings are stored in an efficient binary format (.raptor
|
|
209
|
+
files)
|
|
210
|
+
3. **Search**: When you search, Raptor compares your query embedding against all
|
|
211
|
+
stored embeddings using cosine similarity
|
|
212
|
+
4. **Results**: Returns the most similar results ranked by similarity score
|
|
213
|
+
|
|
214
|
+
**Embedding Model**:
|
|
215
|
+
[BAAI/bge-base-en](https://huggingface.co/BAAI/bge-base-en-v1.5) (768
|
|
216
|
+
dimensions)
|
|
217
|
+
|
|
218
|
+
## Performance
|
|
219
|
+
|
|
220
|
+
- **Batch operations**: Use `storeMany()` for faster bulk inserts
|
|
221
|
+
- **Memory efficient**: Reads file in 64KB chunks, handles large databases
|
|
222
|
+
- **Fast search**: Cosine similarity comparison across all embeddings
|
|
223
|
+
- **Deduplication**: Latest entry automatically used for duplicate keys
|
|
224
|
+
|
|
225
|
+
## Contributing
|
|
226
|
+
|
|
227
|
+
Contributions are welcome! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for
|
|
228
|
+
development setup, architecture details, and guidelines.
|
|
229
|
+
|
|
230
|
+
## License
|
|
231
|
+
|
|
232
|
+
MIT © [Christoffer Artmann](mailto:artgaard@gmail.com)
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { EmbeddingEntry } from './types';
|
|
2
|
+
export declare class BinaryFileReader {
|
|
3
|
+
private storePath;
|
|
4
|
+
constructor(storePath: string);
|
|
5
|
+
entries(): AsyncGenerator<EmbeddingEntry>;
|
|
6
|
+
private parseRecord;
|
|
7
|
+
private parseRecordFromView;
|
|
8
|
+
}
|
|
9
|
+
//# sourceMappingURL=binary-file-reader.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"binary-file-reader.d.ts","sourceRoot":"","sources":["../src/binary-file-reader.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,SAAS,CAAA;AAI7C,qBAAa,gBAAgB;IACf,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM;IAE9B,OAAO,IAAI,cAAc,CAAC,cAAc,CAAC;IAuFhD,OAAO,CAAC,WAAW;IAQnB,OAAO,CAAC,mBAAmB;CAuC5B"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
export declare const magicBytes = "EMBD";
|
|
2
|
+
export declare const currentVersion = 1;
|
|
3
|
+
export declare const headerSize = 16;
|
|
4
|
+
export interface Header {
|
|
5
|
+
version: number;
|
|
6
|
+
dimension: number;
|
|
7
|
+
}
|
|
8
|
+
export interface BinaryRecord {
|
|
9
|
+
key: string;
|
|
10
|
+
embedding: Float32Array;
|
|
11
|
+
recordLength: number;
|
|
12
|
+
}
|
|
13
|
+
export declare function calculateRecordLength(keyLength: number, dimension: number): number;
|
|
14
|
+
export declare function writeHeader(filePath: string, dimension: number): Promise<void>;
|
|
15
|
+
export declare function readHeader(filePath: string): Promise<Header>;
|
|
16
|
+
export declare function writeRecord(filePath: string, key: string, embedding: Float32Array): Promise<void>;
|
|
17
|
+
export declare function writeRecords(filePath: string, records: Array<{
|
|
18
|
+
key: string;
|
|
19
|
+
embedding: Float32Array;
|
|
20
|
+
}>): Promise<void>;
|
|
21
|
+
export declare function readRecordForward(filePath: string, dimension: number, offset: number): Promise<BinaryRecord | null>;
|
|
22
|
+
//# sourceMappingURL=binary-format.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"binary-format.d.ts","sourceRoot":"","sources":["../src/binary-format.ts"],"names":[],"mappings":"AAEA,eAAO,MAAM,UAAU,SAAS,CAAA;AAChC,eAAO,MAAM,cAAc,IAAI,CAAA;AAC/B,eAAO,MAAM,UAAU,KAAK,CAAA;AAE5B,MAAM,WAAW,MAAM;IACrB,OAAO,EAAE,MAAM,CAAA;IACf,SAAS,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAA;IACX,SAAS,EAAE,YAAY,CAAA;IACvB,YAAY,EAAE,MAAM,CAAA;CACrB;AAED,wBAAgB,qBAAqB,CACnC,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,GAChB,MAAM,CAGR;AAED,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,IAAI,CAAC,CAkBf;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CA0ClE;AAED,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,MAAM,EAChB,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,YAAY,GACtB,OAAO,CAAC,IAAI,CAAC,CAgCf;AAED,wBAAsB,YAAY,CAChC,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,YAAY,CAAA;CAAE,CAAC,GACvD,OAAO,CAAC,IAAI,CAAC,CAqDf;AAED,wBAAsB,iBAAiB,CACrC,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CA4E9B"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export declare class CandidateSet {
|
|
2
|
+
readonly size: number;
|
|
3
|
+
protected readonly entries: CandidateSetEntry[];
|
|
4
|
+
constructor(size?: number);
|
|
5
|
+
add(key: string, value: number): void;
|
|
6
|
+
count(): number;
|
|
7
|
+
getEntries(): CandidateSetEntry[];
|
|
8
|
+
getKeys(): string[];
|
|
9
|
+
}
|
|
10
|
+
declare class CandidateSetEntry {
|
|
11
|
+
readonly key: string;
|
|
12
|
+
readonly value: number;
|
|
13
|
+
constructor(key: string, value: number);
|
|
14
|
+
}
|
|
15
|
+
export {};
|
|
16
|
+
//# sourceMappingURL=candidate-set.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"candidate-set.d.ts","sourceRoot":"","sources":["../src/candidate-set.ts"],"names":[],"mappings":"AAEA,qBAAa,YAAY;IACvB,SAAgB,IAAI,EAAE,MAAM,CAAA;IAC5B,SAAS,CAAC,QAAQ,CAAC,OAAO,EAAE,iBAAiB,EAAE,CAAK;gBAExC,IAAI,SAAI;IAMpB,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAyBrC,KAAK,IAAI,MAAM;IAIf,UAAU,IAAI,iBAAiB,EAAE;IAIjC,OAAO,IAAI,MAAM,EAAE;CAGpB;AAED,cAAM,iBAAiB;IACrB,SAAgB,GAAG,EAAE,MAAM,CAAA;IAC3B,SAAgB,KAAK,EAAE,MAAM,CAAA;gBAEjB,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM;CAIvC"}
|
package/dist/cli.cjs
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
const require_engine = require('./engine-KhnrAv7v.cjs');
|
|
3
|
+
let node_fs = require("node:fs");
|
|
4
|
+
node_fs = require_engine.__toESM(node_fs);
|
|
5
|
+
let node_path = require("node:path");
|
|
6
|
+
node_path = require_engine.__toESM(node_path);
|
|
7
|
+
let cleye = require("cleye");
|
|
8
|
+
cleye = require_engine.__toESM(cleye);
|
|
9
|
+
let node_url = require("node:url");
|
|
10
|
+
node_url = require_engine.__toESM(node_url);
|
|
11
|
+
|
|
12
|
+
//#region src/commands/flags.ts
|
|
13
|
+
const sharedFlags = { storePath: {
|
|
14
|
+
type: String,
|
|
15
|
+
description: "Path to the embeddings store file",
|
|
16
|
+
default: "./database.raptor",
|
|
17
|
+
alias: "s"
|
|
18
|
+
} };
|
|
19
|
+
const searchFlags = {
|
|
20
|
+
limit: {
|
|
21
|
+
type: Number,
|
|
22
|
+
description: "Maximum number of results to return",
|
|
23
|
+
default: 10,
|
|
24
|
+
alias: "l"
|
|
25
|
+
},
|
|
26
|
+
minSimilarity: {
|
|
27
|
+
type: Number,
|
|
28
|
+
description: "Minimum similarity threshold (0-1)",
|
|
29
|
+
default: 0,
|
|
30
|
+
alias: "m"
|
|
31
|
+
}
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
//#endregion
|
|
35
|
+
//#region src/commands/store.ts
|
|
36
|
+
const store = (0, cleye.command)({
|
|
37
|
+
name: "store",
|
|
38
|
+
description: "Store a text embedding with a key",
|
|
39
|
+
parameters: ["<key>", "<text>"],
|
|
40
|
+
flags: { ...sharedFlags }
|
|
41
|
+
}, async (argv) => {
|
|
42
|
+
const engine = new require_engine.EmbeddingEngine({ storePath: argv.flags.storePath });
|
|
43
|
+
const [key, text] = argv._;
|
|
44
|
+
await engine.store(key, text);
|
|
45
|
+
console.log(`✓ Stored embedding for key: ${key}`);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
//#endregion
|
|
49
|
+
//#region src/commands/get.ts
|
|
50
|
+
const get = (0, cleye.command)({
|
|
51
|
+
name: "get",
|
|
52
|
+
description: "Retrieve an embedding entry by key",
|
|
53
|
+
parameters: ["<key>"],
|
|
54
|
+
flags: { ...sharedFlags }
|
|
55
|
+
}, async (argv) => {
|
|
56
|
+
const engine = new require_engine.EmbeddingEngine({ storePath: argv.flags.storePath });
|
|
57
|
+
const [key] = argv._;
|
|
58
|
+
const entry = await engine.get(key);
|
|
59
|
+
if (entry) console.log(JSON.stringify({
|
|
60
|
+
key: entry.key,
|
|
61
|
+
text: entry.text,
|
|
62
|
+
embeddingDimensions: entry.embedding.length,
|
|
63
|
+
timestamp: new Date(entry.timestamp).toISOString()
|
|
64
|
+
}, null, 2));
|
|
65
|
+
else {
|
|
66
|
+
console.log(`Key "${key}" not found`);
|
|
67
|
+
process.exit(1);
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
//#endregion
|
|
72
|
+
//#region src/commands/search.ts
|
|
73
|
+
const search = (0, cleye.command)({
|
|
74
|
+
name: "search",
|
|
75
|
+
description: "Search for similar embeddings using a query",
|
|
76
|
+
parameters: ["<query>"],
|
|
77
|
+
flags: {
|
|
78
|
+
...sharedFlags,
|
|
79
|
+
...searchFlags
|
|
80
|
+
}
|
|
81
|
+
}, async (argv) => {
|
|
82
|
+
const engine = new require_engine.EmbeddingEngine({ storePath: argv.flags.storePath });
|
|
83
|
+
const [query] = argv._;
|
|
84
|
+
const results = await engine.search(query, argv.flags.limit, argv.flags.minSimilarity);
|
|
85
|
+
if (results.length === 0) console.log("No results found");
|
|
86
|
+
else {
|
|
87
|
+
console.log(`Found ${results.length} result(s):\n`);
|
|
88
|
+
for (const result of results) console.log(`${result.key}: ${result.similarity.toFixed(6)}`);
|
|
89
|
+
}
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
//#endregion
|
|
93
|
+
//#region src/cli.ts
|
|
94
|
+
const __dirname$1 = (0, node_path.dirname)((0, node_url.fileURLToPath)(require("url").pathToFileURL(__filename).href));
|
|
95
|
+
function main() {
|
|
96
|
+
(0, cleye.cli)({
|
|
97
|
+
name: "raptor",
|
|
98
|
+
version: JSON.parse((0, node_fs.readFileSync)((0, node_path.resolve)(__dirname$1, "../package.json"), "utf-8")).version,
|
|
99
|
+
description: "An embedding database CLI for storing and searching text indexes",
|
|
100
|
+
commands: [
|
|
101
|
+
store,
|
|
102
|
+
get,
|
|
103
|
+
search
|
|
104
|
+
]
|
|
105
|
+
}, () => {});
|
|
106
|
+
}
|
|
107
|
+
main();
|
|
108
|
+
|
|
109
|
+
//#endregion
|
package/dist/cli.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":""}
|
package/dist/cli.mjs
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { t as EmbeddingEngine } from "./engine-DISO9uFr.mjs";
|
|
3
|
+
import { readFileSync } from "node:fs";
|
|
4
|
+
import { dirname, resolve } from "node:path";
|
|
5
|
+
import { cli, command } from "cleye";
|
|
6
|
+
import { fileURLToPath } from "node:url";
|
|
7
|
+
|
|
8
|
+
//#region src/commands/flags.ts
|
|
9
|
+
const sharedFlags = { storePath: {
|
|
10
|
+
type: String,
|
|
11
|
+
description: "Path to the embeddings store file",
|
|
12
|
+
default: "./database.raptor",
|
|
13
|
+
alias: "s"
|
|
14
|
+
} };
|
|
15
|
+
const searchFlags = {
|
|
16
|
+
limit: {
|
|
17
|
+
type: Number,
|
|
18
|
+
description: "Maximum number of results to return",
|
|
19
|
+
default: 10,
|
|
20
|
+
alias: "l"
|
|
21
|
+
},
|
|
22
|
+
minSimilarity: {
|
|
23
|
+
type: Number,
|
|
24
|
+
description: "Minimum similarity threshold (0-1)",
|
|
25
|
+
default: 0,
|
|
26
|
+
alias: "m"
|
|
27
|
+
}
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
//#endregion
|
|
31
|
+
//#region src/commands/store.ts
|
|
32
|
+
const store = command({
|
|
33
|
+
name: "store",
|
|
34
|
+
description: "Store a text embedding with a key",
|
|
35
|
+
parameters: ["<key>", "<text>"],
|
|
36
|
+
flags: { ...sharedFlags }
|
|
37
|
+
}, async (argv) => {
|
|
38
|
+
const engine = new EmbeddingEngine({ storePath: argv.flags.storePath });
|
|
39
|
+
const [key, text] = argv._;
|
|
40
|
+
await engine.store(key, text);
|
|
41
|
+
console.log(`✓ Stored embedding for key: ${key}`);
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
//#endregion
|
|
45
|
+
//#region src/commands/get.ts
|
|
46
|
+
const get = command({
|
|
47
|
+
name: "get",
|
|
48
|
+
description: "Retrieve an embedding entry by key",
|
|
49
|
+
parameters: ["<key>"],
|
|
50
|
+
flags: { ...sharedFlags }
|
|
51
|
+
}, async (argv) => {
|
|
52
|
+
const engine = new EmbeddingEngine({ storePath: argv.flags.storePath });
|
|
53
|
+
const [key] = argv._;
|
|
54
|
+
const entry = await engine.get(key);
|
|
55
|
+
if (entry) console.log(JSON.stringify({
|
|
56
|
+
key: entry.key,
|
|
57
|
+
text: entry.text,
|
|
58
|
+
embeddingDimensions: entry.embedding.length,
|
|
59
|
+
timestamp: new Date(entry.timestamp).toISOString()
|
|
60
|
+
}, null, 2));
|
|
61
|
+
else {
|
|
62
|
+
console.log(`Key "${key}" not found`);
|
|
63
|
+
process.exit(1);
|
|
64
|
+
}
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
//#endregion
|
|
68
|
+
//#region src/commands/search.ts
|
|
69
|
+
const search = command({
|
|
70
|
+
name: "search",
|
|
71
|
+
description: "Search for similar embeddings using a query",
|
|
72
|
+
parameters: ["<query>"],
|
|
73
|
+
flags: {
|
|
74
|
+
...sharedFlags,
|
|
75
|
+
...searchFlags
|
|
76
|
+
}
|
|
77
|
+
}, async (argv) => {
|
|
78
|
+
const engine = new EmbeddingEngine({ storePath: argv.flags.storePath });
|
|
79
|
+
const [query] = argv._;
|
|
80
|
+
const results = await engine.search(query, argv.flags.limit, argv.flags.minSimilarity);
|
|
81
|
+
if (results.length === 0) console.log("No results found");
|
|
82
|
+
else {
|
|
83
|
+
console.log(`Found ${results.length} result(s):\n`);
|
|
84
|
+
for (const result of results) console.log(`${result.key}: ${result.similarity.toFixed(6)}`);
|
|
85
|
+
}
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
//#endregion
|
|
89
|
+
//#region src/cli.ts
|
|
90
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
91
|
+
function main() {
|
|
92
|
+
cli({
|
|
93
|
+
name: "raptor",
|
|
94
|
+
version: JSON.parse(readFileSync(resolve(__dirname, "../package.json"), "utf-8")).version,
|
|
95
|
+
description: "An embedding database CLI for storing and searching text indexes",
|
|
96
|
+
commands: [
|
|
97
|
+
store,
|
|
98
|
+
get,
|
|
99
|
+
search
|
|
100
|
+
]
|
|
101
|
+
}, () => {});
|
|
102
|
+
}
|
|
103
|
+
main();
|
|
104
|
+
|
|
105
|
+
//#endregion
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
export declare const sharedFlags: {
|
|
2
|
+
readonly storePath: {
|
|
3
|
+
readonly type: StringConstructor;
|
|
4
|
+
readonly description: "Path to the embeddings store file";
|
|
5
|
+
readonly default: "./database.raptor";
|
|
6
|
+
readonly alias: "s";
|
|
7
|
+
};
|
|
8
|
+
};
|
|
9
|
+
export declare const searchFlags: {
|
|
10
|
+
readonly limit: {
|
|
11
|
+
readonly type: NumberConstructor;
|
|
12
|
+
readonly description: "Maximum number of results to return";
|
|
13
|
+
readonly default: 10;
|
|
14
|
+
readonly alias: "l";
|
|
15
|
+
};
|
|
16
|
+
readonly minSimilarity: {
|
|
17
|
+
readonly type: NumberConstructor;
|
|
18
|
+
readonly description: "Minimum similarity threshold (0-1)";
|
|
19
|
+
readonly default: 0;
|
|
20
|
+
readonly alias: "m";
|
|
21
|
+
};
|
|
22
|
+
};
|
|
23
|
+
//# sourceMappingURL=flags.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"flags.d.ts","sourceRoot":"","sources":["../../src/commands/flags.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,WAAW;;;;;;;CAOd,CAAA;AAEV,eAAO,MAAM,WAAW;;;;;;;;;;;;;CAad,CAAA"}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
export declare const get: import("cleye").Command<{
|
|
2
|
+
name: "get";
|
|
3
|
+
description: string;
|
|
4
|
+
parameters: "<key>"[];
|
|
5
|
+
flags: {
|
|
6
|
+
storePath: {
|
|
7
|
+
readonly type: StringConstructor;
|
|
8
|
+
readonly description: "Path to the embeddings store file";
|
|
9
|
+
readonly default: "./database.raptor";
|
|
10
|
+
readonly alias: "s";
|
|
11
|
+
};
|
|
12
|
+
};
|
|
13
|
+
}, {
|
|
14
|
+
command: "get";
|
|
15
|
+
} & import("type-flag").TypeFlag<{
|
|
16
|
+
storePath: {
|
|
17
|
+
readonly type: StringConstructor;
|
|
18
|
+
readonly description: "Path to the embeddings store file";
|
|
19
|
+
readonly default: "./database.raptor";
|
|
20
|
+
readonly alias: "s";
|
|
21
|
+
};
|
|
22
|
+
} & {
|
|
23
|
+
help: BooleanConstructor;
|
|
24
|
+
}> & {
|
|
25
|
+
_: {
|
|
26
|
+
key: string;
|
|
27
|
+
};
|
|
28
|
+
showHelp: (options?: {
|
|
29
|
+
version?: string;
|
|
30
|
+
description?: string;
|
|
31
|
+
usage?: false | string | string[];
|
|
32
|
+
examples?: string | string[];
|
|
33
|
+
render?: (nodes: {
|
|
34
|
+
id?: string;
|
|
35
|
+
type: keyof import("cleye").Renderers;
|
|
36
|
+
data: any;
|
|
37
|
+
}[], renderers: import("cleye").Renderers) => string;
|
|
38
|
+
}) => void;
|
|
39
|
+
showVersion: () => void;
|
|
40
|
+
}>;
|
|
41
|
+
//# sourceMappingURL=get.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"get.d.ts","sourceRoot":"","sources":["../../src/commands/get.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;sBAoCk7L,CAAC;;;;;;;;;;;;EADl8L,CAAA"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/commands/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,SAAS,CAAA;AAC/B,OAAO,EAAE,GAAG,EAAE,MAAM,OAAO,CAAA;AAC3B,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAA"}
|