embedded-raptor 1.1.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -11
- package/dist/candidate-set.d.ts +14 -1
- package/dist/candidate-set.d.ts.map +1 -1
- package/dist/cli.cjs +116 -29
- package/dist/cli.mjs +115 -28
- package/dist/commands/delete.d.ts +44 -0
- package/dist/commands/delete.d.ts.map +1 -0
- package/dist/commands/get.d.ts +4 -1
- package/dist/commands/get.d.ts.map +1 -1
- package/dist/commands/index.d.ts +2 -0
- package/dist/commands/index.d.ts.map +1 -1
- package/dist/commands/search.d.ts +4 -1
- package/dist/commands/search.d.ts.map +1 -1
- package/dist/commands/store.d.ts +4 -1
- package/dist/commands/store.d.ts.map +1 -1
- package/dist/commands/wal.d.ts +41 -0
- package/dist/commands/wal.d.ts.map +1 -0
- package/dist/{engine-ez6nFONK.mjs → engine-BvJ0ls3b.mjs} +62 -30
- package/dist/engine-Cuz0P5Od.mjs +1164 -0
- package/dist/engine-DKn_V99U.cjs +1545 -0
- package/dist/engine-DLM7PWhV.cjs +1203 -0
- package/dist/{engine-CTvg_66e.cjs → engine-Iq0_dbnk.cjs} +63 -31
- package/dist/engine-Zax2PJSW.mjs +1488 -0
- package/dist/engine.d.ts +71 -16
- package/dist/engine.d.ts.map +1 -1
- package/dist/index.cjs +1 -1
- package/dist/index.mjs +1 -1
- package/dist/storage-engine/constants.d.ts +54 -0
- package/dist/storage-engine/constants.d.ts.map +1 -0
- package/dist/storage-engine/data-format.d.ts +35 -0
- package/dist/storage-engine/data-format.d.ts.map +1 -0
- package/dist/storage-engine/file-lock.d.ts +35 -0
- package/dist/storage-engine/file-lock.d.ts.map +1 -0
- package/dist/storage-engine/index.d.ts +16 -0
- package/dist/storage-engine/index.d.ts.map +1 -0
- package/dist/storage-engine/integration/helpers.d.ts +32 -0
- package/dist/storage-engine/integration/helpers.d.ts.map +1 -0
- package/dist/storage-engine/key-index.d.ts +59 -0
- package/dist/storage-engine/key-index.d.ts.map +1 -0
- package/dist/storage-engine/migration.d.ts +26 -0
- package/dist/storage-engine/migration.d.ts.map +1 -0
- package/dist/storage-engine/mutex.d.ts +23 -0
- package/dist/storage-engine/mutex.d.ts.map +1 -0
- package/dist/storage-engine/storage-engine.d.ts +102 -0
- package/dist/storage-engine/storage-engine.d.ts.map +1 -0
- package/dist/storage-engine/types.d.ts +104 -0
- package/dist/storage-engine/types.d.ts.map +1 -0
- package/dist/storage-engine/wal-format.d.ts +22 -0
- package/dist/storage-engine/wal-format.d.ts.map +1 -0
- package/dist/storage-engine/wal.d.ts +38 -0
- package/dist/storage-engine/wal.d.ts.map +1 -0
- package/dist/storage-engine/write-batch.d.ts +44 -0
- package/dist/storage-engine/write-batch.d.ts.map +1 -0
- package/dist/storage-engine/write-batcher.d.ts +58 -0
- package/dist/storage-engine/write-batcher.d.ts.map +1 -0
- package/dist/types.d.ts +2 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +8 -3
- package/dist/engine-Bl1yeWoe.cjs +0 -374
- package/dist/engine-D06Gh_gw.mjs +0 -335
- package/dist/engine-DISO9uFr.mjs +0 -31601
- package/dist/engine-KhnrAv7v.cjs +0 -38642
- package/dist/engine-r-qHfsLd.cjs +0 -370
package/README.md
CHANGED
|
@@ -9,16 +9,16 @@
|
|
|
9
9
|
> A lightweight semantic search database with text embeddings for Node.js and
|
|
10
10
|
> Bun
|
|
11
11
|
|
|
12
|
-
Embedded Raptor lets you build semantic search into your applications with just
|
|
13
|
-
lines of code. Store text, search by meaning, and find similar
|
|
14
|
-
for RAG systems, chatbots, and recommendation engines.
|
|
12
|
+
Embedded Raptor lets you build semantic search into your applications with just
|
|
13
|
+
a few lines of code. Store text, search by meaning, and find similar
|
|
14
|
+
content—perfect for RAG systems, chatbots, and recommendation engines.
|
|
15
15
|
|
|
16
16
|
## What is Embedded Raptor?
|
|
17
17
|
|
|
18
|
-
Embedded Raptor is an embedding database that automatically converts text into
|
|
19
|
-
embeddings and stores them in an efficient binary format. Instead of
|
|
20
|
-
by exact keywords, you can search by semantic similarity—finding
|
|
21
|
-
mean the same thing, even if they use different words.
|
|
18
|
+
Embedded Raptor is an embedding database that automatically converts text into
|
|
19
|
+
vector embeddings and stores them in an efficient binary format. Instead of
|
|
20
|
+
searching by exact keywords, you can search by semantic similarity—finding
|
|
21
|
+
documents that mean the same thing, even if they use different words.
|
|
22
22
|
|
|
23
23
|
**Example:** Search for "how to reset password" and find results like "forgot my
|
|
24
24
|
login credentials" or "change account password".
|
|
@@ -203,12 +203,12 @@ raptor store key1 "Some text" --storePath ./data/custom.raptor
|
|
|
203
203
|
|
|
204
204
|
## How It Works
|
|
205
205
|
|
|
206
|
-
1. **Text → Embeddings**: Embedded Raptor uses the BGE-Base-EN model to convert
|
|
207
|
-
768-dimensional vector embeddings
|
|
206
|
+
1. **Text → Embeddings**: Embedded Raptor uses the BGE-Base-EN model to convert
|
|
207
|
+
text into 768-dimensional vector embeddings
|
|
208
208
|
2. **Storage**: Embeddings are stored in an efficient binary format (.raptor
|
|
209
209
|
files)
|
|
210
|
-
3. **Search**: When you search, Embedded Raptor compares your query embedding
|
|
211
|
-
stored embeddings using cosine similarity
|
|
210
|
+
3. **Search**: When you search, Embedded Raptor compares your query embedding
|
|
211
|
+
against all stored embeddings using cosine similarity
|
|
212
212
|
4. **Results**: Returns the most similar results ranked by similarity score
|
|
213
213
|
|
|
214
214
|
**Embedding Model**:
|
package/dist/candidate-set.d.ts
CHANGED
|
@@ -1,11 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A fixed-size collection that maintains the top N highest-value entries.
|
|
3
|
+
* Uses a min-heap internally for O(log n) insertion when at capacity.
|
|
4
|
+
*/
|
|
1
5
|
export declare class CandidateSet {
|
|
2
6
|
readonly size: number;
|
|
3
|
-
|
|
7
|
+
private readonly heap;
|
|
4
8
|
constructor(size?: number);
|
|
5
9
|
add(key: string, value: number): void;
|
|
6
10
|
count(): number;
|
|
7
11
|
getEntries(): CandidateSetEntry[];
|
|
8
12
|
getKeys(): string[];
|
|
13
|
+
/**
|
|
14
|
+
* Bubble up element at index to maintain min-heap property
|
|
15
|
+
*/
|
|
16
|
+
private bubbleUp;
|
|
17
|
+
/**
|
|
18
|
+
* Bubble down element at index to maintain min-heap property
|
|
19
|
+
*/
|
|
20
|
+
private bubbleDown;
|
|
21
|
+
private swap;
|
|
9
22
|
}
|
|
10
23
|
declare class CandidateSetEntry {
|
|
11
24
|
readonly key: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"candidate-set.d.ts","sourceRoot":"","sources":["../src/candidate-set.ts"],"names":[],"mappings":"AAEA,qBAAa,YAAY;IACvB,SAAgB,IAAI,EAAE,MAAM,CAAA;IAC5B,
|
|
1
|
+
{"version":3,"file":"candidate-set.d.ts","sourceRoot":"","sources":["../src/candidate-set.ts"],"names":[],"mappings":"AAEA;;;GAGG;AACH,qBAAa,YAAY;IACvB,SAAgB,IAAI,EAAE,MAAM,CAAA;IAC5B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA0B;gBAEnC,IAAI,SAAI;IAMpB,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAmBrC,KAAK,IAAI,MAAM;IAIf,UAAU,IAAI,iBAAiB,EAAE;IAKjC,OAAO,IAAI,MAAM,EAAE;IAInB;;OAEG;IACH,OAAO,CAAC,QAAQ;IAWhB;;OAEG;IACH,OAAO,CAAC,UAAU;IAgClB,OAAO,CAAC,IAAI;CAKb;AAED,cAAM,iBAAiB;IACrB,SAAgB,GAAG,EAAE,MAAM,CAAA;IAC3B,SAAgB,KAAK,EAAE,MAAM,CAAA;gBAEjB,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM;CAIvC"}
|
package/dist/cli.cjs
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
const require_engine = require('./engine-
|
|
3
|
-
let node_fs = require("node:fs");
|
|
4
|
-
node_fs = require_engine.__toESM(node_fs);
|
|
2
|
+
const require_engine = require('./engine-DKn_V99U.cjs');
|
|
5
3
|
let node_path = require("node:path");
|
|
6
4
|
node_path = require_engine.__toESM(node_path);
|
|
5
|
+
let node_fs = require("node:fs");
|
|
6
|
+
node_fs = require_engine.__toESM(node_fs);
|
|
7
7
|
let cleye = require("cleye");
|
|
8
8
|
cleye = require_engine.__toESM(cleye);
|
|
9
9
|
let node_url = require("node:url");
|
|
@@ -35,36 +35,49 @@ const searchFlags = {
|
|
|
35
35
|
//#region src/commands/store.ts
|
|
36
36
|
const store = (0, cleye.command)({
|
|
37
37
|
name: "store",
|
|
38
|
-
description: "Store a text embedding with a key",
|
|
39
38
|
parameters: ["<key>", "<text>"],
|
|
40
|
-
flags: { ...sharedFlags }
|
|
39
|
+
flags: { ...sharedFlags },
|
|
40
|
+
help: {
|
|
41
|
+
description: "Store a text embedding with a key",
|
|
42
|
+
examples: ["raptor store doc1 \"Machine learning is awesome\"", "raptor store -s ./my-db.raptor mykey \"Some text to embed\""]
|
|
43
|
+
}
|
|
41
44
|
}, async (argv) => {
|
|
42
45
|
const engine = new require_engine.EmbeddingEngine({ storePath: argv.flags.storePath });
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
+
try {
|
|
47
|
+
const [key, text] = argv._;
|
|
48
|
+
await engine.store(key, text);
|
|
49
|
+
console.log(`Stored embedding for key: ${key}`);
|
|
50
|
+
} finally {
|
|
51
|
+
await engine.dispose();
|
|
52
|
+
}
|
|
46
53
|
});
|
|
47
54
|
|
|
48
55
|
//#endregion
|
|
49
56
|
//#region src/commands/get.ts
|
|
50
57
|
const get = (0, cleye.command)({
|
|
51
58
|
name: "get",
|
|
52
|
-
description: "Retrieve an embedding entry by key",
|
|
53
59
|
parameters: ["<key>"],
|
|
54
|
-
flags: { ...sharedFlags }
|
|
60
|
+
flags: { ...sharedFlags },
|
|
61
|
+
help: {
|
|
62
|
+
description: "Retrieve an embedding entry by key",
|
|
63
|
+
examples: ["raptor get doc1", "raptor get -s ./my-db.raptor mykey"]
|
|
64
|
+
}
|
|
55
65
|
}, async (argv) => {
|
|
56
66
|
const engine = new require_engine.EmbeddingEngine({ storePath: argv.flags.storePath });
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
67
|
+
try {
|
|
68
|
+
const [key] = argv._;
|
|
69
|
+
const entry = await engine.get(key);
|
|
70
|
+
if (entry) console.log(JSON.stringify({
|
|
71
|
+
key: entry.key,
|
|
72
|
+
embeddingDimensions: entry.embedding.length,
|
|
73
|
+
timestamp: new Date(entry.timestamp).toISOString()
|
|
74
|
+
}, null, 2));
|
|
75
|
+
else {
|
|
76
|
+
console.log(`Key "${key}" not found`);
|
|
77
|
+
process.exit(1);
|
|
78
|
+
}
|
|
79
|
+
} finally {
|
|
80
|
+
await engine.dispose();
|
|
68
81
|
}
|
|
69
82
|
});
|
|
70
83
|
|
|
@@ -72,20 +85,92 @@ const get = (0, cleye.command)({
|
|
|
72
85
|
//#region src/commands/search.ts
|
|
73
86
|
const search = (0, cleye.command)({
|
|
74
87
|
name: "search",
|
|
75
|
-
description: "Search for similar embeddings using a query",
|
|
76
88
|
parameters: ["<query>"],
|
|
77
89
|
flags: {
|
|
78
90
|
...sharedFlags,
|
|
79
91
|
...searchFlags
|
|
92
|
+
},
|
|
93
|
+
help: {
|
|
94
|
+
description: "Search for similar embeddings using a query",
|
|
95
|
+
examples: [
|
|
96
|
+
"raptor search \"artificial intelligence\"",
|
|
97
|
+
"raptor search -l 5 -m 0.7 \"machine learning\"",
|
|
98
|
+
"raptor search -s ./my-db.raptor \"find similar docs\""
|
|
99
|
+
]
|
|
80
100
|
}
|
|
81
101
|
}, async (argv) => {
|
|
82
102
|
const engine = new require_engine.EmbeddingEngine({ storePath: argv.flags.storePath });
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
103
|
+
try {
|
|
104
|
+
const [query] = argv._;
|
|
105
|
+
const results = await engine.search(query, argv.flags.limit, argv.flags.minSimilarity);
|
|
106
|
+
if (results.length === 0) console.log("No results found");
|
|
107
|
+
else {
|
|
108
|
+
console.log(`Found ${results.length} result(s):\n`);
|
|
109
|
+
for (const result of results) console.log(`${result.key}: ${result.similarity.toFixed(6)}`);
|
|
110
|
+
}
|
|
111
|
+
} finally {
|
|
112
|
+
await engine.dispose();
|
|
113
|
+
}
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
//#endregion
|
|
117
|
+
//#region src/commands/delete.ts
|
|
118
|
+
const deleteCmd = (0, cleye.command)({
|
|
119
|
+
name: "delete",
|
|
120
|
+
parameters: ["<key>"],
|
|
121
|
+
flags: { ...sharedFlags },
|
|
122
|
+
help: {
|
|
123
|
+
description: "Delete an embedding entry by key",
|
|
124
|
+
examples: ["raptor delete doc1", "raptor delete -s ./my-db.raptor mykey"]
|
|
125
|
+
}
|
|
126
|
+
}, async (argv) => {
|
|
127
|
+
const engine = new require_engine.EmbeddingEngine({ storePath: argv.flags.storePath });
|
|
128
|
+
try {
|
|
129
|
+
const [key] = argv._;
|
|
130
|
+
if (await engine.delete(key)) console.log(`Deleted key "${key}"`);
|
|
131
|
+
else {
|
|
132
|
+
console.log(`Key "${key}" not found`);
|
|
133
|
+
process.exit(1);
|
|
134
|
+
}
|
|
135
|
+
} finally {
|
|
136
|
+
await engine.dispose();
|
|
137
|
+
}
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
//#endregion
|
|
141
|
+
//#region src/commands/wal.ts
|
|
142
|
+
const opTypeNames = {
|
|
143
|
+
[require_engine.opType.insert]: "INSERT",
|
|
144
|
+
[require_engine.opType.update]: "UPDATE",
|
|
145
|
+
[require_engine.opType.delete]: "DELETE"
|
|
146
|
+
};
|
|
147
|
+
function formatKeyHash(hash) {
|
|
148
|
+
return Array.from(hash.slice(0, 8)).map((b) => b.toString(16).padStart(2, "0")).join("");
|
|
149
|
+
}
|
|
150
|
+
const walCmd = (0, cleye.command)({
|
|
151
|
+
name: "wal",
|
|
152
|
+
flags: { ...sharedFlags },
|
|
153
|
+
help: {
|
|
154
|
+
description: "Display WAL entries in human-readable format",
|
|
155
|
+
examples: ["raptor wal", "raptor wal -s ./my-db.raptor"]
|
|
156
|
+
}
|
|
157
|
+
}, async (argv) => {
|
|
158
|
+
const wal = new require_engine.Wal(argv.flags.storePath.replace(/\.raptor$/, "") + require_engine.fileExtensions.wal);
|
|
159
|
+
try {
|
|
160
|
+
let count = 0;
|
|
161
|
+
for await (const entry of wal.recover()) {
|
|
162
|
+
const opName = opTypeNames[entry.opType] ?? `UNKNOWN(${entry.opType})`;
|
|
163
|
+
const keyHashHex = formatKeyHash(entry.keyHash);
|
|
164
|
+
console.log(`[${entry.sequenceNumber}] ${opName}`);
|
|
165
|
+
console.log(` offset: ${entry.offset}, length: ${entry.length}`);
|
|
166
|
+
console.log(` keyHash: ${keyHashHex}`);
|
|
167
|
+
console.log();
|
|
168
|
+
count++;
|
|
169
|
+
}
|
|
170
|
+
if (count === 0) console.log("WAL is empty");
|
|
171
|
+
else console.log(`Total: ${count} entries`);
|
|
172
|
+
} finally {
|
|
173
|
+
await wal.close();
|
|
89
174
|
}
|
|
90
175
|
});
|
|
91
176
|
|
|
@@ -100,7 +185,9 @@ function main() {
|
|
|
100
185
|
commands: [
|
|
101
186
|
store,
|
|
102
187
|
get,
|
|
103
|
-
search
|
|
188
|
+
search,
|
|
189
|
+
deleteCmd,
|
|
190
|
+
walCmd
|
|
104
191
|
]
|
|
105
192
|
}, () => {});
|
|
106
193
|
}
|
package/dist/cli.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { t as EmbeddingEngine } from "./engine-
|
|
3
|
-
import { readFileSync } from "node:fs";
|
|
2
|
+
import { i as opType, n as Wal, r as fileExtensions, t as EmbeddingEngine } from "./engine-Zax2PJSW.mjs";
|
|
4
3
|
import { dirname, resolve } from "node:path";
|
|
4
|
+
import { readFileSync } from "node:fs";
|
|
5
5
|
import { cli, command } from "cleye";
|
|
6
6
|
import { fileURLToPath } from "node:url";
|
|
7
7
|
|
|
@@ -31,36 +31,49 @@ const searchFlags = {
|
|
|
31
31
|
//#region src/commands/store.ts
|
|
32
32
|
const store = command({
|
|
33
33
|
name: "store",
|
|
34
|
-
description: "Store a text embedding with a key",
|
|
35
34
|
parameters: ["<key>", "<text>"],
|
|
36
|
-
flags: { ...sharedFlags }
|
|
35
|
+
flags: { ...sharedFlags },
|
|
36
|
+
help: {
|
|
37
|
+
description: "Store a text embedding with a key",
|
|
38
|
+
examples: ["raptor store doc1 \"Machine learning is awesome\"", "raptor store -s ./my-db.raptor mykey \"Some text to embed\""]
|
|
39
|
+
}
|
|
37
40
|
}, async (argv) => {
|
|
38
41
|
const engine = new EmbeddingEngine({ storePath: argv.flags.storePath });
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
+
try {
|
|
43
|
+
const [key, text] = argv._;
|
|
44
|
+
await engine.store(key, text);
|
|
45
|
+
console.log(`Stored embedding for key: ${key}`);
|
|
46
|
+
} finally {
|
|
47
|
+
await engine.dispose();
|
|
48
|
+
}
|
|
42
49
|
});
|
|
43
50
|
|
|
44
51
|
//#endregion
|
|
45
52
|
//#region src/commands/get.ts
|
|
46
53
|
const get = command({
|
|
47
54
|
name: "get",
|
|
48
|
-
description: "Retrieve an embedding entry by key",
|
|
49
55
|
parameters: ["<key>"],
|
|
50
|
-
flags: { ...sharedFlags }
|
|
56
|
+
flags: { ...sharedFlags },
|
|
57
|
+
help: {
|
|
58
|
+
description: "Retrieve an embedding entry by key",
|
|
59
|
+
examples: ["raptor get doc1", "raptor get -s ./my-db.raptor mykey"]
|
|
60
|
+
}
|
|
51
61
|
}, async (argv) => {
|
|
52
62
|
const engine = new EmbeddingEngine({ storePath: argv.flags.storePath });
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
63
|
+
try {
|
|
64
|
+
const [key] = argv._;
|
|
65
|
+
const entry = await engine.get(key);
|
|
66
|
+
if (entry) console.log(JSON.stringify({
|
|
67
|
+
key: entry.key,
|
|
68
|
+
embeddingDimensions: entry.embedding.length,
|
|
69
|
+
timestamp: new Date(entry.timestamp).toISOString()
|
|
70
|
+
}, null, 2));
|
|
71
|
+
else {
|
|
72
|
+
console.log(`Key "${key}" not found`);
|
|
73
|
+
process.exit(1);
|
|
74
|
+
}
|
|
75
|
+
} finally {
|
|
76
|
+
await engine.dispose();
|
|
64
77
|
}
|
|
65
78
|
});
|
|
66
79
|
|
|
@@ -68,20 +81,92 @@ const get = command({
|
|
|
68
81
|
//#region src/commands/search.ts
|
|
69
82
|
const search = command({
|
|
70
83
|
name: "search",
|
|
71
|
-
description: "Search for similar embeddings using a query",
|
|
72
84
|
parameters: ["<query>"],
|
|
73
85
|
flags: {
|
|
74
86
|
...sharedFlags,
|
|
75
87
|
...searchFlags
|
|
88
|
+
},
|
|
89
|
+
help: {
|
|
90
|
+
description: "Search for similar embeddings using a query",
|
|
91
|
+
examples: [
|
|
92
|
+
"raptor search \"artificial intelligence\"",
|
|
93
|
+
"raptor search -l 5 -m 0.7 \"machine learning\"",
|
|
94
|
+
"raptor search -s ./my-db.raptor \"find similar docs\""
|
|
95
|
+
]
|
|
76
96
|
}
|
|
77
97
|
}, async (argv) => {
|
|
78
98
|
const engine = new EmbeddingEngine({ storePath: argv.flags.storePath });
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
99
|
+
try {
|
|
100
|
+
const [query] = argv._;
|
|
101
|
+
const results = await engine.search(query, argv.flags.limit, argv.flags.minSimilarity);
|
|
102
|
+
if (results.length === 0) console.log("No results found");
|
|
103
|
+
else {
|
|
104
|
+
console.log(`Found ${results.length} result(s):\n`);
|
|
105
|
+
for (const result of results) console.log(`${result.key}: ${result.similarity.toFixed(6)}`);
|
|
106
|
+
}
|
|
107
|
+
} finally {
|
|
108
|
+
await engine.dispose();
|
|
109
|
+
}
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
//#endregion
|
|
113
|
+
//#region src/commands/delete.ts
|
|
114
|
+
const deleteCmd = command({
|
|
115
|
+
name: "delete",
|
|
116
|
+
parameters: ["<key>"],
|
|
117
|
+
flags: { ...sharedFlags },
|
|
118
|
+
help: {
|
|
119
|
+
description: "Delete an embedding entry by key",
|
|
120
|
+
examples: ["raptor delete doc1", "raptor delete -s ./my-db.raptor mykey"]
|
|
121
|
+
}
|
|
122
|
+
}, async (argv) => {
|
|
123
|
+
const engine = new EmbeddingEngine({ storePath: argv.flags.storePath });
|
|
124
|
+
try {
|
|
125
|
+
const [key] = argv._;
|
|
126
|
+
if (await engine.delete(key)) console.log(`Deleted key "${key}"`);
|
|
127
|
+
else {
|
|
128
|
+
console.log(`Key "${key}" not found`);
|
|
129
|
+
process.exit(1);
|
|
130
|
+
}
|
|
131
|
+
} finally {
|
|
132
|
+
await engine.dispose();
|
|
133
|
+
}
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
//#endregion
|
|
137
|
+
//#region src/commands/wal.ts
|
|
138
|
+
const opTypeNames = {
|
|
139
|
+
[opType.insert]: "INSERT",
|
|
140
|
+
[opType.update]: "UPDATE",
|
|
141
|
+
[opType.delete]: "DELETE"
|
|
142
|
+
};
|
|
143
|
+
function formatKeyHash(hash) {
|
|
144
|
+
return Array.from(hash.slice(0, 8)).map((b) => b.toString(16).padStart(2, "0")).join("");
|
|
145
|
+
}
|
|
146
|
+
const walCmd = command({
|
|
147
|
+
name: "wal",
|
|
148
|
+
flags: { ...sharedFlags },
|
|
149
|
+
help: {
|
|
150
|
+
description: "Display WAL entries in human-readable format",
|
|
151
|
+
examples: ["raptor wal", "raptor wal -s ./my-db.raptor"]
|
|
152
|
+
}
|
|
153
|
+
}, async (argv) => {
|
|
154
|
+
const wal = new Wal(argv.flags.storePath.replace(/\.raptor$/, "") + fileExtensions.wal);
|
|
155
|
+
try {
|
|
156
|
+
let count = 0;
|
|
157
|
+
for await (const entry of wal.recover()) {
|
|
158
|
+
const opName = opTypeNames[entry.opType] ?? `UNKNOWN(${entry.opType})`;
|
|
159
|
+
const keyHashHex = formatKeyHash(entry.keyHash);
|
|
160
|
+
console.log(`[${entry.sequenceNumber}] ${opName}`);
|
|
161
|
+
console.log(` offset: ${entry.offset}, length: ${entry.length}`);
|
|
162
|
+
console.log(` keyHash: ${keyHashHex}`);
|
|
163
|
+
console.log();
|
|
164
|
+
count++;
|
|
165
|
+
}
|
|
166
|
+
if (count === 0) console.log("WAL is empty");
|
|
167
|
+
else console.log(`Total: ${count} entries`);
|
|
168
|
+
} finally {
|
|
169
|
+
await wal.close();
|
|
85
170
|
}
|
|
86
171
|
});
|
|
87
172
|
|
|
@@ -96,7 +181,9 @@ function main() {
|
|
|
96
181
|
commands: [
|
|
97
182
|
store,
|
|
98
183
|
get,
|
|
99
|
-
search
|
|
184
|
+
search,
|
|
185
|
+
deleteCmd,
|
|
186
|
+
walCmd
|
|
100
187
|
]
|
|
101
188
|
}, () => {});
|
|
102
189
|
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
export declare const deleteCmd: import("cleye").Command<{
|
|
2
|
+
name: "delete";
|
|
3
|
+
parameters: "<key>"[];
|
|
4
|
+
flags: {
|
|
5
|
+
storePath: {
|
|
6
|
+
readonly type: StringConstructor;
|
|
7
|
+
readonly description: "Path to the embeddings store file";
|
|
8
|
+
readonly default: "./database.raptor";
|
|
9
|
+
readonly alias: "s";
|
|
10
|
+
};
|
|
11
|
+
};
|
|
12
|
+
help: {
|
|
13
|
+
description: string;
|
|
14
|
+
examples: string[];
|
|
15
|
+
};
|
|
16
|
+
}, {
|
|
17
|
+
command: "delete";
|
|
18
|
+
} & import("type-flag").TypeFlag<{
|
|
19
|
+
storePath: {
|
|
20
|
+
readonly type: StringConstructor;
|
|
21
|
+
readonly description: "Path to the embeddings store file";
|
|
22
|
+
readonly default: "./database.raptor";
|
|
23
|
+
readonly alias: "s";
|
|
24
|
+
};
|
|
25
|
+
} & {
|
|
26
|
+
help: BooleanConstructor;
|
|
27
|
+
}> & {
|
|
28
|
+
_: {
|
|
29
|
+
key: string;
|
|
30
|
+
};
|
|
31
|
+
showHelp: (options?: {
|
|
32
|
+
version?: string;
|
|
33
|
+
description?: string;
|
|
34
|
+
usage?: false | string | string[];
|
|
35
|
+
examples?: string | string[];
|
|
36
|
+
render?: (nodes: {
|
|
37
|
+
id?: string;
|
|
38
|
+
type: keyof import("cleye").Renderers;
|
|
39
|
+
data: any;
|
|
40
|
+
}[], renderers: import("cleye").Renderers) => string;
|
|
41
|
+
}) => void;
|
|
42
|
+
showVersion: () => void;
|
|
43
|
+
}>;
|
|
44
|
+
//# sourceMappingURL=delete.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"delete.d.ts","sourceRoot":"","sources":["../../src/commands/delete.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;sBAmC88L,CAAC;;;;;;;;;;;;EADp+L,CAAA"}
|
package/dist/commands/get.d.ts
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
export declare const get: import("cleye").Command<{
|
|
2
2
|
name: "get";
|
|
3
|
-
description: string;
|
|
4
3
|
parameters: "<key>"[];
|
|
5
4
|
flags: {
|
|
6
5
|
storePath: {
|
|
@@ -10,6 +9,10 @@ export declare const get: import("cleye").Command<{
|
|
|
10
9
|
readonly alias: "s";
|
|
11
10
|
};
|
|
12
11
|
};
|
|
12
|
+
help: {
|
|
13
|
+
description: string;
|
|
14
|
+
examples: string[];
|
|
15
|
+
};
|
|
13
16
|
}, {
|
|
14
17
|
command: "get";
|
|
15
18
|
} & import("type-flag").TypeFlag<{
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"get.d.ts","sourceRoot":"","sources":["../../src/commands/get.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,GAAG
|
|
1
|
+
{"version":3,"file":"get.d.ts","sourceRoot":"","sources":["../../src/commands/get.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;sBA6CwvL,CAAC;;;;;;;;;;;;EADxwL,CAAA"}
|
package/dist/commands/index.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/commands/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,SAAS,CAAA;AAC/B,OAAO,EAAE,GAAG,EAAE,MAAM,OAAO,CAAA;AAC3B,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAA"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/commands/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,SAAS,CAAA;AAC/B,OAAO,EAAE,GAAG,EAAE,MAAM,OAAO,CAAA;AAC3B,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAA;AACjC,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAA;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,OAAO,CAAA"}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
export declare const search: import("cleye").Command<{
|
|
2
2
|
name: "search";
|
|
3
|
-
description: string;
|
|
4
3
|
parameters: "<query>"[];
|
|
5
4
|
flags: {
|
|
6
5
|
limit: {
|
|
@@ -22,6 +21,10 @@ export declare const search: import("cleye").Command<{
|
|
|
22
21
|
readonly alias: "s";
|
|
23
22
|
};
|
|
24
23
|
};
|
|
24
|
+
help: {
|
|
25
|
+
description: string;
|
|
26
|
+
examples: string[];
|
|
27
|
+
};
|
|
25
28
|
}, {
|
|
26
29
|
command: "search";
|
|
27
30
|
} & import("type-flag").TypeFlag<{
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/commands/search.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,MAAM
|
|
1
|
+
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/commands/search.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;sBA2CgoL,CAAC;;;;;;;;;;;;EADnpL,CAAA"}
|
package/dist/commands/store.d.ts
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
export declare const store: import("cleye").Command<{
|
|
2
2
|
name: "store";
|
|
3
|
-
description: string;
|
|
4
3
|
parameters: ("<key>" | "<text>")[];
|
|
5
4
|
flags: {
|
|
6
5
|
storePath: {
|
|
@@ -10,6 +9,10 @@ export declare const store: import("cleye").Command<{
|
|
|
10
9
|
readonly alias: "s";
|
|
11
10
|
};
|
|
12
11
|
};
|
|
12
|
+
help: {
|
|
13
|
+
description: string;
|
|
14
|
+
examples: string[];
|
|
15
|
+
};
|
|
13
16
|
}, {
|
|
14
17
|
command: "store";
|
|
15
18
|
} & import("type-flag").TypeFlag<{
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/commands/store.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,KAAK
|
|
1
|
+
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/commands/store.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,KAAK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;sBA6BygM,CAAC;;;;;;;;;;;;EAD3hM,CAAA"}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
export declare const walCmd: import("cleye").Command<{
|
|
2
|
+
name: "wal";
|
|
3
|
+
flags: {
|
|
4
|
+
storePath: {
|
|
5
|
+
readonly type: StringConstructor;
|
|
6
|
+
readonly description: "Path to the embeddings store file";
|
|
7
|
+
readonly default: "./database.raptor";
|
|
8
|
+
readonly alias: "s";
|
|
9
|
+
};
|
|
10
|
+
};
|
|
11
|
+
help: {
|
|
12
|
+
description: string;
|
|
13
|
+
examples: string[];
|
|
14
|
+
};
|
|
15
|
+
}, {
|
|
16
|
+
command: "wal";
|
|
17
|
+
} & import("type-flag").TypeFlag<{
|
|
18
|
+
storePath: {
|
|
19
|
+
readonly type: StringConstructor;
|
|
20
|
+
readonly description: "Path to the embeddings store file";
|
|
21
|
+
readonly default: "./database.raptor";
|
|
22
|
+
readonly alias: "s";
|
|
23
|
+
};
|
|
24
|
+
} & {
|
|
25
|
+
help: BooleanConstructor;
|
|
26
|
+
}> & {
|
|
27
|
+
_: {};
|
|
28
|
+
showHelp: (options?: {
|
|
29
|
+
version?: string;
|
|
30
|
+
description?: string;
|
|
31
|
+
usage?: false | string | string[];
|
|
32
|
+
examples?: string | string[];
|
|
33
|
+
render?: (nodes: {
|
|
34
|
+
id?: string;
|
|
35
|
+
type: keyof import("cleye").Renderers;
|
|
36
|
+
data: any;
|
|
37
|
+
}[], renderers: import("cleye").Renderers) => string;
|
|
38
|
+
}) => void;
|
|
39
|
+
showVersion: () => void;
|
|
40
|
+
}>;
|
|
41
|
+
//# sourceMappingURL=wal.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"wal.d.ts","sourceRoot":"","sources":["../../src/commands/wal.ts"],"names":[],"mappings":"AAiBA,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;sBA4CysK,CAAC;;;;;;;;;;;;EAD5tK,CAAA"}
|