@mirk/store 0.4.1 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/sqlite.js +17 -15
- package/dist/{chunk-GC4MYP45.js → chunk-BXM3YDOC.js} +10 -0
- package/dist/{chunk-ZFSKQHBT.js → chunk-EDVHBRXG.js} +5 -3
- package/dist/index.d.ts +1 -1
- package/dist/index.js +4 -2
- package/dist/vector.d.ts +7 -1
- package/dist/vector.js +4 -2
- package/package.json +1 -1
package/dist/adapters/sqlite.js
CHANGED
|
@@ -2,8 +2,9 @@ import {
|
|
|
2
2
|
assertDimensions,
|
|
3
3
|
bufferToVector,
|
|
4
4
|
cosineSimilarity,
|
|
5
|
+
isUsableVector,
|
|
5
6
|
vectorToBuffer
|
|
6
|
-
} from "../chunk-
|
|
7
|
+
} from "../chunk-BXM3YDOC.js";
|
|
7
8
|
|
|
8
9
|
// src/adapters/sqlite.ts
|
|
9
10
|
import Database from "better-sqlite3";
|
|
@@ -25,15 +26,6 @@ function tryLoadSqliteVec(db) {
|
|
|
25
26
|
return false;
|
|
26
27
|
}
|
|
27
28
|
}
|
|
28
|
-
function isUsableVector(v) {
|
|
29
|
-
let nonZero = false;
|
|
30
|
-
for (let i = 0; i < v.length; i++) {
|
|
31
|
-
const x = v[i] ?? 0;
|
|
32
|
-
if (!Number.isFinite(x)) return false;
|
|
33
|
-
if (x !== 0) nonZero = true;
|
|
34
|
-
}
|
|
35
|
-
return nonZero;
|
|
36
|
-
}
|
|
37
29
|
var SqliteAdapter = class {
|
|
38
30
|
db;
|
|
39
31
|
kv;
|
|
@@ -337,6 +329,9 @@ var SqliteVectorFacet = class {
|
|
|
337
329
|
return scored.slice(0, topK);
|
|
338
330
|
}
|
|
339
331
|
};
|
|
332
|
+
function jsonPath(field) {
|
|
333
|
+
return `$."${field.replace(/"/g, '""')}"`;
|
|
334
|
+
}
|
|
340
335
|
function buildWhereClause(filter) {
|
|
341
336
|
if (!filter?.where || Object.keys(filter.where).length === 0) {
|
|
342
337
|
return { clause: "", params: [] };
|
|
@@ -344,18 +339,25 @@ function buildWhereClause(filter) {
|
|
|
344
339
|
const conditions = [];
|
|
345
340
|
const params = [];
|
|
346
341
|
for (const [key, value] of Object.entries(filter.where)) {
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
342
|
+
const path = jsonPath(key);
|
|
343
|
+
if (value === null) {
|
|
344
|
+
conditions.push(`json_type(data, ?) = 'null'`);
|
|
345
|
+
params.push(path);
|
|
346
|
+
} else {
|
|
347
|
+
const bound = typeof value === "boolean" ? value ? 1 : 0 : value;
|
|
348
|
+
conditions.push(`json_extract(data, ?) = ?`);
|
|
349
|
+
params.push(path, bound);
|
|
350
|
+
}
|
|
350
351
|
}
|
|
351
352
|
return { clause: ` WHERE ${conditions.join(" AND ")}`, params };
|
|
352
353
|
}
|
|
353
354
|
function buildOrderBy(filter) {
|
|
354
355
|
if (!filter?.sortBy) return { clause: "", params: [] };
|
|
355
356
|
const dir = filter.sortDir === "desc" ? "DESC" : "ASC";
|
|
357
|
+
const path = jsonPath(filter.sortBy);
|
|
356
358
|
return {
|
|
357
|
-
clause: ` ORDER BY json_extract(data,
|
|
358
|
-
params: [
|
|
359
|
+
clause: ` ORDER BY json_extract(data, ?) IS NULL, json_extract(data, ?) ${dir}`,
|
|
360
|
+
params: [path, path]
|
|
359
361
|
};
|
|
360
362
|
}
|
|
361
363
|
function hashName(s) {
|
|
@@ -29,6 +29,15 @@ function bufferToVector(buf) {
|
|
|
29
29
|
}
|
|
30
30
|
return out;
|
|
31
31
|
}
|
|
32
|
+
function isUsableVector(v) {
|
|
33
|
+
let nonZero = false;
|
|
34
|
+
for (let i = 0; i < v.length; i++) {
|
|
35
|
+
const x = v[i] ?? 0;
|
|
36
|
+
if (!Number.isFinite(x)) return false;
|
|
37
|
+
if (x !== 0) nonZero = true;
|
|
38
|
+
}
|
|
39
|
+
return nonZero;
|
|
40
|
+
}
|
|
32
41
|
function assertDimensions(vector, dimensions) {
|
|
33
42
|
if (vector.length !== dimensions) {
|
|
34
43
|
throw new Error(`Vector dimension mismatch: expected ${dimensions}, got ${vector.length}`);
|
|
@@ -39,5 +48,6 @@ export {
|
|
|
39
48
|
cosineSimilarity,
|
|
40
49
|
vectorToBuffer,
|
|
41
50
|
bufferToVector,
|
|
51
|
+
isUsableVector,
|
|
42
52
|
assertDimensions
|
|
43
53
|
};
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import {
|
|
2
2
|
assertDimensions,
|
|
3
|
-
cosineSimilarity
|
|
4
|
-
|
|
3
|
+
cosineSimilarity,
|
|
4
|
+
isUsableVector
|
|
5
|
+
} from "./chunk-BXM3YDOC.js";
|
|
5
6
|
|
|
6
7
|
// src/vector/memory.ts
|
|
7
8
|
var InMemoryVectorStore = class {
|
|
@@ -39,12 +40,13 @@ var InMemoryVectorStore = class {
|
|
|
39
40
|
if (!coll) return [];
|
|
40
41
|
const scored = [];
|
|
41
42
|
for (const doc of coll.values()) {
|
|
43
|
+
if (!isUsableVector(doc.vector)) continue;
|
|
42
44
|
const score = cosineSimilarity(query, doc.vector);
|
|
43
45
|
if (!Number.isFinite(score)) continue;
|
|
44
46
|
if (minScore !== void 0 && score < minScore) continue;
|
|
45
47
|
scored.push({ id: doc.id, score, metadata: doc.metadata });
|
|
46
48
|
}
|
|
47
|
-
scored.sort((a, b) => b.score - a.score);
|
|
49
|
+
scored.sort((a, b) => b.score - a.score || a.id.localeCompare(b.id));
|
|
48
50
|
return scored.slice(0, topK);
|
|
49
51
|
}
|
|
50
52
|
collectionFor(name) {
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
export { A as AsyncStore, S as StoreFilter, a as StoreMeta, b as SyncStore } from './types-DyQLNtxa.js';
|
|
2
2
|
export { InMemoryKv, toAsync } from './kv.js';
|
|
3
3
|
export { V as Vector, a as VectorDocument, b as VectorSearchOptions, c as VectorSearchResult, d as VectorStore, e as VectorStoreMeta } from './types-BqSZEMAB.js';
|
|
4
|
-
export { InMemoryVectorStore, assertDimensions, bufferToVector, cosineSimilarity, vectorToBuffer } from './vector.js';
|
|
4
|
+
export { InMemoryVectorStore, assertDimensions, bufferToVector, cosineSimilarity, isUsableVector, vectorToBuffer } from './vector.js';
|
package/dist/index.js
CHANGED
|
@@ -4,19 +4,21 @@ import {
|
|
|
4
4
|
} from "./chunk-ZI4JA6IU.js";
|
|
5
5
|
import {
|
|
6
6
|
InMemoryVectorStore
|
|
7
|
-
} from "./chunk-
|
|
7
|
+
} from "./chunk-EDVHBRXG.js";
|
|
8
8
|
import {
|
|
9
9
|
assertDimensions,
|
|
10
10
|
bufferToVector,
|
|
11
11
|
cosineSimilarity,
|
|
12
|
+
isUsableVector,
|
|
12
13
|
vectorToBuffer
|
|
13
|
-
} from "./chunk-
|
|
14
|
+
} from "./chunk-BXM3YDOC.js";
|
|
14
15
|
export {
|
|
15
16
|
InMemoryStore as InMemoryKv,
|
|
16
17
|
InMemoryVectorStore,
|
|
17
18
|
assertDimensions,
|
|
18
19
|
bufferToVector,
|
|
19
20
|
cosineSimilarity,
|
|
21
|
+
isUsableVector,
|
|
20
22
|
toAsync,
|
|
21
23
|
vectorToBuffer
|
|
22
24
|
};
|
package/dist/vector.d.ts
CHANGED
|
@@ -10,6 +10,12 @@ declare function cosineSimilarity(a: Vector, b: Vector): number;
|
|
|
10
10
|
declare function vectorToBuffer(vec: Vector): Buffer;
|
|
11
11
|
/** Inverse of vectorToBuffer. */
|
|
12
12
|
declare function bufferToVector(buf: Buffer | Uint8Array): Vector;
|
|
13
|
+
/** A vector is usable for cosine similarity only if it is all-finite and has a
|
|
14
|
+
* non-zero magnitude. A zero / non-finite vector has no cosine direction, so it is
|
|
15
|
+
* EXCLUDED from search results on EVERY backend (in-memory, sqlite JS path, and
|
|
16
|
+
* sqlite's vec0 path) — that keeps the backends in parity (vec0 would otherwise
|
|
17
|
+
* return a zero vector with a null distance). Shared so the rule lives in one place. */
|
|
18
|
+
declare function isUsableVector(v: Vector): boolean;
|
|
13
19
|
/** Throw if a vector's length doesn't match the store's configured dimensions.
|
|
14
20
|
* Shared by every backend so the check and its message live in one place. */
|
|
15
21
|
declare function assertDimensions(vector: Vector, dimensions: number): void;
|
|
@@ -32,4 +38,4 @@ declare class InMemoryVectorStore implements VectorStore {
|
|
|
32
38
|
private collectionFor;
|
|
33
39
|
}
|
|
34
40
|
|
|
35
|
-
export { InMemoryVectorStore, Vector, VectorDocument, VectorSearchOptions, VectorSearchResult, VectorStore, VectorStoreMeta, assertDimensions, bufferToVector, cosineSimilarity, vectorToBuffer };
|
|
41
|
+
export { InMemoryVectorStore, Vector, VectorDocument, VectorSearchOptions, VectorSearchResult, VectorStore, VectorStoreMeta, assertDimensions, bufferToVector, cosineSimilarity, isUsableVector, vectorToBuffer };
|
package/dist/vector.js
CHANGED
|
@@ -1,16 +1,18 @@
|
|
|
1
1
|
import {
|
|
2
2
|
InMemoryVectorStore
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-EDVHBRXG.js";
|
|
4
4
|
import {
|
|
5
5
|
assertDimensions,
|
|
6
6
|
bufferToVector,
|
|
7
7
|
cosineSimilarity,
|
|
8
|
+
isUsableVector,
|
|
8
9
|
vectorToBuffer
|
|
9
|
-
} from "./chunk-
|
|
10
|
+
} from "./chunk-BXM3YDOC.js";
|
|
10
11
|
export {
|
|
11
12
|
InMemoryVectorStore,
|
|
12
13
|
assertDimensions,
|
|
13
14
|
bufferToVector,
|
|
14
15
|
cosineSimilarity,
|
|
16
|
+
isUsableVector,
|
|
15
17
|
vectorToBuffer
|
|
16
18
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mirk/store",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.2",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Code-split storage ports + source adapters under one namespace. KV + collection store and vector similarity store as interface subpaths; the sqlite source-adapter implements both over one connection.",
|
|
6
6
|
"license": "Apache-2.0",
|