@zukhruf/embeddings 0.8.30 → 0.8.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/collection.d.ts.map +1 -1
- package/dist/collection.js +9 -7
- package/dist/collection.js.map +1 -1
- package/dist/vector-search.d.ts.map +1 -1
- package/dist/vector-search.js +14 -11
- package/dist/vector-search.js.map +1 -1
- package/dist/vector.d.ts +2 -10
- package/dist/vector.d.ts.map +1 -1
- package/dist/vector.js +13 -40
- package/dist/vector.js.map +1 -1
- package/package.json +7 -6
package/README.md
CHANGED
|
@@ -64,7 +64,7 @@ collection.count(); // 2
|
|
|
64
64
|
|
|
65
65
|
### Search a vector table of your own
|
|
66
66
|
|
|
67
|
-
Another program can write the vectors into its own table. A `VectorSearch` searches that table with the same rules as a collection. The table or view must have the columns `id` (text) and `vector`. Each `vector` must be a unit vector, as a BLOB of float32 values
|
|
67
|
+
Another program can write the vectors into its own table. A `VectorSearch` searches that table with the same rules as a collection. The table or view must have the columns `id` (text) and `vector`. Each `vector` must be a unit vector, as a BLOB of float32 values, little-endian, as `toSql` of `@zukhruf/vectors/sqlite` writes it ([ADR 0005](./docs/adr/0005-a-vector-search-reads-a-vector-table-of-unit-vectors.md)). Each call except a query without a date range also reads the column `published_at` (milliseconds since the Unix epoch). Thus a table of documents with no publish date supports a query without a date range ([ADR 0007](./docs/adr/0007-a-query-without-a-date-range-compares-every-document.md)).
|
|
68
68
|
|
|
69
69
|
```ts
|
|
70
70
|
import { DatabaseSync } from 'node:sqlite';
|
|
@@ -103,7 +103,7 @@ To search a table with other column names, make a view that renames the columns,
|
|
|
103
103
|
- One database holds one collection.
|
|
104
104
|
- The collection stores the vector of the text, not the text. The caller chooses the text, for example a title and the first sentences.
|
|
105
105
|
- A document has one vector. A long text gives one mean vector, and its details become a small part of it.
|
|
106
|
-
- A vector search requires unit vectors. It does not examine the length
|
|
106
|
+
- A vector search requires unit vectors. It does not examine the length of a vector. Two vectors of a vector table with other dimensions are the `DimensionMismatchError` of `@zukhruf/vectors`, from its `dot`.
|
|
107
107
|
- The search reads each document in the window ([ADR 0001](./docs/adr/0001-search-compares-each-document-in-the-window.md)). A window of many thousand documents is slow. A neighbour graph decodes each vector once, but it still calculates a similarity for each pair of documents in a window.
|
|
108
108
|
- After a model change, upsert all documents again ([ADR 0002](./docs/adr/0002-the-hash-includes-the-model.md)).
|
|
109
109
|
|
package/dist/collection.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"collection.d.ts","sourceRoot":"","sources":["../src/collection.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,OAAO,EAAE,KAAK,cAAc,EAAa,MAAM,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"collection.d.ts","sourceRoot":"","sources":["../src/collection.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,OAAO,EAAE,KAAK,cAAc,EAAa,MAAM,IAAI,CAAC;AASpD,OAAO,EAAE,KAAK,SAAS,EAAgB,MAAM,oBAAoB,CAAC;AAGlE,MAAM,MAAM,QAAQ,GAAG;IACrB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAEpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAE9B,QAAQ,CAAC,QAAQ,EAAE,YAAY,CAAC;IAGhC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;CACjD,CAAC;AAIF,qBAAa,UAAU;;gBAKT,OAAO,EAAE,iBAAiB;IAYhC,MAAM,CAAC,SAAS,EAAE,SAAS,QAAQ,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAoD3D,OAAO,CACL,EAAE,EAAE,MAAM,EACV,OAAO,EAAE;QAAE,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;KAAE,GAC3D,SAAS,EAAE;IAKd,cAAc,CAAC,OAAO,EAAE;QACtB,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;QACpB,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;QAClB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QACnB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;KAC7B,GAAG,cAAc,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IAKzC,UAAU,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM;IAOxC,KAAK,CACH,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,KAAK,CAAC,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;QAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAA;KAAE,GACjD,SAAS,EAAE;IAMd,KAAK,CAAC,GAAG,EAAE,SAAS,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC;IAexD,MAAM,CAAC,GAAG,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI;IASpC,KAAK,IAAI,MAAM;CA4ChB"}
|
package/dist/collection.js
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import { hash } from 'node:crypto';
|
|
2
2
|
import { embedMany } from 'ai';
|
|
3
3
|
import { transaction } from '@zukhruf/sqlite-transaction';
|
|
4
|
+
import { Vector } from '@zukhruf/vectors';
|
|
5
|
+
import { fromSql, toSql } from '@zukhruf/vectors/sqlite';
|
|
4
6
|
import { DimensionMismatchError } from "./dimension-mismatch-error.js";
|
|
5
7
|
import { collectionSchema } from "./schema.js";
|
|
6
8
|
import { UnknownDocumentError } from "./unknown-document-error.js";
|
|
7
9
|
import { VectorSearch } from "./vector-search.js";
|
|
8
|
-
import {
|
|
10
|
+
import { blob, normalize } from "./vector.js";
|
|
9
11
|
// The embeddings of documents, one vector for each document, in a SQLite
|
|
10
12
|
// database. The collection searches its table through a VectorSearch.
|
|
11
13
|
export class Collection {
|
|
@@ -38,7 +40,7 @@ export class Collection {
|
|
|
38
40
|
const vector = vectors[index];
|
|
39
41
|
if (vector === undefined)
|
|
40
42
|
throw new Error(`The model gave ${vectors.length} embeddings for ${changed.length} texts.`);
|
|
41
|
-
write.run(document.id, hash, document.publishedAt.getTime(), vector
|
|
43
|
+
write.run(document.id, hash, document.publishedAt.getTime(), toSql(vector));
|
|
42
44
|
}
|
|
43
45
|
const move = this.#database.prepare('UPDATE "embedding_document" SET "published_at" = ? WHERE "id" = ?');
|
|
44
46
|
for (const { document } of unchanged)
|
|
@@ -72,7 +74,7 @@ export class Collection {
|
|
|
72
74
|
const row = read.get(id);
|
|
73
75
|
if (row === undefined)
|
|
74
76
|
throw new UnknownDocumentError(id);
|
|
75
|
-
return [id,
|
|
77
|
+
return [id, fromSql(Vector, blob(row.vector)).values];
|
|
76
78
|
}));
|
|
77
79
|
}
|
|
78
80
|
// Removes the documents. An id that the collection does not store is not an
|
|
@@ -97,7 +99,7 @@ export class Collection {
|
|
|
97
99
|
model: this.#model,
|
|
98
100
|
values: texts,
|
|
99
101
|
});
|
|
100
|
-
return embeddings.map((embedding) =>
|
|
102
|
+
return embeddings.map((embedding) => normalize(embedding));
|
|
101
103
|
}
|
|
102
104
|
// The model is part of the hash, so a new model embeds the text again.
|
|
103
105
|
#hash(text) {
|
|
@@ -114,11 +116,11 @@ export class Collection {
|
|
|
114
116
|
.prepare('SELECT length("vector") AS "bytes" FROM "embedding_document" WHERE "id" NOT IN (SELECT "value" FROM json_each(?)) LIMIT 1')
|
|
115
117
|
.get(JSON.stringify(replaced));
|
|
116
118
|
const expected = kept === undefined
|
|
117
|
-
? first.
|
|
119
|
+
? first.dimensions
|
|
118
120
|
: Number(kept.bytes) / Float32Array.BYTES_PER_ELEMENT;
|
|
119
121
|
for (const vector of vectors)
|
|
120
|
-
if (vector.
|
|
121
|
-
throw new DimensionMismatchError(expected, vector.
|
|
122
|
+
if (vector.dimensions !== expected)
|
|
123
|
+
throw new DimensionMismatchError(expected, vector.dimensions);
|
|
122
124
|
}
|
|
123
125
|
}
|
|
124
126
|
//# sourceMappingURL=collection.js.map
|
package/dist/collection.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"collection.js","sourceRoot":"","sources":["../src/collection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAGnC,OAAO,EAAuB,SAAS,EAAE,MAAM,IAAI,CAAC;AAEpD,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"collection.js","sourceRoot":"","sources":["../src/collection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAGnC,OAAO,EAAuB,SAAS,EAAE,MAAM,IAAI,CAAC;AAEpD,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAC1D,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAEzD,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AACvE,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAkB,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAiB9C,yEAAyE;AACzE,sEAAsE;AACtE,MAAM,OAAO,UAAU;IACZ,SAAS,CAAe;IACxB,MAAM,CAAkC;IACxC,OAAO,CAAe;IAE/B,YAAY,OAA0B;QACpC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAC;QAClC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC;QAC5B,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACvC,IAAI,CAAC,OAAO,GAAG,IAAI,YAAY,CAAC;YAC9B,QAAQ,EAAE,IAAI,CAAC,SAAS;YACxB,KAAK,EAAE,oBAAoB;SAC5B,CAAC,CAAC;IACL,CAAC;IAED,4EAA4E;IAC5E,gDAAgD;IAChD,KAAK,CAAC,MAAM,CAAC,SAA8B;QACzC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CACnC,wDAAwD,CACzD,CAAC;QACF,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAC1C,QAAQ;YACR,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC;SAChC,CAAC,CAAC,CAAC;QACJ,MAAM,EAAE,OAAO,GAAG,EAAE,EAAE,SAAS,GAAG,EAAE,EAAE,GAAG,MAAM,CAAC,OAAO,CACrD,MAAM,EACN,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE,CACrB,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CACnE,CAAC;QACF,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAC/B,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAC7C,CAAC;QAEF,WAAW,CACT,IAAI,CAAC,SAAS,EACd,GAAG,EAAE;YACH,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAClC;4IACkI,CACnI,CAAC;YACF,KAAK,MAAM,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;gBAC5D,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;gBAC9B,IAAI,MAAM,KAAK,SAAS;oBACtB,MAAM,IAAI,KAAK,CACb,kBAAkB,OAAO,CAAC,MAAM,mBAAmB,OAAO,CAAC,MAAM,SAAS,CAC3E,CAAC;gBACJ,KAAK,CAAC,GAAG,CACP,QAAQ,CAAC,EAAE,EACX,IAAI,EACJ,QAAQ,CAAC,WAAW,CAAC,OAAO,EAAE,EAC9B,KAAK,CAAC,MAAM,CAAC,CACd,CAAC;YACJ,CAAC;YACD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CACjC,mEAAmE,CACpE,CAAC;YACF,KAAK,MAAM,EAAE,QAAQ,EAAE,IAAI,SAAS;gBAClC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;YACxD,IAAI,CAAC,eAAe,CAClB,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,EAC1C,OAAO,CACR,CAAC;QACJ,CAAC,EACD,EAAE,IAAI,EAAE,WAAW,EAAE,CACtB,CAAC;IACJ,CAAC;IAED,gDAAgD;IAChD,OAAO,CACL,EAAU,EACV,OAA4D;QAE5D,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;IAC3C,CAAC;IAED,iEAAiE;IACjE,cAAc,CAAC,OAKd;QACC,OAAO,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;IAC9C,CAAC;IAED,iDAAiD;IACjD,UAAU,CAAC,CAAS,EAAE,CAAS;QAC7B,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,4EAA4E;IAC5E,0EAA0E;IAC1E,kCAAkC;IAClC,KAAK,CACH,MAAyB,EACzB,KAAkD;QAElD,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC3C,CAAC;IAED,6EAA6E;IAC7E,kCAAkC;IAClC,KAAK,CAAC,GAAsB;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CACjC,0DAA0D,CAC3D,CAAC;QACF,OAAO,IAAI,GAAG,CACZ,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;YACb,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACzB,IAAI,GAAG,KAAK,SAAS;gBAAE,MAAM,IAAI,oBAAoB,CAAC,EAAE,CAAC,CAAC;YAC1D,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACxD,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;IAED,4EAA4E;IAC5E,0EAA0E;IAC1E,MAAM,CAAC,GAAsB;QAC3B,IAAI,CAAC,SAAS;aACX,OAAO,CACN,mFAAmF,CACpF;aACA,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9B,CAAC;IAED,kCAAkC;IAClC,KAAK;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS;aACvB,OAAO,CAAC,sDAAsD,CAAC;aAC/D,GAAG,EAAE,CAAC;QACT,OAAO,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAC5B,CAAC;IAED,sCAAsC;IACtC,KAAK,CAAC,MAAM,CAAC,KAAe;QAC1B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAClC,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,SAAS,CAAC;YACrC,KAAK,EAAE,IAAI,CAAC,MAAM;YAClB,MAAM,EAAE,KAAK;SACd,CAAC,CAAC;QACH,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;IAC7D,CAAC;IAED,uEAAuE;IACvE,KAAK,CAAC,IAAY;QAChB,OAAO,IAAI,CACT,QAAQ,EACR,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAClE,CAAC;IACJ,CAAC;IAED,2EAA2E;IAC3E,2EAA2E;IAC3E,iBAAiB;IACjB,eAAe,CAAC,QAA2B,EAAE,OAA0B;QACrE,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC;QACxB,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS;aACxB,OAAO,CACN,2HAA2H,CAC5H;aACA,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;QACjC,MAAM,QAAQ,GACZ,IAAI,KAAK,SAAS;YAChB,CAAC,CAAC,KAAK,CAAC,UAAU;YAClB,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,YAAY,CAAC,iBAAiB,CAAC;QAC1D,KAAK,MAAM,MAAM,IAAI,OAAO;YAC1B,IAAI,MAAM,CAAC,UAAU,KAAK,QAAQ;gBAChC,MAAM,IAAI,sBAAsB,CAAC,QAAQ,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IACpE,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vector-search.d.ts","sourceRoot":"","sources":["../src/vector-search.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"vector-search.d.ts","sourceRoot":"","sources":["../src/vector-search.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAWhD,MAAM,MAAM,SAAS,GAAG;IACtB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAEhC,QAAQ,CAAC,QAAQ,EAAE,YAAY,CAAC;IAIhC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB,CAAC;AAMF,qBAAa,YAAY;;gBAIX,OAAO,EAAE,mBAAmB;IAQxC,OAAO,CACL,EAAE,EAAE,MAAM,EACV,OAAO,EAAE;QAAE,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;KAAE,GAC3D,SAAS,EAAE;IAoBd,cAAc,CAAC,OAAO,EAAE;QACtB,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;QACpB,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;QAClB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QACnB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;KAC7B,GAAG,cAAc,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IAoCzC,KAAK,CACH,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,KAAK,CAAC,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;QAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAA;KAAE,GACjD,SAAS,EAAE;IA4Bd,UAAU,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM;CAgBzC"}
|
package/dist/vector-search.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
|
+
import { Vector, dot } from '@zukhruf/vectors';
|
|
2
|
+
import { fromSql } from '@zukhruf/vectors/sqlite';
|
|
1
3
|
import { DimensionMismatchError } from "./dimension-mismatch-error.js";
|
|
2
4
|
import { UnknownDocumentError } from "./unknown-document-error.js";
|
|
3
|
-
import {
|
|
5
|
+
import { blob, normalize } from "./vector.js";
|
|
4
6
|
const DAY_MS = 86_400_000;
|
|
5
7
|
// The neighbours of the documents in a vector table. A search compares the
|
|
6
8
|
// document with each document published inside the window, with no index
|
|
@@ -24,7 +26,7 @@ export class VectorSearch {
|
|
|
24
26
|
.all(publishedAt - window, publishedAt + window, id)
|
|
25
27
|
.map((row) => ({
|
|
26
28
|
id: String(row.id),
|
|
27
|
-
vector:
|
|
29
|
+
vector: fromSql(Vector, blob(row.vector)),
|
|
28
30
|
}));
|
|
29
31
|
return ranked(vector, candidates, options.k);
|
|
30
32
|
}
|
|
@@ -43,7 +45,7 @@ export class VectorSearch {
|
|
|
43
45
|
.map((row) => ({
|
|
44
46
|
id: String(row.id),
|
|
45
47
|
publishedAt: Number(row.published_at),
|
|
46
|
-
vector:
|
|
48
|
+
vector: fromSql(Vector, blob(row.vector)),
|
|
47
49
|
}));
|
|
48
50
|
return documents
|
|
49
51
|
.values()
|
|
@@ -60,15 +62,15 @@ export class VectorSearch {
|
|
|
60
62
|
// metadata only when the query names it. Equal similarities sort by id
|
|
61
63
|
// (docs/adr/0006, docs/adr/0007).
|
|
62
64
|
query(vector, range) {
|
|
63
|
-
const query =
|
|
65
|
+
const query = normalize(vector);
|
|
64
66
|
const stored = this.#database
|
|
65
67
|
.prepare(`SELECT length("vector") AS "bytes" FROM ${this.#table} LIMIT 1`)
|
|
66
68
|
.get();
|
|
67
69
|
if (stored === undefined)
|
|
68
70
|
return [];
|
|
69
71
|
const dimension = Number(stored.bytes) / Float32Array.BYTES_PER_ELEMENT;
|
|
70
|
-
if (dimension !== query.
|
|
71
|
-
throw new DimensionMismatchError(dimension, query.
|
|
72
|
+
if (dimension !== query.dimensions)
|
|
73
|
+
throw new DimensionMismatchError(dimension, query.dimensions);
|
|
72
74
|
const rows = range === undefined
|
|
73
75
|
? this.#database
|
|
74
76
|
.prepare(`SELECT "id", "vector" FROM ${this.#table}`)
|
|
@@ -78,13 +80,14 @@ export class VectorSearch {
|
|
|
78
80
|
.all(range.from.getTime(), range.to.getTime());
|
|
79
81
|
const documents = rows.map((row) => ({
|
|
80
82
|
id: String(row.id),
|
|
81
|
-
vector:
|
|
83
|
+
vector: fromSql(Vector, blob(row.vector)),
|
|
82
84
|
}));
|
|
83
85
|
return ranked(query, documents, documents.length);
|
|
84
86
|
}
|
|
85
|
-
// The cosine similarity of two documents
|
|
87
|
+
// The cosine similarity of two documents: the dot product of their unit
|
|
88
|
+
// vectors.
|
|
86
89
|
similarity(a, b) {
|
|
87
|
-
return this.#read(a).vector
|
|
90
|
+
return dot(this.#read(a).vector, this.#read(b).vector);
|
|
88
91
|
}
|
|
89
92
|
#read(id) {
|
|
90
93
|
const row = this.#database
|
|
@@ -93,7 +96,7 @@ export class VectorSearch {
|
|
|
93
96
|
if (row === undefined)
|
|
94
97
|
throw new UnknownDocumentError(id);
|
|
95
98
|
return {
|
|
96
|
-
vector:
|
|
99
|
+
vector: fromSql(Vector, blob(row.vector)),
|
|
97
100
|
publishedAt: Number(row.published_at),
|
|
98
101
|
};
|
|
99
102
|
}
|
|
@@ -103,7 +106,7 @@ function ranked(vector, candidates, k) {
|
|
|
103
106
|
return candidates
|
|
104
107
|
.map((candidate) => ({
|
|
105
108
|
id: candidate.id,
|
|
106
|
-
similarity: vector
|
|
109
|
+
similarity: dot(vector, candidate.vector),
|
|
107
110
|
}))
|
|
108
111
|
.toSorted((left, right) => right.similarity - left.similarity || compareIds(left.id, right.id))
|
|
109
112
|
.slice(0, k);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vector-search.js","sourceRoot":"","sources":["../src/vector-search.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AACvE,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"vector-search.js","sourceRoot":"","sources":["../src/vector-search.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AACvE,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAE9C,MAAM,MAAM,GAAG,UAAU,CAAC;AAgB1B,2EAA2E;AAC3E,yEAAyE;AACzE,4EAA4E;AAC5E,oBAAoB;AACpB,MAAM,OAAO,YAAY;IACd,SAAS,CAAe;IACxB,MAAM,CAAS;IAExB,YAAY,OAA4B;QACtC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAC;QAClC,IAAI,CAAC,MAAM,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC;IAC3D,CAAC;IAED,2EAA2E;IAC3E,sEAAsE;IACtE,4DAA4D;IAC5D,OAAO,CACL,EAAU,EACV,OAA4D;QAE5D,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC/C,MAAM,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,MAAM,CAAC;QAC3C,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS;aAC9B,OAAO,CACN,8BAA8B,IAAI,CAAC,MAAM,qDAAqD,CAC/F;aACA,GAAG,CAAC,WAAW,GAAG,MAAM,EAAE,WAAW,GAAG,MAAM,EAAE,EAAE,CAAC;aACnD,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACb,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YAClB,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;SAC1C,CAAC,CAAC,CAAC;QACN,OAAO,MAAM,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IAC/C,CAAC;IAED,2EAA2E;IAC3E,wEAAwE;IACxE,0DAA0D;IAC1D,4EAA4E;IAC5E,cAAc;IACd,cAAc,CAAC,OAKd;QACC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QACpC,MAAM,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,MAAM,CAAC;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS;aAC7B,OAAO,CACN,8CAA8C,IAAI,CAAC,MAAM,qEAAqE,CAC/H;aACA,GAAG,CAAC,IAAI,GAAG,MAAM,EAAE,EAAE,GAAG,MAAM,CAAC;aAC/B,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACb,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YAClB,WAAW,EAAE,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC;YACrC,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;SAC1C,CAAC,CAAC,CAAC;QACN,OAAO,SAAS;aACb,MAAM,EAAE;aACR,MAAM,CAAC,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC,WAAW,IAAI,IAAI,IAAI,WAAW,IAAI,EAAE,CAAC;aACrE,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC;YACjB,QAAQ,CAAC,EAAE;YACX,MAAM,CACJ,QAAQ,CAAC,MAAM,EACf,SAAS,CAAC,MAAM,CACd,CAAC,KAAK,EAAE,EAAE,CACR,KAAK,CAAC,EAAE,KAAK,QAAQ,CAAC,EAAE;gBACxB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,GAAG,QAAQ,CAAC,WAAW,CAAC,IAAI,MAAM,CAC/D,EACD,OAAO,CAAC,CAAC,CACV;SACF,CAAC,CAAC;IACP,CAAC;IAED,0EAA0E;IAC1E,2EAA2E;IAC3E,2EAA2E;IAC3E,uEAAuE;IACvE,kCAAkC;IAClC,KAAK,CACH,MAAyB,EACzB,KAAkD;QAElD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;QAChC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS;aAC1B,OAAO,CAAC,2CAA2C,IAAI,CAAC,MAAM,UAAU,CAAC;aACzE,GAAG,EAAE,CAAC;QACT,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO,EAAE,CAAC;QACpC,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,YAAY,CAAC,iBAAiB,CAAC;QACxE,IAAI,SAAS,KAAK,KAAK,CAAC,UAAU;YAChC,MAAM,IAAI,sBAAsB,CAAC,SAAS,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;QAChE,MAAM,IAAI,GACR,KAAK,KAAK,SAAS;YACjB,CAAC,CAAC,IAAI,CAAC,SAAS;iBACX,OAAO,CAAC,8BAA8B,IAAI,CAAC,MAAM,EAAE,CAAC;iBACpD,GAAG,EAAE;YACV,CAAC,CAAC,IAAI,CAAC,SAAS;iBACX,OAAO,CACN,8BAA8B,IAAI,CAAC,MAAM,uCAAuC,CACjF;iBACA,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;QACvD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACnC,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YAClB,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;SAC1C,CAAC,CAAC,CAAC;QACJ,OAAO,MAAM,CAAC,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;IACpD,CAAC;IAED,wEAAwE;IACxE,WAAW;IACX,UAAU,CAAC,CAAS,EAAE,CAAS;QAC7B,OAAO,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,EAAU;QACd,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS;aACvB,OAAO,CACN,wCAAwC,IAAI,CAAC,MAAM,iBAAiB,CACrE;aACA,GAAG,CAAC,EAAE,CAAC,CAAC;QACX,IAAI,GAAG,KAAK,SAAS;YAAE,MAAM,IAAI,oBAAoB,CAAC,EAAE,CAAC,CAAC;QAC1D,OAAO;YACL,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACzC,WAAW,EAAE,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC;SACtC,CAAC;IACJ,CAAC;CACF;AAED,8EAA8E;AAC9E,SAAS,MAAM,CACb,MAAc,EACd,UAAuE,EACvE,CAAS;IAET,OAAO,UAAU;SACd,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QACnB,EAAE,EAAE,SAAS,CAAC,EAAE;QAChB,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC;KAC1C,CAAC,CAAC;SACF,QAAQ,CACP,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CACd,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,CAAC,CACtE;SACA,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACjB,CAAC;AAED,0EAA0E;AAC1E,WAAW;AACX,SAAS,UAAU,CAAC,IAAY,EAAE,KAAa;IAC7C,IAAI,IAAI,GAAG,KAAK;QAAE,OAAO,CAAC,CAAC,CAAC;IAC5B,IAAI,IAAI,GAAG,KAAK;QAAE,OAAO,CAAC,CAAC;IAC3B,OAAO,CAAC,CAAC;AACX,CAAC"}
|
package/dist/vector.d.ts
CHANGED
|
@@ -1,12 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
private constructor();
|
|
4
|
-
static normalize(values: readonly number[]): UnitVector;
|
|
5
|
-
static fromBlob(blob: Uint8Array): UnitVector;
|
|
6
|
-
get dimension(): number;
|
|
7
|
-
toFloat32Array(): Float32Array;
|
|
8
|
-
toBlob(): Uint8Array;
|
|
9
|
-
cosine(other: UnitVector): number;
|
|
10
|
-
}
|
|
1
|
+
import { Vector } from '@zukhruf/vectors';
|
|
2
|
+
export declare function normalize(values: readonly number[]): Vector;
|
|
11
3
|
export declare function blob(value: unknown): Uint8Array;
|
|
12
4
|
//# sourceMappingURL=vector.d.ts.map
|
package/dist/vector.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vector.d.ts","sourceRoot":"","sources":["../src/vector.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"vector.d.ts","sourceRoot":"","sources":["../src/vector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAS1C,wBAAgB,SAAS,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAS3D;AAGD,wBAAgB,IAAI,CAAC,KAAK,EAAE,OAAO,GAAG,UAAU,CAI/C"}
|
package/dist/vector.js
CHANGED
|
@@ -1,43 +1,16 @@
|
|
|
1
|
-
|
|
2
|
-
//
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
}
|
|
15
|
-
// The bytes that SQLite gives are copied, because a Float32Array needs an
|
|
16
|
-
// offset that is a multiple of 4.
|
|
17
|
-
static fromBlob(blob) {
|
|
18
|
-
return new UnitVector(new Float32Array(Uint8Array.from(blob).buffer));
|
|
19
|
-
}
|
|
20
|
-
get dimension() {
|
|
21
|
-
return this.#values.length;
|
|
22
|
-
}
|
|
23
|
-
// A copy of the values, so the caller cannot change the vector.
|
|
24
|
-
toFloat32Array() {
|
|
25
|
-
return this.#values.slice();
|
|
26
|
-
}
|
|
27
|
-
toBlob() {
|
|
28
|
-
return new Uint8Array(this.#values.buffer, this.#values.byteOffset, this.#values.byteLength);
|
|
29
|
-
}
|
|
30
|
-
// An indexed loop: an entries() iterator with destructuring made the
|
|
31
|
-
// nearest search of each document in a window of 1,001 documents about two
|
|
32
|
-
// times slower.
|
|
33
|
-
cosine(other) {
|
|
34
|
-
const values = this.#values;
|
|
35
|
-
const others = other.#values;
|
|
36
|
-
let sum = 0;
|
|
37
|
-
for (let index = 0; index < values.length; index++)
|
|
38
|
-
sum += (values[index] ?? 0) * (others[index] ?? 0);
|
|
39
|
-
return sum;
|
|
40
|
-
}
|
|
1
|
+
import { Vector } from '@zukhruf/vectors';
|
|
2
|
+
// Divides each value by the length of the embedding, in float64, and rounds
|
|
3
|
+
// once to float32. The result is a unit vector, so its dot product with
|
|
4
|
+
// another one is their cosine similarity. It is not UnitVector.normalize of
|
|
5
|
+
// @zukhruf/vectors: a length of zero, or one that is not finite, is a
|
|
6
|
+
// RangeError for the callers of a collection (that normalize throws
|
|
7
|
+
// ZeroVectorError), and that normalize rounds each value to float32 before it
|
|
8
|
+
// divides, which can change a stored vector in the last bit.
|
|
9
|
+
export function normalize(values) {
|
|
10
|
+
const length = Math.sqrt(values.reduce((sum, value) => sum + value * value, 0));
|
|
11
|
+
if (length === 0 || !Number.isFinite(length))
|
|
12
|
+
throw new RangeError(`An embedding must have a finite length that is not zero. Its length is ${length}.`);
|
|
13
|
+
return Vector.from(Float32Array.from(values, (value) => value / length));
|
|
41
14
|
}
|
|
42
15
|
// The value of a vector column, which SQLite gives as a Uint8Array.
|
|
43
16
|
export function blob(value) {
|
package/dist/vector.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vector.js","sourceRoot":"","sources":["../src/vector.ts"],"names":[],"mappings":"AAAA,
|
|
1
|
+
{"version":3,"file":"vector.js","sourceRoot":"","sources":["../src/vector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAE1C,4EAA4E;AAC5E,wEAAwE;AACxE,4EAA4E;AAC5E,sEAAsE;AACtE,oEAAoE;AACpE,8EAA8E;AAC9E,6DAA6D;AAC7D,MAAM,UAAU,SAAS,CAAC,MAAyB;IACjD,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CACtB,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,GAAG,KAAK,EAAE,CAAC,CAAC,CACtD,CAAC;IACF,IAAI,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;QAC1C,MAAM,IAAI,UAAU,CAClB,0EAA0E,MAAM,GAAG,CACpF,CAAC;IACJ,OAAO,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC;AAC3E,CAAC;AAED,oEAAoE;AACpE,MAAM,UAAU,IAAI,CAAC,KAAc;IACjC,IAAI,CAAC,CAAC,KAAK,YAAY,UAAU,CAAC;QAChC,MAAM,IAAI,SAAS,CAAC,iCAAiC,CAAC,CAAC;IACzD,OAAO,KAAK,CAAC;AACf,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zukhruf/embeddings",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.32",
|
|
4
4
|
"description": "Document embeddings in SQLite. Embeds only new or changed text through an AI SDK model. Finds the nearest documents of a document by exact cosine similarity in a publish-date window, and the similarity of a query vector to each document of a date range.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -41,13 +41,14 @@
|
|
|
41
41
|
"access": "public"
|
|
42
42
|
},
|
|
43
43
|
"dependencies": {
|
|
44
|
-
"@zukhruf/sqlite-drift": "^0.8.
|
|
45
|
-
"@zukhruf/sqlite-transaction": "^0.8.
|
|
44
|
+
"@zukhruf/sqlite-drift": "^0.8.32",
|
|
45
|
+
"@zukhruf/sqlite-transaction": "^0.8.32",
|
|
46
|
+
"@zukhruf/vectors": "^0.8.32",
|
|
46
47
|
"ai": "^7.0.154"
|
|
47
48
|
},
|
|
48
49
|
"devDependencies": {
|
|
49
|
-
"@zukhruf/fs": "^0.8.
|
|
50
|
-
"@zukhruf/sqlite-open": "^0.8.
|
|
51
|
-
"@zukhruf/testing": "^0.8.
|
|
50
|
+
"@zukhruf/fs": "^0.8.32",
|
|
51
|
+
"@zukhruf/sqlite-open": "^0.8.32",
|
|
52
|
+
"@zukhruf/testing": "^0.8.32"
|
|
52
53
|
}
|
|
53
54
|
}
|