@zukhruf/embeddings 0.8.31 → 0.8.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/collection.d.ts.map +1 -1
- package/dist/collection.js +9 -8
- package/dist/collection.js.map +1 -1
- package/dist/index.d.ts +0 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +0 -1
- package/dist/index.js.map +1 -1
- package/dist/vector-search.d.ts.map +1 -1
- package/dist/vector-search.js +14 -12
- package/dist/vector-search.js.map +1 -1
- package/dist/vector.d.ts +0 -10
- package/dist/vector.d.ts.map +1 -1
- package/dist/vector.js +0 -41
- package/dist/vector.js.map +1 -1
- package/package.json +7 -6
- package/dist/dimension-mismatch-error.d.ts +0 -6
- package/dist/dimension-mismatch-error.d.ts.map +0 -1
- package/dist/dimension-mismatch-error.js +0 -13
- package/dist/dimension-mismatch-error.js.map +0 -1
package/README.md
CHANGED
|
@@ -64,7 +64,7 @@ collection.count(); // 2
|
|
|
64
64
|
|
|
65
65
|
### Search a vector table of your own
|
|
66
66
|
|
|
67
|
-
Another program can write the vectors into its own table. A `VectorSearch` searches that table with the same rules as a collection. The table or view must have the columns `id` (text) and `vector`. Each `vector` must be a unit vector, as a BLOB of float32 values
|
|
67
|
+
Another program can write the vectors into its own table. A `VectorSearch` searches that table with the same rules as a collection. The table or view must have the columns `id` (text) and `vector`. Each `vector` must be a unit vector, as a BLOB of float32 values, little-endian, as `toSql` of `@zukhruf/vectors/sqlite` writes it ([ADR 0005](./docs/adr/0005-a-vector-search-reads-a-vector-table-of-unit-vectors.md)). Each call except a query without a date range also reads the column `published_at` (milliseconds since the Unix epoch). Thus a table of documents with no publish date supports a query without a date range ([ADR 0007](./docs/adr/0007-a-query-without-a-date-range-compares-every-document.md)).
|
|
68
68
|
|
|
69
69
|
```ts
|
|
70
70
|
import { DatabaseSync } from 'node:sqlite';
|
|
@@ -96,14 +96,15 @@ To search a table with other column names, make a view that renames the columns,
|
|
|
96
96
|
## Errors
|
|
97
97
|
|
|
98
98
|
- `UnknownDocumentError`: `nearest`, `similarity` or `fetch` got an id that the vector table does not contain. `fetch` gives no vector then.
|
|
99
|
-
- `DimensionMismatchError`: a vector has a dimension that is not the dimension of the stored vectors. In an upsert, the model gave the vector, and the upsert stores none of its documents. To change to a model with another dimension, upsert all documents in one call. In a query, the caller gave the vector. Give a vector of the model of the stored vectors.
|
|
99
|
+
- `DimensionMismatchError` of `@zukhruf/vectors`: a vector has a dimension that is not the dimension of the stored vectors. `expected` is the stored dimension and `actual` the dimension of the vector. In an upsert, the model gave the vector, and the upsert stores none of its documents. To change to a model with another dimension, upsert all documents in one call. In a query, the caller gave the vector. Give a vector of the model of the stored vectors.
|
|
100
|
+
- `RangeError`: a vector of length 0 (the `ZeroVectorError` of `@zukhruf/vectors`) or with a value that is not finite in float32. A collection makes each vector a unit vector with `UnitVector.normalize` of `@zukhruf/vectors`, which divides the float32 values by their length in float64, as pgvector does. In an upsert, the upsert stores none of its documents.
|
|
100
101
|
|
|
101
102
|
## Limits
|
|
102
103
|
|
|
103
104
|
- One database holds one collection.
|
|
104
105
|
- The collection stores the vector of the text, not the text. The caller chooses the text, for example a title and the first sentences.
|
|
105
106
|
- A document has one vector. A long text gives one mean vector, and its details become a small part of it.
|
|
106
|
-
- A vector search requires unit vectors. It does not examine the length
|
|
107
|
+
- A vector search requires unit vectors. It does not examine the length of a vector. Two vectors of a vector table with other dimensions are the `DimensionMismatchError` of `@zukhruf/vectors`, from its `dot`.
|
|
107
108
|
- The search reads each document in the window ([ADR 0001](./docs/adr/0001-search-compares-each-document-in-the-window.md)). A window of many thousand documents is slow. A neighbour graph decodes each vector once, but it still calculates a similarity for each pair of documents in a window.
|
|
108
109
|
- After a model change, upsert all documents again ([ADR 0002](./docs/adr/0002-the-hash-includes-the-model.md)).
|
|
109
110
|
|
package/dist/collection.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"collection.d.ts","sourceRoot":"","sources":["../src/collection.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,OAAO,EAAE,KAAK,cAAc,EAAa,MAAM,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"collection.d.ts","sourceRoot":"","sources":["../src/collection.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,OAAO,EAAE,KAAK,cAAc,EAAa,MAAM,IAAI,CAAC;AAQpD,OAAO,EAAE,KAAK,SAAS,EAAgB,MAAM,oBAAoB,CAAC;AAGlE,MAAM,MAAM,QAAQ,GAAG;IACrB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAEpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAE9B,QAAQ,CAAC,QAAQ,EAAE,YAAY,CAAC;IAGhC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;CACjD,CAAC;AAIF,qBAAa,UAAU;;gBAKT,OAAO,EAAE,iBAAiB;IAYhC,MAAM,CAAC,SAAS,EAAE,SAAS,QAAQ,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAoD3D,OAAO,CACL,EAAE,EAAE,MAAM,EACV,OAAO,EAAE;QAAE,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;KAAE,GAC3D,SAAS,EAAE;IAKd,cAAc,CAAC,OAAO,EAAE;QACtB,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;QACpB,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;QAClB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QACnB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;KAC7B,GAAG,cAAc,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IAKzC,UAAU,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM;IAOxC,KAAK,CACH,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,KAAK,CAAC,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;QAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAA;KAAE,GACjD,SAAS,EAAE;IAMd,KAAK,CAAC,GAAG,EAAE,SAAS,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC;IAexD,MAAM,CAAC,GAAG,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI;IASpC,KAAK,IAAI,MAAM;CA8ChB"}
|
package/dist/collection.js
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { hash } from 'node:crypto';
|
|
2
2
|
import { embedMany } from 'ai';
|
|
3
3
|
import { transaction } from '@zukhruf/sqlite-transaction';
|
|
4
|
-
import { DimensionMismatchError } from
|
|
4
|
+
import { DimensionMismatchError, UnitVector, Vector } from '@zukhruf/vectors';
|
|
5
|
+
import { fromSql, toSql } from '@zukhruf/vectors/sqlite';
|
|
5
6
|
import { collectionSchema } from "./schema.js";
|
|
6
7
|
import { UnknownDocumentError } from "./unknown-document-error.js";
|
|
7
8
|
import { VectorSearch } from "./vector-search.js";
|
|
8
|
-
import {
|
|
9
|
+
import { blob } from "./vector.js";
|
|
9
10
|
// The embeddings of documents, one vector for each document, in a SQLite
|
|
10
11
|
// database. The collection searches its table through a VectorSearch.
|
|
11
12
|
export class Collection {
|
|
@@ -38,7 +39,7 @@ export class Collection {
|
|
|
38
39
|
const vector = vectors[index];
|
|
39
40
|
if (vector === undefined)
|
|
40
41
|
throw new Error(`The model gave ${vectors.length} embeddings for ${changed.length} texts.`);
|
|
41
|
-
write.run(document.id, hash, document.publishedAt.getTime(), vector
|
|
42
|
+
write.run(document.id, hash, document.publishedAt.getTime(), toSql(vector));
|
|
42
43
|
}
|
|
43
44
|
const move = this.#database.prepare('UPDATE "embedding_document" SET "published_at" = ? WHERE "id" = ?');
|
|
44
45
|
for (const { document } of unchanged)
|
|
@@ -72,7 +73,7 @@ export class Collection {
|
|
|
72
73
|
const row = read.get(id);
|
|
73
74
|
if (row === undefined)
|
|
74
75
|
throw new UnknownDocumentError(id);
|
|
75
|
-
return [id,
|
|
76
|
+
return [id, fromSql(Vector, blob(row.vector)).values];
|
|
76
77
|
}));
|
|
77
78
|
}
|
|
78
79
|
// Removes the documents. An id that the collection does not store is not an
|
|
@@ -97,7 +98,7 @@ export class Collection {
|
|
|
97
98
|
model: this.#model,
|
|
98
99
|
values: texts,
|
|
99
100
|
});
|
|
100
|
-
return embeddings.map((embedding) => UnitVector.normalize(embedding));
|
|
101
|
+
return embeddings.map((embedding) => UnitVector.normalize(Vector.from(embedding)));
|
|
101
102
|
}
|
|
102
103
|
// The model is part of the hash, so a new model embeds the text again.
|
|
103
104
|
#hash(text) {
|
|
@@ -114,11 +115,11 @@ export class Collection {
|
|
|
114
115
|
.prepare('SELECT length("vector") AS "bytes" FROM "embedding_document" WHERE "id" NOT IN (SELECT "value" FROM json_each(?)) LIMIT 1')
|
|
115
116
|
.get(JSON.stringify(replaced));
|
|
116
117
|
const expected = kept === undefined
|
|
117
|
-
? first.
|
|
118
|
+
? first.dimensions
|
|
118
119
|
: Number(kept.bytes) / Float32Array.BYTES_PER_ELEMENT;
|
|
119
120
|
for (const vector of vectors)
|
|
120
|
-
if (vector.
|
|
121
|
-
throw new DimensionMismatchError(expected, vector.
|
|
121
|
+
if (vector.dimensions !== expected)
|
|
122
|
+
throw new DimensionMismatchError(expected, vector.dimensions);
|
|
122
123
|
}
|
|
123
124
|
}
|
|
124
125
|
//# sourceMappingURL=collection.js.map
|
package/dist/collection.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"collection.js","sourceRoot":"","sources":["../src/collection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAGnC,OAAO,EAAuB,SAAS,EAAE,MAAM,IAAI,CAAC;AAEpD,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"collection.js","sourceRoot":"","sources":["../src/collection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAGnC,OAAO,EAAuB,SAAS,EAAE,MAAM,IAAI,CAAC;AAEpD,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAC1D,OAAO,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC9E,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAEzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAkB,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClE,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAiBnC,yEAAyE;AACzE,sEAAsE;AACtE,MAAM,OAAO,UAAU;IACZ,SAAS,CAAe;IACxB,MAAM,CAAkC;IACxC,OAAO,CAAe;IAE/B,YAAY,OAA0B;QACpC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAC;QAClC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC;QAC5B,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACvC,IAAI,CAAC,OAAO,GAAG,IAAI,YAAY,CAAC;YAC9B,QAAQ,EAAE,IAAI,CAAC,SAAS;YACxB,KAAK,EAAE,oBAAoB;SAC5B,CAAC,CAAC;IACL,CAAC;IAED,4EAA4E;IAC5E,gDAAgD;IAChD,KAAK,CAAC,MAAM,CAAC,SAA8B;QACzC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CACnC,wDAAwD,CACzD,CAAC;QACF,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAC1C,QAAQ;YACR,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC;SAChC,CAAC,CAAC,CAAC;QACJ,MAAM,EAAE,OAAO,GAAG,EAAE,EAAE,SAAS,GAAG,EAAE,EAAE,GAAG,MAAM,CAAC,OAAO,CACrD,MAAM,EACN,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE,CACrB,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CACnE,CAAC;QACF,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAC/B,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAC7C,CAAC;QAEF,WAAW,CACT,IAAI,CAAC,SAAS,EACd,GAAG,EAAE;YACH,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAClC;4IACkI,CACnI,CAAC;YACF,KAAK,MAAM,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;gBAC5D,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;gBAC9B,IAAI,MAAM,KAAK,SAAS;oBACtB,MAAM,IAAI,KAAK,CACb,kBAAkB,OAAO,CAAC,MAAM,mBAAmB,OAAO,CAAC,MAAM,SAAS,CAC3E,CAAC;gBACJ,KAAK,CAAC,GAAG,CACP,QAAQ,CAAC,EAAE,EACX,IAAI,EACJ,QAAQ,CAAC,WAAW,CAAC,OAAO,EAAE,EAC9B,KAAK,CAAC,MAAM,CAAC,CACd,CAAC;YACJ,CAAC;YACD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CACjC,mEAAmE,CACpE,CAAC;YACF,KAAK,MAAM,EAAE,QAAQ,EAAE,IAAI,SAAS;gBAClC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;YACxD,IAAI,CAAC,eAAe,CAClB,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,EAC1C,OAAO,CACR,CAAC;QACJ,CAAC,EACD,EAAE,IAAI,EAAE,WAAW,EAAE,CACtB,CAAC;IACJ,CAAC;IAED,gDAAgD;IAChD,OAAO,CACL,EAAU,EACV,OAA4D;QAE5D,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;IAC3C,CAAC;IAED,iEAAiE;IACjE,cAAc,CAAC,OAKd;QACC,OAAO,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;IAC9C,CAAC;IAED,iDAAiD;IACjD,UAAU,CAAC,CAAS,EAAE,CAAS;QAC7B,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,4EAA4E;IAC5E,0EAA0E;IAC1E,kCAAkC;IAClC,KAAK,CACH,MAAyB,EACzB,KAAkD;QAElD,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC3C,CAAC;IAED,6EAA6E;IAC7E,kCAAkC;IAClC,KAAK,CAAC,GAAsB;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CACjC,0DAA0D,CAC3D,CAAC;QACF,OAAO,IAAI,GAAG,CACZ,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;YACb,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACzB,IAAI,GAAG,KAAK,SAAS;gBAAE,MAAM,IAAI,oBAAoB,CAAC,EAAE,CAAC,CAAC;YAC1D,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACxD,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;IAED,4EAA4E;IAC5E,0EAA0E;IAC1E,MAAM,CAAC,GAAsB;QAC3B,IAAI,CAAC,SAAS;aACX,OAAO,CACN,mFAAmF,CACpF;aACA,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9B,CAAC;IAED,kCAAkC;IAClC,KAAK;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS;aACvB,OAAO,CAAC,sDAAsD,CAAC;aAC/D,GAAG,EAAE,CAAC;QACT,OAAO,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAC5B,CAAC;IAED,sCAAsC;IACtC,KAAK,CAAC,MAAM,CAAC,KAAe;QAC1B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAClC,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,SAAS,CAAC;YACrC,KAAK,EAAE,IAAI,CAAC,MAAM;YAClB,MAAM,EAAE,KAAK;SACd,CAAC,CAAC;QACH,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAClC,UAAU,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAC7C,CAAC;IACJ,CAAC;IAED,uEAAuE;IACvE,KAAK,CAAC,IAAY;QAChB,OAAO,IAAI,CACT,QAAQ,EACR,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAClE,CAAC;IACJ,CAAC;IAED,2EAA2E;IAC3E,2EAA2E;IAC3E,iBAAiB;IACjB,eAAe,CAAC,QAA2B,EAAE,OAA0B;QACrE,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC;QACxB,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS;aACxB,OAAO,CACN,2HAA2H,CAC5H;aACA,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;QACjC,MAAM,QAAQ,GACZ,IAAI,KAAK,SAAS;YAChB,CAAC,CAAC,KAAK,CAAC,UAAU;YAClB,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,YAAY,CAAC,iBAAiB,CAAC;QAC1D,KAAK,MAAM,MAAM,IAAI,OAAO;YAC1B,IAAI,MAAM,CAAC,UAAU,KAAK,QAAQ;gBAChC,MAAM,IAAI,sBAAsB,CAAC,QAAQ,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IACpE,CAAC;CACF"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
export { Collection, type CollectionOptions, type Document, } from './collection.ts';
|
|
2
|
-
export { DimensionMismatchError } from './dimension-mismatch-error.ts';
|
|
3
2
|
export { UnknownDocumentError } from './unknown-document-error.ts';
|
|
4
3
|
export { type Neighbour, VectorSearch, type VectorSearchOptions, } from './vector-search.ts';
|
|
5
4
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,UAAU,EACV,KAAK,iBAAiB,EACtB,KAAK,QAAQ,GACd,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,UAAU,EACV,KAAK,iBAAiB,EACtB,KAAK,QAAQ,GACd,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EACL,KAAK,SAAS,EACd,YAAY,EACZ,KAAK,mBAAmB,GACzB,MAAM,oBAAoB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
export { Collection, } from "./collection.js";
|
|
2
|
-
export { DimensionMismatchError } from "./dimension-mismatch-error.js";
|
|
3
2
|
export { UnknownDocumentError } from "./unknown-document-error.js";
|
|
4
3
|
export { VectorSearch, } from "./vector-search.js";
|
|
5
4
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,UAAU,GAGX,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,UAAU,GAGX,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAEL,YAAY,GAEb,MAAM,oBAAoB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vector-search.d.ts","sourceRoot":"","sources":["../src/vector-search.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"vector-search.d.ts","sourceRoot":"","sources":["../src/vector-search.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAehD,MAAM,MAAM,SAAS,GAAG;IACtB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAEhC,QAAQ,CAAC,QAAQ,EAAE,YAAY,CAAC;IAIhC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB,CAAC;AAMF,qBAAa,YAAY;;gBAIX,OAAO,EAAE,mBAAmB;IAQxC,OAAO,CACL,EAAE,EAAE,MAAM,EACV,OAAO,EAAE;QAAE,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;KAAE,GAC3D,SAAS,EAAE;IAoBd,cAAc,CAAC,OAAO,EAAE;QACtB,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;QACpB,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;QAClB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QACnB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;KAC7B,GAAG,cAAc,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IAoCzC,KAAK,CACH,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,KAAK,CAAC,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;QAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAA;KAAE,GACjD,SAAS,EAAE;IA4Bd,UAAU,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM;CAgBzC"}
|
package/dist/vector-search.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { DimensionMismatchError } from
|
|
1
|
+
import { DimensionMismatchError, UnitVector, Vector, dot, } from '@zukhruf/vectors';
|
|
2
|
+
import { fromSql } from '@zukhruf/vectors/sqlite';
|
|
2
3
|
import { UnknownDocumentError } from "./unknown-document-error.js";
|
|
3
|
-
import {
|
|
4
|
+
import { blob } from "./vector.js";
|
|
4
5
|
const DAY_MS = 86_400_000;
|
|
5
6
|
// The neighbours of the documents in a vector table. A search compares the
|
|
6
7
|
// document with each document published inside the window, with no index
|
|
@@ -24,7 +25,7 @@ export class VectorSearch {
|
|
|
24
25
|
.all(publishedAt - window, publishedAt + window, id)
|
|
25
26
|
.map((row) => ({
|
|
26
27
|
id: String(row.id),
|
|
27
|
-
vector:
|
|
28
|
+
vector: fromSql(Vector, blob(row.vector)),
|
|
28
29
|
}));
|
|
29
30
|
return ranked(vector, candidates, options.k);
|
|
30
31
|
}
|
|
@@ -43,7 +44,7 @@ export class VectorSearch {
|
|
|
43
44
|
.map((row) => ({
|
|
44
45
|
id: String(row.id),
|
|
45
46
|
publishedAt: Number(row.published_at),
|
|
46
|
-
vector:
|
|
47
|
+
vector: fromSql(Vector, blob(row.vector)),
|
|
47
48
|
}));
|
|
48
49
|
return documents
|
|
49
50
|
.values()
|
|
@@ -60,15 +61,15 @@ export class VectorSearch {
|
|
|
60
61
|
// metadata only when the query names it. Equal similarities sort by id
|
|
61
62
|
// (docs/adr/0006, docs/adr/0007).
|
|
62
63
|
query(vector, range) {
|
|
63
|
-
const query = UnitVector.normalize(vector);
|
|
64
|
+
const query = UnitVector.normalize(Vector.from(vector));
|
|
64
65
|
const stored = this.#database
|
|
65
66
|
.prepare(`SELECT length("vector") AS "bytes" FROM ${this.#table} LIMIT 1`)
|
|
66
67
|
.get();
|
|
67
68
|
if (stored === undefined)
|
|
68
69
|
return [];
|
|
69
70
|
const dimension = Number(stored.bytes) / Float32Array.BYTES_PER_ELEMENT;
|
|
70
|
-
if (dimension !== query.
|
|
71
|
-
throw new DimensionMismatchError(dimension, query.
|
|
71
|
+
if (dimension !== query.dimensions)
|
|
72
|
+
throw new DimensionMismatchError(dimension, query.dimensions);
|
|
72
73
|
const rows = range === undefined
|
|
73
74
|
? this.#database
|
|
74
75
|
.prepare(`SELECT "id", "vector" FROM ${this.#table}`)
|
|
@@ -78,13 +79,14 @@ export class VectorSearch {
|
|
|
78
79
|
.all(range.from.getTime(), range.to.getTime());
|
|
79
80
|
const documents = rows.map((row) => ({
|
|
80
81
|
id: String(row.id),
|
|
81
|
-
vector:
|
|
82
|
+
vector: fromSql(Vector, blob(row.vector)),
|
|
82
83
|
}));
|
|
83
84
|
return ranked(query, documents, documents.length);
|
|
84
85
|
}
|
|
85
|
-
// The cosine similarity of two documents
|
|
86
|
+
// The cosine similarity of two documents: the dot product of their unit
|
|
87
|
+
// vectors.
|
|
86
88
|
similarity(a, b) {
|
|
87
|
-
return this.#read(a).vector
|
|
89
|
+
return dot(this.#read(a).vector, this.#read(b).vector);
|
|
88
90
|
}
|
|
89
91
|
#read(id) {
|
|
90
92
|
const row = this.#database
|
|
@@ -93,7 +95,7 @@ export class VectorSearch {
|
|
|
93
95
|
if (row === undefined)
|
|
94
96
|
throw new UnknownDocumentError(id);
|
|
95
97
|
return {
|
|
96
|
-
vector:
|
|
98
|
+
vector: fromSql(Vector, blob(row.vector)),
|
|
97
99
|
publishedAt: Number(row.published_at),
|
|
98
100
|
};
|
|
99
101
|
}
|
|
@@ -103,7 +105,7 @@ function ranked(vector, candidates, k) {
|
|
|
103
105
|
return candidates
|
|
104
106
|
.map((candidate) => ({
|
|
105
107
|
id: candidate.id,
|
|
106
|
-
similarity: vector
|
|
108
|
+
similarity: dot(vector, candidate.vector),
|
|
107
109
|
}))
|
|
108
110
|
.toSorted((left, right) => right.similarity - left.similarity || compareIds(left.id, right.id))
|
|
109
111
|
.slice(0, k);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vector-search.js","sourceRoot":"","sources":["../src/vector-search.ts"],"names":[],"mappings":"AAEA,OAAO,
|
|
1
|
+
{"version":3,"file":"vector-search.js","sourceRoot":"","sources":["../src/vector-search.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,sBAAsB,EACtB,UAAU,EACV,MAAM,EACN,GAAG,GACJ,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAEnC,MAAM,MAAM,GAAG,UAAU,CAAC;AAgB1B,2EAA2E;AAC3E,yEAAyE;AACzE,4EAA4E;AAC5E,oBAAoB;AACpB,MAAM,OAAO,YAAY;IACd,SAAS,CAAe;IACxB,MAAM,CAAS;IAExB,YAAY,OAA4B;QACtC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAC;QAClC,IAAI,CAAC,MAAM,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC;IAC3D,CAAC;IAED,2EAA2E;IAC3E,sEAAsE;IACtE,4DAA4D;IAC5D,OAAO,CACL,EAAU,EACV,OAA4D;QAE5D,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC/C,MAAM,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,MAAM,CAAC;QAC3C,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS;aAC9B,OAAO,CACN,8BAA8B,IAAI,CAAC,MAAM,qDAAqD,CAC/F;aACA,GAAG,CAAC,WAAW,GAAG,MAAM,EAAE,WAAW,GAAG,MAAM,EAAE,EAAE,CAAC;aACnD,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACb,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YAClB,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;SAC1C,CAAC,CAAC,CAAC;QACN,OAAO,MAAM,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IAC/C,CAAC;IAED,2EAA2E;IAC3E,wEAAwE;IACxE,0DAA0D;IAC1D,4EAA4E;IAC5E,cAAc;IACd,cAAc,CAAC,OAKd;QACC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QACpC,MAAM,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,MAAM,CAAC;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS;aAC7B,OAAO,CACN,8CAA8C,IAAI,CAAC,MAAM,qEAAqE,CAC/H;aACA,GAAG,CAAC,IAAI,GAAG,MAAM,EAAE,EAAE,GAAG,MAAM,CAAC;aAC/B,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACb,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YAClB,WAAW,EAAE,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC;YACrC,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;SAC1C,CAAC,CAAC,CAAC;QACN,OAAO,SAAS;aACb,MAAM,EAAE;aACR,MAAM,CAAC,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC,WAAW,IAAI,IAAI,IAAI,WAAW,IAAI,EAAE,CAAC;aACrE,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC;YACjB,QAAQ,CAAC,EAAE;YACX,MAAM,CACJ,QAAQ,CAAC,MAAM,EACf,SAAS,CAAC,MAAM,CACd,CAAC,KAAK,EAAE,EAAE,CACR,KAAK,CAAC,EAAE,KAAK,QAAQ,CAAC,EAAE;gBACxB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,GAAG,QAAQ,CAAC,WAAW,CAAC,IAAI,MAAM,CAC/D,EACD,OAAO,CAAC,CAAC,CACV;SACF,CAAC,CAAC;IACP,CAAC;IAED,0EAA0E;IAC1E,2EAA2E;IAC3E,2EAA2E;IAC3E,uEAAuE;IACvE,kCAAkC;IAClC,KAAK,CACH,MAAyB,EACzB,KAAkD;QAElD,MAAM,KAAK,GAAG,UAAU,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QACxD,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS;aAC1B,OAAO,CAAC,2CAA2C,IAAI,CAAC,MAAM,UAAU,CAAC;aACzE,GAAG,EAAE,CAAC;QACT,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO,EAAE,CAAC;QACpC,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,YAAY,CAAC,iBAAiB,CAAC;QACxE,IAAI,SAAS,KAAK,KAAK,CAAC,UAAU;YAChC,MAAM,IAAI,sBAAsB,CAAC,SAAS,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;QAChE,MAAM,IAAI,GACR,KAAK,KAAK,SAAS;YACjB,CAAC,CAAC,IAAI,CAAC,SAAS;iBACX,OAAO,CAAC,8BAA8B,IAAI,CAAC,MAAM,EAAE,CAAC;iBACpD,GAAG,EAAE;YACV,CAAC,CAAC,IAAI,CAAC,SAAS;iBACX,OAAO,CACN,8BAA8B,IAAI,CAAC,MAAM,uCAAuC,CACjF;iBACA,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;QACvD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACnC,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YAClB,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;SAC1C,CAAC,CAAC,CAAC;QACJ,OAAO,MAAM,CAAC,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;IACpD,CAAC;IAED,wEAAwE;IACxE,WAAW;IACX,UAAU,CAAC,CAAS,EAAE,CAAS;QAC7B,OAAO,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,EAAU;QACd,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS;aACvB,OAAO,CACN,wCAAwC,IAAI,CAAC,MAAM,iBAAiB,CACrE;aACA,GAAG,CAAC,EAAE,CAAC,CAAC;QACX,IAAI,GAAG,KAAK,SAAS;YAAE,MAAM,IAAI,oBAAoB,CAAC,EAAE,CAAC,CAAC;QAC1D,OAAO;YACL,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACzC,WAAW,EAAE,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC;SACtC,CAAC;IACJ,CAAC;CACF;AAED,8EAA8E;AAC9E,SAAS,MAAM,CACb,MAAc,EACd,UAAuE,EACvE,CAAS;IAET,OAAO,UAAU;SACd,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QACnB,EAAE,EAAE,SAAS,CAAC,EAAE;QAChB,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC;KAC1C,CAAC,CAAC;SACF,QAAQ,CACP,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CACd,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,CAAC,CACtE;SACA,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACjB,CAAC;AAED,0EAA0E;AAC1E,WAAW;AACX,SAAS,UAAU,CAAC,IAAY,EAAE,KAAa;IAC7C,IAAI,IAAI,GAAG,KAAK;QAAE,OAAO,CAAC,CAAC,CAAC;IAC5B,IAAI,IAAI,GAAG,KAAK;QAAE,OAAO,CAAC,CAAC;IAC3B,OAAO,CAAC,CAAC;AACX,CAAC"}
|
package/dist/vector.d.ts
CHANGED
|
@@ -1,12 +1,2 @@
|
|
|
1
|
-
export declare class UnitVector {
|
|
2
|
-
#private;
|
|
3
|
-
private constructor();
|
|
4
|
-
static normalize(values: readonly number[]): UnitVector;
|
|
5
|
-
static fromBlob(blob: Uint8Array): UnitVector;
|
|
6
|
-
get dimension(): number;
|
|
7
|
-
toFloat32Array(): Float32Array;
|
|
8
|
-
toBlob(): Uint8Array;
|
|
9
|
-
cosine(other: UnitVector): number;
|
|
10
|
-
}
|
|
11
1
|
export declare function blob(value: unknown): Uint8Array;
|
|
12
2
|
//# sourceMappingURL=vector.d.ts.map
|
package/dist/vector.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vector.d.ts","sourceRoot":"","sources":["../src/vector.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"vector.d.ts","sourceRoot":"","sources":["../src/vector.ts"],"names":[],"mappings":"AACA,wBAAgB,IAAI,CAAC,KAAK,EAAE,OAAO,GAAG,UAAU,CAI/C"}
|
package/dist/vector.js
CHANGED
|
@@ -1,44 +1,3 @@
|
|
|
1
|
-
// A unit vector of float32 values. A dot product of two unit vectors is their
|
|
2
|
-
// cosine similarity.
|
|
3
|
-
export class UnitVector {
|
|
4
|
-
#values;
|
|
5
|
-
constructor(values) {
|
|
6
|
-
this.#values = values;
|
|
7
|
-
}
|
|
8
|
-
// Divides each value by the length of the vector.
|
|
9
|
-
static normalize(values) {
|
|
10
|
-
const length = Math.sqrt(values.reduce((sum, value) => sum + value * value, 0));
|
|
11
|
-
if (length === 0 || !Number.isFinite(length))
|
|
12
|
-
throw new RangeError(`An embedding must have a finite length that is not zero. Its length is ${length}.`);
|
|
13
|
-
return new UnitVector(Float32Array.from(values, (value) => value / length));
|
|
14
|
-
}
|
|
15
|
-
// The bytes that SQLite gives are copied, because a Float32Array needs an
|
|
16
|
-
// offset that is a multiple of 4.
|
|
17
|
-
static fromBlob(blob) {
|
|
18
|
-
return new UnitVector(new Float32Array(Uint8Array.from(blob).buffer));
|
|
19
|
-
}
|
|
20
|
-
get dimension() {
|
|
21
|
-
return this.#values.length;
|
|
22
|
-
}
|
|
23
|
-
// A copy of the values, so the caller cannot change the vector.
|
|
24
|
-
toFloat32Array() {
|
|
25
|
-
return this.#values.slice();
|
|
26
|
-
}
|
|
27
|
-
toBlob() {
|
|
28
|
-
return new Uint8Array(this.#values.buffer, this.#values.byteOffset, this.#values.byteLength);
|
|
29
|
-
}
|
|
30
|
-
// An indexed loop: an entries() iterator with destructuring made the
|
|
31
|
-
// nearest search of each document in a window of 1,001 documents about two
|
|
32
|
-
// times slower.
|
|
33
|
-
cosine(other) {
|
|
34
|
-
const values = this.#values;
|
|
35
|
-
const others = other.#values;
|
|
36
|
-
let sum = 0;
|
|
37
|
-
for (let index = 0; index < values.length; index++)
|
|
38
|
-
sum += (values[index] ?? 0) * (others[index] ?? 0);
|
|
39
|
-
return sum;
|
|
40
|
-
}
|
|
41
|
-
}
|
|
42
1
|
// The value of a vector column, which SQLite gives as a Uint8Array.
|
|
43
2
|
export function blob(value) {
|
|
44
3
|
if (!(value instanceof Uint8Array))
|
package/dist/vector.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vector.js","sourceRoot":"","sources":["../src/vector.ts"],"names":[],"mappings":"AAAA,
|
|
1
|
+
{"version":3,"file":"vector.js","sourceRoot":"","sources":["../src/vector.ts"],"names":[],"mappings":"AAAA,oEAAoE;AACpE,MAAM,UAAU,IAAI,CAAC,KAAc;IACjC,IAAI,CAAC,CAAC,KAAK,YAAY,UAAU,CAAC;QAChC,MAAM,IAAI,SAAS,CAAC,iCAAiC,CAAC,CAAC;IACzD,OAAO,KAAK,CAAC;AACf,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zukhruf/embeddings",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.33",
|
|
4
4
|
"description": "Document embeddings in SQLite. Embeds only new or changed text through an AI SDK model. Finds the nearest documents of a document by exact cosine similarity in a publish-date window, and the similarity of a query vector to each document of a date range.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -41,13 +41,14 @@
|
|
|
41
41
|
"access": "public"
|
|
42
42
|
},
|
|
43
43
|
"dependencies": {
|
|
44
|
-
"@zukhruf/sqlite-drift": "^0.8.
|
|
45
|
-
"@zukhruf/sqlite-transaction": "^0.8.
|
|
44
|
+
"@zukhruf/sqlite-drift": "^0.8.33",
|
|
45
|
+
"@zukhruf/sqlite-transaction": "^0.8.33",
|
|
46
|
+
"@zukhruf/vectors": "^0.8.33",
|
|
46
47
|
"ai": "^7.0.154"
|
|
47
48
|
},
|
|
48
49
|
"devDependencies": {
|
|
49
|
-
"@zukhruf/fs": "^0.8.
|
|
50
|
-
"@zukhruf/sqlite-open": "^0.8.
|
|
51
|
-
"@zukhruf/testing": "^0.8.
|
|
50
|
+
"@zukhruf/fs": "^0.8.33",
|
|
51
|
+
"@zukhruf/sqlite-open": "^0.8.33",
|
|
52
|
+
"@zukhruf/testing": "^0.8.33"
|
|
52
53
|
}
|
|
53
54
|
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"dimension-mismatch-error.d.ts","sourceRoot":"","sources":["../src/dimension-mismatch-error.ts"],"names":[],"mappings":"AAEA,qBAAa,sBAAuB,SAAQ,KAAK;IAC/C,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;gBAEZ,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;CAQ7C"}
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
// A vector whose dimension is not the dimension of the stored vectors: a
|
|
2
|
-
// vector of the model in an upsert, or a vector of the caller in a query.
|
|
3
|
-
export class DimensionMismatchError extends Error {
|
|
4
|
-
expected;
|
|
5
|
-
actual;
|
|
6
|
-
constructor(expected, actual) {
|
|
7
|
-
super(`The stored vectors have dimension ${expected}, and the new vector has dimension ${actual}. To change the model, upsert every document again with the new model, in one call. To query, give a vector of the model of the stored vectors.`);
|
|
8
|
-
this.name = 'DimensionMismatchError';
|
|
9
|
-
this.expected = expected;
|
|
10
|
-
this.actual = actual;
|
|
11
|
-
}
|
|
12
|
-
}
|
|
13
|
-
//# sourceMappingURL=dimension-mismatch-error.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"dimension-mismatch-error.js","sourceRoot":"","sources":["../src/dimension-mismatch-error.ts"],"names":[],"mappings":"AAAA,yEAAyE;AACzE,0EAA0E;AAC1E,MAAM,OAAO,sBAAuB,SAAQ,KAAK;IACtC,QAAQ,CAAS;IACjB,MAAM,CAAS;IAExB,YAAY,QAAgB,EAAE,MAAc;QAC1C,KAAK,CACH,qCAAqC,QAAQ,sCAAsC,MAAM,iJAAiJ,CAC3O,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,wBAAwB,CAAC;QACrC,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;CACF"}
|