@zukhruf/embeddings 0.8.32 → 0.8.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/collection.d.ts.map +1 -1
- package/dist/collection.js +3 -4
- package/dist/collection.js.map +1 -1
- package/dist/index.d.ts +0 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +0 -1
- package/dist/index.js.map +1 -1
- package/dist/vector-search.d.ts.map +1 -1
- package/dist/vector-search.js +3 -4
- package/dist/vector-search.js.map +1 -1
- package/dist/vector.d.ts +0 -2
- package/dist/vector.d.ts.map +1 -1
- package/dist/vector.js +0 -14
- package/dist/vector.js.map +1 -1
- package/package.json +7 -7
- package/dist/dimension-mismatch-error.d.ts +0 -6
- package/dist/dimension-mismatch-error.d.ts.map +0 -1
- package/dist/dimension-mismatch-error.js +0 -13
- package/dist/dimension-mismatch-error.js.map +0 -1
package/README.md
CHANGED
|
@@ -96,7 +96,8 @@ To search a table with other column names, make a view that renames the columns,
|
|
|
96
96
|
## Errors
|
|
97
97
|
|
|
98
98
|
- `UnknownDocumentError`: `nearest`, `similarity` or `fetch` got an id that the vector table does not contain. `fetch` gives no vector then.
|
|
99
|
-
- `DimensionMismatchError`: a vector has a dimension that is not the dimension of the stored vectors. In an upsert, the model gave the vector, and the upsert stores none of its documents. To change to a model with another dimension, upsert all documents in one call. In a query, the caller gave the vector. Give a vector of the model of the stored vectors.
|
|
99
|
+
- `DimensionMismatchError` of `@zukhruf/vectors`: a vector has a dimension that is not the dimension of the stored vectors. `expected` is the stored dimension and `actual` the dimension of the vector. In an upsert, the model gave the vector, and the upsert stores none of its documents. To change to a model with another dimension, upsert all documents in one call. In a query, the caller gave the vector. Give a vector of the model of the stored vectors.
|
|
100
|
+
- `RangeError`: a vector of length 0 (the `ZeroVectorError` of `@zukhruf/vectors`) or with a value that is not finite in float32. A collection makes each vector a unit vector with `UnitVector.normalize` of `@zukhruf/vectors`, which divides the float32 values by their length in float64, as pgvector does. In an upsert, the upsert stores none of its documents.
|
|
100
101
|
|
|
101
102
|
## Limits
|
|
102
103
|
|
package/dist/collection.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"collection.d.ts","sourceRoot":"","sources":["../src/collection.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,OAAO,EAAE,KAAK,cAAc,EAAa,MAAM,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"collection.d.ts","sourceRoot":"","sources":["../src/collection.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,OAAO,EAAE,KAAK,cAAc,EAAa,MAAM,IAAI,CAAC;AAQpD,OAAO,EAAE,KAAK,SAAS,EAAgB,MAAM,oBAAoB,CAAC;AAGlE,MAAM,MAAM,QAAQ,GAAG;IACrB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAEpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAE9B,QAAQ,CAAC,QAAQ,EAAE,YAAY,CAAC;IAGhC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;CACjD,CAAC;AAIF,qBAAa,UAAU;;gBAKT,OAAO,EAAE,iBAAiB;IAYhC,MAAM,CAAC,SAAS,EAAE,SAAS,QAAQ,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAoD3D,OAAO,CACL,EAAE,EAAE,MAAM,EACV,OAAO,EAAE;QAAE,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;KAAE,GAC3D,SAAS,EAAE;IAKd,cAAc,CAAC,OAAO,EAAE;QACtB,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;QACpB,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;QAClB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QACnB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;KAC7B,GAAG,cAAc,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IAKzC,UAAU,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM;IAOxC,KAAK,CACH,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,KAAK,CAAC,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;QAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAA;KAAE,GACjD,SAAS,EAAE;IAMd,KAAK,CAAC,GAAG,EAAE,SAAS,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC;IAexD,MAAM,CAAC,GAAG,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI;IASpC,KAAK,IAAI,MAAM;CA8ChB"}
|
package/dist/collection.js
CHANGED
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
import { hash } from 'node:crypto';
|
|
2
2
|
import { embedMany } from 'ai';
|
|
3
3
|
import { transaction } from '@zukhruf/sqlite-transaction';
|
|
4
|
-
import { Vector } from '@zukhruf/vectors';
|
|
4
|
+
import { DimensionMismatchError, UnitVector, Vector } from '@zukhruf/vectors';
|
|
5
5
|
import { fromSql, toSql } from '@zukhruf/vectors/sqlite';
|
|
6
|
-
import { DimensionMismatchError } from "./dimension-mismatch-error.js";
|
|
7
6
|
import { collectionSchema } from "./schema.js";
|
|
8
7
|
import { UnknownDocumentError } from "./unknown-document-error.js";
|
|
9
8
|
import { VectorSearch } from "./vector-search.js";
|
|
10
|
-
import { blob
|
|
9
|
+
import { blob } from "./vector.js";
|
|
11
10
|
// The embeddings of documents, one vector for each document, in a SQLite
|
|
12
11
|
// database. The collection searches its table through a VectorSearch.
|
|
13
12
|
export class Collection {
|
|
@@ -99,7 +98,7 @@ export class Collection {
|
|
|
99
98
|
model: this.#model,
|
|
100
99
|
values: texts,
|
|
101
100
|
});
|
|
102
|
-
return embeddings.map((embedding) => normalize(embedding));
|
|
101
|
+
return embeddings.map((embedding) => UnitVector.normalize(Vector.from(embedding)));
|
|
103
102
|
}
|
|
104
103
|
// The model is part of the hash, so a new model embeds the text again.
|
|
105
104
|
#hash(text) {
|
package/dist/collection.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"collection.js","sourceRoot":"","sources":["../src/collection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAGnC,OAAO,EAAuB,SAAS,EAAE,MAAM,IAAI,CAAC;AAEpD,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAC1D,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"collection.js","sourceRoot":"","sources":["../src/collection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAGnC,OAAO,EAAuB,SAAS,EAAE,MAAM,IAAI,CAAC;AAEpD,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAC1D,OAAO,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC9E,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAEzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAkB,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClE,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAiBnC,yEAAyE;AACzE,sEAAsE;AACtE,MAAM,OAAO,UAAU;IACZ,SAAS,CAAe;IACxB,MAAM,CAAkC;IACxC,OAAO,CAAe;IAE/B,YAAY,OAA0B;QACpC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAC;QAClC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC;QAC5B,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACvC,IAAI,CAAC,OAAO,GAAG,IAAI,YAAY,CAAC;YAC9B,QAAQ,EAAE,IAAI,CAAC,SAAS;YACxB,KAAK,EAAE,oBAAoB;SAC5B,CAAC,CAAC;IACL,CAAC;IAED,4EAA4E;IAC5E,gDAAgD;IAChD,KAAK,CAAC,MAAM,CAAC,SAA8B;QACzC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CACnC,wDAAwD,CACzD,CAAC;QACF,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAC1C,QAAQ;YACR,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC;SAChC,CAAC,CAAC,CAAC;QACJ,MAAM,EAAE,OAAO,GAAG,EAAE,EAAE,SAAS,GAAG,EAAE,EAAE,GAAG,MAAM,CAAC,OAAO,CACrD,MAAM,EACN,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE,CACrB,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CACnE,CAAC;QACF,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAC/B,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAC7C,CAAC;QAEF,WAAW,CACT,IAAI,CAAC,SAAS,EACd,GAAG,EAAE;YACH,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAClC;4IACkI,CACnI,CAAC;YACF,KAAK,MAAM,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;gBAC5D,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;gBAC9B,IAAI,MAAM,KAAK,SAAS;oBACtB,MAAM,IAAI,KAAK,CACb,kBAAkB,OAAO,CAAC,MAAM,mBAAmB,OAAO,CAAC,MAAM,SAAS,CAC3E,CAAC;gBACJ,KAAK,CAAC,GAAG,CACP,QAAQ,CAAC,EAAE,EACX,IAAI,EACJ,QAAQ,CAAC,WAAW,CAAC,OAAO,EAAE,EAC9B,KAAK,CAAC,MAAM,CAAC,CACd,CAAC;YACJ,CAAC;YACD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CACjC,mEAAmE,CACpE,CAAC;YACF,KAAK,MAAM,EAAE,QAAQ,EAAE,IAAI,SAAS;gBAClC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;YACxD,IAAI,CAAC,eAAe,CAClB,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,EAC1C,OAAO,CACR,CAAC;QACJ,CAAC,EACD,EAAE,IAAI,EAAE,WAAW,EAAE,CACtB,CAAC;IACJ,CAAC;IAED,gDAAgD;IAChD,OAAO,CACL,EAAU,EACV,OAA4D;QAE5D,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;IAC3C,CAAC;IAED,iEAAiE;IACjE,cAAc,CAAC,OAKd;QACC,OAAO,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;IAC9C,CAAC;IAED,iDAAiD;IACjD,UAAU,CAAC,CAAS,EAAE,CAAS;QAC7B,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,4EAA4E;IAC5E,0EAA0E;IAC1E,kCAAkC;IAClC,KAAK,CACH,MAAyB,EACzB,KAAkD;QAElD,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC3C,CAAC;IAED,6EAA6E;IAC7E,kCAAkC;IAClC,KAAK,CAAC,GAAsB;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CACjC,0DAA0D,CAC3D,CAAC;QACF,OAAO,IAAI,GAAG,CACZ,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;YACb,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACzB,IAAI,GAAG,KAAK,SAAS;gBAAE,MAAM,IAAI,oBAAoB,CAAC,EAAE,CAAC,CAAC;YAC1D,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACxD,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;IAED,4EAA4E;IAC5E,0EAA0E;IAC1E,MAAM,CAAC,GAAsB;QAC3B,IAAI,CAAC,SAAS;aACX,OAAO,CACN,mFAAmF,CACpF;aACA,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9B,CAAC;IAED,kCAAkC;IAClC,KAAK;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS;aACvB,OAAO,CAAC,sDAAsD,CAAC;aAC/D,GAAG,EAAE,CAAC;QACT,OAAO,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAC5B,CAAC;IAED,sCAAsC;IACtC,KAAK,CAAC,MAAM,CAAC,KAAe;QAC1B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAClC,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,SAAS,CAAC;YACrC,KAAK,EAAE,IAAI,CAAC,MAAM;YAClB,MAAM,EAAE,KAAK;SACd,CAAC,CAAC;QACH,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAClC,UAAU,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAC7C,CAAC;IACJ,CAAC;IAED,uEAAuE;IACvE,KAAK,CAAC,IAAY;QAChB,OAAO,IAAI,CACT,QAAQ,EACR,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAClE,CAAC;IACJ,CAAC;IAED,2EAA2E;IAC3E,2EAA2E;IAC3E,iBAAiB;IACjB,eAAe,CAAC,QAA2B,EAAE,OAA0B;QACrE,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC;QACxB,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS;aACxB,OAAO,CACN,2HAA2H,CAC5H;aACA,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;QACjC,MAAM,QAAQ,GACZ,IAAI,KAAK,SAAS;YAChB,CAAC,CAAC,KAAK,CAAC,UAAU;YAClB,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,YAAY,CAAC,iBAAiB,CAAC;QAC1D,KAAK,MAAM,MAAM,IAAI,OAAO;YAC1B,IAAI,MAAM,CAAC,UAAU,KAAK,QAAQ;gBAChC,MAAM,IAAI,sBAAsB,CAAC,QAAQ,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IACpE,CAAC;CACF"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
export { Collection, type CollectionOptions, type Document, } from './collection.ts';
|
|
2
|
-
export { DimensionMismatchError } from './dimension-mismatch-error.ts';
|
|
3
2
|
export { UnknownDocumentError } from './unknown-document-error.ts';
|
|
4
3
|
export { type Neighbour, VectorSearch, type VectorSearchOptions, } from './vector-search.ts';
|
|
5
4
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,UAAU,EACV,KAAK,iBAAiB,EACtB,KAAK,QAAQ,GACd,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,UAAU,EACV,KAAK,iBAAiB,EACtB,KAAK,QAAQ,GACd,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EACL,KAAK,SAAS,EACd,YAAY,EACZ,KAAK,mBAAmB,GACzB,MAAM,oBAAoB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
export { Collection, } from "./collection.js";
|
|
2
|
-
export { DimensionMismatchError } from "./dimension-mismatch-error.js";
|
|
3
2
|
export { UnknownDocumentError } from "./unknown-document-error.js";
|
|
4
3
|
export { VectorSearch, } from "./vector-search.js";
|
|
5
4
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,UAAU,GAGX,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,UAAU,GAGX,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAEL,YAAY,GAEb,MAAM,oBAAoB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vector-search.d.ts","sourceRoot":"","sources":["../src/vector-search.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"vector-search.d.ts","sourceRoot":"","sources":["../src/vector-search.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAehD,MAAM,MAAM,SAAS,GAAG;IACtB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAEhC,QAAQ,CAAC,QAAQ,EAAE,YAAY,CAAC;IAIhC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB,CAAC;AAMF,qBAAa,YAAY;;gBAIX,OAAO,EAAE,mBAAmB;IAQxC,OAAO,CACL,EAAE,EAAE,MAAM,EACV,OAAO,EAAE;QAAE,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;KAAE,GAC3D,SAAS,EAAE;IAoBd,cAAc,CAAC,OAAO,EAAE;QACtB,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;QACpB,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;QAClB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QACnB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;KAC7B,GAAG,cAAc,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IAoCzC,KAAK,CACH,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,KAAK,CAAC,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;QAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAA;KAAE,GACjD,SAAS,EAAE;IA4Bd,UAAU,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM;CAgBzC"}
|
package/dist/vector-search.js
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import { Vector, dot } from '@zukhruf/vectors';
|
|
1
|
+
import { DimensionMismatchError, UnitVector, Vector, dot, } from '@zukhruf/vectors';
|
|
2
2
|
import { fromSql } from '@zukhruf/vectors/sqlite';
|
|
3
|
-
import { DimensionMismatchError } from "./dimension-mismatch-error.js";
|
|
4
3
|
import { UnknownDocumentError } from "./unknown-document-error.js";
|
|
5
|
-
import { blob
|
|
4
|
+
import { blob } from "./vector.js";
|
|
6
5
|
const DAY_MS = 86_400_000;
|
|
7
6
|
// The neighbours of the documents in a vector table. A search compares the
|
|
8
7
|
// document with each document published inside the window, with no index
|
|
@@ -62,7 +61,7 @@ export class VectorSearch {
|
|
|
62
61
|
// metadata only when the query names it. Equal similarities sort by id
|
|
63
62
|
// (docs/adr/0006, docs/adr/0007).
|
|
64
63
|
query(vector, range) {
|
|
65
|
-
const query = normalize(vector);
|
|
64
|
+
const query = UnitVector.normalize(Vector.from(vector));
|
|
66
65
|
const stored = this.#database
|
|
67
66
|
.prepare(`SELECT length("vector") AS "bytes" FROM ${this.#table} LIMIT 1`)
|
|
68
67
|
.get();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vector-search.js","sourceRoot":"","sources":["../src/vector-search.ts"],"names":[],"mappings":"AAEA,OAAO,
|
|
1
|
+
{"version":3,"file":"vector-search.js","sourceRoot":"","sources":["../src/vector-search.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,sBAAsB,EACtB,UAAU,EACV,MAAM,EACN,GAAG,GACJ,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAEnC,MAAM,MAAM,GAAG,UAAU,CAAC;AAgB1B,2EAA2E;AAC3E,yEAAyE;AACzE,4EAA4E;AAC5E,oBAAoB;AACpB,MAAM,OAAO,YAAY;IACd,SAAS,CAAe;IACxB,MAAM,CAAS;IAExB,YAAY,OAA4B;QACtC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAC;QAClC,IAAI,CAAC,MAAM,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC;IAC3D,CAAC;IAED,2EAA2E;IAC3E,sEAAsE;IACtE,4DAA4D;IAC5D,OAAO,CACL,EAAU,EACV,OAA4D;QAE5D,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC/C,MAAM,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,MAAM,CAAC;QAC3C,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS;aAC9B,OAAO,CACN,8BAA8B,IAAI,CAAC,MAAM,qDAAqD,CAC/F;aACA,GAAG,CAAC,WAAW,GAAG,MAAM,EAAE,WAAW,GAAG,MAAM,EAAE,EAAE,CAAC;aACnD,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACb,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YAClB,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;SAC1C,CAAC,CAAC,CAAC;QACN,OAAO,MAAM,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IAC/C,CAAC;IAED,2EAA2E;IAC3E,wEAAwE;IACxE,0DAA0D;IAC1D,4EAA4E;IAC5E,cAAc;IACd,cAAc,CAAC,OAKd;QACC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QACpC,MAAM,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,MAAM,CAAC;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS;aAC7B,OAAO,CACN,8CAA8C,IAAI,CAAC,MAAM,qEAAqE,CAC/H;aACA,GAAG,CAAC,IAAI,GAAG,MAAM,EAAE,EAAE,GAAG,MAAM,CAAC;aAC/B,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACb,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YAClB,WAAW,EAAE,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC;YACrC,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;SAC1C,CAAC,CAAC,CAAC;QACN,OAAO,SAAS;aACb,MAAM,EAAE;aACR,MAAM,CAAC,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC,WAAW,IAAI,IAAI,IAAI,WAAW,IAAI,EAAE,CAAC;aACrE,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC;YACjB,QAAQ,CAAC,EAAE;YACX,MAAM,CACJ,QAAQ,CAAC,MAAM,EACf,SAAS,CAAC,MAAM,CACd,CAAC,KAAK,EAAE,EAAE,CACR,KAAK,CAAC,EAAE,KAAK,QAAQ,CAAC,EAAE;gBACxB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,GAAG,QAAQ,CAAC,WAAW,CAAC,IAAI,MAAM,CAC/D,EACD,OAAO,CAAC,CAAC,CACV;SACF,CAAC,CAAC;IACP,CAAC;IAED,0EAA0E;IAC1E,2EAA2E;IAC3E,2EAA2E;IAC3E,uEAAuE;IACvE,kCAAkC;IAClC,KAAK,CACH,MAAyB,EACzB,KAAkD;QAElD,MAAM,KAAK,GAAG,UAAU,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QACxD,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS;aAC1B,OAAO,CAAC,2CAA2C,IAAI,CAAC,MAAM,UAAU,CAAC;aACzE,GAAG,EAAE,CAAC;QACT,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO,EAAE,CAAC;QACpC,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,YAAY,CAAC,iBAAiB,CAAC;QACxE,IAAI,SAAS,KAAK,KAAK,CAAC,UAAU;YAChC,MAAM,IAAI,sBAAsB,CAAC,SAAS,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;QAChE,MAAM,IAAI,GACR,KAAK,KAAK,SAAS;YACjB,CAAC,CAAC,IAAI,CAAC,SAAS;iBACX,OAAO,CAAC,8BAA8B,IAAI,CAAC,MAAM,EAAE,CAAC;iBACpD,GAAG,EAAE;YACV,CAAC,CAAC,IAAI,CAAC,SAAS;iBACX,OAAO,CACN,8BAA8B,IAAI,CAAC,MAAM,uCAAuC,CACjF;iBACA,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;QACvD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACnC,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YAClB,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;SAC1C,CAAC,CAAC,CAAC;QACJ,OAAO,MAAM,CAAC,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;IACpD,CAAC;IAED,wEAAwE;IACxE,WAAW;IACX,UAAU,CAAC,CAAS,EAAE,CAAS;QAC7B,OAAO,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,EAAU;QACd,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS;aACvB,OAAO,CACN,wCAAwC,IAAI,CAAC,MAAM,iBAAiB,CACrE;aACA,GAAG,CAAC,EAAE,CAAC,CAAC;QACX,IAAI,GAAG,KAAK,SAAS;YAAE,MAAM,IAAI,oBAAoB,CAAC,EAAE,CAAC,CAAC;QAC1D,OAAO;YACL,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACzC,WAAW,EAAE,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC;SACtC,CAAC;IACJ,CAAC;CACF;AAED,8EAA8E;AAC9E,SAAS,MAAM,CACb,MAAc,EACd,UAAuE,EACvE,CAAS;IAET,OAAO,UAAU;SACd,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QACnB,EAAE,EAAE,SAAS,CAAC,EAAE;QAChB,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC;KAC1C,CAAC,CAAC;SACF,QAAQ,CACP,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CACd,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,CAAC,CACtE;SACA,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACjB,CAAC;AAED,0EAA0E;AAC1E,WAAW;AACX,SAAS,UAAU,CAAC,IAAY,EAAE,KAAa;IAC7C,IAAI,IAAI,GAAG,KAAK;QAAE,OAAO,CAAC,CAAC,CAAC;IAC5B,IAAI,IAAI,GAAG,KAAK;QAAE,OAAO,CAAC,CAAC;IAC3B,OAAO,CAAC,CAAC;AACX,CAAC"}
|
package/dist/vector.d.ts
CHANGED
package/dist/vector.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vector.d.ts","sourceRoot":"","sources":["../src/vector.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"vector.d.ts","sourceRoot":"","sources":["../src/vector.ts"],"names":[],"mappings":"AACA,wBAAgB,IAAI,CAAC,KAAK,EAAE,OAAO,GAAG,UAAU,CAI/C"}
|
package/dist/vector.js
CHANGED
|
@@ -1,17 +1,3 @@
|
|
|
1
|
-
import { Vector } from '@zukhruf/vectors';
|
|
2
|
-
// Divides each value by the length of the embedding, in float64, and rounds
|
|
3
|
-
// once to float32. The result is a unit vector, so its dot product with
|
|
4
|
-
// another one is their cosine similarity. It is not UnitVector.normalize of
|
|
5
|
-
// @zukhruf/vectors: a length of zero, or one that is not finite, is a
|
|
6
|
-
// RangeError for the callers of a collection (that normalize throws
|
|
7
|
-
// ZeroVectorError), and that normalize rounds each value to float32 before it
|
|
8
|
-
// divides, which can change a stored vector in the last bit.
|
|
9
|
-
export function normalize(values) {
|
|
10
|
-
const length = Math.sqrt(values.reduce((sum, value) => sum + value * value, 0));
|
|
11
|
-
if (length === 0 || !Number.isFinite(length))
|
|
12
|
-
throw new RangeError(`An embedding must have a finite length that is not zero. Its length is ${length}.`);
|
|
13
|
-
return Vector.from(Float32Array.from(values, (value) => value / length));
|
|
14
|
-
}
|
|
15
1
|
// The value of a vector column, which SQLite gives as a Uint8Array.
|
|
16
2
|
export function blob(value) {
|
|
17
3
|
if (!(value instanceof Uint8Array))
|
package/dist/vector.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vector.js","sourceRoot":"","sources":["../src/vector.ts"],"names":[],"mappings":"AAAA,
|
|
1
|
+
{"version":3,"file":"vector.js","sourceRoot":"","sources":["../src/vector.ts"],"names":[],"mappings":"AAAA,oEAAoE;AACpE,MAAM,UAAU,IAAI,CAAC,KAAc;IACjC,IAAI,CAAC,CAAC,KAAK,YAAY,UAAU,CAAC;QAChC,MAAM,IAAI,SAAS,CAAC,iCAAiC,CAAC,CAAC;IACzD,OAAO,KAAK,CAAC;AACf,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zukhruf/embeddings",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.33",
|
|
4
4
|
"description": "Document embeddings in SQLite. Embeds only new or changed text through an AI SDK model. Finds the nearest documents of a document by exact cosine similarity in a publish-date window, and the similarity of a query vector to each document of a date range.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -41,14 +41,14 @@
|
|
|
41
41
|
"access": "public"
|
|
42
42
|
},
|
|
43
43
|
"dependencies": {
|
|
44
|
-
"@zukhruf/sqlite-drift": "^0.8.
|
|
45
|
-
"@zukhruf/sqlite-transaction": "^0.8.
|
|
46
|
-
"@zukhruf/vectors": "^0.8.
|
|
44
|
+
"@zukhruf/sqlite-drift": "^0.8.33",
|
|
45
|
+
"@zukhruf/sqlite-transaction": "^0.8.33",
|
|
46
|
+
"@zukhruf/vectors": "^0.8.33",
|
|
47
47
|
"ai": "^7.0.154"
|
|
48
48
|
},
|
|
49
49
|
"devDependencies": {
|
|
50
|
-
"@zukhruf/fs": "^0.8.
|
|
51
|
-
"@zukhruf/sqlite-open": "^0.8.
|
|
52
|
-
"@zukhruf/testing": "^0.8.
|
|
50
|
+
"@zukhruf/fs": "^0.8.33",
|
|
51
|
+
"@zukhruf/sqlite-open": "^0.8.33",
|
|
52
|
+
"@zukhruf/testing": "^0.8.33"
|
|
53
53
|
}
|
|
54
54
|
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"dimension-mismatch-error.d.ts","sourceRoot":"","sources":["../src/dimension-mismatch-error.ts"],"names":[],"mappings":"AAEA,qBAAa,sBAAuB,SAAQ,KAAK;IAC/C,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;gBAEZ,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;CAQ7C"}
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
// A vector whose dimension is not the dimension of the stored vectors: a
|
|
2
|
-
// vector of the model in an upsert, or a vector of the caller in a query.
|
|
3
|
-
export class DimensionMismatchError extends Error {
|
|
4
|
-
expected;
|
|
5
|
-
actual;
|
|
6
|
-
constructor(expected, actual) {
|
|
7
|
-
super(`The stored vectors have dimension ${expected}, and the new vector has dimension ${actual}. To change the model, upsert every document again with the new model, in one call. To query, give a vector of the model of the stored vectors.`);
|
|
8
|
-
this.name = 'DimensionMismatchError';
|
|
9
|
-
this.expected = expected;
|
|
10
|
-
this.actual = actual;
|
|
11
|
-
}
|
|
12
|
-
}
|
|
13
|
-
//# sourceMappingURL=dimension-mismatch-error.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"dimension-mismatch-error.js","sourceRoot":"","sources":["../src/dimension-mismatch-error.ts"],"names":[],"mappings":"AAAA,yEAAyE;AACzE,0EAA0E;AAC1E,MAAM,OAAO,sBAAuB,SAAQ,KAAK;IACtC,QAAQ,CAAS;IACjB,MAAM,CAAS;IAExB,YAAY,QAAgB,EAAE,MAAc;QAC1C,KAAK,CACH,qCAAqC,QAAQ,sCAAsC,MAAM,iJAAiJ,CAC3O,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,wBAAwB,CAAC;QACrC,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;CACF"}
|