@zukhruf/embeddings 0.8.34 → 0.8.35

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -81,8 +81,8 @@ To search a table with other column names, make a view that renames the columns,
81
81
 
82
82
  ## API
83
83
 
84
- - `new Collection({ database, model })`. The caller opens and closes `database`. The collection makes its table `embedding_document` in it. `model` is an AI SDK embedding model object.
85
- - `upsert(documents)` stores each document. It calls the model, through `embedMany`, only for a document whose text or model changed. It stores the new publish date of each document.
84
+ - `new Collection({ database, model })`. The caller opens and closes `database`. The collection makes its table `embedding_document` in it, and keeps the vectors that the model gave in the table `embedding_cache` of [`@zukhruf/embedding-cache`](../embedding-cache). `model` is an AI SDK embedding model object.
85
+ - `upsert(documents)` stores each document. It embeds each text through `embedMany` and the cache, so it calls the model only for a text that the model did not embed before. It stores the new publish date of each document.
86
86
  - `nearest(id, { k, withinDays })` gives at most `k` neighbours of the document, with the highest similarity first. A neighbour has a publish date at most `withinDays` days before or after the publish date of the document. Equal similarities sort by id.
87
87
  - `neighbourGraph({ from, to, k, withinDays })` gives an iterator of `[id, neighbours]` entries, one for each document published from `from` to `to`, in the order of the publish dates and then of the ids. The neighbours of each entry are equal to the result of `nearest(id, { k, withinDays })`. A document outside the range has no entry, but it can be a neighbour of a document in the range. The call reads and decodes each vector one time ([ADR 0004](./docs/adr/0004-a-neighbour-graph-decodes-each-vector-once.md)). Use it when you need the neighbours of many documents.
88
88
  - `similarity(a, b)` gives the cosine similarity of two stored documents.
@@ -111,7 +111,7 @@ To search a table with other column names, make a view that renames the columns,
111
111
  ## Decisions
112
112
 
113
113
  - [ADR 0001: A search compares each document in the window, with no index](./docs/adr/0001-search-compares-each-document-in-the-window.md)
114
- - [ADR 0002: The hash includes the model](./docs/adr/0002-the-hash-includes-the-model.md)
114
+ - [ADR 0002: The hash includes the model (replaced by `@zukhruf/embedding-cache`)](./docs/adr/0002-the-hash-includes-the-model.md)
115
115
  - [ADR 0003: All vectors of a collection have one dimension](./docs/adr/0003-all-vectors-of-a-collection-have-one-dimension.md)
116
116
  - [ADR 0004: A neighbour graph decodes each vector once](./docs/adr/0004-a-neighbour-graph-decodes-each-vector-once.md)
117
117
  - [ADR 0005: A vector search reads a vector table of unit vectors](./docs/adr/0005-a-vector-search-reads-a-vector-table-of-unit-vectors.md)
@@ -1,5 +1,5 @@
1
1
  import type { DatabaseSync } from 'node:sqlite';
2
- import { type EmbeddingModel } from 'ai';
2
+ import { wrapEmbeddingModel } from 'ai';
3
3
  import { type Neighbour } from './vector-search.ts';
4
4
  export type Document = {
5
5
  readonly id: string;
@@ -8,7 +8,7 @@ export type Document = {
8
8
  };
9
9
  export type CollectionOptions = {
10
10
  readonly database: DatabaseSync;
11
- readonly model: Exclude<EmbeddingModel, string>;
11
+ readonly model: Parameters<typeof wrapEmbeddingModel>[0]['model'];
12
12
  };
13
13
  export declare class Collection {
14
14
  #private;
@@ -1 +1 @@
1
- {"version":3,"file":"collection.d.ts","sourceRoot":"","sources":["../src/collection.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,OAAO,EAAE,KAAK,cAAc,EAAa,MAAM,IAAI,CAAC;AAQpD,OAAO,EAAE,KAAK,SAAS,EAAgB,MAAM,oBAAoB,CAAC;AAGlE,MAAM,MAAM,QAAQ,GAAG;IACrB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAEpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAE9B,QAAQ,CAAC,QAAQ,EAAE,YAAY,CAAC;IAGhC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;CACjD,CAAC;AAIF,qBAAa,UAAU;;gBAKT,OAAO,EAAE,iBAAiB;IAYhC,MAAM,CAAC,SAAS,EAAE,SAAS,QAAQ,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAoD3D,OAAO,CACL,EAAE,EAAE,MAAM,EACV,OAAO,EAAE;QAAE,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;KAAE,GAC3D,SAAS,EAAE;IAKd,cAAc,CAAC,OAAO,EAAE;QACtB,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;QACpB,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;QAClB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QACnB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;KAC7B,GAAG,cAAc,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IAKzC,UAAU,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM;IAOxC,KAAK,CACH,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,KAAK,CAAC,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;QAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAA;KAAE,GACjD,SAAS,EAAE;IAMd,KAAK,CAAC,GAAG,EAAE,SAAS,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC;IAexD,MAAM,CAAC,GAAG,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI;IASpC,KAAK,IAAI,MAAM;CA8ChB"}
1
+ {"version":3,"file":"collection.d.ts","sourceRoot":"","sources":["../src/collection.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,OAAO,EAAa,kBAAkB,EAAE,MAAM,IAAI,CAAC;AAUnD,OAAO,EAAE,KAAK,SAAS,EAAgB,MAAM,oBAAoB,CAAC;AAGlE,MAAM,MAAM,QAAQ,GAAG;IACrB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAEpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAE9B,QAAQ,CAAC,QAAQ,EAAE,YAAY,CAAC;IAGhC,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC,OAAO,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;CACnE,CAAC;AAIF,qBAAa,UAAU;;gBAKT,OAAO,EAAE,iBAAiB;IAsBhC,MAAM,CAAC,SAAS,EAAE,SAAS,QAAQ,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAiC3D,OAAO,CACL,EAAE,EAAE,MAAM,EACV,OAAO,EAAE;QAAE,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;KAAE,GAC3D,SAAS,EAAE;IAKd,cAAc,CAAC,OAAO,EAAE;QACtB,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;QACpB,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;QAClB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QACnB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;KAC7B,GAAG,cAAc,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IAKzC,UAAU,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM;IAOxC,KAAK,CACH,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,KAAK,CAAC,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;QAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAA;KAAE,GACjD,SAAS,EAAE;IAMd,KAAK,CAAC,GAAG,EAAE,SAAS,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC;IAexD,MAAM,CAAC,GAAG,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI;IASpC,KAAK,IAAI,MAAM;CA0BhB"}
@@ -1,5 +1,6 @@
1
- import { hash } from 'node:crypto';
2
- import { embedMany } from 'ai';
1
+ import { embedMany, wrapEmbeddingModel } from 'ai';
2
+ import { embeddingCacheMiddleware } from '@zukhruf/embedding-cache';
3
+ import { SqliteEmbeddingCacheStore } from '@zukhruf/embedding-cache/sqlite';
3
4
  import { transaction } from '@zukhruf/sqlite-transaction';
4
5
  import { DimensionMismatchError, UnitVector, Vector } from '@zukhruf/vectors';
5
6
  import { fromSql, toSql } from '@zukhruf/vectors/sqlite';
@@ -15,36 +16,38 @@ export class Collection {
15
16
  #search;
16
17
  constructor(options) {
17
18
  this.#database = options.database;
18
- this.#model = options.model;
19
+ // The cache in the same database calls the model only for a text that
20
+ // this model did not embed before (docs/adr/0002).
21
+ this.#model = wrapEmbeddingModel({
22
+ model: options.model,
23
+ middleware: embeddingCacheMiddleware({
24
+ store: new SqliteEmbeddingCacheStore({
25
+ database: this.#database,
26
+ table: 'embedding_cache',
27
+ }),
28
+ }),
29
+ });
19
30
  collectionSchema.apply(this.#database);
20
31
  this.#search = new VectorSearch({
21
32
  database: this.#database,
22
33
  table: 'embedding_document',
23
34
  });
24
35
  }
25
- // Embeds the documents whose text or model changed since their last upsert,
26
- // and stores the publish date of each document.
36
+ // Embeds the text of each document through the cache, and stores its
37
+ // vector and publish date.
27
38
  async upsert(documents) {
28
- const stored = this.#database.prepare('SELECT "hash" FROM "embedding_document" WHERE "id" = ?');
29
- const hashed = documents.map((document) => ({
30
- document,
31
- hash: this.#hash(document.text),
32
- }));
33
- const { changed = [], unchanged = [] } = Object.groupBy(hashed, ({ document, hash }) => stored.get(document.id)?.hash === hash ? 'unchanged' : 'changed');
34
- const vectors = await this.#embed(changed.map(({ document }) => document.text));
39
+ const { embeddings } = await embedMany({
40
+ model: this.#model,
41
+ values: documents.map(({ text }) => text),
42
+ });
43
+ const vectors = embeddings.map((embedding) => UnitVector.normalize(Vector.from(embedding)));
35
44
  transaction(this.#database, () => {
36
- const write = this.#database.prepare(`INSERT INTO "embedding_document" ("id", "hash", "published_at", "vector") VALUES (?, ?, ?, ?)
37
- ON CONFLICT ("id") DO UPDATE SET "hash" = excluded."hash", "published_at" = excluded."published_at", "vector" = excluded."vector"`);
38
- for (const [index, { document, hash }] of changed.entries()) {
39
- const vector = vectors[index];
40
- if (vector === undefined)
41
- throw new Error(`The model gave ${vectors.length} embeddings for ${changed.length} texts.`);
42
- write.run(document.id, hash, document.publishedAt.getTime(), toSql(vector));
43
- }
44
- const move = this.#database.prepare('UPDATE "embedding_document" SET "published_at" = ? WHERE "id" = ?');
45
- for (const { document } of unchanged)
46
- move.run(document.publishedAt.getTime(), document.id);
47
- this.#checkDimension(changed.map(({ document }) => document.id), vectors);
45
+ const write = this.#database.prepare(`INSERT INTO "embedding_document" ("id", "published_at", "vector") VALUES (?, ?, ?)
46
+ ON CONFLICT ("id") DO UPDATE SET "published_at" = excluded."published_at", "vector" = excluded."vector"`);
47
+ // embedMany gives one embedding for each text, in their order.
48
+ for (const [index, document] of documents.entries())
49
+ write.run(document.id, document.publishedAt.getTime(), toSql(vectors[index]));
50
+ this.#checkDimension(documents.map(({ id }) => id), vectors);
48
51
  }, { mode: 'immediate' });
49
52
  }
50
53
  // The neighbours of a document (docs/adr/0005).
@@ -90,20 +93,6 @@ export class Collection {
90
93
  .get();
91
94
  return Number(row?.count);
92
95
  }
93
- // No texts make no call to the model.
94
- async #embed(texts) {
95
- if (texts.length === 0)
96
- return [];
97
- const { embeddings } = await embedMany({
98
- model: this.#model,
99
- values: texts,
100
- });
101
- return embeddings.map((embedding) => UnitVector.normalize(Vector.from(embedding)));
102
- }
103
- // The model is part of the hash, so a new model embeds the text again.
104
- #hash(text) {
105
- return hash('sha256', JSON.stringify([this.#model.provider, this.#model.modelId, text]));
106
- }
107
96
  // Before the upsert, each stored vector has one dimension. The new vectors
108
97
  // must have the dimension of a vector that the upsert did not replace, and
109
98
  // of each other.
@@ -1 +1 @@
1
- {"version":3,"file":"collection.js","sourceRoot":"","sources":["../src/collection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAGnC,OAAO,EAAuB,SAAS,EAAE,MAAM,IAAI,CAAC;AAEpD,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAC1D,OAAO,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC9E,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAEzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAkB,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClE,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAiBnC,yEAAyE;AACzE,sEAAsE;AACtE,MAAM,OAAO,UAAU;IACZ,SAAS,CAAe;IACxB,MAAM,CAAkC;IACxC,OAAO,CAAe;IAE/B,YAAY,OAA0B;QACpC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAC;QAClC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC;QAC5B,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACvC,IAAI,CAAC,OAAO,GAAG,IAAI,YAAY,CAAC;YAC9B,QAAQ,EAAE,IAAI,CAAC,SAAS;YACxB,KAAK,EAAE,oBAAoB;SAC5B,CAAC,CAAC;IACL,CAAC;IAED,4EAA4E;IAC5E,gDAAgD;IAChD,KAAK,CAAC,MAAM,CAAC,SAA8B;QACzC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CACnC,wDAAwD,CACzD,CAAC;QACF,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAC1C,QAAQ;YACR,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC;SAChC,CAAC,CAAC,CAAC;QACJ,MAAM,EAAE,OAAO,GAAG,EAAE,EAAE,SAAS,GAAG,EAAE,EAAE,GAAG,MAAM,CAAC,OAAO,CACrD,MAAM,EACN,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE,CACrB,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CACnE,CAAC;QACF,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAC/B,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAC7C,CAAC;QAEF,WAAW,CACT,IAAI,CAAC,SAAS,EACd,GAAG,EAAE;YACH,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAClC;4IACkI,CACnI,CAAC;YACF,KAAK,MAAM,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;gBAC5D,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;gBAC9B,IAAI,MAAM,KAAK,SAAS;oBACtB,MAAM,IAAI,KAAK,CACb,kBAAkB,OAAO,CAAC,MAAM,mBAAmB,OAAO,CAAC,MAAM,SAAS,CAC3E,CAAC;gBACJ,KAAK,CAAC,GAAG,CACP,QAAQ,CAAC,EAAE,EACX,IAAI,EACJ,QAAQ,CAAC,WAAW,CAAC,OAAO,EAAE,EAC9B,KAAK,CAAC,MAAM,CAAC,CACd,CAAC;YACJ,CAAC;YACD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CACjC,mEAAmE,CACpE,CAAC;YACF,KAAK,MAAM,EAAE,QAAQ,EAAE,IAAI,SAAS;gBAClC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;YACxD,IAAI,CAAC,eAAe,CAClB,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,EAC1C,OAAO,CACR,CAAC;QACJ,CAAC,EACD,EAAE,IAAI,EAAE,WAAW,EAAE,CACtB,CAAC;IACJ,CAAC;IAED,gDAAgD;IAChD,OAAO,CACL,EAAU,EACV,OAA4D;QAE5D,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;IAC3C,CAAC;IAED,iEAAiE;IACjE,cAAc,CAAC,OAKd;QACC,OAAO,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;IAC9C,CAAC;IAED,iDAAiD;IACjD,UAAU,CAAC,CAAS,EAAE,CAAS;QAC7B,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,4EAA4E;IAC5E,0EAA0E;IAC1E,kCAAkC;IAClC,KAAK,CACH,MAAyB,EACzB,KAAkD;QAElD,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC3C,CAAC;IAED,6EAA6E;IAC7E,kCAAkC;IAClC,KAAK,CAAC,GAAsB;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CACjC,0DAA0D,CAC3D,CAAC;QACF,OAAO,IAAI,GAAG,CACZ,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;YACb,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACzB,IAAI,GAAG,KAAK,SAAS;gBAAE,MAAM,IAAI,oBAAoB,CAAC,EAAE,CAAC,CAAC;YAC1D,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACxD,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;IAED,4EAA4E;IAC5E,0EAA0E;IAC1E,MAAM,CAAC,GAAsB;QAC3B,IAAI,CAAC,SAAS;aACX,OAAO,CACN,mFAAmF,CACpF;aACA,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9B,CAAC;IAED,kCAAkC;IAClC,KAAK;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS;aACvB,OAAO,CAAC,sDAAsD,CAAC;aAC/D,GAAG,EAAE,CAAC;QACT,OAAO,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAC5B,CAAC;IAED,sCAAsC;IACtC,KAAK,CAAC,MAAM,CAAC,KAAe;QAC1B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAClC,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,SAAS,CAAC;YACrC,KAAK,EAAE,IAAI,CAAC,MAAM;YAClB,MAAM,EAAE,KAAK;SACd,CAAC,CAAC;QACH,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAClC,UAAU,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAC7C,CAAC;IACJ,CAAC;IAED,uEAAuE;IACvE,KAAK,CAAC,IAAY;QAChB,OAAO,IAAI,CACT,QAAQ,EACR,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAClE,CAAC;IACJ,CAAC;IAED,2EAA2E;IAC3E,2EAA2E;IAC3E,iBAAiB;IACjB,eAAe,CAAC,QAA2B,EAAE,OAA0B;QACrE,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC;QACxB,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS;aACxB,OAAO,CACN,2HAA2H,CAC5H;aACA,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;QACjC,MAAM,QAAQ,GACZ,IAAI,KAAK,SAAS;YAChB,CAAC,CAAC,KAAK,CAAC,UAAU;YAClB,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,YAAY,CAAC,iBAAiB,CAAC;QAC1D,KAAK,MAAM,MAAM,IAAI,OAAO;YAC1B,IAAI,MAAM,CAAC,UAAU,KAAK,QAAQ;gBAChC,MAAM,IAAI,sBAAsB,CAAC,QAAQ,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IACpE,CAAC;CACF"}
1
+ {"version":3,"file":"collection.js","sourceRoot":"","sources":["../src/collection.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,IAAI,CAAC;AAEnD,OAAO,EAAE,wBAAwB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,EAAE,yBAAyB,EAAE,MAAM,iCAAiC,CAAC;AAC5E,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAC1D,OAAO,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC9E,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAEzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAkB,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClE,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAiBnC,yEAAyE;AACzE,sEAAsE;AACtE,MAAM,OAAO,UAAU;IACZ,SAAS,CAAe;IACxB,MAAM,CAAwC;IAC9C,OAAO,CAAe;IAE/B,YAAY,OAA0B;QACpC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAC;QAClC,sEAAsE;QACtE,mDAAmD;QACnD,IAAI,CAAC,MAAM,GAAG,kBAAkB,CAAC;YAC/B,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,UAAU,EAAE,wBAAwB,CAAC;gBACnC,KAAK,EAAE,IAAI,yBAAyB,CAAC;oBACnC,QAAQ,EAAE,IAAI,CAAC,SAAS;oBACxB,KAAK,EAAE,iBAAiB;iBACzB,CAAC;aACH,CAAC;SACH,CAAC,CAAC;QACH,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACvC,IAAI,CAAC,OAAO,GAAG,IAAI,YAAY,CAAC;YAC9B,QAAQ,EAAE,IAAI,CAAC,SAAS;YACxB,KAAK,EAAE,oBAAoB;SAC5B,CAAC,CAAC;IACL,CAAC;IAED,qEAAqE;IACrE,2BAA2B;IAC3B,KAAK,CAAC,MAAM,CAAC,SAA8B;QACzC,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,SAAS,CAAC;YACrC,KAAK,EAAE,IAAI,CAAC,MAAM;YAClB,MAAM,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC;SAC1C,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAC3C,UAAU,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAC7C,CAAC;QAEF,WAAW,CACT,IAAI,CAAC,SAAS,EACd,GAAG,EAAE;YACH,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAClC;kHACwG,CACzG,CAAC;YACF,+DAA+D;YAC/D,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,SAAS,CAAC,OAAO,EAAE;gBACjD,KAAK,CAAC,GAAG,CACP,QAAQ,CAAC,EAAE,EACX,QAAQ,CAAC,WAAW,CAAC,OAAO,EAAE,EAC9B,KAAK,CAAC,OAAO,CAAC,KAAK,CAAE,CAAC,CACvB,CAAC;YACJ,IAAI,CAAC,eAAe,CAClB,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,EAC7B,OAAO,CACR,CAAC;QACJ,CAAC,EACD,EAAE,IAAI,EAAE,WAAW,EAAE,CACtB,CAAC;IACJ,CAAC;IAED,gDAAgD;IAChD,OAAO,CACL,EAAU,EACV,OAA4D;QAE5D,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;IAC3C,CAAC;IAED,iEAAiE;IACjE,cAAc,CAAC,OAKd;QACC,OAAO,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;IAC9C,CAAC;IAED,iDAAiD;IACjD,UAAU,CAAC,CAAS,EAAE,CAAS;QAC7B,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,4EAA4E;IAC5E,0EAA0E;IAC1E,kCAAkC;IAClC,KAAK,CACH,MAAyB,EACzB,KAAkD;QAElD,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC3C,CAAC;IAED,6EAA6E;IAC7E,kCAAkC;IAClC,KAAK,CAAC,GAAsB;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CACjC,0DAA0D,CAC3D,CAAC;QACF,OAAO,IAAI,GAAG,CACZ,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;YACb,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACzB,IAAI,GAAG,KAAK,SAAS;gBAAE,MAAM,IAAI,oBAAoB,CAAC,EAAE,CAAC,CAAC;YAC1D,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACxD,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;IAED,4EAA4E;IAC5E,0EAA0E;IAC1E,MAAM,CAAC,GAAsB;QAC3B,IAAI,CAAC,SAAS;aACX,OAAO,CACN,mFAAmF,CACpF;aACA,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9B,CAAC;IAED,kCAAkC;IAClC,KAAK;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS;aACvB,OAAO,CAAC,sDAAsD,CAAC;aAC/D,GAAG,EAAE,CAAC;QACT,OAAO,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAC5B,CAAC;IAED,2EAA2E;IAC3E,2EAA2E;IAC3E,iBAAiB;IACjB,eAAe,CAAC,QAA2B,EAAE,OAA0B;QACrE,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC;QACxB,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS;aACxB,OAAO,CACN,2HAA2H,CAC5H;aACA,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;QACjC,MAAM,QAAQ,GACZ,IAAI,KAAK,SAAS;YAChB,CAAC,CAAC,KAAK,CAAC,UAAU;YAClB,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,YAAY,CAAC,iBAAiB,CAAC;QAC1D,KAAK,MAAM,MAAM,IAAI,OAAO;YAC1B,IAAI,MAAM,CAAC,UAAU,KAAK,QAAQ;gBAChC,MAAM,IAAI,sBAAsB,CAAC,QAAQ,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IACpE,CAAC;CACF"}
@@ -1 +1 @@
1
- {"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAKpD,eAAO,MAAM,gBAAgB,aAc5B,CAAC"}
1
+ {"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAKpD,eAAO,MAAM,gBAAgB,aAY5B,CAAC"}
package/dist/schema.js CHANGED
@@ -5,8 +5,6 @@ import { Declaration } from '@zukhruf/sqlite-drift';
5
5
  export const collectionSchema = new Declaration([
6
6
  `CREATE TABLE embedding_document (
7
7
  id TEXT PRIMARY KEY,
8
- -- sha256 of the model's provider and id and the text (docs/adr/0002).
9
- hash TEXT NOT NULL,
10
8
  -- Milliseconds since the Unix epoch.
11
9
  published_at INTEGER NOT NULL,
12
10
  -- A unit vector of float32 values, in the byte order of the platform.
@@ -1 +1 @@
1
- {"version":3,"file":"schema.js","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAEpD,4EAA4E;AAC5E,6EAA6E;AAC7E,4DAA4D;AAC5D,MAAM,CAAC,MAAM,gBAAgB,GAAG,IAAI,WAAW,CAC7C;IACE;;;;;;;;SAQK;IACL,mFAAmF;CACpF,EACD,EAAE,IAAI,EAAE,SAAS,EAAE,CACpB,CAAC"}
1
+ {"version":3,"file":"schema.js","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAEpD,4EAA4E;AAC5E,6EAA6E;AAC7E,4DAA4D;AAC5D,MAAM,CAAC,MAAM,gBAAgB,GAAG,IAAI,WAAW,CAC7C;IACE;;;;;;SAMK;IACL,mFAAmF;CACpF,EACD,EAAE,IAAI,EAAE,SAAS,EAAE,CACpB,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@zukhruf/embeddings",
3
- "version": "0.8.34",
3
+ "version": "0.8.35",
4
4
  "description": "Document embeddings in SQLite. Embeds only new or changed text through an AI SDK model. Finds the nearest documents of a document by exact cosine similarity in a publish-date window, and the similarity of a query vector to each document of a date range.",
5
5
  "license": "MIT",
6
6
  "repository": {
@@ -41,14 +41,15 @@
41
41
  "access": "public"
42
42
  },
43
43
  "dependencies": {
44
- "@zukhruf/sqlite-drift": "^0.8.34",
45
- "@zukhruf/sqlite-transaction": "^0.8.34",
46
- "@zukhruf/vectors": "^0.8.34",
44
+ "@zukhruf/embedding-cache": "^0.8.35",
45
+ "@zukhruf/sqlite-drift": "^0.8.35",
46
+ "@zukhruf/sqlite-transaction": "^0.8.35",
47
+ "@zukhruf/vectors": "^0.8.35",
47
48
  "ai": "^7.0.154"
48
49
  },
49
50
  "devDependencies": {
50
- "@zukhruf/fs": "^0.8.34",
51
- "@zukhruf/sqlite-open": "^0.8.34",
52
- "@zukhruf/testing": "^0.8.34"
51
+ "@zukhruf/fs": "^0.8.35",
52
+ "@zukhruf/sqlite-open": "^0.8.35",
53
+ "@zukhruf/testing": "^0.8.35"
53
54
  }
54
55
  }