agentfootprint 9.2.0 → 9.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/memory/pgVector.js +731 -0
- package/dist/adapters/memory/pgVector.js.map +1 -0
- package/dist/adapters/memory/s3Vectors.js +628 -0
- package/dist/adapters/memory/s3Vectors.js.map +1 -0
- package/dist/adapters/memory/sqliteVector.js +16 -79
- package/dist/adapters/memory/sqliteVector.js.map +1 -1
- package/dist/embedders/index.js +240 -76
- package/dist/embedders/index.js.map +1 -1
- package/dist/esm/adapters/memory/pgVector.d.ts +243 -0
- package/dist/esm/adapters/memory/pgVector.js +726 -0
- package/dist/esm/adapters/memory/pgVector.js.map +1 -0
- package/dist/esm/adapters/memory/s3Vectors.d.ts +208 -0
- package/dist/esm/adapters/memory/s3Vectors.js +624 -0
- package/dist/esm/adapters/memory/s3Vectors.js.map +1 -0
- package/dist/esm/adapters/memory/sqliteVector.d.ts +5 -27
- package/dist/esm/adapters/memory/sqliteVector.js +10 -73
- package/dist/esm/adapters/memory/sqliteVector.js.map +1 -1
- package/dist/esm/embedders/index.d.ts +99 -19
- package/dist/esm/embedders/index.js +240 -76
- package/dist/esm/embedders/index.js.map +1 -1
- package/dist/esm/lib/embedderMismatch.d.ts +67 -0
- package/dist/esm/lib/embedderMismatch.js +96 -0
- package/dist/esm/lib/embedderMismatch.js.map +1 -0
- package/dist/esm/lib/rag/defineRAG.d.ts +28 -6
- package/dist/esm/lib/rag/defineRAG.js +39 -5
- package/dist/esm/lib/rag/defineRAG.js.map +1 -1
- package/dist/esm/memory/define.js +31 -1
- package/dist/esm/memory/define.js.map +1 -1
- package/dist/esm/memory/define.types.d.ts +13 -2
- package/dist/esm/memory/define.types.js.map +1 -1
- package/dist/esm/memory/embedding/loadRelevant.d.ts +10 -2
- package/dist/esm/memory/embedding/loadRelevant.js +36 -18
- package/dist/esm/memory/embedding/loadRelevant.js.map +1 -1
- package/dist/esm/memory/pipeline/semantic.d.ts +13 -2
- package/dist/esm/memory/pipeline/semantic.js +11 -5
- package/dist/esm/memory/pipeline/semantic.js.map +1 -1
- package/dist/esm/memory/store/capability.d.ts +25 -6
- package/dist/esm/memory/store/capability.js +68 -3
- package/dist/esm/memory/store/capability.js.map +1 -1
- package/dist/esm/memory/store/index.d.ts +1 -0
- package/dist/esm/memory/store/index.js +4 -0
- package/dist/esm/memory/store/index.js.map +1 -1
- package/dist/esm/memory/store/types.d.ts +40 -0
- package/dist/esm/memory-providers.d.ts +7 -4
- package/dist/esm/memory-providers.js +18 -4
- package/dist/esm/memory-providers.js.map +1 -1
- package/dist/lib/embedderMismatch.js +103 -0
- package/dist/lib/embedderMismatch.js.map +1 -0
- package/dist/lib/rag/defineRAG.js +39 -5
- package/dist/lib/rag/defineRAG.js.map +1 -1
- package/dist/memory/define.js +31 -1
- package/dist/memory/define.js.map +1 -1
- package/dist/memory/define.types.js.map +1 -1
- package/dist/memory/embedding/loadRelevant.js +36 -18
- package/dist/memory/embedding/loadRelevant.js.map +1 -1
- package/dist/memory/pipeline/semantic.js +11 -5
- package/dist/memory/pipeline/semantic.js.map +1 -1
- package/dist/memory/store/capability.js +70 -4
- package/dist/memory/store/capability.js.map +1 -1
- package/dist/memory/store/index.js +7 -1
- package/dist/memory/store/index.js.map +1 -1
- package/dist/memory-providers.js +22 -5
- package/dist/memory-providers.js.map +1 -1
- package/dist/types/adapters/memory/pgVector.d.ts +244 -0
- package/dist/types/adapters/memory/pgVector.d.ts.map +1 -0
- package/dist/types/adapters/memory/s3Vectors.d.ts +209 -0
- package/dist/types/adapters/memory/s3Vectors.d.ts.map +1 -0
- package/dist/types/adapters/memory/sqliteVector.d.ts +5 -27
- package/dist/types/adapters/memory/sqliteVector.d.ts.map +1 -1
- package/dist/types/embedders/index.d.ts +99 -19
- package/dist/types/embedders/index.d.ts.map +1 -1
- package/dist/types/lib/embedderMismatch.d.ts +68 -0
- package/dist/types/lib/embedderMismatch.d.ts.map +1 -0
- package/dist/types/lib/rag/defineRAG.d.ts +28 -6
- package/dist/types/lib/rag/defineRAG.d.ts.map +1 -1
- package/dist/types/memory/define.d.ts.map +1 -1
- package/dist/types/memory/define.types.d.ts +13 -2
- package/dist/types/memory/define.types.d.ts.map +1 -1
- package/dist/types/memory/embedding/loadRelevant.d.ts +10 -2
- package/dist/types/memory/embedding/loadRelevant.d.ts.map +1 -1
- package/dist/types/memory/pipeline/semantic.d.ts +13 -2
- package/dist/types/memory/pipeline/semantic.d.ts.map +1 -1
- package/dist/types/memory/store/capability.d.ts +25 -6
- package/dist/types/memory/store/capability.d.ts.map +1 -1
- package/dist/types/memory/store/index.d.ts +1 -0
- package/dist/types/memory/store/index.d.ts.map +1 -1
- package/dist/types/memory/store/types.d.ts +40 -0
- package/dist/types/memory/store/types.d.ts.map +1 -1
- package/dist/types/memory-providers.d.ts +7 -4
- package/dist/types/memory-providers.d.ts.map +1 -1
- package/package.json +9 -1
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* adapters/memory/s3Vectors — the corpus outlives the runtime, and can still
|
|
3
|
+
* be updated without one.
|
|
4
|
+
*
|
|
5
|
+
* `sqliteVectorStore` (8.9.0) made a corpus survive a restart by putting it in
|
|
6
|
+
* a file, and `staticVectorStore` (8.20.0) made it survive a runtime with no
|
|
7
|
+
* disk at all by shipping it as a build artifact. Both leave the same gap, and
|
|
8
|
+
* a production field report named it: **a bundle can only change when you
|
|
9
|
+
* redeploy.** Add three documents on a Tuesday and the answer arrives with the
|
|
10
|
+
* next release train. That is a fine trade for product documentation and a bad
|
|
11
|
+
* one for anything a human edits during the day.
|
|
12
|
+
*
|
|
13
|
+
* Amazon S3 Vectors closes it. It is object storage with a native vector index
|
|
14
|
+
* and a query API: durable, serverless, nothing to run, priced like storage
|
|
15
|
+
* rather than like a database cluster. This adapter is the `MemoryStore` port
|
|
16
|
+
* over it, and the two halves that matter are:
|
|
17
|
+
*
|
|
18
|
+
* - `search()` maps 1:1 onto **QueryVectors** — the vector goes as
|
|
19
|
+
* `queryVector.float32`, `k` becomes `topK`, tiers become a metadata
|
|
20
|
+
* `filter`, and the returned `distance` becomes the port's score.
|
|
21
|
+
* - `put()` / `putMany()` map onto **PutVectors** — so `indexCorpus`,
|
|
22
|
+
* `indexFolder` and `indexDocuments` run against it unchanged. **That is
|
|
23
|
+
* the point of writing it rather than only reading it:** a corpus you can
|
|
24
|
+
* add to from a cron job at 14:00 is a different product from one you can
|
|
25
|
+
* add to at the next deploy.
|
|
26
|
+
*
|
|
27
|
+
* ── What it is NOT ──────────────────────────────────────────────────────────
|
|
28
|
+
* A vector index is not a key-value store, and this adapter refuses the
|
|
29
|
+
* operations that would need one rather than faking them:
|
|
30
|
+
*
|
|
31
|
+
* - `putIfVersion` — PutVectors is last-write-wins; there is no
|
|
32
|
+
* compare-and-set. A read-then-write here would be a compare-and-set that
|
|
33
|
+
* another writer can walk through the middle of, which is worse than none.
|
|
34
|
+
* - `recordSignature` / `feedback` — a set of strings and a running average
|
|
35
|
+
* are not vectors. `seen()` answers `false` and `getFeedback()` `null`,
|
|
36
|
+
* which are both TRUE (nothing can be recorded, so nothing has been); the
|
|
37
|
+
* WRITE halves refuse by name.
|
|
38
|
+
*
|
|
39
|
+
* Pair it with a second store for conversation memory — `defineRAG` for the
|
|
40
|
+
* corpus, `defineMemory` for the chat, each with the backend it suits. That is
|
|
41
|
+
* the same shape `staticVectorStore` documents.
|
|
42
|
+
*
|
|
43
|
+
* ── The index you must create first, and why this does not create it ────────
|
|
44
|
+
* A vector index has a DIMENSION and a DISTANCE METRIC fixed at creation, and
|
|
45
|
+
* both are decisions about your embedder that this library must not make on
|
|
46
|
+
* your behalf — creating one silently would pick a size and a metric your
|
|
47
|
+
* corpus then lives with. Create it once, with your infrastructure:
|
|
48
|
+
*
|
|
49
|
+
* ```bash
|
|
50
|
+
* aws s3vectors create-vector-bucket --vector-bucket-name my-corpus
|
|
51
|
+
* aws s3vectors create-index \
|
|
52
|
+
* --vector-bucket-name my-corpus \
|
|
53
|
+
* --index-name docs \
|
|
54
|
+
* --data-type float32 \
|
|
55
|
+
* --dimension 1024 \
|
|
56
|
+
* --distance-metric cosine \
|
|
57
|
+
* --metadata-configuration '{"nonFilterableMetadataKeys":["af"]}'
|
|
58
|
+
* ```
|
|
59
|
+
*
|
|
60
|
+
* `nonFilterableMetadataKeys: ["af"]` is load-bearing, not decoration. This
|
|
61
|
+
* adapter stores the whole entry — the passage the model will read, its
|
|
62
|
+
* provenance, its timestamps — as JSON under the metadata key `af`. Filterable
|
|
63
|
+
* metadata has a small per-vector budget; non-filterable metadata has the large
|
|
64
|
+
* one. Declare `af` non-filterable and a full passage fits; leave it filterable
|
|
65
|
+
* and PutVectors starts refusing your longer chunks partway through an
|
|
66
|
+
* indexing run.
|
|
67
|
+
*
|
|
68
|
+
* Only `ns` (the identity namespace) and `tier` are ever filtered on, and both
|
|
69
|
+
* are short strings.
|
|
70
|
+
*
|
|
71
|
+
* ── Cosine only, said out loud ──────────────────────────────────────────────
|
|
72
|
+
* The port's score is a cosine similarity, and every threshold in this library
|
|
73
|
+
* — starting with `defineRAG`'s 0.7 default — is calibrated on that range. A
|
|
74
|
+
* cosine distance converts exactly (`score = 1 - distance`). A EUCLIDEAN
|
|
75
|
+
* distance does not: any mapping into [-1, 1] produces a number that READS like
|
|
76
|
+
* a cosine and is not one, which is the same class of confident-and-meaningless
|
|
77
|
+
* value the embedder-mismatch refusal exists to stop. So a euclidean index is
|
|
78
|
+
* refused at construction, by name.
|
|
79
|
+
*
|
|
80
|
+
* ── The fingerprint guarantee, stated exactly ───────────────────────────────
|
|
81
|
+
* `sqliteVectorStore` owns its file and can keep a fingerprint row in it. This
|
|
82
|
+
* store owns nothing but vectors, so the guarantee is assembled from what the
|
|
83
|
+
* service actually provides, and it is smaller — so it is spelled out rather
|
|
84
|
+
* than implied:
|
|
85
|
+
*
|
|
86
|
+
* - **Dimensions** are enforced by S3 Vectors itself. The index declares one;
|
|
87
|
+
* a vector of another length is rejected by the service, loudly, at the
|
|
88
|
+
* call. Nothing here can be sloppier than that.
|
|
89
|
+
* - **The model id** is stamped into every vector's metadata (`fp`) and
|
|
90
|
+
* checked in two places: against the fingerprint this process has already
|
|
91
|
+
* seen for the namespace (at write and at search), and against the
|
|
92
|
+
* fingerprint carried by the HITS that come back (at search). The second
|
|
93
|
+
* one is what survives a restart: the first query after an embedder swap
|
|
94
|
+
* sees documents stamped by the old embedder and refuses by name, instead
|
|
95
|
+
* of returning a confident ranking of two incompatible spaces.
|
|
96
|
+
* - What is NOT caught: the first WRITE of a fresh process into a namespace
|
|
97
|
+
* another embedder built. There is no cheap read that would catch it, and a
|
|
98
|
+
* full index scan on every boot is not one either. It is caught at the next
|
|
99
|
+
* search, before a single wrong answer is returned.
|
|
100
|
+
*
|
|
101
|
+
* ── Lazy peer dependency ────────────────────────────────────────────────────
|
|
102
|
+
* `@aws-sdk/client-s3vectors` is an OPTIONAL peer dependency, required at
|
|
103
|
+
* construction time. Importing `agentfootprint/memory` costs nothing for
|
|
104
|
+
* consumers who never build one of these. Pass `client` to share the SDK
|
|
105
|
+
* configuration your app already has.
|
|
106
|
+
*/
|
|
107
|
+
import type { MemoryIdentity } from '../../memory/identity/index.js';
|
|
108
|
+
import type { MemoryStore } from '../../memory/store/types.js';
|
|
109
|
+
/**
|
|
110
|
+
* The slice of an S3 Vectors client this adapter calls.
|
|
111
|
+
*
|
|
112
|
+
* Structural, so the real SDK client, a pre-built one shared with the rest of
|
|
113
|
+
* your app, or a test double all satisfy it without this package taking a hard
|
|
114
|
+
* type dependency on the optional peer.
|
|
115
|
+
*/
|
|
116
|
+
export interface S3VectorsLikeClient {
|
|
117
|
+
/** `send(command, options?)` — the second argument carries `abortSignal`. */
|
|
118
|
+
send(command: unknown, options?: {
|
|
119
|
+
abortSignal?: AbortSignal;
|
|
120
|
+
}): Promise<unknown>;
|
|
121
|
+
/** Optional — released by {@link S3VectorsStore.close} when this store built the client. */
|
|
122
|
+
destroy?(): void;
|
|
123
|
+
}
|
|
124
|
+
/** The constructors this adapter needs out of `@aws-sdk/client-s3vectors`. */
|
|
125
|
+
export interface S3VectorsSdkModule {
|
|
126
|
+
readonly S3VectorsClient?: new (config: {
|
|
127
|
+
region?: string;
|
|
128
|
+
}) => S3VectorsLikeClient;
|
|
129
|
+
readonly PutVectorsCommand?: new (input: unknown) => unknown;
|
|
130
|
+
readonly QueryVectorsCommand?: new (input: unknown) => unknown;
|
|
131
|
+
readonly GetVectorsCommand?: new (input: unknown) => unknown;
|
|
132
|
+
readonly ListVectorsCommand?: new (input: unknown) => unknown;
|
|
133
|
+
readonly DeleteVectorsCommand?: new (input: unknown) => unknown;
|
|
134
|
+
}
|
|
135
|
+
export interface S3VectorsStoreOptions {
|
|
136
|
+
/** The vector bucket (`vectorBucketName`). Created by you, not by this. */
|
|
137
|
+
readonly bucket: string;
|
|
138
|
+
/** The vector index inside it (`indexName`). Created by you, not by this. */
|
|
139
|
+
readonly index: string;
|
|
140
|
+
/** AWS region. Passed to the SDK client when this factory builds one. */
|
|
141
|
+
readonly region?: string;
|
|
142
|
+
/**
|
|
143
|
+
* The metric the index was created with. Only `'cosine'` is supported, and
|
|
144
|
+
* anything else is refused at construction — see the header. This is a
|
|
145
|
+
* DECLARATION about an index this store did not create; state the metric you
|
|
146
|
+
* actually used.
|
|
147
|
+
*/
|
|
148
|
+
readonly distanceMetric?: 'cosine';
|
|
149
|
+
/**
|
|
150
|
+
* How many vectors go in one PutVectors call. Default 100 — deliberately
|
|
151
|
+
* well under the service limit, because the failure mode of guessing that
|
|
152
|
+
* limit high is a corpus that indexes 90% of the way and stops.
|
|
153
|
+
*/
|
|
154
|
+
readonly batchSize?: number;
|
|
155
|
+
/** A pre-built S3 Vectors client, so one SDK config serves the whole app. */
|
|
156
|
+
readonly client?: S3VectorsLikeClient;
|
|
157
|
+
/** @internal Test injection — skips the SDK require entirely. */
|
|
158
|
+
readonly _client?: S3VectorsLikeClient;
|
|
159
|
+
/** @internal Test injection — the AWS SDK module (exercises the real shim with a mock SDK). */
|
|
160
|
+
readonly _sdk?: S3VectorsSdkModule;
|
|
161
|
+
}
|
|
162
|
+
/** A durable vector index in S3, plus the two things this store owns beyond the port. */
|
|
163
|
+
export interface S3VectorsStore extends MemoryStore {
|
|
164
|
+
/** The vector bucket this store reads and writes. */
|
|
165
|
+
readonly bucket: string;
|
|
166
|
+
/** The vector index inside it. */
|
|
167
|
+
readonly index: string;
|
|
168
|
+
/**
|
|
169
|
+
* The embedder fingerprint (`'<id>@<dims>'`) this PROCESS has seen for a
|
|
170
|
+
* namespace, or `undefined` when it has seen none yet.
|
|
171
|
+
*
|
|
172
|
+
* Deliberately not "the fingerprint the index was built with" — see the
|
|
173
|
+
* header for exactly what this store can and cannot know. It is populated by
|
|
174
|
+
* the first write or the first search of the namespace in this process.
|
|
175
|
+
*/
|
|
176
|
+
fingerprintOf(identity: MemoryIdentity): string | undefined;
|
|
177
|
+
/**
|
|
178
|
+
* Release the SDK client, if this store built one. Idempotent. A client you
|
|
179
|
+
* passed in is yours and is left alone.
|
|
180
|
+
*/
|
|
181
|
+
close(): void;
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* Open a `MemoryStore` over an existing S3 Vectors index.
|
|
185
|
+
*
|
|
186
|
+
* @throws when `@aws-sdk/client-s3vectors` is absent and no `client` was passed.
|
|
187
|
+
* @throws when `distanceMetric` is anything but `'cosine'`.
|
|
188
|
+
* @throws EmbedderMismatchError from `put`/`putMany`/`search` when a vector
|
|
189
|
+
* meets a namespace built by a different embedder.
|
|
190
|
+
*
|
|
191
|
+
* @example A corpus you can add to without a redeploy
|
|
192
|
+
* ```ts
|
|
193
|
+
* import { defineRAG, indexDocuments } from 'agentfootprint';
|
|
194
|
+
* import { s3VectorsStore } from 'agentfootprint/memory';
|
|
195
|
+
* import { bedrockEmbedder } from 'agentfootprint/providers';
|
|
196
|
+
*
|
|
197
|
+
* const store = s3VectorsStore({ bucket: 'my-corpus', index: 'docs', region: 'us-east-1' });
|
|
198
|
+
* const embedder = bedrockEmbedder({ region: 'us-east-1' });
|
|
199
|
+
*
|
|
200
|
+
* // Run this from a cron job. No deploy, no restart — the agent sees it next turn.
|
|
201
|
+
* await indexDocuments(store, embedder, newDocs, { embedderId: embedder.id });
|
|
202
|
+
*
|
|
203
|
+
* const agent = Agent.create({ provider })
|
|
204
|
+
* .rag(defineRAG({ id: 'docs', store, embedder, embedderId: embedder.id }))
|
|
205
|
+
* .build();
|
|
206
|
+
* ```
|
|
207
|
+
*/
|
|
208
|
+
export declare function s3VectorsStore(options: S3VectorsStoreOptions): S3VectorsStore;
|
|
209
|
+
//# sourceMappingURL=s3Vectors.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"s3Vectors.d.ts","sourceRoot":"","sources":["../../../../src/adapters/memory/s3Vectors.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyGG;AAWH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAErE,OAAO,KAAK,EAGV,WAAW,EAIZ,MAAM,6BAA6B,CAAC;AAIrC;;;;;;GAMG;AACH,MAAM,WAAW,mBAAmB;IAClC,6EAA6E;IAC7E,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,WAAW,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAClF,4FAA4F;IAC5F,OAAO,CAAC,IAAI,IAAI,CAAC;CAClB;AAED,8EAA8E;AAC9E,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,eAAe,CAAC,EAAE,KAAK,MAAM,EAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,KAAK,mBAAmB,CAAC;IACpF,QAAQ,CAAC,iBAAiB,CAAC,EAAE,KAAK,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC;IAC7D,QAAQ,CAAC,mBAAmB,CAAC,EAAE,KAAK,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC;IAC/D,QAAQ,CAAC,iBAAiB,CAAC,EAAE,KAAK,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC;IAC7D,QAAQ,CAAC,kBAAkB,CAAC,EAAE,KAAK,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC;IAC9D,QAAQ,CAAC,oBAAoB,CAAC,EAAE,KAAK,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC;CACjE;AAID,MAAM,WAAW,qBAAqB;IACpC,2EAA2E;IAC3E,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,6EAA6E;IAC7E,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,yEAAyE;IACzE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;OAKG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,QAAQ,CAAC;IACnC;;;;OAIG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,6EAA6E;IAC7E,QAAQ,CAAC,MAAM,CAAC,EAAE,mBAAmB,CAAC;IACtC,iEAAiE;IACjE,QAAQ,CAAC,OAAO,CAAC,EAAE,mBAAmB,CAAC;IACvC,+FAA+F;IAC/F,QAAQ,CAAC,IAAI,CAAC,EAAE,kBAAkB,CAAC;CACpC;AAED,yFAAyF;AACzF,MAAM,WAAW,cAAe,SAAQ,WAAW;IACjD,qDAAqD;IACrD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,kCAAkC;IAClC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;;;;;OAOG;IACH,aAAa,CAAC,QAAQ,EAAE,cAAc,GAAG,MAAM,GAAG,SAAS,CAAC;IAC5D;;;OAGG;IACH,KAAK,IAAI,IAAI,CAAC;CACf;AAiBD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,qBAAqB,GAAG,cAAc,CA2c7E"}
|
|
@@ -141,34 +141,12 @@ export declare class UnreadableIndexFileError extends Error {
|
|
|
141
141
|
/**
|
|
142
142
|
* Raised when a vector meets an index built by a different embedder.
|
|
143
143
|
*
|
|
144
|
-
*
|
|
145
|
-
*
|
|
146
|
-
*
|
|
147
|
-
*
|
|
148
|
-
* can tell them apart. So the mismatch is refused where it happens, on both
|
|
149
|
-
* sides:
|
|
150
|
-
*
|
|
151
|
-
* - at **write**, so a second embedder's vectors never enter a namespace;
|
|
152
|
-
* - at **query**, so a swapped embedder never scores against the old ones.
|
|
153
|
-
*
|
|
154
|
-
* The named fix is an explicit re-index — delete the namespace and build it
|
|
155
|
-
* again with one embedder, or point the retriever at a different file. It is
|
|
156
|
-
* never a fallback: silently ignoring the mismatch is the failure, and silently
|
|
157
|
-
* re-embedding somebody's corpus is a bill they did not agree to.
|
|
158
|
-
*
|
|
159
|
-
* `problem` says which half is wrong. `'dimensions'` is arithmetically
|
|
160
|
-
* impossible to score at all; `'model'` would score, and lie.
|
|
144
|
+
* Since 9.3.0 this is ONE class shared by every store that can make the check
|
|
145
|
+
* (`lib/embedderMismatch.ts`), re-exported here so the import path that has
|
|
146
|
+
* worked since 8.9.0 keeps working. A second class of the same name would make
|
|
147
|
+
* `instanceof` depend on which store threw.
|
|
161
148
|
*/
|
|
162
|
-
export
|
|
163
|
-
readonly code: "ERR_EMBEDDER_MISMATCH";
|
|
164
|
-
/** The fingerprint the namespace was built with, `'<id>@<dims>'`. */
|
|
165
|
-
readonly indexed: string;
|
|
166
|
-
/** The fingerprint that just arrived. */
|
|
167
|
-
readonly incoming: string;
|
|
168
|
-
/** Which half disagrees. */
|
|
169
|
-
readonly problem: 'dimensions' | 'model';
|
|
170
|
-
constructor(namespace: string, indexed: string, incoming: string, problem: EmbedderMismatchError['problem'], operation: 'write to' | 'search');
|
|
171
|
-
}
|
|
149
|
+
export { EmbedderMismatchError } from '../../lib/embedderMismatch.js';
|
|
172
150
|
/** A durable vector store, plus the three things a file owns beyond the port. */
|
|
173
151
|
export interface SqliteVectorStore extends MemoryStore {
|
|
174
152
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sqliteVector.d.ts","sourceRoot":"","sources":["../../../../src/adapters/memory/sqliteVector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwEG;
|
|
1
|
+
{"version":3,"file":"sqliteVector.d.ts","sourceRoot":"","sources":["../../../../src/adapters/memory/sqliteVector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwEG;AAYH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAErE,OAAO,KAAK,EAGV,WAAW,EAIZ,MAAM,6BAA6B,CAAC;AAIrC,wDAAwD;AACxD,MAAM,WAAW,yBAAyB;IACxC,GAAG,CAAC,GAAG,MAAM,EAAE,SAAS,OAAO,EAAE,GAAG,OAAO,CAAC;IAC5C,GAAG,CAAC,GAAG,MAAM,EAAE,SAAS,OAAO,EAAE,GAAG,OAAO,CAAC;IAC5C,GAAG,CAAC,GAAG,MAAM,EAAE,SAAS,OAAO,EAAE,GAAG,OAAO,EAAE,CAAC;CAC/C;AAED,mDAAmD;AACnD,MAAM,WAAW,wBAAwB;IACvC,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,yBAAyB,CAAC;IAChD,KAAK,IAAI,IAAI,CAAC;CACf;AAED,uEAAuE;AACvE,MAAM,WAAW,sBAAsB;IACrC,KAAK,IAAI,EAAE,MAAM,GAAG,wBAAwB,CAAC;CAC9C;AAID,MAAM,WAAW,wBAAwB;IACvC;;;;;;;;;OASG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB;;;;;OAKG;IACH,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC;;;OAGG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,sBAAsB,CAAC;CAC3C;AAID;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,wBAAyB,SAAQ,KAAK;IACjD,QAAQ,CAAC,IAAI,8BAAwC;IACrD,iCAAiC;IACjC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,wCAAwC;IACxC,QAAQ,CAAC,OAAO,EAAE,aAAa,GAAG,gBAAgB,GAAG,cAAc,CAAC;gBAExD,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,wBAAwB,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,MAAM;CAmBvF;AAED;;;;;;;GAOG;AACH,OAAO,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AA4CtE,iFAAiF;AACjF,MAAM,WAAW,iBAAkB,SAAQ,WAAW;IACpD;;;;;;;;OAQG;IACH,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,wCAAwC;IACxC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB;;;;;;OAMG;IACH,aAAa,CAAC,QAAQ,EAAE,cAAc,GAAG,MAAM,GAAG,SAAS,CAAC;IAC5D;;;;;;;;;;;;;;;;;;;OAmBG;IACH,IAAI,CAAC,QAAQ,EAAE,cAAc,GAAG,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC/E;;;;OAIG;IACH,KAAK,IAAI,IAAI,CAAC;CACf;AAeD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,wBAAwB,GAAG,iBAAiB,CAqetF"}
|
|
@@ -8,7 +8,8 @@
|
|
|
8
8
|
* you use and agentfootprint stays dependency-free.
|
|
9
9
|
*
|
|
10
10
|
* openaiEmbedder() — hosted; needs OPENAI_API_KEY; no extra install (fetch).
|
|
11
|
-
* bedrockEmbedder() — hosted on AWS; Titan Text Embeddings
|
|
11
|
+
* bedrockEmbedder() — hosted on AWS; Titan Text Embeddings and Cohere Embed
|
|
12
|
+
* v3 (the model id picks the body shape); credentials
|
|
12
13
|
* come from the AWS chain, so no key option at all.
|
|
13
14
|
* peer dep: @aws-sdk/client-bedrock-runtime.
|
|
14
15
|
* localEmbedder() — on-device sentence-transformer; no key; offline after a
|
|
@@ -111,25 +112,85 @@ export interface BedrockRuntimeSdkModule {
|
|
|
111
112
|
}) => BedrockRuntimeLikeClient;
|
|
112
113
|
readonly InvokeModelCommand?: new (input: unknown) => unknown;
|
|
113
114
|
}
|
|
115
|
+
/**
|
|
116
|
+
* The request/response SHAPE a Bedrock embedding model speaks (9.3.0).
|
|
117
|
+
*
|
|
118
|
+
* `InvokeModel` is one operation with a vendor-specific body on both sides:
|
|
119
|
+
* Titan takes `{ inputText }` and answers `{ embedding }`, Cohere takes
|
|
120
|
+
* `{ texts, input_type }` and answers `{ embeddings }`. One model id therefore
|
|
121
|
+
* does not describe one call, and until 9.3.0 this factory sent Titan's body to
|
|
122
|
+
* everything — so a Cohere model id was accepted at construction (with
|
|
123
|
+
* `dimensions`) and failed at the first embed, against the real service, with a
|
|
124
|
+
* validation error from AWS rather than a sentence from here.
|
|
125
|
+
*/
|
|
126
|
+
export type BedrockEmbeddingFamily = 'titan' | 'cohere';
|
|
127
|
+
/**
|
|
128
|
+
* Cohere's `input_type`, which is a real parameter and not a hint: the v3
|
|
129
|
+
* models embed a QUERY and a DOCUMENT into deliberately different places, and
|
|
130
|
+
* the two are meant to be compared with each other. Sending one value for both
|
|
131
|
+
* halves is a measurable loss of retrieval quality, not a style choice — and
|
|
132
|
+
* Cohere requires the field, so there is no "unset" to fall back to.
|
|
133
|
+
*/
|
|
134
|
+
export type CohereInputType = 'search_document' | 'search_query';
|
|
114
135
|
export interface BedrockEmbedderOptions {
|
|
115
136
|
/**
|
|
116
137
|
* Bedrock model id. Default `'amazon.titan-embed-text-v2:0'`.
|
|
117
138
|
*
|
|
118
|
-
*
|
|
119
|
-
*
|
|
120
|
-
*
|
|
121
|
-
*
|
|
139
|
+
* Four are known by name — Titan V2, Titan V1, and Cohere Embed English /
|
|
140
|
+
* Multilingual v3 (see {@link BEDROCK_EMBEDDING_MODELS}) — and each brings
|
|
141
|
+
* its own body shape, vector length and input window. An id that WRAPS one of
|
|
142
|
+
* those (a cross-region inference profile `us.amazon.titan-embed-text-v2:0`,
|
|
143
|
+
* or an ARN ending in the model id) is resolved to the model it names.
|
|
144
|
+
*
|
|
145
|
+
* Anything else is a model this library has never met: pass `dimensions`
|
|
146
|
+
* with it (its vector length is not something this can know) and `family` if
|
|
147
|
+
* it is not Titan-shaped.
|
|
122
148
|
*/
|
|
123
149
|
readonly model?: string;
|
|
124
150
|
/**
|
|
125
|
-
* Vector length to request.
|
|
151
|
+
* Vector length to request.
|
|
152
|
+
*
|
|
153
|
+
* Titan V2 is the one CONFIGURABLE model — 1024 (default), 512 or 256 — and
|
|
126
154
|
* the value is SENT to the model AND reported as `.dimensions`, so the two
|
|
127
|
-
* can never disagree.
|
|
155
|
+
* can never disagree. Every other known model has ONE size, and asking for a
|
|
156
|
+
* different one is refused rather than reported: `.dimensions` is what a
|
|
157
|
+
* vector store fingerprints on, and a wrong one corrupts it silently.
|
|
128
158
|
*
|
|
129
|
-
* Required for a model outside {@link
|
|
130
|
-
* `.dimensions` silently corrupts a vector store.
|
|
159
|
+
* Required for a model outside {@link BEDROCK_EMBEDDING_MODELS}.
|
|
131
160
|
*/
|
|
132
161
|
readonly dimensions?: number;
|
|
162
|
+
/**
|
|
163
|
+
* The body shape to speak, when the model id does not say (9.3.0).
|
|
164
|
+
*
|
|
165
|
+
* Inferred for every known model and for anything that wraps one, so this is
|
|
166
|
+
* only for a model id this library has never met — a provisioned-throughput
|
|
167
|
+
* ARN, a custom deployment. Unknown and unstated, the body is **Titan's**,
|
|
168
|
+
* which is the shape every release before 9.3.0 sent to everything.
|
|
169
|
+
*
|
|
170
|
+
* Stating a family that contradicts a known model id is refused by name.
|
|
171
|
+
*/
|
|
172
|
+
readonly family?: BedrockEmbeddingFamily;
|
|
173
|
+
/**
|
|
174
|
+
* Pin Cohere's `input_type` instead of deriving it from the call (9.3.0).
|
|
175
|
+
*
|
|
176
|
+
* Unset — the default — `embed()` sends `'search_query'` and `embedBatch()`
|
|
177
|
+
* sends `'search_document'`, because that is what this library's own two
|
|
178
|
+
* call sites are: retrieval embeds ONE question (`loadRelevant`), indexing
|
|
179
|
+
* embeds MANY passages (`indexDocuments`, `embedMessages`). Pin it when your
|
|
180
|
+
* own code uses the two calls differently — embedding a single document, say,
|
|
181
|
+
* or scoring a batch of queries.
|
|
182
|
+
*
|
|
183
|
+
* Ignored by Titan, which has no such parameter.
|
|
184
|
+
*/
|
|
185
|
+
readonly inputType?: CohereInputType;
|
|
186
|
+
/**
|
|
187
|
+
* The longest input this model reads whole, in CHARACTERS
|
|
188
|
+
* ({@link Embedder.maxInputChars}). Declared for every known model from its
|
|
189
|
+
* documented token window; this option is how a model this library does not
|
|
190
|
+
* know states its own, rather than declaring none and leaving the indexer's
|
|
191
|
+
* conservative default in place. An explicit value always wins.
|
|
192
|
+
*/
|
|
193
|
+
readonly maxInputChars?: number;
|
|
133
194
|
/** AWS region. Passed to the SDK client when this factory builds one. */
|
|
134
195
|
readonly region?: string;
|
|
135
196
|
/** A pre-built Bedrock runtime client, so one SDK config serves the whole app. */
|
|
@@ -167,19 +228,31 @@ export interface BedrockEmbedderOptions {
|
|
|
167
228
|
* q8 and an fp32 build of one model are near-identical spaces, and "near" is
|
|
168
229
|
* exactly the difference that surfaces as a mysteriously worse ranking.)
|
|
169
230
|
*
|
|
231
|
+
* ── One operation, two body shapes (9.3.0) ───────────────────────────────
|
|
232
|
+
* `InvokeModel` is a single API over vendor-specific JSON. Titan takes
|
|
233
|
+
* `{ inputText }` and answers `{ embedding }`; Cohere takes
|
|
234
|
+
* `{ texts, input_type }` and answers `{ embeddings }`, embeds up to
|
|
235
|
+
* {@link COHERE_MAX_TEXTS_PER_CALL} of them per call, and distinguishes a
|
|
236
|
+
* QUERY from a DOCUMENT. So the model id selects a FAMILY
|
|
237
|
+
* ({@link BedrockEmbeddingFamily}), and the family owns the request, the
|
|
238
|
+
* response and the batching. Before this, one shape was sent to everything —
|
|
239
|
+
* a Cohere id constructed fine and failed at the first embed.
|
|
240
|
+
*
|
|
170
241
|
* ── The input ceiling (9.1.0) ────────────────────────────────────────────
|
|
171
|
-
* `.maxInputChars`
|
|
172
|
-
*
|
|
173
|
-
*
|
|
174
|
-
* the indexer's own default, which was measured on an on-device model
|
|
175
|
-
*
|
|
176
|
-
*
|
|
177
|
-
*
|
|
178
|
-
*
|
|
242
|
+
* `.maxInputChars` is per MODEL, from its documented token window converted at
|
|
243
|
+
* the stated {@link CHARS_PER_TOKEN} assumption of 4 characters per token:
|
|
244
|
+
* **32,000** for both Titan text-embedding models (8,192 tokens — sixteen
|
|
245
|
+
* times the indexer's own default, which was measured on an on-device model),
|
|
246
|
+
* and **2,000** for Cohere Embed v3 (512 tokens). Those two numbers are why the
|
|
247
|
+
* ceiling cannot be a per-vendor constant: the same 2,500-character chunk is
|
|
248
|
+
* read whole by Titan and truncated by Cohere. Dense text (code, tables, CJK)
|
|
249
|
+
* tokenises tighter than the assumption; pass an explicit `maxChunkChars` for
|
|
250
|
+
* such a corpus, and it wins over this number.
|
|
179
251
|
*
|
|
180
252
|
* @throws if `model` is unknown and `dimensions` was not supplied; if
|
|
181
|
-
* `dimensions` is
|
|
182
|
-
*
|
|
253
|
+
* `dimensions` is a size the model does not produce; if `family`
|
|
254
|
+
* contradicts a known model id; or if the SDK is missing and no
|
|
255
|
+
* `client` / `_client` / `_sdk` was passed.
|
|
183
256
|
*
|
|
184
257
|
* @example
|
|
185
258
|
* ```ts
|
|
@@ -190,6 +263,13 @@ export interface BedrockEmbedderOptions {
|
|
|
190
263
|
* const embedder = bedrockEmbedder({ region: 'us-east-1', dimensions: 512 });
|
|
191
264
|
* await indexFolder('./docs', { to: sqliteVectorStore({ file: './corpus.db' }), embedder });
|
|
192
265
|
* ```
|
|
266
|
+
*
|
|
267
|
+
* @example A Cohere embedding model on the same runtime
|
|
268
|
+
* ```ts
|
|
269
|
+
* // Body shape, response field, batch size and 512-token window all follow
|
|
270
|
+
* // from the model id — nothing else changes at the call site.
|
|
271
|
+
* const embedder = bedrockEmbedder({ model: 'cohere.embed-english-v3' });
|
|
272
|
+
* ```
|
|
193
273
|
*/
|
|
194
274
|
export declare function bedrockEmbedder(options?: BedrockEmbedderOptions): Embedder;
|
|
195
275
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/embedders/index.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/embedders/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,YAAY,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAC;AAO7D,MAAM,WAAW,qBAAqB;IACpC,2CAA2C;IAC3C,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,yCAAyC;IACzC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;;;;;;;OAWG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,gEAAgE;IAChE,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;CAC3B;AA4DD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,cAAc,CAAC,OAAO,GAAE,qBAA0B,GAAG,QAAQ,CAwD5E;AAMD;;;;;;GAMG;AACH,MAAM,WAAW,wBAAwB;IACvC;;;;;OAKG;IACH,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,WAAW,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACnF;AAED,oDAAoD;AACpD,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,oBAAoB,CAAC,EAAE,KAAK,MAAM,EAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,KAAK,wBAAwB,CAAC;IAC9F,QAAQ,CAAC,kBAAkB,CAAC,EAAE,KAAK,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC;CAC/D;AAED;;;;;;;;;;GAUG;AACH,MAAM,MAAM,sBAAsB,GAAG,OAAO,GAAG,QAAQ,CAAC;AAExD;;;;;;GAMG;AACH,MAAM,MAAM,eAAe,GAAG,iBAAiB,GAAG,cAAc,CAAC;AAEjE,MAAM,WAAW,sBAAsB;IACrC;;;;;;;;;;;;OAYG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B;;;;;;;;;OASG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,sBAAsB,CAAC;IACzC;;;;;;;;;;;OAWG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,eAAe,CAAC;IACrC;;;;;;OAMG;IACH,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,yEAAyE;IACzE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,kFAAkF;IAClF,QAAQ,CAAC,MAAM,CAAC,EAAE,wBAAwB,CAAC;IAC3C,iEAAiE;IACjE,QAAQ,CAAC,OAAO,CAAC,EAAE,wBAAwB,CAAC;IAC5C,+FAA+F;IAC/F,QAAQ,CAAC,IAAI,CAAC,EAAE,uBAAuB,CAAC;CACzC;AAmHD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsEG;AACH,wBAAgB,eAAe,CAAC,OAAO,GAAE,sBAA2B,GAAG,QAAQ,CAqM9E;AA6GD;;;;;;GAMG;AACH,MAAM,WAAW,mBAAmB;IAClC,wDAAwD;IACxD,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5F,mEAAmE;IACnE,GAAG,CAAC,EAAE,OAAO,CAAC;CACf;AAED,MAAM,WAAW,oBAAoB;IACnC,kEAAkE;IAClE,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,+CAA+C;IAC/C,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,0EAA0E;IAC1E,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,qCAAqC;IACrC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B;;;;;;;;;;;OAWG;IACH,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC;;;;;;;;;;;OAWG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,mBAAmB,CAAC;CACxC;AAoBD,wBAAgB,aAAa,CAAC,OAAO,GAAE,oBAAyB,GAAG,QAAQ,CA0C1E;AAMD;;;;;;GAMG;AACH,MAAM,WAAW,gBAAgB;IAC/B,6DAA6D;IAC7D,KAAK,CAAC,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC;IAC1C,wCAAwC;IACxC,MAAM,CAAC,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC;IAC3C,oEAAoE;IACpE,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,qBAAqB;IACpC,wEAAwE;IACxE,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,sEAAsE;IACtE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,gBAAgB,CAAC;CACrC;AAmBD,wBAAgB,cAAc,CAAC,OAAO,GAAE,qBAA0B,GAAG,QAAQ,CA8D5E"}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* EmbedderMismatchError — one class, for every store that can tell.
|
|
3
|
+
*
|
|
4
|
+
* Pattern: shared refusal type (the `SqliteUnavailableError` precedent).
|
|
5
|
+
* Role: lib/, so no store adapter owns it and none has to import another.
|
|
6
|
+
* Emits: N/A.
|
|
7
|
+
*
|
|
8
|
+
* It lived inside `sqliteVector.ts` from 8.9.0 until 9.3.0, when a second and
|
|
9
|
+
* a third store learned to make the same check. A duplicate class of the same
|
|
10
|
+
* name would mean `catch (e) { if (e instanceof EmbedderMismatchError) }`
|
|
11
|
+
* quietly depended on WHICH store threw — so there is exactly one, here, and
|
|
12
|
+
* the stores import it.
|
|
13
|
+
*/
|
|
14
|
+
/**
|
|
15
|
+
* Raised when a vector meets an index built by a different embedder.
|
|
16
|
+
*
|
|
17
|
+
* **This is the refusal that keeps a vector store honest.** Cosine similarity
|
|
18
|
+
* between two different embedding spaces is not a weak signal — it is not a
|
|
19
|
+
* signal at all, and it comes back as a confident number in the same 0-to-1
|
|
20
|
+
* range as a real one. There is no threshold that separates them, and nothing
|
|
21
|
+
* downstream can tell them apart. So the mismatch is refused where it happens,
|
|
22
|
+
* on both sides:
|
|
23
|
+
*
|
|
24
|
+
* - at **write**, so a second embedder's vectors never enter a namespace;
|
|
25
|
+
* - at **query**, so a swapped embedder never scores against the old ones.
|
|
26
|
+
*
|
|
27
|
+
* The named fix is an explicit re-index — delete the namespace and build it
|
|
28
|
+
* again with one embedder, or point the retriever somewhere else. It is never a
|
|
29
|
+
* fallback: silently ignoring the mismatch is the failure, and silently
|
|
30
|
+
* re-embedding somebody's corpus is a bill they did not agree to.
|
|
31
|
+
*
|
|
32
|
+
* `problem` says which half is wrong. `'dimensions'` is arithmetically
|
|
33
|
+
* impossible to score at all; `'model'` would score, and lie.
|
|
34
|
+
*/
|
|
35
|
+
export declare class EmbedderMismatchError extends Error {
|
|
36
|
+
readonly code: "ERR_EMBEDDER_MISMATCH";
|
|
37
|
+
/** The fingerprint the namespace was built with, `'<id>@<dims>'`. */
|
|
38
|
+
readonly indexed: string;
|
|
39
|
+
/** The fingerprint that just arrived. */
|
|
40
|
+
readonly incoming: string;
|
|
41
|
+
/** Which half disagrees. */
|
|
42
|
+
readonly problem: 'dimensions' | 'model';
|
|
43
|
+
/**
|
|
44
|
+
* @param alternative the store-specific second way out, named in the
|
|
45
|
+
* message. A file-backed store says "point this store at a different
|
|
46
|
+
* file"; a service-backed one names its own unit of separation.
|
|
47
|
+
*/
|
|
48
|
+
constructor(namespace: string, indexed: string, incoming: string, problem: EmbedderMismatchError['problem'], operation: 'write to' | 'search', alternative?: string);
|
|
49
|
+
}
|
|
50
|
+
/** An embedder fingerprint, split into the two halves that decide separately. */
|
|
51
|
+
export interface Fingerprint {
|
|
52
|
+
readonly id?: string;
|
|
53
|
+
readonly dims: number;
|
|
54
|
+
}
|
|
55
|
+
/** `'<id>@<dims>'`, with `'?'` for an embedder that did not name itself. */
|
|
56
|
+
export declare function fingerprintText(fp: Fingerprint): string;
|
|
57
|
+
/** Parse the `'<id>@<dims>'` form back into its halves. */
|
|
58
|
+
export declare function parseFingerprint(text: string): Fingerprint;
|
|
59
|
+
/**
|
|
60
|
+
* What, if anything, makes these two incompatible.
|
|
61
|
+
*
|
|
62
|
+
* Dimensions always decide: two lengths cannot be compared at all. Model ids
|
|
63
|
+
* decide only when BOTH sides named themselves — an anonymous vector is not
|
|
64
|
+
* evidence of a different embedder, and refusing on absence would break every
|
|
65
|
+
* caller who never passed an `embedderId`, which is most of them.
|
|
66
|
+
*/
|
|
67
|
+
export declare function fingerprintConflict(stored: Fingerprint, incoming: Fingerprint): 'dimensions' | 'model' | null;
|
|
68
|
+
//# sourceMappingURL=embedderMismatch.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedderMismatch.d.ts","sourceRoot":"","sources":["../../../src/lib/embedderMismatch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,qBAAa,qBAAsB,SAAQ,KAAK;IAC9C,QAAQ,CAAC,IAAI,0BAAoC;IACjD,qEAAqE;IACrE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,yCAAyC;IACzC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,4BAA4B;IAC5B,QAAQ,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC;IAEzC;;;;OAIG;gBAED,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,qBAAqB,CAAC,SAAS,CAAC,EACzC,SAAS,EAAE,UAAU,GAAG,QAAQ,EAChC,WAAW,SAA0C;CAoBxD;AAED,iFAAiF;AACjF,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED,4EAA4E;AAC5E,wBAAgB,eAAe,CAAC,EAAE,EAAE,WAAW,GAAG,MAAM,CAEvD;AAED,2DAA2D;AAC3D,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,CAQ1D;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,WAAW,GACpB,YAAY,GAAG,OAAO,GAAG,IAAI,CAM/B"}
|
|
@@ -139,22 +139,40 @@ export interface DefineRAGOptions {
|
|
|
139
139
|
*/
|
|
140
140
|
readonly description?: string;
|
|
141
141
|
/**
|
|
142
|
-
*
|
|
143
|
-
* `
|
|
144
|
-
*
|
|
145
|
-
*
|
|
142
|
+
* Store containing the indexed corpus. Must implement `search()`. Use
|
|
143
|
+
* `indexDocuments(store, embedder, docs)` at startup to populate it. Ships
|
|
144
|
+
* with `InMemoryStore` for dev/tests; swap to a durable adapter in
|
|
145
|
+
* production.
|
|
146
|
+
*
|
|
147
|
+
* A store that declares `ranksBy: 'server-text'` is served by the backend's
|
|
148
|
+
* own index rather than one built here — see {@link embedder}.
|
|
146
149
|
*/
|
|
147
150
|
readonly store: MemoryStore;
|
|
148
151
|
/**
|
|
149
152
|
* Embedder used for the read-side query. Pass the SAME embedder
|
|
150
153
|
* instance (or one with the same `embedderId`) that was used for
|
|
151
154
|
* indexing — cross-model similarity scores are not comparable.
|
|
155
|
+
*
|
|
156
|
+
* **Optional since 9.3.0, for one case only.** A store that declares
|
|
157
|
+
* `ranksBy: 'server-text'` (see {@link MemoryStore.ranksBy}) takes the
|
|
158
|
+
* question as WORDS and ranks it on the backend's side; there is nothing
|
|
159
|
+
* here for an embedder to do, and embedding the query anyway would be spend
|
|
160
|
+
* on a vector discarded on arrival. Against such a store this must be
|
|
161
|
+
* OMITTED — passing one is refused rather than ignored, because an ignored
|
|
162
|
+
* embedder reads, from the wiring, exactly like a working one.
|
|
163
|
+
*
|
|
164
|
+
* Against every other store it is still required.
|
|
152
165
|
*/
|
|
153
|
-
readonly embedder
|
|
166
|
+
readonly embedder?: Embedder;
|
|
154
167
|
/**
|
|
155
168
|
* Stable id of the embedder. Stored on entries during indexing
|
|
156
169
|
* (via `indexDocuments`) and filtered at search time so a later
|
|
157
170
|
* embedder swap doesn't pollute results.
|
|
171
|
+
*
|
|
172
|
+
* Refused alongside a `'server-text'` store for the same reason
|
|
173
|
+
* {@link embedder} is: the backend's records were never written here and
|
|
174
|
+
* carry no `embeddingModel` to filter on, so the option would name a filter
|
|
175
|
+
* that filtered nothing.
|
|
158
176
|
*/
|
|
159
177
|
readonly embedderId?: string;
|
|
160
178
|
/**
|
|
@@ -264,7 +282,11 @@ export interface DefineRAGOptions {
|
|
|
264
282
|
* (or, equivalently, `.memory(definition)` — same plumbing).
|
|
265
283
|
*
|
|
266
284
|
* @throws when `store` does not implement `search()`. RAG requires a
|
|
267
|
-
*
|
|
285
|
+
* store that can retrieve.
|
|
286
|
+
* @throws when `embedder` is missing and the store does not rank text
|
|
287
|
+
* server-side — somebody has to turn the question into a vector.
|
|
288
|
+
* @throws when `embedder`/`embedderId` is passed to a store that DOES rank
|
|
289
|
+
* text server-side — the option would be read by nothing.
|
|
268
290
|
* @throws when `retrieval` is combined with `topK` or `threshold`.
|
|
269
291
|
*/
|
|
270
292
|
export declare function defineRAG(opts: DefineRAGOptions): MemoryDefinition;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"defineRAG.d.ts","sourceRoot":"","sources":["../../../../src/lib/rag/defineRAG.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsHG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAChE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAC/D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AACrE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AACrE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;
|
|
1
|
+
{"version":3,"file":"defineRAG.d.ts","sourceRoot":"","sources":["../../../../src/lib/rag/defineRAG.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsHG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAChE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAC/D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AACrE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AACrE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAMzE;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,EAAE,cAA8C,CAAC;AAErF,MAAM,WAAW,gBAAgB;IAC/B,kEAAkE;IAClE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAEpB;;;;OAIG;IACH,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAE9B;;;;;;;;OAQG;IACH,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAE5B;;;;;;;;;;;;;;OAcG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC;IAE7B;;;;;;;;;OASG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAE7B;;;;;;;;;;;;;;OAcG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,cAAc,CAAC;IAEjC;;;;;;OAMG;IACH,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAEvB;;;;;;;;;;;;;;;;OAgBG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAyCG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAE3B;;;;;;;;;;;;;OAaG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,iBAAiB,CAAC;CAUxC;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,gBAAgB,GAAG,gBAAgB,CAmGlE"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"define.d.ts","sourceRoot":"","sources":["../../../src/memory/define.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;
|
|
1
|
+
{"version":3,"file":"define.d.ts","sourceRoot":"","sources":["../../../src/memory/define.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAcH,OAAO,EAIL,KAAK,mBAAmB,EAYzB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,MAAM,mBAAmB,CAAC;AAInF;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,gBAAgB,CA4B3E;AAmWD;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,CAAC,EAAE,OAAO,EAAE,uBAAuB,CAAC,CAAC,CAAC,GAAG,OAAO,CAErF"}
|
|
@@ -172,7 +172,17 @@ export interface TopKShorthandStrategy {
|
|
|
172
172
|
readonly topK: number;
|
|
173
173
|
/** Min cosine similarity. Strict — no fallback below this. Default 0.7. */
|
|
174
174
|
readonly threshold?: number;
|
|
175
|
-
|
|
175
|
+
/**
|
|
176
|
+
* The embedder that turns the query into a vector.
|
|
177
|
+
*
|
|
178
|
+
* Optional since 9.3.0 for ONE case: a store declaring
|
|
179
|
+
* `ranksBy: 'server-text'` ranks the question as words on the backend's
|
|
180
|
+
* side, so there is nothing here to embed. Omitted anywhere else,
|
|
181
|
+
* `defineMemory` refuses by name — the requirement moved from the type to a
|
|
182
|
+
* runtime check because it now depends on the STORE, which the type cannot
|
|
183
|
+
* see.
|
|
184
|
+
*/
|
|
185
|
+
readonly embedder?: Embedder;
|
|
176
186
|
/**
|
|
177
187
|
* Stable id of the embedder, filtered against `MemoryEntry.embeddingModel`
|
|
178
188
|
* at search time so a later embedder swap cannot silently mix two vector
|
|
@@ -189,7 +199,8 @@ export interface TopKShorthandStrategy {
|
|
|
189
199
|
/** The spelled-out rule (8.8.0) — and the seam a re-ranker will arrive through. */
|
|
190
200
|
export interface TopKRetrievalStrategy {
|
|
191
201
|
readonly kind: typeof MEMORY_STRATEGIES.TOP_K;
|
|
192
|
-
|
|
202
|
+
/** See {@link TopKShorthandStrategy.embedder} — optional for server-text stores only. */
|
|
203
|
+
readonly embedder?: Embedder;
|
|
193
204
|
readonly retrieval: RetrievalStrategy;
|
|
194
205
|
/** See {@link TopKShorthandStrategy.embedderId}. */
|
|
195
206
|
readonly embedderId?: string;
|