@pithy-sh/vector 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,119 @@
1
+ // SPDX-FileCopyrightText: 2026 Pithy
2
+ // SPDX-License-Identifier: MIT
3
+
4
+ import { VectorMetadataIndexDriftError } from "../error/errors";
5
+ import type { MetadataIndexDescriptor, MetadataIndexType } from "./metadata";
6
+
7
+ /**
8
+ * The check that a declared filterable field actually has a live metadata index.
9
+ *
10
+ * A declared-but-missing metadata index is **fatal**. Failing loudly is deliberate, and the alternative is
11
+ * worse than it looks. Auto-create-and-warn produces two things: a warning in a log nobody reads, and a search
12
+ * that returns partial results for every vector written before the index existed — because Cloudflare states,
13
+ * verbatim, *"Vectors upserted before a metadata index was created won't have their metadata contained in that
14
+ * index."* Nothing errors. The filter just quietly stops matching most of the corpus, and the bug surfaces
15
+ * weeks later as "search feels wrong", which is close to untraceable.
16
+ *
17
+ * **The live comparison runs where the control plane is reachable, which is the CLI — not inside the
18
+ * Worker.** `listMetadataIndexes` is a Cloudflare REST call, and the only credential that authorizes it is an
19
+ * account API token. Handing a Worker one so it could self-check at boot would put a control-plane token on
20
+ * the request path of every search, which is a far worse trade than the bug it would catch. So the live
21
+ * comparison happens in `pithy vector provision`, which already holds the token.
22
+ * {@link assertMetadataIndexes} is the strict form of it, for a caller that wants drift to be an error rather
23
+ * than a report.
24
+ *
25
+ * The Worker still checks — it just checks offline. Provisioning writes down what it observed, and
26
+ * `index/provisioned.ts` compares the config's declarations against that record at boot, reusing
27
+ * {@link compareMetadataIndexes} so both halves apply one rule. That check proves the config declares nothing
28
+ * provisioning did not see *the last time it ran*; it does not prove anything about Cloudflare right now. It
29
+ * catches the case that actually happens: a metadata schema edited and deployed without re-provisioning.
30
+ *
31
+ * A live index the config does not declare is **not** fatal. It may predate the config, or belong to another
32
+ * consumer of the same Vectorize index, and deleting it is destructive; it is reported so `pithy vector
33
+ * provision` can surface it, because it still spends one of the ten slots.
34
+ */
35
+
36
+ /** A metadata index whose live type does not match the type the schema declares. */
37
+ export interface MetadataIndexMismatch {
38
+ /** The metadata property. */
39
+ propertyName: string;
40
+ /** The type the config's metadata schema declares. */
41
+ declared: MetadataIndexType;
42
+ /** The type the live index was created with. */
43
+ live: string;
44
+ }
45
+
46
+ /** What one comparison of declared against live metadata indexes found. */
47
+ export interface MetadataIndexReport {
48
+ /** Declared filterable, with no live metadata index. Fatal: filters on these silently return partial results. */
49
+ missing: MetadataIndexDescriptor[];
50
+ /** Declared and live, but indexed as a different type. Fatal: comparisons against the wrong type never match. */
51
+ mismatched: MetadataIndexMismatch[];
52
+ /** Live, but not declared. Not fatal — but it still spends one of the ten metadata-index slots. */
53
+ extra: { propertyName: string; indexType: string }[];
54
+ }
55
+
56
+ /** The subset of the Vectorize control plane this check reads. Structural, so a test injects a fake. */
57
+ export interface MetadataIndexSource {
58
+ /** The metadata indexes that exist on an index right now. */
59
+ listMetadataIndexes(indexName: string): Promise<{ propertyName: string; indexType: string }[]>;
60
+ }
61
+
62
+ /** Compare what the schema declares against what the index has. Pure — no network, so it is trivially tested. */
63
+ export function compareMetadataIndexes(
64
+ declared: readonly MetadataIndexDescriptor[],
65
+ live: readonly { propertyName: string; indexType: string }[],
66
+ ): MetadataIndexReport {
67
+ const liveByName = new Map(live.map((index) => [index.propertyName, index]));
68
+ const declaredNames = new Set(declared.map((index) => index.propertyName));
69
+
70
+ const missing: MetadataIndexDescriptor[] = [];
71
+ const mismatched: MetadataIndexMismatch[] = [];
72
+ for (const index of declared) {
73
+ const match = liveByName.get(index.propertyName);
74
+ if (!match) {
75
+ missing.push(index);
76
+ continue;
77
+ }
78
+ if (match.indexType !== index.indexType) {
79
+ mismatched.push({ propertyName: index.propertyName, declared: index.indexType, live: match.indexType });
80
+ }
81
+ }
82
+
83
+ const extra = live.filter((index) => !declaredNames.has(index.propertyName)).map((index) => ({ ...index }));
84
+ return { missing, mismatched, extra };
85
+ }
86
+
87
+ /**
88
+ * Fetch the live metadata indexes and refuse to continue if any declared one is missing or mis-typed. Returns
89
+ * the full report so a caller — `pithy vector doctor`, provisioning — can also surface the undeclared ones.
90
+ *
91
+ * Takes a {@link MetadataIndexSource} rather than reaching for one, because the only thing that can implement
92
+ * it is a control-plane client holding an API token. That is the CLI, never the Worker.
93
+ */
94
+ export async function assertMetadataIndexes(
95
+ source: MetadataIndexSource,
96
+ indexName: string,
97
+ declared: readonly MetadataIndexDescriptor[],
98
+ ): Promise<MetadataIndexReport> {
99
+ const report = compareMetadataIndexes(declared, await source.listMetadataIndexes(indexName));
100
+
101
+ if (report.missing.length > 0 || report.mismatched.length > 0) {
102
+ const missing = report.missing.map((index) => `${index.propertyName} (${index.indexType})`);
103
+ const mismatched = report.mismatched.map(
104
+ (index) => `${index.propertyName} (declared ${index.declared}, indexed as ${index.live})`,
105
+ );
106
+ throw new VectorMetadataIndexDriftError({
107
+ message: `Search on \`${indexName}\` is not configured correctly.`,
108
+ action: `Run \`pithy vector provision\` to create the missing metadata indexes on \`${indexName}\`, then re-embed — Vectorize does not index vectors written before an index existed.`,
109
+ detail: [
110
+ missing.length > 0 ? `missing: ${missing.join(", ")}` : "",
111
+ mismatched.length > 0 ? `mismatched: ${mismatched.join(", ")}` : "",
112
+ ]
113
+ .filter(Boolean)
114
+ .join("; "),
115
+ });
116
+ }
117
+
118
+ return report;
119
+ }
@@ -0,0 +1,278 @@
1
+ // SPDX-FileCopyrightText: 2026 Pithy
2
+ // SPDX-License-Identifier: MIT
3
+
4
+ import { ValidationError } from "@pithy-sh/core/src/error/pithyError";
5
+ import type { z } from "zod";
6
+ import { VectorFilterTooLargeError, VectorUnfilterableFieldError } from "../error/errors";
7
+ import { byteLength, filterKeyProblem, MAX_FILTER_BYTES } from "./limits";
8
+ import { introspectMetadata, type MetadataIndexDescriptor } from "./metadata";
9
+
10
+ /**
11
+ * The typed metadata filter builder.
12
+ *
13
+ * Filtering an unindexed field is the failure this package exists to prevent, and it is invisible at runtime:
14
+ * Vectorize accepts the filter and returns a short, plausible result set. So the guard is layered. A filter
15
+ * naming a field that is not marked filterable is a **compile-time type error** for a caller who marked the
16
+ * field with {@link filterable}; `vector/unfilterable_field` is the runtime guard behind it, for a caller
17
+ * arriving through an untyped boundary — an HTTP body, a job payload, JavaScript.
18
+ *
19
+ * `.meta({ filterable: true })` is the runtime marker: it drives provisioning and the drift check. The type
20
+ * system cannot see it — `.meta()` returns the same type it was called on — so {@link filterable} sets the
21
+ * same marker *and* brands the field's type. Mark fields with it and the builder types itself. Mark them with
22
+ * a bare `.meta({ filterable: true })` and everything still provisions and still filters; only the
23
+ * compile-time narrowing is lost.
24
+ */
25
+
26
+ /** A metadata value Vectorize can filter on. Nested objects and arrays are not filterable. */
27
+ export type FilterPrimitive = string | number | boolean;
28
+
29
+ /** The comparison operators Vectorize's metadata filtering accepts. */
30
+ export interface FilterComparison<V extends FilterPrimitive> {
31
+ /** Equal to. */
32
+ $eq?: V;
33
+ /** Not equal to. */
34
+ $ne?: V;
35
+ /** In this set. */
36
+ $in?: readonly V[];
37
+ /** Not in this set. */
38
+ $nin?: readonly V[];
39
+ /** Strictly less than. */
40
+ $lt?: V;
41
+ /** Less than or equal to. */
42
+ $lte?: V;
43
+ /** Strictly greater than. */
44
+ $gt?: V;
45
+ /** Greater than or equal to. */
46
+ $gte?: V;
47
+ }
48
+
49
+ const OPERATORS = ["$eq", "$ne", "$in", "$nin", "$lt", "$lte", "$gt", "$gte"] as const;
50
+ const SET_OPERATORS: readonly string[] = ["$in", "$nin"];
51
+
52
+ /** The brand {@link filterable} stamps on a field's type. Type-level only — it never exists at runtime. */
53
+ declare const filterableBrand: unique symbol;
54
+
55
+ /** A metadata field marked filterable: the same schema, carrying a type-level mark the builder can read. */
56
+ export type Filterable<T extends z.ZodType> = T & { readonly [filterableBrand]: true };
57
+
58
+ /**
59
+ * Mark a metadata field filterable. Sets `.meta({ filterable: true })` — the marker provisioning and drift
60
+ * both read — and brands the type so {@link vectorFilter} accepts this field and rejects the others.
61
+ */
62
+ export function filterable<T extends z.ZodType>(schema: T): Filterable<T> {
63
+ return schema.meta({ filterable: true }) as Filterable<T>;
64
+ }
65
+
66
+ /** The keys of a metadata schema that were marked with {@link filterable}. */
67
+ export type FilterableKeys<S extends z.ZodObject> = {
68
+ [K in keyof S["shape"]]: S["shape"][K] extends { readonly [filterableBrand]: true } ? K : never;
69
+ }[keyof S["shape"]];
70
+
71
+ /** A field's filterable value type, or `never` when it decodes to something Vectorize cannot compare. */
72
+ type FilterValueOf<S extends z.ZodObject, K extends keyof S["shape"]> =
73
+ NonNullable<z.output<S["shape"][K]>> extends infer V ? (V extends FilterPrimitive ? V : never) : never;
74
+
75
+ /**
76
+ * A filter over a metadata schema's filterable fields. A bare value is `$eq` — Vectorize's own shorthand —
77
+ * and an operator object is the long form.
78
+ */
79
+ export type VectorFilter<S extends z.ZodObject> = {
80
+ [K in FilterableKeys<S> & string]?: FilterValueOf<S, K> | FilterComparison<FilterValueOf<S, K>>;
81
+ };
82
+
83
+ /** The compiled filter: the plain object handed to Vectorize's `query`. */
84
+ export type CompiledFilter = Record<string, unknown>;
85
+
86
+ function isPrimitive(value: unknown): value is FilterPrimitive {
87
+ return typeof value === "string" || typeof value === "number" || typeof value === "boolean";
88
+ }
89
+
90
+ /** Whether a value matches the type the property was indexed as. A mismatch never matches anything. */
91
+ function matchesIndexType(value: unknown, indexType: MetadataIndexDescriptor["indexType"]): boolean {
92
+ if (value === null) return true;
93
+ return typeof value === indexType;
94
+ }
95
+
96
+ function assertValueType(key: string, operator: string, value: unknown, descriptor: MetadataIndexDescriptor): void {
97
+ if (SET_OPERATORS.includes(operator)) {
98
+ if (!Array.isArray(value) || value.length === 0) {
99
+ throw new ValidationError({
100
+ message: `\`${operator}\` needs a non-empty list of values.`,
101
+ action: "Pass an array of values to compare against.",
102
+ detail: `filter key '${key}' operator '${operator}' received ${JSON.stringify(value)}`,
103
+ });
104
+ }
105
+ for (const entry of value) {
106
+ if (!matchesIndexType(entry, descriptor.indexType)) {
107
+ throw new ValidationError({
108
+ message: `\`${key}\` is indexed as ${descriptor.indexType}; compare it against ${descriptor.indexType} values.`,
109
+ action: "Match the filter's value type to the field's type in the index's metadata schema.",
110
+ detail: `filter key '${key}' operator '${operator}' received ${JSON.stringify(entry)}`,
111
+ });
112
+ }
113
+ }
114
+ return;
115
+ }
116
+ if (!matchesIndexType(value, descriptor.indexType)) {
117
+ throw new ValidationError({
118
+ message: `\`${key}\` is indexed as ${descriptor.indexType}; compare it against ${descriptor.indexType} values.`,
119
+ action: "Match the filter's value type to the field's type in the index's metadata schema.",
120
+ detail: `filter key '${key}' operator '${operator}' received ${JSON.stringify(value)}`,
121
+ });
122
+ }
123
+ }
124
+
125
+ /**
126
+ * Validate an untyped filter against the metadata indexes an index actually has, and return the object to
127
+ * hand Vectorize. This is the runtime half of the guard: every key must be indexed, every operator must be one
128
+ * Vectorize knows, every value must match the property's index type, and the compact JSON must stay under
129
+ * 2,048 bytes.
130
+ */
131
+ export function compileFilter(descriptors: readonly MetadataIndexDescriptor[], filter: CompiledFilter): CompiledFilter {
132
+ const byName = new Map(descriptors.map((descriptor) => [descriptor.propertyName, descriptor]));
133
+ const compiled: CompiledFilter = {};
134
+
135
+ for (const [key, condition] of Object.entries(filter)) {
136
+ if (condition === undefined) continue;
137
+
138
+ const keyProblem = filterKeyProblem(key);
139
+ if (keyProblem) {
140
+ throw new ValidationError({
141
+ message: "That filter key is not usable.",
142
+ action: "Filter keys are non-empty, carry no dots, do not start with $, and stay under 512 characters.",
143
+ detail: `filter key ${keyProblem}`,
144
+ });
145
+ }
146
+
147
+ const descriptor = byName.get(key);
148
+ if (!descriptor) {
149
+ throw new VectorUnfilterableFieldError({
150
+ detail: `no metadata index for '${key}'; indexed: ${descriptors.map((d) => d.propertyName).join(", ") || "none"}`,
151
+ });
152
+ }
153
+
154
+ if (isPrimitive(condition) || condition === null) {
155
+ // Vectorize's shorthand: a bare value means $eq. Expanded here so one compiled shape reaches the wire.
156
+ assertValueType(key, "$eq", condition, descriptor);
157
+ compiled[key] = { $eq: condition };
158
+ continue;
159
+ }
160
+
161
+ if (typeof condition !== "object") {
162
+ throw new ValidationError({
163
+ message: `\`${key}\` needs a value or a comparison.`,
164
+ action: "Pass a value, or an object of operators such as { $in: [...] }.",
165
+ detail: `filter key '${key}' received ${typeof condition}`,
166
+ });
167
+ }
168
+
169
+ const operators: Record<string, unknown> = {};
170
+ for (const [operator, value] of Object.entries(condition as Record<string, unknown>)) {
171
+ if (value === undefined) continue;
172
+ if (!OPERATORS.includes(operator as (typeof OPERATORS)[number])) {
173
+ throw new ValidationError({
174
+ message: `\`${operator}\` is not a filter operator.`,
175
+ action: `Use one of ${OPERATORS.join(", ")}.`,
176
+ detail: `filter key '${key}' used operator '${operator}'`,
177
+ });
178
+ }
179
+ assertValueType(key, operator, value, descriptor);
180
+ operators[operator] = value;
181
+ }
182
+
183
+ if (Object.keys(operators).length === 0) {
184
+ throw new ValidationError({
185
+ message: `\`${key}\` has no comparison.`,
186
+ action: `Give it a value, or one of ${OPERATORS.join(", ")}.`,
187
+ detail: `filter key '${key}' compiled to an empty comparison`,
188
+ });
189
+ }
190
+ compiled[key] = operators;
191
+ }
192
+
193
+ assertFilterSize(compiled);
194
+ return compiled;
195
+ }
196
+
197
+ /**
198
+ * Reject a filter whose compact JSON reaches Vectorize's ceiling. Checked here, before the call, because
199
+ * Vectorize's own rejection arrives as an opaque error with no indication which filter was at fault.
200
+ */
201
+ export function assertFilterSize(filter: CompiledFilter): void {
202
+ const size = byteLength(JSON.stringify(filter));
203
+ if (size >= MAX_FILTER_BYTES) {
204
+ throw new VectorFilterTooLargeError({
205
+ detail: `filter JSON is ${size} bytes; the limit is under ${MAX_FILTER_BYTES}`,
206
+ });
207
+ }
208
+ }
209
+
210
+ /**
211
+ * Evaluate a compiled filter against one document's metadata, in memory.
212
+ *
213
+ * Vectorize evaluates filters inside the index; this evaluates the same filter against a D1 row. It exists
214
+ * for `pithy vector reprocess --filter`, which selects rows out of the document corpus rather than out of the
215
+ * index. Metadata lives in a JSON column, so no SQL predicate can answer it without hand-written extraction
216
+ * — and re-embedding one document costs orders of magnitude more than reading it, so scanning a page and
217
+ * filtering it here is not the expensive half.
218
+ *
219
+ * The semantics are Vectorize's: an absent property matches only `$ne`/`$nin`, and every named property must
220
+ * match (implicit AND). Pass a filter that {@link compileFilter} produced, so every condition is already in
221
+ * operator form.
222
+ */
223
+ export function matchesCompiledFilter(metadata: Record<string, unknown>, filter: CompiledFilter): boolean {
224
+ for (const [key, condition] of Object.entries(filter)) {
225
+ const value = metadata[key];
226
+ const operators = condition as Record<string, unknown>;
227
+ for (const [operator, operand] of Object.entries(operators)) {
228
+ if (!compare(value, operator, operand)) return false;
229
+ }
230
+ }
231
+ return true;
232
+ }
233
+
234
+ /** One operator against one value. An absent value matches only the negative operators, as Vectorize does. */
235
+ function compare(value: unknown, operator: string, operand: unknown): boolean {
236
+ switch (operator) {
237
+ case "$eq":
238
+ return value === operand;
239
+ case "$ne":
240
+ return value !== operand;
241
+ case "$in":
242
+ return Array.isArray(operand) && operand.includes(value);
243
+ case "$nin":
244
+ return Array.isArray(operand) && !operand.includes(value);
245
+ case "$lt":
246
+ return ordered(value, operand, (a, b) => a < b);
247
+ case "$lte":
248
+ return ordered(value, operand, (a, b) => a <= b);
249
+ case "$gt":
250
+ return ordered(value, operand, (a, b) => a > b);
251
+ case "$gte":
252
+ return ordered(value, operand, (a, b) => a >= b);
253
+ default:
254
+ return false;
255
+ }
256
+ }
257
+
258
+ /**
259
+ * Apply an ordered comparison, but only between two values of the same primitive type. An absent property,
260
+ * or a type mismatch, matches nothing — the same answer Vectorize gives.
261
+ */
262
+ function ordered(
263
+ value: unknown,
264
+ operand: unknown,
265
+ compareTo: (left: number | string, right: number | string) => boolean,
266
+ ): boolean {
267
+ if (typeof value !== typeof operand) return false;
268
+ if (typeof value !== "number" && typeof value !== "string") return false;
269
+ return compareTo(value, operand as number | string);
270
+ }
271
+
272
+ /**
273
+ * Build a filter against an index's metadata schema. The typed entry point: the compiler rejects a field that
274
+ * was not marked {@link filterable}, and the runtime checks below catch the same mistake arriving untyped.
275
+ */
276
+ export function vectorFilter<S extends z.ZodObject>(metadata: S, filter: VectorFilter<S>): CompiledFilter {
277
+ return compileFilter(introspectMetadata(metadata).indexes, filter as CompiledFilter);
278
+ }
@@ -0,0 +1,244 @@
1
+ // SPDX-FileCopyrightText: 2026 Pithy
2
+ // SPDX-License-Identifier: MIT
3
+
4
+ import { InternalError, ValidationError } from "@pithy-sh/core/src/error/pithyError";
5
+ import { z } from "zod";
6
+ import type { VectorIndexConfig } from "../config/config";
7
+ import { VectorDimensionMismatchError, VectorMetadataTooLargeError, VectorTopKExceededError } from "../error/errors";
8
+ import { assertFilterSize, type CompiledFilter } from "./filter";
9
+ import {
10
+ byteLength,
11
+ DEFAULT_TOPK,
12
+ MAX_METADATA_BYTES,
13
+ MAX_NAME_BYTES,
14
+ MAX_TOPK_WITH_PAYLOAD,
15
+ MAX_TOPK_WITHOUT_PAYLOAD,
16
+ MAX_UPSERT_BATCH,
17
+ MAX_VECTOR_ID_BYTES,
18
+ } from "./limits";
19
+
20
+ /**
21
+ * The Vectorize calls, over the `env.VECTORIZE` binding (bindings-first — CLAUDE.md §Cloudflare access).
22
+ *
23
+ * The binding is typed structurally as {@link VectorStore} rather than depending on the exact `VectorizeIndex`
24
+ * shape, so a test injects a fake and the code never reaches for a global. That is not a testing nicety here:
25
+ * Cloudflare ships **no local emulation for Vectorize**, so injection is the only way to unit-test any of this
26
+ * at all. The seam is always the first parameter.
27
+ *
28
+ * Every ceiling Vectorize publishes is checked before the call, not after. Vectorize's own rejections arrive
29
+ * as opaque errors that name neither the limit nor the offending vector, and the worst of them — a topK above
30
+ * the ceiling for the requested payload — is the kind of mistake that ships.
31
+ */
32
+
33
+ /** A vector on its way into the index: an id, its components, and the metadata a filter can match on. */
34
+ export interface VectorUpsert {
35
+ /** The vector's id — at most 64 bytes, and the join key back to the document row. */
36
+ id: string;
37
+ /** The embedding, exactly `dimensions` long. */
38
+ values: number[];
39
+ /** The metadata this vector carries. Under 10 KiB; long text belongs in the document table. */
40
+ metadata?: Record<string, unknown>;
41
+ /** The namespace this vector belongs to, when the index is partitioned. */
42
+ namespace?: string;
43
+ }
44
+
45
+ /** Options a similarity query accepts. */
46
+ export interface VectorQueryOptions {
47
+ /**
48
+ * How many matches to return. Falls back to {@link DEFAULT_TOPK} — a caller that wants the capability's
49
+ * configured `defaultTopK` passes it, because this seam takes one index's config, not the project's.
50
+ */
51
+ topK?: number;
52
+ /** Restrict the search to one namespace. */
53
+ namespace?: string;
54
+ /** A compiled metadata filter — build it with `vectorFilter`, never by hand. */
55
+ filter?: CompiledFilter;
56
+ /** Return each match's components. Costs payload, and lowers the topK ceiling to 50. */
57
+ returnValues?: boolean;
58
+ /** Return each match's metadata. Costs payload, and lowers the topK ceiling to 50. */
59
+ returnMetadata?: boolean;
60
+ }
61
+
62
+ /**
63
+ * The subset of the Vectorize binding this module uses. Responses are `unknown`: their shape is the runtime's,
64
+ * so it is validated here rather than trusted.
65
+ */
66
+ export interface VectorStore {
67
+ /** Insert or replace vectors. Asynchronous — the returned mutation id is an acknowledgement, not a write. */
68
+ upsert(vectors: VectorUpsert[]): Promise<unknown>;
69
+ /** Find the nearest neighbors of a vector. */
70
+ query(vector: number[], options?: Record<string, unknown>): Promise<unknown>;
71
+ /** Delete vectors by id. Optional here so a fake may omit it; guarded before use. */
72
+ deleteByIds?: (ids: string[]) => Promise<unknown>;
73
+ }
74
+
75
+ const MutationAck = z
76
+ .object({ mutationId: z.string().describe("Vectorize's id for the enqueued mutation.") })
77
+ .describe("The acknowledgement Vectorize returns for an enqueued write — the write itself lands later.");
78
+
79
+ const QueryMatch = z
80
+ .object({
81
+ id: z.string().describe("The matched vector's id."),
82
+ score: z.number().describe("The similarity score, as scored by the index's metric."),
83
+ values: z.array(z.number()).optional().describe("The matched vector's components, when values were requested."),
84
+ metadata: z
85
+ .record(z.string(), z.unknown())
86
+ .optional()
87
+ .describe("The matched vector's metadata, when metadata was requested."),
88
+ namespace: z.string().nullish().describe("The namespace the match belongs to, when the index is partitioned."),
89
+ })
90
+ .describe("One nearest-neighbor match Vectorize returned.");
91
+
92
+ const QueryResponse = z
93
+ .object({
94
+ count: z.number().describe("How many matches came back."),
95
+ matches: z.array(QueryMatch).describe("The matches, nearest first."),
96
+ })
97
+ .describe("The shape of a Vectorize query response.");
98
+
99
+ /** A query's result: the matches, nearest first. */
100
+ export type VectorQueryResult = z.infer<typeof QueryResponse>;
101
+
102
+ /** A namespace is a Vectorize identifier, so it carries the same 64-byte ceiling as an index name. */
103
+ function assertNamespace(namespace: string | undefined): void {
104
+ if (namespace === undefined) return;
105
+ if (namespace.length === 0 || byteLength(namespace) > MAX_NAME_BYTES) {
106
+ throw new ValidationError({
107
+ message: "That namespace is not usable.",
108
+ action: `A namespace is between 1 and ${MAX_NAME_BYTES} bytes.`,
109
+ detail: `namespace '${namespace}' is ${byteLength(namespace)} bytes`,
110
+ });
111
+ }
112
+ }
113
+
114
+ /** The vector must be exactly as long as the index was created for. Dimensions cannot be changed later. */
115
+ function assertDimensions(index: VectorIndexConfig, values: number[], context: string): void {
116
+ if (values.length !== index.dimensions) {
117
+ throw new VectorDimensionMismatchError({
118
+ detail: `${context}: got ${values.length} components, index expects ${index.dimensions} (model ${index.model})`,
119
+ });
120
+ }
121
+ }
122
+
123
+ /**
124
+ * Validate and upsert a batch. Returns Vectorize's mutation id — an acknowledgement, not a completed write:
125
+ * Vectorize applies writes asynchronously, so nothing here may be read back immediately.
126
+ */
127
+ export async function upsertVectors(
128
+ store: VectorStore,
129
+ index: VectorIndexConfig,
130
+ vectors: VectorUpsert[],
131
+ ): Promise<string> {
132
+ if (vectors.length === 0) {
133
+ throw new ValidationError({
134
+ message: "Nothing to write.",
135
+ action: "Pass at least one vector.",
136
+ detail: "upsertVectors received an empty batch",
137
+ });
138
+ }
139
+ if (vectors.length > MAX_UPSERT_BATCH) {
140
+ throw new ValidationError({
141
+ message: "Too many vectors in one write.",
142
+ action: `Write at most ${MAX_UPSERT_BATCH} vectors per call and repeat.`,
143
+ detail: `upsertVectors received ${vectors.length} vectors; the Workers binding accepts ${MAX_UPSERT_BATCH}`,
144
+ });
145
+ }
146
+
147
+ for (const vector of vectors) {
148
+ if (vector.id.length === 0 || byteLength(vector.id) > MAX_VECTOR_ID_BYTES) {
149
+ throw new ValidationError({
150
+ message: "That vector id is not usable.",
151
+ action: `A vector id is between 1 and ${MAX_VECTOR_ID_BYTES} bytes.`,
152
+ detail: `vector id '${vector.id.slice(0, 32)}…' is ${byteLength(vector.id)} bytes`,
153
+ });
154
+ }
155
+ assertDimensions(index, vector.values, `vector '${vector.id}'`);
156
+ if (vector.metadata) {
157
+ const size = byteLength(JSON.stringify(vector.metadata));
158
+ if (size >= MAX_METADATA_BYTES) {
159
+ throw new VectorMetadataTooLargeError({
160
+ detail: `vector '${vector.id}' carries ${size} bytes of metadata; the limit is ${MAX_METADATA_BYTES}`,
161
+ });
162
+ }
163
+ }
164
+ assertNamespace(vector.namespace);
165
+ }
166
+
167
+ const raw = await store.upsert(vectors);
168
+ const parsed = MutationAck.safeParse(raw);
169
+ if (!parsed.success) {
170
+ throw new InternalError({
171
+ message: "The vector store returned something unexpected.",
172
+ detail: "Vectorize upsert returned an unexpected shape",
173
+ });
174
+ }
175
+ return parsed.data.mutationId;
176
+ }
177
+
178
+ /**
179
+ * Validate and run a similarity query. `topK` is checked against the ceiling that applies to *this* query:
180
+ * Vectorize allows 100 matches when it returns neither values nor metadata, and 50 when it returns either.
181
+ */
182
+ export async function queryVectors(
183
+ store: VectorStore,
184
+ index: VectorIndexConfig,
185
+ vector: number[],
186
+ options: VectorQueryOptions = {},
187
+ ): Promise<VectorQueryResult> {
188
+ assertDimensions(index, vector, "query vector");
189
+ assertNamespace(options.namespace ?? index.namespace);
190
+
191
+ const returnsPayload = options.returnValues === true || options.returnMetadata === true;
192
+ const ceiling = returnsPayload ? MAX_TOPK_WITH_PAYLOAD : MAX_TOPK_WITHOUT_PAYLOAD;
193
+ const topK = options.topK ?? DEFAULT_TOPK;
194
+ if (topK < 1) {
195
+ throw new ValidationError({
196
+ message: "Ask for at least one match.",
197
+ action: "topK is a positive whole number.",
198
+ detail: `topK was ${topK}`,
199
+ });
200
+ }
201
+ if (topK > ceiling) {
202
+ throw new VectorTopKExceededError({
203
+ detail: `topK ${topK} exceeds ${ceiling} for a query that returns ${returnsPayload ? "values or metadata" : "neither values nor metadata"}`,
204
+ });
205
+ }
206
+ if (options.filter) assertFilterSize(options.filter);
207
+
208
+ const namespace = options.namespace ?? index.namespace;
209
+ const raw = await store.query(vector, {
210
+ topK,
211
+ returnValues: options.returnValues === true,
212
+ returnMetadata: options.returnMetadata === true,
213
+ ...(namespace ? { namespace } : {}),
214
+ ...(options.filter ? { filter: options.filter } : {}),
215
+ });
216
+
217
+ const parsed = QueryResponse.safeParse(raw);
218
+ if (!parsed.success) {
219
+ throw new InternalError({
220
+ message: "The vector store returned something unexpected.",
221
+ detail: "Vectorize query returned an unexpected shape",
222
+ });
223
+ }
224
+ return parsed.data;
225
+ }
226
+
227
+ /** Delete vectors by id. Guarded, because a fake — or an older binding — may not carry the method. */
228
+ export async function deleteVectors(store: VectorStore, ids: string[]): Promise<string> {
229
+ if (!store.deleteByIds) {
230
+ throw new InternalError({
231
+ message: "This vector index cannot delete.",
232
+ detail: "the Vectorize binding does not expose deleteByIds",
233
+ });
234
+ }
235
+ const raw = await store.deleteByIds(ids);
236
+ const parsed = MutationAck.safeParse(raw);
237
+ if (!parsed.success) {
238
+ throw new InternalError({
239
+ message: "The vector store returned something unexpected.",
240
+ detail: "Vectorize deleteByIds returned an unexpected shape",
241
+ });
242
+ }
243
+ return parsed.data.mutationId;
244
+ }