@volter/twin-upstashvector 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +233 -0
- package/dist/src/cli.d.ts +2 -0
- package/dist/src/cli.js +45 -0
- package/dist/src/index.d.ts +14 -0
- package/dist/src/index.js +106 -0
- package/dist/src/upstashvector-budget.d.ts +84 -0
- package/dist/src/upstashvector-budget.js +443 -0
- package/dist/src/upstashvector-capabilities.d.ts +4 -0
- package/dist/src/upstashvector-capabilities.js +1108 -0
- package/dist/src/upstashvector-conformance.d.ts +7 -0
- package/dist/src/upstashvector-conformance.js +163 -0
- package/dist/src/upstashvector-connector.d.ts +109 -0
- package/dist/src/upstashvector-connector.js +286 -0
- package/dist/src/upstashvector-filter.d.ts +80 -0
- package/dist/src/upstashvector-filter.js +564 -0
- package/dist/src/upstashvector-server.d.ts +32 -0
- package/dist/src/upstashvector-server.js +56 -0
- package/dist/src/upstashvector-store.d.ts +248 -0
- package/dist/src/upstashvector-store.js +883 -0
- package/dist/src/upstashvector-twin.d.ts +67 -0
- package/dist/src/upstashvector-twin.js +287 -0
- package/package.json +51 -0
- package/src/cli.ts +47 -0
- package/src/index.ts +193 -0
- package/src/upstashvector-budget.ts +489 -0
- package/src/upstashvector-capabilities.ts +1242 -0
- package/src/upstashvector-conformance.ts +175 -0
- package/src/upstashvector-connector.ts +328 -0
- package/src/upstashvector-filter.ts +525 -0
- package/src/upstashvector-server.ts +86 -0
- package/src/upstashvector-store.ts +944 -0
- package/src/upstashvector-twin.ts +347 -0
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
// upstashvector conformance (dev-only; lazy-imported by the CLI, NEVER from index.ts/runtime — E2).
|
|
2
|
+
//
|
|
3
|
+
// This is NOT the static self-referential snapshot check some packs ship. §9 refuted that pattern
|
|
4
|
+
// elsewhere in this repo on the grounds that comparing one constant to another cannot detect a dead
|
|
5
|
+
// handler — so this check ISSUES REAL REQUESTS (the azureformrecognizer `conformance.endpoint_probe`
|
|
6
|
+
// pattern, as adopted by the sibling upstash pack): it drives every endpoint the snapshot
|
|
7
|
+
// claims through `handleUpstashVectorTwinRequest` against a throwaway root and requires each to be
|
|
8
|
+
// genuinely routed and to answer with the right envelope AND the right values. A twin whose handler
|
|
9
|
+
// returned `{}` fails this, which is the whole point.
|
|
10
|
+
import { mkdtempSync, rmSync } from 'node:fs';
|
|
11
|
+
import { tmpdir } from 'node:os';
|
|
12
|
+
import { join } from 'node:path';
|
|
13
|
+
import { handleUpstashVectorTwinRequest, upstashvectorTwinSnapshot } from "./upstashvector-twin.js";
|
|
14
|
+
import { UPSTASHVECTOR_RESOURCE_TYPES } from "./upstashvector-store.js";
|
|
15
|
+
const AT = '2026-01-01T00:00:00.000Z';
|
|
16
|
+
export async function checkUpstashVectorConformance() {
|
|
17
|
+
const snapshot = upstashvectorTwinSnapshot();
|
|
18
|
+
const violations = [];
|
|
19
|
+
const root = mkdtempSync(join(tmpdir(), 'upstashvector-conf-'));
|
|
20
|
+
const auth = { authorization: 'Bearer twin-conformance-token' };
|
|
21
|
+
const result = (r) => r.body.result;
|
|
22
|
+
const probes = [
|
|
23
|
+
{
|
|
24
|
+
label: 'POST /upsert[/{namespace}] (one vector object or an array of them)',
|
|
25
|
+
method: 'POST', path: '/upsert', body: JSON.stringify([{ id: 'c-1', vector: [1, 0], metadata: { kind: 'a' } }, { id: 'c-2', vector: [0, 1] }]),
|
|
26
|
+
expect: (r) => r.status === 200 && result(r) === 'Success',
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
label: 'POST /query[/{namespace}] (an object = one query; an array = queryMany)',
|
|
30
|
+
method: 'POST', path: '/query', body: JSON.stringify({ vector: [1, 0], topK: 1, includeMetadata: true }),
|
|
31
|
+
expect: (r) => {
|
|
32
|
+
const rows = result(r);
|
|
33
|
+
// The nearest vector to [1,0] IS c-1, and under COSINE its score is exactly 1 — a value,
|
|
34
|
+
// not a status. A handler returning a constant or an empty array fails here.
|
|
35
|
+
return r.status === 200 && Array.isArray(rows) && rows.length === 1 && rows[0].id === 'c-1' && rows[0].score === 1
|
|
36
|
+
&& JSON.stringify(rows[0].metadata) === JSON.stringify({ kind: 'a' });
|
|
37
|
+
},
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
label: 'POST /fetch[/{namespace}] (by ids — positionally aligned, null for a miss — or by prefix)',
|
|
41
|
+
method: 'POST', path: '/fetch', body: JSON.stringify({ ids: ['c-2', 'c-missing'], includeVectors: true }),
|
|
42
|
+
expect: (r) => {
|
|
43
|
+
const rows = result(r);
|
|
44
|
+
return r.status === 200 && Array.isArray(rows) && rows.length === 2
|
|
45
|
+
&& rows[0]?.id === 'c-2' && JSON.stringify(rows[0]?.vector) === JSON.stringify([0, 1])
|
|
46
|
+
&& rows[1] === null;
|
|
47
|
+
},
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
label: 'POST /range[/{namespace}] (offset cursor, nextCursor "" when exhausted)',
|
|
51
|
+
method: 'POST', path: '/range', body: JSON.stringify({ cursor: '', limit: 1 }),
|
|
52
|
+
expect: (r) => {
|
|
53
|
+
const page = result(r);
|
|
54
|
+
return r.status === 200 && page.nextCursor === '1' && page.vectors?.length === 1 && page.vectors[0].id === 'c-1';
|
|
55
|
+
},
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
label: 'POST /update[/{namespace}] ({updated:0|1}, metadataUpdateMode OVERWRITE|PATCH)',
|
|
59
|
+
method: 'POST', path: '/update', body: JSON.stringify({ id: 'c-2', metadata: { kind: 'b' } }),
|
|
60
|
+
expect: (r) => JSON.stringify(result(r)) === JSON.stringify({ updated: 1 }),
|
|
61
|
+
},
|
|
62
|
+
{
|
|
63
|
+
label: 'GET|POST /info (counts + configuration + the per-namespace map)',
|
|
64
|
+
method: 'GET', path: '/info',
|
|
65
|
+
expect: (r) => {
|
|
66
|
+
const info = result(r);
|
|
67
|
+
return r.status === 200 && info.vectorCount === 2 && info.dimension === 2
|
|
68
|
+
&& info.similarityFunction === 'COSINE' && info.namespaces !== undefined && '' in info.namespaces;
|
|
69
|
+
},
|
|
70
|
+
},
|
|
71
|
+
{
|
|
72
|
+
label: 'POST /upsert[/{namespace}] into a namespace, then GET|POST /list-namespaces',
|
|
73
|
+
method: 'POST', path: '/upsert/team-a', body: JSON.stringify({ id: 'n-1', vector: [1, 1] }),
|
|
74
|
+
expect: (r) => r.status === 200 && result(r) === 'Success',
|
|
75
|
+
},
|
|
76
|
+
{
|
|
77
|
+
label: 'GET|POST /list-namespaces',
|
|
78
|
+
method: 'GET', path: '/list-namespaces',
|
|
79
|
+
expect: (r) => JSON.stringify(result(r)) === JSON.stringify(['', 'team-a']),
|
|
80
|
+
},
|
|
81
|
+
{
|
|
82
|
+
label: 'POST|DELETE /delete[/{namespace}] (by ids, prefix or metadata filter)',
|
|
83
|
+
method: 'POST', path: '/delete', body: JSON.stringify({ ids: ['c-1'] }),
|
|
84
|
+
expect: (r) => JSON.stringify(result(r)) === JSON.stringify({ deleted: 1 }),
|
|
85
|
+
},
|
|
86
|
+
{
|
|
87
|
+
label: 'POST|DELETE /delete-namespace/{namespace}',
|
|
88
|
+
method: 'DELETE', path: '/delete-namespace/team-a',
|
|
89
|
+
expect: (r) => r.status === 200 && result(r) === 'Success',
|
|
90
|
+
},
|
|
91
|
+
{
|
|
92
|
+
label: 'POST|DELETE /reset[/{namespace}] and /reset?all',
|
|
93
|
+
method: 'POST', path: '/reset?all',
|
|
94
|
+
expect: (r) => r.status === 200 && result(r) === 'Success',
|
|
95
|
+
},
|
|
96
|
+
{
|
|
97
|
+
label: 'OPTIONS (CORS preflight)',
|
|
98
|
+
method: 'OPTIONS', path: '/query',
|
|
99
|
+
expect: (r) => r.status === 200 && r.body === null,
|
|
100
|
+
},
|
|
101
|
+
];
|
|
102
|
+
try {
|
|
103
|
+
for (const probe of probes) {
|
|
104
|
+
// The namespace-upsert probe is a SETUP step for list-namespaces and deliberately shares that
|
|
105
|
+
// endpoint's inventory entry, so it is exempt from the label check below.
|
|
106
|
+
const setupProbe = probe.label.startsWith('POST /upsert[/{namespace}] into a namespace');
|
|
107
|
+
if (!setupProbe && !snapshot.implementedEndpoints.includes(probe.label)) {
|
|
108
|
+
violations.push(`probe '${probe.label}' has no matching entry in implementedEndpoints`);
|
|
109
|
+
continue;
|
|
110
|
+
}
|
|
111
|
+
const response = await handleUpstashVectorTwinRequest({
|
|
112
|
+
method: probe.method,
|
|
113
|
+
path: probe.path,
|
|
114
|
+
headers: auth,
|
|
115
|
+
root,
|
|
116
|
+
occurredAt: AT,
|
|
117
|
+
...(probe.body !== undefined ? { body: probe.body } : {}),
|
|
118
|
+
});
|
|
119
|
+
if (!probe.expect(response)) {
|
|
120
|
+
violations.push(`endpoint '${probe.label}' did not answer as declared (status ${response.status}, body ${JSON.stringify(response.body)})`);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
// Every claimed endpoint must have a probe — otherwise the inventory could grow entries nothing
|
|
124
|
+
// exercises, which is exactly the drift a self-referential snapshot cannot see.
|
|
125
|
+
for (const endpoint of snapshot.implementedEndpoints) {
|
|
126
|
+
if (!probes.some((p) => p.label === endpoint))
|
|
127
|
+
violations.push(`declared endpoint '${endpoint}' has no conformance probe`);
|
|
128
|
+
}
|
|
129
|
+
// The inventory must match what the handler WRITES, in BOTH directions. Asserting only one
|
|
130
|
+
// direction leaves the loop vacuous against the drift it claims to catch (the bug §9 round 2
|
|
131
|
+
// found in the sibling upstash pack's first version of this check).
|
|
132
|
+
const written = ['vector', 'namespace', 'config'];
|
|
133
|
+
for (const type of UPSTASHVECTOR_RESOURCE_TYPES) {
|
|
134
|
+
if (!written.includes(type))
|
|
135
|
+
violations.push(`declared resource type '${type}' is not written by any handler path`);
|
|
136
|
+
}
|
|
137
|
+
for (const type of written) {
|
|
138
|
+
if (!UPSTASHVECTOR_RESOURCE_TYPES.includes(type)) {
|
|
139
|
+
violations.push(`the handler writes subject type '${type}' but UPSTASHVECTOR_RESOURCE_TYPES does not declare it`);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
// …and the reset above must have genuinely emptied the index, so the probes' writes are proven
|
|
143
|
+
// to have reached real state rather than merely having returned 200s.
|
|
144
|
+
const after = await handleUpstashVectorTwinRequest({ method: 'GET', path: '/info', headers: auth, root, occurredAt: AT });
|
|
145
|
+
const info = after.body.result;
|
|
146
|
+
if (info?.vectorCount !== 0) {
|
|
147
|
+
violations.push(`/reset?all ran but the index still reports ${String(info?.vectorCount)} vectors`);
|
|
148
|
+
}
|
|
149
|
+
// The 'config' subject must SURVIVE a reset — the index keeps its dimension when emptied.
|
|
150
|
+
if (info?.dimension !== 2) {
|
|
151
|
+
violations.push(`the 'config' subject is declared but the index's dimension did not persist across reset (dimension=${String(info?.dimension)})`);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
finally {
|
|
155
|
+
rmSync(root, { recursive: true, force: true });
|
|
156
|
+
}
|
|
157
|
+
return {
|
|
158
|
+
ok: violations.length === 0,
|
|
159
|
+
endpointsChecked: snapshot.implementedEndpoints.length,
|
|
160
|
+
resourceTypesChecked: snapshot.resourceTypes.length,
|
|
161
|
+
violations,
|
|
162
|
+
};
|
|
163
|
+
}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import type { PerformContext, PushOutcome, RemoteExecute, SyncResource, TwinAction } from '@volter/world-core';
|
|
2
|
+
import { type UpstashVectorBudgetedOptions } from './upstashvector-budget.js';
|
|
3
|
+
export type { UpstashVectorBudgetedOptions };
|
|
4
|
+
/**
|
|
5
|
+
* The subset of a real Upstash Vector client this connector calls. Every member is OPTIONAL: a
|
|
6
|
+
* client that cannot do one of these makes the corresponding pull observe nothing, which is the
|
|
7
|
+
* honest outcome — inventing a value would turn "cannot see" into "saw an empty index", and a
|
|
8
|
+
* subsequent push would then delete real data.
|
|
9
|
+
*/
|
|
10
|
+
export interface UpstashVectorLikeClient {
|
|
11
|
+
info?: () => Promise<unknown>;
|
|
12
|
+
listNamespaces?: () => Promise<string[]>;
|
|
13
|
+
range?: (args: {
|
|
14
|
+
cursor: string;
|
|
15
|
+
limit: number;
|
|
16
|
+
includeMetadata?: boolean;
|
|
17
|
+
includeVectors?: boolean;
|
|
18
|
+
includeData?: boolean;
|
|
19
|
+
}, opts?: {
|
|
20
|
+
namespace?: string;
|
|
21
|
+
}) => Promise<unknown>;
|
|
22
|
+
}
|
|
23
|
+
/** The twin-side shape of one pulled vector, before it becomes a kernel resource. */
|
|
24
|
+
export type UpstashVectorRealVector = {
|
|
25
|
+
id: string;
|
|
26
|
+
namespace?: string;
|
|
27
|
+
vector: number[];
|
|
28
|
+
metadata?: Record<string, unknown> | null;
|
|
29
|
+
data?: string | null;
|
|
30
|
+
};
|
|
31
|
+
/**
|
|
32
|
+
* Pure mapper (a real vector → a kernel `SyncResource`). Never touches a client, so the mutation
|
|
33
|
+
* harness's connector-seam sweep (which sabotages exports matching sync/push/pull/fullSync) leaves
|
|
34
|
+
* it real — the pack convention, mirroring qstash's `mapMessage` and upstash's `mapKey`.
|
|
35
|
+
*
|
|
36
|
+
* The field names deliberately avoid the kernel's reserved META keys (`type`/`id`/`updatedAt`,
|
|
37
|
+
* which `projectResources` SILENTLY DROPS): the vector's own id is stored as `vid` and its
|
|
38
|
+
* namespace as `ns`. This is the same layout `upstashvector-store.ts` writes, so a pulled vector
|
|
39
|
+
* and a locally upserted one are indistinguishable to every read path.
|
|
40
|
+
*/
|
|
41
|
+
export declare function mapVector(real: UpstashVectorRealVector): SyncResource;
|
|
42
|
+
/**
|
|
43
|
+
* A namespace observed on the vendor, so `list-namespaces` reflects a pull.
|
|
44
|
+
*
|
|
45
|
+
* Uses the store's `namespaceSubjectId` rather than re-implementing it. §9 round 2 found this
|
|
46
|
+
* function had drifted into its own bare `encodeURIComponent` call, which meant (a) two
|
|
47
|
+
* implementations of one subject key, a latent collision risk, and (b) the surrogate guard added
|
|
48
|
+
* to `encodeKeyPart` did not cover it — so `mapNamespace('\ud800')` still threw a raw `URIError`
|
|
49
|
+
* off the pull's hot path, after `listNamespaces()` had already been paid for. One implementation,
|
|
50
|
+
* one guard.
|
|
51
|
+
*/
|
|
52
|
+
export declare function mapNamespace(name: string): SyncResource;
|
|
53
|
+
/**
|
|
54
|
+
* Walk ONE namespace with `/range`, bounded by `limit`.
|
|
55
|
+
*
|
|
56
|
+
* The bound is the point: an unbounded walk of a production index is precisely the spend this
|
|
57
|
+
* connector's budget exists to stop, and an entrypoint that offered one would be inviting the
|
|
58
|
+
* caller to burn a whole allowance in a loop.
|
|
59
|
+
*/
|
|
60
|
+
export declare function pullUpstashVectorRange(rawClient: UpstashVectorLikeClient, opts?: {
|
|
61
|
+
namespace?: string;
|
|
62
|
+
limit?: number;
|
|
63
|
+
pageSize?: number;
|
|
64
|
+
} & UpstashVectorBudgetedOptions): Promise<UpstashVectorRealVector[]>;
|
|
65
|
+
/**
|
|
66
|
+
* Discover the namespaces that exist on the vendor. A client without `listNamespaces` observes
|
|
67
|
+
* only the default namespace, which is the honest floor rather than an invented list.
|
|
68
|
+
*/
|
|
69
|
+
export declare function pullUpstashVectorNamespaces(rawClient: UpstashVectorLikeClient, opts?: UpstashVectorBudgetedOptions): Promise<string[]>;
|
|
70
|
+
/**
|
|
71
|
+
* D7 entry point: pull the current real state of an index and fold it into the twin via ONE
|
|
72
|
+
* one observation. Returns `{observed, deltasAppended}` — a re-pull of identical
|
|
73
|
+
* state appends ZERO deltas, which is what makes the pull idempotent.
|
|
74
|
+
*/
|
|
75
|
+
export declare function syncUpstashVectorFromReal(rawClient: UpstashVectorLikeClient, opts?: {
|
|
76
|
+
root?: string;
|
|
77
|
+
occurredAt?: string;
|
|
78
|
+
namespaces?: string[];
|
|
79
|
+
limit?: number;
|
|
80
|
+
pageSize?: number;
|
|
81
|
+
} & UpstashVectorBudgetedOptions): Promise<{
|
|
82
|
+
observed: number;
|
|
83
|
+
deltasAppended: number;
|
|
84
|
+
}>;
|
|
85
|
+
/**
|
|
86
|
+
* An `UpstashVectorLikeClient` over the kernel's executor. At a REAL boundary the kernel sets the
|
|
87
|
+
* sealed credential over these headers (executor.ts); at the twin's own wire any credential is one.
|
|
88
|
+
*
|
|
89
|
+
* EVERY call is a POST, whatever the docs spell: `@upstash/vector`'s `HttpClient.request` hardcodes
|
|
90
|
+
* it, so a client that sent the documented method would be talking to a surface the real SDK never
|
|
91
|
+
* uses. Namespaces are a PATH SUFFIX, not a body field.
|
|
92
|
+
*/
|
|
93
|
+
export declare function upstashVectorClientOver(execute: RemoteExecute): UpstashVectorLikeClient;
|
|
94
|
+
/** The refresh adapter: Upstash Vector enumerates (`/list-namespaces`, then `/range` per
|
|
95
|
+
* namespace), so the whole index comes back without the world having to say what it holds. */
|
|
96
|
+
export declare function syncUpstashVectorFromRemote(execute: RemoteExecute, opts?: {
|
|
97
|
+
root?: string;
|
|
98
|
+
origin?: string;
|
|
99
|
+
occurredAt?: string;
|
|
100
|
+
} & UpstashVectorBudgetedOptions): Promise<{
|
|
101
|
+
observed: number;
|
|
102
|
+
deltasAppended: number;
|
|
103
|
+
}>;
|
|
104
|
+
/**
|
|
105
|
+
* The perform adapter: an upsert and a delete cross; a namespace reset does too. Everything else a
|
|
106
|
+
* world records about its index — a query it ran, a count it took — is a READ at the vendor and has
|
|
107
|
+
* nothing to write.
|
|
108
|
+
*/
|
|
109
|
+
export declare function performUpstashVectorAction(execute: RemoteExecute, action: TwinAction, _ctx: PerformContext): Promise<PushOutcome>;
|
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
// upstashvector CONNECTOR — the live-vendor pull path that gives this twin the "git for SaaS"
|
|
2
|
+
// lifecycle over an INJECTED client (the auth boundary). The pack imports NO SDK and holds NO
|
|
3
|
+
// token; a consumer injects something structurally satisfying `UpstashVectorLikeClient` (a real
|
|
4
|
+
// `@upstash/vector` `Index` is assignable as-is, since its `info`/`range`/`listNamespaces` methods
|
|
5
|
+
// have exactly these shapes — asserted at the type level in the SDK integration test).
|
|
6
|
+
//
|
|
7
|
+
// ── PULL WALKS `/range`, AND THE WALK IS BOUNDED ──────────────────────────────────────────────
|
|
8
|
+
// Upstash Vector has no "give me everything" endpoint; `/range` is the supported pagination and it
|
|
9
|
+
// is unbounded in the size of the caller's index. A vector index is routinely millions of rows, so
|
|
10
|
+
// an unbounded walk is exactly the shape that turns one careless call into a very large bill —
|
|
11
|
+
// Upstash Vector bills PER REQUEST ($0.4/100K on Pay-As-You-Go) and the free tier caps queries per
|
|
12
|
+
// DAY. So every entrypoint takes a hard `limit`, loops a bounded number of pages, and GUARDS the
|
|
13
|
+
// injected client before touching it (`guardUpstashVectorClient`, which is idempotent). There is
|
|
14
|
+
// deliberately no option that turns the guard off.
|
|
15
|
+
//
|
|
16
|
+
// ── PUSH IS A FILED GAP, NOT AN OVERSIGHT ─────────────────────────────────────────────────────
|
|
17
|
+
// See `upstashvector.connector.push` in the manifest: writing local vectors back into a real index
|
|
18
|
+
// is destructive (an upsert REPLACES) and the safe design — an explicit id allowlist plus a
|
|
19
|
+
// dry-run diff — has not been settled.
|
|
20
|
+
import { observeResources } from '@volter/world-core';
|
|
21
|
+
import { DEFAULT_NAMESPACE, VectorApiError, namespaceSubjectId, vectorSubjectId } from "./upstashvector-store.js";
|
|
22
|
+
import { upstashvectorBudgetOf, guardUpstashVectorClient } from "./upstashvector-budget.js";
|
|
23
|
+
const SERVICE = 'upstashvector';
|
|
24
|
+
/**
|
|
25
|
+
* Pure mapper (a real vector → a kernel `SyncResource`). Never touches a client, so the mutation
|
|
26
|
+
* harness's connector-seam sweep (which sabotages exports matching sync/push/pull/fullSync) leaves
|
|
27
|
+
* it real — the pack convention, mirroring qstash's `mapMessage` and upstash's `mapKey`.
|
|
28
|
+
*
|
|
29
|
+
* The field names deliberately avoid the kernel's reserved META keys (`type`/`id`/`updatedAt`,
|
|
30
|
+
* which `projectResources` SILENTLY DROPS): the vector's own id is stored as `vid` and its
|
|
31
|
+
* namespace as `ns`. This is the same layout `upstashvector-store.ts` writes, so a pulled vector
|
|
32
|
+
* and a locally upserted one are indistinguishable to every read path.
|
|
33
|
+
*/
|
|
34
|
+
export function mapVector(real) {
|
|
35
|
+
const ns = real.namespace ?? DEFAULT_NAMESPACE;
|
|
36
|
+
// Validate the COORDINATES here, at the ingest boundary, with the same rule the HTTP write path
|
|
37
|
+
// applies. §9 round 2 found the hole: a vendor row carrying a non-numeric or overflow-scale
|
|
38
|
+
// coordinate flowed straight into the kernel, and a `NaN` serialised to JSON as `null` came back
|
|
39
|
+
// as 0 — a silently substituted coordinate — while an overflow-scale one poisoned every score it
|
|
40
|
+
// touched into `"score": null` on the wire, ranking FIRST under DOT_PRODUCT. Refusing loudly, by
|
|
41
|
+
// id, is the only answer that neither corrupts data nor hides it.
|
|
42
|
+
assertStorableVector(real.vector, real.id);
|
|
43
|
+
return {
|
|
44
|
+
type: 'vector',
|
|
45
|
+
id: vectorSubjectId(ns, real.id),
|
|
46
|
+
// Gotcha (kernel field MERGE, never a deep merge or an omission-clear): every field a later
|
|
47
|
+
// local write could overwrite is written explicitly here, so a pulled vector can never inherit
|
|
48
|
+
// stale metadata or a dead `gone` flag from a previous incarnation of the same subject.
|
|
49
|
+
fields: {
|
|
50
|
+
ns,
|
|
51
|
+
vid: real.id,
|
|
52
|
+
values: real.vector,
|
|
53
|
+
metadata: real.metadata ?? null,
|
|
54
|
+
data: real.data ?? '',
|
|
55
|
+
gone: false,
|
|
56
|
+
},
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* A namespace observed on the vendor, so `list-namespaces` reflects a pull.
|
|
61
|
+
*
|
|
62
|
+
* Uses the store's `namespaceSubjectId` rather than re-implementing it. §9 round 2 found this
|
|
63
|
+
* function had drifted into its own bare `encodeURIComponent` call, which meant (a) two
|
|
64
|
+
* implementations of one subject key, a latent collision risk, and (b) the surrogate guard added
|
|
65
|
+
* to `encodeKeyPart` did not cover it — so `mapNamespace('\ud800')` still threw a raw `URIError`
|
|
66
|
+
* off the pull's hot path, after `listNamespaces()` had already been paid for. One implementation,
|
|
67
|
+
* one guard.
|
|
68
|
+
*/
|
|
69
|
+
export function mapNamespace(name) {
|
|
70
|
+
return { type: 'namespace', id: namespaceSubjectId(name), fields: { name, gone: false } };
|
|
71
|
+
}
|
|
72
|
+
/** The float32 width the index stores — see `upstashvector-store.ts`'s `readVector`. */
|
|
73
|
+
const FLOAT32_MAX = 3.4028234663852886e38;
|
|
74
|
+
/** Refuse a vendor vector this twin cannot store faithfully, naming the id. */
|
|
75
|
+
function assertStorableVector(vector, id) {
|
|
76
|
+
if (!Array.isArray(vector) || vector.length === 0) {
|
|
77
|
+
throw new VectorApiError(`upstashvector connector: vector '${id}' arrived with no coordinates`, 422);
|
|
78
|
+
}
|
|
79
|
+
for (const n of vector) {
|
|
80
|
+
if (typeof n !== 'number' || !Number.isFinite(n) || Math.abs(n) > FLOAT32_MAX) {
|
|
81
|
+
throw new VectorApiError(`upstashvector connector: vector '${id}' has a coordinate this index cannot store (${String(n)}) — `
|
|
82
|
+
+ 'refusing rather than coercing it, which would substitute a different vector for the real one', 422);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
/** One page of the vendor's `/range` response, defensively narrowed. */
|
|
87
|
+
function readRangePage(raw) {
|
|
88
|
+
const o = (raw ?? {});
|
|
89
|
+
const vectors = [];
|
|
90
|
+
for (const item of Array.isArray(o.vectors) ? o.vectors : []) {
|
|
91
|
+
const v = (item ?? {});
|
|
92
|
+
if (typeof v.id !== 'string' && typeof v.id !== 'number')
|
|
93
|
+
continue;
|
|
94
|
+
// A row with no coordinates is SKIPPED rather than stored as an empty vector: a zero-length
|
|
95
|
+
// vector scores against nothing, so persisting one would put a permanently unrankable row in
|
|
96
|
+
// the twin that looks like real data. (`includeVectors` is always requested below, so a
|
|
97
|
+
// missing `vector` here means the vendor genuinely did not return one.)
|
|
98
|
+
if (!Array.isArray(v.vector) || v.vector.length === 0)
|
|
99
|
+
continue;
|
|
100
|
+
vectors.push({
|
|
101
|
+
id: String(v.id),
|
|
102
|
+
// NOT `.map(Number)`: coercing here is what turned a non-numeric vendor coordinate into a
|
|
103
|
+
// silent 0 (via NaN -> JSON null -> Number(null)). The raw values are carried through and
|
|
104
|
+
// `mapVector` refuses anything unstorable, by id (§9 round 2).
|
|
105
|
+
vector: v.vector,
|
|
106
|
+
metadata: v.metadata !== null && typeof v.metadata === 'object' ? v.metadata : null,
|
|
107
|
+
data: typeof v.data === 'string' ? v.data : null,
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
return { nextCursor: typeof o.nextCursor === 'string' ? o.nextCursor : '', vectors };
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Walk ONE namespace with `/range`, bounded by `limit`.
|
|
114
|
+
*
|
|
115
|
+
* The bound is the point: an unbounded walk of a production index is precisely the spend this
|
|
116
|
+
* connector's budget exists to stop, and an entrypoint that offered one would be inviting the
|
|
117
|
+
* caller to burn a whole allowance in a loop.
|
|
118
|
+
*/
|
|
119
|
+
export async function pullUpstashVectorRange(rawClient, opts = {}) {
|
|
120
|
+
const client = guardUpstashVectorClient(rawClient, upstashvectorBudgetOf(opts));
|
|
121
|
+
if (!client.range)
|
|
122
|
+
return [];
|
|
123
|
+
const namespace = opts.namespace ?? DEFAULT_NAMESPACE;
|
|
124
|
+
const limit = opts.limit ?? 100;
|
|
125
|
+
const pageSize = Math.max(1, Math.min(opts.pageSize ?? 50, limit));
|
|
126
|
+
const found = [];
|
|
127
|
+
let cursor = '';
|
|
128
|
+
// A bounded loop, not a `while (cursor !== '')`: a vendor that never returns an empty cursor must
|
|
129
|
+
// not be able to spin this forever, and the budget must not be the only thing that stops it.
|
|
130
|
+
for (let page = 0; page < 100 && found.length < limit; page++) {
|
|
131
|
+
const raw = await client.range({ cursor, limit: pageSize, includeMetadata: true, includeVectors: true, includeData: true }, { namespace });
|
|
132
|
+
const { nextCursor, vectors } = readRangePage(raw);
|
|
133
|
+
for (const v of vectors)
|
|
134
|
+
found.push({ ...v, namespace });
|
|
135
|
+
if (nextCursor === '' || vectors.length === 0)
|
|
136
|
+
break;
|
|
137
|
+
cursor = nextCursor;
|
|
138
|
+
}
|
|
139
|
+
return found.slice(0, limit);
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Discover the namespaces that exist on the vendor. A client without `listNamespaces` observes
|
|
143
|
+
* only the default namespace, which is the honest floor rather than an invented list.
|
|
144
|
+
*/
|
|
145
|
+
export async function pullUpstashVectorNamespaces(rawClient, opts = {}) {
|
|
146
|
+
const client = guardUpstashVectorClient(rawClient, upstashvectorBudgetOf(opts));
|
|
147
|
+
if (!client.listNamespaces)
|
|
148
|
+
return [DEFAULT_NAMESPACE];
|
|
149
|
+
const names = await client.listNamespaces();
|
|
150
|
+
const out = Array.isArray(names) ? names.filter((n) => typeof n === 'string') : [];
|
|
151
|
+
return out.includes(DEFAULT_NAMESPACE) ? out : [DEFAULT_NAMESPACE, ...out];
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* D7 entry point: pull the current real state of an index and fold it into the twin via ONE
|
|
155
|
+
* one observation. Returns `{observed, deltasAppended}` — a re-pull of identical
|
|
156
|
+
* state appends ZERO deltas, which is what makes the pull idempotent.
|
|
157
|
+
*/
|
|
158
|
+
export async function syncUpstashVectorFromReal(rawClient, opts = {}) {
|
|
159
|
+
// Guard ONCE here and hand the guarded client down: the per-namespace loop is unbounded in
|
|
160
|
+
// namespace count, so this is the entrypoint that must be unable to run unbudgeted.
|
|
161
|
+
const client = guardUpstashVectorClient(rawClient, upstashvectorBudgetOf(opts));
|
|
162
|
+
const occurredAt = opts.occurredAt ?? new Date().toISOString();
|
|
163
|
+
const namespaces = opts.namespaces ?? await pullUpstashVectorNamespaces(client);
|
|
164
|
+
const resources = [];
|
|
165
|
+
for (const ns of namespaces) {
|
|
166
|
+
// The DEFAULT namespace ('') is not state — it always exists on every index and the twin's own
|
|
167
|
+
// store synthesizes it in `listNamespaces` rather than holding a subject for it. Minting one
|
|
168
|
+
// from the pull gave an observed subject the write handler could never produce, which is a
|
|
169
|
+
// shape a branch would behave differently on. (Shape parity, 2026-09-08.)
|
|
170
|
+
if (ns !== '')
|
|
171
|
+
resources.push(mapNamespace(ns));
|
|
172
|
+
const budgetOpts = {
|
|
173
|
+
...(opts.limit !== undefined ? { limit: opts.limit } : {}),
|
|
174
|
+
...(opts.pageSize !== undefined ? { pageSize: opts.pageSize } : {}),
|
|
175
|
+
};
|
|
176
|
+
for (const v of await pullUpstashVectorRange(client, { namespace: ns, ...budgetOpts })) {
|
|
177
|
+
resources.push(mapVector(v));
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
const result = ((__at) => observeResources(SERVICE, resources, { ...(opts.root !== undefined ? { root: opts.root } : {}), at: __at, batch: `obs:${SERVICE}:${__at}` }))(occurredAt);
|
|
181
|
+
return { observed: resources.length, deltasAppended: result.appended };
|
|
182
|
+
}
|
|
183
|
+
// ── PROTOCOL 2: the pack's half of the real state system ────────────────────────────────────
|
|
184
|
+
/**
|
|
185
|
+
* An `UpstashVectorLikeClient` over the kernel's executor. At a REAL boundary the kernel sets the
|
|
186
|
+
* sealed credential over these headers (executor.ts); at the twin's own wire any credential is one.
|
|
187
|
+
*
|
|
188
|
+
* EVERY call is a POST, whatever the docs spell: `@upstash/vector`'s `HttpClient.request` hardcodes
|
|
189
|
+
* it, so a client that sent the documented method would be talking to a surface the real SDK never
|
|
190
|
+
* uses. Namespaces are a PATH SUFFIX, not a body field.
|
|
191
|
+
*/
|
|
192
|
+
export function upstashVectorClientOver(execute) {
|
|
193
|
+
const post = async (path, body) => {
|
|
194
|
+
const res = await execute({
|
|
195
|
+
method: 'POST', path,
|
|
196
|
+
headers: { accept: 'application/json', 'content-type': 'application/json', authorization: 'Bearer twin' },
|
|
197
|
+
...(body === undefined ? {} : { body: JSON.stringify(body) }),
|
|
198
|
+
});
|
|
199
|
+
let parsed = {};
|
|
200
|
+
try {
|
|
201
|
+
parsed = JSON.parse(res.body || '{}');
|
|
202
|
+
}
|
|
203
|
+
catch {
|
|
204
|
+
parsed = {};
|
|
205
|
+
}
|
|
206
|
+
if (res.status < 200 || res.status >= 300) {
|
|
207
|
+
throw new Error(`upstash vector POST ${path} refused: HTTP ${res.status} ${JSON.stringify(parsed).slice(0, 200)}`);
|
|
208
|
+
}
|
|
209
|
+
// UNWRAP `result`, because the real SDK does. Every Upstash Vector reply is `{result: …}` and
|
|
210
|
+
// `@upstash/vector`'s HttpClient hands the caller the INNER value; a client that returned the
|
|
211
|
+
// envelope would be a shape this connector's parsers have never seen, and they would quietly
|
|
212
|
+
// find no vectors rather than fail. (Shape parity caught exactly that, 2026-09-08.)
|
|
213
|
+
return parsed?.result !== undefined ? parsed.result : parsed;
|
|
214
|
+
};
|
|
215
|
+
return {
|
|
216
|
+
info: () => post('/info'),
|
|
217
|
+
listNamespaces: async () => {
|
|
218
|
+
const answered = await post('/list-namespaces');
|
|
219
|
+
return (Array.isArray(answered) ? answered : []).map((n) => String(n));
|
|
220
|
+
},
|
|
221
|
+
range: (args, opts) => post(opts?.namespace ? `/range/${encodeURIComponent(opts.namespace)}` : '/range', args),
|
|
222
|
+
};
|
|
223
|
+
}
|
|
224
|
+
/** The refresh adapter: Upstash Vector enumerates (`/list-namespaces`, then `/range` per
|
|
225
|
+
* namespace), so the whole index comes back without the world having to say what it holds. */
|
|
226
|
+
export async function syncUpstashVectorFromRemote(execute, opts = {}) {
|
|
227
|
+
// Guarded here as well as inside `syncUpstashVectorFromReal`: the guard is idempotent (an
|
|
228
|
+
// already-guarded client comes back unchanged), and stating it at every entrypoint is what the
|
|
229
|
+
// source tooth in upstashvector-budget.test.ts holds this connector to.
|
|
230
|
+
const client = guardUpstashVectorClient(upstashVectorClientOver(execute), upstashvectorBudgetOf(opts));
|
|
231
|
+
return syncUpstashVectorFromReal(client, {
|
|
232
|
+
...(opts.root !== undefined ? { root: opts.root } : {}),
|
|
233
|
+
occurredAt: opts.occurredAt ?? new Date().toISOString(),
|
|
234
|
+
});
|
|
235
|
+
}
|
|
236
|
+
/**
|
|
237
|
+
* The perform adapter: an upsert and a delete cross; a namespace reset does too. Everything else a
|
|
238
|
+
* world records about its index — a query it ran, a count it took — is a READ at the vendor and has
|
|
239
|
+
* nothing to write.
|
|
240
|
+
*/
|
|
241
|
+
export async function performUpstashVectorAction(execute, action, _ctx) {
|
|
242
|
+
const op = action.operation ?? `${action.subject.type}.update`;
|
|
243
|
+
const fields = (action.fields ?? {});
|
|
244
|
+
// `ns`, not `namespace` — the store's own name for it, and the default namespace is the empty
|
|
245
|
+
// string, which is a PATH SUFFIX of nothing rather than a segment.
|
|
246
|
+
const nsName = typeof fields.ns === 'string' ? fields.ns : typeof fields.namespace === 'string' ? fields.namespace : '';
|
|
247
|
+
const ns = nsName === '' ? '' : `/${encodeURIComponent(nsName)}`;
|
|
248
|
+
const post = async (path, body) => {
|
|
249
|
+
const res = await execute({
|
|
250
|
+
method: 'POST', path,
|
|
251
|
+
headers: { accept: 'application/json', 'content-type': 'application/json', authorization: 'Bearer twin' },
|
|
252
|
+
body: JSON.stringify(body),
|
|
253
|
+
});
|
|
254
|
+
if (res.status < 200 || res.status >= 300)
|
|
255
|
+
throw new Error(`upstash vector POST ${path} refused: HTTP ${res.status} ${res.body.slice(0, 200)}`);
|
|
256
|
+
try {
|
|
257
|
+
return JSON.parse(res.body || '{}');
|
|
258
|
+
}
|
|
259
|
+
catch {
|
|
260
|
+
return {};
|
|
261
|
+
}
|
|
262
|
+
};
|
|
263
|
+
// READ THE FIELD NAMES THE STORE ACTUALLY USES. The subject is `vec:<namespace>:<id>` and its
|
|
264
|
+
// fields are `{ ns, vid, values, data, metadata }` — not `{ id, vector }`. The first version of
|
|
265
|
+
// this adapter guessed the latter and therefore sent NO vector, an id still carrying its `vec::`
|
|
266
|
+
// prefix, and `data: ''` — which asks the vendor to EMBED an empty string and is refused by an
|
|
267
|
+
// index with no embedding model. Nothing in the gates exercises a perform, so it passed them all.
|
|
268
|
+
const vid = typeof fields.vid === 'string' && fields.vid ? fields.vid : action.subject.id.replace(/^vec:[^:]*:/, '');
|
|
269
|
+
const values = Array.isArray(fields.values) ? fields.values : Array.isArray(fields.vector) ? fields.vector : null;
|
|
270
|
+
const data = typeof fields.data === 'string' && fields.data !== '' ? fields.data : null;
|
|
271
|
+
if (op === 'vector.upsert' || op === 'vector.update') {
|
|
272
|
+
const answered = await post(`/upsert${ns}`, {
|
|
273
|
+
id: vid,
|
|
274
|
+
// A vector OR data — never both, and never an empty `data`, which is not "no data" to this
|
|
275
|
+
// vendor but "embed the empty string".
|
|
276
|
+
...(values === null ? {} : { vector: values }),
|
|
277
|
+
...(values === null && data !== null ? { data } : {}),
|
|
278
|
+
...(fields.metadata !== undefined && fields.metadata !== null ? { metadata: fields.metadata } : {}),
|
|
279
|
+
});
|
|
280
|
+
return { externalId: vid, data: answered };
|
|
281
|
+
}
|
|
282
|
+
if (op === 'vector.delete') {
|
|
283
|
+
return { externalId: vid, data: await post(`/delete${ns}`, { ids: [vid] }) };
|
|
284
|
+
}
|
|
285
|
+
return { externalId: action.subject.id, data: { performed: false, reason: `${op} is a read at Upstash Vector — there is nothing to write for it` } };
|
|
286
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/** A filter that could not be parsed. The handler maps this to the vendor's 400 envelope. */
|
|
2
|
+
export declare class FilterError extends Error {
|
|
3
|
+
constructor(message: string);
|
|
4
|
+
}
|
|
5
|
+
export type TokenKind = 'ident' | 'string' | 'number' | 'op' | 'word' | 'punct';
|
|
6
|
+
export type Token = {
|
|
7
|
+
kind: TokenKind;
|
|
8
|
+
value: string;
|
|
9
|
+
pos: number;
|
|
10
|
+
};
|
|
11
|
+
/**
|
|
12
|
+
* Split a filter string into tokens.
|
|
13
|
+
*
|
|
14
|
+
* Identifiers carry their `.`/`[]` accessors as ONE token (`geography.coordinates.latitude`,
|
|
15
|
+
* `major_industries[0]`, `tags[#-1]`) because the path is resolved as a unit at evaluation time;
|
|
16
|
+
* splitting them would force the parser to re-join them and lose the distinction between the
|
|
17
|
+
* accessor dot and a decimal point.
|
|
18
|
+
*/
|
|
19
|
+
export declare function tokenizeFilter(input: string): Token[];
|
|
20
|
+
export type FilterValue = string | number | boolean;
|
|
21
|
+
export type ComparisonOp = '=' | '!=' | '<' | '>' | '<=' | '>=' | 'GLOB' | 'NOT GLOB' | 'CONTAINS' | 'NOT CONTAINS';
|
|
22
|
+
export type FilterNode = {
|
|
23
|
+
kind: 'and';
|
|
24
|
+
left: FilterNode;
|
|
25
|
+
right: FilterNode;
|
|
26
|
+
} | {
|
|
27
|
+
kind: 'or';
|
|
28
|
+
left: FilterNode;
|
|
29
|
+
right: FilterNode;
|
|
30
|
+
} | {
|
|
31
|
+
kind: 'compare';
|
|
32
|
+
path: string;
|
|
33
|
+
op: ComparisonOp;
|
|
34
|
+
value: FilterValue;
|
|
35
|
+
} | {
|
|
36
|
+
kind: 'in';
|
|
37
|
+
path: string;
|
|
38
|
+
negated: boolean;
|
|
39
|
+
values: FilterValue[];
|
|
40
|
+
} | {
|
|
41
|
+
kind: 'hasField';
|
|
42
|
+
path: string;
|
|
43
|
+
negated: boolean;
|
|
44
|
+
};
|
|
45
|
+
/** Parse a filter string into an AST. Throws `FilterError` on anything malformed. */
|
|
46
|
+
export declare function parseFilter(filter: string): FilterNode;
|
|
47
|
+
/** One step of a metadata path: an object key, or an array subscript. */
|
|
48
|
+
type PathStep = {
|
|
49
|
+
key: string;
|
|
50
|
+
} | {
|
|
51
|
+
index: number;
|
|
52
|
+
fromEnd: boolean;
|
|
53
|
+
};
|
|
54
|
+
/**
|
|
55
|
+
* Split `geography.coordinates[0]` / `tags[#-1]` into steps.
|
|
56
|
+
*
|
|
57
|
+
* `[#-1]` is the docs' "index from the back using the `#` character with negative values": the
|
|
58
|
+
* `#` marks end-relative addressing and the value that follows is the offset, so `[#-1]` is the
|
|
59
|
+
* LAST element. A plain `[0]` is head-relative.
|
|
60
|
+
*/
|
|
61
|
+
export declare function parsePath(path: string): PathStep[];
|
|
62
|
+
/** The sentinel for "this path is not present". Distinct from a stored `null`, which IS present. */
|
|
63
|
+
export declare const MISSING: unique symbol;
|
|
64
|
+
/** Resolve a metadata path. Returns `MISSING` when any step is absent. */
|
|
65
|
+
export declare function resolvePath(metadata: unknown, path: string): unknown | typeof MISSING;
|
|
66
|
+
/**
|
|
67
|
+
* SQLite-style GLOB: `*` any run, `?` one character, `[abc]` a class, `[^abc]` a negated class,
|
|
68
|
+
* `[a-z]` a range. Case-SENSITIVE (that is what distinguishes GLOB from LIKE).
|
|
69
|
+
*
|
|
70
|
+
* Implemented as a backtracking matcher rather than by translating to a RegExp: a translation has
|
|
71
|
+
* to escape every regex metacharacter that is ORDINARY in a glob (`.`, `+`, `(`, `$`, …), and
|
|
72
|
+
* missing one turns a filter into a different filter — a silent wrong-answer bug rather than an
|
|
73
|
+
* error. The docs' own example, `city GLOB '?[sz]*[^m-z]'`, exercises all four constructs.
|
|
74
|
+
*/
|
|
75
|
+
export declare function globMatch(pattern: string, subject: string): boolean;
|
|
76
|
+
/** Evaluate a parsed filter against one vector's metadata. */
|
|
77
|
+
export declare function evaluateFilter(node: FilterNode, metadata: unknown): boolean;
|
|
78
|
+
/** Parse + evaluate in one step. Exported for the handler's single call site. */
|
|
79
|
+
export declare function matchesFilter(filter: string, metadata: unknown): boolean;
|
|
80
|
+
export {};
|