@volter/twin-pinecone 0.1.0

package/src/pinecone-connector.ts

@@ -0,0 +1,130 @@
+// Pinecone CONNECTOR — the live-vendor pull path over an INJECTED client (the auth boundary; a
+// fake in tests, the real `@pinecone-database/pinecone` SDK in prod). The pack imports NO SDK and
+// holds NO key at runtime — only this file's *type* shapes describe what a real client call would
+// return; the actual SDK is a devDependency imported ONLY in *.test.ts (architecture.test.ts).
+//
+// PULL (real -> twin): fetch real indexes + vectors via the injected client, map them to
+// SyncResource[], and fold them into the event log via `syncPull` (shadow-diff dedup, so a
+// re-pull of identical state is a no-op). Kernel subject ids are TYPE-PREFIXED to stay
+// collision-safe (mirrors pinecone-twin.ts).
+//
+// Vectors are genuinely HANDLE-DRIVEN, the same honest shape fal's connector uses (see fal-
+// connector.ts / pull-audit.json's fal notes): Pinecone's real SDK has NO cross-index "list every
+// vector in my project" endpoint — `Index.listPaginated()` is scoped to one already-known index +
+// namespace. So `PineconeLikeClient.listVectors` is this connector's OWN pull-source shape (a
+// real production implementation loops `pc.listIndexes()` then `pc.index(name).listPaginated()`
+// per namespace and flattens the result into exactly this shape) rather than a literal single
+// vendor endpoint — same honesty framing as fal's `syncFalFromReal`.
+//
+// A pulled index is re-minted a LOCAL data-plane host (`mintHost`) rather than keeping whatever
+// host the real vendor reported — the twin's own data-plane router only ever recognizes its own
+// minted hosts (pinecone-twin.ts's `recoverIndexFromHost`), so a pulled index must get one too to
+// stay queryable through this twin.
+import { syncPull } from '@volter/world-core';
+import type { SyncResource } from '@volter/world-core';
+import { mintHost } from './pinecone-twin.ts';
+//
+// ── The client-side RATE BUDGET is not optional here ────────────────────────────────────────
+// Pinecone publishes a detailed DATA-plane limits table but NO row for the CONTROL-plane listIndexes
+// this connector calls, and `listVectors` here is a cross-index, per-namespace paginated aggregation
+// rather than one vendor endpoint.
+// Every entrypoint below GUARDS the injected client before touching it (`guardPineconeClient`, which is
+// idempotent — a caller who already wrapped is not double-charged, a caller who forgot is protected
+// anyway); there is deliberately no option that turns the budget off. See pinecone-budget.ts.
+import { pineconeBudgetOf, guardPineconeClient, type PineconeBudgetedOptions } from './pinecone-budget.ts';
+
+const SERVICE = 'pinecone';
+
+export type { PineconeBudgetedOptions };
+
+export type PineconeRealIndex = {
+  name: string;
+  dimension?: number | null;
+  metric?: string | null;
+  spec?: unknown;
+  deletionProtection?: string | null;
+  tags?: Record<string, string> | null;
+  vectorType?: string | null;
+};
+export type PineconeRealVector = {
+  index: string;
+  namespace?: string | null;
+  id: string;
+  values?: number[] | null;
+  metadata?: Record<string, unknown> | null;
+};
+
+// The injected client exposes the minimal subset the connector calls; a real Pinecone-backed
+// implementation is structurally assignable (see file header for why `listVectors` is a
+// connector-owned aggregation shape, not a literal SDK method name).
+export interface PineconeLikeClient {
+  listIndexes?: () => Promise<{ indexes: PineconeRealIndex[] }>;
+  listVectors?: () => Promise<{ vectors: PineconeRealVector[] }>;
+}
+
+function indexKey(name: string): string {
+  return `index:${name}`;
+}
+function vectorKey(index: string, namespace: string, id: string): string {
+  return `vector:${index}::${namespace}::${id}`;
+}
+
+// ── PULL mappers (real Pinecone object → SyncResource) ─────────────────────────────────────
+export function mapIndex(m: PineconeRealIndex): SyncResource {
+  return {
+    type: 'index',
+    id: indexKey(m.name),
+    fields: {
+      name: m.name,
+      dimension: m.dimension ?? null,
+      metric: m.metric ?? 'cosine',
+      vectorType: m.vectorType ?? 'dense',
+      spec: m.spec ?? { serverless: { cloud: 'aws', region: 'us-east-1' } },
+      deletionProtection: m.deletionProtection ?? 'disabled',
+      tags: m.tags ?? {},
+      host: mintHost(m.name),
+      ready: true,
+      state: 'Ready',
+    },
+  };
+}
+export function mapVector(v: PineconeRealVector): SyncResource {
+  const namespace = v.namespace ?? '';
+  return {
+    type: 'vector',
+    id: vectorKey(v.index, namespace, v.id),
+    fields: { values: v.values ?? [], metadata: v.metadata ?? {} },
+  };
+}
+
+export async function pullPineconeIndexes(rawClient: PineconeLikeClient, opts: { root?: string } & PineconeBudgetedOptions = {}): Promise<SyncResource[]> {
+  const client = guardPineconeClient(rawClient, pineconeBudgetOf(opts));
+  if (!client.listIndexes) return [];
+  return (await client.listIndexes()).indexes.map((m) => mapIndex(m));
+}
+export async function pullPineconeVectors(rawClient: PineconeLikeClient, opts: PineconeBudgetedOptions = {}): Promise<SyncResource[]> {
+  const client = guardPineconeClient(rawClient, opts);
+  if (!client.listVectors) return [];
+  return (await client.listVectors()).vectors.map(mapVector);
+}
+
+/**
+ * D7 entry point: pull from real Pinecone (indexes + vectors) and fold EVERY domain into the twin
+ * via ONE syncPull (shadow-diff dedup). Returns { observed, deltasAppended } — a re-pull of
+ * identical state appends ZERO deltas (observed stays, deltasAppended drops to 0).
+ */
+export async function syncPineconeFromReal(
+  rawClient: PineconeLikeClient,
+  opts: { root?: string; occurredAt?: string } & PineconeBudgetedOptions = {},
+): Promise<{ observed: number; deltasAppended: number }> {
+  // Guard ONCE here and hand the guarded client down: this is the entrypoint whose vector
+  // aggregation fans out, so it is the one that must be unable to run unbudgeted.
+  const client = guardPineconeClient(rawClient, pineconeBudgetOf(opts));
+  const occurredAt = opts.occurredAt ?? new Date().toISOString();
+  const resources: SyncResource[] = [
+    ...(await pullPineconeIndexes(client, opts)),
+    ...(await pullPineconeVectors(client)),
+  ];
+  const result = syncPull({ service: SERVICE, resources, occurredAt, ...(opts.root !== undefined ? { root: opts.root } : {}) });
+  return { observed: result.observed, deltasAppended: result.deltasAppended };
+}

package/src/pinecone-filter.ts

@@ -0,0 +1,84 @@
+// Pinecone METADATA-FILTER evaluator — net-new, ported CONCEPTUALLY (not copied) from
+// aws-dynamodb-expr.ts's pattern: a genuine interpreter that evaluates real operators against
+// real stored data, not a name/keyword match. The two engines differ in shape because the input
+// differs — DynamoDB's ConditionExpression is a STRING that must be tokenized/parsed, while a
+// Pinecone filter arrives as already-parsed JSON (Mongo-style query syntax) — so this file is a
+// direct recursive evaluator over that JSON, with no tokenizer/parser stage. What is genuinely
+// carried over from aws-dynamodb-expr.ts is the DISCIPLINE: real operator semantics (not string
+// containment), used to gate query/delete results for real, with a crafted query.filter_* test
+// suite proving a filter-ignoring implementation fails (see pinecone-capabilities.ts).
+//
+// Modeled operator subset (Pinecone-documented — see spec-sources.json): field-level
+// `$eq $ne $gt $gte $lt $lte $in $nin`, plus a bare `{field: literal}` shorthand for `$eq`
+// (Pinecone's own documented shorthand), and top-level combinators `$and` / `$or` (arrays of
+// sub-filters). NOT modeled (todo, honest v1 cut): `$exists`, deep dot-path/nested-object
+// metadata addressing, and `null`-value comparison edge semantics — see
+// `pinecone.filters.exists` / `pinecone.filters.deep_nested` / `pinecone.filters.null_semantics`.
+export type PineconeFilter = Record<string, unknown>;
+
+const COMPARISON_OPERATORS = new Set(['$eq', '$ne', '$gt', '$gte', '$lt', '$lte', '$in', '$nin']);
+
+function isPlainObject(v: unknown): v is Record<string, unknown> {
+  return typeof v === 'object' && v !== null && !Array.isArray(v);
+}
+
+function valuesEqual(a: unknown, b: unknown): boolean {
+  if (Array.isArray(a) && Array.isArray(b)) {
+    return a.length === b.length && a.every((x, i) => valuesEqual(x, b[i]));
+  }
+  return a === b;
+}
+
+function evalComparison(op: string, fieldValue: unknown, operand: unknown): boolean {
+  switch (op) {
+    case '$eq':
+      return valuesEqual(fieldValue, operand);
+    case '$ne':
+      return !valuesEqual(fieldValue, operand);
+    case '$in':
+      return Array.isArray(operand) && operand.some((o) => valuesEqual(fieldValue, o));
+    case '$nin':
+      return Array.isArray(operand) && !operand.some((o) => valuesEqual(fieldValue, o));
+    case '$gt':
+    case '$gte':
+    case '$lt':
+    case '$lte': {
+      if (typeof fieldValue !== 'number' || typeof operand !== 'number') return false;
+      if (op === '$gt') return fieldValue > operand;
+      if (op === '$gte') return fieldValue >= operand;
+      if (op === '$lt') return fieldValue < operand;
+      return fieldValue <= operand;
+    }
+    default:
+      return false;
+  }
+}
+
+/** Evaluate a single field's clause: either an operator object `{$gt: 5}` or a bare `$eq` literal. */
+function evalFieldClause(fieldValue: unknown, clause: unknown): boolean {
+  if (isPlainObject(clause)) {
+    const keys = Object.keys(clause);
+    if (keys.length > 0 && keys.every((k) => COMPARISON_OPERATORS.has(k))) {
+      return keys.every((op) => evalComparison(op, fieldValue, clause[op]));
+    }
+  }
+  return valuesEqual(fieldValue, clause); // bare-literal shorthand for $eq
+}
+
+/** Evaluate a Pinecone metadata filter (Mongo-style) against a stored metadata object. An absent/empty filter matches everything. */
+export function matchesFilter(metadata: Record<string, unknown> | undefined, filter: PineconeFilter | undefined | null): boolean {
+  if (!filter || Object.keys(filter).length === 0) return true;
+  const md = metadata ?? {};
+  for (const [key, clause] of Object.entries(filter)) {
+    if (key === '$and') {
+      if (!Array.isArray(clause) || !clause.every((c) => matchesFilter(md, c as PineconeFilter))) return false;
+      continue;
+    }
+    if (key === '$or') {
+      if (!Array.isArray(clause) || clause.length === 0 || !clause.some((c) => matchesFilter(md, c as PineconeFilter))) return false;
+      continue;
+    }
+    if (!evalFieldClause(md[key], clause)) return false;
+  }
+  return true;
+}

package/src/pinecone-server.ts

@@ -0,0 +1,75 @@
+// Pinecone twin HTTP server — ONE server hosting BOTH the control plane (api.pinecone.io) and
+// every index's data plane (a per-index minted host, see pinecone-twin.ts's `mintHost`), so an
+// unmodified `@pinecone-database/pinecone` SDK works end-to-end against a single local process.
+//
+// THE ROUTING TRICK (verified against the installed 8.0.0 package's own source — pinecone-sdk.
+// integration.test.ts's header has the full trail): `PineconeConfiguration.fetchApi` is a
+// genuine full-fetch override — BOTH `indexOperationsBuilder` (control plane) and
+// `VectorOperationsProvider`/etc (data plane) read `getFetch(config)`, which prefers
+// `config.fetchApi` over the ambient global `fetch` — so ONE override at `new Pinecone({
+// fetchApi })` covers every request the SDK makes, control AND data plane alike. The twin's own
+// `mintHost()` output is a synthetic, non-resolvable hostname; a caller (real SDK or otherwise)
+// that supplies a `fetchApi` rewriting the URL's host to this server's real `127.0.0.1:<port>`
+// while stashing the ORIGINAL host in the `x-pinecone-target-host` header (exactly what
+// pinecone-sdk.integration.test.ts's `fetchApi` does) reaches this ONE process for every call —
+// the real network is never touched, keeping every capability verify offline + deterministic.
+// Without that header (a plain caller hitting this server directly), the server falls back to
+// the literal HTTP `Host` header, so a caller who already points requests straight at THIS
+// server's address (e.g. `controllerHostUrl: 'http://127.0.0.1:<port>'` for the control plane,
+// `pc.index({ name, host: 'http://127.0.0.1:<port>' })` for the data plane — both real,
+// documented SDK config fields) also works without any header trick at all; the twin's own
+// `pinecone-twin.test.ts` HTTP-server tests drive it exactly that way.
+//
+// Writable by default; pass readOnly to reject writes (per-operation — see pinecone-twin.ts).
+//
+// FETCH-FIRST (runtime contract R12b): the surface is the plain `createPineconeTwinFetch` and the
+// SERVER is one line of `Bun.serve` around it. This is a CUSTOM fetch, not the kernel adapter
+// (`createTwinFetchFromHandler`): the 202/204 replies are body-suppressed by STATUS whatever the
+// handler returned, which the adapter's body-shaped rule would change.
+import { serveHttp } from '@volter/world-core';
+import { handlePineconeTwinRequest } from './pinecone-twin.ts';
+import { worldNow, statefulTwinManifest} from '@volter/world-core';
+
+/** Options every Pinecone-twin HTTP surface needs, independent of who owns the socket. */
+export interface PineconeTwinFetchOptions {
+  root?: string;
+  readOnly?: boolean;
+}
+
+export function createPineconeTwinFetch(options: PineconeTwinFetchOptions): (request: Request) => Promise<Response> {
+  const readOnly = options.readOnly ?? false;
+  return async function pineconeTwinFetch(request: Request): Promise<Response> {
+    const url = new URL(request.url);
+    // GET /twin — the discovery manifest (education inside the twin).
+    if (request.method === 'GET' && url.pathname.replace(/\/+$/, '') === '/twin') {
+      return Response.json(statefulTwinManifest({ vendor: 'pinecone', twinOf: 'the Pinecone vector database', stores: 'indexes, collections and vectors, searched by a real brute-force scorer' }));
+    }
+    const headers: Record<string, string> = {};
+    request.headers.forEach((value, key) => { headers[key.toLowerCase()] = value; });
+    const host = headers['x-pinecone-target-host'] || headers['host'] || url.host;
+    const body = request.method !== 'GET' && request.method !== 'HEAD' ? await request.text() : '';
+    const { status, body: out, headers: outHeaders } = await handlePineconeTwinRequest({
+      method: request.method,
+      path: url.pathname + (url.search || ''),
+      body,
+      headers,
+      host,
+      readOnly,
+      occurredAt: worldNow(),
+      ...(options.root !== undefined ? { root: options.root } : {}),
+    });
+    if (status === 202 || status === 204 || out === null) {
+      return new Response(null, { status, headers: { ...(outHeaders ?? {}) } });
+    }
+    return new Response(JSON.stringify(out), { status, headers: { 'content-type': 'application/json', ...(outHeaders ?? {}) } });
+  };
+}
+
+export async function createPineconeTwinServer(options: { root?: string; port?: number; readOnly?: boolean } = {}): Promise<{ port: number; stop: () => void }> {
+  const server = await serveHttp({
+    port: options.port ?? 0,
+    idleTimeout: 60,
+    fetch: createPineconeTwinFetch(options),
+  });
+  return { port: server.port ?? options.port ?? 0, stop: () => server.stop(true) };
+}

package/src/pinecone-similarity.ts

@@ -0,0 +1,71 @@
+// Pinecone SIMILARITY MATH — net-new, no clone (ADDING_A_TWIN.md's "real computed op as strong
+// done" pattern; the analog of aws-dynamodb's Query returning computed results offline). This is
+// REAL brute-force top-K search over the stored vector values, not a stub: exact cosine /
+// dotproduct / euclidean, computed and sorted for real. Exact search is a faithful SUPERSET of
+// Pinecone's real approximate nearest-neighbor (ANN) index — the matches/scores this twin
+// returns are CORRECT, just not approximated the way a real HNSW/IVF index would be.
+//
+// Metric semantics (grounded facts + one doc-UNVERIFIED ⚠, recorded in spec-sources.json):
+//   cosine     — cosine similarity in [-1, 1]; HIGHER score = more similar (descending sort).
+//   dotproduct — raw dot product; HIGHER score = more similar (descending sort).
+//   euclidean  ⚠ — this twin returns the SQUARED L2 distance (sum of squared per-dimension
+//     differences — the conventional Pinecone computation, avoiding an unnecessary sqrt: squaring
+//     is monotonic so it never changes the ranking). LOWER score = more similar (ascending sort).
+//     The public Pinecone docs fetched during this build did not state the exact wire-level score
+//     value or sort direction for euclidean (WebFetch against docs.pinecone.io/guides/index-data/
+//     indexing-overview came back with no coverage of distance-metric scoring) — this is doc-
+//     UNVERIFIED. The ascending-is-closer DIRECTION is not a guessed API convention, though: it is
+//     the definitional meaning of "distance" (0 = identical vectors = most similar), so it is
+//     modeled as `done` rather than demoted to `todo`; only the exact SIGN/SCALE of the number
+//     Pinecone's real API returns is unverified.
+export type PineconeMetric = 'cosine' | 'dotproduct' | 'euclidean';
+
+export function dotProduct(a: readonly number[], b: readonly number[]): number {
+  const n = Math.min(a.length, b.length);
+  let sum = 0;
+  for (let i = 0; i < n; i++) sum += a[i]! * b[i]!;
+  return sum;
+}
+
+export function magnitude(v: readonly number[]): number {
+  let sum = 0;
+  for (const x of v) sum += x * x;
+  return Math.sqrt(sum);
+}
+
+/** Cosine similarity in [-1, 1]; 0 if either vector has zero magnitude (undefined direction). */
+export function cosineSimilarity(a: readonly number[], b: readonly number[]): number {
+  const magA = magnitude(a);
+  const magB = magnitude(b);
+  if (magA === 0 || magB === 0) return 0;
+  return dotProduct(a, b) / (magA * magB);
+}
+
+/** Squared L2 distance (sum of squared differences). Mismatched lengths pad the shorter with 0. */
+export function squaredEuclideanDistance(a: readonly number[], b: readonly number[]): number {
+  const n = Math.max(a.length, b.length);
+  let sum = 0;
+  for (let i = 0; i < n; i++) {
+    const d = (a[i] ?? 0) - (b[i] ?? 0);
+    sum += d * d;
+  }
+  return sum;
+}
+
+/** Score one candidate against a query vector for the given metric. */
+export function scoreVector(metric: PineconeMetric, query: readonly number[], candidate: readonly number[]): number {
+  if (metric === 'cosine') return cosineSimilarity(query, candidate);
+  if (metric === 'dotproduct') return dotProduct(query, candidate);
+  return squaredEuclideanDistance(query, candidate);
+}
+
+/** True when a HIGHER score means more similar (false only for euclidean: lower/ascending). */
+export function higherIsBetter(metric: PineconeMetric): boolean {
+  return metric !== 'euclidean';
+}
+
+/** Sort scored candidates best-match-first for the given metric (stable). */
+export function sortByScore<T extends { score: number }>(metric: PineconeMetric, scored: readonly T[]): T[] {
+  const ascending = !higherIsBetter(metric);
+  return [...scored].sort((a, b) => (ascending ? a.score - b.score : b.score - a.score));
+}