@pithy-sh/vector 0.1.0

package/src/provision/provisionVector.ts

@@ -0,0 +1,250 @@
+// SPDX-FileCopyrightText: 2026 Pithy
+// SPDX-License-Identifier: MIT
+
+import { PithyError, ValidationError } from "@pithy-sh/core/src/error/pithyError";
+import { resourceNames } from "@pithy-sh/core/src/naming/resourceNames";
+import type { VectorConfig } from "../config/config";
+import type { MetadataIndexReport } from "../index/drift";
+import { type MetadataIndexDescriptor, metadataIndexes } from "../index/metadata";
+import type { ObservedMetadataIndex, ProvisionedIndex, VectorProvisionRecord } from "../index/provisioned";
+import { VECTOR_CAPABILITY } from "../workflows/specs";
+
+/**
+ * The provisioning orchestration for the vector capability — the live counterpart to `pithy add vector`'s
+ * config wiring. `pithy add` writes the bindings it can; this creates what they point at.
+ *
+ * The `vectorize` and `workflows` bindings are not among the ones it can. Wrangler requires an `index_name`
+ * on the first and a `name` + `class_name` on the second, and every one of those values is an output of this
+ * run — so `add` writes neither (a partial entry stops wrangler loading the config at all) and the CLI writes
+ * both, complete, after provisioning succeeds.
+ *
+ * **The order is the whole contract.** An index is created, then every metadata index its schema declares,
+ * and only then is the worker that writes vectors deployed. Cloudflare states it plainly: *"Vectors upserted
+ * before a metadata index was created won't have their metadata contained in that index."* A vector written
+ * into the window between those two steps is permanently invisible to any filter on that field — no error,
+ * no warning, just a search that quietly returns less than it should. Doing this in the wrong order is
+ * unrecoverable without a full re-embed, which is exactly why the order lives in tested code rather than in
+ * a runbook.
+ *
+ * The live Cloudflare/wrangler steps sit behind the {@link VectorProvisioner} seam, so the ordering and the
+ * idempotency contract are unit-tested without an account. **Every step is idempotent** — find-then-create
+ * for the index and its metadata indexes, and a deploy that overwrites — so a re-run reconciles rather than
+ * fails. Vectorize applies both index and metadata-index changes asynchronously, so nothing here writes and
+ * immediately reads back.
+ */
+
+/**
+ * The Vectorize index name for one configured index in one environment —
+ * `<project>-<env>-vector-<index>`.
+ *
+ * Per environment, never shared: a staging index and a prod index that share vectors mean staging's
+ * test corpus answers prod's searches, and a staging teardown deletes prod's embeddings.
+ *
+ * Per **project** for a sharper reason. Vectorize's index namespace is account-wide, and provisioning
+ * reuses an index it finds by name — so an unscoped `pithy-vector-docs-prod` would let a second Pithy
+ * project adopt this one's index, mix two corpora into one search, and delete both on either teardown.
+ * Dimensions and metric are fixed at creation, so the adoption would also silently pin the second
+ * project to the first's embedding model.
+ *
+ * **Composed through the naming facade**, which carries Vectorize's own rule: 64 bytes — not the 63
+ * every namespace was once held to — refused rather than truncated, because a shortened index name is
+ * a *different, empty* index and re-embedding a corpus is neither free nor quick. The facade also
+ * validates the project once and the environment once, so `production` is refused here rather than
+ * standing up a fourth environment nothing else knows about.
+ */
+export function vectorIndexName(project: string, index: string, env: string): string {
+  const scope = resourceNames(project).env(env);
+  try {
+    return scope.vectorizeIndex(`${VECTOR_CAPABILITY}-${index}`);
+  } catch (error) {
+    // The facade refuses an over-long Vectorize name as an `InternalError`, on the reasoning that the
+    // project and the environment are already validated so only author code can overflow it. That
+    // reasoning does not hold here: the trailing segment is a key out of the adopter's
+    // `pithy.config.ts`, and config validates it against Vectorize's 64 without knowing what the
+    // project and environment will spend of it. So the refusal is restated in the adopter's terms —
+    // same limit, same number, named as the fixable thing it is.
+    if (!(error instanceof PithyError)) throw error;
+    throw new ValidationError({
+      message: `The \`${index}\` index cannot be named: ${error.payload.message}`,
+      action: `Shorten the \`${index}\` index's key in pithy.config.ts, or the project name — the deployed name is <project>-<env>-${VECTOR_CAPABILITY}-<index>.`,
+      detail: error.payload.detail ?? error.payload.message,
+    });
+  }
+}
+
+/** The shape an index is created with. Fixed at creation — neither value can be changed afterwards. */
+export interface VectorIndexShape {
+  /** Components per vector, matching the model that fills the index. */
+  dimensions: number;
+  /** How nearest neighbors are scored. */
+  metric: "cosine" | "euclidean" | "dot-product";
+}
+
+/** The live Cloudflare/wrangler seam. Each step must be idempotent. */
+export interface VectorProvisioner {
+  /** Verify account prerequisites before anything is created (a workers.dev subdomain, Vectorize access). */
+  preflight(): Promise<void>;
+  /** Create (or reuse) the index. Idempotent — an existing index with the same shape is reused as-is. */
+  ensureIndex(indexName: string, shape: VectorIndexShape): Promise<{ name: string }>;
+  /**
+   * Create every declared metadata index that is missing, and report what was found. Idempotent.
+   *
+   * **Returning is the guarantee.** An implementation returns only once every declared index is live on the
+   * index — waiting out Cloudflare's asynchronous apply — and throws otherwise. That is what lets
+   * {@link provisionVector} record the declared set as observed, and what keeps the very next step (deploying
+   * the worker that writes vectors) out of the window where a write would be permanently unfilterable.
+   */
+  ensureMetadataIndexes(indexName: string, declared: readonly MetadataIndexDescriptor[]): Promise<MetadataIndexReport>;
+  /** Deploy the prebuilt vector worker for this environment, bound to the provisioned indexes. */
+  deployWorker(env: string, indexNames: Record<string, string>): Promise<void>;
+  /** Delete an index and everything in it. **Destructive** — every vector is lost. Idempotent. */
+  deleteIndex(indexName: string): Promise<void>;
+  /** Start a reprocess run for one configured index and wait for it. */
+  reprocess(env: string, index: string, options: { all?: boolean; filter?: Record<string, unknown> }): Promise<unknown>;
+}
+
+/** What provisioning did for one configured index. */
+export interface VectorIndexResult {
+  /** The config's name for the index. */
+  index: string;
+  /** The Vectorize index name it was provisioned as. */
+  indexName: string;
+  /** The metadata indexes created on this run — empty on a re-run, which is what idempotent looks like. */
+  created: MetadataIndexDescriptor[];
+  /** Metadata indexes that exist but the config does not declare. Not fatal; they still spend a slot. */
+  extra: { propertyName: string; indexType: string }[];
+  /**
+   * Every metadata index live on this index when provisioning finished: the declared ones (each confirmed
+   * visible by `ensureMetadataIndexes`, which throws rather than return early) plus the undeclared ones.
+   * This is what {@link toProvisionRecord} writes into the Worker's env, and what the Worker's boot check
+   * compares its declarations against.
+   */
+  observed: ObservedMetadataIndex[];
+}
+
+/** What provisioning produced. */
+export interface VectorProvisionResult {
+  /** The environment provisioned. */
+  env: string;
+  /** Each configured index, with what was created for it. */
+  indexes: VectorIndexResult[];
+}
+
+/** Provisioning inputs: the resolved config, the project that owns it, and the environment to stand it up in. */
+export interface VectorProvisionOptions {
+  /**
+   * The project name — the `<project>` segment every index name leads with. The root
+   * `pithy.config.ts` `name`, resolved by `requireProjectName` and never guessed: an index found by
+   * name is *reused*, so a wrong value adopts another project's corpus.
+   */
+  project: string;
+  /** The app's resolved vector config. */
+  config: VectorConfig;
+  /** The environment to provision. Vectorize has no local emulation, so `dev` is a real remote index too. */
+  env: string;
+}
+
+/** The declared metadata indexes for one configured index, or none when it declares no metadata schema. */
+function declaredIndexes(config: VectorConfig, index: string): MetadataIndexDescriptor[] {
+  const metadata = config.indexes[index]?.metadata;
+  return metadata ? metadataIndexes(metadata) : [];
+}
+
+/**
+ * Provision every configured index for one environment: the index, then its metadata indexes, then the
+ * worker that will write to it. Idempotent end to end.
+ */
+export async function provisionVector(
+  provisioner: VectorProvisioner,
+  options: VectorProvisionOptions,
+): Promise<VectorProvisionResult> {
+  await provisioner.preflight();
+
+  const indexNames: Record<string, string> = {};
+  const indexes: VectorIndexResult[] = [];
+
+  for (const [index, indexConfig] of Object.entries(options.config.indexes)) {
+    const indexName = vectorIndexName(options.project, index, options.env);
+    await provisioner.ensureIndex(indexName, { dimensions: indexConfig.dimensions, metric: indexConfig.metric });
+
+    // Immediately after the index, and before any worker that could write to it exists.
+    const declared = declaredIndexes(options.config, index);
+    const report = await provisioner.ensureMetadataIndexes(indexName, declared);
+
+    indexNames[index] = indexName;
+    indexes.push({
+      index,
+      indexName,
+      created: report.missing,
+      extra: report.extra,
+      // `ensureMetadataIndexes` returned, so every declared index is live. Record the declarations rather
+      // than re-listing the index: a fresh list would be a second eventually-consistent read, and could
+      // report an index that was just confirmed as absent again.
+      observed: [...declared, ...report.extra],
+    });
+  }
+
+  // The worker last: it is the thing that writes vectors, and every metadata index now exists.
+  await provisioner.deployWorker(options.env, indexNames);
+
+  return { env: options.env, indexes };
+}
+
+/**
+ * Project a provisioning result into the record the Worker boots against — the `VECTOR_PROVISIONED` var.
+ *
+ * This is the whole reason the Worker can check drift without a Cloudflare token: provisioning already holds
+ * one, already compared declared against live, and writes down what it saw. The record is therefore a
+ * statement about the last provisioning run, not about Cloudflare now, and the boot check says so.
+ */
+export function toProvisionRecord(result: VectorProvisionResult): VectorProvisionRecord {
+  const indexes: Record<string, ProvisionedIndex> = {};
+  for (const entry of result.indexes) {
+    indexes[entry.index] = { indexName: entry.indexName, metadataIndexes: entry.observed };
+  }
+  return { indexes };
+}
+
+/** What a reset did. */
+export interface VectorResetResult extends VectorProvisionResult {
+  /** The indexes that were deleted and rebuilt, by their Vectorize names. */
+  deleted: string[];
+  /** The reprocess runs started, one per configured index. */
+  reprocessed: string[];
+}
+
+/**
+ * Delete every configured index, rebuild it, re-provision its metadata indexes, and re-embed the corpus.
+ *
+ * This is the repair for the one failure Vectorize cannot repair in place: a metadata index added after
+ * vectors were written covers none of them, and there is no backfill. Rebuilding the index and re-embedding
+ * from the document corpus is the only way back — which is the third reason `pithy_vector_documents` exists.
+ *
+ * **Destructive by definition.** Every vector is deleted; only the D1 corpus survives, and only what it holds
+ * comes back. The confirmation gate lives in the CLI, where the environment is known.
+ */
+export async function resetVector(
+  provisioner: VectorProvisioner,
+  options: VectorProvisionOptions,
+): Promise<VectorResetResult> {
+  await provisioner.preflight();
+
+  const deleted: string[] = [];
+  for (const index of Object.keys(options.config.indexes)) {
+    const indexName = vectorIndexName(options.project, index, options.env);
+    await provisioner.deleteIndex(indexName);
+    deleted.push(indexName);
+  }
+
+  // Rebuild through the ordinary path, so creation order and the metadata-index rule are stated once.
+  const provisioned = await provisionVector(provisioner, options);
+
+  const reprocessed: string[] = [];
+  for (const index of Object.keys(options.config.indexes)) {
+    // `all`: the rebuilt index holds nothing, so every document must be re-embedded regardless of its model.
+    await provisioner.reprocess(options.env, index, { all: true });
+    reprocessed.push(index);
+  }
+
+  return { ...provisioned, deleted, reprocessed };
+}

package/src/provision/resolveVectorConfig.ts

@@ -0,0 +1,85 @@
+// SPDX-FileCopyrightText: 2026 Pithy
+// SPDX-License-Identifier: MIT
+
+import {
+  type HostVectorizeBinding,
+  hostWorkflowsFor,
+  resolveWorkflowHost,
+  type WorkflowHostTemplate,
+} from "@pithy-sh/core/src/workflow/host";
+import type { VectorConfig } from "../config/config";
+import { toWorkerConfig } from "../config/workerConfig";
+import { VECTOR_CAPABILITY, vectorWorkflowRegistry } from "../workflows/specs";
+
+/**
+ * Resolve the vector worker's committed `wrangler.jsonc` template into one environment's standalone config.
+ * Thin over core's {@link resolveWorkflowHost}: the generic resolver owns the mechanics (clone, fill by
+ * binding name, stamp `ENVIRONMENT`), and this file owns only what is vector's. Pure — the caller parses
+ * the template and writes the result.
+ *
+ * Two things are vector's own.
+ *
+ * **The `vectorize` array is rebuilt, not filled.** A template declares the bindings a capability has *by
+ * default*; how many Vectorize bindings a project needs is a fact about its config, because one binding
+ * addresses exactly one index. So the array is regenerated from the configured indexes and then handed to the
+ * generic resolver to fill and mark remote.
+ *
+ * **`VECTOR_CONFIG` is a projection.** An index's `metadata` is a live Zod schema; `JSON.stringify` would
+ * hand the worker an unusable husk. {@link toWorkerConfig} projects the config into the serializable facts
+ * the worker needs — including the *result* of introspecting each metadata schema.
+ */
+
+/** The resolved ids and per-env values for one environment's vector-worker deploy. */
+export interface VectorConfigParams {
+  /**
+   * The project name — the `<project>` segment the deployed worker and Workflow names lead with. The
+   * root `pithy.config.ts` `name`, resolved by `requireProjectName` and never guessed.
+   */
+  project: string;
+  /** The target environment. */
+  env: string;
+  /** The app database id for this environment — where the document corpus lives. */
+  appDatabaseId: string;
+  /** Config index name → provisioned Vectorize index name, from `provisionVector`. */
+  indexNames: Record<string, string>;
+  /** The app's resolved vector config. */
+  config: VectorConfig;
+}
+
+/** Fill the template for one environment. */
+export function resolveVectorConfig(template: WorkflowHostTemplate, params: VectorConfigParams): WorkflowHostTemplate {
+  const { project, env, appDatabaseId, indexNames, config } = params;
+
+  // One Vectorize binding per configured index, in config order. `index_name` is a placeholder here; the
+  // generic resolver fills it from `vectorizeIndexNames` below, so the mapping lives in exactly one place.
+  const vectorize: HostVectorizeBinding[] = Object.entries(config.indexes).map(([index, indexConfig]) => ({
+    binding: indexConfig.binding,
+    index_name: indexNames[index] ?? "",
+    remote: true,
+  }));
+
+  const vectorizeIndexNames: Record<string, string> = {};
+  for (const [index, indexConfig] of Object.entries(config.indexes)) {
+    const name = indexNames[index];
+    if (name) vectorizeIndexNames[indexConfig.binding] = name;
+  }
+
+  return resolveWorkflowHost(
+    { ...template, vectorize },
+    {
+      project,
+      capability: VECTOR_CAPABILITY,
+      env,
+      databaseIds: { DB: appDatabaseId },
+      vectorizeIndexNames,
+      // The reprocess Workflow, derived from vector's own specs. A Workflow name is account-scoped, so
+      // the project has to reach it — and only the registry knows both the project and the job.
+      workflows: hostWorkflowsFor(vectorWorkflowRegistry, { project, capability: VECTOR_CAPABILITY, env }).workflows,
+      // Neither Vectorize nor Workers AI has a local emulation, and a Workflow host always runs locally in
+      // `wrangler dev` — without `remote` the bindings resolve to nothing and every re-embed fails locally
+      // for a reason that reads like a code fault.
+      remoteBindings: [...Object.values(config.indexes).map((index) => index.binding), "AI"],
+      vars: { VECTOR_CONFIG: JSON.stringify(toWorkerConfig(config, (index) => indexNames[index] ?? "")) },
+    },
+  );
+}

package/src/seeds/example.ts

@@ -0,0 +1,79 @@
+// SPDX-FileCopyrightText: 2026 Pithy
+// SPDX-License-Identifier: MIT
+
+import { EXAMPLE_ADA, EXAMPLE_ALAN, EXAMPLE_GRACE } from "@pithy-sh/core/src/seed/exampleIdentities";
+import { d1SeedGroup, defineSeed, type SeedSet } from "@pithy-sh/core/src/seed/seed";
+import { VectorDocument } from "../data/document";
+import { VECTOR_DOCUMENTS_TABLE } from "../data/tables";
+
+/**
+ * Where the example set sorts in the project's seed registry. It runs after `auth` (100), whose example
+ * seeds the users these documents belong to, so the owning identities exist before anything references them.
+ *
+ * It need not line up with `VECTOR_MIGRATION_ORDER` — a different registry, composed separately by
+ * `pithy seed`.
+ */
+const VECTOR_EXAMPLE_SEED_ORDER = 240;
+
+/** The index these demo documents belong to — the name `pithy add vector` scaffolds into pithy.config.ts. */
+const EXAMPLE_INDEX = "docs";
+
+const now = () => new Date();
+
+/**
+ * A small demo corpus owned by the canonical example cast ({@link EXAMPLE_ADA} et al.). The owner rides in
+ * `metadata.ownerId` rather than in a column, because that is how a real corpus scopes: a document's owner
+ * is exactly the kind of field an index marks `filterable`, and putting it in metadata means the demo shows
+ * the filter path rather than a shape only the seed uses.
+ *
+ * **`model` is null on purpose.** A seed writes rows into D1; it cannot call Workers AI, so it cannot
+ * produce vectors. These documents are a durable corpus that is not yet searchable — which is precisely the
+ * state `pithy vector reprocess` exists to resolve, and running it against a freshly seeded dev environment
+ * is the shortest honest demonstration of the whole capability.
+ *
+ * Composed in only when the project turns on `seed.includeExamples`, and only for `dev` and `staging` — an
+ * example fixture never targets `prod`.
+ */
+export const vectorExampleSeed: SeedSet = defineSeed({
+  name: "example",
+  order: VECTOR_EXAMPLE_SEED_ORDER,
+  environments: ["dev", "staging"],
+  example: true,
+  d1: [
+    d1SeedGroup("app", VECTOR_DOCUMENTS_TABLE, VectorDocument, [
+      {
+        id: "example-doc-ada",
+        indexName: EXAMPLE_INDEX,
+        namespace: null,
+        content:
+          "An analytical engine can weave algebraic patterns the way a loom weaves flowers and leaves. Its power is in the general rule, not the particular sum.",
+        metadata: { ownerId: EXAMPLE_ADA.id, title: "On the analytical engine" },
+        model: null,
+        createdAt: now(),
+        updatedAt: now(),
+      },
+      {
+        id: "example-doc-grace",
+        indexName: EXAMPLE_INDEX,
+        namespace: null,
+        content:
+          "A program should be written in words people can read. Compilers exist so that the machine, not the person, does the translating.",
+        metadata: { ownerId: EXAMPLE_GRACE.id, title: "On compilers" },
+        model: null,
+        createdAt: now(),
+        updatedAt: now(),
+      },
+      {
+        id: "example-doc-alan",
+        indexName: EXAMPLE_INDEX,
+        namespace: null,
+        content:
+          "A machine may be said to think if a person conversing with it cannot tell it from another person. The question is behavior, not substance.",
+        metadata: { ownerId: EXAMPLE_ALAN.id, title: "On machine intelligence" },
+        model: null,
+        createdAt: now(),
+        updatedAt: now(),
+      },
+    ]),
+  ],
+});

package/src/version.generated.ts

@@ -0,0 +1,16 @@
+// SPDX-FileCopyrightText: 2026 Pithy
+// SPDX-License-Identifier: MIT
+
+// GENERATED by scripts/stampVersions.ts — do not edit by hand. Regenerate with `bun run stamp-versions`.
+//
+// A Worker cannot read its own package.json, so this is how @pithy-sh/vector knows its own version at
+// runtime. The capability attaches it, and `GET /control-plane/manifest` reports it per capability —
+// which is what answers "should this project upgrade" and "is this customer exposed to what we just
+// fixed". Those questions are only answerable per module, because a project composes some capabilities
+// and not others.
+
+/** This package's npm name — the join key against a release feed. */
+export const PACKAGE_NAME = "@pithy-sh/vector";
+
+/** This package's version, stamped from its own package.json at generation time. */
+export const PACKAGE_VERSION = "0.1.0";

package/src/workflows/reprocess.ts

@@ -0,0 +1,180 @@
+// SPDX-FileCopyrightText: 2026 Pithy
+// SPDX-License-Identifier: MIT
+
+import type { VectorIndexConfig } from "../config/config";
+import type { VectorDocument } from "../data/document";
+import type { DocumentStore } from "../data/documents";
+import { embedBatched, type VectorAi } from "../embed/embed";
+import { type CompiledFilter, matchesCompiledFilter } from "../index/filter";
+import { upsertVectors, type VectorStore, type VectorUpsert } from "../index/index";
+import { MAX_UPSERT_BATCH } from "../index/limits";
+
+/**
+ * Re-embed a corpus, one durable page at a time.
+ *
+ * The orchestration is pure: the Workflow step runner, the document store, the index, and the AI binding all
+ * arrive as parameters, so the whole of it — including resuming mid-run — is tested with fakes and no
+ * network. `workflows/worker.ts` is the thin `WorkflowEntrypoint` that supplies the real ones.
+ *
+ * **Pagination is keyset, not offset, and that is the correctness argument for the whole file.** The default
+ * pass selects rows whose `model` differs from the configured one and then *sets* that model on the rows it
+ * writes — so the result set shrinks underneath the scan. With `LIMIT/OFFSET` that is the classic skip bug:
+ * page two starts at row 1,000 of a set that just lost its first 1,000 members, and half the corpus is never
+ * re-embedded, silently. Reading `id > cursor` in id order cannot skip and cannot repeat, whatever the
+ * predicate does to the rows behind the cursor.
+ *
+ * **Each page is one `step.do`.** Workflow steps are journalled, so an instance that dies at page 4,000
+ * resumes at page 4,000 with the same cursor rather than re-embedding four million documents. The step names
+ * are derived from a page counter, so a replay asks for the same steps in the same order — which is what
+ * makes the journal usable at all.
+ *
+ * A row written during the run whose id sorts *behind* the cursor is not picked up. That is the honest cost
+ * of keyset order, and it is not a problem in practice: a document written after the model changed is
+ * embedded with the new model by the write path anyway.
+ */
+
+/** The Workflow step runner, structurally. Injectable, so a test can journal and replay it. */
+export interface ReprocessStep {
+  /** Run a named step, or return its journalled result if this instance already completed it. */
+  do<T>(name: string, callback: () => Promise<T>): Promise<T>;
+}
+
+/** What a reprocess run needs. Every one is a seam. */
+export interface ReprocessDeps {
+  /** The document corpus — the source of the text and the destination of the new model stamp. */
+  documents: DocumentStore;
+  /** The Vectorize index the re-embedded vectors are written to. */
+  store: VectorStore;
+  /** The Workers AI binding. */
+  ai: VectorAi;
+  /** The index's resolved config — the model to embed with and the shape to prove. */
+  index: VectorIndexConfig;
+  /** The index's config name, which scopes the corpus rows. */
+  indexName: string;
+  /** Clock, injected so the stamped `updatedAt` is assertable. */
+  now(): Date;
+}
+
+/** How much of the corpus a run covers. */
+export interface ReprocessOptions {
+  /** Re-embed every document, not only the ones whose model drifted. */
+  all?: boolean;
+  /** Narrow the run to documents whose metadata matches. Compile it before passing it in. */
+  filter?: CompiledFilter;
+  /** Documents per step. Defaults to — and is capped at — the Vectorize upsert ceiling. */
+  pageSize?: number;
+}
+
+/** What one page did. Returned through the step, so a replay restores it exactly. */
+interface PageResult {
+  /** The id to read after on the next page, or null when the corpus is exhausted. */
+  cursor: string | null;
+  /** Rows read. */
+  scanned: number;
+  /** Rows re-embedded and written. */
+  reembedded: number;
+  /** Rows that matched but hold no content, so there is nothing to embed. */
+  skipped: number;
+  /** Whether this was the last page. */
+  done: boolean;
+}
+
+/** The tally a run reports. */
+export interface ReprocessReport {
+  /** The index that was reprocessed. */
+  indexName: string;
+  /** How many pages the run took — one Workflow step each. */
+  pages: number;
+  /** Rows read. */
+  scanned: number;
+  /** Rows re-embedded and written back. */
+  reembedded: number;
+  /** Rows selected but skipped because the corpus holds no text for them. */
+  skipped: number;
+}
+
+/** Re-embed one page's documents and write them back. The body of a step. */
+async function reembedPage(deps: ReprocessDeps, documents: readonly VectorDocument[]): Promise<number> {
+  const embeddable = documents.filter((document) => document.content !== null);
+  if (embeddable.length === 0) return 0;
+
+  const vectors = await embedBatched(
+    deps.ai,
+    deps.index,
+    embeddable.map((document) => document.content as string),
+  );
+
+  const upserts: VectorUpsert[] = embeddable.map((document, position) => ({
+    id: document.id,
+    values: vectors[position] as number[],
+    metadata: document.metadata,
+    ...(document.namespace ? { namespace: document.namespace } : {}),
+  }));
+
+  await upsertVectors(deps.store, deps.index, upserts);
+  // Stamped only after the write is accepted, so a step that dies mid-page leaves the rows selectable by the
+  // next run rather than marked done.
+  await deps.documents.markEmbedded(
+    deps.indexName,
+    embeddable.map((document) => document.id),
+    deps.index.model,
+    deps.now(),
+  );
+  return embeddable.length;
+}
+
+/**
+ * Re-embed an index's documents. Resumable by construction: every page is a journalled step keyed by its
+ * page number, and the cursor that drives the next page comes out of the previous step's result.
+ */
+export async function reprocessIndex(
+  deps: ReprocessDeps,
+  step: ReprocessStep,
+  options: ReprocessOptions = {},
+): Promise<ReprocessReport> {
+  const limit = Math.min(options.pageSize ?? MAX_UPSERT_BATCH, MAX_UPSERT_BATCH);
+  const report: ReprocessReport = { indexName: deps.indexName, pages: 0, scanned: 0, reembedded: 0, skipped: 0 };
+
+  let cursor: string | null = null;
+  let page = 0;
+
+  for (;;) {
+    page += 1;
+    const after = cursor;
+    // Zero-padded so the step names sort the way the pages ran — the journal is read by humans too.
+    const result: PageResult = await step.do(`page-${String(page).padStart(6, "0")}`, async () => {
+      const rows = await deps.documents.page({
+        indexName: deps.indexName,
+        after,
+        limit,
+        ...(options.all ? {} : { staleModel: deps.index.model }),
+      });
+      if (rows.length === 0) return { cursor: after, scanned: 0, reembedded: 0, skipped: 0, done: true };
+
+      const last = rows[rows.length - 1] as VectorDocument;
+      const selected = options.filter
+        ? rows.filter((row) => matchesCompiledFilter(row.metadata, options.filter as CompiledFilter))
+        : rows;
+      const reembedded = await reembedPage(deps, selected);
+
+      return {
+        cursor: last.id,
+        scanned: rows.length,
+        reembedded,
+        skipped: selected.length - reembedded,
+        // A short page means the corpus is exhausted. A full page might still be the last one; the next
+        // step reads zero rows and ends the run, which costs one cheap query and no special case.
+        done: rows.length < limit,
+      };
+    });
+
+    report.pages = page;
+    report.scanned += result.scanned;
+    report.reembedded += result.reembedded;
+    report.skipped += result.skipped;
+    cursor = result.cursor;
+    if (result.done) break;
+  }
+
+  return report;
+}

package/src/workflows/retryPolicy.ts

@@ -0,0 +1,54 @@
+// SPDX-FileCopyrightText: 2026 Pithy
+// SPDX-License-Identifier: MIT
+
+import type { WorkflowRetryPolicy } from "@pithy-sh/core/src/workflow/faults";
+
+/**
+ * **What a reprocess retries, and what it refuses to.**
+ *
+ * A reprocess is a corpus-sized job: one journalled step per keyset page, thousands of them for a
+ * corpus worth re-embedding at all. That shape decides the policy in both directions. A page lost to a
+ * model blip takes the whole instance's remaining pages with it, so an outage must be retryable; and a
+ * refusal that will not change — a model pinned to the wrong dimensions, a filter naming a field the
+ * index cannot filter on — must fail on page one rather than on page one after five attempts
+ * (pithy-sh/pithy#348).
+ *
+ * ## Retryable, and why
+ *
+ * - **`core/upstream_failed`** — Workers AI rejected the embedding call. Raised at the one wrap around
+ *   `ai.run` in `embedTexts`, and it means the model did not answer. Re-embedding a page is idempotent —
+ *   the rows are stamped with the new model only *after* the upsert is accepted, so a page that died
+ *   mid-write is still selectable by the next attempt rather than marked done.
+ * - **A transient D1 fault** — the document corpus: the page read and the `markEmbedded` write.
+ *   Classified in core by `withD1Retry`, never restated here.
+ *
+ * ## Terminal, and why
+ *
+ * - **`vector/dimension_mismatch`** — the configured model produced vectors the index cannot hold. An
+ *   index's dimensions are fixed at creation; this is a config error caught on the first page, which is
+ *   exactly where you want it, and it wants a new index rather than a backoff.
+ * - **`vector/index_not_found`** — no such index in `VECTOR_CONFIG`. Raised before the first step even
+ *   runs, and answered by `pithy vector provision`.
+ * - **`vector/unfilterable_field`, `vector/filter_too_large`, `vector/topk_exceeded`,
+ *   `vector/metadata_too_large`** — limits and shapes. Every one is deterministic in the input, and
+ *   three of them are compiled against the provisioned metadata indexes *before* a single document is
+ *   re-embedded.
+ * - **`vector/metadata_index_drift`** — Vectorize is missing an index the config declares. Provisioning,
+ *   not weather.
+ * - **`core/internal`** — the model or the store answered in a shape nobody recognizes, or the binding
+ *   does not expose the method. A shape is not a transient.
+ * - **`validation/invalid_input`** — an empty batch, a `topK` below one, a namespace the schema refuses.
+ *
+ * **What this cannot say.** A Vectorize `upsert` that fails at the binding throws whatever the binding
+ * threw — not a `PithyError`, so `unclassified`, so terminal. The step stamps `markEmbedded` only after
+ * the upsert is accepted, so the page is re-selected by the next run rather than skipped; the run is
+ * lost, the corpus is not. That is a default rather than a decision, and no policy record can turn it
+ * into one.
+ */
+export const vectorWorkflowRetry: WorkflowRetryPolicy = {
+  capability: "vector",
+  retryable: {
+    "core/upstream_failed":
+      "The embedding model did not answer; a page is re-selectable until its upsert is accepted, so re-driving it repeats nothing.",
+  },
+};