@pithy-sh/vector 0.1.0

package/src/workflows/specs.ts

@@ -0,0 +1,71 @@
+// SPDX-FileCopyrightText: 2026 Pithy
+// SPDX-License-Identifier: MIT
+
+import { workflowKey } from "@pithy-sh/core/src/workflow/naming";
+import type { WorkflowRegistry, WorkflowSpecMap } from "@pithy-sh/core/src/workflow/spec";
+import { z } from "zod";
+
+/**
+ * The one durable job vector owns, declared once.
+ *
+ * Re-embedding a corpus is the job that has to be a Workflow. Ten million vectors is an ordinary size for a
+ * Vectorize index, and re-embedding them is hours of Workers AI calls: a request would time out, a queue
+ * consumer would lose its place, and a script on someone's laptop would die with the lid. A Workflow's steps
+ * are journalled, so an interrupted run resumes at the page it reached rather than starting over — and
+ * starting over is not merely slow, it is what makes people not re-embed at all.
+ *
+ * The spec is `optional: true`: the binding exists only once `pithy vector provision` has deployed the
+ * reprocess worker, and a project that has not provisioned must still boot and serve every search route.
+ */
+
+/** The capability name — the first segment of the dispatch key and of every deployed workflow name. */
+export const VECTOR_CAPABILITY = "vector";
+
+/** The parameters a reprocess run takes: which index, how much of it, and how it is scoped. */
+export const VectorReprocessParams = z
+  .object({
+    index: z.string().min(1).describe("The index to re-embed, as named in pithy.config.ts. One run covers one index."),
+    all: z
+      .boolean()
+      .optional()
+      .describe(
+        "Re-embed every document, not just the drifted ones. The default pass selects only rows whose model differs from the configured one — including rows that were never embedded at all.",
+      ),
+    filter: z
+      .record(z.string(), z.unknown())
+      .optional()
+      .describe(
+        "Narrow the run to documents whose metadata matches this filter. Evaluated against the corpus rows, so it uses the same operators a query filter does.",
+      ),
+    pageSize: z
+      .number()
+      .int()
+      .min(1)
+      .optional()
+      .describe(
+        "Documents per Workflow step. Defaults to the Vectorize upsert ceiling of 1,000, which is also the largest batch one step can write.",
+      ),
+  })
+  .describe("The instance parameters of a reprocess run — which index to re-embed, and how much of it.");
+export type VectorReprocessParams = z.infer<typeof VectorReprocessParams>;
+
+/** Vector's durable jobs, keyed by job name. The key is the second segment of the `vector/<job>` dispatch key. */
+export const vectorWorkflows = {
+  reprocess: {
+    binding: "VECTOR_REPROCESS",
+    className: "VectorReprocessWorkflow",
+    params: VectorReprocessParams,
+    optional: true,
+  },
+} as const satisfies WorkflowSpecMap;
+
+/**
+ * Vector's jobs as a dispatch registry, keyed `vector/<job>`. Built from the same map and core's
+ * {@link workflowKey}, so the dispatch key, the deployed name, and the binding cannot drift apart.
+ */
+export const vectorWorkflowRegistry: WorkflowRegistry = Object.fromEntries(
+  Object.entries(vectorWorkflows).map(([job, spec]) => {
+    const key = workflowKey(VECTOR_CAPABILITY, job);
+    return [key, { key, capability: VECTOR_CAPABILITY, job, spec }];
+  }),
+);

package/src/workflows/worker.ts

@@ -0,0 +1,130 @@
+// SPDX-FileCopyrightText: 2026 Pithy
+// SPDX-License-Identifier: MIT
+
+import { WorkflowEntrypoint, type WorkflowEvent, type WorkflowStep } from "cloudflare:workers";
+import { NonRetryableError } from "cloudflare:workflows";
+import type { D1Database } from "@cloudflare/workers-types";
+import { InternalError } from "@pithy-sh/core/src/error/pithyError";
+import { classifiedSteps } from "@pithy-sh/core/src/workflow/faults";
+import { workflowHostEntry } from "@pithy-sh/core/src/workflow/hostEntry";
+import type { VectorIndexConfig } from "../config/config";
+import { VectorWorkerConfig, type VectorWorkerIndex } from "../config/workerConfig";
+import { vectorDocuments } from "../data/documents";
+import { vectorDatabase } from "../data/tables";
+import type { VectorAi } from "../embed/embed";
+import { VectorIndexNotFoundError } from "../error/errors";
+import { compileFilter } from "../index/filter";
+import type { VectorStore } from "../index/index";
+import { type ReprocessReport, reprocessIndex } from "./reprocess";
+import { vectorWorkflowRetry } from "./retryPolicy";
+import { VECTOR_CAPABILITY, VectorReprocessParams } from "./specs";
+
+/**
+ * The prebuilt vector worker. `pithy vector provision` deploys one per environment; the adopter authors no
+ * code for it. It hosts the reprocess Workflow — a thin durable shell around the tested orchestration in
+ * `reprocess.ts` — and nothing else. Every search route lives in the app worker.
+ *
+ * This module imports `cloudflare:workers`, so it runs only in the Workers runtime and is excluded from the
+ * node meta-test glob.
+ *
+ * Its config arrives as `VECTOR_CONFIG`, a **projection** of the capability's config rather than the config
+ * itself: an index's `metadata` is a live Zod schema, which cannot travel through a wrangler var. The
+ * projection carries the introspected filterable descriptors instead, which is all this worker needs to
+ * validate a `--filter`.
+ *
+ * **The default export is what makes this an ES module** (#426). It exports one Workflow class and has no
+ * cron, so until now it exported no default — and wrangler infers a worker's module format from exactly
+ * that, so the build read it as a service worker and refused `cloudflare:workers` outright. The host did not
+ * build, `pithy dev` carried on past it, and a reprocess dispatched at it went nowhere. The refusal it
+ * exports is the honest body for a host with no request surface; see `@pithy-sh/core/src/workflow/hostEntry`.
+ */
+
+/** The vector worker's env: the corpus database, the AI binding, its config, and one Vectorize binding per index. */
+export interface VectorWorkerEnv {
+  /** The app database the document corpus lives in. */
+  DB: D1Database;
+  /** The Workers AI binding. Marked remote in the host config — Workers AI has no local emulation. */
+  AI: VectorAi;
+  /** The projected vector config as one JSON string, filled at provision. */
+  VECTOR_CONFIG?: string;
+  /** Each index's Vectorize binding, addressed by the name its config declares. */
+  [binding: string]: unknown;
+}
+
+/** Parse the projected config off the env. A worker with no config cannot do anything but say so. */
+function workerConfig(env: VectorWorkerEnv): VectorWorkerConfig {
+  if (!env.VECTOR_CONFIG) {
+    throw new InternalError({
+      message: "The vector worker has no configuration.",
+      action: "Run `pithy vector provision` — it deploys this worker with its config.",
+      detail: "VECTOR_CONFIG was absent from the worker env",
+    });
+  }
+  return VectorWorkerConfig.parse(JSON.parse(env.VECTOR_CONFIG));
+}
+
+/** The index's config in the shape the shared code takes. Metadata is absent by design — see the file note. */
+function indexConfig(index: VectorWorkerIndex): VectorIndexConfig {
+  return {
+    model: index.model,
+    dimensions: index.dimensions,
+    metric: index.metric,
+    binding: index.binding,
+    ...(index.namespace !== undefined ? { namespace: index.namespace } : {}),
+  };
+}
+
+/** Re-embed a corpus, one journalled page per step. */
+export class VectorReprocessWorkflow extends WorkflowEntrypoint<VectorWorkerEnv, VectorReprocessParams> {
+  override async run(event: WorkflowEvent<VectorReprocessParams>, step: WorkflowStep): Promise<ReprocessReport> {
+    const params = VectorReprocessParams.parse(event.payload);
+    const config = workerConfig(this.env);
+
+    const index = config.indexes[params.index];
+    if (!index) {
+      throw new VectorIndexNotFoundError({
+        detail: `no index '${params.index}' in VECTOR_CONFIG; deployed: ${Object.keys(config.indexes).join(", ") || "none"}`,
+      });
+    }
+
+    const store = this.env[index.binding] as VectorStore | undefined;
+    if (!store) {
+      throw new InternalError({
+        message: "The vector worker cannot reach that index.",
+        action: "Re-run `pithy vector provision` so the worker binds every configured index.",
+        detail: `the \`${index.binding}\` Vectorize binding was not present on the worker env`,
+      });
+    }
+
+    // Compiled against the *provisioned* metadata indexes, so a filter naming a field this index cannot
+    // filter on fails before a single document is re-embedded.
+    const filter = params.filter ? compileFilter(index.filterable, params.filter) : undefined;
+
+    return reprocessIndex(
+      {
+        documents: vectorDocuments(vectorDatabase(this.env.DB)),
+        store,
+        ai: this.env.AI,
+        index: indexConfig(index),
+        indexName: params.index,
+        now: () => new Date(),
+      },
+      // Under `vectorWorkflowRetry`: an unreachable embedding model re-drives the page, and a pinned
+      // dimension, a filter the index cannot answer, or a shape nobody recognizes fails on page one —
+      // which is where a config error belongs in a job thousands of pages long.
+      classifiedSteps(step, vectorWorkflowRetry, NonRetryableError),
+      {
+        ...(params.all !== undefined ? { all: params.all } : {}),
+        ...(filter ? { filter } : {}),
+        ...(params.pageSize !== undefined ? { pageSize: params.pageSize } : {}),
+      },
+    );
+  }
+}
+
+/**
+ * The module's default export, and therefore its format. See `hostEntry` for why a Workflow host needs one
+ * at all, and why this one refuses rather than being empty: nothing reaches this worker over HTTP — search
+ * lives in the app worker, and a re-embed starts on the `VECTOR_REPROCESS` binding.
+ */
+export default workflowHostEntry(VECTOR_CAPABILITY);

package/src/workflows/wrangler.jsonc

@@ -0,0 +1,42 @@
+{
+  // The prebuilt vector worker. Like the email and media workers, this is a TEMPLATE, not a wrangler
+  // env-stanza file: each environment is a separate worker. `pithy vector provision` resolves it into one
+  // complete config per environment — filling the `<...>` placeholders and the per-index Vectorize bindings
+  // — and deploys each with `wrangler deploy --config <resolved>`. The adopter authors none of it.
+  // Resolved per project and env → <project>-staging-vector / <project>-prod-vector. Worker
+  // script names are account-scoped, so the project segment is what stops a second Pithy project's
+  // deploy overwriting this one's running worker instead of colliding with it.
+  "name": "pithy-vector",
+  "main": "./worker.ts",
+  // The compatibility date every Worker in this repository runs on. Stated once in the repository
+  // root's `compatibility.ts` and copied here because JSONC cannot import it —
+  // `cli/src/ci/compatibilityDates.test.ts` fails on any Worker older than it.
+  "compatibility_date": "2026-06-01",
+  "compatibility_flags": ["nodejs_compat"],
+
+  // No public URL. Search lives in the app worker; this one is reached only by Workflow dispatch.
+  "workers_dev": false,
+
+  // The app database — the document corpus a re-embed reads its text out of.
+  "d1_databases": [{ "binding": "DB", "database_name": "pithy-app", "database_id": "<filled-at-provision>" }],
+
+  // One entry per configured index, rewritten at provision from the config's `binding` names. `remote` is
+  // not optional here: Vectorize has no local emulation, and a Workflow host always runs locally in dev.
+  "vectorize": [{ "binding": "VECTORIZE", "index_name": "<filled-at-provision>", "remote": true }],
+
+  // Workers AI — the embedding model. Remote for the same reason.
+  "ai": { "binding": "AI", "remote": true },
+
+  // The one Workflow this worker hosts. `class_name` matches the exported WorkflowEntrypoint subclass;
+  // `binding` is what the app worker and `pithy vector reprocess` dispatch to.
+  "workflows": [
+    { "binding": "VECTOR_REPROCESS", "name": "pithy-vector-reprocess", "class_name": "VectorReprocessWorkflow" }
+  ],
+
+  "vars": {
+    // The projected vector config as one JSON blob, filled at provision. A projection, not the config: an
+    // index's metadata is a Zod schema, and a schema cannot travel through a wrangler var.
+    "VECTOR_CONFIG": "<filled-at-provision>",
+    "ENVIRONMENT": "<filled-at-provision>"
+  }
+}