@pithy-sh/cli 0.1.0

package/src/capabilities/vectorProvisioner.ts

@@ -0,0 +1,260 @@
+// SPDX-FileCopyrightText: 2026 Pithy
+// SPDX-License-Identifier: MIT
+
+import { readFile, unlink, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import type { CloudflareClients } from "@pithy-sh/cloudflare/src/client/clients";
+import type { CloudflareWorkflowsClient } from "@pithy-sh/cloudflare/src/workflows/workflowsClient";
+import { InternalError, ValidationError } from "@pithy-sh/core/src/error/pithyError";
+import { resourceNames } from "@pithy-sh/core/src/naming/resourceNames";
+import type { WorkflowHostTemplate } from "@pithy-sh/core/src/workflow/host";
+import { parse } from "comment-json";
+import { runWrangler } from "../project/wrangler";
+import { capabilityLoadError } from "./loadFailure";
+
+/**
+ * The live vector provisioner — the Cloudflare + wrangler implementation behind `@pithy-sh/vector`'s
+ * `VectorProvisioner` seam. Control-plane calls go through `@pithy-sh/cloudflare` (CLAUDE.md: the CF API
+ * only via that client); the worker deploy shells out to wrangler with the bootstrap token.
+ *
+ * `@pithy-sh/vector` is an **optional** capability, so the CLI must not hard-depend on it. Types arrive
+ * through type-only imports (erased at build) and every runtime value through {@link loadVector} — a guarded
+ * dynamic import that turns "the package isn't installed" into an actionable error rather than an
+ * unresolved-module crash.
+ *
+ * **Vectorize is eventually consistent, and this file never assumes otherwise.** Creating an index or a
+ * metadata index is *accepted*, not applied. Where the ordering matters — a metadata index must exist before
+ * any vector is written, or that vector is permanently unfilterable — the provisioner polls the list endpoint
+ * with a bounded retry rather than reading back what it just wrote.
+ */
+
+type VectorProvisionModule = typeof import("@pithy-sh/vector/src/provision/provisionVector");
+type VectorResolveModule = typeof import("@pithy-sh/vector/src/provision/resolveVectorConfig");
+type VectorCapabilityModule = typeof import("@pithy-sh/vector/src/capability");
+type VectorDriftModule = typeof import("@pithy-sh/vector/src/index/drift");
+type VectorProvisionedModule = typeof import("@pithy-sh/vector/src/index/provisioned");
+type VectorSpecsModule = typeof import("@pithy-sh/vector/src/workflows/specs");
+
+/** The seams, referenced by type only so the CLI gains no dependency on the package. */
+type VectorProvisioner = import("@pithy-sh/vector/src/provision/provisionVector").VectorProvisioner;
+type VectorIndexShape = import("@pithy-sh/vector/src/provision/provisionVector").VectorIndexShape;
+type MetadataIndexDescriptor = import("@pithy-sh/vector/src/index/metadata").MetadataIndexDescriptor;
+type MetadataIndexReport = import("@pithy-sh/vector/src/index/drift").MetadataIndexReport;
+type VectorConfig = import("@pithy-sh/vector/src/config/config").VectorConfig;
+
+/** Everything `pithy vector` loads out of the optional package, in one guarded import. */
+export type VectorModule = VectorProvisionModule &
+  VectorResolveModule &
+  VectorCapabilityModule &
+  VectorDriftModule &
+  VectorProvisionedModule &
+  VectorSpecsModule;
+
+/**
+ * Load `@pithy-sh/vector` from the project's own install. The one place the optional dependency is resolved,
+ * so a project that has not added vector gets one clear instruction instead of a module error from whichever
+ * call site happened to run first.
+ */
+export async function loadVector(): Promise<VectorModule> {
+  try {
+    const [provision, resolve, capability, drift, provisioned, specs] = await Promise.all([
+      import("@pithy-sh/vector/src/provision/provisionVector"),
+      import("@pithy-sh/vector/src/provision/resolveVectorConfig"),
+      import("@pithy-sh/vector/src/capability"),
+      import("@pithy-sh/vector/src/index/drift"),
+      import("@pithy-sh/vector/src/index/provisioned"),
+      import("@pithy-sh/vector/src/workflows/specs"),
+    ]);
+    return { ...provision, ...resolve, ...capability, ...drift, ...provisioned, ...specs };
+  } catch (error) {
+    throw capabilityLoadError("vector", "@pithy-sh/vector", error);
+  }
+}
+
+/** The directory of the prebuilt vector worker inside the installed package (holds `wrangler.jsonc`). */
+async function vectorWorkerDir(): Promise<string> {
+  try {
+    return dirname(fileURLToPath(import.meta.resolve("@pithy-sh/vector/src/workflows/worker")));
+  } catch (error) {
+    throw capabilityLoadError("vector", "@pithy-sh/vector/src/workflows/worker", error);
+  }
+}
+
+/** Resolve the app database id for an environment — where the document corpus lives. */
+export type ResolveVectorEnv = (env: string) => Promise<{ appDatabaseId: string }>;
+
+/** How long to wait for an accepted metadata index to become visible, and how often to look. */
+const METADATA_POLL_ATTEMPTS = 10;
+const METADATA_POLL_INTERVAL_MS = 1_000;
+
+export interface CloudflareVectorProvisionerOptions {
+  cf: CloudflareClients;
+  accountId: string;
+  /**
+   * The project name, from `requireProjectName(await loadProject(projectDir))` — never
+   * `resolveProjectName`. The deployed worker and the reprocess Workflow both lead with it, and a
+   * guessed value dispatches into a Workflow name nothing deployed.
+   */
+  project: string;
+  /** The bootstrap token (`.dev.vars` `CLOUDFLARE_API_TOKEN`) that authenticates the worker deploy. */
+  apiToken: string;
+  /** The app's resolved vector config — the indexes to create and the config the worker is deployed with. */
+  config: VectorConfig;
+  /** Resolve the per-env app database id. Injected so it is testable and decoupled from wrangler parsing. */
+  resolveEnv: ResolveVectorEnv;
+  /** The Workflows REST client, for triggering a reprocess run in a deployed environment. */
+  workflows: CloudflareWorkflowsClient;
+  /** Sleep between metadata-index polls. Injected so tests run with no real delay. */
+  sleep?: (ms: number) => Promise<void>;
+}
+
+/** The live {@link VectorProvisioner}. Every step is idempotent, so provisioning is safe to re-run. */
+export class CloudflareVectorProvisioner implements VectorProvisioner {
+  readonly #cf: CloudflareClients;
+  readonly #accountId: string;
+  readonly #project: string;
+  readonly #apiToken: string;
+  readonly #config: VectorConfig;
+  readonly #resolveEnv: ResolveVectorEnv;
+  readonly #workflows: CloudflareWorkflowsClient;
+  readonly #sleep: (ms: number) => Promise<void>;
+
+  constructor(options: CloudflareVectorProvisionerOptions) {
+    this.#cf = options.cf;
+    this.#accountId = options.accountId;
+    this.#project = options.project;
+    this.#apiToken = options.apiToken;
+    this.#config = options.config;
+    this.#resolveEnv = options.resolveEnv;
+    this.#workflows = options.workflows;
+    this.#sleep = options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+  }
+
+  /** Require Vectorize access and a workers.dev subdomain — Cloudflare needs one to deploy a Workflow host. */
+  async preflight(): Promise<void> {
+    if (!(await this.#cf.vectorizeProvisioner().validateServiceAccess())) {
+      throw new ValidationError({
+        message: "This Cloudflare account cannot reach Vectorize.",
+        action: "Vectorize needs a paid Workers plan, and the API token needs Vectorize Read and Write.",
+      });
+    }
+    if (!(await this.#cf.workers().accountSubdomain())) {
+      throw new ValidationError({
+        message: "This Cloudflare account has no workers.dev subdomain, which Workflows require.",
+        action: "Open Workers & Pages in the dashboard once to create one, then re-run.",
+      });
+    }
+  }
+
+  /** Reuse the index if it exists, otherwise create it. An index's shape is fixed, so a mismatch is fatal. */
+  async ensureIndex(indexName: string, shape: VectorIndexShape): Promise<{ name: string }> {
+    const existing = await this.#cf.vectorizeProvisioner().findIndexByName(indexName);
+    if (existing) {
+      if (existing.config.dimensions !== shape.dimensions || existing.config.metric !== shape.metric) {
+        throw new ValidationError({
+          message: `The \`${indexName}\` index already exists with a different shape.`,
+          action:
+            "Dimensions and metric are fixed when an index is created. Change the config back, or run `pithy vector reset` to rebuild the index from the document corpus.",
+          detail: `live ${existing.config.dimensions}/${existing.config.metric}, config ${shape.dimensions}/${shape.metric}`,
+        });
+      }
+      return { name: existing.name };
+    }
+    const created = await this.#cf.vectorizeProvisioner().createIndex(indexName, shape);
+    return { name: created.name };
+  }
+
+  /**
+   * Create every declared metadata index that is missing, then wait until each one is visible.
+   *
+   * The wait is the point. Cloudflare applies a metadata index asynchronously, and a vector written before it
+   * lands is never covered by it — silently, with no error and no way back but a full re-embed. Returning
+   * before the index is live would put the caller's very next step (deploying the worker that writes vectors)
+   * inside that window.
+   */
+  async ensureMetadataIndexes(
+    indexName: string,
+    declared: readonly MetadataIndexDescriptor[],
+  ): Promise<MetadataIndexReport> {
+    const { compareMetadataIndexes } = await loadVector();
+    const provisioner = this.#cf.vectorizeProvisioner();
+
+    const before = compareMetadataIndexes(declared, await provisioner.listMetadataIndexes(indexName));
+    if (before.mismatched.length > 0) {
+      throw new ValidationError({
+        message: `A metadata index on \`${indexName}\` was created with a different type.`,
+        action:
+          "A metadata index's type is fixed. Change the field's type back in the index's metadata schema, or run `pithy vector reset` to rebuild the index.",
+        detail: before.mismatched
+          .map((entry) => `${entry.propertyName}: declared ${entry.declared}, indexed as ${entry.live}`)
+          .join("; "),
+      });
+    }
+    if (before.missing.length === 0) return before;
+
+    for (const descriptor of before.missing) {
+      await provisioner.createMetadataIndex(indexName, descriptor.propertyName, descriptor.indexType);
+    }
+
+    for (let attempt = 0; attempt < METADATA_POLL_ATTEMPTS; attempt += 1) {
+      await this.#sleep(METADATA_POLL_INTERVAL_MS);
+      const now = compareMetadataIndexes(declared, await provisioner.listMetadataIndexes(indexName));
+      if (now.missing.length === 0) return { ...now, missing: [...before.missing] };
+    }
+
+    throw new InternalError({
+      message: `The metadata indexes on \`${indexName}\` were accepted but are not live yet.`,
+      action: "Re-run `pithy vector provision` in a minute. Do not write vectors until it reports them ready.",
+      detail: `waited ${(METADATA_POLL_ATTEMPTS * METADATA_POLL_INTERVAL_MS) / 1000}s for ${before.missing
+        .map((entry) => entry.propertyName)
+        .join(", ")}`,
+    });
+  }
+
+  /** Resolve the env's wrangler config from the committed template + provisioned ids, then `wrangler deploy`. */
+  async deployWorker(env: string, indexNames: Record<string, string>): Promise<void> {
+    const { resolveVectorConfig } = await loadVector();
+    const { appDatabaseId } = await this.#resolveEnv(env);
+    const dir = await vectorWorkerDir();
+    const template = parse(await readFile(join(dir, "wrangler.jsonc"), "utf8")) as unknown as WorkflowHostTemplate;
+    const config = resolveVectorConfig(template, {
+      project: this.#project,
+      env,
+      appDatabaseId,
+      indexNames,
+      config: this.#config,
+    });
+
+    const configPath = join(dir, `.wrangler.${env}.json`);
+    await writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`);
+    try {
+      await runWrangler(["deploy", "--config", configPath], {
+        cwd: dir,
+        env: { CLOUDFLARE_API_TOKEN: this.#apiToken, CLOUDFLARE_ACCOUNT_ID: this.#accountId },
+      });
+    } finally {
+      await unlink(configPath).catch(() => {});
+    }
+  }
+
+  /** Delete an index. Destructive, and idempotent — a missing index is a no-op. */
+  async deleteIndex(indexName: string): Promise<void> {
+    await this.#cf.vectorizeProvisioner().deleteIndex(indexName);
+  }
+
+  /** Trigger the deployed reprocess Workflow for this environment and wait for it to finish. */
+  async reprocess(
+    env: string,
+    index: string,
+    options: { all?: boolean; filter?: Record<string, unknown> },
+  ): Promise<unknown> {
+    const { VECTOR_CAPABILITY } = await loadVector();
+    const name = resourceNames(this.#project).env(env).workflow(VECTOR_CAPABILITY, "reprocess");
+    return this.#workflows.dispatchAndPoll(name, {
+      index,
+      ...(options.all !== undefined ? { all: options.all } : {}),
+      ...(options.filter ? { filter: options.filter } : {}),
+    });
+  }
+}

package/src/ci/fileModes.ts

@@ -0,0 +1,223 @@
+// SPDX-FileCopyrightText: 2026 Pithy
+// SPDX-License-Identifier: MIT
+
+import { execFileSync } from "node:child_process";
+import { chmodSync, lstatSync } from "node:fs";
+import { join } from "node:path";
+
+/**
+ * **The permission bits of this repository's own files, as a rule rather than as a habit.**
+ *
+ * `bun install` chmods `packages/cli/src/bin.ts` to `0777`. It is the only `bin` declared anywhere in
+ * the workspace, `tooling/browser-scopes` depends on `@pithy-sh/cli`, and bun links a workspace bin by
+ * symlinking `node_modules/.bin/pithy` straight at the source file — then makes the target executable
+ * with a mode nobody chose. Not `0755`. `0777` (#345).
+ *
+ * Two harms, and they are not the same size.
+ *
+ * 1. **`rwxrwxrwx` on the program that reads an adopter's dev secrets and holds their Cloudflare
+ *    credentials.** Any local account can rewrite it, and the next `pithy` the user runs is whatever
+ *    that account wrote. This is the one that matters.
+ * 2. **Every worktree dirty at birth.** git records the file `100644`, so `git status` reported
+ *    `mode change 100644 => 100755` in a tree nobody had touched — six of thirteen worktrees on one
+ *    machine at once. A tree that is dirty by default is how an unrelated change gets swept into
+ *    somebody's commit, and this repository already carries that defect class.
+ *
+ * **The exec bit is wanted; `0777` is not.** `bin.ts` opens `#!/usr/bin/env bun` and is declared as a
+ * `bin`, and bun's workspace link points at the source rather than at a shim, so
+ * `node_modules/.bin/pithy` is only runnable if the source file itself is executable. Clearing the bit
+ * would fix `git status` by breaking the link bun had just made. So git records `100755` and the rule
+ * below keeps the mode narrow, rather than the other way round.
+ *
+ * ## The rule, frozen
+ *
+ * For every path git tracks, or that is written and not ignored:
+ *
+ * - **No other-write.** `0777`'s `o+w` is the whole security finding, and no umask a person actually
+ *   runs produces it.
+ * - **No setuid, setgid or sticky bit.** Nothing in a source tree has any use for one.
+ * - **The exec bits agree with what git records** — set for `100755`, clear for `100644`. This is the
+ *   half that keeps `git status` clean, and it only applies to a path git has an opinion about.
+ * - **And a file git records executable is not group-writable either**, so the three programs in this
+ *   tree that something else runs by path land at `0755` and no wider.
+ *
+ * **Group-write is permitted everywhere else, and that is a limit rather than a decision.** git checks a
+ * file out at `0666 & ~umask`, so under this repository's own reproduction umask of `0002` all 2,368
+ * ordinary tracked files land at `0664`. A rule banning `g+w` across the tree would be red on files git
+ * itself had just written, red again after every `git switch` and every `git pull`, and muted within a
+ * day — and a muted gate is the defect it was built to catch, shipping. So the tree-wide invariant is the
+ * one that survives any umask a person runs, and the tighter one is spent where it costs least and buys
+ * most: three files, rewritten by git only when they change, each of them a program.
+ *
+ * ## Where it runs
+ *
+ * Both halves, from one rule. {@link violations} is the gate in `./fileModes.test.ts`; {@link repair}
+ * is what the repo root's `postinstall` runs, so the tree is corrected by the same command that breaks
+ * it and an already-affected worktree is fixed by re-running `bun install` rather than by hand.
+ *
+ * **Builtins only, on purpose.** This module is executed by `bun install` itself, through
+ * `scripts/fileModes.ts`. A bare specifier here would be resolved through the very `node_modules` the
+ * install is still assembling, which is the one import graph a postinstall hook cannot assume. It is
+ * also why the errors below are plain — this is build-time tooling in the same class as
+ * `scripts/worktree.ts`, not the shipped runtime that owes a `PithyError`.
+ */
+
+/** What `git ls-files -s` writes for a regular file. */
+const RECORDED_REGULAR = "100644";
+
+/** And for one git records executable. */
+const RECORDED_EXECUTABLE = "100755";
+
+/** The byte `-z` separates paths with. Never a literal one; a path may hold anything else. */
+const NUL = String.fromCharCode(0);
+
+/** Every permission bit a chmod can carry, `setuid` through `other-execute`. */
+const PERMISSIONS = 0o7777;
+
+/** setuid, setgid and sticky. Nothing in a source tree has any use for one. */
+const SPECIAL = 0o7000;
+
+/** Write for other. The whole of the security finding in #345, and no umask anyone runs produces it. */
+const OTHER_WRITE = 0o002;
+
+/** Write for group. Refused on a file git records executable, permitted on the rest. See above. */
+const GROUP_WRITE = 0o020;
+
+/** The three execute bits. */
+const EXECUTE = 0o111;
+
+/** The owner's, which is the one a file git records executable always gets back. */
+const OWNER_EXECUTE = 0o100;
+
+/** The three read bits — where an execute bit belongs, when one belongs at all. */
+const READ = 0o444;
+
+/** One path, what git records about it, and what the filesystem says. */
+export interface TrackedFile {
+  /** Repository-relative, forward slashes, exactly as git wrote it. */
+  readonly path: string;
+  /**
+   * Whether git records the file executable, or `null` for a path git records nothing about — one
+   * that is written and not ignored, and so has no recorded mode to agree with yet.
+   */
+  readonly recorded: boolean | null;
+  /** The on-disk permission bits, or `null` when the path was not there to stat. */
+  readonly mode: number | null;
+}
+
+/** Run git in `root` and hand back what it wrote. */
+function git(root: string, args: string[]): string[] {
+  const written = execFileSync("git", args, { cwd: root, encoding: "utf8", maxBuffer: 64 * 1024 * 1024 });
+  return written.split(NUL).filter((entry) => entry !== "");
+}
+
+/**
+ * What git records for one index entry.
+ *
+ * **An unrecognised mode throws rather than being skipped.** git can record a symlink (`120000`) and a
+ * submodule (`160000`), and this repository has neither; the rule above has no sentence about either
+ * one, and a gate that quietly drops what it has no rule for is a gate that shrinks as the tree grows.
+ * Whoever adds the first one gets to decide what the rule is, once, out loud.
+ */
+export function recordedExecutable(mode: string, path: string): boolean {
+  if (mode === RECORDED_REGULAR) return false;
+  if (mode === RECORDED_EXECUTABLE) return true;
+  throw new Error(`git records ${path} as mode ${mode}, and packages/cli/src/ci/fileModes.ts has no rule for it.`);
+}
+
+/** `path`'s permission bits, or `null` if it is not there. `lstat`, so a link is never followed. */
+function onDisk(path: string): number | null {
+  try {
+    return lstatSync(path).mode & PERMISSIONS;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Every path git carries: the index, **plus what is written and not ignored**.
+ *
+ * The index alone would make the world-writable half of the rule one run late by construction — a file
+ * exists before anything stages it, and the pre-commit hook runs Biome rather than this suite. The
+ * second listing closes that, and closes it without noise: `--exclude-standard` drops `node_modules`,
+ * `dist`, and the whole-project scaffolds this repository's own suites write into `packages/cli/.smoke-*`
+ * and `.e2e-*`. The same argument `sourceFiles.test.ts` makes about a NUL byte, about a mode.
+ */
+export function trackedFiles(root: string): TrackedFile[] {
+  const found: TrackedFile[] = [];
+  for (const entry of git(root, ["ls-files", "-s", "-z"])) {
+    // `<mode> <sha> <stage>\t<path>` — and a path may hold a tab, so the first one ends the header.
+    const tab = entry.indexOf("\t");
+    if (tab < 0) throw new Error(`git ls-files -s wrote an entry with no path: ${entry}`);
+    const path = entry.slice(tab + 1);
+    const record = recordedExecutable(entry.slice(0, entry.indexOf(" ")), path);
+    found.push({ path, recorded: record, mode: onDisk(join(root, path)) });
+  }
+  for (const path of git(root, ["ls-files", "-z", "--others", "--exclude-standard"])) {
+    found.push({ path, recorded: null, mode: onDisk(join(root, path)) });
+  }
+  return found.sort((left, right) => (left.path < right.path ? -1 : 1));
+}
+
+/**
+ * What `file` should have been, given what git records about it.
+ *
+ * **It only ever narrows.** For an ordinary file it is not `chmod 644`: forcing an absolute mode would
+ * fight the umask of whoever ran it and then disagree with what `git checkout` writes on the next branch
+ * switch, so only the bits the rule refuses are cleared and `0664` stays `0664`. For a file git records
+ * executable there is an absolute answer, and it is `0755` — the exec bits go back wherever a read bit
+ * is, group-write comes off, and `0777` lands where the issue asked for it. Running it twice changes
+ * nothing either way.
+ */
+export function repairedMode(recordedExecutable: boolean, mode: number): number {
+  const forbidden = SPECIAL | OTHER_WRITE | (recordedExecutable ? GROUP_WRITE : 0);
+  const narrowed = mode & ~forbidden & ~EXECUTE;
+  if (!recordedExecutable) return narrowed;
+  // The owner's bit unconditionally, because a file git records executable and nobody can read — `0000`,
+  // and the rest of that corner — would otherwise come back still not executable, which is the one thing
+  // the repair was asked for. Everyone else gets one only where they already have a read bit.
+  return narrowed | OWNER_EXECUTE | ((narrowed & READ) >> 2);
+}
+
+/** Why one file breaks the rule, in the words the gate prints. Null when it does not. */
+export function offense(file: TrackedFile): string | null {
+  if (file.mode === null) return null;
+  const found: string[] = [];
+  const octal = `0${file.mode.toString(8).padStart(3, "0")}`;
+  if ((file.mode & SPECIAL) !== 0) found.push(`${octal} carries a setuid, setgid or sticky bit`);
+  if ((file.mode & OTHER_WRITE) !== 0) found.push(`${octal} is world-writable`);
+  if (file.recorded === true && (file.mode & GROUP_WRITE) !== 0) {
+    found.push(`${octal} is group-writable and git records it executable`);
+  }
+  if (file.recorded !== null && ((file.mode & EXECUTE) !== 0) !== file.recorded) {
+    const records = file.recorded ? "100755" : "100644";
+    found.push(`${octal} is ${file.recorded ? "not " : ""}executable and git records it ${records}`);
+  }
+  return found.length === 0 ? null : found.join("; ");
+}
+
+/** Every file breaking the rule, path → why. Empty is what passing looks like. */
+export function violations(files: readonly TrackedFile[]): Record<string, string> {
+  const found: Record<string, string> = {};
+  for (const file of files) {
+    const why = offense(file);
+    if (why !== null) found[file.path] = why;
+  }
+  return found;
+}
+
+/**
+ * Narrow every file that breaks the rule, and say which ones those were.
+ *
+ * Only the offenders are touched. Chmodding all ~2,371 tracked files on every install would churn a
+ * tree that is already correct and hide, in the noise, the one file that was not.
+ */
+export function repair(root: string): string[] {
+  const fixed: string[] = [];
+  for (const file of trackedFiles(root)) {
+    if (file.mode === null || offense(file) === null) continue;
+    chmodSync(join(root, file.path), repairedMode(file.recorded === true, file.mode));
+    fixed.push(file.path);
+  }
+  return fixed;
+}