@dench.com/cli 3.7.0-staging.387 → 3.8.1-staging.390

package/environments.ts

@@ -0,0 +1,694 @@
+import { randomUUID } from "node:crypto";
+import {
+  lstat,
+  mkdtemp,
+  readdir,
+  readFile,
+  rename,
+  rm,
+  writeFile,
+} from "node:fs/promises";
+import { homedir, tmpdir } from "node:os";
+import { join } from "node:path";
+import type { ConvexHttpClient } from "convex/browser";
+import { makeFunctionReference } from "convex/server";
+import { CliArgError } from "./lib/cli-args";
+import { fetchEnvironmentBytes, sha256 } from "./lib/environment-archive";
+import {
+  integerOption,
+  isEnvironmentHelp,
+  parseEnvironmentOptions,
+  requireEnvironmentName,
+} from "./lib/environment-cli-args";
+import {
+  type EnvironmentRevision,
+  type EnvironmentSourceBundle,
+  isForbiddenEnvironmentSourcePath,
+  MAX_ENVIRONMENT_FILES,
+  MAX_ENVIRONMENT_SOURCE_BYTES,
+} from "./lib/environment-contract";
+import {
+  acquireEnvironmentWorkspaceLock,
+  activatedEnvironment,
+  EnvironmentOutputRedactor,
+  environmentPrivatePath,
+  environmentSecretMatches,
+  prepareEnvironment,
+  type ResolvedEnvironment,
+  runEnvironmentProcess,
+} from "./lib/environment-execution";
+import {
+  collectEnvironmentSource,
+  validateSourceBundle,
+} from "./lib/environment-source";
+
+const HELP = `dench env — verified setups reused across chats
+
+  dench env list [--limit N] [--cursor CURSOR] [--json]
+  dench env show NAME [--revision SHA256] [--json]
+  dench env save NAME --path DIRECTORY [--wait SECONDS | --no-wait] [--json]
+  dench env run NAME [--revision SHA256] [--state-key KEY] [--prepare-local] [--json] -- COMMAND [ARGS...]
+  dench env delete NAME [--json]
+
+Save requires dench-env.json, SKILL.md, an explicit files allowlist and verify
+commands. Credentials, caches and symlinks are excluded from source uploads.
+Save returns an authoritative preparation receipt; default observation is 10s.
+Ready runs reuse prepared dependencies. --prepare-local explicitly prepares a
+separate local platform cache once; ordinary ready runs never reinstall.
+State is job-local by default; choose an explicit state key to share it.
+Example: dench env run openai-report -- node report.mjs`;
+export function environmentHelpText(): string {
+  return HELP;
+}
+
+export function validateEnvironmentArgs(input: readonly string[]) {
+  if (isEnvironmentHelp(input)) return { verb: "help" as const };
+  const verb = input[0];
+  if (!verb || !["list", "show", "save", "run", "delete"].includes(verb))
+    throw new CliArgError(
+      "Use dench env list, show, save, run, or delete; see --help.",
+    );
+  const values =
+    verb === "list"
+      ? ["--limit", "--cursor"]
+      : verb === "show"
+        ? ["--revision"]
+        : verb === "save"
+          ? ["--path", "--wait"]
+          : verb === "run"
+            ? ["--revision", "--state-key"]
+            : [];
+  const booleans = [
+    "--json",
+    ...(verb === "save" ? ["--no-wait"] : []),
+    ...(verb === "run" ? ["--prepare-local"] : []),
+  ];
+  const parsed = parseEnvironmentOptions(input.slice(1), values, booleans);
+  if (
+    parsed.positional.length !== (verb === "list" ? 0 : 1) ||
+    (verb !== "run" && parsed.child.length)
+  )
+    throw new CliArgError(
+      `Unexpected or missing arguments for dench env ${verb}.`,
+    );
+  const name =
+    verb === "list" ? undefined : requireEnvironmentName(parsed.positional[0]);
+  const revision = parsed.flags["--revision"] as string | undefined;
+  if (revision !== undefined && !/^[a-f0-9]{64}$/.test(revision))
+    throw new CliArgError(
+      "--revision must be the SHA-256 revision from env show.",
+    );
+  const stateKey = parsed.flags["--state-key"] as string | undefined;
+  if (
+    stateKey !== undefined &&
+    !/^[a-zA-Z0-9][a-zA-Z0-9:_-]{0,159}$/.test(stateKey)
+  )
+    throw new CliArgError(
+      "--state-key must contain 1–160 letters, numbers, colons, underscores or hyphens, beginning with a letter or number.",
+    );
+  if (verb === "save" && !parsed.flags["--path"])
+    throw new CliArgError(
+      "env save requires --path DIRECTORY containing dench-env.json and SKILL.md.",
+    );
+  if (parsed.flags["--wait"] !== undefined && parsed.flags["--no-wait"])
+    throw new CliArgError("Use --wait or --no-wait, not both.");
+  if (verb === "run" && !parsed.child.length)
+    throw new CliArgError("env run requires -- COMMAND [ARGS...].");
+  return {
+    verb: verb as "list" | "show" | "save" | "run" | "delete",
+    name,
+    revision,
+    stateKey,
+    limit: integerOption(parsed.flags["--limit"], "--limit", 20, 1, 100),
+    cursor: parsed.flags["--cursor"] as string | undefined,
+    path: parsed.flags["--path"] as string | undefined,
+    wait: parsed.flags["--no-wait"]
+      ? 0
+      : integerOption(parsed.flags["--wait"], "--wait", 10, 0, 600),
+    prepareLocal: Boolean(parsed.flags["--prepare-local"]),
+    json: Boolean(parsed.flags["--json"]),
+    command: parsed.child,
+  };
+}
+
+type ClientOptions = {
+  convex: ConvexHttpClient;
+  args: string[];
+  sessionToken?: string;
+  apiUrl?: string;
+  execution?: { workspace?: string; cacheRoot?: string };
+};
+const queryRef = (name: string) =>
+  makeFunctionReference<"query">(`functions/environments:${name}`);
+const mutationRef = (name: string) =>
+  makeFunctionReference<"mutation">(`functions/environments:${name}`);
+const actionRef = (name: string) =>
+  makeFunctionReference<"action">(`functions/environmentsNode:${name}`);
+
+async function uploadBundle(
+  opts: ClientOptions,
+  kind: "source" | "state",
+  bytes: Buffer,
+) {
+  const token = opts.sessionToken ? { sessionToken: opts.sessionToken } : {};
+  const hash = sha256(bytes);
+  const grant = (await opts.convex.mutation(mutationRef("requestUpload"), {
+    ...token,
+    kind,
+    size: bytes.length,
+    sha256: hash,
+  })) as { uploadId: string; uploadUrl: string; prefix: string };
+  const url = new URL(grant.uploadUrl);
+  if (
+    url.protocol !== "https:" &&
+    !(
+      url.protocol === "http:" &&
+      ["localhost", "127.0.0.1", "[::1]"].includes(url.hostname)
+    )
+  )
+    throw new Error("invalid_upload_url: Upload URL must use HTTPS.");
+  const response = await fetch(grant.uploadUrl, {
+    method: "POST",
+    headers: { "Content-Type": "application/octet-stream" },
+    body: new Uint8Array(
+      Buffer.concat([Buffer.from(grant.prefix, "utf8"), bytes]),
+    ),
+    redirect: "error",
+    signal: AbortSignal.timeout(120_000),
+  });
+  if (!response.ok)
+    throw new Error(
+      "environment_upload_failed: No environment or state was published.",
+    );
+  const result = (await response.json()) as { storageId?: string };
+  if (!result.storageId)
+    throw new Error(
+      "environment_upload_failed: Upload did not return a storage receipt.",
+    );
+  return {
+    uploadId: grant.uploadId,
+    storageId: result.storageId,
+    sha256: hash,
+  };
+}
+
+async function captureState(
+  environment: ResolvedEnvironment,
+  workspace: string,
+  secretValues: string[],
+): Promise<Buffer> {
+  const files: EnvironmentSourceBundle["files"] = [];
+  const seen = new Set<string>();
+  // Expand after command completion, once refreshed credential-file values
+  // have joined the bound values. Never upload known tokens as plaintext state.
+  const secretMatches = environmentSecretMatches(secretValues).map((value) =>
+    Buffer.from(value),
+  );
+  let bytes = 0;
+  const secretPaths = environment.manifest.secretFiles.map(
+    (secret) => secret.path,
+  );
+  const visit = async (path: string) => {
+    if (seen.has(path)) return;
+    seen.add(path);
+    if (
+      isForbiddenEnvironmentSourcePath(path) ||
+      secretPaths.some(
+        (secret) => path === secret || path.startsWith(`${secret}/`),
+      )
+    )
+      throw new Error(
+        "environment_state_rejected: Credential/cache paths cannot be saved as application state.",
+      );
+    const absolute = await environmentPrivatePath(workspace, path);
+    let stat: Awaited<ReturnType<typeof lstat>>;
+    try {
+      stat = await lstat(absolute);
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code === "ENOENT") return;
+      throw error;
+    }
+    if (stat.isDirectory()) {
+      for (const child of (await readdir(absolute)).sort())
+        await visit(`${path}/${child}`);
+      return;
+    }
+    if (!stat.isFile() || stat.size > MAX_ENVIRONMENT_SOURCE_BYTES)
+      throw new Error(
+        "environment_state_rejected: State must contain bounded regular files.",
+      );
+    const value = await readFile(absolute);
+    if (secretMatches.some((secret) => value.includes(secret)))
+      throw new Error(
+        "environment_state_rejected: State contains a bound credential; use a mutable encrypted credential file instead.",
+      );
+    bytes += value.length;
+    if (
+      bytes > MAX_ENVIRONMENT_SOURCE_BYTES ||
+      files.length >= MAX_ENVIRONMENT_FILES
+    )
+      throw new Error(
+        "environment_state_rejected: State exceeds its file or byte limit.",
+      );
+    files.push({
+      path,
+      contentBase64: value.toString("base64"),
+      mode: stat.mode & 0o111 ? 493 : 420,
+    });
+  };
+  for (const path of environment.manifest.statePaths) await visit(path);
+  return Buffer.from(
+    JSON.stringify(validateSourceBundle({ version: 1, files })),
+    "utf8",
+  );
+}
+
+async function restoreState(
+  environment: ResolvedEnvironment,
+  workspace: string,
+) {
+  if (!environment.state?.url) return;
+  const payload = await fetchEnvironmentBytes(
+    environment.state.url,
+    MAX_ENVIRONMENT_SOURCE_BYTES + 512,
+  );
+  const prefix = Buffer.from(environment.state.prefix ?? "", "utf8");
+  const bytes = payload.subarray(prefix.length);
+  if (
+    !payload.subarray(0, prefix.length).equals(prefix) ||
+    !environment.state.sha256 ||
+    sha256(bytes) !== environment.state.sha256
+  )
+    throw new Error(
+      "invalid_environment_state: Saved state failed integrity verification.",
+    );
+  const bundle = validateSourceBundle(JSON.parse(bytes.toString("utf8")));
+  for (const file of bundle.files) {
+    if (
+      !environment.manifest.statePaths.some(
+        (path) => file.path === path || file.path.startsWith(`${path}/`),
+      ) ||
+      environment.manifest.secretFiles.some(
+        (secret) =>
+          file.path === secret.path || file.path.startsWith(`${secret.path}/`),
+      )
+    )
+      throw new Error(
+        "invalid_environment_state: State contains an undeclared or credential path.",
+      );
+  }
+  for (const relative of environment.manifest.statePaths)
+    await rm(await environmentPrivatePath(workspace, relative), {
+      recursive: true,
+      force: true,
+    });
+  for (const file of bundle.files)
+    await writeFile(
+      await environmentPrivatePath(workspace, file.path),
+      Buffer.from(file.contentBase64, "base64"),
+      { mode: file.mode, flag: "wx" },
+    );
+}
+
+async function executeSavedEnvironment(
+  opts: ClientOptions,
+  parsed: Exclude<ReturnType<typeof validateEnvironmentArgs>, { verb: "help" }>,
+) {
+  const token = opts.sessionToken ? { sessionToken: opts.sessionToken } : {};
+  const job =
+    process.env.DENCH_HARNESS_A_JOB_ID ?? process.env.DENCH_HARNESS_JOB_ID;
+  const stateKey =
+    parsed.stateKey ??
+    (job ? `job:${job}` : `local:${sha256(process.cwd()).slice(0, 32)}`);
+  const environment = (await opts.convex.action(actionRef("resolve"), {
+    ...token,
+    name: parsed.name,
+    ...(parsed.revision ? { revision: parsed.revision } : {}),
+    stateKey,
+  })) as ResolvedEnvironment;
+  const suppliedWorkspace =
+    opts.execution?.workspace ?? process.env.DENCH_ENVIRONMENT_WORKSPACE;
+  const workspace =
+    suppliedWorkspace ??
+    (job
+      ? join(
+          process.env.HOME ?? homedir(),
+          ".dench",
+          "environment-workspaces",
+          sha256(environment.organizationId).slice(0, 24),
+          environment.revision,
+        )
+      : await mkdtemp(join(tmpdir(), "dench-env-run-")));
+  const privatePaths: string[] = [];
+  const secretValues = [
+    ...Object.values(environment.credentialEnv).map((entry) => entry.value),
+    ...environment.credentialFiles.map((entry) => entry.value),
+  ];
+  const sanitizeError = (error: unknown) => {
+    const redactor = new EnvironmentOutputRedactor(secretValues);
+    const message = redactor.push(
+      error instanceof Error ? error.message : String(error),
+      true,
+    );
+    const data = (
+      error as {
+        data?: { code?: string; receiptId?: string; retryable?: boolean };
+      }
+    )?.data;
+    return Object.assign(new Error(message), {
+      data: {
+        message,
+        ...(data?.code ? { code: data.code } : {}),
+        ...(data?.receiptId ? { receiptId: data.receiptId } : {}),
+        ...(typeof data?.retryable === "boolean"
+          ? { retryable: data.retryable }
+          : {}),
+      },
+    });
+  };
+  let completedCommand:
+    | Awaited<ReturnType<typeof runEnvironmentProcess>>
+    | undefined;
+  let pendingState: Buffer | undefined;
+  let preserveWorkspace = false;
+  const releaseExecution =
+    environment.manifest.secretFiles.length ||
+    environment.manifest.statePaths.length
+      ? await acquireEnvironmentWorkspaceLock(workspace)
+      : undefined;
+  try {
+    if (releaseExecution) {
+      // Another command may have committed state or rotated credentials while
+      // this process waited. Read again under the lock, retaining this revision.
+      Object.assign(
+        environment,
+        await opts.convex.action(actionRef("resolve"), {
+          ...token,
+          name: environment.name,
+          revision: environment.revision,
+          stateKey,
+        }),
+      );
+      secretValues.push(
+        ...Object.values(environment.credentialEnv).map((entry) => entry.value),
+        ...environment.credentialFiles.map((entry) => entry.value),
+      );
+    }
+    const prepared = await prepareEnvironment(environment, {
+      workspace,
+      prepareLocal: parsed.prepareLocal,
+      cacheRoot: opts.execution?.cacheRoot,
+      report: (text) => process.stderr.write(`${text}\n`),
+    });
+    await restoreState(environment, prepared.workspace);
+    for (const secret of environment.credentialFiles) {
+      const path = await environmentPrivatePath(
+        prepared.workspace,
+        secret.path,
+      );
+      privatePaths.push(path);
+      const temporary = `${path}.dench-secret-${randomUUID()}`;
+      try {
+        await writeFile(temporary, secret.value, { mode: 0o600, flag: "wx" });
+        await rename(temporary, path);
+      } finally {
+        await rm(temporary, { force: true });
+      }
+    }
+    const result = await runEnvironmentProcess(parsed.command, {
+      cwd: prepared.workspace,
+      env: activatedEnvironment(prepared, environment, {
+        sessionToken: opts.sessionToken,
+        convexUrl: opts.convex.url,
+        apiUrl: opts.apiUrl,
+      }),
+      secretValues,
+      ...(parsed.json
+        ? {}
+        : {
+            stdout: (text: string) => process.stdout.write(text),
+            stderr: (text: string) => process.stderr.write(text),
+          }),
+    });
+    completedCommand = result;
+    const observations: {
+      state?: unknown;
+      credentialUpdates: unknown[];
+      conflicts: { kind: string; receipt?: unknown }[];
+    } = { credentialUpdates: [], conflicts: [] };
+    for (const secret of environment.credentialFiles.filter(
+      (file) => file.mutable,
+    )) {
+      const value = await readFile(
+        await environmentPrivatePath(prepared.workspace, secret.path),
+        "utf8",
+      );
+      if (value === secret.value) continue;
+      // New token values are also redacted from errors and never included in the result.
+      secretValues.push(value);
+      try {
+        const receipt = await opts.convex.action(
+          makeFunctionReference<"action">("functions/workspaceSecretsNode:set"),
+          {
+            ...token,
+            name: secret.secretName,
+            kind: "file",
+            value,
+            expectedVersion: secret.version,
+          },
+        );
+        observations.credentialUpdates.push({
+          name: secret.secretName,
+          version: (receipt as { version?: number }).version,
+        });
+      } catch (error) {
+        const data = (error as { data?: { receiptId?: string; code?: string } })
+          .data;
+        let localRecovery:
+          | { path: string; name: string; expectedVersion: number }
+          | undefined;
+        if (!data?.receiptId) {
+          const recoveryDirectory = await mkdtemp(
+            join(tmpdir(), "dench-secret-recovery-"),
+          );
+          const path = join(recoveryDirectory, "credential");
+          await writeFile(path, value, { mode: 0o600, flag: "wx" });
+          localRecovery = {
+            path,
+            name: secret.secretName,
+            expectedVersion: secret.version,
+          };
+        }
+        observations.conflicts.push({
+          kind: data?.code ?? "credential_state_unconfirmed",
+          ...(data?.receiptId
+            ? { receipt: { receiptId: data.receiptId } }
+            : { receipt: localRecovery }),
+        });
+      }
+    }
+    if (environment.manifest.statePaths.length) {
+      const state = await captureState(
+        environment,
+        prepared.workspace,
+        secretValues,
+      );
+      pendingState = state;
+      if (sha256(state) !== environment.state?.sha256) {
+        const upload = await uploadBundle(opts, "state", state);
+        try {
+          observations.state = await opts.convex.action(
+            actionRef("saveState"),
+            {
+              ...token,
+              name: environment.name,
+              revision: environment.revision,
+              stateKey: environment.state?.key ?? stateKey,
+              expectedVersion: environment.state?.version ?? 0,
+              uploadId: upload.uploadId,
+              storageId: upload.storageId,
+            },
+          );
+        } catch {
+          observations.conflicts.push({
+            kind: "state_commit_unconfirmed",
+            receipt: {
+              ...upload,
+              expectedVersion: environment.state?.version ?? 0,
+              stateKey,
+            },
+          });
+        }
+      }
+    }
+    const output = {
+      name: environment.name,
+      revision: environment.revision,
+      exitCode: result.exitCode,
+      signal: result.signal,
+      preparation: {
+        cacheHit: prepared.cacheHit,
+        source: prepared.preparation,
+        platform: prepared.platform,
+      },
+      ...observations,
+      ...(observations.conflicts.length
+        ? {
+            commandExecuted: true,
+            guidance:
+              "The business command already ran. Reconcile the saved receipts; do not rerun it to recover state.",
+          }
+        : {}),
+      ...(parsed.json
+        ? {
+            stdout: result.stdout,
+            stderr: result.stderr,
+            outputTruncated: result.outputTruncated,
+          }
+        : {}),
+    };
+    if (parsed.json) console.log(JSON.stringify(output, null, 2));
+    else if (observations.conflicts.length)
+      process.stderr.write(
+        `${JSON.stringify({ code: "environment_state_conflict", commandExitCode: result.exitCode, conflicts: observations.conflicts, guidance: "The business command already ran. Reconcile the saved receipts; do not rerun it to recover state." })}\n`,
+      );
+    process.exitCode = observations.conflicts.length ? 75 : result.exitCode;
+    return output;
+  } catch (error) {
+    if (completedCommand) {
+      preserveWorkspace = true;
+      // A post-execution storage failure must never look like a failed dispatch.
+      // A verified, nonsecret state candidate remains recoverable without running
+      // the business operation again, even if its upload failed before a receipt.
+      let recoveryPath: string | undefined;
+      if (pendingState) {
+        const recoveryDirectory = await mkdtemp(
+          join(tmpdir(), "dench-env-state-recovery-"),
+        );
+        recoveryPath = join(recoveryDirectory, "state.json");
+        await writeFile(recoveryPath, pendingState, {
+          mode: 0o600,
+          flag: "wx",
+        });
+      }
+      const output = {
+        name: environment.name,
+        revision: environment.revision,
+        code: "environment_state_unconfirmed",
+        commandExecuted: true,
+        exitCode: completedCommand.exitCode,
+        signal: completedCommand.signal,
+        error: sanitizeError(error).message,
+        recovery: {
+          path: recoveryPath ?? workspace,
+          verified: Boolean(recoveryPath),
+          expectedVersion: environment.state?.version ?? 0,
+          stateKey,
+        },
+        guidance:
+          "The business command already ran. Recover the saved state; do not rerun the business operation.",
+        ...(parsed.json
+          ? {
+              stdout: completedCommand.stdout,
+              stderr: completedCommand.stderr,
+              outputTruncated: completedCommand.outputTruncated,
+            }
+          : {}),
+      };
+      if (parsed.json) console.log(JSON.stringify(output, null, 2));
+      else process.stderr.write(`${JSON.stringify(output)}\n`);
+      process.exitCode = 75;
+      return output;
+    }
+    throw sanitizeError(error);
+  } finally {
+    try {
+      for (const path of privatePaths) await rm(path, { force: true });
+    } finally {
+      await releaseExecution?.();
+    }
+    if (!suppliedWorkspace && !job && !preserveWorkspace)
+      await rm(workspace, { recursive: true, force: true });
+  }
+}
+
+export async function runEnvironmentCommand(
+  opts: ClientOptions,
+): Promise<void> {
+  const parsed = validateEnvironmentArgs(opts.args);
+  if (parsed.verb === "help") {
+    console.log(HELP);
+    return;
+  }
+  const token = opts.sessionToken ? { sessionToken: opts.sessionToken } : {};
+  if (parsed.verb === "list") {
+    console.log(
+      JSON.stringify(
+        await opts.convex.query(queryRef("list"), {
+          ...token,
+          limit: parsed.limit,
+          ...(parsed.cursor ? { cursor: parsed.cursor } : {}),
+        }),
+        null,
+        2,
+      ),
+    );
+    return;
+  }
+  if (parsed.verb === "show") {
+    console.log(
+      JSON.stringify(
+        await opts.convex.query(queryRef("get"), {
+          ...token,
+          name: parsed.name,
+          ...(parsed.revision ? { revision: parsed.revision } : {}),
+        }),
+        null,
+        2,
+      ),
+    );
+    return;
+  }
+  if (parsed.verb === "delete") {
+    console.log(
+      JSON.stringify(
+        await opts.convex.mutation(mutationRef("remove"), {
+          ...token,
+          name: parsed.name,
+        }),
+        null,
+        2,
+      ),
+    );
+    return;
+  }
+  if (parsed.verb === "run") {
+    await executeSavedEnvironment(opts, parsed);
+    return;
+  }
+  if (!parsed.path || !parsed.name)
+    throw new CliArgError("env save requires a name and --path.");
+  const source = await collectEnvironmentSource(parsed.path, parsed.name);
+  const uploaded = await uploadBundle(opts, "source", source.bytes);
+  let receipt = (await opts.convex.action(actionRef("save"), {
+    ...token,
+    name: parsed.name,
+    manifest: source.manifest,
+    ...uploaded,
+  })) as EnvironmentRevision;
+  const deadline = Date.now() + parsed.wait * 1000;
+  while (
+    (receipt.status === "pending" || receipt.status === "building") &&
+    Date.now() < deadline
+  ) {
+    await new Promise((done) =>
+      setTimeout(done, Math.min(1000, deadline - Date.now())),
+    );
+    receipt = (await opts.convex.query(queryRef("get"), {
+      ...token,
+      name: parsed.name,
+      revision: receipt.revision,
+    })) as EnvironmentRevision;
+  }
+  console.log(JSON.stringify(receipt, null, 2));
+}