chainlesschain 0.163.6 → 0.163.7

package/src/commands/scheduler-daemon.js

@@ -1,4 +1,5 @@
 import { randomUUID } from "node:crypto";
+import { hostname, userInfo } from "node:os";
 import chalk from "chalk";
 import { logger } from "../lib/logger.js";
 import {
@@ -7,6 +8,18 @@ import {
   MIN_SERVICE_INTERVAL_MS,
   createSchedulerService,
 } from "../lib/scheduler-kernel/service.js";
+import {
+  createSchedulerMigrationAdminRepository,
+  listSchedulerMigrations,
+} from "../lib/scheduler-kernel/migration-admin.js";
+import {
+  rollbackSchedulerMigration,
+  showSchedulerMigrationRollback,
+} from "../lib/scheduler-kernel/migration-rollback-admin.js";
+import {
+  schedulerAdjudicationOperatorDigest,
+  schedulerAdjudicationReasonDigest,
+} from "../lib/scheduler-kernel/store.js";
 
 export const SCHEDULER_DAEMON_DOMAINS = Object.freeze(["agenda", "cowork"]);
 
@@ -64,17 +77,204 @@ export function parseSchedulerCapabilities(value) {
 }
 
 async function withSchedulerStore(dependencies, operation) {
+  if (dependencies.schedulerStore) {
+    return operation(dependencies.schedulerStore);
+  }
   const openStore =
     dependencies.openSchedulerStore ||
     (await import("../lib/scheduler-kernel/store.js")).openSchedulerStore;
   const store = openStore();
   try {
-    return operation(store);
+    return await operation(store);
   } finally {
     store.close();
   }
 }
 
+async function withSchedulerMigrationRepository(dependencies, operation) {
+  if (dependencies.migrationRepository) {
+    return operation(dependencies.migrationRepository);
+  }
+  return withSchedulerStore(dependencies, (store) =>
+    operation(createSchedulerMigrationAdminRepository(store)),
+  );
+}
+
+export async function listSchedulerMigrationJournal(
+  options = {},
+  dependencies = {},
+) {
+  return withSchedulerMigrationRepository(dependencies, (repository) =>
+    listSchedulerMigrations(repository, options),
+  );
+}
+
+export async function showSchedulerMigrationJournal(
+  migrationId,
+  dependencies = {},
+) {
+  return withSchedulerMigrationRepository(dependencies, (repository) =>
+    showSchedulerMigrationRollback(repository, migrationId),
+  );
+}
+
+export async function rollbackSchedulerMigrationJournal(
+  migrationId,
+  options = {},
+  dependencies = {},
+) {
+  return withSchedulerStore(dependencies, (schedulerStore) => {
+    const repository =
+      dependencies.migrationRepository ||
+      createSchedulerMigrationAdminRepository(schedulerStore);
+    return rollbackSchedulerMigration({
+      schedulerStore,
+      repository,
+      migrationId,
+      options,
+      dependencies,
+    });
+  });
+}
+
+export function buildSchedulerAdjudicationChallenge({
+  occurrenceId,
+  decision,
+  evidenceDigest,
+  expectedAttempt,
+  expectedFence,
+} = {}) {
+  if (
+    typeof occurrenceId !== "string" ||
+    !["confirmed_applied", "confirmed_not_applied"].includes(decision) ||
+    typeof evidenceDigest !== "string" ||
+    !/^sha256:[a-f0-9]{64}$/.test(evidenceDigest) ||
+    !Number.isSafeInteger(expectedAttempt) ||
+    expectedAttempt < 1 ||
+    !Number.isSafeInteger(expectedFence) ||
+    expectedFence < 1
+  ) {
+    throw schedulerDaemonError(
+      "SCHEDULER_ADJUDICATION_INPUT_INVALID",
+      "Scheduler adjudication challenge input is invalid",
+    );
+  }
+  return (
+    `HOST STOPPED AND SCHEDULER DISPATCH DRAINED; ADJUDICATE ${occurrenceId} ` +
+    `${decision} ${evidenceDigest} ATTEMPT ${expectedAttempt} FENCE ${expectedFence}`
+  );
+}
+
+async function requireSchedulerAdjudicationTTY(request, dependencies = {}) {
+  const stdin = dependencies.stdin || process.stdin;
+  const stdout = dependencies.stdout || process.stdout;
+  if (stdin?.isTTY !== true || stdout?.isTTY !== true) {
+    throw schedulerDaemonError(
+      "SCHEDULER_ADJUDICATION_NON_INTERACTIVE",
+      "Scheduler adjudication requires an interactive TTY",
+    );
+  }
+  const readReason =
+    dependencies.readReason ||
+    (async () => {
+      const { input } = await import("@inquirer/prompts");
+      return input({
+        message: "Reason (only its digest is stored; do not enter secrets)",
+      });
+    });
+  const reason = await readReason();
+  const challenge = buildSchedulerAdjudicationChallenge(request);
+  const readChallenge =
+    dependencies.readChallenge ||
+    (async (expected) => {
+      const { input } = await import("@inquirer/prompts");
+      return input({
+        message:
+          "Stop every scheduler host, drain already-dispatched work, and verify the external outcome. " +
+          `Type this authorization exactly:\n${expected}`,
+      });
+    });
+  if ((await readChallenge(challenge)) !== challenge) {
+    throw schedulerDaemonError(
+      "SCHEDULER_ADJUDICATION_CHALLENGE_FAILED",
+      "Scheduler adjudication challenge did not match; no change was made",
+    );
+  }
+  return { reasonDigest: schedulerAdjudicationReasonDigest(reason), challenge };
+}
+
+export async function listSchedulerAdjudicationCases(
+  options = {},
+  dependencies = {},
+) {
+  return withSchedulerStore(dependencies, (store) =>
+    store.listAdjudicationCases({ limit: options.limit }),
+  );
+}
+
+export async function getSchedulerAdjudicationCase(
+  occurrenceId,
+  dependencies = {},
+) {
+  return withSchedulerStore(dependencies, (store) => {
+    const adjudicationCase = store.getAdjudicationCase(occurrenceId);
+    if (!adjudicationCase) {
+      throw schedulerDaemonError(
+        "SCHEDULER_NOT_FOUND",
+        `Scheduler occurrence does not exist: ${occurrenceId}`,
+      );
+    }
+    return adjudicationCase;
+  });
+}
+
+export async function adjudicateSchedulerOccurrence(
+  occurrenceId,
+  options,
+  dependencies = {},
+) {
+  const request = {
+    occurrenceId,
+    decision: options.decision,
+    evidenceDigest: options.expectedEvidenceDigest,
+    expectedAttempt: positiveInteger(
+      options.expectedAttempt,
+      "expectedAttempt",
+    ),
+    expectedFence: positiveInteger(options.expectedFence, "expectedFence"),
+  };
+  const confirmation = await requireSchedulerAdjudicationTTY(
+    request,
+    dependencies,
+  );
+  const operatorIdentity =
+    dependencies.operatorIdentity ||
+    (() => {
+      let username = "unknown";
+      try {
+        username = userInfo().username || username;
+      } catch {
+        // The digest still binds the host and uid when username lookup fails.
+      }
+      return {
+        username,
+        hostname: hostname(),
+        uid: typeof process.getuid === "function" ? process.getuid() : null,
+      };
+    })();
+  return withSchedulerStore(dependencies, (store) =>
+    store.adjudicateOccurrence({
+      occurrenceId,
+      decision: request.decision,
+      expectedEvidenceDigest: request.evidenceDigest,
+      expectedAttempt: request.expectedAttempt,
+      expectedFence: request.expectedFence,
+      reasonDigest: confirmation.reasonDigest,
+      operatorDigest: schedulerAdjudicationOperatorDigest(operatorIdentity),
+    }),
+  );
+}
+
 export async function getSchedulerAuthorityPolicy(
   principalType,
   principalId,
@@ -359,7 +559,7 @@ export async function runSchedulerDaemon(options = {}, dependencies = {}) {
   }
 }
 
-export function registerSchedulerDaemonCommands(daemon) {
+export function registerSchedulerDaemonCommands(daemon, dependencies = {}) {
   const scheduler = daemon
     .command("scheduler")
     .description("Run durable unattended scheduler domains in one process");
@@ -447,4 +647,161 @@ export function registerSchedulerDaemonCommands(daemon) {
         process.exitCode = 1;
       }
     });
+
+  const migration = scheduler
+    .command("migration")
+    .description("Inspect scheduler domain migration journals");
+
+  migration
+    .command("list", { isDefault: true })
+    .description("List scheduler domain migration journals")
+    .option("--state <state>", "Filter by migration state")
+    .option("--domain <domain>", "Filter by scheduler source domain")
+    .option("--limit <n>", "Maximum migrations to return", "50")
+    .action(async (options) => {
+      try {
+        logger.log(
+          JSON.stringify(
+            await listSchedulerMigrationJournal(options, dependencies),
+            null,
+            2,
+          ),
+        );
+      } catch (error) {
+        logger.error(chalk.red(error.message));
+        process.exitCode = 1;
+      }
+    });
+
+  migration
+    .command("rollback")
+    .description(
+      "Rollback one scheduler migration after source, target, CAS and TTY checks",
+    )
+    .argument("<migration-id>", "Scheduler domain migration ID")
+    .requiredOption(
+      "--expected-evidence-digest <digest>",
+      "Exact rollback.evidenceDigest from the latest migration show",
+    )
+    .option("--source-directory <path>", "Agenda or routine source directory")
+    .option("--workspace <path>", "Cowork source workspace")
+    .option("--session-id <id>", "Saved loop session ID")
+    .option("--automation-database <identity>", "Automation database identity")
+    .action(async (migrationId, options) => {
+      try {
+        logger.log(
+          JSON.stringify(
+            await rollbackSchedulerMigrationJournal(
+              migrationId,
+              options,
+              dependencies,
+            ),
+            null,
+            2,
+          ),
+        );
+      } catch (error) {
+        logger.error(chalk.red(error.message));
+        process.exitCode = 1;
+      }
+    });
+
+  migration
+    .command("show")
+    .description("Show a sanitized migration journal and rollback readiness")
+    .argument("<migration-id>", "Scheduler domain migration ID")
+    .action(async (migrationId) => {
+      try {
+        logger.log(
+          JSON.stringify(
+            await showSchedulerMigrationJournal(migrationId, dependencies),
+            null,
+            2,
+          ),
+        );
+      } catch (error) {
+        logger.error(chalk.red(error.message));
+        process.exitCode = 1;
+      }
+    });
+
+  const adjudication = scheduler
+    .command("adjudication")
+    .description("Inspect or resolve scheduler outcome-unknown dead letters");
+
+  adjudication
+    .command("list", { isDefault: true })
+    .description("List unadjudicated outcome-unknown dead letters")
+    .option("--limit <n>", "Maximum cases to return", "100")
+    .action(async (options) => {
+      try {
+        logger.log(
+          JSON.stringify(
+            await listSchedulerAdjudicationCases({
+              limit: positiveInteger(options.limit, "limit"),
+            }),
+            null,
+            2,
+          ),
+        );
+      } catch (error) {
+        logger.error(chalk.red(error.message));
+        process.exitCode = 1;
+      }
+    });
+
+  adjudication
+    .command("show")
+    .description("Show one case and its current CAS evidence")
+    .argument("<occurrence-id>", "Scheduler occurrence ID")
+    .action(async (occurrenceId) => {
+      try {
+        logger.log(
+          JSON.stringify(
+            await getSchedulerAdjudicationCase(occurrenceId),
+            null,
+            2,
+          ),
+        );
+      } catch (error) {
+        logger.error(chalk.red(error.message));
+        process.exitCode = 1;
+      }
+    });
+
+  adjudication
+    .command("decide")
+    .description(
+      "After host stop, dispatch drain and external verification, record one monotonic decision",
+    )
+    .argument("<occurrence-id>", "Scheduler occurrence ID")
+    .requiredOption(
+      "--decision <decision>",
+      "confirmed_applied or confirmed_not_applied",
+    )
+    .requiredOption(
+      "--expected-evidence-digest <digest>",
+      "Exact evidenceDigest from the latest show",
+    )
+    .requiredOption(
+      "--expected-attempt <n>",
+      "Exact attempt from the latest show",
+    )
+    .requiredOption("--expected-fence <n>", "Exact fence from the latest show")
+    .action(async (occurrenceId, options) => {
+      try {
+        const result = await adjudicateSchedulerOccurrence(
+          occurrenceId,
+          options,
+        );
+        logger.log(JSON.stringify(result, null, 2));
+        logger.warn(
+          "The typed challenge is operational evidence, not a machine-wide process lease. " +
+            "Restart a scheduler host to apply the durable decision; confirmed_not_applied permits one bounded claim.",
+        );
+      } catch (error) {
+        logger.error(chalk.red(error.message));
+        process.exitCode = 1;
+      }
+    });
 }

package/src/commands/session-observability.js

@@ -0,0 +1,305 @@
+import fs from "node:fs";
+import path from "node:path";
+import { randomUUID } from "node:crypto";
+import { TextDecoder } from "node:util";
+import { numericOption } from "../lib/cli-numeric.js";
+import { loadConfig } from "../lib/config-manager.js";
+import { mergePricing } from "../lib/llm-pricing.js";
+import {
+  buildCausalObservabilityReport,
+  causalSessionIds,
+  createCausalObservabilityLimitTracker,
+  createVerifiedSessionObservabilityProjection,
+  normalizeCausalRequest,
+  projectVerifiedDelivery,
+  selectCausalDeliveries,
+} from "../lib/causal-observability.js";
+import { restoreDeliveryFlow } from "../lib/delivery-coordinator.js";
+import { readVerifiedProjection } from "../harness/jsonl-session-store.js";
+import { ensurePrivateFile } from "../lib/secure-fs.js";
+import {
+  sameFileStatIdentity,
+  samePathHandleFileIdentity,
+  withTrustedFileParentSync,
+} from "../lib/secure-file-identity.js";
+
+const MAX_INPUT_BYTES = 16 * 1024 * 1024;
+
+function readTrustedJson(filePath, deps = {}) {
+  const runtimeFs = deps.fs || fs;
+  const trustedParent =
+    deps.withTrustedFileParentSync || withTrustedFileParentSync;
+  return trustedParent(
+    runtimeFs,
+    filePath,
+    ({ canonicalPath, parentDevice }) => {
+      const before = runtimeFs.lstatSync(canonicalPath, { bigint: true });
+      if (
+        before.isSymbolicLink() ||
+        !before.isFile() ||
+        Number(before.nlink) !== 1
+      ) {
+        throw new Error(
+          `input must be a regular, single-link file: ${filePath}`,
+        );
+      }
+      const size = Number(before.size);
+      if (size <= 0 || size > MAX_INPUT_BYTES) {
+        throw new Error(
+          `input exceeds the safe ${MAX_INPUT_BYTES}-byte limit: ${filePath}`,
+        );
+      }
+      let descriptor = null;
+      try {
+        descriptor = runtimeFs.openSync(
+          canonicalPath,
+          runtimeFs.constants.O_RDONLY |
+            Number(runtimeFs.constants.O_NOFOLLOW || 0),
+        );
+        const opened = runtimeFs.fstatSync(descriptor, { bigint: true });
+        if (
+          !opened.isFile() ||
+          Number(opened.nlink) !== 1 ||
+          !samePathHandleFileIdentity(before, opened, parentDevice)
+        ) {
+          throw new Error(`input identity changed while opening: ${filePath}`);
+        }
+        const bounded = Buffer.allocUnsafe(MAX_INPUT_BYTES + 1);
+        let bytesRead = 0;
+        while (bytesRead < bounded.length) {
+          const count = runtimeFs.readSync(
+            descriptor,
+            bounded,
+            bytesRead,
+            bounded.length - bytesRead,
+            null,
+          );
+          if (count === 0) break;
+          bytesRead += count;
+        }
+        if (bytesRead > MAX_INPUT_BYTES) {
+          throw new Error(
+            `input exceeds the safe ${MAX_INPUT_BYTES}-byte limit: ${filePath}`,
+          );
+        }
+        const body = bounded.subarray(0, bytesRead);
+        const after = runtimeFs.fstatSync(descriptor, { bigint: true });
+        if (
+          Number(after.size) !== body.length ||
+          !sameFileStatIdentity(opened, after)
+        ) {
+          throw new Error(`input changed while being read: ${filePath}`);
+        }
+        return Object.freeze({
+          canonicalPath,
+          value: JSON.parse(
+            new TextDecoder("utf-8", { fatal: true }).decode(body),
+          ),
+        });
+      } finally {
+        if (descriptor !== null) runtimeFs.closeSync(descriptor);
+      }
+    },
+  );
+}
+
+function atomicWritePrivate(outputPath, contents, deps = {}) {
+  const runtimeFs = deps.fs || fs;
+  const absolute = path.resolve(outputPath);
+  const trustedParent =
+    deps.withTrustedFileParentSync || withTrustedFileParentSync;
+  return trustedParent(runtimeFs, absolute, ({ canonicalPath, parentPath }) => {
+    if (runtimeFs.existsSync(canonicalPath)) {
+      throw new Error(
+        `output already exists; refusing to overwrite: ${absolute}`,
+      );
+    }
+    const temporary = path.join(
+      parentPath,
+      `.${path.basename(canonicalPath)}.${randomUUID()}.tmp`,
+    );
+    let published = false;
+    try {
+      runtimeFs.writeFileSync(temporary, contents, {
+        encoding: "utf8",
+        mode: 0o600,
+        flag: "wx",
+      });
+      (deps.ensurePrivateFile || ensurePrivateFile)(temporary, {
+        applyWindowsAcl: true,
+        failIfUnavailable: true,
+      });
+      // Publishing a fully-written hard link is atomic and, unlike rename,
+      // fails closed if another writer created the requested output first.
+      runtimeFs.linkSync(temporary, canonicalPath);
+      published = true;
+      runtimeFs.rmSync(temporary);
+      (deps.ensurePrivateFile || ensurePrivateFile)(canonicalPath, {
+        applyWindowsAcl: true,
+        failIfUnavailable: true,
+      });
+      return canonicalPath;
+    } catch (error) {
+      try {
+        runtimeFs.rmSync(temporary, { force: true });
+        if (published) runtimeFs.rmSync(canonicalPath, { force: true });
+      } catch {
+        // Preserve the authoritative write error.
+      }
+      throw error;
+    }
+  });
+}
+
+function budgetOverrides(options) {
+  const parse = (value, name, extra = {}) =>
+    value == null
+      ? undefined
+      : numericOption(value, { name, min: 0, ...extra });
+  return {
+    maxTokens: parse(options.maxTokens, "--max-tokens", { integer: true }),
+    maxUsd: parse(options.maxUsd, "--max-usd"),
+    maxRetries: parse(options.maxRetries, "--max-retries", { integer: true }),
+    maxRetryRatio: parse(options.maxRetryRatio, "--max-retry-ratio", {
+      max: 1,
+    }),
+    maxToolP95Ms: parse(options.maxToolP95Ms, "--max-tool-p95-ms"),
+  };
+}
+
+function nonEmptyFilterOption(value, name) {
+  if (value == null) return undefined;
+  if (typeof value !== "string" || value.trim() === "") {
+    throw new Error(`${name} must be a non-empty string`);
+  }
+  return value;
+}
+
+function optionFilter(options) {
+  return {
+    workspaceId: nonEmptyFilterOption(options.workspace, "--workspace"),
+    teamId: nonEmptyFilterOption(options.team, "--team"),
+    policyId: nonEmptyFilterOption(options.policy, "--policy"),
+  };
+}
+
+function mergeDefined(base, overrides) {
+  const definedOverrides = Object.fromEntries(
+    Object.entries(overrides).filter(([, value]) => value !== undefined),
+  );
+  return Object.fromEntries(
+    Object.entries({ ...base, ...definedOverrides }).filter(
+      ([, value]) => value !== undefined,
+    ),
+  );
+}
+
+export function runSessionObservability(requestPath, options = {}, deps = {}) {
+  try {
+    const trustedRequest = readTrustedJson(requestPath, deps);
+    const request = normalizeCausalRequest(trustedRequest.value);
+    const requestDir = path.dirname(trustedRequest.canonicalPath);
+    const deliveries = request.deliveryStates.map((statePath) => {
+      const resolved = path.resolve(requestDir, statePath);
+      const state = (deps.restoreDeliveryFlow || restoreDeliveryFlow)(
+        readTrustedJson(resolved, deps).value,
+      );
+      return projectVerifiedDelivery(state);
+    });
+    const filter = mergeDefined(request.filter, optionFilter(options));
+    const selectedDeliveries = (
+      deps.selectCausalDeliveries || selectCausalDeliveries
+    )(deliveries, filter);
+    const selectedSessionIds = (deps.causalSessionIds || causalSessionIds)(
+      selectedDeliveries,
+    );
+    const limitTracker = (
+      deps.createCausalObservabilityLimitTracker ||
+      createCausalObservabilityLimitTracker
+    )();
+    for (const delivery of selectedDeliveries) {
+      limitTracker.acceptDelivery(delivery);
+    }
+    const sessionsById = new Map();
+    const pricingTable = (deps.mergePricing || mergePricing)(
+      (deps.loadConfig || loadConfig)().llm?.pricing,
+    );
+    for (const sessionId of selectedSessionIds) {
+      const session = (deps.readVerifiedProjection || readVerifiedProjection)(
+        sessionId,
+        () =>
+          (
+            deps.createVerifiedSessionObservabilityProjection ||
+            createVerifiedSessionObservabilityProjection
+          )(sessionId, { pricingTable }),
+      );
+      limitTracker.acceptSession(session);
+      sessionsById.set(sessionId, session);
+    }
+    const budgets = mergeDefined(request.budgets, budgetOverrides(options));
+    const report = (
+      deps.buildCausalObservabilityReport || buildCausalObservabilityReport
+    )({
+      deliveries: selectedDeliveries,
+      sessionsById,
+      filter,
+      budgets,
+      pricingTable,
+      generatedAt: deps.now ? deps.now() : new Date().toISOString(),
+    });
+    const output = `${JSON.stringify(report, null, 2)}\n`;
+    if (options.output) {
+      const written = atomicWritePrivate(options.output, output, deps);
+      if (!options.json) process.stdout.write(`${written}\n`);
+    } else {
+      process.stdout.write(output);
+    }
+    if (
+      options.strictBudget === true &&
+      ["exceeded", "unknown"].includes(report.budget.status)
+    ) {
+      return 2;
+    }
+    return 0;
+  } catch (error) {
+    process.stderr.write(`Causal observability failed: ${error.message}\n`);
+    return 1;
+  }
+}
+
+export function registerSessionObservabilitySubcommand(session) {
+  session
+    .command("observability <request>")
+    .description(
+      "Export verified token/USD, retry, tool latency, diff, gate, artifact and PR causality",
+    )
+    .option("--workspace <id>", "Only delivery authority for this workspace")
+    .option("--team <id>", "Only delivery authority for this team")
+    .option("--policy <id>", "Only delivery authority for this policy")
+    .option("--max-tokens <n>", "Alert when aggregate tokens exceed n")
+    .option(
+      "--max-usd <amount>",
+      "Alert when estimated priced USD exceeds amount",
+    )
+    .option("--max-retries <n>", "Alert when automatic LLM retries exceed n")
+    .option(
+      "--max-retry-ratio <ratio>",
+      "Alert when approximate retry/call ratio exceeds 0..1",
+    )
+    .option(
+      "--max-tool-p95-ms <ms>",
+      "Alert when fully observed tool P95 exceeds ms",
+    )
+    .option(
+      "--strict-budget",
+      "Exit 2 when a budget is exceeded or cannot be evaluated",
+    )
+    .option(
+      "--output <path>",
+      "Atomically write owner-only JSON instead of stdout",
+    )
+    .option("--json", "Machine-readable output (default when stdout is used)")
+    .action((requestPath, options) => {
+      process.exitCode = runSessionObservability(requestPath, options);
+    });
+}