@mettlelabs/mettle 0.0.0-beta.1 → 0.0.0-beta.1-linux-x64

package/harness/chunk-msg872ea.mjs

@@ -0,0 +1,334 @@
+import {
+  MAX_LABEL,
+  MAX_PREVIEW_LINES,
+  MAX_PREVIEW_LINE,
+  HarnessError,
+  canonicalJson,
+  digest,
+  captureEvidence,
+  RUNTIME_VERSION
+} from "./chunk-2gt4cy7x.mjs";
+import {
+  PostgresJournal
+} from "./chunk-7bg2gnv2.mjs";
+import {
+  PROBE_TOOL,
+  SANDBOX_TOOLS,
+  dockerJson,
+  SandboxWorker
+} from "./chunk-xqyc4ca2.mjs";
+
+// apps/harness/src/execution/tool-executor.ts
+import { setTimeout as sleep } from "node:timers/promises";
+var APPROVAL_TTL_MS = 600000;
+var INTERRUPTED = "The operator interrupted this call while it ran; its outcome is unknown.";
+var SETTLED_FAILURES = new Set([
+  "tool_denied",
+  "approval_denied",
+  "approval_expired",
+  "invalid_artifact",
+  "sandbox_policy",
+  "sandbox_identity",
+  "sandbox_probe_failed",
+  "sandbox_output",
+  "sandbox_missing",
+  "sandbox_timeout",
+  "sandbox_unavailable",
+  "sandbox_image",
+  "candidate_denied",
+  "candidate_conflict",
+  "repository_bound",
+  "scanner_denied",
+  "web_denied",
+  "sandbox_task_failed",
+  "sandbox_output_limit",
+  "tool_interrupted"
+]);
+function settledFailure(error) {
+  return error instanceof HarnessError && SETTLED_FAILURES.has(error.code);
+}
+
+class ToolExecutor {
+  journal;
+  lease;
+  signal;
+  registry;
+  hooks;
+  constructor(journal, lease, signal, registry, hooks = {}) {
+    this.journal = journal;
+    this.lease = lease;
+    this.signal = signal;
+    this.registry = registry;
+    this.hooks = hooks;
+  }
+  async execute(callId, name, input, turn) {
+    if (this.signal.aborted)
+      throw new HarnessError("stop_requested", "Execution cancelled before admission.");
+    if (!callId)
+      throw new HarnessError("tool_denied", "A finalized tool call identity is required.");
+    let args;
+    let bytes;
+    try {
+      bytes = canonicalJson(input);
+      if (Buffer.byteLength(bytes, "utf8") > 65536)
+        throw new Error("Input limit exceeded.");
+      args = JSON.parse(bytes);
+    } catch {
+      throw new HarnessError("tool_denied", "Tool arguments must be bounded JSON.");
+    }
+    const tool = this.registry.admit(name, args, bytes);
+    const summary = tool.summarizeArgs(args).slice(0, 512);
+    const cachedRef = await this.journal.prepareAction(this.lease, callId, tool.name, args, {
+      label: tool.label(args).slice(0, MAX_LABEL),
+      summary,
+      quiet: tool.quiet?.(args) === true
+    }, tool.idempotent === true);
+    if (cachedRef)
+      return this.cached(cachedRef, tool);
+    const approval = await this.requested(callId, tool, args, summary);
+    await this.hooks.at?.("after-intent");
+    if (this.signal.aborted)
+      throw new HarnessError("stop_requested", "Execution cancelled before approval.");
+    if (approval)
+      await this.decided(callId, approval);
+    if (this.hooks.delayBeforeDispatchMs) {
+      try {
+        await sleep(this.hooks.delayBeforeDispatchMs, undefined, {
+          signal: this.signal
+        });
+      } catch {
+        throw new HarnessError("stop_requested", "Fixture wait cancelled before dispatch.");
+      }
+    }
+    if (this.signal.aborted)
+      throw new HarnessError("stop_requested", "Execution cancelled before dispatch.");
+    await this.journal.dispatchAction(this.lease, callId);
+    await this.hooks.at?.("after-dispatch");
+    let ref;
+    try {
+      const output = await tool.execute(args, turn ? AbortSignal.any([this.signal, turn]) : this.signal, callId);
+      await this.hooks.at?.("after-effect");
+      const result = tool.outputSchema.safeParse(output);
+      if (!result.success)
+        throw new HarnessError("invalid_artifact", "Tool output does not match its registered schema.");
+      const captured = captureEvidence(result.data);
+      const admitted = tool.outputSchema.safeParse(captured.value);
+      if (!admitted.success || canonicalJson(admitted.data) !== captured.bytes)
+        throw new HarnessError("invalid_artifact", "Redacted output is incompatible with the registered tool schema.");
+      const told = tool.summarizeResult(captured.value, args);
+      ref = await this.journal.completeAction(this.lease, callId, {
+        value: captured.value,
+        summary: told.summary.slice(0, 512),
+        ...told.rejected ? { rejected: true } : {},
+        ...told.preview?.length ? {
+          preview: told.preview.slice(0, MAX_PREVIEW_LINES).map((line) => line.slice(0, MAX_PREVIEW_LINE))
+        } : {}
+      });
+    } catch (error) {
+      if (turn?.aborted && !this.signal.aborted && !(error instanceof HarnessError && error.code === "lease_lost")) {
+        await this.journal.failAction(this.lease, callId, INTERRUPTED, "tool_interrupted", false);
+        throw new HarnessError("tool_interrupted", INTERRUPTED);
+      }
+      await this.settle(callId, error);
+      throw error;
+    }
+    await this.hooks.at?.("after-result");
+    return (await this.journal.artifact(this.lease.runId, ref)).bytes;
+  }
+  async requested(callId, tool, args, summary) {
+    const permission = tool.permission?.(args) ?? "local";
+    if (permission === "local")
+      return;
+    return this.journal.requestApproval(this.lease, callId, {
+      permission,
+      action: (tool.approval?.action ?? `${tool.name} ${summary}`).slice(0, 512),
+      detail: tool.approval?.detail ?? `${permission} against the live target; the effect may persist and the agent cannot undo it.`,
+      why: tool.approval?.why ?? `The ${permission} permission reaches outside the agent's sandbox, so an operator must authorize it before the call runs.`,
+      cleanup: tool.approval?.cleanup ?? "No automated rollback; an operator reverses any persisted effect manually if the call must be undone."
+    }, this.hooks.approvalTtlMs ?? APPROVAL_TTL_MS);
+  }
+  async decided(callId, approval) {
+    const current = await this.journal.approvalState(this.lease, approval.approvalId);
+    if (current.decision === null)
+      throw new HarnessError("approval_pending", "Execution is suspended until the recorded approval is decided or expires.");
+    if (current.decision === "allow")
+      return;
+    const error = current.decision === "deny" ? new HarnessError("approval_denied", `The operator denied ${current.permission} for this action.`) : new HarnessError("approval_expired", `The ${current.permission} approval expired before a decision.`);
+    await this.journal.failAction(this.lease, callId, error.message, error.code);
+    throw error;
+  }
+  async settle(callId, error) {
+    if (!(error instanceof HarnessError) || !SETTLED_FAILURES.has(error.code))
+      return;
+    await this.journal.failAction(this.lease, callId, error.message, error.code).catch(() => {
+      return;
+    });
+  }
+  async cached(ref, tool) {
+    const artifact = await this.journal.artifact(this.lease.runId, ref);
+    try {
+      if (!tool.outputSchema.safeParse(JSON.parse(artifact.bytes)).success)
+        throw new Error("Invalid result.");
+    } catch {
+      throw new HarnessError("invalid_artifact", "Stored tool output does not match its registered schema.");
+    }
+    return artifact.bytes;
+  }
+}
+
+// apps/harness/src/storage/sandbox-journal.ts
+import { randomUUID } from "node:crypto";
+class SandboxProbeJournal extends PostgresJournal {
+  get runtimeVersion() {
+    return `${RUNTIME_VERSION}/sandbox-probe-v1`;
+  }
+}
+
+class SandboxJournal {
+  journal;
+  lease;
+  constructor(journal, lease) {
+    this.journal = journal;
+    this.lease = lease;
+  }
+  async reserve(callId, workerId, image, policyDigest, binding = { tool: PROBE_TOOL, args: {} }) {
+    if (!SANDBOX_TOOLS.includes(binding.tool) || binding.tool === PROBE_TOOL && digest(binding.args) !== digest({}))
+      throw new HarnessError("sandbox_denied", "Only an approved isolated observation or fixed probe may reserve a sandbox.");
+    return this.journal.fenced(this.lease, async ({ client }) => {
+      const action = await client.query("SELECT 1 FROM harness_mastra.actions WHERE run_id=$1 AND call_id=$2 AND tool=$4 AND args_digest=$3 AND state='dispatched'", [this.lease.runId, callId, digest(binding.args), binding.tool]);
+      if (!action.rowCount)
+        throw new HarnessError("sandbox_denied", "A sandbox requires the exact committed observation dispatch.");
+      const record = {
+        runId: this.lease.runId,
+        callId,
+        epoch: this.lease.epoch,
+        workerId,
+        name: `mettle-${binding.tool.replace(/^sandbox_/, "")}-${randomUUID()}`,
+        image,
+        policyDigest,
+        containerId: null,
+        state: "reserved"
+      };
+      const inserted = await client.query("INSERT INTO harness_mastra.sandboxes(run_id,call_id,epoch,worker_id,name,image,policy_digest,state) VALUES ($1,$2,$3,$4,$5,$6,$7,'reserved') ON CONFLICT DO NOTHING", [
+        record.runId,
+        callId,
+        record.epoch,
+        workerId,
+        record.name,
+        image,
+        policyDigest
+      ]);
+      if (!inserted.rowCount)
+        throw new HarnessError("uncertain_execution", "A prior sandbox reservation cannot be executed again.");
+      return record;
+    });
+  }
+  async provision(record, operation) {
+    await this.journal.fenced(this.lease, async ({ client }) => {
+      await this.assertState(record, "reserved", client);
+      const containerId = await operation();
+      await this.transition(record, "reserved", "created", containerId, client);
+    });
+  }
+  async start(record, operation) {
+    await this.journal.fenced(this.lease, async ({ client }) => {
+      await this.assertState(record, "created", client);
+      await operation();
+      await this.transition(record, "created", "running", record.containerId, client);
+    });
+  }
+  async transition(record, from, to, containerId, client) {
+    const result = await client.query("UPDATE harness_mastra.sandboxes SET state=$5,container_id=$6 WHERE run_id=$1 AND call_id=$2 AND epoch=$3 AND name=$4 AND state=$7", [
+      this.lease.runId,
+      record.callId,
+      this.lease.epoch,
+      record.name,
+      to,
+      containerId,
+      from
+    ]);
+    if (result.rowCount !== 1)
+      throw new HarnessError("sandbox_denied", "Sandbox ownership or lifecycle no longer permits this operation.");
+    record.containerId = containerId;
+    record.state = to;
+  }
+  async assertTaskRunning(record) {
+    await this.journal.fenced(this.lease, async ({ client }) => {
+      await this.assertState(record, "running", client);
+    });
+  }
+  async assertState(record, state, client) {
+    const result = await client.query("SELECT 1 FROM harness_mastra.sandboxes WHERE run_id=$1 AND call_id=$2 AND epoch=$3 AND name=$4 AND state=$5", [this.lease.runId, record.callId, this.lease.epoch, record.name, state]);
+    if (record.runId !== this.lease.runId || record.epoch !== this.lease.epoch || result.rowCount !== 1)
+      throw new HarnessError("sandbox_denied", "Sandbox ownership or lifecycle no longer permits this operation.");
+  }
+  async removed(record) {
+    await this.journal.fenced(this.lease, async ({ client }) => {
+      const result = await client.query("UPDATE harness_mastra.sandboxes SET state='removed',removed_at=clock_timestamp() WHERE run_id=$1 AND call_id=$2 AND name=$3 AND worker_id=$4", [this.lease.runId, record.callId, record.name, record.workerId]);
+      if (record.runId !== this.lease.runId || !result.rowCount)
+        throw new HarnessError("sandbox_denied", "Cleanup receipt does not belong to this run.");
+      record.state = "removed";
+    }, "settlement");
+  }
+  async pending() {
+    return this.journal.fenced(this.lease, async ({ client }) => {
+      const result = await client.query(`SELECT run_id AS "runId",call_id AS "callId",epoch,worker_id AS "workerId",name,image,policy_digest AS "policyDigest",container_id AS "containerId",state FROM harness_mastra.sandboxes WHERE run_id=$1 AND state!='removed' ORDER BY name`, [this.lease.runId]);
+      return result.rows;
+    }, "settlement");
+  }
+  async markCleanupAttempt() {
+    await this.journal.fenced(this.lease, async ({ client }) => {
+      await client.query("UPDATE harness_mastra.sandboxes SET cleanup_attempted_at=clock_timestamp() WHERE run_id=$1 AND state!='removed'", [this.lease.runId]);
+    }, "settlement");
+  }
+}
+
+// apps/harness/src/sandbox/reaper.ts
+async function reapSandboxes(config, engine, journal, signal, runId, journalForRun) {
+  const info = dockerJson(await engine.call("GET", "/info"));
+  if (typeof info?.ID !== "string" || !info.ID)
+    throw new HarnessError("sandbox_unavailable", "The original engine identity is required.");
+  const candidates = runId ? [runId] : await journal.orphanedRuns(info.ID);
+  const outcomes = [];
+  for (const runId of candidates) {
+    if (signal.aborted)
+      break;
+    let lease;
+    let selected = journal;
+    try {
+      selected = journalForRun ? await journalForRun(runId) : journal;
+      lease = await selected.acquire(runId, 30000);
+      const lifecycle = new SandboxJournal(selected, lease);
+      await lifecycle.markCleanupAttempt();
+      const worker = new SandboxWorker(config, engine, lifecycle);
+      const removed = await worker.reap(info.ID);
+      outcomes.push({ runId, status: "removed", removed });
+    } catch (error) {
+      const code = error instanceof HarnessError ? error.code : "internal_failure";
+      outcomes.push({
+        runId,
+        status: code === "run_busy" ? "busy" : "blocked",
+        removed: 0,
+        code
+      });
+    } finally {
+      if (lease)
+        await selected.release(lease).catch(() => {
+          return;
+        });
+      if (selected !== journal)
+        await selected.close();
+    }
+  }
+  return {
+    workerId: info.ID,
+    scanned: candidates.length,
+    outcomes,
+    interrupted: signal.aborted,
+    actionReplayed: false,
+    actionReconciled: false,
+    productionApproved: false
+  };
+}
+
+export { settledFailure, ToolExecutor, SandboxProbeJournal, SandboxJournal, reapSandboxes };