@h-ai/runner 1.0.0-beta.24 → 1.0.0-beta.25

package/dist/index.js

@@ -4,7 +4,7 @@ import { randomUUID, createHash } from 'crypto';
 import { core, ok, err } from '@h-ai/core';
 import { execFile, spawn } from 'child_process';
 import { once } from 'events';
-import { lstat, chmod, rm, readFile, stat, realpath, mkdtemp, writeFile, mkdir } from 'fs/promises';
+import { lstat, chmod, rm, readFile, readlink, stat, realpath, mkdtemp, writeFile, mkdir } from 'fs/promises';
 import { tmpdir } from 'os';
 import { join, posix, basename } from 'path';
 import { promisify } from 'util';
@@ -62,6 +62,8 @@ var RunnerConfigSchema = z.strictObject({
   maxControlRunning: z.number().int().min(1).max(1e3).default(8),
   maxQueued: z.number().int().min(1).max(1e5).default(5e3),
   maxEvents: z.number().int().min(1).max(1e5).default(1e4),
+  /** 已结算任务、事件和幂等键的保留期限。未核验的 unknown 不受该期限影响。 */
+  retentionMs: z.number().int().min(36e5).max(365 * 864e5).default(7 * 864e5),
   closeTimeoutMs: z.number().int().min(100).max(12e4).default(2e4),
   environments: z.array(environment).max(1e3).default([])
 }).refine((value) => value.pollMs * 3 < value.leaseMs && new Set(value.environments.map((entry) => entry.id)).size === value.environments.length);
@@ -499,7 +501,7 @@ async function startContainer(engine, environment2, resources2, launch, signal,
       return closing;
     };
     let activated = false;
-    const process2 = {
+    const process3 = {
       stdin: child.stdin,
       stdout: child.stdout,
       stderr: child.stderr,
@@ -530,7 +532,7 @@ async function startContainer(engine, environment2, resources2, launch, signal,
           await new Promise((resolve, reject) => {
             child.stdin.write("+", (error) => error ? reject(error) : resolve());
           });
-          return ok(process2);
+          return ok(process3);
         } catch {
           return unavailable();
         }
@@ -580,7 +582,7 @@ async function startContainer(engine, environment2, resources2, launch, signal,
         return (await stop()).success ? unavailable() : uncertain();
       }
     }
-    return ok(process2);
+    return ok(process3);
   } catch {
     if (envFile) {
       try {
@@ -609,10 +611,83 @@ async function prepareWarmContainer(engine, environment2, signal, record) {
     inputs: {}
   }, signal, record, true);
 }
+var processIdentity = z.strictObject({
+  pid: z.number().int().positive().max(2 ** 31 - 1),
+  hostKey: z.string().regex(/^[a-f\d]{64}$/u),
+  started: z.string().regex(/^\d+$/u).max(20)
+});
+async function hostKey() {
+  const [machine, boot, namespace] = await Promise.all([
+    readFile("/etc/machine-id", "utf8"),
+    readFile("/proc/sys/kernel/random/boot_id", "utf8"),
+    readlink("/proc/self/ns/pid")
+  ]);
+  if (!/^[a-f\d]{32}$/u.test(machine.trim()) || !/^[a-f\d-]{36}$/u.test(boot.trim()) || !/^pid:\[\d+\]$/u.test(namespace)) {
+    throw new Error("Runner host identity unavailable");
+  }
+  return createHash("sha256").update(JSON.stringify([
+    machine.trim(),
+    boot.trim(),
+    namespace
+  ])).digest("hex");
+}
+async function startTime(pid) {
+  const raw = await readFile(`/proc/${pid}/stat`, "utf8");
+  const index = raw.lastIndexOf(") ");
+  const field = index < 0 ? void 0 : raw.slice(index + 2).split(/\s+/u)[19];
+  if (!field || !/^\d+$/u.test(field))
+    throw new Error("Runner process identity unavailable");
+  return field;
+}
+async function readRunnerProcess() {
+  if (process.platform !== "linux")
+    return void 0;
+  try {
+    const [host, started] = await Promise.all([
+      hostKey(),
+      startTime(process.pid)
+    ]);
+    return { pid: process.pid, hostKey: host, started };
+  } catch {
+    return void 0;
+  }
+}
+async function confirmRunnerProcessReleased(identity) {
+  const record = processIdentity.safeParse(identity);
+  if (!record.success || process.platform !== "linux")
+    return false;
+  const { pid, hostKey: previous, started } = record.data;
+  try {
+    if (previous !== await hostKey())
+      return false;
+    try {
+      return await startTime(pid) !== started;
+    } catch (error) {
+      if (!(error instanceof Error && "code" in error && error.code === "ENOENT"))
+        return false;
+      try {
+        process.kill(pid, 0);
+        return false;
+      } catch (cause) {
+        return cause instanceof Error && "code" in cause && cause.code === "ESRCH";
+      }
+    }
+  } catch {
+    return false;
+  }
+}
 var EXECUTIONS = "hai_runner_executions";
 var EVENTS = "hai_runner_events";
 var GATE = "hai_runner_gate";
 var ACTIVE = "(state IN ('running', 'cancelling') OR (state = 'unknown' AND reconciled_at IS NULL))";
+var RETENTION_STATES = [
+  "succeeded",
+  "failed",
+  "cancelled",
+  "unknown"
+];
+var RETENTION_BATCH = 128;
+var MAX_EVENT_BYTES = 16 * 1024 * 1024;
 var text = { type: "TEXT", notNull: true };
 var nullableText = { type: "TEXT" };
 var integer = { type: "BIGINT", notNull: true };
@@ -629,6 +704,7 @@ var executions = {
   reason: nullableText,
   owner_node: nullableText,
   owner_generation: nullableText,
+  owner_process_json: payload,
   pinned_node: nullableText,
   pinned_generation: nullableText,
   parent_id: nullableText,
@@ -636,6 +712,7 @@ var executions = {
   queue_until: integer,
   run_until: integer,
   event_sequence: integer,
+  event_bytes: { ...integer, defaultValue: 0 },
   cpu_milli: integer,
   memory_mib: integer,
   pids: integer,
@@ -673,13 +750,12 @@ function hash(value) {
 function databaseValue(value) {
   return typeof value === "string" ? JSON.parse(value) : value;
 }
-function project(row) {
-  const request = databaseValue(row.request_json);
+function project(row, environmentId) {
   return {
     id: row.id,
     status: row.state,
     owner: row.owner_node && row.owner_generation ? { nodeId: row.owner_node, generation: row.owner_generation } : void 0,
-    environmentId: request.environment?.id,
+    environmentId,
     output: row.result_json === null ? void 0 : databaseValue(row.result_json).value,
     failure: row.reason ?? void 0,
     createdAt: Number(row.created_at),
@@ -687,6 +763,9 @@ function project(row) {
     reconciledAt: row.reconciled_at === null ? void 0 : Number(row.reconciled_at)
   };
 }
+function projectRequest(row) {
+  return project(row, databaseValue(row.request_json).environment?.id);
+}
 function unavailable2() {
   return err(HaiRunnerError.UNAVAILABLE, runnerM("runner_unavailable"));
 }
@@ -731,6 +810,7 @@ function within(actual, ceiling) {
 async function readyStore() {
   if (!reldb.isInitialized)
     return unavailable2();
+  const addedColumns = ["event_bytes", "owner_process_json"];
   for (const [name, def] of [
     [
       GATE,
@@ -753,8 +833,19 @@ async function readyStore() {
   ]) {
     const result = await reldb.ddl.createTable(name, def);
     if (!result.success) {
-      const existing = await reldb.schemaColumns(name);
-      if (!existing.success || Object.keys(def).some((column) => !existing.data.includes(column)))
+      const existing2 = await reldb.schemaColumns(name);
+      if (!existing2.success || Object.keys(def).some((column) => !existing2.data.includes(column) && !(name === EXECUTIONS && addedColumns.includes(column))))
+        return unavailable2();
+    }
+  }
+  const existing = await reldb.schemaColumns(EXECUTIONS);
+  if (!existing.success)
+    return unavailable2();
+  for (const column of addedColumns.filter((column2) => !existing.data.includes(column2))) {
+    const added = await reldb.ddl.addColumn(EXECUTIONS, column, executions[column]);
+    if (!added.success) {
+      const refreshed = await reldb.schemaColumns(EXECUTIONS);
+      if (!refreshed.success || !refreshed.data.includes(column))
         return unavailable2();
     }
   }
@@ -779,6 +870,36 @@ async function readyStore() {
       "hai_runner_concurrency_idx",
       ["concurrency_key", "state"]
     ],
+    [
+      EXECUTIONS,
+      "hai_runner_parent_idx",
+      ["parent_id"]
+    ],
+    [
+      EXECUTIONS,
+      "hai_runner_retention_idx",
+      [
+        "state",
+        "reconciled_at",
+        "updated_at",
+        "id"
+      ]
+    ],
+    [
+      EXECUTIONS,
+      "hai_runner_queue_due_idx",
+      ["state", "queue_until"]
+    ],
+    [
+      EXECUTIONS,
+      "hai_runner_lease_due_idx",
+      ["state", "leased_until"]
+    ],
+    [
+      EXECUTIONS,
+      "hai_runner_run_due_idx",
+      ["state", "run_until"]
+    ],
     [
       EVENTS,
       "hai_runner_event_cursor_idx",
@@ -789,11 +910,11 @@ async function readyStore() {
     if (!result.success) {
       if (reldb.config?.type !== "mysql")
         return unavailable2();
-      const existing = await reldb.sql.query(
+      const existing2 = await reldb.sql.query(
         "SELECT COLUMN_NAME AS column_name FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND INDEX_NAME = ? ORDER BY SEQ_IN_INDEX",
         [table, name]
       );
-      if (!existing.success || JSON.stringify(existing.data.map((row) => row.column_name)) !== JSON.stringify(fields))
+      if (!existing2.success || JSON.stringify(existing2.data.map((row) => row.column_name)) !== JSON.stringify(fields))
         return unavailable2();
     }
   }
@@ -801,11 +922,35 @@ async function readyStore() {
   const seed = await reldb.sql.execute(`INSERT INTO ${GATE} (id, revision) VALUES (1, 0)${syntax}`);
   return seed.success ? ok(void 0) : unavailable2();
 }
-function makeStore(config, owner) {
+function makeStore(config, owner, ownerProcess) {
   const envById = new Map(config.environments.map((item) => [item.id, item]));
   const envKeys = config.environments.filter((item) => !item.architecture || item.architecture === process.arch).map((item) => hash(item));
   const now = clock();
   const resultValue = reldb.config?.type === "postgresql" ? "CAST(? AS JSONB)" : reldb.config?.type === "mysql" ? "CAST(CONVERT(? USING utf8mb4) AS JSON)" : "?";
+  const statementTime = reldb.config?.type === "postgresql" ? "CAST(FLOOR(EXTRACT(EPOCH FROM statement_timestamp()) * 1000) AS BIGINT)" : reldb.config?.type === "mysql" ? "CAST(UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000 AS SIGNED)" : now;
+  const processJson = ownerProcess ? JSON.stringify(ownerProcess) : null;
+  const environmentPath = reldb.json.extract("request_json", "$.environment.id");
+  const environmentProjection = reldb.config?.type === "mysql" ? `JSON_UNQUOTE(${environmentPath.sql})` : reldb.config?.type === "postgresql" ? "request_json #>> ?::text[]" : environmentPath.sql;
+  async function summary(id2, scopeKey) {
+    const row = data(await reldb.sql.get(`
+      SELECT id, state, reason, owner_node, owner_generation, result_json, created_at, updated_at, reconciled_at,
+        ${environmentProjection} AS environment_id
+      FROM ${EXECUTIONS} WHERE id = ? AND scope_key = ?`, [...environmentPath.params, id2, scopeKey]));
+    return row ? project(row, row.environment_id ?? void 0) : void 0;
+  }
+  async function parentSettled(sql, row) {
+    if (!row.parent_id)
+      return true;
+    const parent = data(await sql.get(`
+      SELECT state, reconciled_at FROM ${EXECUTIONS}
+      WHERE id = ? AND owner_node = ? AND owner_generation = ? AND scope_key = ?`, [
+      row.parent_id,
+      row.owner_node,
+      row.owner_generation,
+      row.scope_key
+    ]));
+    return !!parent && (parent.state === "unknown" ? parent.reconciled_at !== null : ["succeeded", "failed", "cancelled"].includes(parent.state));
+  }
   async function sweep(sql) {
     data(await sql.execute(`UPDATE ${EXECUTIONS} SET state = 'failed', reason = 'queue-timeout', reuse_key = NULL, updated_at = ${now}
       WHERE state = 'queued' AND queue_until <= ${now}`));
@@ -814,13 +959,51 @@ function makeStore(config, owner) {
     data(await sql.execute(`UPDATE ${EXECUTIONS} SET state = 'cancelling', reason = 'run-timeout', updated_at = ${now}
       WHERE state = 'running' AND run_until <= ${now} AND leased_until > ${now}`));
   }
+  async function expired(sql, limit) {
+    const pages = await Promise.all(RETENTION_STATES.map((state) => {
+      const endedAt = state === "unknown" ? "q.reconciled_at" : "q.updated_at";
+      return sql.query(`
+        SELECT q.id, ${endedAt} AS ended_at FROM ${EXECUTIONS} q
+        WHERE q.state = ? AND q.reconciled_at ${state === "unknown" ? "IS NOT NULL" : "IS NULL"}
+          AND ${endedAt} <= ${statementTime} - ?
+          AND NOT EXISTS (SELECT 1 FROM ${EXECUTIONS} child WHERE child.parent_id = q.id)
+        ORDER BY ${endedAt}, q.updated_at, q.id LIMIT ?`, [state, config.retentionMs, limit]);
+    }));
+    return pages.flatMap((page) => data(page)).sort((left, right) => Number(left.ended_at) - Number(right.ended_at)).slice(0, limit);
+  }
   return {
     async sweep() {
+      const due = await reldb.sql.get(`
+        SELECT id FROM ${EXECUTIONS} WHERE (state = 'queued' AND queue_until <= ${statementTime})
+          OR (state IN ('running', 'cancelling') AND leased_until <= ${statementTime})
+          OR (state = 'running' AND run_until <= ${statementTime}) LIMIT 1`);
+      if (!due.success)
+        return unavailable2();
+      if (!due.data)
+        return ok(void 0);
       return serialized(async (sql) => {
         await sweep(sql);
         return ok(void 0);
       });
     },
+    async prune() {
+      try {
+        if (!(await expired(reldb.sql, 1)).length)
+          return ok(0);
+      } catch {
+        return unavailable2();
+      }
+      return serialized(async (sql) => {
+        const entries = await expired(sql, RETENTION_BATCH);
+        if (!entries.length)
+          return ok(0);
+        const ids = entries.map((entry) => entry.id);
+        const keys = ids.map(() => "?").join(", ");
+        data(await sql.execute(`DELETE FROM ${EVENTS} WHERE execution_id IN (${keys})`, ids));
+        const removed = data(await sql.execute(`DELETE FROM ${EXECUTIONS} WHERE id IN (${keys})`, ids));
+        return ok(removed.changes);
+      });
+    },
     async submit(request, pool) {
       const scopeKey = hash([request.scope.tenantId, request.scope.userId]);
       const id2 = hash([scopeKey, request.kind, request.idempotencyKey ?? randomUUID()]);
@@ -841,7 +1024,7 @@ function makeStore(config, owner) {
       return serialized(async (sql) => {
         const existing = await find(sql, id2, scopeKey) ?? (reuse ? data(await sql.get(`SELECT * FROM ${EXECUTIONS} WHERE reuse_key = ?`, [reuse])) : null);
         if (existing) {
-          return existing.request_key === requestKey ? ok(project(existing)) : conflict();
+          return existing.request_key === requestKey ? ok(projectRequest(existing)) : conflict();
         }
         const environment2 = request.environmentId ? envById.get(request.environmentId) : void 0;
         if (request.warmPool && (!pool || !environment2))
@@ -902,22 +1085,21 @@ function makeStore(config, owner) {
         const row = await find(sql, id2, scopeKey);
         if (!row)
           throw new Error("Newly inserted execution could not be read");
-        return ok(project(row));
+        return ok(projectRequest(row));
       });
     },
     async find(scope, kind, key) {
       try {
         const scopeKey = hash([scope.tenantId, scope.userId]);
-        const row = await find(reldb.sql, hash([scopeKey, kind, key]), scopeKey);
-        return ok(row ? project(row) : void 0);
+        return ok(await summary(hash([scopeKey, kind, key]), scopeKey));
       } catch {
         return unavailable2();
       }
     },
     async get(scope, id2) {
       try {
-        const row = await find(reldb.sql, id2, hash([scope.tenantId, scope.userId]));
-        return row ? ok(project(row)) : missing();
+        const found = await summary(id2, hash([scope.tenantId, scope.userId]));
+        return found ? ok(found) : missing();
       } catch {
         return unavailable2();
       }
@@ -932,7 +1114,7 @@ function makeStore(config, owner) {
         data(await sql.execute(`UPDATE ${EXECUTIONS} SET state = 'cancelled', reason = 'parent-ended', updated_at = ${now} WHERE parent_id = ? AND state = 'queued'`, [id2]));
         data(await sql.execute(`UPDATE ${EXECUTIONS} SET state = 'cancelling', reason = 'parent-ended', updated_at = ${now} WHERE parent_id = ? AND state = 'running'`, [id2]));
         const changed = await find(sql, id2, row.scope_key);
-        return changed ? ok(project(changed)) : missing();
+        return changed ? ok(projectRequest(changed)) : missing();
       });
     },
     async reserveWarm(pool) {
@@ -944,7 +1126,8 @@ function makeStore(config, owner) {
         environment: environment2
       });
       const key = hash([pool.kind, pool.key]);
-      return serialized(async (sql) => {
+      const usage = columns(environment2.resources);
+      async function canReserve(sql) {
         const [totalsResult, reservedResult] = await Promise.all([
           sql.get(`
             SELECT COUNT(*) AS total, COALESCE(SUM(CASE WHEN state = 'running' AND pool_fingerprint = ?
@@ -959,9 +1142,8 @@ function makeStore(config, owner) {
         const totals = data(totalsResult);
         const reserved = data(reservedResult);
         if (!totals || !reserved || Number(totals.warm) >= pool.minReady || Number(totals.total) >= Math.min(pool.maxTotal, Number(totals.ceiling)) || Number(reserved.active) >= config.toolMaxRunning) {
-          return ok(null);
+          return false;
         }
-        const usage = columns(environment2.resources);
         const capacity = columns(config.toolCapacity);
         const occupied = [
           Number(reserved.cpu),
@@ -969,7 +1151,16 @@ function makeStore(config, owner) {
           Number(reserved.pids),
           Number(reserved.scratch)
         ];
-        if (usage.some((amount, index) => amount + occupied[index] > capacity[index]))
+        return !usage.some((amount, index) => amount + occupied[index] > capacity[index]);
+      }
+      try {
+        if (!await canReserve(reldb.sql))
+          return ok(null);
+      } catch {
+        return unavailable2();
+      }
+      return serialized(async (sql) => {
+        if (!await canReserve(sql))
           return ok(null);
         const id2 = hash([
           owner.generation,
@@ -1014,12 +1205,25 @@ function makeStore(config, owner) {
       return result.success && result.data.changes === 1 ? ok(void 0) : lost();
     },
     async claim(kinds, category, pools = [], warmIds = []) {
-      return serialized(async (sql) => {
-        if (!kinds.length)
-          return ok(null);
+      if (!kinds.length)
+        return ok(null);
+      const capacityClass = category === "tool" ? "tool" : "default";
+      const parentFilter = category === "control" ? "IS NOT NULL" : "IS NULL";
+      const queued = await reldb.sql.get(`
+        SELECT id FROM ${EXECUTIONS} WHERE state = 'queued' AND capacity_class = ? AND parent_id ${parentFilter}
+          AND queue_until > ${now} AND kind IN (${kinds.map(() => "?").join(",")})
+          AND (pinned_node IS NULL OR pinned_node = ?) AND (pinned_generation IS NULL OR pinned_generation = ?) LIMIT 1`, [
+        capacityClass,
+        ...kinds,
+        owner.nodeId,
+        owner.generation
+      ]);
+      if (!queued.success)
+        return unavailable2();
+      if (!queued.data)
+        return ok(null);
+      async function eligible(sql, includeRequest) {
         const maxRunning = category === "tool" ? config.toolMaxRunning : category === "control" ? config.maxControlRunning : config.maxRunning;
-        const capacityClass = category === "tool" ? "tool" : "default";
-        const parentFilter = category === "control" ? "IS NOT NULL" : "IS NULL";
         const reserved = data(await sql.get(
           `SELECT COUNT(*) AS active, COALESCE(SUM(cpu_milli),0) AS cpu, COALESCE(SUM(memory_mib),0) AS memory,
              COALESCE(SUM(pids),0) AS pids, COALESCE(SUM(scratch_mib),0) AS scratch
@@ -1027,7 +1231,7 @@ function makeStore(config, owner) {
           [owner.nodeId, capacityClass]
         ));
         if (!reserved)
-          return ok(null);
+          return null;
         const fingerprints = category === "tool" ? pools.flatMap((pool) => {
           const environment2 = envById.get(pool.environmentId);
           return environment2 ? [
@@ -1043,8 +1247,8 @@ function makeStore(config, owner) {
         const canStartCold = Number(reserved.active) < maxRunning;
         const eligibleEnvironment = envKeys.length ? ` OR q.environment_key IN (${envKeys.map(() => "?").join(",")})` : "";
         const [cpu, memory, pids, scratch] = columns(category === "tool" ? config.toolCapacity : config.capacity);
-        const row = data(await sql.get(`
-          SELECT q.* FROM ${EXECUTIONS} q
+        return data(await sql.get(`
+          SELECT q.id${includeRequest ? ", q.kind, q.request_json, q.parent_id, q.pool_fingerprint" : ""} FROM ${EXECUTIONS} q
           WHERE q.state = 'queued' AND q.capacity_class = ? AND q.queue_until > ${now} AND q.parent_id ${parentFilter}
           AND (q.pinned_node IS NULL OR q.pinned_node = ?) AND (q.pinned_generation IS NULL OR q.pinned_generation = ?)
           AND q.kind IN (${kinds.map(() => "?").join(",")}) AND (q.environment_key IS NULL${eligibleEnvironment})
@@ -1083,6 +1287,15 @@ function makeStore(config, owner) {
           scratch - Number(reserved.scratch),
           config.maxRunningPerScope
         ]));
+      }
+      try {
+        if (!await eligible(reldb.sql, false))
+          return ok(null);
+      } catch {
+        return unavailable2();
+      }
+      return serialized(async (sql) => {
+        const row = await eligible(sql, true);
         if (!row)
           return ok(null);
         const request = databaseValue(row.request_json);
@@ -1090,11 +1303,12 @@ function makeStore(config, owner) {
           SELECT * FROM ${EXECUTIONS} WHERE id IN (${warmIds.map(() => "?").join(",")}) AND pool_fingerprint = ?
             AND state = 'running' AND warm_state = 'ready' AND leased_until > ${now} ORDER BY created_at, id LIMIT 1`, [...warmIds, row.pool_fingerprint])) : null;
         const changed = data(await sql.execute(`UPDATE ${EXECUTIONS} SET state = 'running', owner_node = ?, owner_generation = ?,
-          container_json = ${resultValue}, leased_until = ${now} + ?, run_until = ${now} + ?, updated_at = ${now}
+          container_json = ${resultValue}, owner_process_json = ${resultValue}, leased_until = ${now} + ?, run_until = ${now} + ?, updated_at = ${now}
           WHERE id = ? AND state = 'queued' AND queue_until > ${now}`, [
           owner.nodeId,
           owner.generation,
           standby?.container_json ? JSON.stringify(databaseValue(standby.container_json)) : null,
+          request.environment ? null : processJson,
           config.leaseMs,
           request.runTimeoutMs,
           row.id
@@ -1134,11 +1348,20 @@ function makeStore(config, owner) {
         const row = await find(reldb.sql, id2, hash([scope.tenantId, scope.userId]));
         if (!row)
           return missing();
-        if (row.state !== "unknown" || row.owner_node !== owner.nodeId || row.reconciled_at !== null || !row.owner_generation || !row.container_json) {
+        if (row.state !== "unknown" || row.owner_node !== owner.nodeId || row.reconciled_at !== null || !row.owner_generation) {
           return conflict();
         }
-        const container = databaseValue(row.container_json);
-        return container.engine === config.engine ? ok({ container, generation: row.owner_generation }) : conflict();
+        if (row.container_json) {
+          const container = databaseValue(row.container_json);
+          return container.engine === config.engine ? ok({ container, generation: row.owner_generation }) : conflict();
+        }
+        if (row.environment_key !== null || !row.owner_process_json || !await parentSettled(reldb.sql, row))
+          return conflict();
+        return ok({
+          container: void 0,
+          process: databaseValue(row.owner_process_json),
+          generation: row.owner_generation
+        });
       } catch {
         return unavailable2();
       }
@@ -1146,10 +1369,16 @@ function makeStore(config, owner) {
     async reconcile(scope, id2, generation, review) {
       return serialized(async (sql) => {
         const scopeKey = hash([scope.tenantId, scope.userId]);
+        if (review.hostResourcesStopped) {
+          const row2 = await find(sql, id2, scopeKey);
+          if (!row2 || !await parentSettled(sql, row2))
+            return conflict();
+        }
+        const proof = review.hostResourcesStopped ? "container_json IS NULL AND environment_key IS NULL AND owner_process_json IS NOT NULL" : "container_json IS NOT NULL";
         const result = data(await sql.execute(`UPDATE ${EXECUTIONS}
           SET reconciled_at = ${now}, reconciliation_json = ${resultValue}, reuse_key = NULL, updated_at = ${now}
           WHERE id = ? AND scope_key = ? AND owner_node = ? AND owner_generation = ? AND state = 'unknown'
-            AND reconciled_at IS NULL AND container_json IS NOT NULL`, [
+            AND reconciled_at IS NULL AND ${proof}`, [
           JSON.stringify(review),
           id2,
           scopeKey,
@@ -1159,7 +1388,7 @@ function makeStore(config, owner) {
         if (result.changes !== 1)
           return conflict();
         const row = await find(sql, id2, scopeKey);
-        return row ? ok(project(row)) : missing();
+        return row ? ok(projectRequest(row)) : missing();
       });
     },
     async renew(id2) {
@@ -1190,10 +1419,18 @@ function makeStore(config, owner) {
       return ok(row.data.state);
     },
     async emit(id2, value) {
+      const encoded = JSON.stringify({ value });
+      const bytes = Buffer.byteLength(encoded);
       const result = await reldb.tx.wrap(async (sql) => {
-        const changed = data(await sql.execute(`UPDATE ${EXECUTIONS} SET event_sequence = event_sequence + 1
+        const changed = data(await sql.execute(`UPDATE ${EXECUTIONS} SET event_sequence = event_sequence + 1, event_bytes = event_bytes + ?
           WHERE id = ? AND owner_generation = ? AND state = 'running' AND leased_until > ${now}
-            AND run_until > ${now} AND event_sequence < ?`, [id2, owner.generation, config.maxEvents]));
+            AND run_until > ${now} AND event_sequence < ? AND event_bytes <= ?`, [
+          bytes,
+          id2,
+          owner.generation,
+          config.maxEvents,
+          MAX_EVENT_BYTES - bytes
+        ]));
         if (changed.changes !== 1)
           return lost();
         const row = data(await sql.get(`SELECT event_sequence FROM ${EXECUTIONS} WHERE id = ?`, [id2]));
@@ -1206,7 +1443,7 @@ function makeStore(config, owner) {
           hash([id2, sequence]),
           id2,
           sequence,
-          JSON.stringify({ value }),
+          encoded,
           id2
         ]));
         if (published.changes !== 1)
@@ -1216,9 +1453,14 @@ function makeStore(config, owner) {
       return result.success ? result.data : unavailable2();
     },
     async events(scope, id2, after, limit) {
-      const result = await this.get(scope, id2);
-      if (!result.success)
-        return result;
+      const owner2 = await reldb.sql.get(`SELECT id FROM ${EXECUTIONS} WHERE id = ? AND scope_key = ?`, [
+        id2,
+        hash([scope.tenantId, scope.userId])
+      ]);
+      if (!owner2.success)
+        return unavailable2();
+      if (!owner2.data)
+        return missing();
       const page = await reldb.sql.query(
         `SELECT sequence_number, event_json, created_at FROM ${EVENTS}
           WHERE execution_id = ? AND sequence_number > ? ORDER BY sequence_number ASC LIMIT ?`,
@@ -1315,6 +1557,7 @@ function createRunner() {
   let timer;
   let tick;
   let sweepAt = 0;
+  let pruneAt = 0;
   let unconfirmedCleanup = false;
   const running = /* @__PURE__ */ new Map();
   const handlers = /* @__PURE__ */ new Map();
@@ -1379,19 +1622,19 @@ function createRunner() {
   }
   function launch(claim, handler, current, settings, holder, warmed) {
     const abort = new AbortController();
-    let process2;
+    let process3;
     let uncertain2 = false;
     const closeProcess = async () => {
-      const started = await process2;
+      const started = await process3;
       if (started?.success)
         return (await started.data.close()).success;
       return warmed ? (await warmed.close()).success : !uncertain2;
     };
     async function start(options) {
       const { environment: environment2, resources: resources2 } = claim.request;
-      if (abort.signal.aborted || process2 || !environment2 || !resources2)
+      if (abort.signal.aborted || process3 || !environment2 || !resources2)
         return invalid2();
-      process2 = (async () => {
+      process3 = (async () => {
         const renewed = await current.renew(claim.id);
         if (!renewed.success)
           return renewed;
@@ -1405,7 +1648,7 @@ function createRunner() {
           uncertain2 = true;
         return started;
       })();
-      return process2;
+      return process3;
     }
     async function execute2() {
       const context = {
@@ -1484,6 +1727,12 @@ function createRunner() {
       }
       if (closingRequested)
         return;
+      if (pruneAt <= performance.now()) {
+        const pruned = await current.prune();
+        if (!pruned.success)
+          logger.warn("Managed execution retention failed", { code: pruned.error.code });
+        pruneAt = performance.now() + (pruned.success && pruned.data ? 250 : 1e3);
+      }
       const kinds = [
         ...handlers.keys()
       ];
@@ -1543,9 +1792,10 @@ function createRunner() {
         generation: randomUUID()
       };
       config = parsed.data;
-      store = makeStore(config, owner);
+      store = makeStore(config, owner, config.executor ? await readRunnerProcess() : void 0);
       unconfirmedCleanup = false;
       sweepAt = 0;
+      pruneAt = 0;
       warmAt = 0;
       if (config.executor) {
         timer = setInterval(poll, config.pollMs);
@@ -1692,7 +1942,7 @@ function createRunner() {
       const current = store;
       if (!current || !config?.executor || closingRequested)
         return inactive();
-      if (!validAddress(scope, id2) || !review || typeof review.operatorId !== "string" || !runnerKind.safeParse(review.operatorId).success || typeof review.reason !== "string" || !review.reason.trim() || review.reason.length > 300 || /[\r\n\0]/u.test(review.reason)) {
+      if (!validAddress(scope, id2) || !review || typeof review.operatorId !== "string" || !runnerKind.safeParse(review.operatorId).success || typeof review.reason !== "string" || review.hostResourcesStopped !== void 0 && review.hostResourcesStopped !== true || !review.reason.trim() || review.reason.length > 300 || /[\r\n\0]/u.test(review.reason)) {
         return invalid2();
       }
       if (running.has(id2))
@@ -1700,14 +1950,19 @@ function createRunner() {
       const evidence = await current.reconciliation(scope, id2);
       if (!evidence.success)
         return evidence;
-      const verified = await confirmContainerReleased(evidence.data.container);
-      if (!verified.success)
-        return verified;
+      if (evidence.data.container) {
+        const verified = await confirmContainerReleased(evidence.data.container);
+        if (!verified.success)
+          return verified;
+      } else if (review.hostResourcesStopped !== true || !await confirmRunnerProcessReleased(evidence.data.process)) {
+        return cleanupUnconfirmed();
+      }
       if (current !== store || closingRequested)
         return inactive();
       const reconciled = await current.reconcile(scope, id2, evidence.data.generation, {
         operatorId: review.operatorId,
-        reason: review.reason.trim()
+        reason: review.reason.trim(),
+        ...!evidence.data.container && { hostResourcesStopped: true }
       });
       if (reconciled.success)
         logger.info("Unknown execution resources reconciled", { executionId: id2, operatorId: review.operatorId });