deepline 0.3.88 → 0.3.90

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/worker-resource-metrics.ts

@@ -0,0 +1,249 @@
+import { statfsSync } from 'node:fs';
+import { cpus } from 'node:os';
+
+type ReadFile = (path: string, encoding: 'utf8') => string;
+
+export type WorkerResourceMetrics = {
+  cpuUtilizationRatio: number;
+  cpuLimitCores: number;
+  memoryUsageBytes: number;
+  memoryLimitBytes: number;
+  memoryUtilizationRatio: number;
+  filesystemUsageBytes: number;
+  filesystemAvailableBytes: number;
+  filesystemTotalBytes: number;
+  filesystemUtilizationRatio: number;
+};
+
+/** A zero-sized filesystem report means the probe itself is unavailable; a
+ * nonzero filesystem with zero free bytes is the most urgent pressure state. */
+export function hasFilesystemPressure(
+  metrics: Pick<
+    WorkerResourceMetrics,
+    'filesystemAvailableBytes' | 'filesystemTotalBytes'
+  >,
+  reserveBytes: number,
+): boolean {
+  return (
+    metrics.filesystemTotalBytes > 0 &&
+    metrics.filesystemAvailableBytes < reserveBytes
+  );
+}
+
+type FileSystemStats = {
+  bsize: number;
+  blocks: number;
+  bfree: number;
+  bavail: number;
+};
+
+/** Renders the small common CPU/RAM surface used by runtime processes. */
+export function renderRuntimeProcessResourceMetrics(input: {
+  service: 'scheduler' | 'receipt_gateway' | 'execution_gateway' | 'scaler';
+  machine: string;
+  metrics: WorkerResourceMetrics;
+}): string {
+  const labels = `service="${input.service}",machine="${input.machine}"`;
+  return [
+    '# HELP deepline_runtime_process_cpu_utilization_percent Container CPU utilization normalized to its CPU limit.',
+    '# TYPE deepline_runtime_process_cpu_utilization_percent gauge',
+    `deepline_runtime_process_cpu_utilization_percent{${labels}} ${input.metrics.cpuUtilizationRatio * 100}`,
+    '# HELP deepline_runtime_process_cpu_limit_cores Container CPU limit in cores.',
+    '# TYPE deepline_runtime_process_cpu_limit_cores gauge',
+    `deepline_runtime_process_cpu_limit_cores{${labels}} ${input.metrics.cpuLimitCores}`,
+    '# HELP deepline_runtime_process_memory_usage_bytes Container memory currently used.',
+    '# TYPE deepline_runtime_process_memory_usage_bytes gauge',
+    `deepline_runtime_process_memory_usage_bytes{${labels}} ${input.metrics.memoryUsageBytes}`,
+    '# HELP deepline_runtime_process_memory_limit_bytes Container memory limit.',
+    '# TYPE deepline_runtime_process_memory_limit_bytes gauge',
+    `deepline_runtime_process_memory_limit_bytes{${labels}} ${input.metrics.memoryLimitBytes}`,
+    '# HELP deepline_runtime_process_memory_utilization_percent Container memory utilization normalized to its memory limit.',
+    '# TYPE deepline_runtime_process_memory_utilization_percent gauge',
+    `deepline_runtime_process_memory_utilization_percent{${labels}} ${input.metrics.memoryUtilizationRatio * 100}`,
+    '# HELP deepline_runtime_process_filesystem_usage_bytes Local /tmp filesystem space used.',
+    '# TYPE deepline_runtime_process_filesystem_usage_bytes gauge',
+    `deepline_runtime_process_filesystem_usage_bytes{${labels}} ${input.metrics.filesystemUsageBytes}`,
+    '# HELP deepline_runtime_process_filesystem_available_bytes Local /tmp filesystem space available.',
+    '# TYPE deepline_runtime_process_filesystem_available_bytes gauge',
+    `deepline_runtime_process_filesystem_available_bytes{${labels}} ${input.metrics.filesystemAvailableBytes}`,
+    '# HELP deepline_runtime_process_filesystem_total_bytes Local /tmp filesystem total capacity.',
+    '# TYPE deepline_runtime_process_filesystem_total_bytes gauge',
+    `deepline_runtime_process_filesystem_total_bytes{${labels}} ${input.metrics.filesystemTotalBytes}`,
+    '# HELP deepline_runtime_process_filesystem_utilization_percent Local /tmp filesystem utilization normalized to its capacity.',
+    '# TYPE deepline_runtime_process_filesystem_utilization_percent gauge',
+    `deepline_runtime_process_filesystem_utilization_percent{${labels}} ${input.metrics.filesystemUtilizationRatio * 100}`,
+  ].join('\n');
+}
+
+type CpuSample = {
+  usageMicroseconds: number;
+  sampledAtMs: number;
+};
+
+function finiteNonNegative(value: number, fallback: number): number {
+  return Number.isFinite(value) && value >= 0 ? value : fallback;
+}
+
+export function parseCgroupCpuLimit(raw: string, fallback: number): number {
+  const [quota, period] = raw.trim().split(/\s+/, 2);
+  if (!quota || quota === 'max' || !period) return fallback;
+  const parsedQuota = Number(quota);
+  const parsedPeriod = Number(period);
+  if (
+    !Number.isFinite(parsedQuota) ||
+    !Number.isFinite(parsedPeriod) ||
+    parsedQuota <= 0 ||
+    parsedPeriod <= 0
+  ) {
+    return fallback;
+  }
+  return Math.max(parsedQuota / parsedPeriod, 0.01);
+}
+
+/** cgroup v1 reports aggregate CPU time in nanoseconds. */
+export function parseCgroupV1CpuUsageMicroseconds(raw: string): number | null {
+  const nanoseconds = Number(raw.trim());
+  return Number.isFinite(nanoseconds) && nanoseconds >= 0
+    ? nanoseconds / 1_000
+    : null;
+}
+
+export function parseCgroupCpuUsageMicroseconds(raw: string): number | null {
+  const match = raw.match(/^usage_usec\s+(\d+)$/m);
+  if (!match) return null;
+  const usage = Number(match[1]);
+  return Number.isFinite(usage) && usage >= 0 ? usage : null;
+}
+
+export function parseCgroupLimit(raw: string, fallback: number): number {
+  const value = raw.trim();
+  if (!value || value === 'max') return fallback;
+  return finiteNonNegative(Number(value), fallback);
+}
+
+export function parseFileSystemStats(stats: FileSystemStats | null): {
+  usageBytes: number;
+  availableBytes: number;
+  totalBytes: number;
+} {
+  if (!stats) {
+    return { usageBytes: 0, availableBytes: 0, totalBytes: 0 };
+  }
+  const blockSize = finiteNonNegative(stats.bsize, 0);
+  const blocks = finiteNonNegative(stats.blocks, 0);
+  const freeBlocks = Math.min(blocks, finiteNonNegative(stats.bfree, 0));
+  const availableBlocks = Math.min(blocks, finiteNonNegative(stats.bavail, 0));
+  const totalBytes = blocks * blockSize;
+  const availableBytes = availableBlocks * blockSize;
+  return {
+    // `bavail` excludes filesystem-reserved blocks. It is right for admission
+    // (what user work can still allocate), but treating the reservation as
+    // used makes a fresh Fly worker appear 5–6% full. `bfree` is the actual
+    // used-space denominator reported by df.
+    usageBytes: Math.max(0, (blocks - freeBlocks) * blockSize),
+    availableBytes,
+    totalBytes,
+  };
+}
+
+/**
+ * Produces container-scoped CPU and memory gauges for one Fly worker process.
+ * Fly's guest exposes cgroup v1 memory files and may expose cgroup v2 CPU
+ * files. Local development falls back to process RSS and host core count,
+ * preserving a useful metric endpoint without pretending local values are
+ * container quotas.
+ */
+export function createWorkerResourceMetricsReader(input: {
+  readFile: ReadFile;
+  now?: () => number;
+  cpuCount?: () => number;
+  memoryUsage?: () => NodeJS.MemoryUsage;
+  statfs?: (path: string) => FileSystemStats;
+}) {
+  const now = input.now ?? Date.now;
+  const cpuCount = input.cpuCount ?? (() => cpus().length || 1);
+  const memoryUsage = input.memoryUsage ?? process.memoryUsage;
+  const statfs = input.statfs ?? statfsSync;
+  let previousCpu: CpuSample | null = null;
+
+  const read = (path: string): string | null => {
+    try {
+      return input.readFile(path, 'utf8');
+    } catch {
+      return null;
+    }
+  };
+
+  return (): WorkerResourceMetrics => {
+    const fallbackCpuLimit = Math.max(cpuCount(), 1);
+    const cpuQuota = read('/sys/fs/cgroup/cpu/cpu.cfs_quota_us');
+    const cpuPeriod = read('/sys/fs/cgroup/cpu/cpu.cfs_period_us');
+    const cpuLimitCores = parseCgroupCpuLimit(
+      read('/sys/fs/cgroup/cpu.max') ??
+        (cpuQuota !== null && cpuPeriod !== null
+          ? `${cpuQuota} ${cpuPeriod}`
+          : 'max'),
+      fallbackCpuLimit,
+    );
+    const sampledAtMs = now();
+    const cpuUsageMicroseconds =
+      parseCgroupCpuUsageMicroseconds(read('/sys/fs/cgroup/cpu.stat') ?? '') ??
+      parseCgroupV1CpuUsageMicroseconds(
+        read('/sys/fs/cgroup/cpuacct/cpuacct.usage') ?? '',
+      );
+    const cpuUtilizationRatio =
+      cpuUsageMicroseconds === null || previousCpu === null
+        ? 0
+        : Math.max(
+            0,
+            Math.min(
+              1,
+              (cpuUsageMicroseconds - previousCpu.usageMicroseconds) /
+                ((sampledAtMs - previousCpu.sampledAtMs) *
+                  1_000 *
+                  cpuLimitCores),
+            ),
+          );
+    if (cpuUsageMicroseconds !== null) {
+      previousCpu = { usageMicroseconds: cpuUsageMicroseconds, sampledAtMs };
+    }
+
+    const fallbackMemory = memoryUsage().rss;
+    const filesystem = (() => {
+      try {
+        return parseFileSystemStats(statfs('/tmp'));
+      } catch {
+        return parseFileSystemStats(null);
+      }
+    })();
+    const memoryUsageBytes = parseCgroupLimit(
+      read('/sys/fs/cgroup/memory.current') ??
+        read('/sys/fs/cgroup/memory/memory.usage_in_bytes') ??
+        '',
+      fallbackMemory,
+    );
+    const memoryLimitBytes = parseCgroupLimit(
+      read('/sys/fs/cgroup/memory.max') ??
+        read('/sys/fs/cgroup/memory/memory.limit_in_bytes') ??
+        '',
+      fallbackMemory,
+    );
+    return {
+      cpuUtilizationRatio,
+      cpuLimitCores,
+      memoryUsageBytes,
+      memoryLimitBytes,
+      memoryUtilizationRatio:
+        memoryLimitBytes > 0
+          ? Math.min(1, memoryUsageBytes / memoryLimitBytes)
+          : 0,
+      filesystemUsageBytes: filesystem.usageBytes,
+      filesystemAvailableBytes: filesystem.availableBytes,
+      filesystemTotalBytes: filesystem.totalBytes,
+      filesystemUtilizationRatio:
+        filesystem.totalBytes > 0
+          ? Math.min(1, filesystem.usageBytes / filesystem.totalBytes)
+          : 0,
+    };
+  };
+}

package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts

@@ -40,6 +40,13 @@ export function releasableRuntimeSheetAttemptFenceSql(
   )`;
 }
 
+/**
+ * Predicate for durable replay idempotency: matches rows that are already
+ * terminal (enriched or failed) from the same attempt owner + epoch. Used to
+ * suppress same-version conflicts when a worker restarts and re-runs map rows
+ * that it already completed. First terminal state wins; payload/error drift on
+ * replay is accepted as a no-op rather than rejected as a conflict.
+ */
 export function sameOwnerTerminalAttemptEpochSql(
   tableAlias: string,
   attemptOwnerRunIdExpression: string,

package/dist/bundling-sources/shared_libs/play-runtime/step-progress.ts

@@ -0,0 +1,296 @@
+import type { PlayStaticSubstep } from '../plays/static-pipeline';
+import {
+  normalizePlayDocflowNodeIoState,
+  type PlayDocflowNodeIoState,
+} from './/docflow-node-io';
+import type { PlayVisualNodeUsageSnapshot } from './/live-state-contract';
+
+export type PlayStepMetricLike = {
+  pulse?: boolean;
+  tone?: string;
+};
+
+export type PlayVisualNodeStatus =
+  | 'idle'
+  | 'running'
+  | 'completed'
+  | 'failed'
+  | 'skipped';
+
+export type PlayVisualNodeProgress = {
+  completed?: number;
+  total?: number;
+  failed?: number;
+  startedRows?: number;
+  activeRows?: number;
+  waitingRows?: number;
+  completedRows?: number;
+  message?: string;
+  updatedAt?: number | null;
+  startedAt?: number | null;
+  completedAt?: number | null;
+  artifactTableNamespace?: string | null;
+  nodeIo?: PlayDocflowNodeIoState;
+};
+
+export type PlayVisualNodeState = {
+  nodeId: string;
+  status: PlayVisualNodeStatus;
+  artifactTableNamespace?: string | null;
+  progress?: PlayVisualNodeProgress | null;
+  startedAt?: number | null;
+  completedAt?: number | null;
+  updatedAt?: number | null;
+  nodeIo?: PlayDocflowNodeIoState;
+  /** Observed provider usage for this node (ADR 0018). */
+  usage?: PlayVisualNodeUsageSnapshot;
+};
+
+export type PlayVisualNodeProgressMap = Record<string, PlayVisualNodeProgress>;
+
+export function getStaticSubstepNodeId(
+  substep: PlayStaticSubstep,
+  index: number,
+): string {
+  switch (substep.type) {
+    case 'csv':
+      return `csv:${substep.field || index}`;
+    case 'dataset':
+      return `map:${substep.tableNamespace ?? substep.field}`;
+    case 'tool':
+      return `tool:${substep.field}:${substep.toolId}`;
+    case 'waterfall':
+      return `waterfall:${substep.id ?? substep.field}`;
+    case 'step_suite':
+      return `step_suite:${substep.field}`;
+    case 'control_flow':
+      return `control_flow:${substep.kind}:${substep.field}`;
+    case 'play_call':
+      return `play_call:${substep.field}:${substep.playId}`;
+    case 'play_launch':
+      return `play_launch:${substep.field}:${substep.playId}`;
+    case 'run_javascript':
+      return `run_javascript:${substep.alias}`;
+    case 'code':
+      return `code:${substep.field || index}`;
+    default:
+      return `node:${index}`;
+  }
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
+}
+
+export function normalizePlayVisualNodeProgressMap(
+  value: unknown,
+): PlayVisualNodeProgressMap {
+  if (!isRecord(value)) {
+    return {};
+  }
+
+  const normalized: PlayVisualNodeProgressMap = {};
+  for (const [nodeId, rawProgress] of Object.entries(value)) {
+    if (!nodeId.trim() || !isRecord(rawProgress)) {
+      continue;
+    }
+    const completed =
+      typeof rawProgress.completed === 'number' &&
+      Number.isFinite(rawProgress.completed)
+        ? rawProgress.completed
+        : undefined;
+    const total =
+      typeof rawProgress.total === 'number' &&
+      Number.isFinite(rawProgress.total)
+        ? rawProgress.total
+        : undefined;
+    const failed =
+      typeof rawProgress.failed === 'number' &&
+      Number.isFinite(rawProgress.failed)
+        ? rawProgress.failed
+        : undefined;
+    const startedRows =
+      typeof rawProgress.startedRows === 'number' &&
+      Number.isFinite(rawProgress.startedRows)
+        ? rawProgress.startedRows
+        : undefined;
+    const activeRows =
+      typeof rawProgress.activeRows === 'number' &&
+      Number.isFinite(rawProgress.activeRows)
+        ? rawProgress.activeRows
+        : undefined;
+    const waitingRows =
+      typeof rawProgress.waitingRows === 'number' &&
+      Number.isFinite(rawProgress.waitingRows)
+        ? rawProgress.waitingRows
+        : undefined;
+    const completedRows =
+      typeof rawProgress.completedRows === 'number' &&
+      Number.isFinite(rawProgress.completedRows)
+        ? rawProgress.completedRows
+        : undefined;
+    const updatedAt =
+      typeof rawProgress.updatedAt === 'number' &&
+      Number.isFinite(rawProgress.updatedAt)
+        ? rawProgress.updatedAt
+        : undefined;
+    const startedAt =
+      typeof rawProgress.startedAt === 'number' &&
+      Number.isFinite(rawProgress.startedAt)
+        ? rawProgress.startedAt
+        : undefined;
+    const completedAt =
+      typeof rawProgress.completedAt === 'number' &&
+      Number.isFinite(rawProgress.completedAt)
+        ? rawProgress.completedAt
+        : undefined;
+    const message =
+      typeof rawProgress.message === 'string' && rawProgress.message.trim()
+        ? rawProgress.message.trim()
+        : undefined;
+    const artifactTableNamespace =
+      typeof rawProgress.artifactTableNamespace === 'string' &&
+      rawProgress.artifactTableNamespace.trim()
+        ? rawProgress.artifactTableNamespace.trim()
+        : rawProgress.artifactTableNamespace === null
+          ? null
+          : undefined;
+    const nodeIo = normalizePlayDocflowNodeIoState(rawProgress.nodeIo);
+
+    normalized[nodeId] = {
+      ...(completed !== undefined ? { completed } : {}),
+      ...(total !== undefined ? { total } : {}),
+      ...(failed !== undefined ? { failed } : {}),
+      ...(startedRows !== undefined ? { startedRows } : {}),
+      ...(activeRows !== undefined ? { activeRows } : {}),
+      ...(waitingRows !== undefined ? { waitingRows } : {}),
+      ...(completedRows !== undefined ? { completedRows } : {}),
+      ...(updatedAt !== undefined ? { updatedAt } : {}),
+      ...(startedAt !== undefined ? { startedAt } : {}),
+      ...(completedAt !== undefined ? { completedAt } : {}),
+      ...(message !== undefined ? { message } : {}),
+      ...(artifactTableNamespace !== undefined
+        ? { artifactTableNamespace }
+        : {}),
+      ...(nodeIo ? { nodeIo } : {}),
+    };
+  }
+  return normalized;
+}
+
+export function applyVisualNodeProgress(input: {
+  nodeStates: readonly PlayVisualNodeState[];
+  progressByNode: PlayVisualNodeProgressMap;
+}): PlayVisualNodeState[] {
+  const activeProgressIndex = input.nodeStates.findIndex((nodeState) => {
+    const progress = input.progressByNode[nodeState.nodeId];
+    if (!progress) {
+      return false;
+    }
+    const completed = Number(progress.completed ?? 0);
+    const total = Number(progress.total ?? 0);
+    return (
+      Number.isFinite(total) &&
+      total > 0 &&
+      Number.isFinite(completed) &&
+      completed < total
+    );
+  });
+
+  return input.nodeStates.map((nodeState, index) => {
+    const progress = input.progressByNode[nodeState.nodeId] ?? null;
+    if (
+      !progress &&
+      activeProgressIndex >= 0 &&
+      index > activeProgressIndex &&
+      nodeState.status === 'running'
+    ) {
+      return {
+        ...nodeState,
+        status: 'idle',
+      };
+    }
+    if (!progress) {
+      return nodeState;
+    }
+    const completed = Number(progress.completed ?? 0);
+    const total = Number(progress.total ?? 0);
+    const progressStatus =
+      nodeState.status === 'failed'
+        ? 'failed'
+        : Number.isFinite(total) && total > 0
+          ? Number.isFinite(completed) && completed >= total
+            ? 'completed'
+            : 'running'
+          : progress.completedAt
+            ? 'completed'
+            : nodeState.status;
+    return {
+      ...nodeState,
+      status: progressStatus,
+      artifactTableNamespace:
+        nodeState.artifactTableNamespace ??
+        progress.artifactTableNamespace ??
+        null,
+      startedAt: nodeState.startedAt ?? progress.startedAt ?? null,
+      completedAt: nodeState.completedAt ?? progress.completedAt ?? null,
+      progress,
+    };
+  });
+}
+
+export function findActiveStepMetricIndex(
+  metrics: readonly (PlayStepMetricLike | null | undefined)[],
+): number | null {
+  const runningIndex = metrics.findIndex(
+    (metric) => metric?.pulse || metric?.tone === 'running',
+  );
+  if (runningIndex >= 0) {
+    return runningIndex;
+  }
+
+  const waitingIndex = metrics.findIndex(
+    (metric) => metric?.tone === 'waiting',
+  );
+  return waitingIndex >= 0 ? waitingIndex : null;
+}
+
+export function resolveSubstepSequenceStatus(input: {
+  substeps: readonly PlayStaticSubstep[];
+  substep: PlayStaticSubstep;
+  nodeStates?: readonly PlayVisualNodeState[];
+  liveStatus?: string;
+}): string | undefined {
+  const nodeId = getStaticSubstepNodeId(
+    input.substep,
+    input.substeps.indexOf(input.substep),
+  );
+  const derivedStatus = input.nodeStates?.find(
+    (candidate) => candidate.nodeId === nodeId,
+  )?.status;
+  if (derivedStatus) {
+    if (derivedStatus === 'idle') {
+      return undefined;
+    }
+    return derivedStatus;
+  }
+
+  const normalizedLiveStatus = input.liveStatus?.toLowerCase();
+  if (!normalizedLiveStatus) {
+    return undefined;
+  }
+
+  if (
+    normalizedLiveStatus === 'completed' ||
+    normalizedLiveStatus === 'failed' ||
+    normalizedLiveStatus === 'cancelled' ||
+    normalizedLiveStatus === 'terminated'
+  ) {
+    return normalizedLiveStatus;
+  }
+
+  if (normalizedLiveStatus !== 'running') {
+    return normalizedLiveStatus;
+  }
+  return 'running';
+}

package/dist/bundling-sources/shared_libs/play-runtime/tool-call/native-batch.ts

@@ -35,7 +35,7 @@ export type NativeBatchToolCallInput = {
    */
   buildPhysicalCall(input: {
     members: NativeBatchOwnedToolCallMember[];
-  }): ToolCallInput;
+  }): ToolCallInput | Promise<ToolCallInput>;
   /**
    * Decodes one aggregate provider result into exactly one result for every
    * owned receipt. The Module validates the correspondence before publishing
@@ -88,9 +88,9 @@ export function createNativeBatchToolCallJob(dependencies: {
       const executeOwned = async (): Promise<void> => {
         const owned = ownedMembers(cohort, input.members);
         if (owned.length === 0) return;
-        const physical = input.buildPhysicalCall({ members: owned });
-        validatePhysicalCall(physical, input.owner);
         try {
+          const physical = await input.buildPhysicalCall({ members: owned });
+          validatePhysicalCall(physical, input.owner);
           const aggregate = await dependencies.physicalToolCall.call(physical);
           const split = await input.splitResult({
             result: aggregate,

package/dist/bundling-sources/shared_libs/play-runtime/tool-call/runtime-step-receipts-adapter.ts

@@ -81,6 +81,15 @@ export type RuntimeStepToolResultReceiptsInput = {
     resultReceiptKey: string;
     source: ToolCallReceiptRecoverySource | 'owner';
   }) => ToolExecuteResult;
+  /**
+   * Best-effort observer after a canonical completed result converges. The
+   * adapter keeps receipt semantics; callers can use this to record a compact
+   * source-run replay pointer without another provider operation.
+   */
+  onStored?: (input: {
+    result: ToolExecuteResult;
+    resultReceiptKey: string;
+  }) => void | Promise<void>;
   /** Injectable only for deterministic adapter conformance tests. */
   sleep?: (milliseconds: number) => Promise<void>;
 };
@@ -433,6 +442,10 @@ export function createRuntimeStepToolResultReceipts(
             ? null
             : completedResult(persisted, record.resultReceiptKey);
         if (result) {
+          await input.onStored?.({
+            result,
+            resultReceiptKey: record.resultReceiptKey,
+          });
           return {
             // The legacy storage operation returns the converged completed row
             // for both an insert and a concurrent winner. The exact distinction
@@ -443,7 +456,13 @@ export function createRuntimeStepToolResultReceipts(
           };
         }
         const converged = await readCompleted(record.resultReceiptKey, 'owner');
-        if (converged) return { kind: 'existing', result: converged };
+        if (converged) {
+          await input.onStored?.({
+            result: converged,
+            resultReceiptKey: record.resultReceiptKey,
+          });
+          return { kind: 'existing', result: converged };
+        }
         throw new Error(
           `Tool Result Receipt ${record.resultReceiptKey} could not publish a completed result.`,
         );