@bastani/atomic 0.9.44-alpha.2 → 0.9.44-alpha.4

package/dist/builtin/workflows/src/extension/local-worker-entry.mjs

@@ -1661,12 +1661,16 @@ function removeProcessExitCleanup(cleanup) {
   if (processExitCleanups.delete(cleanup) && processExitCleanups.size === 0)
     process.off("exit", runProcessExitCleanups);
 }
+function writtenHandle(event) {
+  if (event.type === RUN_EVENT_TYPES.promptAsked || event.type === RUN_EVENT_TYPES.promptReleased)
+    return event.payload.handle;
+  if (event.type === RUN_EVENT_TYPES.checkpointRecorded || event.type === RUN_EVENT_TYPES.messageSettled || event.type === RUN_EVENT_TYPES.messageDelivered)
+    return;
+  return event.payload;
+}
 function snapshotNeedsHistory(tail) {
-  const event = tail.snapshot.event;
-  if (event.type !== RUN_EVENT_TYPES.messageQueued)
-    return false;
-  const payload = event.payload;
-  return hasPendingStageDeliveryDelta(payload.pendingStageMessages);
+  const handle = writtenHandle(tail.snapshot.event);
+  return handle !== undefined && (handle.pendingStageMessageChange !== undefined || hasPendingStageDeliveryDelta(handle.pendingStageMessages));
 }
 function* invoke(action) {
   try {
@@ -1780,14 +1784,11 @@ class JournalDurableBackend {
       const payload = event.payload;
       this.projection.archivePendingStageDeliveryReceipt(entry.runId, payload.messageId, payload.delivery, payload.accounting);
     } else if (event.type === RUN_EVENT_TYPES.promptAsked || event.type === RUN_EVENT_TYPES.promptReleased) {
-      this.projection.applyWorkflowSnapshot(event.payload.handle);
+      this.projection.applyWorkflowSnapshot(decodeHandle(event.payload.handle, this.projection.getWorkflow(entry.runId)?.pendingStageMessages));
       this.prompts.set(entry.runId, new PromptReservationState(event.payload.handle.pendingPrompts, event.payload.prompts));
     } else {
       const previous = this.projection.getWorkflow(entry.runId);
-      const handle = event.type === RUN_EVENT_TYPES.messageQueued && hasPendingStageDeliveryDelta(event.payload.pendingStageMessages) ? {
-        ...event.payload,
-        pendingStageMessages: decodePendingStageDeliveries(previous?.pendingStageMessages ?? [], event.payload.pendingStageMessages ?? [])
-      } : event.payload;
+      const handle = decodeHandle(event.payload, previous?.pendingStageMessages);
       this.projection.applyWorkflowSnapshot(handle);
       if (event.type === RUN_EVENT_TYPES.runStarted || previous?.pendingPrompts !== handle.pendingPrompts)
         this.prompts.delete(entry.runId);
@@ -1907,6 +1908,24 @@ class JournalDurableBackend {
   }
   *append(runId, event, options) {
     const lease = yield* this.ownership(runId);
+    const pinned = options?.expectSeq;
+    for (let attempt = 0;attempt < CHANGE_APPEND_ATTEMPTS; attempt++) {
+      const base = this.sequences.get(runId);
+      const encoded = base !== undefined && (pinned === undefined || pinned === base) && event.type !== RUN_EVENT_TYPES.runStarted ? encodeHandleEvent(event, this.projection.getWorkflow(runId)?.pendingStageMessages) : undefined;
+      if (encoded === undefined)
+        break;
+      try {
+        return yield* this.write(runId, lease, encoded, { ...options, expectSeq: base });
+      } catch (error) {
+        if (!(error instanceof JournalCasConflictError) || pinned !== undefined)
+          throw error;
+      }
+      for (const entry of yield* this.entries(runId, (this.sequences.get(runId) ?? 0) + 1))
+        this.fold(entry);
+    }
+    yield* this.write(runId, lease, event, options);
+  }
+  *write(runId, lease, event, options) {
     try {
       const outcome = yield* invoke(() => this.journal.append(lease, event, options));
       const nextSeq = (this.sequences.get(runId) ?? 0) + 1;
@@ -2088,17 +2107,6 @@ class JournalDurableBackend {
       pendingStageMessages: replacePendingStageMessagesForRun(handle.pendingStageMessages ?? [], logicalRunId, messages),
       updatedAt: Math.max(Date.now(), handle.updatedAt + 1)
     };
-    const encoded = encodePendingStageDeliveries(handle.pendingStageMessages ?? [], next.pendingStageMessages ?? []);
-    const base = this.sequences.get(runId);
-    if (encoded !== undefined && base !== undefined) {
-      try {
-        yield* this.append(runId, { type: RUN_EVENT_TYPES.messageQueued, payload: { ...next, pendingStageMessages: encoded } }, { expectSeq: base });
-        return true;
-      } catch (error) {
-        if (!(error instanceof JournalCasConflictError))
-          throw error;
-      }
-    }
     yield* this.snapshot(next, RUN_EVENT_TYPES.messageQueued);
     return true;
   }
@@ -2438,31 +2446,143 @@ function uniqueByIdentity(messages) {
 function isDeltaEncoded(message) {
   return message.deliveryDelta !== undefined;
 }
-function encodePendingStageDeliveries(previous, next) {
-  const bases = uniqueByIdentity(previous);
-  const nextIdentities = uniqueByIdentity(next);
-  let encodedAny = false;
-  const encoded = next.map((message) => {
-    const base = bases.get(identity(message))?.deliveries;
-    const deliveries = message.deliveries;
-    if (base === undefined || deliveries === undefined || !nextIdentities.has(identity(message)))
-      return message;
-    const drop = [];
-    let kept = 0;
-    base.forEach((delivery, index) => {
-      if (kept < deliveries.length && JSON.stringify(delivery) === JSON.stringify(deliveries[kept]))
-        kept++;
-      else
-        drop.push(index);
-    });
-    const add = deliveries.slice(kept);
-    if (drop.length + add.length >= deliveries.length)
-      return message;
-    encodedAny = true;
-    const { deliveries: _full, ...rest } = message;
-    return { ...rest, deliveryDelta: drop.length === 0 ? { add } : { drop, add } };
+function deliveryDelta(base, deliveries) {
+  const drop = [];
+  let kept = 0;
+  base.forEach((delivery, index) => {
+    if (kept < deliveries.length && JSON.stringify(delivery) === JSON.stringify(deliveries[kept]))
+      kept++;
+    else
+      drop.push(index);
+  });
+  const add = deliveries.slice(kept);
+  return drop.length === 0 ? { add } : { drop, add };
+}
+function pendingStageFields(message) {
+  return Object.keys(message);
+}
+function copyField(target, source, key) {
+  target[key] = source[key];
+}
+function isRemovedField(message, key) {
+  return message[key] === undefined;
+}
+function json(message) {
+  let text = entryJson.get(message);
+  if (text === undefined) {
+    text = JSON.stringify(message);
+    entryJson.set(message, text);
+  }
+  return text;
+}
+function patchEntry(index, base, message) {
+  const set = {};
+  const unset = pendingStageFields(base).filter((key) => base[key] !== undefined && isRemovedField(message, key));
+  for (const key of pendingStageFields(message)) {
+    if (message[key] === undefined || key === "deliveries")
+      continue;
+    if (JSON.stringify(message[key]) !== JSON.stringify(base[key]))
+      copyField(set, message, key);
+  }
+  let delta;
+  if (message.deliveries !== undefined) {
+    if (base.deliveries === undefined)
+      set.deliveries = message.deliveries;
+    else if (JSON.stringify(base.deliveries) !== JSON.stringify(message.deliveries))
+      delta = deliveryDelta(base.deliveries, message.deliveries);
+  }
+  return {
+    patch: index,
+    ...Object.keys(set).length === 0 ? {} : { set },
+    ...unset.length === 0 ? {} : { unset },
+    ...delta === undefined ? {} : { deliveryDelta: delta }
+  };
+}
+function encodePendingStageList(previous, next) {
+  const positions = new Map;
+  const repeated = new Set;
+  previous.forEach((message, index) => {
+    const key = identity(message);
+    if (positions.has(key))
+      repeated.add(key);
+    positions.set(key, index);
   });
-  return encodedAny ? encoded : undefined;
+  for (const key of repeated)
+    positions.delete(key);
+  const items = [];
+  let reused = 0;
+  for (const message of next) {
+    const index = positions.get(identity(message));
+    const base = index === undefined ? undefined : previous[index];
+    if (base === undefined) {
+      items.push(message);
+      continue;
+    }
+    reused++;
+    const patched = base === message || json(base) === json(message) ? undefined : patchEntry(index, base, message);
+    if (patched === undefined || Object.keys(patched).length === 1) {
+      const last = items.at(-1);
+      if (Array.isArray(last) && last[0] + last[1] === index)
+        last[1]++;
+      else
+        items.push([index, 1]);
+      continue;
+    }
+    items.push(JSON.stringify(patched).length < json(message).length ? patched : message);
+  }
+  return reused > 0 ? items : undefined;
+}
+function decodePendingStageList(previous, items) {
+  const base = (index) => {
+    if (!Number.isInteger(index) || index < 0 || index >= previous.length)
+      throw new Error(`atomic-workflows: journal pending-stage change names missing entry ${index}`);
+    return previous[index];
+  };
+  const result = [];
+  for (const item of items) {
+    if (Array.isArray(item)) {
+      const [start, count] = item;
+      for (let index = start;index < start + count; index++)
+        result.push(base(index));
+    } else if (typeof item.patch === "number") {
+      const { patch, set, unset, deliveryDelta: delta } = item;
+      const entry = { ...base(patch), ...set };
+      for (const key of unset ?? [])
+        delete entry[key];
+      if (delta !== undefined) {
+        const dropped = new Set(delta.drop ?? []);
+        const deliveries = base(patch).deliveries ?? [];
+        entry.deliveries = [...deliveries.filter((_, index) => !dropped.has(index)), ...delta.add];
+      }
+      result.push(entry);
+    } else
+      result.push(item);
+  }
+  return result;
+}
+function encodeHandleEvent(event, previous) {
+  const handle = writtenHandle(event);
+  if (handle?.pendingStageMessages === undefined || previous === undefined)
+    return;
+  const change = encodePendingStageList(previous, handle.pendingStageMessages);
+  if (change === undefined)
+    return;
+  const { pendingStageMessages: _full, ...rest } = handle;
+  const encoded = { ...rest, pendingStageMessageChange: change };
+  if (event.type === RUN_EVENT_TYPES.promptAsked || event.type === RUN_EVENT_TYPES.promptReleased)
+    return { ...event, payload: { ...event.payload, handle: encoded } };
+  return { ...event, payload: encoded };
+}
+function decodeHandle(handle, previous) {
+  const { pendingStageMessageChange: change, ...rest } = handle;
+  if (change !== undefined)
+    return { ...rest, pendingStageMessages: decodePendingStageList(previous ?? [], change) };
+  if (!hasPendingStageDeliveryDelta(handle.pendingStageMessages))
+    return handle;
+  return {
+    ...handle,
+    pendingStageMessages: decodePendingStageDeliveries(previous ?? [], handle.pendingStageMessages ?? [])
+  };
 }
 function decodePendingStageDeliveries(previous, messages) {
   if (!messages.some(isDeltaEncoded))
@@ -2482,7 +2602,7 @@ function decodePendingStageDeliveries(previous, messages) {
 function hasPendingStageDeliveryDelta(messages) {
   return messages?.some(isDeltaEncoded) === true;
 }
-var PROCESS_HOLDER_ID, processExitCleanups, JournalOwnershipError, JournalBackendClosedError, JournalOperationAbortedError, JournalExecutionRetiredError, JournalMessageScopeError;
+var PROCESS_HOLDER_ID, CHANGE_APPEND_ATTEMPTS = 4, processExitCleanups, JournalOwnershipError, JournalBackendClosedError, JournalOperationAbortedError, JournalExecutionRetiredError, JournalMessageScopeError, entryJson;
 var init_journal_backend = __esm(() => {
   init_workflow_instance_ownership_error();
   init_backend();
@@ -2540,6 +2660,7 @@ var init_journal_backend = __esm(() => {
       this.name = "JournalMessageScopeError";
     }
   };
+  entryJson = new WeakMap;
 });
 
 // dist/builtin/workflows/src/durable/journal/legacy-journal-tables.ts
@@ -57532,7 +57653,20 @@ function createWorkflowBoundarySegmentsResolver(runs) {
   };
 }
 function workflowBoundaryHops(runs, runId) {
+  return boundaryHopsFrom(new Map(runs.map((run) => [run.id, run])), runId);
+}
+function createWorkflowBoundaryHopsResolver(runs) {
   const runById = new Map(runs.map((run) => [run.id, run]));
+  const cache = new Map;
+  return (runId) => {
+    if (cache.has(runId))
+      return cache.get(runId);
+    const hops = boundaryHopsFrom(runById, runId);
+    cache.set(runId, hops);
+    return hops;
+  };
+}
+function boundaryHopsFrom(runById, runId) {
   const hops = [];
   let current = runById.get(runId);
   while (current !== undefined && current.parentRunId !== undefined) {
@@ -57558,6 +57692,35 @@ function workflowStagePathVariants(hops, stageSegments) {
 function stageMatchesPathPattern(patternSegments, hops, stageSegments) {
   return workflowStagePathVariants(hops, stageSegments).some((candidate) => matchStagePathSegments(patternSegments, candidate));
 }
+function createStagePathPatternIndex(patterns) {
+  const byFirstSegment = new Map;
+  const globFirst = [];
+  patterns.forEach((pattern, index) => {
+    const first = pattern[0];
+    if (first === undefined || first.includes("*")) {
+      globFirst.push(index);
+      return;
+    }
+    const bucket = byFirstSegment.get(first);
+    if (bucket === undefined)
+      byFirstSegment.set(first, [index]);
+    else
+      bucket.push(index);
+  });
+  return {
+    matching(hops, stageSegments) {
+      if (patterns.length === 0)
+        return [];
+      const variants = workflowStagePathVariants(hops, stageSegments);
+      const candidates = new Set(globFirst);
+      for (const variant of variants) {
+        for (const index of byFirstSegment.get(variant[0]) ?? [])
+          candidates.add(index);
+      }
+      return [...candidates].sort((left, right) => left - right).filter((index) => variants.some((variant) => matchStagePathSegments(patterns[index], variant)));
+    }
+  };
+}
 var init_pending_stage_status = __esm(() => {
   init_store_internal();
   init_workflow_stage_target();
@@ -57568,24 +57731,63 @@ function stickyStageDeliveryStatuses(runs, rootRunId, availableBackend) {
   const root = runs.find((run) => run.id === rootRunId);
   const entries = (root?.pendingStageMessages ?? []).filter((entry) => entry.sticky === true);
   const result = [];
+  const patternByTarget = new Map;
+  const patterns = [];
+  const entriesByPattern = [];
+  const ownEntries = [];
+  entries.forEach((entry, index) => {
+    const target = entry.targetPath ?? entry.stageKey;
+    let pattern = patternByTarget.get(target);
+    if (pattern === undefined) {
+      const parsed = parseWorkflowStageTarget(target);
+      pattern = parsed?.rootRunId === rootRunId ? patterns.push(parsed.segments) - 1 : -1;
+      if (pattern >= 0)
+        entriesByPattern.push([]);
+      patternByTarget.set(target, pattern);
+    }
+    if (pattern < 0)
+      return;
+    entriesByPattern[pattern].push(index);
+    ownEntries.push(index);
+  });
+  if (ownEntries.length === 0)
+    return result;
+  const patternIndex = createStagePathPatternIndex(patterns);
+  const deliveriesByEntry = new Map;
+  const entryReceipts = (index, runId, stageId) => {
+    let byStage = deliveriesByEntry.get(index);
+    if (byStage === undefined) {
+      byStage = new Map;
+      for (const delivery of entries[index].deliveries ?? []) {
+        const key = `${delivery.runId}\x00${delivery.stageId}`;
+        const list = byStage.get(key);
+        if (list === undefined)
+          byStage.set(key, [delivery]);
+        else
+          list.push(delivery);
+      }
+      deliveriesByEntry.set(index, byStage);
+    }
+    return byStage.get(`${runId}\x00${stageId}`) ?? [];
+  };
   availableBackend = availableRunDurableBackend(runs, rootRunId) ?? availableBackend;
   const backend = availableBackend === undefined ? undefined : durableBackendForRun(availableBackend, runs, rootRunId);
+  const runById = new Map(runs.map((run) => [run.id, run]));
+  const hopsFor = createWorkflowBoundaryHopsResolver(runs);
   for (const run of runs) {
-    if (run.id !== rootRunId && run.rootRunId !== rootRunId && durableRootRunIdForRun(runs, run.id) !== rootRunId)
+    const durableRootRunId = reciprocalWorkflowRootRunId(runById, run.id);
+    if (run.id !== rootRunId && run.rootRunId !== rootRunId && durableRootRunId !== rootRunId)
       continue;
-    const resolved = durableRootRunIdForRun(runs, run.id) === rootRunId;
-    const hops = workflowBoundaryHops(runs, run.id);
+    const resolved = durableRootRunId === rootRunId;
+    const hops = hopsFor(run.id);
     for (const stage of run.stages) {
       if (stage.nodeKind === "tool" || stage.replayKey?.startsWith("prompt:") === true || stage.replayKey?.startsWith("workflow:") === true || stage.workflowChildRun !== undefined || stage.workflowChild !== undefined)
         continue;
-      for (const entry of entries) {
+      const matchedEntries = resolved && hops ? patternIndex.matching(hops, [stage.id, stage.name]).flatMap((pattern) => entriesByPattern[pattern]).sort((left, right) => left - right) : ownEntries;
+      for (const index of matchedEntries) {
+        const entry = entries[index];
         const target = entry.targetPath ?? entry.stageKey;
-        const parsed = parseWorkflowStageTarget(target);
-        if (!parsed || parsed.rootRunId !== rootRunId)
-          continue;
-        if (resolved && hops && !stageMatchesPathPattern(parsed.segments, hops, [stage.id, stage.name]))
-          continue;
-        const receipts = (entry.deliveries ?? []).filter((delivery) => delivery.runId === run.id && delivery.stageId === stage.id);
+        const receipts = entryReceipts(index, run.id, stage.id);
         const delivered = backend?.hasCachedPendingStageDeliveryReceipt?.(rootRunId, entry.id, {
           runId: run.id,
           stageId: stage.id,
@@ -57903,20 +58105,27 @@ function possibleStageRows(runs, rootRunId, rootRun, backend) {
   if (isTerminalRunStatus(rootRun.status))
     return [];
   const stickyQueued = (rootRun.pendingStageMessages ?? []).filter((entry) => entry.sticky === true && entry.status === "queued");
+  const segmentsByTarget = new Map;
   const stickySegments = (entry) => {
-    const parsed = parseWorkflowStageTarget(entry.targetPath ?? entry.stageKey);
-    return parsed?.rootRunId === rootRunId ? parsed.segments : undefined;
+    const target = entry.targetPath ?? entry.stageKey;
+    if (segmentsByTarget.has(target))
+      return segmentsByTarget.get(target);
+    const parsed = parseWorkflowStageTarget(target);
+    const segments = parsed?.rootRunId === rootRunId ? parsed.segments : undefined;
+    segmentsByTarget.set(target, segments);
+    return segments;
   };
   const scanEntries = stickyQueued.filter((entry) => {
     const segments = stickySegments(entry);
     return !(segments !== undefined && segments.length === 1 && segments[0] === "**");
   });
+  const liveStageMatches = materializedStageMatcher(runs, rootRunId);
   const rows = [];
   for (const scanEntry of rootRun.possibleStages ?? []) {
     const rowSegments = splitStagePathSegments(scanEntry);
     if (rowSegments.length === 0 || rowSegments.some((segment) => segment.length === 0))
       continue;
-    if (!rowSegments.some((segment) => segment.includes("*")) && materializedStageMatches(runs, rootRunId, rowSegments)) {
+    if (!rowSegments.some((segment) => segment.includes("*")) && liveStageMatches(rowSegments)) {
       continue;
     }
     rows.push({
@@ -57937,28 +58146,43 @@ function possibleStageRows(runs, rootRunId, rootRun, backend) {
     }).length
   });
   const statuses = stickyStageDeliveryStatuses(runs, rootRunId, backend);
-  for (const target of new Set(statuses.map((status) => status.target))) {
-    if (rows.some((row) => row.target === target))
+  const statusesByTarget = new Map;
+  for (const status of statuses) {
+    const forTarget = statusesByTarget.get(status.target);
+    if (forTarget === undefined)
+      statusesByTarget.set(status.target, [status]);
+    else
+      forTarget.push(status);
+  }
+  const queuedByTarget = new Map;
+  for (const entry of stickyQueued) {
+    const target = entry.targetPath ?? entry.stageKey;
+    queuedByTarget.set(target, (queuedByTarget.get(target) ?? 0) + 1);
+  }
+  const rowTargets = new Set(rows.map((row) => row.target));
+  for (const target of statusesByTarget.keys()) {
+    if (rowTargets.has(target))
       continue;
-    rows.push({
-      target,
-      queuedCount: stickyQueued.filter((entry) => (entry.targetPath ?? entry.stageKey) === target).length,
-      deliveryOnly: true
-    });
+    rowTargets.add(target);
+    rows.push({ target, queuedCount: queuedByTarget.get(target) ?? 0, deliveryOnly: true });
   }
   for (let index = 0;index < rows.length; index++) {
     const row = rows[index];
-    const deliveryStates = statuses.filter((status) => status.target === row.target);
-    if (deliveryStates.length > 0)
+    const deliveryStates = statusesByTarget.get(row.target);
+    if (deliveryStates !== undefined)
       rows[index] = { ...row, deliveryStates };
   }
   return rows;
 }
-function materializedStageMatches(runs, rootRunId, entrySegments) {
-  for (const run of runs) {
-    if (durableRootRunIdForRun(runs, run.id) !== rootRunId)
-      continue;
-    const hops = workflowBoundaryHops(runs, run.id);
+function materializedStageMatcher(runs, rootRunId) {
+  const runById = new Map(runs.map((run) => [run.id, run]));
+  const hopsFor = createWorkflowBoundaryHopsResolver(runs);
+  const invocationRuns = runs.filter((run) => reciprocalWorkflowRootRunId(runById, run.id) === rootRunId);
+  return (entrySegments) => materializedStageMatches(invocationRuns, hopsFor, entrySegments);
+}
+function materializedStageMatches(invocationRuns, hopsFor, entrySegments) {
+  for (const run of invocationRuns) {
+    const hops = hopsFor(run.id);
     if (hops === undefined)
       continue;
     for (const stage of run.stages) {
@@ -58233,10 +58457,12 @@ async function recordConfirmedStickyDeliveries(activeStore, event) {
 }
 function matchingLiveStages(runs, rootRunId, patternSegments) {
   const matches = [];
+  const runById = new Map(runs.map((run) => [run.id, run]));
+  const hopsFor = createWorkflowBoundaryHopsResolver(runs);
   for (const run of runs) {
-    if (durableRootRunIdForRun(runs, run.id) !== rootRunId)
+    if (reciprocalWorkflowRootRunId(runById, run.id) !== rootRunId)
       continue;
-    const hops = workflowBoundaryHops(runs, run.id);
+    const hops = hopsFor(run.id);
     if (hops === undefined)
       continue;
     for (const stage of run.stages) {
@@ -58372,11 +58598,26 @@ function writeMessage(socket, msg, callback) {
 }
 function createMessageReader(onMessage, onError) {
   let buffer = Buffer.alloc(0);
+  let pending = [];
+  let pendingLength = 0;
+  let awaitedLength = 0;
   return (data) => {
-    buffer = Buffer.concat([buffer, data]);
+    if (pendingLength + data.length < awaitedLength) {
+      pending.push(data);
+      pendingLength += data.length;
+      return;
+    }
+    buffer = pending.length === 0 ? Buffer.concat([buffer, data]) : Buffer.concat([...pending, data]);
+    pending = [];
+    pendingLength = 0;
+    awaitedLength = 0;
     while (buffer.length >= 4) {
       const length = buffer.readUInt32BE(0);
       if (buffer.length < 4 + length) {
+        pending = [buffer];
+        pendingLength = buffer.length;
+        awaitedLength = 4 + length;
+        buffer = Buffer.alloc(0);
         break;
       }
       const payload = buffer.subarray(4, 4 + length);