@bastani/atomic 0.9.44-alpha.2 → 0.9.44-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/dist/builtin/intercom/CHANGELOG.md +7 -0
- package/dist/builtin/intercom/broker/framing.ts +18 -1
- package/dist/builtin/intercom/index.bundle.mjs +19 -4
- package/dist/builtin/intercom/package.json +1 -1
- package/dist/builtin/mcp/package.json +1 -1
- package/dist/builtin/subagents/package.json +1 -1
- package/dist/builtin/web-access/index.bundle.mjs +2510 -1308
- package/dist/builtin/web-access/package.json +1 -1
- package/dist/builtin/workflows/CHANGELOG.md +13 -0
- package/dist/builtin/workflows/package.json +1 -1
- package/dist/builtin/workflows/src/extension/index.bundle.mjs +317 -76
- package/dist/builtin/workflows/src/extension/local-worker-entry.mjs +317 -76
- package/dist/builtin/workflows/src/index.js +317 -76
- package/dist/builtin/workflows/src/local-worker-entry.mjs +317 -76
- package/examples/extensions/gondolin/package-lock.json +3 -3
- package/npm-shrinkwrap.json +67 -51
- package/package.json +4 -4
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bastani/web-access",
|
|
3
|
-
"version": "0.9.44-alpha.
|
|
3
|
+
"version": "0.9.44-alpha.4",
|
|
4
4
|
"private": true,
|
|
5
5
|
"description": "Atomic extension for web search, URL fetching, GitHub repo cloning, PDF/video extraction. Fork of: https://github.com/nicobailon/pi-web-access",
|
|
6
6
|
"contributors": [
|
|
@@ -6,6 +6,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
8
|
|
|
9
|
+
## [0.9.44-alpha.4] - 2026-10-25
|
|
10
|
+
|
|
11
|
+
### Fixed
|
|
12
|
+
|
|
13
|
+
- A pending-message write that loses a race with another writer to the same run (for example a stage's delivery receipt) no longer stores the run's whole list of pending stage messages in the journal. It is rebuilt as a change on top of the other writer's entry and retried, and only falls back to the full list after repeated conflicts. In a 160-worker fan-out, 2% of queued events had fallen back and written 93% of their bytes, up to 1.5 MB each and growing with the list ([#3929](https://github.com/bastani-inc/atomic/issues/3929))
|
|
14
|
+
|
|
15
|
+
## [0.9.44-alpha.3] - 2026-10-25
|
|
16
|
+
|
|
17
|
+
### Fixed
|
|
18
|
+
|
|
19
|
+
- A workflow whose stages send many Intercom messages to stage patterns (for example `workflow:<rootRunId>/child-*/**`) no longer saturates the workflow host. The host matched every queued pattern message against every stage on each change; it now indexes the patterns and each stage's paths once. In a 150-child fan-out test, delivered sends went from 566 of 900 to 900 of 900 and host event-loop lag p99 from about 1 s to under 200 ms ([#3931](https://github.com/bastani-inc/atomic/issues/3931))
|
|
20
|
+
- Workflow journal events no longer store a run's whole list of pending stage messages each time. Pattern sends stay queued until the root run ends, so every queued message rewrote the growing list and the journal grew quadratically with the number of sends. Each event now records only what changed; older journals still read as before ([#3929](https://github.com/bastani-inc/atomic/issues/3929))
|
|
21
|
+
|
|
9
22
|
## [0.9.44-alpha.2] - 2026-10-24
|
|
10
23
|
|
|
11
24
|
### Fixed
|
|
@@ -2120,7 +2120,20 @@ function createWorkflowBoundarySegmentsResolver(runs) {
|
|
|
2120
2120
|
};
|
|
2121
2121
|
}
|
|
2122
2122
|
function workflowBoundaryHops(runs, runId) {
|
|
2123
|
+
return boundaryHopsFrom(new Map(runs.map((run) => [run.id, run])), runId);
|
|
2124
|
+
}
|
|
2125
|
+
function createWorkflowBoundaryHopsResolver(runs) {
|
|
2123
2126
|
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
2127
|
+
const cache = new Map;
|
|
2128
|
+
return (runId) => {
|
|
2129
|
+
if (cache.has(runId))
|
|
2130
|
+
return cache.get(runId);
|
|
2131
|
+
const hops = boundaryHopsFrom(runById, runId);
|
|
2132
|
+
cache.set(runId, hops);
|
|
2133
|
+
return hops;
|
|
2134
|
+
};
|
|
2135
|
+
}
|
|
2136
|
+
function boundaryHopsFrom(runById, runId) {
|
|
2124
2137
|
const hops = [];
|
|
2125
2138
|
let current = runById.get(runId);
|
|
2126
2139
|
while (current !== undefined && current.parentRunId !== undefined) {
|
|
@@ -2146,6 +2159,35 @@ function workflowStagePathVariants(hops, stageSegments) {
|
|
|
2146
2159
|
function stageMatchesPathPattern(patternSegments, hops, stageSegments) {
|
|
2147
2160
|
return workflowStagePathVariants(hops, stageSegments).some((candidate) => matchStagePathSegments(patternSegments, candidate));
|
|
2148
2161
|
}
|
|
2162
|
+
function createStagePathPatternIndex(patterns) {
|
|
2163
|
+
const byFirstSegment = new Map;
|
|
2164
|
+
const globFirst = [];
|
|
2165
|
+
patterns.forEach((pattern, index) => {
|
|
2166
|
+
const first = pattern[0];
|
|
2167
|
+
if (first === undefined || first.includes("*")) {
|
|
2168
|
+
globFirst.push(index);
|
|
2169
|
+
return;
|
|
2170
|
+
}
|
|
2171
|
+
const bucket = byFirstSegment.get(first);
|
|
2172
|
+
if (bucket === undefined)
|
|
2173
|
+
byFirstSegment.set(first, [index]);
|
|
2174
|
+
else
|
|
2175
|
+
bucket.push(index);
|
|
2176
|
+
});
|
|
2177
|
+
return {
|
|
2178
|
+
matching(hops, stageSegments) {
|
|
2179
|
+
if (patterns.length === 0)
|
|
2180
|
+
return [];
|
|
2181
|
+
const variants = workflowStagePathVariants(hops, stageSegments);
|
|
2182
|
+
const candidates = new Set(globFirst);
|
|
2183
|
+
for (const variant of variants) {
|
|
2184
|
+
for (const index of byFirstSegment.get(variant[0]) ?? [])
|
|
2185
|
+
candidates.add(index);
|
|
2186
|
+
}
|
|
2187
|
+
return [...candidates].sort((left, right) => left - right).filter((index) => variants.some((variant) => matchStagePathSegments(patterns[index], variant)));
|
|
2188
|
+
}
|
|
2189
|
+
};
|
|
2190
|
+
}
|
|
2149
2191
|
function pendingStageTarget(runId, stage) {
|
|
2150
2192
|
return "workflowGraphTarget" in stage ? stage.workflowGraphTarget : { runId, stageId: stage.id };
|
|
2151
2193
|
}
|
|
@@ -6808,12 +6850,16 @@ function removeProcessExitCleanup(cleanup) {
|
|
|
6808
6850
|
if (processExitCleanups.delete(cleanup) && processExitCleanups.size === 0)
|
|
6809
6851
|
process.off("exit", runProcessExitCleanups);
|
|
6810
6852
|
}
|
|
6853
|
+
function writtenHandle(event) {
|
|
6854
|
+
if (event.type === RUN_EVENT_TYPES.promptAsked || event.type === RUN_EVENT_TYPES.promptReleased)
|
|
6855
|
+
return event.payload.handle;
|
|
6856
|
+
if (event.type === RUN_EVENT_TYPES.checkpointRecorded || event.type === RUN_EVENT_TYPES.messageSettled || event.type === RUN_EVENT_TYPES.messageDelivered)
|
|
6857
|
+
return;
|
|
6858
|
+
return event.payload;
|
|
6859
|
+
}
|
|
6811
6860
|
function snapshotNeedsHistory(tail) {
|
|
6812
|
-
const
|
|
6813
|
-
|
|
6814
|
-
return false;
|
|
6815
|
-
const payload = event.payload;
|
|
6816
|
-
return hasPendingStageDeliveryDelta(payload.pendingStageMessages);
|
|
6861
|
+
const handle = writtenHandle(tail.snapshot.event);
|
|
6862
|
+
return handle !== undefined && (handle.pendingStageMessageChange !== undefined || hasPendingStageDeliveryDelta(handle.pendingStageMessages));
|
|
6817
6863
|
}
|
|
6818
6864
|
function* invoke(action) {
|
|
6819
6865
|
try {
|
|
@@ -6927,14 +6973,11 @@ class JournalDurableBackend {
|
|
|
6927
6973
|
const payload = event.payload;
|
|
6928
6974
|
this.projection.archivePendingStageDeliveryReceipt(entry.runId, payload.messageId, payload.delivery, payload.accounting);
|
|
6929
6975
|
} else if (event.type === RUN_EVENT_TYPES.promptAsked || event.type === RUN_EVENT_TYPES.promptReleased) {
|
|
6930
|
-
this.projection.applyWorkflowSnapshot(event.payload.handle);
|
|
6976
|
+
this.projection.applyWorkflowSnapshot(decodeHandle(event.payload.handle, this.projection.getWorkflow(entry.runId)?.pendingStageMessages));
|
|
6931
6977
|
this.prompts.set(entry.runId, new PromptReservationState(event.payload.handle.pendingPrompts, event.payload.prompts));
|
|
6932
6978
|
} else {
|
|
6933
6979
|
const previous = this.projection.getWorkflow(entry.runId);
|
|
6934
|
-
const handle =
|
|
6935
|
-
...event.payload,
|
|
6936
|
-
pendingStageMessages: decodePendingStageDeliveries(previous?.pendingStageMessages ?? [], event.payload.pendingStageMessages ?? [])
|
|
6937
|
-
} : event.payload;
|
|
6980
|
+
const handle = decodeHandle(event.payload, previous?.pendingStageMessages);
|
|
6938
6981
|
this.projection.applyWorkflowSnapshot(handle);
|
|
6939
6982
|
if (event.type === RUN_EVENT_TYPES.runStarted || previous?.pendingPrompts !== handle.pendingPrompts)
|
|
6940
6983
|
this.prompts.delete(entry.runId);
|
|
@@ -7054,6 +7097,24 @@ class JournalDurableBackend {
|
|
|
7054
7097
|
}
|
|
7055
7098
|
*append(runId, event, options) {
|
|
7056
7099
|
const lease = yield* this.ownership(runId);
|
|
7100
|
+
const pinned = options?.expectSeq;
|
|
7101
|
+
for (let attempt = 0;attempt < CHANGE_APPEND_ATTEMPTS; attempt++) {
|
|
7102
|
+
const base = this.sequences.get(runId);
|
|
7103
|
+
const encoded = base !== undefined && (pinned === undefined || pinned === base) && event.type !== RUN_EVENT_TYPES.runStarted ? encodeHandleEvent(event, this.projection.getWorkflow(runId)?.pendingStageMessages) : undefined;
|
|
7104
|
+
if (encoded === undefined)
|
|
7105
|
+
break;
|
|
7106
|
+
try {
|
|
7107
|
+
return yield* this.write(runId, lease, encoded, { ...options, expectSeq: base });
|
|
7108
|
+
} catch (error) {
|
|
7109
|
+
if (!(error instanceof JournalCasConflictError) || pinned !== undefined)
|
|
7110
|
+
throw error;
|
|
7111
|
+
}
|
|
7112
|
+
for (const entry of yield* this.entries(runId, (this.sequences.get(runId) ?? 0) + 1))
|
|
7113
|
+
this.fold(entry);
|
|
7114
|
+
}
|
|
7115
|
+
yield* this.write(runId, lease, event, options);
|
|
7116
|
+
}
|
|
7117
|
+
*write(runId, lease, event, options) {
|
|
7057
7118
|
try {
|
|
7058
7119
|
const outcome = yield* invoke(() => this.journal.append(lease, event, options));
|
|
7059
7120
|
const nextSeq = (this.sequences.get(runId) ?? 0) + 1;
|
|
@@ -7235,17 +7296,6 @@ class JournalDurableBackend {
|
|
|
7235
7296
|
pendingStageMessages: replacePendingStageMessagesForRun(handle.pendingStageMessages ?? [], logicalRunId, messages),
|
|
7236
7297
|
updatedAt: Math.max(Date.now(), handle.updatedAt + 1)
|
|
7237
7298
|
};
|
|
7238
|
-
const encoded = encodePendingStageDeliveries(handle.pendingStageMessages ?? [], next.pendingStageMessages ?? []);
|
|
7239
|
-
const base = this.sequences.get(runId);
|
|
7240
|
-
if (encoded !== undefined && base !== undefined) {
|
|
7241
|
-
try {
|
|
7242
|
-
yield* this.append(runId, { type: RUN_EVENT_TYPES.messageQueued, payload: { ...next, pendingStageMessages: encoded } }, { expectSeq: base });
|
|
7243
|
-
return true;
|
|
7244
|
-
} catch (error) {
|
|
7245
|
-
if (!(error instanceof JournalCasConflictError))
|
|
7246
|
-
throw error;
|
|
7247
|
-
}
|
|
7248
|
-
}
|
|
7249
7299
|
yield* this.snapshot(next, RUN_EVENT_TYPES.messageQueued);
|
|
7250
7300
|
return true;
|
|
7251
7301
|
}
|
|
@@ -7585,31 +7635,143 @@ function uniqueByIdentity(messages) {
|
|
|
7585
7635
|
function isDeltaEncoded(message) {
|
|
7586
7636
|
return message.deliveryDelta !== undefined;
|
|
7587
7637
|
}
|
|
7588
|
-
function
|
|
7589
|
-
const
|
|
7590
|
-
|
|
7591
|
-
|
|
7592
|
-
|
|
7593
|
-
|
|
7594
|
-
|
|
7595
|
-
|
|
7596
|
-
return message;
|
|
7597
|
-
const drop = [];
|
|
7598
|
-
let kept = 0;
|
|
7599
|
-
base.forEach((delivery, index) => {
|
|
7600
|
-
if (kept < deliveries.length && JSON.stringify(delivery) === JSON.stringify(deliveries[kept]))
|
|
7601
|
-
kept++;
|
|
7602
|
-
else
|
|
7603
|
-
drop.push(index);
|
|
7604
|
-
});
|
|
7605
|
-
const add = deliveries.slice(kept);
|
|
7606
|
-
if (drop.length + add.length >= deliveries.length)
|
|
7607
|
-
return message;
|
|
7608
|
-
encodedAny = true;
|
|
7609
|
-
const { deliveries: _full, ...rest } = message;
|
|
7610
|
-
return { ...rest, deliveryDelta: drop.length === 0 ? { add } : { drop, add } };
|
|
7638
|
+
function deliveryDelta(base, deliveries) {
|
|
7639
|
+
const drop = [];
|
|
7640
|
+
let kept = 0;
|
|
7641
|
+
base.forEach((delivery, index) => {
|
|
7642
|
+
if (kept < deliveries.length && JSON.stringify(delivery) === JSON.stringify(deliveries[kept]))
|
|
7643
|
+
kept++;
|
|
7644
|
+
else
|
|
7645
|
+
drop.push(index);
|
|
7611
7646
|
});
|
|
7612
|
-
|
|
7647
|
+
const add = deliveries.slice(kept);
|
|
7648
|
+
return drop.length === 0 ? { add } : { drop, add };
|
|
7649
|
+
}
|
|
7650
|
+
function pendingStageFields(message) {
|
|
7651
|
+
return Object.keys(message);
|
|
7652
|
+
}
|
|
7653
|
+
function copyField(target, source, key) {
|
|
7654
|
+
target[key] = source[key];
|
|
7655
|
+
}
|
|
7656
|
+
function isRemovedField(message, key) {
|
|
7657
|
+
return message[key] === undefined;
|
|
7658
|
+
}
|
|
7659
|
+
function json(message) {
|
|
7660
|
+
let text = entryJson.get(message);
|
|
7661
|
+
if (text === undefined) {
|
|
7662
|
+
text = JSON.stringify(message);
|
|
7663
|
+
entryJson.set(message, text);
|
|
7664
|
+
}
|
|
7665
|
+
return text;
|
|
7666
|
+
}
|
|
7667
|
+
function patchEntry(index, base, message) {
|
|
7668
|
+
const set = {};
|
|
7669
|
+
const unset = pendingStageFields(base).filter((key) => base[key] !== undefined && isRemovedField(message, key));
|
|
7670
|
+
for (const key of pendingStageFields(message)) {
|
|
7671
|
+
if (message[key] === undefined || key === "deliveries")
|
|
7672
|
+
continue;
|
|
7673
|
+
if (JSON.stringify(message[key]) !== JSON.stringify(base[key]))
|
|
7674
|
+
copyField(set, message, key);
|
|
7675
|
+
}
|
|
7676
|
+
let delta;
|
|
7677
|
+
if (message.deliveries !== undefined) {
|
|
7678
|
+
if (base.deliveries === undefined)
|
|
7679
|
+
set.deliveries = message.deliveries;
|
|
7680
|
+
else if (JSON.stringify(base.deliveries) !== JSON.stringify(message.deliveries))
|
|
7681
|
+
delta = deliveryDelta(base.deliveries, message.deliveries);
|
|
7682
|
+
}
|
|
7683
|
+
return {
|
|
7684
|
+
patch: index,
|
|
7685
|
+
...Object.keys(set).length === 0 ? {} : { set },
|
|
7686
|
+
...unset.length === 0 ? {} : { unset },
|
|
7687
|
+
...delta === undefined ? {} : { deliveryDelta: delta }
|
|
7688
|
+
};
|
|
7689
|
+
}
|
|
7690
|
+
function encodePendingStageList(previous, next) {
|
|
7691
|
+
const positions = new Map;
|
|
7692
|
+
const repeated = new Set;
|
|
7693
|
+
previous.forEach((message, index) => {
|
|
7694
|
+
const key = identity(message);
|
|
7695
|
+
if (positions.has(key))
|
|
7696
|
+
repeated.add(key);
|
|
7697
|
+
positions.set(key, index);
|
|
7698
|
+
});
|
|
7699
|
+
for (const key of repeated)
|
|
7700
|
+
positions.delete(key);
|
|
7701
|
+
const items = [];
|
|
7702
|
+
let reused = 0;
|
|
7703
|
+
for (const message of next) {
|
|
7704
|
+
const index = positions.get(identity(message));
|
|
7705
|
+
const base = index === undefined ? undefined : previous[index];
|
|
7706
|
+
if (base === undefined) {
|
|
7707
|
+
items.push(message);
|
|
7708
|
+
continue;
|
|
7709
|
+
}
|
|
7710
|
+
reused++;
|
|
7711
|
+
const patched = base === message || json(base) === json(message) ? undefined : patchEntry(index, base, message);
|
|
7712
|
+
if (patched === undefined || Object.keys(patched).length === 1) {
|
|
7713
|
+
const last = items.at(-1);
|
|
7714
|
+
if (Array.isArray(last) && last[0] + last[1] === index)
|
|
7715
|
+
last[1]++;
|
|
7716
|
+
else
|
|
7717
|
+
items.push([index, 1]);
|
|
7718
|
+
continue;
|
|
7719
|
+
}
|
|
7720
|
+
items.push(JSON.stringify(patched).length < json(message).length ? patched : message);
|
|
7721
|
+
}
|
|
7722
|
+
return reused > 0 ? items : undefined;
|
|
7723
|
+
}
|
|
7724
|
+
function decodePendingStageList(previous, items) {
|
|
7725
|
+
const base = (index) => {
|
|
7726
|
+
if (!Number.isInteger(index) || index < 0 || index >= previous.length)
|
|
7727
|
+
throw new Error(`atomic-workflows: journal pending-stage change names missing entry ${index}`);
|
|
7728
|
+
return previous[index];
|
|
7729
|
+
};
|
|
7730
|
+
const result = [];
|
|
7731
|
+
for (const item of items) {
|
|
7732
|
+
if (Array.isArray(item)) {
|
|
7733
|
+
const [start, count] = item;
|
|
7734
|
+
for (let index = start;index < start + count; index++)
|
|
7735
|
+
result.push(base(index));
|
|
7736
|
+
} else if (typeof item.patch === "number") {
|
|
7737
|
+
const { patch, set, unset, deliveryDelta: delta } = item;
|
|
7738
|
+
const entry = { ...base(patch), ...set };
|
|
7739
|
+
for (const key of unset ?? [])
|
|
7740
|
+
delete entry[key];
|
|
7741
|
+
if (delta !== undefined) {
|
|
7742
|
+
const dropped = new Set(delta.drop ?? []);
|
|
7743
|
+
const deliveries = base(patch).deliveries ?? [];
|
|
7744
|
+
entry.deliveries = [...deliveries.filter((_, index) => !dropped.has(index)), ...delta.add];
|
|
7745
|
+
}
|
|
7746
|
+
result.push(entry);
|
|
7747
|
+
} else
|
|
7748
|
+
result.push(item);
|
|
7749
|
+
}
|
|
7750
|
+
return result;
|
|
7751
|
+
}
|
|
7752
|
+
function encodeHandleEvent(event, previous) {
|
|
7753
|
+
const handle = writtenHandle(event);
|
|
7754
|
+
if (handle?.pendingStageMessages === undefined || previous === undefined)
|
|
7755
|
+
return;
|
|
7756
|
+
const change = encodePendingStageList(previous, handle.pendingStageMessages);
|
|
7757
|
+
if (change === undefined)
|
|
7758
|
+
return;
|
|
7759
|
+
const { pendingStageMessages: _full, ...rest } = handle;
|
|
7760
|
+
const encoded = { ...rest, pendingStageMessageChange: change };
|
|
7761
|
+
if (event.type === RUN_EVENT_TYPES.promptAsked || event.type === RUN_EVENT_TYPES.promptReleased)
|
|
7762
|
+
return { ...event, payload: { ...event.payload, handle: encoded } };
|
|
7763
|
+
return { ...event, payload: encoded };
|
|
7764
|
+
}
|
|
7765
|
+
function decodeHandle(handle, previous) {
|
|
7766
|
+
const { pendingStageMessageChange: change, ...rest } = handle;
|
|
7767
|
+
if (change !== undefined)
|
|
7768
|
+
return { ...rest, pendingStageMessages: decodePendingStageList(previous ?? [], change) };
|
|
7769
|
+
if (!hasPendingStageDeliveryDelta(handle.pendingStageMessages))
|
|
7770
|
+
return handle;
|
|
7771
|
+
return {
|
|
7772
|
+
...handle,
|
|
7773
|
+
pendingStageMessages: decodePendingStageDeliveries(previous ?? [], handle.pendingStageMessages ?? [])
|
|
7774
|
+
};
|
|
7613
7775
|
}
|
|
7614
7776
|
function decodePendingStageDeliveries(previous, messages) {
|
|
7615
7777
|
if (!messages.some(isDeltaEncoded))
|
|
@@ -7629,7 +7791,7 @@ function decodePendingStageDeliveries(previous, messages) {
|
|
|
7629
7791
|
function hasPendingStageDeliveryDelta(messages) {
|
|
7630
7792
|
return messages?.some(isDeltaEncoded) === true;
|
|
7631
7793
|
}
|
|
7632
|
-
var PROCESS_HOLDER_ID, processExitCleanups, JournalOwnershipError, JournalBackendClosedError, JournalOperationAbortedError, JournalExecutionRetiredError, JournalMessageScopeError;
|
|
7794
|
+
var PROCESS_HOLDER_ID, CHANGE_APPEND_ATTEMPTS = 4, processExitCleanups, JournalOwnershipError, JournalBackendClosedError, JournalOperationAbortedError, JournalExecutionRetiredError, JournalMessageScopeError, entryJson;
|
|
7633
7795
|
var init_journal_backend = __esm(() => {
|
|
7634
7796
|
init_workflow_instance_ownership_error();
|
|
7635
7797
|
init_backend();
|
|
@@ -7687,6 +7849,7 @@ var init_journal_backend = __esm(() => {
|
|
|
7687
7849
|
this.name = "JournalMessageScopeError";
|
|
7688
7850
|
}
|
|
7689
7851
|
};
|
|
7852
|
+
entryJson = new WeakMap;
|
|
7690
7853
|
});
|
|
7691
7854
|
|
|
7692
7855
|
// dist/builtin/workflows/src/durable/journal/legacy-journal-tables.ts
|
|
@@ -60812,24 +60975,63 @@ function stickyStageDeliveryStatuses(runs, rootRunId, availableBackend) {
|
|
|
60812
60975
|
const root = runs.find((run) => run.id === rootRunId);
|
|
60813
60976
|
const entries = (root?.pendingStageMessages ?? []).filter((entry) => entry.sticky === true);
|
|
60814
60977
|
const result = [];
|
|
60978
|
+
const patternByTarget = new Map;
|
|
60979
|
+
const patterns = [];
|
|
60980
|
+
const entriesByPattern = [];
|
|
60981
|
+
const ownEntries = [];
|
|
60982
|
+
entries.forEach((entry, index) => {
|
|
60983
|
+
const target = entry.targetPath ?? entry.stageKey;
|
|
60984
|
+
let pattern = patternByTarget.get(target);
|
|
60985
|
+
if (pattern === undefined) {
|
|
60986
|
+
const parsed = parseWorkflowStageTarget(target);
|
|
60987
|
+
pattern = parsed?.rootRunId === rootRunId ? patterns.push(parsed.segments) - 1 : -1;
|
|
60988
|
+
if (pattern >= 0)
|
|
60989
|
+
entriesByPattern.push([]);
|
|
60990
|
+
patternByTarget.set(target, pattern);
|
|
60991
|
+
}
|
|
60992
|
+
if (pattern < 0)
|
|
60993
|
+
return;
|
|
60994
|
+
entriesByPattern[pattern].push(index);
|
|
60995
|
+
ownEntries.push(index);
|
|
60996
|
+
});
|
|
60997
|
+
if (ownEntries.length === 0)
|
|
60998
|
+
return result;
|
|
60999
|
+
const patternIndex = createStagePathPatternIndex(patterns);
|
|
61000
|
+
const deliveriesByEntry = new Map;
|
|
61001
|
+
const entryReceipts = (index, runId, stageId) => {
|
|
61002
|
+
let byStage = deliveriesByEntry.get(index);
|
|
61003
|
+
if (byStage === undefined) {
|
|
61004
|
+
byStage = new Map;
|
|
61005
|
+
for (const delivery of entries[index].deliveries ?? []) {
|
|
61006
|
+
const key = `${delivery.runId}\x00${delivery.stageId}`;
|
|
61007
|
+
const list = byStage.get(key);
|
|
61008
|
+
if (list === undefined)
|
|
61009
|
+
byStage.set(key, [delivery]);
|
|
61010
|
+
else
|
|
61011
|
+
list.push(delivery);
|
|
61012
|
+
}
|
|
61013
|
+
deliveriesByEntry.set(index, byStage);
|
|
61014
|
+
}
|
|
61015
|
+
return byStage.get(`${runId}\x00${stageId}`) ?? [];
|
|
61016
|
+
};
|
|
60815
61017
|
availableBackend = availableRunDurableBackend(runs, rootRunId) ?? availableBackend;
|
|
60816
61018
|
const backend = availableBackend === undefined ? undefined : durableBackendForRun(availableBackend, runs, rootRunId);
|
|
61019
|
+
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
61020
|
+
const hopsFor = createWorkflowBoundaryHopsResolver(runs);
|
|
60817
61021
|
for (const run of runs) {
|
|
60818
|
-
|
|
61022
|
+
const durableRootRunId = reciprocalWorkflowRootRunId(runById, run.id);
|
|
61023
|
+
if (run.id !== rootRunId && run.rootRunId !== rootRunId && durableRootRunId !== rootRunId)
|
|
60819
61024
|
continue;
|
|
60820
|
-
const resolved =
|
|
60821
|
-
const hops =
|
|
61025
|
+
const resolved = durableRootRunId === rootRunId;
|
|
61026
|
+
const hops = hopsFor(run.id);
|
|
60822
61027
|
for (const stage of run.stages) {
|
|
60823
61028
|
if (stage.nodeKind === "tool" || stage.replayKey?.startsWith("prompt:") === true || stage.replayKey?.startsWith("workflow:") === true || stage.workflowChildRun !== undefined || stage.workflowChild !== undefined)
|
|
60824
61029
|
continue;
|
|
60825
|
-
|
|
61030
|
+
const matchedEntries = resolved && hops ? patternIndex.matching(hops, [stage.id, stage.name]).flatMap((pattern) => entriesByPattern[pattern]).sort((left, right) => left - right) : ownEntries;
|
|
61031
|
+
for (const index of matchedEntries) {
|
|
61032
|
+
const entry = entries[index];
|
|
60826
61033
|
const target = entry.targetPath ?? entry.stageKey;
|
|
60827
|
-
const
|
|
60828
|
-
if (!parsed || parsed.rootRunId !== rootRunId)
|
|
60829
|
-
continue;
|
|
60830
|
-
if (resolved && hops && !stageMatchesPathPattern(parsed.segments, hops, [stage.id, stage.name]))
|
|
60831
|
-
continue;
|
|
60832
|
-
const receipts = (entry.deliveries ?? []).filter((delivery) => delivery.runId === run.id && delivery.stageId === stage.id);
|
|
61034
|
+
const receipts = entryReceipts(index, run.id, stage.id);
|
|
60833
61035
|
const delivered = backend?.hasCachedPendingStageDeliveryReceipt?.(rootRunId, entry.id, {
|
|
60834
61036
|
runId: run.id,
|
|
60835
61037
|
stageId: stage.id,
|
|
@@ -61147,20 +61349,27 @@ function possibleStageRows(runs, rootRunId, rootRun, backend) {
|
|
|
61147
61349
|
if (isTerminalRunStatus(rootRun.status))
|
|
61148
61350
|
return [];
|
|
61149
61351
|
const stickyQueued = (rootRun.pendingStageMessages ?? []).filter((entry) => entry.sticky === true && entry.status === "queued");
|
|
61352
|
+
const segmentsByTarget = new Map;
|
|
61150
61353
|
const stickySegments = (entry) => {
|
|
61151
|
-
const
|
|
61152
|
-
|
|
61354
|
+
const target = entry.targetPath ?? entry.stageKey;
|
|
61355
|
+
if (segmentsByTarget.has(target))
|
|
61356
|
+
return segmentsByTarget.get(target);
|
|
61357
|
+
const parsed = parseWorkflowStageTarget(target);
|
|
61358
|
+
const segments = parsed?.rootRunId === rootRunId ? parsed.segments : undefined;
|
|
61359
|
+
segmentsByTarget.set(target, segments);
|
|
61360
|
+
return segments;
|
|
61153
61361
|
};
|
|
61154
61362
|
const scanEntries = stickyQueued.filter((entry) => {
|
|
61155
61363
|
const segments = stickySegments(entry);
|
|
61156
61364
|
return !(segments !== undefined && segments.length === 1 && segments[0] === "**");
|
|
61157
61365
|
});
|
|
61366
|
+
const liveStageMatches = materializedStageMatcher(runs, rootRunId);
|
|
61158
61367
|
const rows = [];
|
|
61159
61368
|
for (const scanEntry of rootRun.possibleStages ?? []) {
|
|
61160
61369
|
const rowSegments = splitStagePathSegments(scanEntry);
|
|
61161
61370
|
if (rowSegments.length === 0 || rowSegments.some((segment) => segment.length === 0))
|
|
61162
61371
|
continue;
|
|
61163
|
-
if (!rowSegments.some((segment) => segment.includes("*")) &&
|
|
61372
|
+
if (!rowSegments.some((segment) => segment.includes("*")) && liveStageMatches(rowSegments)) {
|
|
61164
61373
|
continue;
|
|
61165
61374
|
}
|
|
61166
61375
|
rows.push({
|
|
@@ -61181,28 +61390,43 @@ function possibleStageRows(runs, rootRunId, rootRun, backend) {
|
|
|
61181
61390
|
}).length
|
|
61182
61391
|
});
|
|
61183
61392
|
const statuses = stickyStageDeliveryStatuses(runs, rootRunId, backend);
|
|
61184
|
-
|
|
61185
|
-
|
|
61393
|
+
const statusesByTarget = new Map;
|
|
61394
|
+
for (const status of statuses) {
|
|
61395
|
+
const forTarget = statusesByTarget.get(status.target);
|
|
61396
|
+
if (forTarget === undefined)
|
|
61397
|
+
statusesByTarget.set(status.target, [status]);
|
|
61398
|
+
else
|
|
61399
|
+
forTarget.push(status);
|
|
61400
|
+
}
|
|
61401
|
+
const queuedByTarget = new Map;
|
|
61402
|
+
for (const entry of stickyQueued) {
|
|
61403
|
+
const target = entry.targetPath ?? entry.stageKey;
|
|
61404
|
+
queuedByTarget.set(target, (queuedByTarget.get(target) ?? 0) + 1);
|
|
61405
|
+
}
|
|
61406
|
+
const rowTargets = new Set(rows.map((row) => row.target));
|
|
61407
|
+
for (const target of statusesByTarget.keys()) {
|
|
61408
|
+
if (rowTargets.has(target))
|
|
61186
61409
|
continue;
|
|
61187
|
-
|
|
61188
|
-
|
|
61189
|
-
queuedCount: stickyQueued.filter((entry) => (entry.targetPath ?? entry.stageKey) === target).length,
|
|
61190
|
-
deliveryOnly: true
|
|
61191
|
-
});
|
|
61410
|
+
rowTargets.add(target);
|
|
61411
|
+
rows.push({ target, queuedCount: queuedByTarget.get(target) ?? 0, deliveryOnly: true });
|
|
61192
61412
|
}
|
|
61193
61413
|
for (let index = 0;index < rows.length; index++) {
|
|
61194
61414
|
const row = rows[index];
|
|
61195
|
-
const deliveryStates =
|
|
61196
|
-
if (deliveryStates
|
|
61415
|
+
const deliveryStates = statusesByTarget.get(row.target);
|
|
61416
|
+
if (deliveryStates !== undefined)
|
|
61197
61417
|
rows[index] = { ...row, deliveryStates };
|
|
61198
61418
|
}
|
|
61199
61419
|
return rows;
|
|
61200
61420
|
}
|
|
61201
|
-
function
|
|
61202
|
-
|
|
61203
|
-
|
|
61204
|
-
|
|
61205
|
-
|
|
61421
|
+
function materializedStageMatcher(runs, rootRunId) {
|
|
61422
|
+
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
61423
|
+
const hopsFor = createWorkflowBoundaryHopsResolver(runs);
|
|
61424
|
+
const invocationRuns = runs.filter((run) => reciprocalWorkflowRootRunId(runById, run.id) === rootRunId);
|
|
61425
|
+
return (entrySegments) => materializedStageMatches(invocationRuns, hopsFor, entrySegments);
|
|
61426
|
+
}
|
|
61427
|
+
function materializedStageMatches(invocationRuns, hopsFor, entrySegments) {
|
|
61428
|
+
for (const run of invocationRuns) {
|
|
61429
|
+
const hops = hopsFor(run.id);
|
|
61206
61430
|
if (hops === undefined)
|
|
61207
61431
|
continue;
|
|
61208
61432
|
for (const stage of run.stages) {
|
|
@@ -61477,10 +61701,12 @@ async function recordConfirmedStickyDeliveries(activeStore, event) {
|
|
|
61477
61701
|
}
|
|
61478
61702
|
function matchingLiveStages(runs, rootRunId, patternSegments) {
|
|
61479
61703
|
const matches = [];
|
|
61704
|
+
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
61705
|
+
const hopsFor = createWorkflowBoundaryHopsResolver(runs);
|
|
61480
61706
|
for (const run of runs) {
|
|
61481
|
-
if (
|
|
61707
|
+
if (reciprocalWorkflowRootRunId(runById, run.id) !== rootRunId)
|
|
61482
61708
|
continue;
|
|
61483
|
-
const hops =
|
|
61709
|
+
const hops = hopsFor(run.id);
|
|
61484
61710
|
if (hops === undefined)
|
|
61485
61711
|
continue;
|
|
61486
61712
|
for (const stage of run.stages) {
|
|
@@ -61616,11 +61842,26 @@ function writeMessage(socket, msg, callback) {
|
|
|
61616
61842
|
}
|
|
61617
61843
|
function createMessageReader(onMessage, onError) {
|
|
61618
61844
|
let buffer = Buffer.alloc(0);
|
|
61845
|
+
let pending = [];
|
|
61846
|
+
let pendingLength = 0;
|
|
61847
|
+
let awaitedLength = 0;
|
|
61619
61848
|
return (data) => {
|
|
61620
|
-
|
|
61849
|
+
if (pendingLength + data.length < awaitedLength) {
|
|
61850
|
+
pending.push(data);
|
|
61851
|
+
pendingLength += data.length;
|
|
61852
|
+
return;
|
|
61853
|
+
}
|
|
61854
|
+
buffer = pending.length === 0 ? Buffer.concat([buffer, data]) : Buffer.concat([...pending, data]);
|
|
61855
|
+
pending = [];
|
|
61856
|
+
pendingLength = 0;
|
|
61857
|
+
awaitedLength = 0;
|
|
61621
61858
|
while (buffer.length >= 4) {
|
|
61622
61859
|
const length = buffer.readUInt32BE(0);
|
|
61623
61860
|
if (buffer.length < 4 + length) {
|
|
61861
|
+
pending = [buffer];
|
|
61862
|
+
pendingLength = buffer.length;
|
|
61863
|
+
awaitedLength = 4 + length;
|
|
61864
|
+
buffer = Buffer.alloc(0);
|
|
61624
61865
|
break;
|
|
61625
61866
|
}
|
|
61626
61867
|
const payload = buffer.subarray(4, 4 + length);
|