@hunterzhu/pulse-runtime 0.1.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/context/builder.d.ts +6 -1
- package/dist/context/builder.js +37 -4
- package/dist/core/inbox.d.ts +6 -0
- package/dist/core/inbox.js +31 -7
- package/dist/core/mutations.d.ts +5 -1
- package/dist/core/mutations.js +5 -1
- package/dist/core/types.d.ts +26 -1
- package/dist/core/types.js +1 -1
- package/dist/dsl/program.d.ts +4 -1
- package/dist/dsl/program.js +147 -24
- package/dist/dsl/session.d.ts +7 -0
- package/dist/dsl/session.js +37 -10
- package/dist/dsl/templates.d.ts +2 -5
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/scheduler/human-arbitration.d.ts +59 -0
- package/dist/scheduler/human-arbitration.js +62 -0
- package/dist/scheduler/runtime.d.ts +51 -3
- package/dist/scheduler/runtime.js +445 -40
- package/dist/scheduler/worker.d.ts +2 -0
- package/dist/scheduler/worker.js +24 -4
- package/dist/storage/session.d.ts +2 -1
- package/dist/storage/session.js +14 -0
- package/dist/transitions/validate.js +14 -4
- package/package.json +1 -1
|
@@ -24,6 +24,7 @@ import { prepareFindingPublication } from '../storage/findings.js';
|
|
|
24
24
|
import { runtimeErrorFromCause } from '../core/errors.js';
|
|
25
25
|
import { RuntimeToolRegistry } from '../tools/registry.js';
|
|
26
26
|
import { SchedulerDecisionCoordinator, schedulerDecisionCandidateFromLane } from './decision.js';
|
|
27
|
+
import { HumanArbitrationCoordinator, ruleHumanArbitration } from './human-arbitration.js';
|
|
27
28
|
function encodeEffectExecution(execution) {
|
|
28
29
|
const encoded = structuredClone(execution);
|
|
29
30
|
const artifact = execution.artifact;
|
|
@@ -111,6 +112,19 @@ function validateHostCommand(command) {
|
|
|
111
112
|
}
|
|
112
113
|
return;
|
|
113
114
|
}
|
|
115
|
+
if (value.type === 'human_input') {
|
|
116
|
+
if (typeof value.agentId !== 'string' || value.agentId.length === 0 || typeof value.inputId !== 'string' || value.inputId.length === 0)
|
|
117
|
+
throw new Error('INVALID_HOST_COMMAND');
|
|
118
|
+
if (value.targetEffectId !== undefined && (typeof value.targetEffectId !== 'string' || value.targetEffectId.length === 0))
|
|
119
|
+
throw new Error('INVALID_HOST_COMMAND');
|
|
120
|
+
try {
|
|
121
|
+
strictJsonValue(value.value);
|
|
122
|
+
}
|
|
123
|
+
catch {
|
|
124
|
+
throw new Error('INVALID_HOST_COMMAND_VALUE');
|
|
125
|
+
}
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
114
128
|
if (value.type === 'cancel' || value.type === 'cancel_effect') {
|
|
115
129
|
if (typeof value.agentId !== 'string' || value.agentId.length === 0 || typeof value.reason !== 'string' || value.reason.length === 0)
|
|
116
130
|
throw new Error('INVALID_HOST_COMMAND');
|
|
@@ -125,6 +139,31 @@ function validateHostCommand(command) {
|
|
|
125
139
|
}
|
|
126
140
|
throw new Error('INVALID_HOST_COMMAND');
|
|
127
141
|
}
|
|
142
|
+
function parseHumanArbitrationDecision(value) {
|
|
143
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
144
|
+
return undefined;
|
|
145
|
+
const candidate = value;
|
|
146
|
+
const actions = new Set(['respond', 'steer', 'spawn', 'defer', 'cancel']);
|
|
147
|
+
if (typeof candidate.decisionId !== 'string' || typeof candidate.inputId !== 'string' || typeof candidate.agentId !== 'string' || typeof candidate.modelId !== 'string' || typeof candidate.action !== 'string' || !actions.has(candidate.action))
|
|
148
|
+
return undefined;
|
|
149
|
+
if (candidate.targetLaneId !== undefined && typeof candidate.targetLaneId !== 'string')
|
|
150
|
+
return undefined;
|
|
151
|
+
if (candidate.targetEffectId !== undefined && typeof candidate.targetEffectId !== 'string')
|
|
152
|
+
return undefined;
|
|
153
|
+
if (candidate.reason !== undefined && typeof candidate.reason !== 'string')
|
|
154
|
+
return undefined;
|
|
155
|
+
return {
|
|
156
|
+
schemaVersion: 1,
|
|
157
|
+
decisionId: candidate.decisionId,
|
|
158
|
+
inputId: candidate.inputId,
|
|
159
|
+
agentId: candidate.agentId,
|
|
160
|
+
action: candidate.action,
|
|
161
|
+
...(candidate.targetLaneId === undefined ? {} : { targetLaneId: candidate.targetLaneId }),
|
|
162
|
+
...(candidate.targetEffectId === undefined ? {} : { targetEffectId: candidate.targetEffectId }),
|
|
163
|
+
...(candidate.reason === undefined ? {} : { reason: candidate.reason }),
|
|
164
|
+
modelId: candidate.modelId,
|
|
165
|
+
};
|
|
166
|
+
}
|
|
128
167
|
function artifactOutput(value) {
|
|
129
168
|
if (value instanceof Uint8Array)
|
|
130
169
|
return { mediaType: 'application/octet-stream', content: new Uint8Array(value) };
|
|
@@ -414,6 +453,13 @@ export class PulseRuntime {
|
|
|
414
453
|
auditLogPrivacy;
|
|
415
454
|
persistenceBackend;
|
|
416
455
|
sessionStore;
|
|
456
|
+
/**
|
|
457
|
+
* The file session store is synchronous by design, so avoid rewriting an
|
|
458
|
+
* unchanged warm-start snapshot on every scheduler tick. Without this
|
|
459
|
+
* guard an idle run can spend its whole event loop serializing and fsyncing
|
|
460
|
+
* the same snapshot, starving effect dispatch and human input handling.
|
|
461
|
+
*/
|
|
462
|
+
sessionStoreDigests = new Map();
|
|
417
463
|
enforcingRecoveryPrograms;
|
|
418
464
|
toolVersions;
|
|
419
465
|
recoveryCompatibility;
|
|
@@ -456,6 +502,7 @@ export class PulseRuntime {
|
|
|
456
502
|
factInboxDedupeArchive;
|
|
457
503
|
customExecutor;
|
|
458
504
|
builtinHumanEffects;
|
|
505
|
+
humanInputProgram;
|
|
459
506
|
enqueueSeq = 1;
|
|
460
507
|
maxSteps;
|
|
461
508
|
maxTickMs;
|
|
@@ -468,6 +515,10 @@ export class PulseRuntime {
|
|
|
468
515
|
maxPreparedLLMs;
|
|
469
516
|
effectSubmissionPreparer;
|
|
470
517
|
schedulerDecisionCoordinator;
|
|
518
|
+
humanArbitrationCoordinator;
|
|
519
|
+
humanArbitrationModelId;
|
|
520
|
+
humanArbitrationRequestSeq = 1;
|
|
521
|
+
humanArbitrationRequests = new Map();
|
|
471
522
|
schedulerDecisionConfig;
|
|
472
523
|
schedulerDecisionModelId;
|
|
473
524
|
schedulerDecisionMaxReorderDistance;
|
|
@@ -482,7 +533,15 @@ export class PulseRuntime {
|
|
|
482
533
|
sessionId;
|
|
483
534
|
hostCommandSeq = 1;
|
|
484
535
|
factWaiters = [];
|
|
536
|
+
activityWaiters = new Set();
|
|
485
537
|
wakeScheduled = false;
|
|
538
|
+
/**
|
|
539
|
+
* A lock grant can happen synchronously while an effect completion is being
|
|
540
|
+
* applied inside tick(). scheduleWake() intentionally avoids re-entering the
|
|
541
|
+
* drain in that case, so remember forced wake requests and service them once
|
|
542
|
+
* the current drain has finished.
|
|
543
|
+
*/
|
|
544
|
+
wakeAfterDrain = false;
|
|
486
545
|
inDrain = false;
|
|
487
546
|
wakeError;
|
|
488
547
|
tickBudget;
|
|
@@ -616,6 +675,8 @@ export class PulseRuntime {
|
|
|
616
675
|
timeoutMs: this.schedulerDecisionConfig.decisionTimeoutMs,
|
|
617
676
|
maxOutstanding: this.schedulerDecisionConfig.maxOutstandingDecisions,
|
|
618
677
|
});
|
|
678
|
+
this.humanArbitrationModelId = config.humanArbitration?.model?.id;
|
|
679
|
+
this.humanArbitrationCoordinator = config.humanArbitration?.model === undefined ? undefined : new HumanArbitrationCoordinator(config.humanArbitration.model, config.humanArbitration.timeoutMs ?? 250);
|
|
619
680
|
this.telemetryExporter = config.telemetryExporter;
|
|
620
681
|
this.auditLogSink = config.auditLogSink;
|
|
621
682
|
this.auditLogPrivacy = config.auditLogPrivacy;
|
|
@@ -639,6 +700,11 @@ export class PulseRuntime {
|
|
|
639
700
|
this.syncStoragePolicy();
|
|
640
701
|
}
|
|
641
702
|
register(program) { this.programs.register(program); }
|
|
703
|
+
/** Register the program used for urgent, concurrent human interactions. */
|
|
704
|
+
setHumanInputProgram(program) {
|
|
705
|
+
this.register(program);
|
|
706
|
+
this.humanInputProgram = program;
|
|
707
|
+
}
|
|
642
708
|
createAgent(goalOrRequest, program, agentId) {
|
|
643
709
|
if (this.shuttingDown)
|
|
644
710
|
throw new Error('RUNTIME_SHUTTING_DOWN');
|
|
@@ -750,6 +816,190 @@ export class PulseRuntime {
|
|
|
750
816
|
}
|
|
751
817
|
start(agentId) { if (!this.state.agents.has(agentId))
|
|
752
818
|
throw new Error(`UNKNOWN_AGENT:${agentId}`); return new PulseSession(this, agentId); }
|
|
819
|
+
/** Accept external human input without waiting for the current Effect to settle. */
|
|
820
|
+
submitHumanInput(agentId, inputId, value, targetEffectId) {
|
|
821
|
+
if (!agentId)
|
|
822
|
+
throw new Error('INVALID_AGENT_ID');
|
|
823
|
+
if (!inputId)
|
|
824
|
+
throw new Error('INVALID_HUMAN_INPUT_ID');
|
|
825
|
+
strictJsonValue(value);
|
|
826
|
+
return this.enqueueHostCommand({ type: 'human_input', agentId, inputId, value, ...(targetEffectId === undefined ? {} : { targetEffectId }) });
|
|
827
|
+
}
|
|
828
|
+
humanInputsFor(agentId) {
|
|
829
|
+
return [...this.state.humanInputs.values()].filter((input) => input.agentId === agentId).map((input) => structuredClone(input));
|
|
830
|
+
}
|
|
831
|
+
humanArbitrationRequest(agent, input) {
|
|
832
|
+
const lanes = [...this.state.lanes.values()].filter((lane) => lane.agentId === agent.id || this.ownsAgentForRuntime(lane.agentId, agent.id)).map((lane) => ({ laneId: lane.id, agentId: lane.agentId, status: lane.status, priority: lane.priority, ...(lane.goal ? { goal: lane.goal } : {}), ...(lane.activeWaitId === undefined ? {} : { activeWaitId: lane.activeWaitId }) }));
|
|
833
|
+
const effects = [...this.state.effects.values()].filter((effect) => effect.agentId === agent.id || this.ownsAgentForRuntime(effect.agentId, agent.id)).map((effect) => ({ effectId: effect.id, agentId: effect.agentId, laneId: effect.ownerLaneId, kind: effect.kind, state: effect.state, ...(effect.sideEffectPolicy === undefined ? {} : { sideEffectPolicy: effect.sideEffectPolicy }), sideEffectState: effect.sideEffectState }));
|
|
834
|
+
const llmLimit = this.state.maxRunning.llm;
|
|
835
|
+
const availableLLMSlots = Number.isFinite(llmLimit) ? Math.max(0, llmLimit - this.runningCount('llm') - 1) : Number.POSITIVE_INFINITY;
|
|
836
|
+
return { schemaVersion: 1, decisionId: `human-decision-${this.sessionId}-${this.humanArbitrationRequestSeq}`, agentId: agent.id, input: structuredClone(input), lanes, effects, availableLLMSlots };
|
|
837
|
+
}
|
|
838
|
+
ownsAgentForRuntime(candidateAgentId, ancestorAgentId) {
|
|
839
|
+
let current = this.state.agents.get(candidateAgentId);
|
|
840
|
+
const seen = new Set();
|
|
841
|
+
while (current && !seen.has(current.id)) {
|
|
842
|
+
if (current.id === ancestorAgentId)
|
|
843
|
+
return true;
|
|
844
|
+
seen.add(current.id);
|
|
845
|
+
current = current.parentAgentId === undefined ? undefined : this.state.agents.get(current.parentAgentId);
|
|
846
|
+
}
|
|
847
|
+
return false;
|
|
848
|
+
}
|
|
849
|
+
commitHumanInputDecision(record, decision, eventType = 'human.input.decided') {
|
|
850
|
+
const next = structuredClone(record);
|
|
851
|
+
next.decision = decision.action;
|
|
852
|
+
if (decision.reason === undefined)
|
|
853
|
+
delete next.decisionReason;
|
|
854
|
+
else
|
|
855
|
+
next.decisionReason = decision.reason;
|
|
856
|
+
next.decisionModelId = decision.modelId;
|
|
857
|
+
next.decidedAt = this.state.now;
|
|
858
|
+
const event = { type: eventType, agentId: record.agentId, data: { inputId: record.id, action: decision.action, modelId: decision.modelId, ...(decision.reason === undefined ? {} : { reason: decision.reason }), ...(decision.targetLaneId === undefined ? {} : { targetLaneId: decision.targetLaneId }), ...(decision.targetEffectId === undefined ? {} : { targetEffectId: decision.targetEffectId }) } };
|
|
859
|
+
const mutations = [{ op: 'setHumanInput', inputId: record.id, record: next }, { op: 'appendEvent', event }];
|
|
860
|
+
this.assertStorageAdmission(mutations);
|
|
861
|
+
commitMutationTransaction(this.state, this.mutationLog, `human-input:${record.id}:decision:${decision.action}`, mutations, this.state.now, this.sessionId);
|
|
862
|
+
Object.assign(record, next);
|
|
863
|
+
this.state.humanInputs.set(record.id, record);
|
|
864
|
+
}
|
|
865
|
+
applyHumanArbitrationDecision(decision) {
|
|
866
|
+
const record = this.state.humanInputs.get(decision.inputId);
|
|
867
|
+
if (!record || record.agentId !== decision.agentId || record.status !== 'pending')
|
|
868
|
+
return false;
|
|
869
|
+
const targetEffectId = decision.targetEffectId ?? record.targetEffectId;
|
|
870
|
+
if (decision.action === 'respond') {
|
|
871
|
+
if (targetEffectId !== undefined) {
|
|
872
|
+
const effect = this.state.effects.get(targetEffectId);
|
|
873
|
+
if (!effect || effect.agentId !== record.agentId || effect.kind !== 'human' || effect.outcome)
|
|
874
|
+
return false;
|
|
875
|
+
this.commitHumanInputDecision(record, decision);
|
|
876
|
+
record.status = 'consumed';
|
|
877
|
+
const consumed = structuredClone(record);
|
|
878
|
+
const mutations = [{ op: 'setHumanInput', inputId: record.id, record: consumed }];
|
|
879
|
+
this.assertStorageAdmission(mutations);
|
|
880
|
+
commitMutationTransaction(this.state, this.mutationLog, `human-input:${record.id}:respond`, mutations, this.state.now, this.sessionId);
|
|
881
|
+
Object.assign(record, consumed);
|
|
882
|
+
this.completeEffect(effect.id, { value: record.value }, 'succeeded');
|
|
883
|
+
return true;
|
|
884
|
+
}
|
|
885
|
+
const next = structuredClone(record);
|
|
886
|
+
next.status = 'consumed';
|
|
887
|
+
this.commitHumanInputDecision(record, decision, 'human.input.responded');
|
|
888
|
+
const mutations = [{ op: 'setHumanInput', inputId: record.id, record: next }];
|
|
889
|
+
this.assertStorageAdmission(mutations);
|
|
890
|
+
commitMutationTransaction(this.state, this.mutationLog, `human-input:${record.id}:responded`, mutations, this.state.now, this.sessionId);
|
|
891
|
+
Object.assign(record, next);
|
|
892
|
+
this.state.humanInputs.set(record.id, record);
|
|
893
|
+
return true;
|
|
894
|
+
}
|
|
895
|
+
if (decision.action === 'steer') {
|
|
896
|
+
const lane = this.state.lanes.get(decision.targetLaneId ?? this.state.agents.get(record.agentId)?.rootLaneId ?? '');
|
|
897
|
+
if (!lane || lane.agentId !== record.agentId || ['succeeded', 'failed', 'cancelled'].includes(lane.status)) {
|
|
898
|
+
const deferred = structuredClone(record);
|
|
899
|
+
deferred.status = 'deferred';
|
|
900
|
+
this.commitHumanInputDecision(record, { ...decision, action: 'defer', reason: decision.reason ?? 'Target lane is no longer active.' }, 'human.input.deferred');
|
|
901
|
+
Object.assign(record, deferred);
|
|
902
|
+
return true;
|
|
903
|
+
}
|
|
904
|
+
const nextLane = structuredClone(lane);
|
|
905
|
+
nextLane.pendingHumanInputs = [...(nextLane.pendingHumanInputs ?? []), structuredClone(record)];
|
|
906
|
+
nextLane.priority = Math.max(nextLane.priority, 2);
|
|
907
|
+
nextLane.version++;
|
|
908
|
+
const next = structuredClone(record);
|
|
909
|
+
next.status = 'consumed';
|
|
910
|
+
next.handledByLaneId = lane.id;
|
|
911
|
+
this.commitHumanInputDecision(record, decision);
|
|
912
|
+
const mutations = [{ op: 'setHumanInput', inputId: record.id, record: next }, { op: 'setLane', laneId: lane.id, record: nextLane }, { op: 'appendEvent', event: { type: 'human.input.steered', agentId: record.agentId, laneId: lane.id, data: { inputId: record.id, priority: 'human' } } }];
|
|
913
|
+
this.assertStorageAdmission(mutations);
|
|
914
|
+
commitMutationTransaction(this.state, this.mutationLog, `human-input:${record.id}:steer`, mutations, this.state.now, this.sessionId);
|
|
915
|
+
Object.assign(record, next);
|
|
916
|
+
this.state.humanInputs.set(record.id, record);
|
|
917
|
+
Object.assign(lane, nextLane);
|
|
918
|
+
this.state.lanes.set(lane.id, lane);
|
|
919
|
+
if (lane.status === 'ready')
|
|
920
|
+
this.enqueueReadyItem(readyItemFromLane(lane));
|
|
921
|
+
return true;
|
|
922
|
+
}
|
|
923
|
+
if (decision.action === 'spawn') {
|
|
924
|
+
if (this.humanInputProgram === undefined) {
|
|
925
|
+
const deferred = { ...decision, action: 'defer', reason: decision.reason ?? 'No Human interaction program is registered.' };
|
|
926
|
+
return this.applyHumanArbitrationDecision(deferred);
|
|
927
|
+
}
|
|
928
|
+
try {
|
|
929
|
+
const parentAgent = this.state.agents.get(record.agentId);
|
|
930
|
+
const parentLane = parentAgent === undefined ? undefined : this.state.lanes.get(parentAgent.rootLaneId);
|
|
931
|
+
const inheritedResults = parentLane?.visibleResultRefs === undefined ? [] : [...parentLane.visibleResultRefs].slice(-64);
|
|
932
|
+
const child = this.createAgent({ goal: `Human input: ${typeof record.value === 'string' ? record.value : JSON.stringify(record.value)}`, program: this.humanInputProgram, priority: 'urgent', parentAgentId: record.agentId, warmStart: { agentId: record.agentId, globalVersion: 'latest', include: 'facts_and_findings', ...(inheritedResults.length ? { relevanceRefs: inheritedResults } : {}) } });
|
|
933
|
+
const next = structuredClone(record);
|
|
934
|
+
next.status = 'consumed';
|
|
935
|
+
next.handledByLaneId = child.laneId;
|
|
936
|
+
this.commitHumanInputDecision(record, decision);
|
|
937
|
+
const mutations = [{ op: 'setHumanInput', inputId: record.id, record: next }, { op: 'appendEvent', event: { type: 'human.input.dispatched', agentId: record.agentId, laneId: child.laneId, data: { inputId: record.id, decision: 'spawn', childAgentId: child.agentId } } }];
|
|
938
|
+
this.assertStorageAdmission(mutations);
|
|
939
|
+
commitMutationTransaction(this.state, this.mutationLog, `human-input:${record.id}:spawn`, mutations, this.state.now, this.sessionId);
|
|
940
|
+
Object.assign(record, next);
|
|
941
|
+
this.state.humanInputs.set(record.id, record);
|
|
942
|
+
}
|
|
943
|
+
catch (cause) {
|
|
944
|
+
return this.applyHumanArbitrationDecision({ ...decision, action: 'defer', reason: cause instanceof Error ? cause.message : String(cause) });
|
|
945
|
+
}
|
|
946
|
+
return true;
|
|
947
|
+
}
|
|
948
|
+
if (decision.action === 'cancel') {
|
|
949
|
+
const targetEffect = targetEffectId === undefined ? undefined : this.state.effects.get(targetEffectId);
|
|
950
|
+
if (targetEffect && (targetEffect.agentId !== record.agentId || targetEffect.outcome))
|
|
951
|
+
return false;
|
|
952
|
+
this.commitHumanInputDecision(record, decision);
|
|
953
|
+
const next = structuredClone(record);
|
|
954
|
+
next.status = 'consumed';
|
|
955
|
+
const mutations = [{ op: 'setHumanInput', inputId: record.id, record: next }];
|
|
956
|
+
this.assertStorageAdmission(mutations);
|
|
957
|
+
commitMutationTransaction(this.state, this.mutationLog, `human-input:${record.id}:cancel`, mutations, this.state.now, this.sessionId);
|
|
958
|
+
Object.assign(record, next);
|
|
959
|
+
this.state.humanInputs.set(record.id, record);
|
|
960
|
+
if (targetEffect)
|
|
961
|
+
this.cancelEffect(targetEffect.id, targetEffect.cancelGraceMs ?? 0, decision.reason ?? 'HUMAN_CANCELLED');
|
|
962
|
+
else
|
|
963
|
+
this.cancelAgent(record.agentId, decision.reason ?? 'HUMAN_CANCELLED');
|
|
964
|
+
return true;
|
|
965
|
+
}
|
|
966
|
+
const deferred = structuredClone(record);
|
|
967
|
+
deferred.status = 'deferred';
|
|
968
|
+
this.commitHumanInputDecision(record, decision, 'human.input.deferred');
|
|
969
|
+
const mutations = [{ op: 'setHumanInput', inputId: record.id, record: deferred }];
|
|
970
|
+
this.assertStorageAdmission(mutations);
|
|
971
|
+
commitMutationTransaction(this.state, this.mutationLog, `human-input:${record.id}:defer`, mutations, this.state.now, this.sessionId);
|
|
972
|
+
Object.assign(record, deferred);
|
|
973
|
+
this.state.humanInputs.set(record.id, record);
|
|
974
|
+
return true;
|
|
975
|
+
}
|
|
976
|
+
requestHumanArbitration(agent, input) {
|
|
977
|
+
const coordinator = this.humanArbitrationCoordinator;
|
|
978
|
+
if (!coordinator)
|
|
979
|
+
return false;
|
|
980
|
+
const request = this.humanArbitrationRequest(agent, input);
|
|
981
|
+
const requestId = request.decisionId;
|
|
982
|
+
this.humanArbitrationRequestSeq++;
|
|
983
|
+
this.humanArbitrationRequests.set(requestId, { agentId: agent.id, inputId: input.id });
|
|
984
|
+
const accepted = coordinator.request(request, (decision) => {
|
|
985
|
+
this.humanArbitrationRequests.delete(requestId);
|
|
986
|
+
if (decision.decisionId !== requestId || decision.inputId !== input.id || decision.agentId !== agent.id || decision.modelId !== this.humanArbitrationModelId)
|
|
987
|
+
return;
|
|
988
|
+
try {
|
|
989
|
+
this.enqueueFact({ type: 'human_arbitration', decision: strictJsonValue(decision) }, `human-arbitration:${requestId}`, true);
|
|
990
|
+
}
|
|
991
|
+
catch { /* malformed model output is handled as a deferred input */ }
|
|
992
|
+
}, () => {
|
|
993
|
+
this.humanArbitrationRequests.delete(requestId);
|
|
994
|
+
try {
|
|
995
|
+
this.enqueueFact({ type: 'human_arbitration', decision: strictJsonValue({ schemaVersion: 1, decisionId: requestId, inputId: input.id, agentId: agent.id, action: 'defer', reason: 'Human arbitration timed out.', modelId: this.humanArbitrationModelId ?? 'model' }) }, `human-arbitration:${requestId}:timeout`, true);
|
|
996
|
+
}
|
|
997
|
+
catch { /* runtime shutdown */ }
|
|
998
|
+
});
|
|
999
|
+
if (!accepted)
|
|
1000
|
+
this.humanArbitrationRequests.delete(requestId);
|
|
1001
|
+
return accepted;
|
|
1002
|
+
}
|
|
753
1003
|
requestCancel(agentId, reason = 'USER_REQUESTED') {
|
|
754
1004
|
if (!agentId)
|
|
755
1005
|
throw new Error('INVALID_AGENT_ID');
|
|
@@ -834,14 +1084,25 @@ export class PulseRuntime {
|
|
|
834
1084
|
async persist(backend) {
|
|
835
1085
|
const persistedPolicy = this.storagePolicy.clone();
|
|
836
1086
|
persistedPolicy.markPersisted();
|
|
837
|
-
|
|
1087
|
+
// The saved state is already authoritative: restore does not replay the
|
|
1088
|
+
// journal unless the snapshot is a checkpoint. Rewriting every historical
|
|
1089
|
+
// mutation on each flush copies full Effect records into a file that grows
|
|
1090
|
+
// without bound and blocks the next dispatch. For the attached backend,
|
|
1091
|
+
// persist the watermark only and drop those entries after the save succeeds.
|
|
1092
|
+
const attached = backend === this.persistenceBackend;
|
|
1093
|
+
const journalWatermark = attached ? this.mutationLog.lastSequence : undefined;
|
|
1094
|
+
const exportedLog = journalWatermark === undefined ? this.mutationLog : new MutationLog([], journalWatermark);
|
|
1095
|
+
const exported = exportRuntimePersistence(this.persistenceState(), exportedLog, this.outbox, this.quarantine, persistedPolicy, this.factInbox.snapshot(), this.persistenceCompatibility());
|
|
838
1096
|
// Never write a snapshot that the constructor would refuse to load; failing here is recoverable, a poisoned store is not.
|
|
839
1097
|
validateRuntimePersistenceSnapshot(exported);
|
|
840
1098
|
const withResults = backend.resultStore === undefined ? exported : await externalizeRuntimeResultBodies(exported, backend.resultStore);
|
|
841
1099
|
const snapshot = backend.snapshotStore === undefined ? withResults : await externalizeRuntimeSnapshotBodies(withResults, backend.snapshotStore);
|
|
842
|
-
await backend.save(snapshot,
|
|
843
|
-
if (
|
|
1100
|
+
await backend.save(snapshot, attached ? this.persistenceDigest : undefined);
|
|
1101
|
+
if (attached) {
|
|
844
1102
|
this.persistenceDigest = snapshot.integrity?.digest;
|
|
1103
|
+
if (journalWatermark !== undefined && journalWatermark > this.mutationLog.watermark && journalWatermark <= this.mutationLog.lastSequence)
|
|
1104
|
+
this.mutationLog.truncateThrough(journalWatermark);
|
|
1105
|
+
}
|
|
845
1106
|
this.storagePolicy.markPersisted();
|
|
846
1107
|
this.markArtifactsPersisted();
|
|
847
1108
|
this.syncStoragePolicy();
|
|
@@ -849,9 +1110,15 @@ export class PulseRuntime {
|
|
|
849
1110
|
async flushPersistence() {
|
|
850
1111
|
if (!this.persistenceBackend)
|
|
851
1112
|
return;
|
|
852
|
-
// An explicit flush
|
|
1113
|
+
// An explicit flush retries a pending/dirty snapshot, but must not mark an
|
|
1114
|
+
// already clean runtime dirty on every scheduler tick. The run loop calls
|
|
1115
|
+
// this after each tick; forcing a new write there can keep persistence
|
|
1116
|
+
// permanently dirty and starve the durable-dispatch gate.
|
|
853
1117
|
this.persistenceBackoff = false;
|
|
854
|
-
this.
|
|
1118
|
+
if (!this.persistenceDirty && !this.persistenceScheduled)
|
|
1119
|
+
return;
|
|
1120
|
+
if (this.persistenceDirty)
|
|
1121
|
+
this.schedulePersistence();
|
|
855
1122
|
while (true) {
|
|
856
1123
|
await this.persistencePending;
|
|
857
1124
|
if (!this.persistenceDirty)
|
|
@@ -1073,9 +1340,9 @@ export class PulseRuntime {
|
|
|
1073
1340
|
}
|
|
1074
1341
|
catch (cause) {
|
|
1075
1342
|
if (signal.aborted && isSideEffectful(definition.manifest.sideEffectPolicy))
|
|
1076
|
-
return { value: null, executionState: 'remote_unknown', sideEffectState: 'unknown', ...(executionRef === undefined ? {} : { executionRef }), metadata: { toolVersion: definition.manifest.version, reconcileRequired: true },
|
|
1343
|
+
return { value: null, executionState: 'remote_unknown', sideEffectState: 'unknown', ...(executionRef === undefined ? {} : { executionRef }), metadata: { toolVersion: definition.manifest.version, reconcileRequired: true }, error: runtimeErrorFromCause(cause, 'TOOL_CANCELLED_UNKNOWN') };
|
|
1077
1344
|
const error = runtimeErrorFromCause(cause, 'TOOL_EXECUTION_FAILED');
|
|
1078
|
-
return { value: null, status: signal.aborted ? 'cancelled' : 'failed', executionState: 'failed', sideEffectState: 'none', ...(executionRef === undefined ? {} : { executionRef }),
|
|
1345
|
+
return { value: null, status: signal.aborted ? 'cancelled' : 'failed', executionState: 'failed', sideEffectState: 'none', ...(executionRef === undefined ? {} : { executionRef }), error, ...(observations.length ? { observations } : {}) };
|
|
1079
1346
|
}
|
|
1080
1347
|
}
|
|
1081
1348
|
if (effect.kind !== 'llm')
|
|
@@ -1156,18 +1423,19 @@ export class PulseRuntime {
|
|
|
1156
1423
|
}
|
|
1157
1424
|
this.mutationLog.append(transactionId, mutations, this.state.now);
|
|
1158
1425
|
}
|
|
1159
|
-
enqueueFact(fact, eventId) {
|
|
1426
|
+
enqueueFact(fact, eventId, urgent = false) {
|
|
1160
1427
|
const candidateInbox = FactInbox.fromSnapshot(this.factInbox.snapshot());
|
|
1161
|
-
if (!candidateInbox.enqueue(fact, eventId))
|
|
1428
|
+
if (!(urgent ? candidateInbox.enqueueUrgent(fact, eventId) : candidateInbox.enqueue(fact, eventId)))
|
|
1162
1429
|
return false;
|
|
1163
1430
|
const candidatePolicy = this.storagePolicy.clone();
|
|
1164
1431
|
this.syncStoragePolicy(candidatePolicy, this.state, candidateInbox);
|
|
1165
|
-
const envelope = this.factInbox.enqueue(fact, eventId);
|
|
1432
|
+
const envelope = urgent ? this.factInbox.enqueueUrgent(fact, eventId) : this.factInbox.enqueue(fact, eventId);
|
|
1166
1433
|
if (!envelope)
|
|
1167
1434
|
return false;
|
|
1168
1435
|
this.syncStoragePolicy();
|
|
1169
1436
|
this.schedulePersistence();
|
|
1170
1437
|
this.scheduleWake();
|
|
1438
|
+
this.notifyActivity();
|
|
1171
1439
|
for (const resolve of this.factWaiters.splice(0))
|
|
1172
1440
|
resolve();
|
|
1173
1441
|
return true;
|
|
@@ -1175,9 +1443,11 @@ export class PulseRuntime {
|
|
|
1175
1443
|
enqueueHostCommand(command) {
|
|
1176
1444
|
validateHostCommand(command);
|
|
1177
1445
|
const eventId = `host-command-${this.hostCommandSeq}`;
|
|
1178
|
-
|
|
1179
|
-
|
|
1446
|
+
const urgent = command.type === 'human_input' || command.type === 'reply' || command.type === 'cancel' || command.type === 'cancel_effect';
|
|
1447
|
+
if (!this.enqueueFact(command, eventId, urgent))
|
|
1448
|
+
return false;
|
|
1180
1449
|
this.hostCommandSeq++;
|
|
1450
|
+
return true;
|
|
1181
1451
|
}
|
|
1182
1452
|
enqueueEffectCompletion(effectId, attemptId, execution, status = 'succeeded', error, dispatchError) {
|
|
1183
1453
|
const fact = { type: 'effect_completion', effectId, attemptId, execution: encodeEffectExecution(execution), status, ...(error === undefined ? {} : { error: error }), ...(dispatchError === undefined ? {} : { dispatchError: dispatchError }) };
|
|
@@ -1193,7 +1463,12 @@ export class PulseRuntime {
|
|
|
1193
1463
|
this.enqueueFact(fact, `effect-reconcile:${effect.id}:${effect.attemptId}:${status}`);
|
|
1194
1464
|
}
|
|
1195
1465
|
scheduleWake(force = false) {
|
|
1196
|
-
if (this.
|
|
1466
|
+
if (this.inDrain) {
|
|
1467
|
+
if (force)
|
|
1468
|
+
this.wakeAfterDrain = true;
|
|
1469
|
+
return;
|
|
1470
|
+
}
|
|
1471
|
+
if (this.wakeScheduled)
|
|
1197
1472
|
return;
|
|
1198
1473
|
if (!force && this.factInbox.size === 0)
|
|
1199
1474
|
return;
|
|
@@ -1211,7 +1486,9 @@ export class PulseRuntime {
|
|
|
1211
1486
|
}
|
|
1212
1487
|
finally {
|
|
1213
1488
|
this.inDrain = false;
|
|
1214
|
-
|
|
1489
|
+
const wakeAfterDrain = this.wakeAfterDrain;
|
|
1490
|
+
this.wakeAfterDrain = false;
|
|
1491
|
+
if (this.wakeError === undefined && (wakeAfterDrain || this.factInbox.size > 0 || this.hasPendingTickCleanup()))
|
|
1215
1492
|
this.scheduleWake(true);
|
|
1216
1493
|
}
|
|
1217
1494
|
});
|
|
@@ -1370,6 +1647,7 @@ export class PulseRuntime {
|
|
|
1370
1647
|
this.wakeError = undefined;
|
|
1371
1648
|
this.assertRecoveryPrograms();
|
|
1372
1649
|
this.state.now = this.clock.now();
|
|
1650
|
+
const mutationSequenceBeforeTick = this.mutationLog.lastSequence;
|
|
1373
1651
|
const tickStartedAt = performance.now();
|
|
1374
1652
|
let tickOperations = 0;
|
|
1375
1653
|
const canStartTickOperation = () => tickOperations === 0 || performance.now() - tickStartedAt < this.maxTickMs;
|
|
@@ -1409,6 +1687,17 @@ export class PulseRuntime {
|
|
|
1409
1687
|
else if (envelope.fact.type === 'scheduler_decision') {
|
|
1410
1688
|
commandApplied = this.applySchedulerDecision(envelope.fact);
|
|
1411
1689
|
}
|
|
1690
|
+
else if (envelope.fact.type === 'human_arbitration') {
|
|
1691
|
+
const decision = parseHumanArbitrationDecision(envelope.fact.decision);
|
|
1692
|
+
if (decision !== undefined)
|
|
1693
|
+
commandApplied = this.applyHumanArbitrationDecision(decision);
|
|
1694
|
+
else {
|
|
1695
|
+
const raw = envelope.fact.decision && typeof envelope.fact.decision === 'object' && !Array.isArray(envelope.fact.decision) ? envelope.fact.decision : {};
|
|
1696
|
+
const record = typeof raw.inputId === 'string' ? this.state.humanInputs.get(raw.inputId) : undefined;
|
|
1697
|
+
if (record && record.status === 'pending')
|
|
1698
|
+
commandApplied = this.applyHumanArbitrationDecision({ schemaVersion: 1, decisionId: typeof raw.decisionId === 'string' ? raw.decisionId : `invalid:${record.id}`, inputId: record.id, agentId: record.agentId, action: 'defer', reason: 'Human arbitration returned an invalid decision.', modelId: 'runtime-safety' });
|
|
1699
|
+
}
|
|
1700
|
+
}
|
|
1412
1701
|
else if (envelope.fact.type === 'effect_completion') {
|
|
1413
1702
|
if (envelope.fact.dispatchError !== undefined)
|
|
1414
1703
|
this.tryEmit({ type: 'effect.dispatch_failed', effectId: envelope.fact.effectId, data: envelope.fact.dispatchError });
|
|
@@ -1434,6 +1723,43 @@ export class PulseRuntime {
|
|
|
1434
1723
|
commandApplied = true;
|
|
1435
1724
|
}
|
|
1436
1725
|
}
|
|
1726
|
+
else if (envelope.fact.type === 'human_input') {
|
|
1727
|
+
const agent = this.state.agents.get(envelope.fact.agentId);
|
|
1728
|
+
if (!agent) {
|
|
1729
|
+
this.rejectHostCommand(envelope.eventId, 'AGENT_NOT_FOUND');
|
|
1730
|
+
commandApplied = true;
|
|
1731
|
+
}
|
|
1732
|
+
else if (this.state.humanInputs.has(envelope.fact.inputId)) {
|
|
1733
|
+
this.tryEmit({ type: 'human.input.duplicate', agentId: agent.id, data: { inputId: envelope.fact.inputId } });
|
|
1734
|
+
this.emit({ type: 'command.applied', data: { eventId: envelope.eventId, duplicate: true } });
|
|
1735
|
+
commandApplied = true;
|
|
1736
|
+
}
|
|
1737
|
+
else {
|
|
1738
|
+
const target = envelope.fact.targetEffectId === undefined ? undefined : this.state.effects.get(envelope.fact.targetEffectId);
|
|
1739
|
+
if (target !== undefined && (target.agentId !== agent.id || target.kind !== 'human' || target.outcome !== undefined)) {
|
|
1740
|
+
this.rejectHostCommand(envelope.eventId, target.agentId !== agent.id ? 'EFFECT_NOT_OWNED' : 'EFFECT_NOT_REPLYABLE');
|
|
1741
|
+
commandApplied = true;
|
|
1742
|
+
}
|
|
1743
|
+
else {
|
|
1744
|
+
const record = { id: envelope.fact.inputId, agentId: agent.id, value: structuredClone(envelope.fact.value), receivedAt: this.state.now, status: 'pending', ...(target === undefined ? {} : { targetEffectId: target.id }) };
|
|
1745
|
+
const event = { type: 'human.input.received', agentId: agent.id, ...(target === undefined ? {} : { effectId: target.id }), data: { inputId: record.id, value: record.value, priority: 'human', ...(target === undefined ? {} : { targetEffectId: target.id }) } };
|
|
1746
|
+
const mutations = [{ op: 'setHumanInput', inputId: record.id, record }, { op: 'appendEvent', event }, { op: 'appendEvent', event: { type: 'command.applied', data: { eventId: envelope.eventId } } }];
|
|
1747
|
+
this.assertStorageAdmission(mutations);
|
|
1748
|
+
commitMutationTransaction(this.state, this.mutationLog, `human-input:${record.id}`, mutations, this.state.now, this.sessionId);
|
|
1749
|
+
commandApplied = true;
|
|
1750
|
+
if (target !== undefined) {
|
|
1751
|
+
this.applyHumanArbitrationDecision({ schemaVersion: 1, decisionId: `target:${record.id}`, inputId: record.id, agentId: agent.id, action: 'respond', targetEffectId: target.id, modelId: 'target-effect' });
|
|
1752
|
+
}
|
|
1753
|
+
else {
|
|
1754
|
+
const rule = ruleHumanArbitration(record.value, agent.id, record.id);
|
|
1755
|
+
if (rule !== undefined)
|
|
1756
|
+
this.applyHumanArbitrationDecision(rule);
|
|
1757
|
+
else if (!this.requestHumanArbitration(agent, record) && this.humanInputProgram !== undefined)
|
|
1758
|
+
this.applyHumanArbitrationDecision({ schemaVersion: 1, decisionId: `default:${record.id}`, inputId: record.id, agentId: agent.id, action: 'spawn', modelId: 'runtime-default' });
|
|
1759
|
+
}
|
|
1760
|
+
}
|
|
1761
|
+
}
|
|
1762
|
+
}
|
|
1437
1763
|
else if (envelope.fact.type === 'reply') {
|
|
1438
1764
|
const effect = this.state.effects.get(envelope.fact.effectId);
|
|
1439
1765
|
if (effect?.agentId === envelope.fact.agentId && effect.kind === 'human' && !effect.outcome)
|
|
@@ -1458,7 +1784,7 @@ export class PulseRuntime {
|
|
|
1458
1784
|
else
|
|
1459
1785
|
commandApplied = this.cancelEffect(envelope.fact.effectId, 0, envelope.fact.reason, [{ op: 'appendEvent', event: { type: 'command.applied', data: { eventId: envelope.eventId } } }]);
|
|
1460
1786
|
}
|
|
1461
|
-
else {
|
|
1787
|
+
else if (envelope.fact.type === 'set_lane_priority') {
|
|
1462
1788
|
const lane = this.state.lanes.get(envelope.fact.laneId);
|
|
1463
1789
|
if (!lane) {
|
|
1464
1790
|
this.rejectHostCommand(envelope.eventId, 'LANE_NOT_FOUND');
|
|
@@ -1484,6 +1810,9 @@ export class PulseRuntime {
|
|
|
1484
1810
|
commandApplied = true;
|
|
1485
1811
|
}
|
|
1486
1812
|
}
|
|
1813
|
+
else {
|
|
1814
|
+
commandApplied = false;
|
|
1815
|
+
}
|
|
1487
1816
|
if (!commandApplied && envelope.fact.type !== 'effect_completion' && envelope.fact.type !== 'scheduler_decision')
|
|
1488
1817
|
this.emit({ type: 'command.applied', data: { eventId: envelope.eventId } });
|
|
1489
1818
|
}
|
|
@@ -1507,6 +1836,12 @@ export class PulseRuntime {
|
|
|
1507
1836
|
timer.callback();
|
|
1508
1837
|
tickOperations++;
|
|
1509
1838
|
}
|
|
1839
|
+
// A restored snapshot may contain a settled Effect and its still-pending
|
|
1840
|
+
// Wait when the process stopped between the two persistence writes. Re-run
|
|
1841
|
+
// wait resolution after recovery timers have fired so the Lane receives its
|
|
1842
|
+
// ResumeInput before the next program step is evaluated.
|
|
1843
|
+
if ([...this.state.waits.values()].some((wait) => wait.state === 'pending'))
|
|
1844
|
+
this.refreshWaits();
|
|
1510
1845
|
let progressed = 0;
|
|
1511
1846
|
while (progressed < this.maxSteps && canStartTickOperation()) {
|
|
1512
1847
|
const laneId = this.selectReadyLane(this.state.now, this.maxSteps - progressed);
|
|
@@ -1528,7 +1863,7 @@ export class PulseRuntime {
|
|
|
1528
1863
|
continue;
|
|
1529
1864
|
}
|
|
1530
1865
|
let output;
|
|
1531
|
-
const stepContext = { lane: stepLane, state: structuredClone(this.state), ...(lane.pendingResumeInput ? { resumeInput: structuredClone(lane.pendingResumeInput) } : {}), now: this.state.now, observe: (event) => { this.observationInbox.enqueue({ ...event, agentId: lane.agentId, laneId: lane.id, timestamp: this.state.now }); } };
|
|
1866
|
+
const stepContext = { lane: stepLane, state: structuredClone(this.state), ...(lane.pendingResumeInput ? { resumeInput: structuredClone(lane.pendingResumeInput) } : {}), ...(lane.pendingHumanInputs?.length ? { humanInputs: structuredClone(lane.pendingHumanInputs) } : {}), now: this.state.now, observe: (event) => { this.observationInbox.enqueue({ ...event, agentId: lane.agentId, laneId: lane.id, timestamp: this.state.now }); this.notifyActivity(); } };
|
|
1532
1867
|
try {
|
|
1533
1868
|
output = withPureStepGuard(() => lane.series || program.seriesMember ? this.seriesStep(program, stepContext, lane.series) : program.step(stepContext));
|
|
1534
1869
|
}
|
|
@@ -1602,6 +1937,7 @@ export class PulseRuntime {
|
|
|
1602
1937
|
return mutation;
|
|
1603
1938
|
const nextLane = structuredClone(mutation.record);
|
|
1604
1939
|
delete nextLane.consecutiveControlErrors;
|
|
1940
|
+
delete nextLane.pendingHumanInputs;
|
|
1605
1941
|
replaceResumeInput(nextLane, undefined);
|
|
1606
1942
|
nextLane.progressWatchdog = watchdog.state;
|
|
1607
1943
|
return { ...mutation, record: nextLane };
|
|
@@ -1642,12 +1978,19 @@ export class PulseRuntime {
|
|
|
1642
1978
|
this.dispatchQueuedEffects();
|
|
1643
1979
|
this.completeFinishedChildAgents();
|
|
1644
1980
|
this.finalizeCancellations();
|
|
1645
|
-
this.
|
|
1646
|
-
|
|
1981
|
+
const tickMutated = this.mutationLog.lastSequence !== mutationSequenceBeforeTick;
|
|
1982
|
+
if (tickMutated)
|
|
1983
|
+
this.syncStoragePolicy();
|
|
1984
|
+
// A wake can be scheduled solely to retry dispatch or drain an empty
|
|
1985
|
+
// queue. Persist only when this tick committed a mutation; otherwise a
|
|
1986
|
+
// waiting run rewrites the full runtime snapshot on every wake.
|
|
1987
|
+
if (tickMutated)
|
|
1988
|
+
this.schedulePersistence();
|
|
1647
1989
|
const pendingTickCleanup = this.hasPendingTickCleanup();
|
|
1648
1990
|
this.tickBudget = undefined;
|
|
1649
1991
|
if (pendingTickCleanup)
|
|
1650
1992
|
this.scheduleWake(true);
|
|
1993
|
+
this.notifyActivity();
|
|
1651
1994
|
return progressed;
|
|
1652
1995
|
}
|
|
1653
1996
|
async run(agentOrMaxTicks = 10_000, requestedMaxTicks = 10_000) {
|
|
@@ -1662,7 +2005,7 @@ export class PulseRuntime {
|
|
|
1662
2005
|
}
|
|
1663
2006
|
if (this.ready.size === 0 && this.executions.size === 0 && !this.hasQueuedEffects() && !this.hasPendingTickCleanup()) {
|
|
1664
2007
|
if (this.preparingLLMs.size) {
|
|
1665
|
-
await
|
|
2008
|
+
await this.waitForFact();
|
|
1666
2009
|
continue;
|
|
1667
2010
|
}
|
|
1668
2011
|
if (this.factInbox.size > 0)
|
|
@@ -1731,7 +2074,7 @@ export class PulseRuntime {
|
|
|
1731
2074
|
}
|
|
1732
2075
|
if (this.ready.size === 0 && this.executions.size === 0 && !this.hasQueuedEffects() && !this.hasPendingTickCleanup()) {
|
|
1733
2076
|
if (this.preparingLLMs.size) {
|
|
1734
|
-
await
|
|
2077
|
+
await this.waitForFact();
|
|
1735
2078
|
continue;
|
|
1736
2079
|
}
|
|
1737
2080
|
if (this.factInbox.size > 0)
|
|
@@ -1784,13 +2127,15 @@ export class PulseRuntime {
|
|
|
1784
2127
|
if (this.executions.size)
|
|
1785
2128
|
await Promise.race([...this.executions.values()].map((execution) => execution.promise));
|
|
1786
2129
|
else if (this.preparingLLMs.size)
|
|
1787
|
-
await
|
|
2130
|
+
await this.waitForFact();
|
|
1788
2131
|
else if (this.hasQueuedEffects() || this.factInbox.size || this.hasPendingTickCleanup() || this.hasDueTimer())
|
|
1789
2132
|
await new Promise((resolve) => setImmediate(resolve));
|
|
1790
2133
|
} await this.flushPersistence(); }
|
|
1791
2134
|
async shutdown(timeoutMs = 5_000) {
|
|
1792
2135
|
this.shuttingDown = true;
|
|
1793
2136
|
this.schedulerDecisionCoordinator?.cancel();
|
|
2137
|
+
this.humanArbitrationCoordinator?.cancel();
|
|
2138
|
+
this.humanArbitrationRequests.clear();
|
|
1794
2139
|
this.schedulerDecisionRequests.clear();
|
|
1795
2140
|
this.schedulerDecisionCache = undefined;
|
|
1796
2141
|
for (const agent of this.state.agents.values())
|
|
@@ -1984,9 +2329,16 @@ export class PulseRuntime {
|
|
|
1984
2329
|
if (result)
|
|
1985
2330
|
Object.assign(result, value);
|
|
1986
2331
|
}
|
|
1987
|
-
if (this.sessionStore)
|
|
1988
|
-
for (const agent of state.agents.values())
|
|
1989
|
-
|
|
2332
|
+
if (this.sessionStore) {
|
|
2333
|
+
for (const agent of state.agents.values()) {
|
|
2334
|
+
const snapshot = exportWarmStartSession(state, agent.id);
|
|
2335
|
+
const digest = contentHash(snapshot);
|
|
2336
|
+
if (this.sessionStoreDigests.get(agent.id) === digest)
|
|
2337
|
+
continue;
|
|
2338
|
+
this.sessionStore.put(snapshot);
|
|
2339
|
+
this.sessionStoreDigests.set(agent.id, digest);
|
|
2340
|
+
}
|
|
2341
|
+
}
|
|
1990
2342
|
}
|
|
1991
2343
|
}
|
|
1992
2344
|
hasQueuedEffects() { return [...this.state.effects.values()].some((effect) => effect.state === 'queued' && !this.executions.has(effect.id)); }
|
|
@@ -2025,7 +2377,39 @@ export class PulseRuntime {
|
|
|
2025
2377
|
}
|
|
2026
2378
|
hasPendingHostInteraction(agentId) { return [...this.state.effects.values()].some((effect) => effect.kind === 'human' && !effect.outcome && (agentId === undefined || effect.agentId === agentId)); }
|
|
2027
2379
|
waitForFact() { return new Promise((resolve) => this.factWaiters.push(resolve)); }
|
|
2380
|
+
/** Wait for a state or observation change without polling the event loop. */
|
|
2381
|
+
waitForActivity(timeoutMs = 250) {
|
|
2382
|
+
return new Promise((resolve) => {
|
|
2383
|
+
let settled = false;
|
|
2384
|
+
const finish = () => {
|
|
2385
|
+
if (settled)
|
|
2386
|
+
return;
|
|
2387
|
+
settled = true;
|
|
2388
|
+
this.activityWaiters.delete(finish);
|
|
2389
|
+
clearTimeout(timer);
|
|
2390
|
+
resolve();
|
|
2391
|
+
};
|
|
2392
|
+
const timer = setTimeout(finish, timeoutMs);
|
|
2393
|
+
this.activityWaiters.add(finish);
|
|
2394
|
+
});
|
|
2395
|
+
}
|
|
2396
|
+
notifyActivity() {
|
|
2397
|
+
for (const resolve of [...this.activityWaiters])
|
|
2398
|
+
resolve();
|
|
2399
|
+
}
|
|
2028
2400
|
completeFinishedChildAgents() {
|
|
2401
|
+
// Human interaction Agents are detached from the main Lane rather than
|
|
2402
|
+
// represented by an agent Effect. Settle their Agent record when the root
|
|
2403
|
+
// Lane reaches a terminal state so session streams can close correctly.
|
|
2404
|
+
for (const child of this.state.agents.values()) {
|
|
2405
|
+
if (child.parentAgentId === undefined || ['succeeded', 'failed', 'cancelled'].includes(child.state ?? ''))
|
|
2406
|
+
continue;
|
|
2407
|
+
const root = this.state.lanes.get(child.rootLaneId);
|
|
2408
|
+
if (!root || !['succeeded', 'failed', 'cancelled'].includes(root.status))
|
|
2409
|
+
continue;
|
|
2410
|
+
const status = root.status === 'succeeded' ? 'succeeded' : root.status === 'cancelled' ? 'cancelled' : 'failed';
|
|
2411
|
+
this.commitAgentState(child.id, status, `agent:${child.id}:interaction-settled:${root.version}`);
|
|
2412
|
+
}
|
|
2029
2413
|
for (const effect of this.state.effects.values()) {
|
|
2030
2414
|
if (effect.kind !== 'agent' || !effect.childAgentId || effect.outcome)
|
|
2031
2415
|
continue;
|
|
@@ -2229,8 +2613,10 @@ export class PulseRuntime {
|
|
|
2229
2613
|
}
|
|
2230
2614
|
this.releaseEffectLocks(effectId);
|
|
2231
2615
|
this.outbox.ack(`${effect.id}:${effect.attemptId}`);
|
|
2232
|
-
for (const observation of effectiveExecution.observations ?? [])
|
|
2616
|
+
for (const observation of effectiveExecution.observations ?? []) {
|
|
2233
2617
|
this.observationInbox.enqueue({ ...observation, agentId: effect.agentId, laneId: effect.ownerLaneId, timestamp: this.state.now });
|
|
2618
|
+
this.notifyActivity();
|
|
2619
|
+
}
|
|
2234
2620
|
const settlementTransactionId = `effect:${effect.id}:${settledAttemptId}:settled`;
|
|
2235
2621
|
const settlementMutations = [...publicationMutations];
|
|
2236
2622
|
commitMutationTransaction(this.state, this.mutationLog, settlementTransactionId, settlementMutations, this.state.now, this.sessionId);
|
|
@@ -2464,8 +2850,9 @@ export class PulseRuntime {
|
|
|
2464
2850
|
...cancellableEffects.map((effect) => ({ type: 'effect.cancel_requested', effectId: effect.id, data: { reason } })),
|
|
2465
2851
|
...cancellableEffects.flatMap((effect) => {
|
|
2466
2852
|
if (this.executions.has(effect.id) && (effect.cancelGraceMs ?? 0) === 0) {
|
|
2467
|
-
|
|
2468
|
-
|
|
2853
|
+
if (isSideEffectful(effect.sideEffectPolicy))
|
|
2854
|
+
return [{ type: 'effect.quarantined', effectId: effect.id, data: { reason, state: 'reconcile_required' } }];
|
|
2855
|
+
return [{ type: 'effect.settled', effectId: effect.id, data: { status: 'cancelled', error: { code: 'CANCELLED', message: reason } } }];
|
|
2469
2856
|
}
|
|
2470
2857
|
if (!this.executions.has(effect.id))
|
|
2471
2858
|
return [{ type: 'effect.settled', effectId: effect.id, data: { status: 'cancelled', error: { code: 'CANCELLED', message: reason } } }];
|
|
@@ -2499,9 +2886,10 @@ export class PulseRuntime {
|
|
|
2499
2886
|
const candidate = structuredClone(effect);
|
|
2500
2887
|
candidate.cancelRequested = { reason, at: this.state.now };
|
|
2501
2888
|
if (this.executions.has(effect.id) && (effect.cancelGraceMs ?? 0) === 0) {
|
|
2502
|
-
|
|
2503
|
-
candidate.
|
|
2504
|
-
candidate.
|
|
2889
|
+
const shouldQuarantine = isSideEffectful(candidate.sideEffectPolicy);
|
|
2890
|
+
candidate.executionState = shouldQuarantine ? 'remote_unknown' : 'local_closed';
|
|
2891
|
+
candidate.sideEffectState = shouldQuarantine ? 'unknown' : 'none';
|
|
2892
|
+
candidate.state = shouldQuarantine ? 'reconcile_required' : 'cancelled';
|
|
2505
2893
|
if (candidate.state === 'cancelled')
|
|
2506
2894
|
candidate.outcome = { status: 'cancelled', reason, error: { code: reason, message: reason } };
|
|
2507
2895
|
}
|
|
@@ -2620,7 +3008,8 @@ export class PulseRuntime {
|
|
|
2620
3008
|
return { id: lane.id, agentId: lane.agentId, status: lane.status, cancelReason: lane.cancelReason ?? null, failure: lane.failure?.error ?? null, goal: lane.goal, basePriority: lane.priority, effectivePriority: ready?.effectivePriority ?? lane.priority, queueWaitMs: ready ? Math.max(0, this.state.now - lane.readySince) : 0, blockedBy, activeWaitId: lane.activeWaitId ?? null, lastEventSeq: lastEvent('lane', lane.id), watchdog: lane.progressWatchdog ?? null, consecutiveControlErrors: lane.consecutiveControlErrors ?? 0, lastInterventionReason: lane.progressWatchdog?.lastReason ?? null, unresolvedEffectIds: lane.unresolvedEffectIds ?? [] };
|
|
2621
3009
|
});
|
|
2622
3010
|
const effects = [...this.state.effects.values()].filter((effect) => laneId === undefined || effect.ownerLaneId === laneId).map((effect) => ({ id: effect.id, state: effect.state, executionState: effect.executionState, sideEffectState: effect.sideEffectState, attemptId: effect.attemptId, inheritedFloor: effect.inheritedFloor ?? null, deadlineAt: effect.deadlineAt ?? null, lastEventSeq: lastEvent('effect', effect.id), preparation: effect.preparation ?? null, metadata: latestEffectMetadata(effect.id) }));
|
|
2623
|
-
|
|
3011
|
+
const humanInputs = [...this.state.humanInputs.values()].filter((input) => laneId === undefined || input.agentId === this.state.lanes.get(laneId)?.agentId).map((input) => structuredClone(input));
|
|
3012
|
+
return { now: this.state.now, lanes, effects, humanInputs, preparation: { preparing: this.preparingLLMs.size, prepared: [...this.state.effects.values()].filter((effect) => effect.state === 'queued' && effect.preparation?.state === 'prepared').length, maxPreparing: this.maxPreparingLLMs, maxPrepared: this.maxPreparedLLMs }, quarantine: this.quarantine.unresolvedEffectIds };
|
|
2624
3013
|
}
|
|
2625
3014
|
retryEffect(effectId, delayMs) {
|
|
2626
3015
|
const effect = this.state.effects.get(effectId);
|
|
@@ -2687,9 +3076,13 @@ export class PulseRuntime {
|
|
|
2687
3076
|
this.dispatchQueuedEffectsNow();
|
|
2688
3077
|
return;
|
|
2689
3078
|
}
|
|
2690
|
-
this.schedulePersistence();
|
|
2691
3079
|
if (this.dispatchPersistencePending)
|
|
2692
3080
|
return;
|
|
3081
|
+
// Do not mark persistence dirty again while the durability gate is
|
|
3082
|
+
// already flushing. The scheduler calls this method on every tick; a
|
|
3083
|
+
// write scheduled before this guard would keep the flush promise alive
|
|
3084
|
+
// forever and starve the queued effects behind it.
|
|
3085
|
+
this.schedulePersistence();
|
|
2693
3086
|
this.dispatchPersistencePending = true;
|
|
2694
3087
|
void this.flushPersistence().then(() => {
|
|
2695
3088
|
this.dispatchPersistencePending = false;
|
|
@@ -2709,6 +3102,13 @@ export class PulseRuntime {
|
|
|
2709
3102
|
// Higher effective priority (max of own priority and inherited floor) dispatches first, matching ReadyQueue semantics.
|
|
2710
3103
|
const effectivePriority = (effect) => Math.max(effect.schedulePriority ?? 0, effect.inheritedFloor ?? Number.NEGATIVE_INFINITY);
|
|
2711
3104
|
const queued = [...this.state.effects.values()].filter((effect) => effect.state === 'queued' && !this.executions.has(effect.id)).sort((a, b) => (effectivePriority(b) - effectivePriority(a)) || a.id.localeCompare(b.id));
|
|
3105
|
+
const llmLimit = () => {
|
|
3106
|
+
const configured = this.state.maxRunning.llm;
|
|
3107
|
+
if ((this.humanInputProgram === undefined && this.humanArbitrationCoordinator === undefined) || !Number.isFinite(configured))
|
|
3108
|
+
return configured;
|
|
3109
|
+
const hasUrgentHumanWork = queued.some((candidate) => candidate.kind === 'llm' && effectivePriority(candidate) >= 2);
|
|
3110
|
+
return hasUrgentHumanWork ? configured : Math.max(0, configured - 1);
|
|
3111
|
+
};
|
|
2712
3112
|
for (const effect of queued) {
|
|
2713
3113
|
if (this.tickBudget && !this.tickBudget.canStart())
|
|
2714
3114
|
break;
|
|
@@ -2716,7 +3116,7 @@ export class PulseRuntime {
|
|
|
2716
3116
|
continue;
|
|
2717
3117
|
if (effect.kind === 'llm' && !this.prepareLLMEffect(effect))
|
|
2718
3118
|
continue;
|
|
2719
|
-
if (effect.concurrencyClass !== 'none' && this.runningCount(effect.concurrencyClass) >= this.state.maxRunning[effect.concurrencyClass])
|
|
3119
|
+
if (effect.concurrencyClass !== 'none' && this.runningCount(effect.concurrencyClass) >= (effect.concurrencyClass === 'llm' ? llmLimit() : this.state.maxRunning[effect.concurrencyClass]))
|
|
2720
3120
|
continue;
|
|
2721
3121
|
const budgetError = this.budgetRejection(effect);
|
|
2722
3122
|
if (budgetError) {
|
|
@@ -2823,6 +3223,7 @@ export class PulseRuntime {
|
|
|
2823
3223
|
return;
|
|
2824
3224
|
}
|
|
2825
3225
|
this.observationInbox.enqueue({ ...observation, agentId: effect.agentId, laneId: effect.ownerLaneId, timestamp: this.state.now });
|
|
3226
|
+
this.notifyActivity();
|
|
2826
3227
|
};
|
|
2827
3228
|
const attemptId = effect.attemptId;
|
|
2828
3229
|
const promise = this.executor(effect, controller.signal, emitObservation).then((execution) => {
|
|
@@ -2909,25 +3310,28 @@ export class PulseRuntime {
|
|
|
2909
3310
|
if (!effect || effect.outcome)
|
|
2910
3311
|
return false;
|
|
2911
3312
|
const candidate = structuredClone(effect);
|
|
3313
|
+
const shouldQuarantine = isSideEffectful(candidate.sideEffectPolicy);
|
|
2912
3314
|
if (precedingEvent?.type === 'effect.cancel_requested' || precedingEvent?.type === 'limit.rejected')
|
|
2913
3315
|
candidate.cancelRequested = { reason, at: this.state.now };
|
|
2914
|
-
candidate.executionState = 'remote_unknown';
|
|
2915
|
-
candidate.sideEffectState =
|
|
2916
|
-
candidate.state =
|
|
3316
|
+
candidate.executionState = shouldQuarantine ? 'remote_unknown' : 'local_closed';
|
|
3317
|
+
candidate.sideEffectState = shouldQuarantine ? 'unknown' : 'none';
|
|
3318
|
+
candidate.state = shouldQuarantine ? 'reconcile_required' : 'cancelled';
|
|
2917
3319
|
if (candidate.state === 'cancelled')
|
|
2918
3320
|
candidate.outcome = { status: 'cancelled', reason, error: { code: reason, message: reason } };
|
|
2919
3321
|
const lane = this.state.lanes.get(effect.ownerLaneId);
|
|
2920
3322
|
const candidateLane = lane === undefined ? undefined : structuredClone(lane);
|
|
2921
|
-
if (candidateLane)
|
|
3323
|
+
if (shouldQuarantine && candidateLane)
|
|
2922
3324
|
candidateLane.unresolvedEffectIds = [...new Set([...(candidateLane.unresolvedEffectIds ?? []), effectId])];
|
|
2923
|
-
const
|
|
3325
|
+
const terminalEvent = shouldQuarantine
|
|
3326
|
+
? { type: 'effect.quarantined', effectId, data: { reason, state: candidate.state } }
|
|
3327
|
+
: { type: 'effect.settled', effectId, data: candidate.outcome };
|
|
2924
3328
|
const admission = [{ op: 'setEffect', effectId, record: candidate }];
|
|
2925
3329
|
if (candidateLane)
|
|
2926
3330
|
admission.push({ op: 'setLane', laneId: candidateLane.id, record: candidateLane });
|
|
2927
3331
|
if (precedingEvent)
|
|
2928
3332
|
admission.push({ op: 'appendEvent', event: precedingEvent });
|
|
2929
3333
|
admission.push(...additionalMutations.map((mutation) => structuredClone(mutation)));
|
|
2930
|
-
admission.push({ op: 'appendEvent', event:
|
|
3334
|
+
admission.push({ op: 'appendEvent', event: terminalEvent });
|
|
2931
3335
|
this.assertStorageAdmission(admission);
|
|
2932
3336
|
if (execution) {
|
|
2933
3337
|
execution.controller.abort();
|
|
@@ -2941,7 +3345,8 @@ export class PulseRuntime {
|
|
|
2941
3345
|
Object.assign(lane, candidateLane);
|
|
2942
3346
|
this.state.lanes.set(candidateLane.id, lane);
|
|
2943
3347
|
}
|
|
2944
|
-
|
|
3348
|
+
if (shouldQuarantine)
|
|
3349
|
+
this.quarantine.add(effectId, this.state.now, reason);
|
|
2945
3350
|
this.refreshWaits();
|
|
2946
3351
|
this.schedulePersistence();
|
|
2947
3352
|
return true;
|