@atolis-hq/wake 0.4.94 → 0.4.96
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/artifacts/application/delegation-artifacts.d.ts +26 -0
- package/dist/src/artifacts/application/delegation-artifacts.js +33 -0
- package/dist/src/artifacts/index.d.ts +1 -0
- package/dist/src/artifacts/index.js +1 -0
- package/dist/src/bootstrap/composition-root.js +16 -3
- package/dist/src/bootstrap/doctor-processor-health.js +19 -3
- package/dist/src/bootstrap/integration-runtime.d.ts +2 -1
- package/dist/src/bootstrap/integration-runtime.js +13 -16
- package/dist/src/bootstrap/poll-coordinator.d.ts +36 -0
- package/dist/src/bootstrap/poll-coordinator.js +93 -0
- package/dist/src/bootstrap/version.js +1 -1
- package/dist/src/control-plane/application/activation-scheduler.js +28 -43
- package/dist/src/control-plane/application/advance-once-dispatch.d.ts +0 -13
- package/dist/src/control-plane/application/advance-once-dispatch.js +10 -24
- package/dist/src/control-plane/application/in-process-scheduler-serialiser.d.ts +1 -0
- package/dist/src/control-plane/application/in-process-scheduler-serialiser.js +15 -0
- package/dist/src/control-plane/application/runner-quota-outcome.d.ts +13 -0
- package/dist/src/control-plane/application/runner-quota-outcome.js +23 -0
- package/dist/src/integrations/application/adapter-health-tracker.d.ts +2 -0
- package/dist/src/integrations/application/adapter-health-tracker.js +3 -0
- package/dist/src/integrations/application/poll-service.js +4 -1
- package/dist/src/integrations/contracts/provider.d.ts +4 -0
- package/dist/src/integrations/github/infrastructure/source.js +13 -5
- package/dist/src/integrations/github/provider.d.ts +2 -0
- package/dist/src/integrations/github/provider.js +28 -2
- package/dist/src/orchestration/application/accept-signal.d.ts +3 -1
- package/dist/src/orchestration/application/accept-signal.js +2 -0
- package/dist/src/orchestration/application/advance-workflow.d.ts +0 -1
- package/dist/src/orchestration/application/advance-workflow.js +0 -4
- package/dist/src/orchestration/application/deferred-signals.d.ts +21 -0
- package/dist/src/orchestration/application/deferred-signals.js +119 -0
- package/dist/src/orchestration/application/delegated-work.d.ts +51 -0
- package/dist/src/orchestration/application/delegated-work.js +420 -0
- package/dist/src/orchestration/application/delegation-gate.d.ts +17 -0
- package/dist/src/orchestration/application/delegation-gate.js +31 -0
- package/dist/src/orchestration/application/delegation-ledger.d.ts +28 -0
- package/dist/src/orchestration/application/delegation-ledger.js +79 -0
- package/dist/src/orchestration/application/delegation-reservation.d.ts +15 -0
- package/dist/src/orchestration/application/delegation-reservation.js +54 -0
- package/dist/src/orchestration/application/expire-timed-out-waits.d.ts +1 -1
- package/dist/src/orchestration/application/expire-timed-out-waits.js +2 -2
- package/dist/src/orchestration/application/orchestration-projection.js +5 -1
- package/dist/src/orchestration/application/orchestration-repository.d.ts +5 -0
- package/dist/src/orchestration/application/orchestration-repository.js +69 -11
- package/dist/src/orchestration/application/orchestration-service.d.ts +28 -3
- package/dist/src/orchestration/application/orchestration-service.js +69 -20
- package/dist/src/orchestration/application/resource-transition-matching.js +11 -6
- package/dist/src/orchestration/application/start-workflow.js +4 -2
- package/dist/src/orchestration/application/workflow-definition-registry.d.ts +5 -1
- package/dist/src/orchestration/application/workflow-definition-registry.js +16 -3
- package/dist/src/orchestration/contracts/commands.d.ts +1 -0
- package/dist/src/orchestration/contracts/delegation-ports.d.ts +14 -0
- package/dist/src/orchestration/contracts/delegation-ports.js +1 -0
- package/dist/src/orchestration/contracts/delegation.d.ts +98 -0
- package/dist/src/orchestration/contracts/delegation.js +70 -0
- package/dist/src/orchestration/contracts/event-decoder.js +19 -0
- package/dist/src/orchestration/contracts/event-factory.js +16 -0
- package/dist/src/orchestration/contracts/events.d.ts +39 -0
- package/dist/src/orchestration/contracts/events.js +8 -0
- package/dist/src/orchestration/contracts/views.d.ts +8 -0
- package/dist/src/orchestration/domain/workflow-completion.d.ts +3 -0
- package/dist/src/orchestration/domain/workflow-completion.js +9 -0
- package/dist/src/orchestration/domain/workflow-instance-events.d.ts +2 -0
- package/dist/src/orchestration/domain/workflow-instance-events.js +37 -0
- package/dist/src/orchestration/index.d.ts +2 -0
- package/dist/src/orchestration/index.js +1 -0
- package/dist/src/work/application/work-projection.js +3 -0
- package/dist/src/work/application/work-service.js +16 -0
- package/dist/src/work/contracts/commands.d.ts +6 -0
- package/dist/src/work/contracts/events.d.ts +2 -1
- package/dist/src/work/contracts/events.js +8 -0
- package/dist/src/work/contracts/views.d.ts +2 -1
- package/dist/src/work/domain/work-item.js +3 -0
- package/package.json +1 -1
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { ArtifactRevisionReference } from '../contracts/context.js';
|
|
2
|
+
import type { ArtifactContextService } from './artifact-context-service.js';
|
|
3
|
+
import type { ArtifactService } from './artifact-service.js';
|
|
4
|
+
/** Binds delegation to Artifacts grants and accepted exact revisions; children never see the join. */
|
|
5
|
+
export declare function createDelegationArtifacts(input: {
|
|
6
|
+
readonly artifacts: ArtifactService;
|
|
7
|
+
readonly contexts: ArtifactContextService;
|
|
8
|
+
readonly accepted: (activationId: string, workItemId?: string) => Promise<boolean>;
|
|
9
|
+
}): {
|
|
10
|
+
validateInputs(_parent: {
|
|
11
|
+
readonly workItemId: string;
|
|
12
|
+
}, references: readonly ArtifactRevisionReference[]): Promise<void>;
|
|
13
|
+
registerChild: (args_0: import("../contracts/context.js").ArtifactDelegationGrant, args_1: import("@atolis-hq/eventing").CommandContext) => Promise<void>;
|
|
14
|
+
qualifyingOutputs(child: {
|
|
15
|
+
readonly workItemId: ArtifactRevisionReference["workItemId"];
|
|
16
|
+
}, outputs: readonly {
|
|
17
|
+
readonly name: string;
|
|
18
|
+
readonly path: string;
|
|
19
|
+
}[]): Promise<{
|
|
20
|
+
name: string;
|
|
21
|
+
reference: {
|
|
22
|
+
workItemId: import("../../work/index.js").WorkItemId;
|
|
23
|
+
revisionId: string;
|
|
24
|
+
};
|
|
25
|
+
}[] | null>;
|
|
26
|
+
};
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/** Binds delegation to Artifacts grants and accepted exact revisions; children never see the join. */
|
|
2
|
+
export function createDelegationArtifacts(input) {
|
|
3
|
+
return {
|
|
4
|
+
async validateInputs(_parent, references) {
|
|
5
|
+
const accepted = input.accepted;
|
|
6
|
+
for (const reference of references) {
|
|
7
|
+
const revision = (await input.artifacts.get(reference.workItemId))?.revisions.find((candidate) => candidate.revisionId === reference.revisionId);
|
|
8
|
+
if (revision === undefined ||
|
|
9
|
+
revision.deleted ||
|
|
10
|
+
!(await accepted(revision.activationId, reference.workItemId)))
|
|
11
|
+
throw new Error('Delegation context must reference accepted artifact revisions');
|
|
12
|
+
}
|
|
13
|
+
},
|
|
14
|
+
registerChild: async (...args) => input.contexts.registerChild(args[0], input.accepted, args[1]),
|
|
15
|
+
async qualifyingOutputs(child, outputs) {
|
|
16
|
+
const published = await input.artifacts.latestPublished(child.workItemId, (activationId) => input.accepted(activationId, child.workItemId));
|
|
17
|
+
const selected = outputs.map((output) => ({
|
|
18
|
+
name: output.name,
|
|
19
|
+
revision: published.find((revision) => revision.path === output.path),
|
|
20
|
+
}));
|
|
21
|
+
const references = [];
|
|
22
|
+
for (const entry of selected) {
|
|
23
|
+
if (entry.revision === undefined)
|
|
24
|
+
return null;
|
|
25
|
+
references.push({
|
|
26
|
+
name: entry.name,
|
|
27
|
+
reference: { workItemId: child.workItemId, revisionId: entry.revision.revisionId },
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
return references;
|
|
31
|
+
},
|
|
32
|
+
};
|
|
33
|
+
}
|
|
@@ -18,3 +18,4 @@ export * from './contracts/context-events.js';
|
|
|
18
18
|
export * from './contracts/context.js';
|
|
19
19
|
export * from './application/operator-instructions-service.js';
|
|
20
20
|
export * from './application/operator-instructions-projection.js';
|
|
21
|
+
export { createDelegationArtifacts } from './application/delegation-artifacts.js';
|
|
@@ -18,3 +18,4 @@ export * from './contracts/context-events.js';
|
|
|
18
18
|
export * from './contracts/context.js';
|
|
19
19
|
export * from './application/operator-instructions-service.js';
|
|
20
20
|
export * from './application/operator-instructions-projection.js';
|
|
21
|
+
export { createDelegationArtifacts } from './application/delegation-artifacts.js';
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { acquireFileLock } from '@atolis-hq/eventing-filesystem';
|
|
2
2
|
import { createPullRequestService } from '../activities/index.js';
|
|
3
|
-
import { ArtifactContextService, FileArtifactMcpSessionStore, FileArtifactStore, OperatorInstructionsService, createArtifactService, } from '../artifacts/index.js';
|
|
3
|
+
import { ArtifactContextService, FileArtifactMcpSessionStore, FileArtifactStore, OperatorInstructionsService, createArtifactService, createDelegationArtifacts, } from '../artifacts/index.js';
|
|
4
4
|
import { DispatchPolicy, createActivationScheduler, createActivationSchedulerSubscriber, createControlPlaneService, createDurableRunnerIneligibility, createRunnerControlService, } from '../control-plane/index.js';
|
|
5
5
|
import { createConversationService } from '../conversations/index.js';
|
|
6
|
-
import { ExecutionCancellationReason, GitWorkspaceProvider, RecoveryService, TranscriptStore, createExecutionService, createLocalProcessInspector, createRunnerMemoryProfileDecorator, } from '../execution/index.js';
|
|
6
|
+
import { ExecutionCancellationReason, GitWorkspaceProvider, RecoveryService, RunStatus, TranscriptStore, createExecutionService, createLocalProcessInspector, createRunnerMemoryProfileDecorator, isActiveRunStatus, } from '../execution/index.js';
|
|
7
7
|
import { createGitHubAgentContextReader, gitHubProviderDefinition, gitHubRepositoryIdentity, resolveGitHubResourceUrl, } from '../integrations/github/index.js';
|
|
8
8
|
import { SystemClock, UlidIdGenerator } from '../kernel/index.js';
|
|
9
9
|
import { compileWorkflow, createAgentActivityClassifier, createOrchestrationService, } from '../orchestration/index.js';
|
|
@@ -63,7 +63,20 @@ async function createApplicationServices(wakeRoot, options = {}) {
|
|
|
63
63
|
process.stderr.write(`Warning: ${diagnostic.message}\n`);
|
|
64
64
|
}),
|
|
65
65
|
]));
|
|
66
|
-
const orchestration = createOrchestrationService(journal, work, definitions, projections, createAgentActivityClassifier(activities))
|
|
66
|
+
const orchestration = createOrchestrationService(journal, work, definitions, projections, createAgentActivityClassifier(activities), (queries) => ({
|
|
67
|
+
artifacts: createDelegationArtifacts({
|
|
68
|
+
artifacts,
|
|
69
|
+
contexts: artifactContexts,
|
|
70
|
+
accepted: queries.isActivationAccepted,
|
|
71
|
+
}),
|
|
72
|
+
runHasEnded: async (runId) => {
|
|
73
|
+
const run = (await execution.list()).find((entry) => entry.runId === runId);
|
|
74
|
+
return (run !== undefined &&
|
|
75
|
+
!isActiveRunStatus(run.status) &&
|
|
76
|
+
run.status !== RunStatus.Ambiguous &&
|
|
77
|
+
!execution.isLocallyActive(runId));
|
|
78
|
+
},
|
|
79
|
+
}));
|
|
67
80
|
const workspaces = new GitWorkspaceProvider(paths.workspacesRoot, {
|
|
68
81
|
async cloneLocator(id) {
|
|
69
82
|
const resource = await resources.get(resourceId(id));
|
|
@@ -10,6 +10,15 @@ const healthSchema = z.object({
|
|
|
10
10
|
kind: z.string().optional(),
|
|
11
11
|
detail: z.string().optional(),
|
|
12
12
|
})),
|
|
13
|
+
adapters: z
|
|
14
|
+
.array(z.object({
|
|
15
|
+
adapter: z.string(),
|
|
16
|
+
scope: z.string(),
|
|
17
|
+
channel: z.string(),
|
|
18
|
+
status: z.string(),
|
|
19
|
+
detail: z.string().optional(),
|
|
20
|
+
}))
|
|
21
|
+
.optional(),
|
|
13
22
|
}),
|
|
14
23
|
});
|
|
15
24
|
function unavailable(reason) {
|
|
@@ -59,10 +68,17 @@ export async function doctorProcessorHealth(wakeRoot, config) {
|
|
|
59
68
|
const checks = parsed.data.data.checks.filter(({ kind }) => kind === ApiHealthCheckKind.Processor);
|
|
60
69
|
if (checks.length === 0)
|
|
61
70
|
return unavailable('the API reported no processors');
|
|
71
|
+
// Only degraded adapters are reported; 'unknown' means never exercised.
|
|
72
|
+
const adapterFailures = (parsed.data.data.adapters ?? [])
|
|
73
|
+
.filter(({ status }) => status === 'degraded')
|
|
74
|
+
.map(({ adapter, scope, channel, detail }) => `Adapter ${adapter} ${scope} (${channel}): ${detail ?? 'degraded'}`);
|
|
62
75
|
return {
|
|
63
|
-
failures:
|
|
64
|
-
|
|
65
|
-
|
|
76
|
+
failures: [
|
|
77
|
+
...checks
|
|
78
|
+
.filter(({ status }) => status !== healthyStatus)
|
|
79
|
+
.map(({ name, detail }) => `Processor ${name}: ${detail ?? 'degraded'}`),
|
|
80
|
+
...adapterFailures,
|
|
81
|
+
],
|
|
66
82
|
notices: checks
|
|
67
83
|
.filter(({ status }) => status === healthyStatus)
|
|
68
84
|
.map(({ name, detail }) => `Processor ${name}: ${detail ?? 'ok'}`),
|
|
@@ -10,6 +10,7 @@ import { type createResourceLookup, type createResourceService } from '../resour
|
|
|
10
10
|
import { type createWorkService } from '../work/index.js';
|
|
11
11
|
import type { ResolvedWakeModulesConfig } from './config/load-config.js';
|
|
12
12
|
import type { EventProcessorRuntime } from './event-processor-runtime.js';
|
|
13
|
+
import { type PollCoordinator } from './poll-coordinator.js';
|
|
13
14
|
import { type RuntimeProjectionSubscriptions } from './projection-runtime.js';
|
|
14
15
|
export interface IntegrationRuntime {
|
|
15
16
|
readonly triggers: TriggerService;
|
|
@@ -53,5 +54,5 @@ export interface IntegrationRuntimeInput {
|
|
|
53
54
|
}
|
|
54
55
|
export declare function composeIntegrationRuntime(input: IntegrationRuntimeInput): Promise<IntegrationRuntime>;
|
|
55
56
|
/** Coalesces provider webhook requests while retaining the source's durable poll semantics. */
|
|
56
|
-
export declare function createImmediatePollRequester(providers: readonly ProviderInstance[], journal: EventJournal, wakeRoot: string): (adapter: string) => void;
|
|
57
|
+
export declare function createImmediatePollRequester(providers: readonly ProviderInstance[], journal: EventJournal, wakeRoot: string, coordinator?: PollCoordinator): (adapter: string) => void;
|
|
57
58
|
export declare function composeProviders(input: Pick<IntegrationRuntimeInput, 'wakeRoot' | 'config' | 'work' | 'conversations' | 'resources' | 'lookup' | 'pullRequests' | 'journal' | 'orchestration' | 'ids' | 'clock' | 'checkpoints' | 'execution' | 'providerDefinitions'>): Promise<import("../integrations/index.js").ProviderCompositionResult>;
|
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
import { EventActorKind, correlationId, } from '@atolis-hq/eventing';
|
|
2
|
-
import { withFileLock } from '@atolis-hq/eventing-filesystem';
|
|
3
2
|
import { join } from 'node:path';
|
|
4
3
|
import { z } from 'zod';
|
|
5
4
|
import { ActivityExecutionKind, activityName, } from '../activities/index.js';
|
|
6
5
|
import { ControlEventType, ControlStreamKind, TriggerService, createDurableRunnerIneligibility, createIntakePipeline, createRunnerPipeline, createTriggerReactor, createWorkCancellationPolicy, hasOpenWorkCapacity, readOpenWorkItemIds, selectControlEvent, triggerInvocationProjection, triggerInvocationStream, } from '../control-plane/index.js';
|
|
7
6
|
import { ConversationOriginKind, workItemIdForConversation } from '../conversations/index.js';
|
|
8
7
|
import { ExecutionStreamKind, NoEligibleRunnerError, RunRepository, createRuntimeMemoryProfile, decodeActivationExecutionEvent, isActiveRunStatus, isRunStream, runId, } from '../execution/index.js';
|
|
9
|
-
import { AgentRunPublicationReactor, ArtifactRegistrationReactor, DeliveryOutcomeReactor, DeliveryService, IntegrationStreamKind,
|
|
8
|
+
import { AgentRunPublicationReactor, ArtifactRegistrationReactor, DeliveryOutcomeReactor, DeliveryService, IntegrationStreamKind, ProviderRegistry, deliveryProjectionDefinitions, fakeProviderDefinition, publishConversationCommandFeedback, reconcileMergedDeliveries, } from '../integrations/index.js';
|
|
10
9
|
import { compileWorkflowSelectors, createConversationCommandReactor, createPullRequestTransitionEvidence, createResourceTransitionReactor, createWatchReactor, createWatchReconciler, isWorkflowInstanceStream, selectWorkflow, workflowInstanceId, workflowName, } from '../orchestration/index.js';
|
|
11
10
|
import { BuiltInResourceCapability, resourceId, } from '../resources/index.js';
|
|
12
11
|
import { isWorkItemStream } from '../work/index.js';
|
|
13
12
|
import { decodeCanonicalEvent } from './canonical-events.js';
|
|
14
13
|
import { hydrateFakeProviderEvidence } from './fake-provider-files.js';
|
|
14
|
+
import { createPollCoordinator } from './poll-coordinator.js';
|
|
15
15
|
import { createRuntimeProjectionSubscriptions, } from './projection-runtime.js';
|
|
16
16
|
import { createCapabilityResourceTransitionEvidence } from './resource-transition-evidence.js';
|
|
17
17
|
export async function composeIntegrationRuntime(input) {
|
|
@@ -220,13 +220,17 @@ export async function composeIntegrationRuntime(input) {
|
|
|
220
220
|
// Inbound translation reads durable provider facts through its own services,
|
|
221
221
|
// so it has no projection freshness dependency that could hold shutdown or
|
|
222
222
|
// unrelated resident work behind a blocked projection consumer.
|
|
223
|
+
const pollCoordinator = createPollCoordinator({
|
|
224
|
+
journal: input.journal,
|
|
225
|
+
wakeRoot: input.wakeRoot,
|
|
226
|
+
});
|
|
223
227
|
const intakePipeline = createIntakePipeline({
|
|
224
228
|
isPaused: input.isPaused,
|
|
225
229
|
poll: async (signal) => {
|
|
226
230
|
return observeMemory(memoryProfile, 'intake.poll', async () => {
|
|
227
231
|
let appended = 0;
|
|
228
232
|
for (const provider of providers)
|
|
229
|
-
appended += await
|
|
233
|
+
appended += (await pollCoordinator.poll(provider, signal)).appended;
|
|
230
234
|
return appended;
|
|
231
235
|
});
|
|
232
236
|
},
|
|
@@ -274,7 +278,7 @@ export async function composeIntegrationRuntime(input) {
|
|
|
274
278
|
await delivery.deliverNext(signal);
|
|
275
279
|
}),
|
|
276
280
|
});
|
|
277
|
-
const requestImmediatePoll = createImmediatePollRequester(providers, input.journal, input.wakeRoot);
|
|
281
|
+
const requestImmediatePoll = createImmediatePollRequester(providers, input.journal, input.wakeRoot, pollCoordinator);
|
|
278
282
|
return {
|
|
279
283
|
triggers,
|
|
280
284
|
projectionSubscriptions,
|
|
@@ -298,22 +302,15 @@ export async function composeIntegrationRuntime(input) {
|
|
|
298
302
|
};
|
|
299
303
|
}
|
|
300
304
|
/** Coalesces provider webhook requests while retaining the source's durable poll semantics. */
|
|
301
|
-
export function createImmediatePollRequester(providers, journal, wakeRoot) {
|
|
302
|
-
const immediatePolls = new Map();
|
|
305
|
+
export function createImmediatePollRequester(providers, journal, wakeRoot, coordinator = createPollCoordinator({ journal, wakeRoot })) {
|
|
303
306
|
return (adapter) => {
|
|
304
|
-
if (immediatePolls.has(adapter))
|
|
305
|
-
return;
|
|
306
307
|
const provider = providers.find((candidate) => candidate.adapter === adapter);
|
|
307
308
|
if (provider === undefined)
|
|
308
309
|
return;
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
});
|
|
313
|
-
})
|
|
314
|
-
.catch((error) => console.error(`Webhook polling ${adapter} failed`, error))
|
|
315
|
-
.finally(() => immediatePolls.delete(adapter));
|
|
316
|
-
immediatePolls.set(adapter, run);
|
|
310
|
+
// Joins any in-flight poll, so a webhook burst never queues behind intake.
|
|
311
|
+
coordinator
|
|
312
|
+
.poll(provider, new AbortController().signal, { bypassInterval: true })
|
|
313
|
+
.catch((error) => console.error(`Webhook polling ${adapter} failed`, error));
|
|
317
314
|
};
|
|
318
315
|
}
|
|
319
316
|
async function observeMemory(profile, phase, operation) {
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { EventJournal } from '@atolis-hq/eventing';
|
|
2
|
+
import { type ProviderInstance } from '../integrations/index.js';
|
|
3
|
+
export declare const PollOutcome: {
|
|
4
|
+
readonly Polled: "polled";
|
|
5
|
+
readonly LockContended: "lock-contended";
|
|
6
|
+
};
|
|
7
|
+
export interface PollCoordinatorOptions {
|
|
8
|
+
readonly journal: EventJournal;
|
|
9
|
+
readonly wakeRoot: string;
|
|
10
|
+
/** Wall-clock bound for one poll run; the lock is released when it elapses. */
|
|
11
|
+
readonly deadlineMs?: number;
|
|
12
|
+
/** How long to wait for another process holding the poll lock before skipping the tick. */
|
|
13
|
+
readonly lockWaitMs?: number;
|
|
14
|
+
}
|
|
15
|
+
export interface PollRunOptions {
|
|
16
|
+
readonly bypassInterval?: boolean;
|
|
17
|
+
}
|
|
18
|
+
export interface PollRunResult {
|
|
19
|
+
readonly outcome: (typeof PollOutcome)[keyof typeof PollOutcome];
|
|
20
|
+
readonly appended: number;
|
|
21
|
+
}
|
|
22
|
+
export declare const DEFAULT_POLL_DEADLINE_MS: number;
|
|
23
|
+
export declare const DEFAULT_POLL_LOCK_WAIT_MS = 5000;
|
|
24
|
+
export type PollCoordinator = ReturnType<typeof createPollCoordinator>;
|
|
25
|
+
/**
|
|
26
|
+
* Single-flight per adapter inside the process (intake and webhook polls join
|
|
27
|
+
* one another); the file lock only guards against other processes. Every run
|
|
28
|
+
* is bounded so a hung request cannot pin the lock for the life of the process.
|
|
29
|
+
*/
|
|
30
|
+
export declare function createPollCoordinator(options: PollCoordinatorOptions): {
|
|
31
|
+
/**
|
|
32
|
+
* Joins the existing run using its original signal; a joining caller cannot
|
|
33
|
+
* cancel it. Changes arriving after its reads wait for the next interval.
|
|
34
|
+
*/
|
|
35
|
+
poll(provider: ProviderInstance, signal: AbortSignal, pollOptions?: PollRunOptions): Promise<PollRunResult>;
|
|
36
|
+
};
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
import { acquireFileLock } from '@atolis-hq/eventing-filesystem';
|
|
3
|
+
import { PollService } from '../integrations/index.js';
|
|
4
|
+
export const PollOutcome = {
|
|
5
|
+
Polled: 'polled',
|
|
6
|
+
LockContended: 'lock-contended',
|
|
7
|
+
};
|
|
8
|
+
export const DEFAULT_POLL_DEADLINE_MS = 5 * 60_000;
|
|
9
|
+
export const DEFAULT_POLL_LOCK_WAIT_MS = 5_000;
|
|
10
|
+
/**
|
|
11
|
+
* Single-flight per adapter inside the process (intake and webhook polls join
|
|
12
|
+
* one another); the file lock only guards against other processes. Every run
|
|
13
|
+
* is bounded so a hung request cannot pin the lock for the life of the process.
|
|
14
|
+
*/
|
|
15
|
+
export function createPollCoordinator(options) {
|
|
16
|
+
const deadlineMs = options.deadlineMs ?? DEFAULT_POLL_DEADLINE_MS;
|
|
17
|
+
const lockWaitMs = options.lockWaitMs ?? DEFAULT_POLL_LOCK_WAIT_MS;
|
|
18
|
+
const inFlight = new Map();
|
|
19
|
+
const run = async (provider, signal, pollOptions) => {
|
|
20
|
+
const lockPath = join(options.wakeRoot, '.wake', 'locks', `poll-${provider.adapter}.lock`);
|
|
21
|
+
const lock = await acquireWithin(lockPath, lockWaitMs);
|
|
22
|
+
if (!lock.acquired) {
|
|
23
|
+
const error = new Error(`Poll lock is held by another process: ${lockPath}`);
|
|
24
|
+
provider.recordPollFailure?.(error);
|
|
25
|
+
console.warn(`Skipping ${provider.adapter} poll: ${error.message}`);
|
|
26
|
+
return { outcome: PollOutcome.LockContended, appended: 0 };
|
|
27
|
+
}
|
|
28
|
+
try {
|
|
29
|
+
const result = await runWithDeadline((boundedSignal) => new PollService(options.journal, provider).pollOnce(boundedSignal, pollOptions), signal, deadlineMs, provider.adapter);
|
|
30
|
+
return { outcome: PollOutcome.Polled, appended: result.appended };
|
|
31
|
+
}
|
|
32
|
+
catch (error) {
|
|
33
|
+
provider.recordPollFailure?.(error);
|
|
34
|
+
throw error;
|
|
35
|
+
}
|
|
36
|
+
finally {
|
|
37
|
+
await lock.release();
|
|
38
|
+
}
|
|
39
|
+
};
|
|
40
|
+
return {
|
|
41
|
+
/**
|
|
42
|
+
* Joins the existing run using its original signal; a joining caller cannot
|
|
43
|
+
* cancel it. Changes arriving after its reads wait for the next interval.
|
|
44
|
+
*/
|
|
45
|
+
poll(provider, signal, pollOptions = {}) {
|
|
46
|
+
const existing = inFlight.get(provider.adapter);
|
|
47
|
+
if (existing !== undefined)
|
|
48
|
+
return existing;
|
|
49
|
+
const started = run(provider, signal, pollOptions).finally(() => {
|
|
50
|
+
inFlight.delete(provider.adapter);
|
|
51
|
+
});
|
|
52
|
+
inFlight.set(provider.adapter, started);
|
|
53
|
+
return started;
|
|
54
|
+
},
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
async function acquireWithin(path, waitMs) {
|
|
58
|
+
const deadline = Date.now() + waitMs;
|
|
59
|
+
for (;;) {
|
|
60
|
+
const lock = await acquireFileLock(path, {
|
|
61
|
+
staleAfterMs: 60_000,
|
|
62
|
+
staleRequiresDeadProcess: true,
|
|
63
|
+
});
|
|
64
|
+
if (lock.acquired || Date.now() >= deadline)
|
|
65
|
+
return lock;
|
|
66
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
async function runWithDeadline(operation, outer, deadlineMs, adapter) {
|
|
70
|
+
const controller = new AbortController();
|
|
71
|
+
const abort = () => controller.abort(outer.reason);
|
|
72
|
+
if (outer.aborted)
|
|
73
|
+
abort();
|
|
74
|
+
else
|
|
75
|
+
outer.addEventListener('abort', abort, { once: true });
|
|
76
|
+
let timer;
|
|
77
|
+
const deadline = new Promise((_, reject) => {
|
|
78
|
+
timer = setTimeout(() => {
|
|
79
|
+
const error = new Error(`Polling ${adapter} exceeded its ${deadlineMs}ms deadline`);
|
|
80
|
+
// Abort first so cooperative work stops, but reject regardless: release
|
|
81
|
+
// of the lock must not depend on the hung operation honouring the signal.
|
|
82
|
+
controller.abort(error);
|
|
83
|
+
reject(error);
|
|
84
|
+
}, deadlineMs);
|
|
85
|
+
});
|
|
86
|
+
try {
|
|
87
|
+
return await Promise.race([operation(controller.signal), deadline]);
|
|
88
|
+
}
|
|
89
|
+
finally {
|
|
90
|
+
clearTimeout(timer);
|
|
91
|
+
outer.removeEventListener('abort', abort);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
@@ -124,6 +124,6 @@ export function resolveWakeVersion(options = {}) {
|
|
|
124
124
|
return `g${headHash.slice(0, 7)}`;
|
|
125
125
|
return '0.1.0-dev';
|
|
126
126
|
}
|
|
127
|
-
export const wakeVersion = "v0.4.
|
|
127
|
+
export const wakeVersion = "v0.4.96";
|
|
128
128
|
/** The registry-installable version used by packaged sandbox images. */
|
|
129
129
|
export const wakePackageVersion = packageVersion(defaultRepoRoot) ?? wakeVersion;
|
|
@@ -1,11 +1,13 @@
|
|
|
1
|
+
import { createInProcessSerialiser } from './in-process-scheduler-serialiser.js';
|
|
2
|
+
import { isRunnerQuotaOutcome, reportUnappliedRunnerQuotaRetry, runnerQuotaMessage, } from './runner-quota-outcome.js';
|
|
1
3
|
/* eslint-disable complexity, max-lines-per-function */
|
|
2
4
|
import { correlationId, EventActorKind } from '@atolis-hq/eventing';
|
|
3
5
|
import { ExecutionFailureCode, RunStatus } from '../../execution/index.js';
|
|
4
|
-
import { isAmbiguityResolutionBlock, WorkflowStatus, } from '../../orchestration/index.js';
|
|
6
|
+
import { DelegationPendingError, isAmbiguityResolutionBlock, WorkflowStatus, } from '../../orchestration/index.js';
|
|
5
7
|
import { WorkStatus } from '../../work/index.js';
|
|
6
8
|
import { ControlStreamKind } from '../contracts/streams.js';
|
|
7
9
|
import { DispatchPolicy } from '../domain/dispatch-policy.js';
|
|
8
|
-
import { advancementWasDurablyApplied, blockIsolatedWorkflow,
|
|
10
|
+
import { advancementWasDurablyApplied, blockIsolatedWorkflow, runDispatchLoop, } from './advance-once-dispatch.js';
|
|
9
11
|
import { findUnresolvedSucceededTerminal, findUnresolvedTerminal, } from './execution-reconciliation.js';
|
|
10
12
|
export function createActivationScheduler(orchestration, execution, resources, clock, dependencies) {
|
|
11
13
|
const runnerIneligibility = dependencies.runnerIneligibility ?? (async () => new Set());
|
|
@@ -21,6 +23,19 @@ export function createActivationScheduler(orchestration, execution, resources, c
|
|
|
21
23
|
occurredAt: clock.now().toISOString(),
|
|
22
24
|
actor: { kind: EventActorKind.System, id: ControlStreamKind.Global },
|
|
23
25
|
});
|
|
26
|
+
const dispatchContext = {
|
|
27
|
+
orchestration,
|
|
28
|
+
execution,
|
|
29
|
+
resources,
|
|
30
|
+
dispatchPolicy,
|
|
31
|
+
maxConcurrentRuns,
|
|
32
|
+
maxOpenWorkItems: dependencies.maxOpenWorkItems ?? null,
|
|
33
|
+
work: dependencies.work,
|
|
34
|
+
maxDispatches,
|
|
35
|
+
runnerIneligibility,
|
|
36
|
+
isDispatchPaused,
|
|
37
|
+
commandContext: context,
|
|
38
|
+
};
|
|
24
39
|
const maintain = async () => {
|
|
25
40
|
if (await isDispatchPaused())
|
|
26
41
|
return { kind: 'paused' };
|
|
@@ -161,22 +176,19 @@ export function createActivationScheduler(orchestration, execution, resources, c
|
|
|
161
176
|
};
|
|
162
177
|
}
|
|
163
178
|
catch (error) {
|
|
179
|
+
if (error instanceof DelegationPendingError) {
|
|
180
|
+
const result = await runDispatchLoop(pending.filter((item) => item.workflow.workflowInstanceId !== recovery.item.workflow.workflowInstanceId), dispatchContext);
|
|
181
|
+
return result.kind === 'no-work' || result.kind === WorkflowStatus.Waiting
|
|
182
|
+
? {
|
|
183
|
+
kind: WorkflowStatus.Waiting,
|
|
184
|
+
workflowInstanceId: recovery.item.workflow.workflowInstanceId,
|
|
185
|
+
}
|
|
186
|
+
: result;
|
|
187
|
+
}
|
|
164
188
|
if (await advancementWasDurablyApplied(orchestration, recovery.item.workflow.workflowInstanceId, recovery.item.activation.activationId))
|
|
165
189
|
throw error;
|
|
166
190
|
const reason = await blockIsolatedWorkflow(orchestration, recovery.item.workflow.workflowInstanceId, error, context(recovery.run.runId));
|
|
167
|
-
const result = await runDispatchLoop(pending.filter((item) => item.workflow.workflowInstanceId !== recovery.item.workflow.workflowInstanceId),
|
|
168
|
-
orchestration,
|
|
169
|
-
execution,
|
|
170
|
-
resources,
|
|
171
|
-
dispatchPolicy,
|
|
172
|
-
maxConcurrentRuns,
|
|
173
|
-
maxOpenWorkItems: dependencies.maxOpenWorkItems ?? null,
|
|
174
|
-
work: dependencies.work,
|
|
175
|
-
maxDispatches,
|
|
176
|
-
runnerIneligibility,
|
|
177
|
-
isDispatchPaused,
|
|
178
|
-
commandContext: context,
|
|
179
|
-
});
|
|
191
|
+
const result = await runDispatchLoop(pending.filter((item) => item.workflow.workflowInstanceId !== recovery.item.workflow.workflowInstanceId), dispatchContext);
|
|
180
192
|
return result.kind === 'progressed'
|
|
181
193
|
? result
|
|
182
194
|
: {
|
|
@@ -186,19 +198,7 @@ export function createActivationScheduler(orchestration, execution, resources, c
|
|
|
186
198
|
};
|
|
187
199
|
}
|
|
188
200
|
}
|
|
189
|
-
const result = await runDispatchLoop(pending,
|
|
190
|
-
orchestration,
|
|
191
|
-
execution,
|
|
192
|
-
resources,
|
|
193
|
-
dispatchPolicy,
|
|
194
|
-
maxConcurrentRuns,
|
|
195
|
-
maxOpenWorkItems: dependencies.maxOpenWorkItems ?? null,
|
|
196
|
-
work: dependencies.work,
|
|
197
|
-
maxDispatches,
|
|
198
|
-
runnerIneligibility,
|
|
199
|
-
isDispatchPaused,
|
|
200
|
-
commandContext: context,
|
|
201
|
-
});
|
|
201
|
+
const result = await runDispatchLoop(pending, dispatchContext);
|
|
202
202
|
if (expired.length === 0 ||
|
|
203
203
|
(result.kind !== 'no-work' && result.kind !== WorkflowStatus.Waiting))
|
|
204
204
|
return result;
|
|
@@ -221,18 +221,3 @@ export function createActivationScheduler(orchestration, execution, resources, c
|
|
|
221
221
|
dispatch: (options, signal) => serialise(() => dispatch(options), signal),
|
|
222
222
|
};
|
|
223
223
|
}
|
|
224
|
-
function createInProcessSerialiser() {
|
|
225
|
-
let prior = Promise.resolve();
|
|
226
|
-
return async (operation, signal) => {
|
|
227
|
-
const current = prior.then(async () => {
|
|
228
|
-
throwIfAborted(signal);
|
|
229
|
-
return operation();
|
|
230
|
-
});
|
|
231
|
-
prior = current.then(() => undefined, () => undefined);
|
|
232
|
-
return current;
|
|
233
|
-
};
|
|
234
|
-
}
|
|
235
|
-
function throwIfAborted(signal) {
|
|
236
|
-
if (signal?.aborted)
|
|
237
|
-
throw signal.reason ?? new Error('Activation scheduler run aborted');
|
|
238
|
-
}
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import type { CommandContext } from '@atolis-hq/eventing';
|
|
2
|
-
import { type ActivityOutcome } from '../../activities/index.js';
|
|
3
2
|
import type { ActivityActivationView, PendingActivationPriority, WorkflowInstanceView } from '../../orchestration/index.js';
|
|
4
3
|
import type { ResourceService } from '../../resources/index.js';
|
|
5
4
|
import type { AdvanceResult } from '../contracts/views.js';
|
|
@@ -29,18 +28,6 @@ interface PendingActivation {
|
|
|
29
28
|
export declare function advancementWasDurablyApplied(orchestration: OrchestrationPort, workflowInstanceId: WorkflowInstanceView['workflowInstanceId'], activationId: ActivityActivationView['activationId']): Promise<boolean>;
|
|
30
29
|
export declare function isolatedWorkflowReason(error: unknown): string;
|
|
31
30
|
export declare function blockIsolatedWorkflow(orchestration: OrchestrationPort, workflowInstanceId: string, error: unknown, context: CommandContext): Promise<string>;
|
|
32
|
-
export declare function isRunnerQuotaOutcome(outcome: ActivityOutcome): boolean;
|
|
33
|
-
export declare function runnerQuotaMessage(outcome: ActivityOutcome): string;
|
|
34
|
-
/**
|
|
35
|
-
* A quota retry deliberately leaves the outcome unaccepted, so a port that
|
|
36
|
-
* never applied it (absent method, or a workflow instance that vanished)
|
|
37
|
-
* leaves the Run permanently unresolved. Surface that rather than going
|
|
38
|
-
* silent; it is an operational defect, not a workflow state.
|
|
39
|
-
*/
|
|
40
|
-
export declare function reportUnappliedRunnerQuotaRetry(result: unknown, input: {
|
|
41
|
-
readonly activationId: string;
|
|
42
|
-
readonly runId: string;
|
|
43
|
-
}): void;
|
|
44
31
|
/**
|
|
45
32
|
* Fills open capacity within one Advancement call: dispatches ready
|
|
46
33
|
* activations one at a time, rechecking `maxConcurrentRuns` and reselecting
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { isRunnerQuotaOutcome, reportUnappliedRunnerQuotaRetry, runnerQuotaMessage, } from './runner-quota-outcome.js';
|
|
2
2
|
import { ActivationClaimConflictError, isActiveRunStatus, NoEligibleRunnerError, RunStatus, WorkspaceMode, } from '../../execution/index.js';
|
|
3
|
-
import { WorkflowStatus } from '../../orchestration/index.js';
|
|
3
|
+
import { DelegationPendingError, WorkflowStatus } from '../../orchestration/index.js';
|
|
4
4
|
import { canDispatchWorkItem, readOpenWorkItemIds, } from '../domain/open-work-policy.js';
|
|
5
5
|
import { isExecutionFailureTerminal } from './execution-reconciliation.js';
|
|
6
6
|
// Ports without priority (older fakes) rank neutrally and fall back to oldest-first.
|
|
@@ -27,28 +27,6 @@ export async function blockIsolatedWorkflow(orchestration, workflowInstanceId, e
|
|
|
27
27
|
await orchestration.block(workflowInstanceId, reason, context);
|
|
28
28
|
return reason;
|
|
29
29
|
}
|
|
30
|
-
export function isRunnerQuotaOutcome(outcome) {
|
|
31
|
-
return (outcome.kind === ActivityOutcomeKind.Failed &&
|
|
32
|
-
typeof outcome.data === 'object' &&
|
|
33
|
-
outcome.data !== null &&
|
|
34
|
-
'reason' in outcome.data &&
|
|
35
|
-
outcome.data.reason === ActivityFailureCode.RunnerQuotaExceeded);
|
|
36
|
-
}
|
|
37
|
-
export function runnerQuotaMessage(outcome) {
|
|
38
|
-
const data = outcome.data;
|
|
39
|
-
return typeof data.message === 'string' ? data.message : 'runner reported quota exhaustion';
|
|
40
|
-
}
|
|
41
|
-
/**
|
|
42
|
-
* A quota retry deliberately leaves the outcome unaccepted, so a port that
|
|
43
|
-
* never applied it (absent method, or a workflow instance that vanished)
|
|
44
|
-
* leaves the Run permanently unresolved. Surface that rather than going
|
|
45
|
-
* silent; it is an operational defect, not a workflow state.
|
|
46
|
-
*/
|
|
47
|
-
export function reportUnappliedRunnerQuotaRetry(result, input) {
|
|
48
|
-
if (result !== null && result !== undefined)
|
|
49
|
-
return;
|
|
50
|
-
console.error(`Runner-quota retry was not applied for activation ${input.activationId} (run ${input.runId}); its outcome remains unaccepted.`);
|
|
51
|
-
}
|
|
52
30
|
/**
|
|
53
31
|
* Fills open capacity within one Advancement call: dispatches ready
|
|
54
32
|
* activations one at a time, rechecking `maxConcurrentRuns` and reselecting
|
|
@@ -181,6 +159,14 @@ export async function runDispatchLoop(pending, ctx) {
|
|
|
181
159
|
dispatchedIds.add(selected.activation.activationId);
|
|
182
160
|
}
|
|
183
161
|
catch (error) {
|
|
162
|
+
if (error instanceof DelegationPendingError) {
|
|
163
|
+
isolatedIds.add(selected.workflow.workflowInstanceId);
|
|
164
|
+
stopReason = {
|
|
165
|
+
kind: WorkflowStatus.Waiting,
|
|
166
|
+
workflowInstanceId: selected.workflow.workflowInstanceId,
|
|
167
|
+
};
|
|
168
|
+
continue;
|
|
169
|
+
}
|
|
184
170
|
if (error instanceof ActivationClaimConflictError) {
|
|
185
171
|
stopReason = { kind: 'no-work' };
|
|
186
172
|
break;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function createInProcessSerialiser(): <Value>(operation: () => Promise<Value>, signal?: AbortSignal) => Promise<Value>;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export function createInProcessSerialiser() {
|
|
2
|
+
let prior = Promise.resolve();
|
|
3
|
+
return async (operation, signal) => {
|
|
4
|
+
const current = prior.then(async () => {
|
|
5
|
+
throwIfAborted(signal);
|
|
6
|
+
return operation();
|
|
7
|
+
});
|
|
8
|
+
prior = current.then(() => undefined, () => undefined);
|
|
9
|
+
return current;
|
|
10
|
+
};
|
|
11
|
+
}
|
|
12
|
+
function throwIfAborted(signal) {
|
|
13
|
+
if (signal?.aborted)
|
|
14
|
+
throw signal.reason ?? new Error('Activation scheduler run aborted');
|
|
15
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { type ActivityOutcome } from '../../activities/index.js';
|
|
2
|
+
export declare function isRunnerQuotaOutcome(outcome: ActivityOutcome): boolean;
|
|
3
|
+
export declare function runnerQuotaMessage(outcome: ActivityOutcome): string;
|
|
4
|
+
/**
|
|
5
|
+
* A quota retry deliberately leaves the outcome unaccepted, so a port that
|
|
6
|
+
* never applied it (absent method, or a workflow instance that vanished)
|
|
7
|
+
* leaves the Run permanently unresolved. Surface that rather than going
|
|
8
|
+
* silent; it is an operational defect, not a workflow state.
|
|
9
|
+
*/
|
|
10
|
+
export declare function reportUnappliedRunnerQuotaRetry(result: unknown, input: {
|
|
11
|
+
readonly activationId: string;
|
|
12
|
+
readonly runId: string;
|
|
13
|
+
}): void;
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { ActivityFailureCode, ActivityOutcomeKind, } from '../../activities/index.js';
|
|
2
|
+
export function isRunnerQuotaOutcome(outcome) {
|
|
3
|
+
return (outcome.kind === ActivityOutcomeKind.Failed &&
|
|
4
|
+
typeof outcome.data === 'object' &&
|
|
5
|
+
outcome.data !== null &&
|
|
6
|
+
'reason' in outcome.data &&
|
|
7
|
+
outcome.data.reason === ActivityFailureCode.RunnerQuotaExceeded);
|
|
8
|
+
}
|
|
9
|
+
export function runnerQuotaMessage(outcome) {
|
|
10
|
+
const data = outcome.data;
|
|
11
|
+
return typeof data.message === 'string' ? data.message : 'runner reported quota exhaustion';
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* A quota retry deliberately leaves the outcome unaccepted, so a port that
|
|
15
|
+
* never applied it (absent method, or a workflow instance that vanished)
|
|
16
|
+
* leaves the Run permanently unresolved. Surface that rather than going
|
|
17
|
+
* silent; it is an operational defect, not a workflow state.
|
|
18
|
+
*/
|
|
19
|
+
export function reportUnappliedRunnerQuotaRetry(result, input) {
|
|
20
|
+
if (result !== null && result !== undefined)
|
|
21
|
+
return;
|
|
22
|
+
console.error(`Runner-quota retry was not applied for activation ${input.activationId} (run ${input.runId}); its outcome remains unaccepted.`);
|
|
23
|
+
}
|
|
@@ -3,6 +3,8 @@ export interface AdapterHealth {
|
|
|
3
3
|
readonly detail?: string;
|
|
4
4
|
readonly successCount: number;
|
|
5
5
|
readonly failureCount: number;
|
|
6
|
+
/** Epoch milliseconds of the latest success; absent until one happens. */
|
|
7
|
+
readonly lastSuccessAt?: number;
|
|
6
8
|
}
|
|
7
9
|
export interface AdapterHealthTracker {
|
|
8
10
|
recordSuccess(): void;
|