@atolis-hq/wake 0.4.92 → 0.4.93
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/bootstrap/doctor-processor-health.d.ts +6 -0
- package/dist/src/bootstrap/doctor-processor-health.js +74 -0
- package/dist/src/bootstrap/event-processor-runtime.d.ts +7 -1
- package/dist/src/bootstrap/event-processor-runtime.js +40 -7
- package/dist/src/bootstrap/surface-api-applications.js +2 -1
- package/dist/src/bootstrap/surface-cli-applications.js +10 -3
- package/dist/src/bootstrap/version.js +1 -1
- package/dist/src/node_modules/@atolis-hq/eventing/dist/index.d.ts +1 -0
- package/dist/src/node_modules/@atolis-hq/eventing/dist/index.d.ts.map +1 -1
- package/dist/src/node_modules/@atolis-hq/eventing/dist/index.js +1 -0
- package/dist/src/node_modules/@atolis-hq/eventing/dist/runtime/event-processor-host.d.ts +5 -0
- package/dist/src/node_modules/@atolis-hq/eventing/dist/runtime/event-processor-host.d.ts.map +1 -1
- package/dist/src/node_modules/@atolis-hq/eventing/dist/runtime/event-processor-host.js +57 -8
- package/dist/src/node_modules/@atolis-hq/eventing/dist/runtime/processor-failure.d.ts +10 -0
- package/dist/src/node_modules/@atolis-hq/eventing/dist/runtime/processor-failure.d.ts.map +1 -0
- package/dist/src/node_modules/@atolis-hq/eventing/dist/runtime/processor-failure.js +15 -0
- package/dist/src/node_modules/@atolis-hq/eventing/dist/runtime/processor-health.d.ts.map +1 -1
- package/dist/src/node_modules/@atolis-hq/eventing/dist/runtime/processor-health.js +7 -0
- package/dist/src/surfaces/api/contracts/system.d.ts +6 -0
- package/dist/src/surfaces/api/contracts/system.js +4 -0
- package/dist/src/surfaces/auth/credentials.d.ts +2 -0
- package/dist/src/surfaces/auth/credentials.js +4 -0
- package/package.json +1 -1
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { type SurfacesConfig } from '../surfaces/index.js';
|
|
2
|
+
/** Read live, volatile health without creating credentials or starting processors. */
|
|
3
|
+
export declare function doctorProcessorHealth(wakeRoot: string, config: SurfacesConfig): Promise<{
|
|
4
|
+
failures: string[];
|
|
5
|
+
notices: string[];
|
|
6
|
+
}>;
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { ApiHealthCheckKind, loadCredentials, } from '../surfaces/index.js';
|
|
3
|
+
// Any status other than the healthy one is reported as a failure.
|
|
4
|
+
const healthyStatus = 'ok';
|
|
5
|
+
const healthSchema = z.object({
|
|
6
|
+
data: z.object({
|
|
7
|
+
checks: z.array(z.object({
|
|
8
|
+
name: z.string(),
|
|
9
|
+
status: z.string(),
|
|
10
|
+
kind: z.string().optional(),
|
|
11
|
+
detail: z.string().optional(),
|
|
12
|
+
})),
|
|
13
|
+
}),
|
|
14
|
+
});
|
|
15
|
+
function unavailable(reason) {
|
|
16
|
+
return { failures: [], notices: [`Processor health unavailable: ${reason}.`] };
|
|
17
|
+
}
|
|
18
|
+
/** Read live, volatile health without creating credentials or starting processors. */
|
|
19
|
+
export async function doctorProcessorHealth(wakeRoot, config) {
|
|
20
|
+
if (!config.api.enabled)
|
|
21
|
+
return unavailable('the configured API is disabled');
|
|
22
|
+
try {
|
|
23
|
+
const host = ['0.0.0.0', '::'].includes(config.api.host) ? '127.0.0.1' : config.api.host;
|
|
24
|
+
const base = new URL(`http://${host.includes(':') ? `[${host}]` : host}:${config.api.port}`);
|
|
25
|
+
const signal = AbortSignal.timeout(2_000);
|
|
26
|
+
let cookie = '';
|
|
27
|
+
if (!config.web.auth.disabled) {
|
|
28
|
+
let accessKey;
|
|
29
|
+
try {
|
|
30
|
+
accessKey = (await loadCredentials(wakeRoot)).accessKey;
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
return unavailable('no readable API credentials');
|
|
34
|
+
}
|
|
35
|
+
const login = await fetch(new URL('/api/v1/auth/login', base), {
|
|
36
|
+
method: 'POST',
|
|
37
|
+
redirect: 'error',
|
|
38
|
+
signal,
|
|
39
|
+
headers: { 'content-type': 'application/json' },
|
|
40
|
+
body: JSON.stringify({ accessKey }),
|
|
41
|
+
});
|
|
42
|
+
if (!login.ok)
|
|
43
|
+
return unavailable(`API authentication failed (HTTP ${login.status})`);
|
|
44
|
+
cookie = login.headers
|
|
45
|
+
.getSetCookie()
|
|
46
|
+
.map((value) => value.split(';')[0])
|
|
47
|
+
.join('; ');
|
|
48
|
+
}
|
|
49
|
+
const response = await fetch(new URL('/api/v1/system/health', base), {
|
|
50
|
+
redirect: 'error',
|
|
51
|
+
signal,
|
|
52
|
+
headers: { cookie },
|
|
53
|
+
});
|
|
54
|
+
if (!response.ok)
|
|
55
|
+
return unavailable(`health request failed (HTTP ${response.status})`);
|
|
56
|
+
const parsed = healthSchema.safeParse(await response.json());
|
|
57
|
+
if (!parsed.success)
|
|
58
|
+
return unavailable('the API returned an unrecognised health response');
|
|
59
|
+
const checks = parsed.data.data.checks.filter(({ kind }) => kind === ApiHealthCheckKind.Processor);
|
|
60
|
+
if (checks.length === 0)
|
|
61
|
+
return unavailable('the API reported no processors');
|
|
62
|
+
return {
|
|
63
|
+
failures: checks
|
|
64
|
+
.filter(({ status }) => status !== healthyStatus)
|
|
65
|
+
.map(({ name, detail }) => `Processor ${name}: ${detail ?? 'degraded'}`),
|
|
66
|
+
notices: checks
|
|
67
|
+
.filter(({ status }) => status === healthyStatus)
|
|
68
|
+
.map(({ name, detail }) => `Processor ${name}: ${detail ?? 'ok'}`),
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
catch {
|
|
72
|
+
return unavailable('the configured API is unreachable');
|
|
73
|
+
}
|
|
74
|
+
}
|
|
@@ -1,5 +1,11 @@
|
|
|
1
|
-
import type { CheckpointStore, EventJournal } from '@atolis-hq/eventing';
|
|
1
|
+
import type { CheckpointStore, EventJournal, EventProcessorFailure } from '@atolis-hq/eventing';
|
|
2
2
|
import { type EventProcessor, type EventProcessorHealth, type EventProcessorHostOptions, type EventProcessorHostRun, type EventingClock, type ProcessorRunSerialiser } from '@atolis-hq/eventing';
|
|
3
|
+
/**
|
|
4
|
+
* Logs the full stack when a processor first fails or its failure changes;
|
|
5
|
+
* identical repeats log one short line only at each power-of-ten count so a
|
|
6
|
+
* poison event retried on backoff cannot flood the log.
|
|
7
|
+
*/
|
|
8
|
+
export declare function createProcessorFailureLogger(write: (line: string) => void): (error: EventProcessorFailure) => void;
|
|
3
9
|
/**
|
|
4
10
|
* Bootstrap's single registry for every deployed event processor. It owns
|
|
5
11
|
* runtime supervision only; processor definitions remain with their modules.
|
|
@@ -1,4 +1,24 @@
|
|
|
1
1
|
import { EventProcessorHealthStatus, EventProcessorHost, } from '@atolis-hq/eventing';
|
|
2
|
+
import { inspect } from 'node:util';
|
|
3
|
+
/**
|
|
4
|
+
* Logs the full stack when a processor first fails or its failure changes;
|
|
5
|
+
* identical repeats log one short line only at each power-of-ten count so a
|
|
6
|
+
* poison event retried on backoff cannot flood the log.
|
|
7
|
+
*/
|
|
8
|
+
export function createProcessorFailureLogger(write) {
|
|
9
|
+
const seen = new Map();
|
|
10
|
+
return (error) => {
|
|
11
|
+
const previous = seen.get(error.consumer);
|
|
12
|
+
if (previous?.message !== error.message) {
|
|
13
|
+
seen.set(error.consumer, { message: error.message, count: 1 });
|
|
14
|
+
write(`${inspect(error, { depth: 8 })}\n`);
|
|
15
|
+
return;
|
|
16
|
+
}
|
|
17
|
+
previous.count += 1;
|
|
18
|
+
if (Number.isInteger(Math.log10(previous.count)))
|
|
19
|
+
write(`${error.message} (repeated ${previous.count} times; stack logged on first failure)\n`);
|
|
20
|
+
};
|
|
21
|
+
}
|
|
2
22
|
/**
|
|
3
23
|
* Bootstrap's single registry for every deployed event processor. It owns
|
|
4
24
|
* runtime supervision only; processor definitions remain with their modules.
|
|
@@ -13,7 +33,10 @@ export class EventProcessorRuntime {
|
|
|
13
33
|
constructor(journal, checkpoints, serialiseRun, clock, options = {}) {
|
|
14
34
|
this.journal = journal;
|
|
15
35
|
this.checkpoints = checkpoints;
|
|
16
|
-
this.host = new EventProcessorHost(journal, checkpoints, serialiseRun, clock,
|
|
36
|
+
this.host = new EventProcessorHost(journal, checkpoints, serialiseRun, clock, {
|
|
37
|
+
onFailure: createProcessorFailureLogger((line) => process.stderr.write(line)),
|
|
38
|
+
...options,
|
|
39
|
+
});
|
|
17
40
|
}
|
|
18
41
|
get processors() {
|
|
19
42
|
return [...this.registrations.values()];
|
|
@@ -54,12 +77,22 @@ export class EventProcessorRuntime {
|
|
|
54
77
|
throw new Error('Event processor barrier name is required');
|
|
55
78
|
for (const processor of processors)
|
|
56
79
|
this.assertRegistered(processor);
|
|
57
|
-
const
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
80
|
+
const results = await Promise.allSettled(processors.map(async (processor) => {
|
|
81
|
+
try {
|
|
82
|
+
const checkpoint = await this.checkpoints.load(processor.consumer);
|
|
83
|
+
if (checkpoint >= targetGlobalPosition &&
|
|
84
|
+
this.host.health(processor.consumer)?.status !== EventProcessorHealthStatus.Degraded)
|
|
85
|
+
return { checkpoint, eventCount: 0, handledCount: 0 };
|
|
86
|
+
}
|
|
87
|
+
catch {
|
|
88
|
+
// Let the host attribute and supervise storage failures.
|
|
89
|
+
}
|
|
90
|
+
return this.host.runThrough(processor, targetGlobalPosition, signal);
|
|
91
|
+
}));
|
|
92
|
+
const failures = results.flatMap((result) => result.status !== 'fulfilled' ? [result.reason] : []);
|
|
93
|
+
if (failures.length > 0)
|
|
94
|
+
throw new AggregateError(failures, `${name}: ${failures.map((error) => (error instanceof Error ? error.message : String(error))).join('; ')}`);
|
|
95
|
+
return results.reduce((total, result) => total + (result.status === 'fulfilled' ? result.value.eventCount : 0), 0);
|
|
63
96
|
}
|
|
64
97
|
async health() {
|
|
65
98
|
const head = await this.journal.latestGlobalPosition();
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { ActivationSchedulerSubscriptionStatus, ControlStreamKind, } from '../control-plane/index.js';
|
|
2
|
-
import { ApiCommandStatus, fromWorkItemKey, presentBoardCard, presentResource, presentStatus, presentWorkflowInstance, redactConfiguration, } from '../surfaces/index.js';
|
|
2
|
+
import { ApiCommandStatus, ApiHealthCheckKind, fromWorkItemKey, presentBoardCard, presentResource, presentStatus, presentWorkflowInstance, redactConfiguration, } from '../surfaces/index.js';
|
|
3
3
|
import { analyticsProjection } from './analytics-projection.js';
|
|
4
4
|
import { createAnalyticsReports } from './analytics-reports.js';
|
|
5
5
|
import { boardConditionCounts, boardProjection, normalizeBoardActiveRunPhase, } from './board-projection.js';
|
|
@@ -312,6 +312,7 @@ function subscriptionHealthCheck(name, snapshot) {
|
|
|
312
312
|
const consecutiveFailures = snapshot?.consecutiveFailures ?? 0;
|
|
313
313
|
return {
|
|
314
314
|
name,
|
|
315
|
+
kind: ApiHealthCheckKind.Processor,
|
|
315
316
|
status: status === ActivationSchedulerSubscriptionStatus.Healthy
|
|
316
317
|
? 'ok'
|
|
317
318
|
: 'degraded',
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { EventActorKind, JOURNAL_CHANGE_FALLBACK_MS, correlationId } from '@atolis-hq/eventing';
|
|
1
|
+
import { EventActorKind, EventProcessorFailure, JOURNAL_CHANGE_FALLBACK_MS, correlationId, } from '@atolis-hq/eventing';
|
|
2
2
|
import { withFileLock } from '@atolis-hq/eventing-filesystem';
|
|
3
3
|
import { execFile as nodeExecFile, spawn } from 'node:child_process';
|
|
4
4
|
import { access, copyFile, mkdir, writeFile as writeFileContent } from 'node:fs/promises';
|
|
5
5
|
import { join } from 'node:path';
|
|
6
6
|
import { createInterface } from 'node:readline/promises';
|
|
7
|
-
import { promisify } from 'node:util';
|
|
7
|
+
import { inspect, promisify } from 'node:util';
|
|
8
8
|
import { BuiltInActivityName, agentActivityDefinition } from '../activities/index.js';
|
|
9
9
|
import { FileArtifactMcpSessionStore } from '../artifacts/index.js';
|
|
10
10
|
import { IntakeHost, ResidentHost, TickHost, } from '../control-plane/index.js';
|
|
@@ -16,6 +16,7 @@ import { WorkStreamKind, workItemId } from '../work/index.js';
|
|
|
16
16
|
import { createArtifactMcpServerForSession } from './artifact-mcp.js';
|
|
17
17
|
import { createDiagnosticRoot } from './composition-root.js';
|
|
18
18
|
import { loadConfig } from './config/load-config.js';
|
|
19
|
+
import { doctorProcessorHealth } from './doctor-processor-health.js';
|
|
19
20
|
import { createNpmUpdatePort } from './npm-update-port.js';
|
|
20
21
|
import { resolveWakePaths } from './paths.js';
|
|
21
22
|
import { composePersistence } from './persistence-composition.js';
|
|
@@ -70,7 +71,10 @@ export function createSurfaceCliApplications(root, api, now) {
|
|
|
70
71
|
const intakeHost = new IntakeHost((signal) => root.intakePipeline.run(signal));
|
|
71
72
|
const runnerIdleWait = createRunnerIdleWait(root, root.config.controlPlane?.resident);
|
|
72
73
|
const reportResidentError = (label) => async (error) => {
|
|
73
|
-
|
|
74
|
+
// Processor failures were already logged with their stack by the runtime.
|
|
75
|
+
const attributed = error instanceof AggregateError &&
|
|
76
|
+
error.errors.every((cause) => cause instanceof EventProcessorFailure);
|
|
77
|
+
process.stderr.write(`Wake ${label} tick failed: ${attributed ? error.message : inspect(error, { depth: 8 })}\n`);
|
|
74
78
|
};
|
|
75
79
|
// Intake owns provider polling, so only its resident loop backs off
|
|
76
80
|
// exponentially when idle. The runner loop owns triggers, reconciliation,
|
|
@@ -738,6 +742,9 @@ async function doctorDiagnostics(root) {
|
|
|
738
742
|
failures.push(name + ' runtime directory is not accessible: ' + path);
|
|
739
743
|
}
|
|
740
744
|
}
|
|
745
|
+
const processors = await doctorProcessorHealth(root.paths.wakeRoot, root.config.surfaces);
|
|
746
|
+
failures.push(...processors.failures);
|
|
747
|
+
notices.push(...processors.notices);
|
|
741
748
|
return { failures, notices };
|
|
742
749
|
}
|
|
743
750
|
// Derives the prompt template names doctor must validate from what configured
|
|
@@ -124,6 +124,6 @@ export function resolveWakeVersion(options = {}) {
|
|
|
124
124
|
return `g${headHash.slice(0, 7)}`;
|
|
125
125
|
return '0.1.0-dev';
|
|
126
126
|
}
|
|
127
|
-
export const wakeVersion = "v0.4.
|
|
127
|
+
export const wakeVersion = "v0.4.93";
|
|
128
128
|
/** The registry-installable version used by packaged sandbox images. */
|
|
129
129
|
export const wakePackageVersion = packageVersion(defaultRepoRoot) ?? wakeVersion;
|
|
@@ -10,6 +10,7 @@ export * from './projections/projection-store.js';
|
|
|
10
10
|
export * from './runtime/cached-journal-view.js';
|
|
11
11
|
export * from './runtime/clock.js';
|
|
12
12
|
export * from './runtime/event-processor-host.js';
|
|
13
|
+
export * from './runtime/processor-failure.js';
|
|
13
14
|
export * from './runtime/processor-health.js';
|
|
14
15
|
export * from './runtime/processor-run-serialiser.js';
|
|
15
16
|
export * from './store/checkpoint-store.js';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,yBAAyB,CAAC;AAExC,cAAc,+BAA+B,CAAC;AAE9C,cAAc,6BAA6B,CAAC;AAE5C,cAAc,uBAAuB,CAAC;AAEtC,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAEjF,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAEtF,cAAc,sCAAsC,CAAC;AAErD,cAAc,uCAAuC,CAAC;AAEtD,cAAc,mCAAmC,CAAC;AAElD,cAAc,kCAAkC,CAAC;AAEjD,cAAc,oBAAoB,CAAC;AAEnC,cAAc,mCAAmC,CAAC;AAElD,cAAc,+BAA+B,CAAC;AAE9C,cAAc,uCAAuC,CAAC;AAEtD,cAAc,6BAA6B,CAAC;AAE5C,cAAc,0BAA0B,CAAC;AAEzC,cAAc,oCAAoC,CAAC;AAEnD,cAAc,0CAA0C,CAAC;AAEzD,cAAc,yBAAyB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,yBAAyB,CAAC;AAExC,cAAc,+BAA+B,CAAC;AAE9C,cAAc,6BAA6B,CAAC;AAE5C,cAAc,uBAAuB,CAAC;AAEtC,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAEjF,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAEtF,cAAc,sCAAsC,CAAC;AAErD,cAAc,uCAAuC,CAAC;AAEtD,cAAc,mCAAmC,CAAC;AAElD,cAAc,kCAAkC,CAAC;AAEjD,cAAc,oBAAoB,CAAC;AAEnC,cAAc,mCAAmC,CAAC;AAElD,cAAc,gCAAgC,CAAC;AAE/C,cAAc,+BAA+B,CAAC;AAE9C,cAAc,uCAAuC,CAAC;AAEtD,cAAc,6BAA6B,CAAC;AAE5C,cAAc,0BAA0B,CAAC;AAEzC,cAAc,oCAAoC,CAAC;AAEnD,cAAc,0CAA0C,CAAC;AAEzD,cAAc,yBAAyB,CAAC"}
|
|
@@ -9,6 +9,7 @@ export * from './projections/projection-store.js';
|
|
|
9
9
|
export * from './runtime/cached-journal-view.js';
|
|
10
10
|
export * from './runtime/clock.js';
|
|
11
11
|
export * from './runtime/event-processor-host.js';
|
|
12
|
+
export * from './runtime/processor-failure.js';
|
|
12
13
|
export * from './runtime/processor-health.js';
|
|
13
14
|
export * from './runtime/processor-run-serialiser.js';
|
|
14
15
|
export * from './store/checkpoint-store.js';
|
|
@@ -2,9 +2,11 @@ import type { CheckpointStore } from '../store/checkpoint-store.js';
|
|
|
2
2
|
import type { EventJournal } from '../store/event-journal.js';
|
|
3
3
|
import { type EventProcessor } from '../subscriptions/event-processor.js';
|
|
4
4
|
import type { EventingClock } from './clock.js';
|
|
5
|
+
import { EventProcessorFailure } from './processor-failure.js';
|
|
5
6
|
import { type EventProcessorHealth } from './processor-health.js';
|
|
6
7
|
import type { ProcessorRunSerialiser } from './processor-run-serialiser.js';
|
|
7
8
|
export interface EventProcessorHostOptions {
|
|
9
|
+
readonly onFailure?: (error: EventProcessorFailure) => void;
|
|
8
10
|
readonly fallbackMs?: number;
|
|
9
11
|
readonly retryBackoff?: (consecutiveFailures: number, signal: AbortSignal) => Promise<void>;
|
|
10
12
|
}
|
|
@@ -21,6 +23,7 @@ export declare class EventProcessorHost {
|
|
|
21
23
|
private readonly journal;
|
|
22
24
|
private readonly checkpoints;
|
|
23
25
|
private readonly serialiseRun;
|
|
26
|
+
private readonly options;
|
|
24
27
|
private readonly fallbackMs;
|
|
25
28
|
private readonly retryBackoff;
|
|
26
29
|
private readonly healthRegistry;
|
|
@@ -28,6 +31,8 @@ export declare class EventProcessorHost {
|
|
|
28
31
|
start(processors: readonly EventProcessor[], parentSignal?: AbortSignal): EventProcessorHostRun;
|
|
29
32
|
runOnce(processor: EventProcessor, signal?: AbortSignal): Promise<EventProcessorPass>;
|
|
30
33
|
runThrough(processor: EventProcessor, targetGlobalPosition: number, signal?: AbortSignal): Promise<EventProcessorPass>;
|
|
34
|
+
private observedRun;
|
|
35
|
+
private failure;
|
|
31
36
|
health(consumer: string): EventProcessorHealth | undefined;
|
|
32
37
|
private runProcessor;
|
|
33
38
|
private runProcessorPass;
|
package/dist/src/node_modules/@atolis-hq/eventing/dist/runtime/event-processor-host.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"event-processor-host.d.ts","sourceRoot":"","sources":["../../src/runtime/event-processor-host.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,KAAK,cAAc,EAAE,MAAM,qCAAqC,CAAC;AAC1E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,EAGL,KAAK,oBAAoB,EAC1B,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AAY5E,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,mBAAmB,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC7F;AAED,MAAM,WAAW,qBAAqB;IACpC,KAAK,IAAI,IAAI,CAAC;IACd,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;CAC9B;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B;AAED,qBAAa,kBAAkB;IAU3B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,YAAY;
|
|
1
|
+
{"version":3,"file":"event-processor-host.d.ts","sourceRoot":"","sources":["../../src/runtime/event-processor-host.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,KAAK,cAAc,EAAE,MAAM,qCAAqC,CAAC;AAC1E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AAC/D,OAAO,EAGL,KAAK,oBAAoB,EAC1B,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AAY5E,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,KAAK,EAAE,qBAAqB,KAAK,IAAI,CAAC;IAC5D,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,mBAAmB,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC7F;AAED,MAAM,WAAW,qBAAqB;IACpC,KAAK,IAAI,IAAI,CAAC;IACd,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;CAC9B;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B;AAED,qBAAa,kBAAkB;IAU3B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,YAAY;IAE7B,OAAO,CAAC,QAAQ,CAAC,OAAO;IAb1B,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAGV;IAEnB,OAAO,CAAC,QAAQ,CAAC,cAAc,CAA0B;gBAGtC,OAAO,EAAE,YAAY,EACrB,WAAW,EAAE,eAAe,EAC5B,YAAY,EAAE,sBAAsB,EACrD,KAAK,EAAE,aAAa,EACH,OAAO,GAAE,yBAA8B;IAU1D,KAAK,CAAC,UAAU,EAAE,SAAS,cAAc,EAAE,EAAE,YAAY,CAAC,EAAE,WAAW,GAAG,qBAAqB;IAczF,OAAO,CAAC,SAAS,EAAE,cAAc,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAQrF,UAAU,CACd,SAAS,EAAE,cAAc,EACzB,oBAAoB,EAAE,MAAM,EAC5B,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,kBAAkB,CAAC;YAShB,WAAW;IA6CzB,OAAO,CAAC,OAAO;IAWf,MAAM,CAAC,QAAQ,EAAE,MAAM,GAAG,oBAAoB,GAAG,SAAS;YAI5C,YAAY;YAkEZ,gBAAgB;YAwChB,mBAAmB;CAiBlC;AAuBD,qBAAa,6BAA8B,SAAQ,KAAK;;CAIvD"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { EventProcessorFailure } from './processor-failure.js';
|
|
1
2
|
import { EventProcessorHealthStatus, ProcessorHealthRegistry, } from './processor-health.js';
|
|
2
3
|
import { assertDistinctProcessorConsumers, assertNonNegativeSafeInteger, assertPositiveSafeInteger, assertProcessor, } from './processor-validation.js';
|
|
3
4
|
const defaultBatchSize = 100;
|
|
@@ -7,6 +8,7 @@ export class EventProcessorHost {
|
|
|
7
8
|
journal;
|
|
8
9
|
checkpoints;
|
|
9
10
|
serialiseRun;
|
|
11
|
+
options;
|
|
10
12
|
fallbackMs;
|
|
11
13
|
retryBackoff;
|
|
12
14
|
healthRegistry;
|
|
@@ -14,6 +16,7 @@ export class EventProcessorHost {
|
|
|
14
16
|
this.journal = journal;
|
|
15
17
|
this.checkpoints = checkpoints;
|
|
16
18
|
this.serialiseRun = serialiseRun;
|
|
19
|
+
this.options = options;
|
|
17
20
|
this.fallbackMs = options.fallbackMs ?? defaultFallbackMs;
|
|
18
21
|
this.retryBackoff = options.retryBackoff ?? defaultRetryBackoff;
|
|
19
22
|
this.healthRegistry = new ProcessorHealthRegistry(journal, clock);
|
|
@@ -37,13 +40,54 @@ export class EventProcessorHost {
|
|
|
37
40
|
async runOnce(processor, signal) {
|
|
38
41
|
assertProcessor(processor);
|
|
39
42
|
const runSignal = signal ?? new AbortController().signal;
|
|
40
|
-
return this.
|
|
43
|
+
return this.observedRun(processor, runSignal, () => this.runProcessorPass(processor, runSignal));
|
|
41
44
|
}
|
|
42
45
|
async runThrough(processor, targetGlobalPosition, signal) {
|
|
43
46
|
assertProcessor(processor);
|
|
44
47
|
assertNonNegativeSafeInteger(targetGlobalPosition, 'Processor target global position');
|
|
45
48
|
const runSignal = signal ?? new AbortController().signal;
|
|
46
|
-
return this.
|
|
49
|
+
return this.observedRun(processor, runSignal, () => this.runProcessorThrough(processor, targetGlobalPosition, runSignal));
|
|
50
|
+
}
|
|
51
|
+
async observedRun(processor, signal, operation) {
|
|
52
|
+
try {
|
|
53
|
+
return await this.serialiseRun(processor.consumer, signal, async () => {
|
|
54
|
+
const previous = this.healthRegistry.get(processor.consumer);
|
|
55
|
+
if (previous?.status === EventProcessorHealthStatus.Degraded)
|
|
56
|
+
await this.retryBackoff(previous.consecutiveFailures, signal);
|
|
57
|
+
throwIfAborted(signal);
|
|
58
|
+
const pass = await operation();
|
|
59
|
+
const head = await this.journal.latestGlobalPosition();
|
|
60
|
+
await this.healthRegistry.update(processor, pass.checkpoint >= head
|
|
61
|
+
? EventProcessorHealthStatus.Healthy
|
|
62
|
+
: EventProcessorHealthStatus.CatchingUp, pass.checkpoint, 0, { lastSuccessAt: this.healthRegistry.now() }, head);
|
|
63
|
+
return pass;
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
catch (error) {
|
|
67
|
+
if (signal.aborted)
|
|
68
|
+
throw error;
|
|
69
|
+
const previous = this.healthRegistry.get(processor.consumer);
|
|
70
|
+
const failure = this.failure(processor, error);
|
|
71
|
+
let checkpoint = previous?.checkpoint ?? 0;
|
|
72
|
+
try {
|
|
73
|
+
checkpoint = await this.checkpoints.load(processor.consumer);
|
|
74
|
+
}
|
|
75
|
+
catch {
|
|
76
|
+
/* Retain known position. */
|
|
77
|
+
}
|
|
78
|
+
await this.healthRegistry.degrade(processor, checkpoint, (previous?.consecutiveFailures ?? 0) + 1, failure);
|
|
79
|
+
throw failure;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
failure(processor, error) {
|
|
83
|
+
const failure = error instanceof EventProcessorFailure ? error : new EventProcessorFailure(processor, error);
|
|
84
|
+
try {
|
|
85
|
+
this.options.onFailure?.(failure);
|
|
86
|
+
}
|
|
87
|
+
catch {
|
|
88
|
+
/* Diagnostics cannot stop supervision. */
|
|
89
|
+
}
|
|
90
|
+
return failure;
|
|
47
91
|
}
|
|
48
92
|
health(consumer) {
|
|
49
93
|
return this.healthRegistry.get(consumer);
|
|
@@ -82,7 +126,7 @@ export class EventProcessorHost {
|
|
|
82
126
|
catch {
|
|
83
127
|
// The next supervised attempt re-reads the durable checkpoint.
|
|
84
128
|
}
|
|
85
|
-
await this.healthRegistry.degrade(processor, checkpoint, consecutiveFailures, error);
|
|
129
|
+
await this.healthRegistry.degrade(processor, checkpoint, consecutiveFailures, this.failure(processor, error));
|
|
86
130
|
await this.retryBackoff(consecutiveFailures, signal);
|
|
87
131
|
}
|
|
88
132
|
}
|
|
@@ -118,11 +162,16 @@ export class EventProcessorHost {
|
|
|
118
162
|
let handledCount = 0;
|
|
119
163
|
for (const event of bounded) {
|
|
120
164
|
throwIfAborted(signal);
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
165
|
+
try {
|
|
166
|
+
const message = processor.select(event);
|
|
167
|
+
if (message === null)
|
|
168
|
+
continue;
|
|
169
|
+
handledCount += 1;
|
|
170
|
+
await processor.handle(message, event, signal);
|
|
171
|
+
}
|
|
172
|
+
catch (error) {
|
|
173
|
+
throw new EventProcessorFailure(processor, error, event);
|
|
174
|
+
}
|
|
126
175
|
}
|
|
127
176
|
const nextCheckpoint = bounded.at(-1).globalPosition;
|
|
128
177
|
await this.checkpoints.save(processor.consumer, nextCheckpoint);
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { EventEnvelope } from '../contracts/events.js';
|
|
2
|
+
import type { EventProcessor } from '../subscriptions/event-processor.js';
|
|
3
|
+
/** Preserves the original stack as cause and attributes only known event context. */
|
|
4
|
+
export declare class EventProcessorFailure extends Error {
|
|
5
|
+
readonly consumer: string;
|
|
6
|
+
readonly eventId?: string;
|
|
7
|
+
readonly globalPosition?: number;
|
|
8
|
+
constructor(processor: Pick<EventProcessor, 'name' | 'consumer'>, cause: unknown, event?: EventEnvelope);
|
|
9
|
+
}
|
|
10
|
+
//# sourceMappingURL=processor-failure.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"processor-failure.d.ts","sourceRoot":"","sources":["../../src/runtime/processor-failure.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qCAAqC,CAAC;AAE1E,qFAAqF;AACrF,qBAAa,qBAAsB,SAAQ,KAAK;IAC9C,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;gBAG/B,SAAS,EAAE,IAAI,CAAC,cAAc,EAAE,MAAM,GAAG,UAAU,CAAC,EACpD,KAAK,EAAE,OAAO,EACd,KAAK,CAAC,EAAE,aAAa;CAaxB"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/** Preserves the original stack as cause and attributes only known event context. */
|
|
2
|
+
export class EventProcessorFailure extends Error {
|
|
3
|
+
consumer;
|
|
4
|
+
eventId;
|
|
5
|
+
globalPosition;
|
|
6
|
+
constructor(processor, cause, event) {
|
|
7
|
+
super(`Processor ${processor.name} (${processor.consumer}) failed${event === undefined ? '' : ` on event ${event.event.eventId} at global position ${event.globalPosition}`}: ${cause instanceof Error ? cause.message : String(cause)}`, { cause });
|
|
8
|
+
this.name = 'EventProcessorFailure';
|
|
9
|
+
this.consumer = processor.consumer;
|
|
10
|
+
if (event !== undefined) {
|
|
11
|
+
this.eventId = event.event.eventId;
|
|
12
|
+
this.globalPosition = event.globalPosition;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"processor-health.d.ts","sourceRoot":"","sources":["../../src/runtime/processor-health.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,KAAK,cAAc,EAAE,MAAM,qCAAqC,CAAC;AAC1E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"processor-health.d.ts","sourceRoot":"","sources":["../../src/runtime/processor-health.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,KAAK,cAAc,EAAE,MAAM,qCAAqC,CAAC;AAC1E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAKhD,eAAO,MAAM,0BAA0B;;;;;;CAM7B,CAAC;AAEX,MAAM,MAAM,0BAA0B,GACpC,CAAC,OAAO,0BAA0B,CAAC,CAAC,MAAM,OAAO,0BAA0B,CAAC,CAAC;AAE/E,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;IAC9C,QAAQ,CAAC,MAAM,EAAE,0BAA0B,CAAC;IAC5C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,SAAS,CAAC,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IACzE,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;CACjC;AAED,KAAK,YAAY,GAAG,OAAO,CACzB,IAAI,CAAC,oBAAoB,EAAE,WAAW,GAAG,eAAe,GAAG,eAAe,GAAG,eAAe,CAAC,CAC9F,CAAC;AAEF,qBAAa,uBAAuB;IAIhC,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,KAAK;IAJxB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAA2C;gBAGlD,OAAO,EAAE,YAAY,EACrB,KAAK,EAAE,aAAa;IAGvC,GAAG,CAAC,QAAQ,EAAE,MAAM,GAAG,oBAAoB,GAAG,SAAS;IAIjD,MAAM,CACV,SAAS,EAAE,cAAc,EACzB,MAAM,EAAE,0BAA0B,EAClC,UAAU,EAAE,MAAM,EAClB,mBAAmB,EAAE,MAAM,EAC3B,MAAM,GAAE,YAAiB,EACzB,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,CAAC,oBAAoB,CAAC;IAO1B,OAAO,CACX,SAAS,EAAE,cAAc,EACzB,UAAU,EAAE,MAAM,EAClB,mBAAmB,EAAE,MAAM,EAC3B,KAAK,EAAE,OAAO,GACb,OAAO,CAAC,IAAI,CAAC;IA0BhB,GAAG,IAAI,MAAM;IAIb,OAAO,CAAC,MAAM;CA0Bf"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { EventProcessorFailure } from './processor-failure.js';
|
|
1
2
|
const maximumErrorLength = 1_000;
|
|
2
3
|
export const EventProcessorHealthStatus = {
|
|
3
4
|
Starting: 'starting',
|
|
@@ -57,6 +58,12 @@ export class ProcessorHealthRegistry {
|
|
|
57
58
|
}
|
|
58
59
|
}
|
|
59
60
|
function boundedError(error) {
|
|
61
|
+
// Keep the attributed message but report the original error class.
|
|
62
|
+
if (error instanceof EventProcessorFailure && error.cause instanceof Error)
|
|
63
|
+
return {
|
|
64
|
+
name: error.cause.name.slice(0, maximumErrorLength),
|
|
65
|
+
message: error.message.slice(0, maximumErrorLength),
|
|
66
|
+
};
|
|
60
67
|
if (error instanceof Error)
|
|
61
68
|
return {
|
|
62
69
|
name: error.name.slice(0, maximumErrorLength),
|
|
@@ -3,6 +3,11 @@ export declare const ApiAdapterHealthStatus: {
|
|
|
3
3
|
readonly Unknown: "unknown";
|
|
4
4
|
};
|
|
5
5
|
export type ApiAdapterHealthStatus = 'ok' | 'degraded' | ValueOf<typeof ApiAdapterHealthStatus>;
|
|
6
|
+
/** Marks checks that report an event processor, so clients need not infer it from names. */
|
|
7
|
+
export declare const ApiHealthCheckKind: {
|
|
8
|
+
readonly Processor: "event-processor";
|
|
9
|
+
};
|
|
10
|
+
export type ApiHealthCheckKind = ValueOf<typeof ApiHealthCheckKind>;
|
|
6
11
|
export interface HealthResponse {
|
|
7
12
|
readonly status: 'ok' | 'degraded';
|
|
8
13
|
readonly version: string;
|
|
@@ -10,6 +15,7 @@ export interface HealthResponse {
|
|
|
10
15
|
readonly checks?: readonly {
|
|
11
16
|
readonly name: string;
|
|
12
17
|
readonly status: 'ok' | 'degraded';
|
|
18
|
+
readonly kind?: ApiHealthCheckKind;
|
|
13
19
|
readonly detail?: string;
|
|
14
20
|
}[];
|
|
15
21
|
readonly adapters?: readonly {
|
|
@@ -2,3 +2,7 @@ import { defineClosedVocabulary } from '../../../kernel/index.js';
|
|
|
2
2
|
export const ApiAdapterHealthStatus = defineClosedVocabulary({
|
|
3
3
|
Unknown: 'unknown',
|
|
4
4
|
});
|
|
5
|
+
/** Marks checks that report an event processor, so clients need not infer it from names. */
|
|
6
|
+
export const ApiHealthCheckKind = defineClosedVocabulary({
|
|
7
|
+
Processor: 'event-processor',
|
|
8
|
+
});
|
|
@@ -14,6 +14,8 @@ export interface SurfacePairingGrant {
|
|
|
14
14
|
}
|
|
15
15
|
export type CredentialMutationSerialiser = <Value>(lockPath: string, operation: () => Promise<Value>) => Promise<Value>;
|
|
16
16
|
export declare function credentialsPath(wakeRoot: string): string;
|
|
17
|
+
/** Read existing credentials without creating auth state. */
|
|
18
|
+
export declare function loadCredentials(wakeRoot: string): Promise<SurfaceCredentials>;
|
|
17
19
|
export declare function loadOrCreateCredentials(wakeRoot: string): Promise<SurfaceCredentials>;
|
|
18
20
|
export declare function replaceAccessKey(wakeRoot: string, accessKey: string, serialise: CredentialMutationSerialiser): Promise<SurfaceCredentials>;
|
|
19
21
|
export declare function createPairingGrant(wakeRoot: string, now: Date | undefined, serialise: CredentialMutationSerialiser): Promise<SurfacePairingGrant>;
|
|
@@ -6,6 +6,10 @@ const credentialLockName = 'surface-credentials.lock';
|
|
|
6
6
|
export function credentialsPath(wakeRoot) {
|
|
7
7
|
return join(wakeRoot, '.wake', 'auth', 'credentials.json');
|
|
8
8
|
}
|
|
9
|
+
/** Read existing credentials without creating auth state. */
|
|
10
|
+
export async function loadCredentials(wakeRoot) {
|
|
11
|
+
return decodeCredentials(JSON.parse(await readFile(credentialsPath(wakeRoot), 'utf8')));
|
|
12
|
+
}
|
|
9
13
|
export async function loadOrCreateCredentials(wakeRoot) {
|
|
10
14
|
const path = credentialsPath(wakeRoot);
|
|
11
15
|
try {
|