agents-relay 2.0.37 → 2.0.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/dist/observability.js +186 -0
- package/dist/pool.js +28 -4
- package/dist/projected-store.js +38 -2
- package/dist/projection-outbox.js +4 -1
- package/dist/reconciler.js +40 -1
- package/dist/relay-store.js +19 -3
- package/dist/relayd.js +14 -6
- package/dist/scheduler.js +20 -2
- package/dist/workflow/authoritative-unblock-v2.js +2 -2
- package/dist/workflow/blocker-resolution-v2.js +2 -2
- package/dist/workflow/dispatcher.js +61 -0
- package/dist/workflow/durable-transition.js +19 -2
- package/dist/workflow/effects.js +23 -0
- package/dist/workflow/lifecycle.js +7 -0
- package/dist/workflow/planner-orchestration-v2.js +2 -2
- package/dist/workflow/planner-trigger-v2.js +1 -1
- package/dist/workflow/review-v2.js +2 -2
- package/dist/workflow/schedule-cycle-v2.js +1 -1
- package/dist/workflow/scheduled-run-v2.js +2 -2
- package/dist/workflow/task-related-action-v2.js +1 -0
- package/dist/workflow/troubleshooter-v2.js +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -42,6 +42,10 @@ RelayStore owns local operational execution facts; GitHub remains the human-read
|
|
|
42
42
|
|
|
43
43
|
These paths always represent the current canonical graphs. Historical graph definitions are available through Git history, tags, and commits rather than parallel versioned files in `workflow/`. See [`docs/workflow-engine.md`](docs/workflow-engine.md) for the executable profile and runtime boundary.
|
|
44
44
|
|
|
45
|
+
### Structured runtime observability
|
|
46
|
+
|
|
47
|
+
`src/observability.ts` is the single structured logging/event boundary that records why a lifecycle or projection decision happened. One correlated record chain (`wake -> graph -> transition -> store commit -> projection outcome`) lets an operator reconstruct a representative task lifecycle without ad-hoc stderr parsing. Records are observational only, never a second durable store; they correlate by repository/PR/job/task/execution, classify failures as deterministic `conflict` vs `transient` retry vs `terminal`, redact secrets/prompts, bound large payloads, and keep routine idle/no-op cycles quiet at the default level. See [`docs/structured-observability.md`](docs/structured-observability.md) for the record contract, fields, noise policy, and redaction rules.
|
|
48
|
+
|
|
45
49
|
## Quick start
|
|
46
50
|
|
|
47
51
|
For GitHub-backed work, create or adopt the PR through Agents Relay itself. This guarantees the durable job marker exists before any managed task is submitted.
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
2
|
+
const LEVEL_RANK = { debug: 0, info: 1, warn: 2, error: 3 };
|
|
3
|
+
export const OBSERVABILITY_MAX_STRING = 512;
|
|
4
|
+
export const OBSERVABILITY_MAX_ITEMS = 20;
|
|
5
|
+
export const OBSERVABILITY_MAX_DEPTH = 4;
|
|
6
|
+
export const OBSERVABILITY_REDACTED = '[redacted]';
|
|
7
|
+
const SENSITIVE_KEY = /(secret|token|password|passwd|authorization|cookie|credential|api[_-]?key|private[_-]?key|bearer|prompt)/i;
|
|
8
|
+
const BULK_KEY = /^(payload|input|body|diff|patch|content|messages?)$/i;
|
|
9
|
+
export function errorMessage(error) {
|
|
10
|
+
if (error instanceof Error)
|
|
11
|
+
return error.message;
|
|
12
|
+
if (typeof error === 'string')
|
|
13
|
+
return error;
|
|
14
|
+
if (error && typeof error === 'object') {
|
|
15
|
+
const message = error.message;
|
|
16
|
+
if (typeof message === 'string')
|
|
17
|
+
return message;
|
|
18
|
+
}
|
|
19
|
+
return String(error);
|
|
20
|
+
}
|
|
21
|
+
function failureStatus(error) {
|
|
22
|
+
if (!error || typeof error !== 'object')
|
|
23
|
+
return undefined;
|
|
24
|
+
const value = error;
|
|
25
|
+
for (const candidate of [value.status, value.statusCode, value.response?.status]) {
|
|
26
|
+
if (typeof candidate === 'number' && Number.isFinite(candidate))
|
|
27
|
+
return candidate;
|
|
28
|
+
}
|
|
29
|
+
return undefined;
|
|
30
|
+
}
|
|
31
|
+
function failureRetryAfterMs(error) {
|
|
32
|
+
if (!error || typeof error !== 'object')
|
|
33
|
+
return undefined;
|
|
34
|
+
const value = error;
|
|
35
|
+
const retryAfterSeconds = Number(value.retryAfter);
|
|
36
|
+
if (Number.isFinite(retryAfterSeconds) && retryAfterSeconds > 0)
|
|
37
|
+
return retryAfterSeconds * 1000;
|
|
38
|
+
const retryAfterMs = Number(value.retryAfterMs);
|
|
39
|
+
if (Number.isFinite(retryAfterMs) && retryAfterMs > 0)
|
|
40
|
+
return retryAfterMs;
|
|
41
|
+
for (const headers of [value.response?.headers, value.headers]) {
|
|
42
|
+
const raw = headerValue(headers, 'retry-after');
|
|
43
|
+
const seconds = Number(raw);
|
|
44
|
+
if (Number.isFinite(seconds) && seconds > 0)
|
|
45
|
+
return seconds * 1000;
|
|
46
|
+
}
|
|
47
|
+
return undefined;
|
|
48
|
+
}
|
|
49
|
+
function headerValue(headers, name) {
|
|
50
|
+
if (!headers || typeof headers !== 'object')
|
|
51
|
+
return null;
|
|
52
|
+
if (typeof headers.get === 'function')
|
|
53
|
+
return headers.get(name);
|
|
54
|
+
const record = headers;
|
|
55
|
+
const hit = Object.entries(record).find(([key]) => key.toLowerCase() === name);
|
|
56
|
+
return hit && hit[1] !== undefined ? String(hit[1]) : null;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Deterministic conflict vs transient retry classification.
|
|
60
|
+
*
|
|
61
|
+
* `conflict` means the durable record rejects a divergent definition and retrying
|
|
62
|
+
* will not help. `transient` means the same request may succeed later (rate limit,
|
|
63
|
+
* 5xx, network interruption, retry-after). `terminal` covers everything else.
|
|
64
|
+
*/
|
|
65
|
+
export function classifyFailure(error) {
|
|
66
|
+
const reason = errorMessage(error);
|
|
67
|
+
const status = failureStatus(error);
|
|
68
|
+
const retryAfterMs = failureRetryAfterMs(error);
|
|
69
|
+
const classified = { classification: 'terminal', reason };
|
|
70
|
+
if (status !== undefined)
|
|
71
|
+
classified.status = status;
|
|
72
|
+
if (retryAfterMs !== undefined)
|
|
73
|
+
classified.retryAfterMs = retryAfterMs;
|
|
74
|
+
if (/already exists with a different definition/i.test(reason) || status === 409) {
|
|
75
|
+
return { ...classified, classification: 'conflict' };
|
|
76
|
+
}
|
|
77
|
+
if (status === 429
|
|
78
|
+
|| (status !== undefined && status >= 500)
|
|
79
|
+
|| retryAfterMs !== undefined
|
|
80
|
+
|| /\b(ECONNRESET|ETIMEDOUT|ENOTFOUND|EAI_AGAIN|ECONNREFUSED|EPIPE|UND_ERR)\b/.test(reason)
|
|
81
|
+
|| /fetch failed|network|socket hang up|rate limit|secondary rate|temporarily unavailable/i.test(reason)) {
|
|
82
|
+
return { ...classified, classification: 'transient' };
|
|
83
|
+
}
|
|
84
|
+
return classified;
|
|
85
|
+
}
|
|
86
|
+
/** Bound and redact observation data so records never leak secrets or large payloads. */
|
|
87
|
+
export function sanitizeObservationValue(value, depth = 0) {
|
|
88
|
+
if (value === null || value === undefined)
|
|
89
|
+
return value;
|
|
90
|
+
if (typeof value === 'string') {
|
|
91
|
+
return value.length > OBSERVABILITY_MAX_STRING
|
|
92
|
+
? `${value.slice(0, OBSERVABILITY_MAX_STRING)}…(+${value.length - OBSERVABILITY_MAX_STRING})`
|
|
93
|
+
: value;
|
|
94
|
+
}
|
|
95
|
+
if (typeof value === 'number' || typeof value === 'boolean')
|
|
96
|
+
return value;
|
|
97
|
+
if (typeof value === 'bigint')
|
|
98
|
+
return value.toString();
|
|
99
|
+
if (typeof value === 'function')
|
|
100
|
+
return '[function]';
|
|
101
|
+
if (Array.isArray(value)) {
|
|
102
|
+
const items = value.slice(0, OBSERVABILITY_MAX_ITEMS).map(item => sanitizeObservationValue(item, depth + 1));
|
|
103
|
+
return value.length > OBSERVABILITY_MAX_ITEMS ? [...items, `+${value.length - OBSERVABILITY_MAX_ITEMS} more`] : items;
|
|
104
|
+
}
|
|
105
|
+
if (typeof value === 'object') {
|
|
106
|
+
if (depth >= OBSERVABILITY_MAX_DEPTH)
|
|
107
|
+
return '[truncated]';
|
|
108
|
+
const out = {};
|
|
109
|
+
for (const [key, item] of Object.entries(value)) {
|
|
110
|
+
if (SENSITIVE_KEY.test(key)) {
|
|
111
|
+
out[key] = OBSERVABILITY_REDACTED;
|
|
112
|
+
continue;
|
|
113
|
+
}
|
|
114
|
+
if (BULK_KEY.test(key) && typeof item === 'string' && item.length > OBSERVABILITY_MAX_STRING) {
|
|
115
|
+
out[key] = `[omitted ${item.length} chars]`;
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
out[key] = sanitizeObservationValue(item, depth + 1);
|
|
119
|
+
}
|
|
120
|
+
return out;
|
|
121
|
+
}
|
|
122
|
+
return '[unsupported]';
|
|
123
|
+
}
|
|
124
|
+
export function sanitizeObservationData(data) {
|
|
125
|
+
if (!data)
|
|
126
|
+
return undefined;
|
|
127
|
+
return sanitizeObservationValue(data, 0);
|
|
128
|
+
}
|
|
129
|
+
function writeStderrObservation(record) {
|
|
130
|
+
try {
|
|
131
|
+
process.stderr.write(`${JSON.stringify(record)}\n`);
|
|
132
|
+
}
|
|
133
|
+
catch { /* observability must never break execution */ }
|
|
134
|
+
}
|
|
135
|
+
const correlationStorage = new AsyncLocalStorage();
|
|
136
|
+
export function withObservabilityCorrelation(correlation, work) {
|
|
137
|
+
return correlationStorage.run({ ...currentObservabilityCorrelation(), ...correlation }, work);
|
|
138
|
+
}
|
|
139
|
+
export function currentObservabilityCorrelation() {
|
|
140
|
+
return correlationStorage.getStore() ?? {};
|
|
141
|
+
}
|
|
142
|
+
export class RelayObservability {
|
|
143
|
+
sink;
|
|
144
|
+
minimumLevel;
|
|
145
|
+
constructor(sink = writeStderrObservation, minimumLevel = 'info') {
|
|
146
|
+
this.sink = sink;
|
|
147
|
+
this.minimumLevel = minimumLevel;
|
|
148
|
+
}
|
|
149
|
+
/** No-op observer used as the safe default until a runtime configures logging. */
|
|
150
|
+
static silent() { return new RelayObservability(() => undefined, 'error'); }
|
|
151
|
+
static stderr(minimumLevel = 'info') {
|
|
152
|
+
return new RelayObservability(writeStderrObservation, minimumLevel);
|
|
153
|
+
}
|
|
154
|
+
static fromEnvironment(env = process.env) {
|
|
155
|
+
const configured = (env.AGENTS_RELAY_OBSERVABILITY ?? env.RELAY_OBSERVABILITY_LEVEL ?? '').trim();
|
|
156
|
+
if (/^(silent|off|none|0|false)$/i.test(configured))
|
|
157
|
+
return RelayObservability.silent();
|
|
158
|
+
const level = /^(debug|info|warn|error)$/i.test(configured) ? configured.toLowerCase() : 'info';
|
|
159
|
+
return RelayObservability.stderr(level);
|
|
160
|
+
}
|
|
161
|
+
isEnabled(level) { return LEVEL_RANK[level] >= LEVEL_RANK[this.minimumLevel]; }
|
|
162
|
+
record(input) {
|
|
163
|
+
const level = input.level ?? 'info';
|
|
164
|
+
if (!this.isEnabled(level))
|
|
165
|
+
return;
|
|
166
|
+
const observation = {
|
|
167
|
+
ts: new Date().toISOString(),
|
|
168
|
+
stage: input.stage,
|
|
169
|
+
event: input.event,
|
|
170
|
+
level,
|
|
171
|
+
outcome: input.outcome ?? 'succeeded',
|
|
172
|
+
message: input.message,
|
|
173
|
+
correlation: { ...currentObservabilityCorrelation(), ...input.correlation },
|
|
174
|
+
...(input.transition ? { transition: input.transition } : {}),
|
|
175
|
+
...(input.data ? { data: sanitizeObservationData(input.data) } : {}),
|
|
176
|
+
};
|
|
177
|
+
this.sink(observation);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
let activeObservability = RelayObservability.silent();
|
|
181
|
+
export function observability() { return activeObservability; }
|
|
182
|
+
export function setObservability(next) { activeObservability = next; }
|
|
183
|
+
/** In-memory sink for tests and diagnostics. */
|
|
184
|
+
export function collectObservations(records) {
|
|
185
|
+
return record => { records.push(record); };
|
|
186
|
+
}
|
package/dist/pool.js
CHANGED
|
@@ -3,6 +3,7 @@ import { eventFor } from './events.js';
|
|
|
3
3
|
import { runScheduledBucket } from './scheduled-jobs.js';
|
|
4
4
|
import { scheduleBucket } from './scheduler.js';
|
|
5
5
|
import { ReconcileCoordinator } from './reconcile-coordinator.js';
|
|
6
|
+
import { errorMessage, observability, withObservabilityCorrelation } from './observability.js';
|
|
6
7
|
const RANK = { P0: 0, P1: 1, P2: 2, P3: 3 };
|
|
7
8
|
export function compareJobPriority(a, b) { return RANK[a.priority ?? 'P2'] - RANK[b.priority ?? 'P2'] || a.createdAt.localeCompare(b.createdAt) || a.id.localeCompare(b.id); }
|
|
8
9
|
export function runnableManagedJobs(items) { return items.filter(item => item.githubState === 'OPEN' && item.job.state === 'ACTIVE').map(item => item.job).sort(compareJobPriority); }
|
|
@@ -47,7 +48,12 @@ export class RepositoryWorkerPool {
|
|
|
47
48
|
this.makeStore = makeStore;
|
|
48
49
|
this.repositories = [...new Set(typeof repository === 'string' ? [repository] : repository)];
|
|
49
50
|
const reportCoordinatorError = (error, signal) => {
|
|
50
|
-
|
|
51
|
+
observability().record({
|
|
52
|
+
stage: 'reconcile', event: 'reconcile.coordinator_failed', level: 'error', outcome: 'failed',
|
|
53
|
+
message: `Reconcile coordinator failed for ${signal.jobId} (${signal.reason}): ${errorMessage(error)}`,
|
|
54
|
+
correlation: { repository: signal.repository, prNumber: signal.prNumber, jobId: signal.jobId },
|
|
55
|
+
data: { reason: signal.reason },
|
|
56
|
+
});
|
|
51
57
|
};
|
|
52
58
|
this.coordinator = new ReconcileCoordinator((jobId, signals) => this.consumeSignals(jobId, signals), reportCoordinatorError);
|
|
53
59
|
this.githubSignalCoordinator = new ReconcileCoordinator((_key, signals) => this.consumeGitHubSignals(signals), reportCoordinatorError);
|
|
@@ -103,7 +109,12 @@ export class RepositoryWorkerPool {
|
|
|
103
109
|
}
|
|
104
110
|
const target = this.jobTargets.get(jobId);
|
|
105
111
|
if (!target || !this.repositories.includes(target.repository)) {
|
|
106
|
-
|
|
112
|
+
observability().record({
|
|
113
|
+
stage: 'reconcile', event: 'reconcile.signal_unrouted', level: 'warn', outcome: 'skipped',
|
|
114
|
+
message: `Reconcile signal target unknown for job ${jobId}`,
|
|
115
|
+
correlation: { jobId },
|
|
116
|
+
data: { reasons: signals.map(signal => signal.reason) },
|
|
117
|
+
});
|
|
107
118
|
return;
|
|
108
119
|
}
|
|
109
120
|
const seedKey = `${target.repository}#${target.prNumber}:${jobId}`;
|
|
@@ -129,11 +140,24 @@ export class RepositoryWorkerPool {
|
|
|
129
140
|
const result = await withGitHubRequestContext(this.client, `pool.coordinator-apply:${signal.reason}`, () => reconciler.applyEvent(signal.event));
|
|
130
141
|
shouldReconcile ||= result.reconcile;
|
|
131
142
|
}
|
|
132
|
-
if (!shouldReconcile)
|
|
143
|
+
if (!shouldReconcile) {
|
|
144
|
+
observability().record({
|
|
145
|
+
stage: 'wake', event: 'wake.signal', level: 'debug', outcome: 'noop',
|
|
146
|
+
message: `Wake for ${target.repository}#${target.prNumber} produced no reconcile work`,
|
|
147
|
+
correlation: { repository: target.repository, prNumber: target.prNumber, jobId },
|
|
148
|
+
data: { reasons: [...new Set(reasons)] },
|
|
149
|
+
});
|
|
133
150
|
return;
|
|
151
|
+
}
|
|
134
152
|
const reason = [...new Set(reasons)].join('+') || 'unknown';
|
|
153
|
+
observability().record({
|
|
154
|
+
stage: 'wake', event: 'wake.signal', outcome: 'started',
|
|
155
|
+
message: `Wake for ${target.repository}#${target.prNumber}: ${reason}`,
|
|
156
|
+
correlation: { repository: target.repository, prNumber: target.prNumber, jobId },
|
|
157
|
+
data: { reasons: [...new Set(reasons)] },
|
|
158
|
+
});
|
|
135
159
|
await this.reconcileExclusive(jobId, async () => {
|
|
136
|
-
await withGitHubRequestContext(this.client, `pool.coordinator-reconcile:${reason}`, () => reconciler.reconcile(jobId));
|
|
160
|
+
await withObservabilityCorrelation({ repository: target.repository, prNumber: target.prNumber, jobId }, () => withGitHubRequestContext(this.client, `pool.coordinator-reconcile:${reason}`, () => reconciler.reconcile(jobId)));
|
|
137
161
|
});
|
|
138
162
|
}
|
|
139
163
|
async reconcileDiscovered(discovered) {
|
package/dist/projected-store.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { registerBlockerDependency, resolveBlockerDependencies } from './blockers.js';
|
|
2
2
|
import { DurableDefinitionConflictError, isDurableDefinitionConflict, sameTaskDefinition } from './store-task-definition.js';
|
|
3
3
|
import { decideProjectionFailure } from './workflow/projection-conflict-v2.js';
|
|
4
|
+
import { classifyFailure, errorMessage, observability } from './observability.js';
|
|
4
5
|
function taskKey(task) { return `task:${task.jobId ?? ''}:${task.id}`; }
|
|
5
6
|
function planKey(plan) { return `plan:${plan.jobId}:${plan.id}`; }
|
|
6
7
|
function retryHeader(error, name) {
|
|
@@ -92,19 +93,36 @@ export class GitHubProjectionWriter {
|
|
|
92
93
|
await this.apply(item);
|
|
93
94
|
await this.relay.completeProjection(item.id);
|
|
94
95
|
projected += 1;
|
|
96
|
+
observability().record({
|
|
97
|
+
stage: 'projection', event: 'projection.completed', outcome: 'succeeded',
|
|
98
|
+
message: `Projection ${item.operation} applied`,
|
|
99
|
+
data: { operation: item.operation, idempotencyKey: item.idempotencyKey, coalesceKey: item.coalesceKey, attempts: item.attempts },
|
|
100
|
+
});
|
|
95
101
|
}
|
|
96
102
|
catch (error) {
|
|
97
|
-
const message =
|
|
103
|
+
const message = errorMessage(error);
|
|
98
104
|
const deterministic = isTerminalProjectionFailure(item, error);
|
|
105
|
+
const classified = classifyFailure(error);
|
|
99
106
|
const decision = decideProjectionFailure({ deterministicDefinitionConflict: deterministic });
|
|
100
107
|
if (decision.accepted && decision.target === 'conflict') {
|
|
101
108
|
const diagnostic = conflictDiagnostic(item, error, now.toISOString());
|
|
102
109
|
await this.relay.recordProjectionConflict(item.id, diagnostic);
|
|
103
110
|
await this.notifyConflict(diagnostic, message);
|
|
111
|
+
observability().record({
|
|
112
|
+
stage: 'projection', event: 'projection.conflict', level: 'error', outcome: 'failed',
|
|
113
|
+
message: `Projection ${item.operation} has a deterministic conflict: ${message}`,
|
|
114
|
+
data: { operation: item.operation, idempotencyKey: item.idempotencyKey, coalesceKey: item.coalesceKey, attempts: item.attempts, classification: 'conflict', reason: classified.reason },
|
|
115
|
+
});
|
|
104
116
|
}
|
|
105
117
|
else {
|
|
106
|
-
|
|
118
|
+
const retryAt = githubProjectionRetryAt(error, item.attempts, now);
|
|
119
|
+
await this.relay.retryProjection(item.id, message, retryAt);
|
|
107
120
|
deferred += 1;
|
|
121
|
+
observability().record({
|
|
122
|
+
stage: 'projection', event: 'projection.retry_scheduled', level: 'warn', outcome: 'failed',
|
|
123
|
+
message: `Projection ${item.operation} deferred for retry: ${message}`,
|
|
124
|
+
data: { operation: item.operation, idempotencyKey: item.idempotencyKey, attempts: item.attempts, classification: classified.classification, retryAt: retryAt.toISOString(), status: classified.status, reason: classified.reason },
|
|
125
|
+
});
|
|
108
126
|
}
|
|
109
127
|
}
|
|
110
128
|
}
|
|
@@ -195,6 +213,12 @@ export class ProjectedDurableStore {
|
|
|
195
213
|
payload: { operation: 'save_job', value },
|
|
196
214
|
priority: ['COMPLETED', 'CANCELLED'].includes(job.state) ? 100 : 20,
|
|
197
215
|
});
|
|
216
|
+
observability().record({
|
|
217
|
+
stage: 'store', event: 'store.commit', outcome: 'succeeded',
|
|
218
|
+
message: `Job ${job.id} committed`,
|
|
219
|
+
correlation: { repository: job.repository, prNumber: job.prNumber, jobId: job.id },
|
|
220
|
+
data: { operation: 'save_job', state: job.state, tasks: job.tasks.length },
|
|
221
|
+
});
|
|
198
222
|
}
|
|
199
223
|
async appendPlan(plan) {
|
|
200
224
|
const job = await this.load(plan.jobId);
|
|
@@ -246,6 +270,12 @@ export class ProjectedDurableStore {
|
|
|
246
270
|
payload: { operation: 'append_task', value: task },
|
|
247
271
|
priority: 40,
|
|
248
272
|
});
|
|
273
|
+
observability().record({
|
|
274
|
+
stage: 'store', event: 'store.commit', outcome: 'succeeded',
|
|
275
|
+
message: `Task ${task.id} appended`,
|
|
276
|
+
correlation: { repository: job.repository, prNumber: job.prNumber, jobId: task.jobId, taskId: task.id },
|
|
277
|
+
data: { operation: 'append_task', agentName: task.agentName, storySize: task.storySize },
|
|
278
|
+
});
|
|
249
279
|
}
|
|
250
280
|
async saveTask(task) {
|
|
251
281
|
if (!task.jobId)
|
|
@@ -267,6 +297,12 @@ export class ProjectedDurableStore {
|
|
|
267
297
|
payload: { operation: 'save_task', value },
|
|
268
298
|
priority: terminal ? 100 : 50,
|
|
269
299
|
});
|
|
300
|
+
observability().record({
|
|
301
|
+
stage: 'store', event: 'store.commit', outcome: 'succeeded',
|
|
302
|
+
message: `Task ${task.id} committed (${task.state})`,
|
|
303
|
+
correlation: { repository: job.repository, prNumber: job.prNumber, jobId: task.jobId, taskId: task.id, executionId: task.executionId ?? undefined },
|
|
304
|
+
data: { operation: 'save_task', state: task.state, attempt: task.attempt },
|
|
305
|
+
});
|
|
270
306
|
}
|
|
271
307
|
async listAgents() {
|
|
272
308
|
const local = await this.relay.loadAgents();
|
|
@@ -140,11 +140,13 @@ export class ProjectionOutbox {
|
|
|
140
140
|
}
|
|
141
141
|
enqueue(request, now) {
|
|
142
142
|
const operationKey = logicalOperationKey(request.idempotencyKey);
|
|
143
|
+
let coalesced = false;
|
|
143
144
|
if (request.coalesceKey) {
|
|
144
|
-
this.db.prepare(`
|
|
145
|
+
const removed = this.db.prepare(`
|
|
145
146
|
DELETE FROM projection_outbox
|
|
146
147
|
WHERE coalesce_key=? AND status='pending' AND operation_key<>?
|
|
147
148
|
`).run(request.coalesceKey, operationKey);
|
|
149
|
+
coalesced = Number(removed.changes) > 0;
|
|
148
150
|
}
|
|
149
151
|
const existing = this.db.prepare('SELECT status FROM projection_outbox WHERE operation_key=? ORDER BY id DESC LIMIT 1')
|
|
150
152
|
.get(operationKey);
|
|
@@ -171,6 +173,7 @@ export class ProjectionOutbox {
|
|
|
171
173
|
last_error=NULL,
|
|
172
174
|
updated_at=excluded.updated_at
|
|
173
175
|
`).run(idempotencyKey, operationKey, request.coalesceKey ?? null, request.operation, JSON.stringify(request.payload), request.priority ?? 0, now, now);
|
|
176
|
+
return coalesced;
|
|
174
177
|
}
|
|
175
178
|
claimDue(limit, now) {
|
|
176
179
|
const observedAt = now.toISOString();
|
package/dist/reconciler.js
CHANGED
|
@@ -21,6 +21,7 @@ import { taskTagWorkerPrompt } from './task-tag.js';
|
|
|
21
21
|
import { recoverExpiredExecutions, recoverStartupOrphans as recoverStartupExecutionOrphans } from './execution-recovery.js';
|
|
22
22
|
import { dispatchJobReconcile } from './workflow/job-reconcile-v2.js';
|
|
23
23
|
import { launchPlanExecution } from './planner-runtime.js';
|
|
24
|
+
import { classifyFailure, errorMessage, observability, withObservabilityCorrelation } from './observability.js';
|
|
24
25
|
const PRIORITY_RANK = { P0: 0, P1: 1, P2: 2, P3: 3 };
|
|
25
26
|
export function compareTaskPriority(a, b, jobPriority = 'P2') { const ar = PRIORITY_RANK[a.priority ?? jobPriority]; const br = PRIORITY_RANK[b.priority ?? jobPriority]; return ar - br || a.createdAt.localeCompare(b.createdAt) || a.id.localeCompare(b.id); }
|
|
26
27
|
export async function managedWorkerInput(job, task, context) {
|
|
@@ -106,6 +107,8 @@ export class Reconciler extends ReconcilerReviewBase {
|
|
|
106
107
|
try {
|
|
107
108
|
const job = await this.store.load(jobId);
|
|
108
109
|
const previousState = job.state;
|
|
110
|
+
const lifecycleFingerprint = (value) => `${value.state}|${value.tasks.map(task => `${task.id}:${task.state}`).join(',')}`;
|
|
111
|
+
const beforeFingerprint = lifecycleFingerprint(job);
|
|
109
112
|
const prState = await this.store.pullRequestState?.() ?? 'OPEN';
|
|
110
113
|
const headSha = prState === 'OPEN' ? await this.currentHeadSha(job) : null;
|
|
111
114
|
if (headSha && job.headSha !== headSha) {
|
|
@@ -201,7 +204,33 @@ export class Reconciler extends ReconcilerReviewBase {
|
|
|
201
204
|
const outcomes = await this.workflowEffects.execute([actionId], { jobId: finalState.id, transitionKey: `reconcile:${headSha ?? 'no-head'}`, data: { headSha } });
|
|
202
205
|
await this.applyJobEffectOutcomes(finalState, outcomes);
|
|
203
206
|
}, async () => { await this.store.saveJob(finalState); });
|
|
204
|
-
|
|
207
|
+
const settled = await this.store.load(jobId);
|
|
208
|
+
const changed = lifecycleFingerprint(settled) !== beforeFingerprint;
|
|
209
|
+
observability().record({
|
|
210
|
+
stage: 'reconcile',
|
|
211
|
+
event: 'reconcile',
|
|
212
|
+
level: changed ? 'info' : 'debug',
|
|
213
|
+
outcome: changed ? 'succeeded' : 'noop',
|
|
214
|
+
message: changed
|
|
215
|
+
? `Reconcile updated ${settled.repository}#${settled.prNumber} lifecycle`
|
|
216
|
+
: `Reconcile ${settled.repository}#${settled.prNumber} made no lifecycle change`,
|
|
217
|
+
correlation: { repository: settled.repository, prNumber: settled.prNumber, jobId: settled.id },
|
|
218
|
+
transition: { graph: 'job-lifecycle', from: previousState, to: settled.state },
|
|
219
|
+
data: { prState, tasks: settled.tasks.length },
|
|
220
|
+
});
|
|
221
|
+
return settled;
|
|
222
|
+
}
|
|
223
|
+
catch (error) {
|
|
224
|
+
observability().record({
|
|
225
|
+
stage: 'reconcile',
|
|
226
|
+
event: 'reconcile.failed',
|
|
227
|
+
level: 'error',
|
|
228
|
+
outcome: 'failed',
|
|
229
|
+
message: `Reconcile failed for ${jobId}: ${errorMessage(error)}`,
|
|
230
|
+
correlation: { jobId },
|
|
231
|
+
data: { classification: classifyFailure(error).classification, reason: errorMessage(error) },
|
|
232
|
+
});
|
|
233
|
+
throw error;
|
|
205
234
|
}
|
|
206
235
|
finally {
|
|
207
236
|
this.reconciling = false;
|
|
@@ -343,6 +372,9 @@ export class Reconciler extends ReconcilerReviewBase {
|
|
|
343
372
|
});
|
|
344
373
|
}
|
|
345
374
|
async launch(job, task) {
|
|
375
|
+
return withObservabilityCorrelation({ repository: job.repository, prNumber: job.prNumber, jobId: job.id, taskId: task.id }, () => this.launchCorrelated(job, task));
|
|
376
|
+
}
|
|
377
|
+
async launchCorrelated(job, task) {
|
|
346
378
|
const adapter = this.options.adapters.find(x => x.name === task.adapter);
|
|
347
379
|
if (!adapter) {
|
|
348
380
|
await transitionTaskEvent(task, 'fail');
|
|
@@ -436,6 +468,13 @@ export class Reconciler extends ReconcilerReviewBase {
|
|
|
436
468
|
void startAckSetup.ack.finally(() => { void startAckSetup.stop(); }).catch(() => { });
|
|
437
469
|
this.live.set(task.id, { jobId: job.id, taskId: task.id, executionId: durableExecutionId, execution, controller, launchBarrier: Date.now(), workerStarted: false, processSettled: false });
|
|
438
470
|
this.registerExecutionOwner(job.id, task.id, durableExecutionId);
|
|
471
|
+
observability().record({
|
|
472
|
+
stage: 'transition', event: 'task.dispatch', outcome: 'succeeded',
|
|
473
|
+
message: `Task ${task.id} dispatched to ${task.adapter}`,
|
|
474
|
+
correlation: { repository: job.repository, prNumber: job.prNumber, jobId: job.id, taskId: task.id, executionId: durableExecutionId },
|
|
475
|
+
transition: { graph: 'task-lifecycle', from: 'QUEUED', to: 'QUEUED', actions: ['effect.launchTask'] },
|
|
476
|
+
data: { adapter: adapter.id, attempt: task.attempt, model: task.routing?.model ?? null },
|
|
477
|
+
});
|
|
439
478
|
const terminalTimer = setTimeout(() => {
|
|
440
479
|
this.detach(this.timeoutTask(job.id, task.id, durableExecutionId), job.id, task.id, 'worker terminal-event timeout');
|
|
441
480
|
}, Math.max(1, task.timeoutMs));
|
package/dist/relay-store.js
CHANGED
|
@@ -2,6 +2,7 @@ import { DatabaseSync } from 'node:sqlite';
|
|
|
2
2
|
import { dirname } from 'node:path';
|
|
3
3
|
import { mkdirSync } from 'node:fs';
|
|
4
4
|
import { readdir } from 'node:fs/promises';
|
|
5
|
+
import { observability } from './observability.js';
|
|
5
6
|
import { ProjectionOutbox, SCHEMA_VERSION, } from './projection-outbox.js';
|
|
6
7
|
function newerRecord(local, remote) {
|
|
7
8
|
if (!local)
|
|
@@ -117,8 +118,16 @@ export class SQLiteRelayStore {
|
|
|
117
118
|
this.db.exec('BEGIN IMMEDIATE');
|
|
118
119
|
try {
|
|
119
120
|
this.writeJobSnapshot(job, now);
|
|
120
|
-
this.enqueueProjectionInTransaction(request, now);
|
|
121
|
+
const coalesced = this.enqueueProjectionInTransaction(request, now);
|
|
121
122
|
this.db.exec('COMMIT');
|
|
123
|
+
observability().record({
|
|
124
|
+
stage: 'projection', event: 'projection.enqueued', outcome: coalesced ? 'deduplicated' : 'succeeded',
|
|
125
|
+
message: coalesced
|
|
126
|
+
? `Projection ${request.operation} coalesced`
|
|
127
|
+
: `Projection ${request.operation} enqueued`,
|
|
128
|
+
correlation: { repository: job.repository, prNumber: job.prNumber, jobId: job.id },
|
|
129
|
+
data: { operation: request.operation, idempotencyKey: request.idempotencyKey, coalesceKey: request.coalesceKey, priority: request.priority ?? 0, coalesced },
|
|
130
|
+
});
|
|
122
131
|
}
|
|
123
132
|
catch (error) {
|
|
124
133
|
this.db.exec('ROLLBACK');
|
|
@@ -199,14 +208,21 @@ export class SQLiteRelayStore {
|
|
|
199
208
|
return Number(result.changes) === 1;
|
|
200
209
|
}
|
|
201
210
|
enqueueProjectionInTransaction(request, now) {
|
|
202
|
-
this.outbox.enqueue(request, now);
|
|
211
|
+
return this.outbox.enqueue(request, now);
|
|
203
212
|
}
|
|
204
213
|
async enqueueProjection(request) {
|
|
205
214
|
const now = new Date().toISOString();
|
|
206
215
|
this.db.exec('BEGIN IMMEDIATE');
|
|
207
216
|
try {
|
|
208
|
-
this.
|
|
217
|
+
const coalesced = this.enqueueProjectionInTransaction(request, now);
|
|
209
218
|
this.db.exec('COMMIT');
|
|
219
|
+
observability().record({
|
|
220
|
+
stage: 'projection', event: 'projection.enqueued', outcome: coalesced ? 'deduplicated' : 'succeeded',
|
|
221
|
+
message: coalesced
|
|
222
|
+
? `Projection ${request.operation} coalesced`
|
|
223
|
+
: `Projection ${request.operation} enqueued`,
|
|
224
|
+
data: { operation: request.operation, idempotencyKey: request.idempotencyKey, coalesceKey: request.coalesceKey, priority: request.priority ?? 0, coalesced },
|
|
225
|
+
});
|
|
210
226
|
}
|
|
211
227
|
catch (error) {
|
|
212
228
|
this.db.exec('ROLLBACK');
|
package/dist/relayd.js
CHANGED
|
@@ -27,6 +27,13 @@ import { summarizeRuntime } from './runtime-summary.js';
|
|
|
27
27
|
import { executionRouterRulesPath } from './routing-policy.js';
|
|
28
28
|
import { workerCandidates } from './worker-routing.js';
|
|
29
29
|
import { routeExecution } from './execution-routing.js';
|
|
30
|
+
import { RelayObservability, classifyFailure, errorMessage, observability, setObservability } from './observability.js';
|
|
31
|
+
/** Initialize the process-wide structured observability sink once, from environment. */
|
|
32
|
+
export function configureDaemonObservability(env = process.env) {
|
|
33
|
+
const configured = RelayObservability.fromEnvironment(env);
|
|
34
|
+
setObservability(configured);
|
|
35
|
+
return configured;
|
|
36
|
+
}
|
|
30
37
|
export async function overlayOperationalOverviews(overviews, storeFor) {
|
|
31
38
|
return await Promise.all(overviews.map(async (item) => {
|
|
32
39
|
const store = storeFor(item.job.repository, item.job.prNumber);
|
|
@@ -128,6 +135,7 @@ export async function runDaemon(argv) {
|
|
|
128
135
|
console.log(daemonHelp);
|
|
129
136
|
return;
|
|
130
137
|
}
|
|
138
|
+
configureDaemonObservability();
|
|
131
139
|
configureExecutionRouterRulesPath(argv);
|
|
132
140
|
await ensureChatGptBrowserWorkspace();
|
|
133
141
|
const normalized = normalizeDaemonArguments(argv);
|
|
@@ -198,7 +206,7 @@ export async function runDaemon(argv) {
|
|
|
198
206
|
return cachedOverviews;
|
|
199
207
|
}
|
|
200
208
|
catch (error) {
|
|
201
|
-
|
|
209
|
+
observability().record({ stage: 'daemon', event: 'daemon.refresh_failed', level: 'warn', outcome: 'failed', message: `Workspace job refresh unavailable: ${errorMessage(error)}`, data: { classification: classifyFailure(error).classification, reason: errorMessage(error) } });
|
|
202
210
|
return cachedOverviews;
|
|
203
211
|
}
|
|
204
212
|
};
|
|
@@ -364,7 +372,7 @@ export async function runDaemon(argv) {
|
|
|
364
372
|
return { ...refreshed.job, githubState: refreshed.githubState, runtimeStatus: deriveJobRuntimeStatus(refreshed.job, refreshed.githubState), mergedAt: refreshed.mergedAt, closedAt: refreshed.closedAt, draft: refreshed.draft, publication: refreshed.publication };
|
|
365
373
|
}
|
|
366
374
|
catch (error) {
|
|
367
|
-
|
|
375
|
+
observability().record({ stage: 'daemon', event: 'daemon.refresh_failed', level: 'warn', outcome: 'failed', message: `Selected job refresh unavailable: ${errorMessage(error)}`, data: { classification: classifyFailure(error).classification, reason: errorMessage(error) } });
|
|
368
376
|
return cachedJob();
|
|
369
377
|
}
|
|
370
378
|
};
|
|
@@ -448,7 +456,7 @@ export async function runDaemon(argv) {
|
|
|
448
456
|
await pool.reconcile();
|
|
449
457
|
}
|
|
450
458
|
catch (error) {
|
|
451
|
-
|
|
459
|
+
observability().record({ stage: 'daemon', event: 'daemon.reconcile_failed', level: 'warn', outcome: 'failed', message: `Workspace reconcile unavailable: ${errorMessage(error)}`, data: { classification: classifyFailure(error).classification, reason: errorMessage(error) } });
|
|
452
460
|
}
|
|
453
461
|
};
|
|
454
462
|
const scheduleHeartbeatSafely = async () => {
|
|
@@ -458,7 +466,7 @@ export async function runDaemon(argv) {
|
|
|
458
466
|
await pool.scheduleTickSignal(bus, item.job, now);
|
|
459
467
|
}
|
|
460
468
|
catch (error) {
|
|
461
|
-
|
|
469
|
+
observability().record({ stage: 'daemon', event: 'daemon.schedule_heartbeat_failed', level: 'warn', outcome: 'failed', message: `Scheduled heartbeat unavailable: ${errorMessage(error)}`, correlation: { repository: item.job.repository, prNumber: item.job.prNumber, jobId: item.job.id }, data: { classification: classifyFailure(error).classification, reason: errorMessage(error) } });
|
|
462
470
|
}
|
|
463
471
|
}
|
|
464
472
|
};
|
|
@@ -468,7 +476,7 @@ export async function runDaemon(argv) {
|
|
|
468
476
|
await writer.flushOnce();
|
|
469
477
|
}
|
|
470
478
|
catch (error) {
|
|
471
|
-
|
|
479
|
+
observability().record({ stage: 'daemon', event: 'daemon.projection_flush_failed', level: 'warn', outcome: 'failed', message: `GitHub projection flush unavailable: ${errorMessage(error)}`, data: { classification: classifyFailure(error).classification, reason: errorMessage(error) } });
|
|
472
480
|
}
|
|
473
481
|
}
|
|
474
482
|
};
|
|
@@ -495,7 +503,7 @@ export async function runDaemon(argv) {
|
|
|
495
503
|
console.log('Dashboard listening; repository worker pool active');
|
|
496
504
|
void refreshAll(true)
|
|
497
505
|
.then(items => pool.reconcileKnown(items))
|
|
498
|
-
.catch(error =>
|
|
506
|
+
.catch(error => observability().record({ stage: 'daemon', event: 'daemon.startup_refresh_failed', level: 'warn', outcome: 'failed', message: 'Startup remote refresh unavailable: ' + errorMessage(error), data: { classification: classifyFailure(error).classification, reason: errorMessage(error) } }));
|
|
499
507
|
}
|
|
500
508
|
if (isEntrypoint(import.meta.url))
|
|
501
509
|
runDaemon(process.argv.slice(2)).catch(error => { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; });
|
package/dist/scheduler.js
CHANGED
|
@@ -3,6 +3,7 @@ import { dispatchJobLifecycle, dispatchTaskLifecycle, dispatchTaskLifecycleSync
|
|
|
3
3
|
import { reconcileTaskDependencies } from './workflow/task-reconcile.js';
|
|
4
4
|
export { dependencyReady, detectCycles } from './workflow/task-reconcile.js';
|
|
5
5
|
import { dispatchScheduleCycle } from './workflow/schedule-cycle-v2.js';
|
|
6
|
+
import { observability } from './observability.js';
|
|
6
7
|
export function completionGateSatisfied(job) {
|
|
7
8
|
if (job.schedule)
|
|
8
9
|
return false;
|
|
@@ -52,14 +53,31 @@ export async function transitionTaskEvent(task, event, context = {}, executeEffe
|
|
|
52
53
|
throw new Error(decision.reason === 'ambiguous'
|
|
53
54
|
? `Ambiguous workflow transition from ${from} on ${event}`
|
|
54
55
|
: `No enabled workflow transition from ${from} on ${event}`);
|
|
56
|
+
observability().record({
|
|
57
|
+
stage: 'transition',
|
|
58
|
+
event,
|
|
59
|
+
outcome: 'succeeded',
|
|
60
|
+
message: `Task ${task.id} ${from} -> ${decision.target}`,
|
|
61
|
+
correlation: { jobId: task.jobId, taskId: task.id, executionId: task.executionId ?? undefined },
|
|
62
|
+
transition: { graph: 'task-lifecycle', from, to: decision.target, actions: decision.actions },
|
|
63
|
+
});
|
|
55
64
|
return decision;
|
|
56
65
|
}
|
|
57
66
|
export async function transitionJobEvent(job, event, context = {}, executeEffect, persist) {
|
|
67
|
+
const from = job.state;
|
|
58
68
|
const decision = await dispatchJobLifecycle(job, event, context, executeEffect, persist);
|
|
59
69
|
if (!decision.accepted || !decision.target)
|
|
60
70
|
throw new Error(decision.reason === 'ambiguous'
|
|
61
|
-
? `Ambiguous workflow transition from ${
|
|
62
|
-
: `No enabled workflow transition from ${
|
|
71
|
+
? `Ambiguous workflow transition from ${from} on ${event}`
|
|
72
|
+
: `No enabled workflow transition from ${from} on ${event}`);
|
|
73
|
+
observability().record({
|
|
74
|
+
stage: 'transition',
|
|
75
|
+
event,
|
|
76
|
+
outcome: 'succeeded',
|
|
77
|
+
message: `Job ${job.id} ${from} -> ${decision.target}`,
|
|
78
|
+
correlation: { repository: job.repository, prNumber: job.prNumber, jobId: job.id },
|
|
79
|
+
transition: { graph: 'job-lifecycle', from, to: decision.target, actions: decision.actions },
|
|
80
|
+
});
|
|
63
81
|
return decision;
|
|
64
82
|
}
|
|
65
83
|
export const SCHEDULE_RESOLUTION_MS = 600_000;
|
|
@@ -13,8 +13,8 @@ export const authoritativeUnblockV2 = new GraphRuntime(source, {
|
|
|
13
13
|
events: { 'task.authoritative_unblock': { id: 'task.authoritative_unblock' } },
|
|
14
14
|
});
|
|
15
15
|
export async function dispatchAuthoritativeUnblock(autonomousJob, executeAction) {
|
|
16
|
-
return dispatchWorkflowGraph(authoritativeUnblockV2, { initialState: 'blocked', input: { transport: 'event', id: 'task.authoritative_unblock', payload: {} }, durableFacts: { autonomousJob }, executeAction });
|
|
16
|
+
return dispatchWorkflowGraph(authoritativeUnblockV2, { initialState: 'blocked', input: { transport: 'event', id: 'task.authoritative_unblock', payload: {} }, graphName: 'authoritative-unblock', durableFacts: { autonomousJob }, executeAction });
|
|
17
17
|
}
|
|
18
18
|
export function decideAuthoritativeUnblock(autonomousJob) {
|
|
19
|
-
return dispatchWorkflowGraphSync(authoritativeUnblockV2, { initialState: 'blocked', input: { transport: 'event', id: 'task.authoritative_unblock', payload: {} }, durableFacts: { autonomousJob } });
|
|
19
|
+
return dispatchWorkflowGraphSync(authoritativeUnblockV2, { initialState: 'blocked', input: { transport: 'event', id: 'task.authoritative_unblock', payload: {} }, graphName: 'authoritative-unblock', durableFacts: { autonomousJob } });
|
|
20
20
|
}
|
|
@@ -26,8 +26,8 @@ export const blockerResolutionV2 = new GraphRuntime(source, {
|
|
|
26
26
|
events: { 'blocker.resolved': { id: 'blocker.resolved' } },
|
|
27
27
|
});
|
|
28
28
|
export async function dispatchBlockerResolution(facts, executeAction) {
|
|
29
|
-
return dispatchWorkflowGraph(blockerResolutionV2, { initialState: 'waiting', input: { transport: 'event', id: 'blocker.resolved', payload: {} }, durableFacts: facts, isTransientState: state => state.startsWith('check_'), maxSteps: 10, executeAction });
|
|
29
|
+
return dispatchWorkflowGraph(blockerResolutionV2, { initialState: 'waiting', input: { transport: 'event', id: 'blocker.resolved', payload: {} }, durableFacts: facts, graphName: 'blocker-resolution', isTransientState: state => state.startsWith('check_'), maxSteps: 10, executeAction });
|
|
30
30
|
}
|
|
31
31
|
export function decideBlockerResolution(facts) {
|
|
32
|
-
return dispatchWorkflowGraphSync(blockerResolutionV2, { initialState: 'waiting', input: { transport: 'event', id: 'blocker.resolved', payload: {} }, durableFacts: facts, isTransientState: state => state.startsWith('check_'), maxSteps: 10 });
|
|
32
|
+
return dispatchWorkflowGraphSync(blockerResolutionV2, { initialState: 'waiting', input: { transport: 'event', id: 'blocker.resolved', payload: {} }, durableFacts: facts, graphName: 'blocker-resolution', isTransientState: state => state.startsWith('check_'), maxSteps: 10 });
|
|
33
33
|
}
|
|
@@ -1,4 +1,54 @@
|
|
|
1
1
|
import { GraphDispatcher } from './graph-v2.js';
|
|
2
|
+
import { observability } from '../observability.js';
|
|
3
|
+
/**
|
|
4
|
+
* Record graph/decision identity and selected transition/action.
|
|
5
|
+
*
|
|
6
|
+
* `no_transition` is emitted at debug so normal idle watchdog cycles stay quiet;
|
|
7
|
+
* accepted transitions and ambiguous paths are visible at info/warn.
|
|
8
|
+
*/
|
|
9
|
+
function observeDecision(graphName, input, decision) {
|
|
10
|
+
if (decision.accepted) {
|
|
11
|
+
const noOp = decision.source === decision.target && decision.actions.length === 0;
|
|
12
|
+
observability().record({
|
|
13
|
+
stage: 'graph',
|
|
14
|
+
event: input.id,
|
|
15
|
+
level: noOp ? 'debug' : undefined,
|
|
16
|
+
outcome: noOp ? 'noop' : 'succeeded',
|
|
17
|
+
message: noOp
|
|
18
|
+
? `Graph ${graphName ?? 'workflow'} accepted no-op ${decision.source} -> ${decision.target}`
|
|
19
|
+
: `Graph ${graphName ?? 'workflow'} accepted ${decision.source} -> ${decision.target}`,
|
|
20
|
+
transition: {
|
|
21
|
+
graph: graphName ?? null,
|
|
22
|
+
from: decision.source,
|
|
23
|
+
to: decision.target,
|
|
24
|
+
actions: decision.actions.map(action => action.id),
|
|
25
|
+
},
|
|
26
|
+
data: { transport: input.transport },
|
|
27
|
+
});
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
30
|
+
if (decision.reason === 'ambiguous') {
|
|
31
|
+
observability().record({
|
|
32
|
+
stage: 'graph',
|
|
33
|
+
event: input.id,
|
|
34
|
+
level: 'warn',
|
|
35
|
+
outcome: 'failed',
|
|
36
|
+
message: `Graph ${graphName ?? 'workflow'} found ambiguous path from ${decision.source}`,
|
|
37
|
+
transition: { graph: graphName ?? null, from: decision.source, to: null },
|
|
38
|
+
data: { transport: input.transport, reason: decision.reason },
|
|
39
|
+
});
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
observability().record({
|
|
43
|
+
stage: 'graph',
|
|
44
|
+
event: input.id,
|
|
45
|
+
level: 'debug',
|
|
46
|
+
outcome: 'noop',
|
|
47
|
+
message: `Graph ${graphName ?? 'workflow'} has no transition from ${decision.source} on ${input.id}`,
|
|
48
|
+
transition: { graph: graphName ?? null, from: decision.source, to: null },
|
|
49
|
+
data: { transport: input.transport, reason: decision.reason },
|
|
50
|
+
});
|
|
51
|
+
}
|
|
2
52
|
/**
|
|
3
53
|
* The single production graph execution boundary.
|
|
4
54
|
* Ingress adapters normalize input/facts, while GraphDispatcher alone evaluates,
|
|
@@ -18,6 +68,8 @@ export async function dispatchWorkflowGraph(runtime, options) {
|
|
|
18
68
|
input: options.input,
|
|
19
69
|
durableFacts: options.durableFacts ?? {},
|
|
20
70
|
});
|
|
71
|
+
if (options.observe !== false)
|
|
72
|
+
observeDecision(options.graphName, options.input, result.decision);
|
|
21
73
|
if (!result.decision.accepted)
|
|
22
74
|
return result.decision;
|
|
23
75
|
current = result.decision.target;
|
|
@@ -39,6 +91,8 @@ export function dispatchWorkflowGraphSync(runtime, options) {
|
|
|
39
91
|
throw new Error('sync workflow publishEvent returned Promise'); },
|
|
40
92
|
});
|
|
41
93
|
const result = dispatcher.dispatchSync({ currentState: current, input: options.input, durableFacts: options.durableFacts ?? {} });
|
|
94
|
+
if (options.observe !== false)
|
|
95
|
+
observeDecision(options.graphName, options.input, result.decision);
|
|
42
96
|
if (!result.decision.accepted)
|
|
43
97
|
return result.decision;
|
|
44
98
|
current = result.decision.target;
|
|
@@ -52,5 +106,12 @@ export async function dispatchWorkflowTargetOverride(runtime, options) {
|
|
|
52
106
|
// It intentionally executes no workflow actions/effects: this is a state repair, not an event replay.
|
|
53
107
|
const { assertV2StateTransition } = await import('./definitions-v2.js');
|
|
54
108
|
assertV2StateTransition(runtime.graph, options.currentState, options.targetState, options.kind);
|
|
109
|
+
observability().record({
|
|
110
|
+
stage: 'graph',
|
|
111
|
+
event: 'target.override',
|
|
112
|
+
outcome: 'succeeded',
|
|
113
|
+
message: `Administrative ${options.kind} state repair ${options.currentState} -> ${options.targetState}`,
|
|
114
|
+
transition: { graph: `${options.kind}.override`, from: options.currentState, to: options.targetState, actions: [] },
|
|
115
|
+
});
|
|
55
116
|
await options.persistTransition(options.currentState, options.targetState);
|
|
56
117
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { tryTransitionTaskEvent } from '../scheduler.js';
|
|
2
|
+
import { observability } from '../observability.js';
|
|
2
3
|
/**
|
|
3
4
|
* Runtime primitive for applying one canonical Task-graph event to fresh durable state.
|
|
4
5
|
*
|
|
@@ -15,14 +16,30 @@ export class DurableTaskTransitionRuntime {
|
|
|
15
16
|
return this.withLock(`${jobId}/${taskId}`, async () => {
|
|
16
17
|
const job = await this.store.load(jobId);
|
|
17
18
|
const task = job.tasks.find(item => item.id === taskId);
|
|
18
|
-
if (!task || (options.eligible && !options.eligible(job, task)))
|
|
19
|
+
if (!task || (options.eligible && !options.eligible(job, task))) {
|
|
20
|
+
observability().record({
|
|
21
|
+
stage: 'transition', event, level: 'debug', outcome: 'skipped',
|
|
22
|
+
message: `Task ${taskId} transition ${event} not eligible`,
|
|
23
|
+
correlation: { repository: job.repository, prNumber: job.prNumber, jobId, taskId },
|
|
24
|
+
transition: { graph: 'task-lifecycle', from: task?.state ?? null, to: null },
|
|
25
|
+
});
|
|
19
26
|
return null;
|
|
27
|
+
}
|
|
20
28
|
const from = task.state;
|
|
21
29
|
const decision = tryTransitionTaskEvent(task, event, options.facts?.(job, task) ?? {});
|
|
22
|
-
if (!decision.accepted || !decision.target)
|
|
30
|
+
if (!decision.accepted || !decision.target) {
|
|
23
31
|
return { job, task, from, decision };
|
|
32
|
+
}
|
|
24
33
|
await options.mutate?.(job, task, decision);
|
|
25
34
|
await this.store.saveTask(task);
|
|
35
|
+
observability().record({
|
|
36
|
+
stage: 'transition',
|
|
37
|
+
event,
|
|
38
|
+
outcome: 'succeeded',
|
|
39
|
+
message: `Task ${taskId} ${from} -> ${decision.target}`,
|
|
40
|
+
correlation: { repository: job.repository, prNumber: job.prNumber, jobId, taskId, executionId: task.executionId ?? undefined },
|
|
41
|
+
transition: { graph: 'task-lifecycle', from, to: decision.target, actions: decision.actions },
|
|
42
|
+
});
|
|
26
43
|
return { job, task, from, decision };
|
|
27
44
|
});
|
|
28
45
|
}
|
package/dist/workflow/effects.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { classifyFailure, observability } from '../observability.js';
|
|
1
2
|
export class MemoryWorkflowEffectReceiptStore {
|
|
2
3
|
completed = new Set();
|
|
3
4
|
async has(key) { return this.completed.has(key); }
|
|
@@ -48,6 +49,7 @@ export class WorkflowEffectExecutor {
|
|
|
48
49
|
if (await this.receipts.has(key)) {
|
|
49
50
|
const outcome = { effect, key, status: 'deduplicated' };
|
|
50
51
|
outcomes.push(outcome);
|
|
52
|
+
this.recordEffect(context, outcome);
|
|
51
53
|
await onOutcome?.(outcome);
|
|
52
54
|
continue;
|
|
53
55
|
}
|
|
@@ -56,6 +58,7 @@ export class WorkflowEffectExecutor {
|
|
|
56
58
|
await this.receipts.markCompleted(key);
|
|
57
59
|
const outcome = { effect, key, status: 'completed' };
|
|
58
60
|
outcomes.push(outcome);
|
|
61
|
+
this.recordEffect(context, outcome);
|
|
59
62
|
await onOutcome?.(outcome);
|
|
60
63
|
}
|
|
61
64
|
catch (error) {
|
|
@@ -66,9 +69,29 @@ export class WorkflowEffectExecutor {
|
|
|
66
69
|
error: error instanceof Error ? error.message : String(error),
|
|
67
70
|
};
|
|
68
71
|
outcomes.push(outcome);
|
|
72
|
+
this.recordEffect(context, outcome, error);
|
|
69
73
|
await onOutcome?.(outcome);
|
|
70
74
|
}
|
|
71
75
|
}
|
|
72
76
|
return outcomes;
|
|
73
77
|
}
|
|
78
|
+
recordEffect(context, outcome, error) {
|
|
79
|
+
observability().record({
|
|
80
|
+
stage: 'transition',
|
|
81
|
+
event: outcome.status === 'failed' ? 'workflow.effect.failed' : 'workflow.effect',
|
|
82
|
+
level: outcome.status === 'failed' ? 'error' : 'info',
|
|
83
|
+
outcome: outcome.status === 'deduplicated' ? 'deduplicated' : outcome.status === 'failed' ? 'failed' : 'succeeded',
|
|
84
|
+
message: outcome.status === 'failed'
|
|
85
|
+
? `Effect ${outcome.effect} failed: ${outcome.error ?? 'unknown error'}`
|
|
86
|
+
: `Effect ${outcome.effect} ${outcome.status}`,
|
|
87
|
+
correlation: { jobId: context.jobId, taskId: context.taskId },
|
|
88
|
+
transition: { graph: 'workflow.effect', from: null, to: null, actions: [outcome.effect] },
|
|
89
|
+
data: {
|
|
90
|
+
effect: outcome.effect,
|
|
91
|
+
idempotencyKey: outcome.key,
|
|
92
|
+
transitionKey: context.transitionKey,
|
|
93
|
+
...(error ? { classification: classifyFailure(error).classification } : {}),
|
|
94
|
+
},
|
|
95
|
+
});
|
|
96
|
+
}
|
|
74
97
|
}
|
|
@@ -3,11 +3,14 @@ import { jobLifecycleV2, taskLifecycleV2 } from './definitions-v2.js';
|
|
|
3
3
|
import { dispatchWorkflowGraph, dispatchWorkflowGraphSync } from './dispatcher.js';
|
|
4
4
|
import { applyJobGraphTarget, applyTaskGraphTarget, applyTaskMutation } from './runtime.js';
|
|
5
5
|
function v2WorkflowDecision(runtime, from, event, facts = {}) {
|
|
6
|
+
// Pure lifecycle predicates still enter through the dispatcher facade, but
|
|
7
|
+
// suppress observability because no durable transition is being applied.
|
|
6
8
|
const actions = [];
|
|
7
9
|
const decision = dispatchWorkflowGraphSync(runtime, {
|
|
8
10
|
initialState: from,
|
|
9
11
|
input: { transport: 'event', id: event, payload: {} },
|
|
10
12
|
durableFacts: facts,
|
|
13
|
+
observe: false,
|
|
11
14
|
executeAction: actionId => { actions.push(actionId); return {}; },
|
|
12
15
|
});
|
|
13
16
|
if (!decision.accepted)
|
|
@@ -34,6 +37,7 @@ export function dispatchTaskLifecycleSync(task, event, facts = {}) {
|
|
|
34
37
|
initialState: task.state,
|
|
35
38
|
input: { transport: 'event', id: event, payload: {} },
|
|
36
39
|
durableFacts: facts,
|
|
40
|
+
graphName: 'task-lifecycle',
|
|
37
41
|
persistTransition: (_source, target) => applyTaskGraphTarget(task, target),
|
|
38
42
|
executeAction: actionId => { actions.push(actionId); if (actionId.startsWith('mutation.'))
|
|
39
43
|
applyTaskMutation(task, actionId); return {}; },
|
|
@@ -46,6 +50,7 @@ export async function dispatchTaskLifecycle(task, event, facts = {}, executeEffe
|
|
|
46
50
|
const actions = [];
|
|
47
51
|
const result = await dispatchWorkflowGraph(taskLifecycleV2, {
|
|
48
52
|
initialState: task.state, input: { transport: 'event', id: event, payload: {} }, durableFacts: facts,
|
|
53
|
+
graphName: 'task-lifecycle',
|
|
49
54
|
persistTransition: async (_source, target) => { applyTaskGraphTarget(task, target); await persist?.(); },
|
|
50
55
|
executeAction: async (actionId) => {
|
|
51
56
|
actions.push(actionId);
|
|
@@ -66,6 +71,7 @@ export function dispatchJobLifecycleSync(job, event, facts = {}) {
|
|
|
66
71
|
initialState: job.state,
|
|
67
72
|
input: { transport: 'event', id: event, payload: {} },
|
|
68
73
|
durableFacts: facts,
|
|
74
|
+
graphName: 'job-lifecycle',
|
|
69
75
|
persistTransition: (_source, target) => applyJobGraphTarget(job, target),
|
|
70
76
|
executeAction: actionId => { actions.push(actionId); return {}; },
|
|
71
77
|
});
|
|
@@ -77,6 +83,7 @@ export async function dispatchJobLifecycle(job, event, facts = {}, executeEffect
|
|
|
77
83
|
const actions = [];
|
|
78
84
|
const result = await dispatchWorkflowGraph(jobLifecycleV2, {
|
|
79
85
|
initialState: job.state, input: { transport: 'event', id: event, payload: {} }, durableFacts: facts,
|
|
86
|
+
graphName: 'job-lifecycle',
|
|
80
87
|
persistTransition: async (_source, target) => { applyJobGraphTarget(job, target); await persist?.(); },
|
|
81
88
|
executeAction: async (actionId) => { actions.push(actionId); await executeEffect?.(actionId); return {}; },
|
|
82
89
|
});
|
|
@@ -27,10 +27,10 @@ export const plannerOrchestrationV2 = new GraphRuntime(source, {
|
|
|
27
27
|
events: { 'planner.reconcile': { id: 'planner.reconcile' } },
|
|
28
28
|
});
|
|
29
29
|
export async function dispatchPlannerOrchestration(facts, executeAction) {
|
|
30
|
-
return dispatchWorkflowGraph(plannerOrchestrationV2, { initialState: 'idle', input: { transport: 'event', id: 'planner.reconcile', payload: {} }, durableFacts: facts, isTransientState: state => state.startsWith('check_'), maxSteps: 9, executeAction });
|
|
30
|
+
return dispatchWorkflowGraph(plannerOrchestrationV2, { initialState: 'idle', input: { transport: 'event', id: 'planner.reconcile', payload: {} }, durableFacts: facts, graphName: 'planner-orchestration', isTransientState: state => state.startsWith('check_'), maxSteps: 9, executeAction });
|
|
31
31
|
}
|
|
32
32
|
export function decidePlannerOrchestration(facts) {
|
|
33
|
-
return dispatchWorkflowGraphSync(plannerOrchestrationV2, { initialState: 'idle', input: { transport: 'event', id: 'planner.reconcile', payload: {} }, durableFacts: facts, isTransientState: state => state.startsWith('check_'), maxSteps: 9 });
|
|
33
|
+
return dispatchWorkflowGraphSync(plannerOrchestrationV2, { initialState: 'idle', input: { transport: 'event', id: 'planner.reconcile', payload: {} }, durableFacts: facts, graphName: 'planner-orchestration', isTransientState: state => state.startsWith('check_'), maxSteps: 9 });
|
|
34
34
|
}
|
|
35
35
|
export function plannerOrchestrationFacts(job, recoverablePendingPlan) {
|
|
36
36
|
const slot = job.schedule?.currentRunSlot ? Date.parse(job.schedule.currentRunSlot) : Number.NaN;
|
|
@@ -16,7 +16,7 @@ export const plannerTriggerV2 = new GraphRuntime(source, {
|
|
|
16
16
|
});
|
|
17
17
|
export function dispatchPlannerTrigger(facts) {
|
|
18
18
|
return dispatchWorkflowGraphSync(plannerTriggerV2, {
|
|
19
|
-
initialState: 'idle', input: { transport: 'event', id: 'plan.requested', payload: {} }, durableFacts: facts,
|
|
19
|
+
initialState: 'idle', input: { transport: 'event', id: 'plan.requested', payload: {} }, durableFacts: facts, graphName: 'planner-trigger',
|
|
20
20
|
isTransientState: state => state.startsWith('check_'), maxSteps: 4,
|
|
21
21
|
});
|
|
22
22
|
}
|
|
@@ -35,8 +35,8 @@ export function reviewOutcome(state) {
|
|
|
35
35
|
return state.toLowerCase();
|
|
36
36
|
}
|
|
37
37
|
export async function dispatchReviewLifecycle(outcome, event, facts = {}, executeAction) {
|
|
38
|
-
return dispatchWorkflowGraph(reviewLifecycleV2, { initialState: reviewGraphState(outcome), input: { transport: 'event', id: event, payload: {} }, durableFacts: facts, executeAction });
|
|
38
|
+
return dispatchWorkflowGraph(reviewLifecycleV2, { initialState: reviewGraphState(outcome), input: { transport: 'event', id: event, payload: {} }, durableFacts: facts, graphName: 'review-lifecycle', executeAction });
|
|
39
39
|
}
|
|
40
40
|
export function decideReviewLifecycle(outcome, event, facts = {}) {
|
|
41
|
-
return dispatchWorkflowGraphSync(reviewLifecycleV2, { initialState: reviewGraphState(outcome), input: { transport: 'event', id: event, payload: {} }, durableFacts: facts });
|
|
41
|
+
return dispatchWorkflowGraphSync(reviewLifecycleV2, { initialState: reviewGraphState(outcome), input: { transport: 'event', id: event, payload: {} }, durableFacts: facts, graphName: 'review-lifecycle' });
|
|
42
42
|
}
|
|
@@ -14,7 +14,7 @@ export const scheduleCycleV2 = new GraphRuntime(source, {
|
|
|
14
14
|
});
|
|
15
15
|
export function dispatchScheduleCycle(facts, onFinish) {
|
|
16
16
|
return dispatchWorkflowGraphSync(scheduleCycleV2, {
|
|
17
|
-
initialState: 'idle', input: { transport: 'event', id: 'schedule.reconcile', payload: {} }, durableFacts: facts,
|
|
17
|
+
initialState: 'idle', input: { transport: 'event', id: 'schedule.reconcile', payload: {} }, durableFacts: facts, graphName: 'schedule-cycle',
|
|
18
18
|
isTransientState: state => state.startsWith('check_'), maxSteps: 2,
|
|
19
19
|
executeAction: actionId => { if (actionId === 'schedule.finishSlot')
|
|
20
20
|
onFinish();
|
|
@@ -23,8 +23,8 @@ export const scheduledRunV2 = new GraphRuntime(source, {
|
|
|
23
23
|
events: { 'schedule.tick': { id: 'schedule.tick' } },
|
|
24
24
|
});
|
|
25
25
|
export async function dispatchScheduledRun(facts, executeAction) {
|
|
26
|
-
return dispatchWorkflowGraph(scheduledRunV2, { initialState: 'idle', input: { transport: 'event', id: 'schedule.tick', payload: {} }, durableFacts: facts, isTransientState: state => state.startsWith('check_'), maxSteps: 8, executeAction });
|
|
26
|
+
return dispatchWorkflowGraph(scheduledRunV2, { initialState: 'idle', input: { transport: 'event', id: 'schedule.tick', payload: {} }, durableFacts: facts, graphName: 'scheduled-run', isTransientState: state => state.startsWith('check_'), maxSteps: 8, executeAction });
|
|
27
27
|
}
|
|
28
28
|
export function decideScheduledRun(facts) {
|
|
29
|
-
return dispatchWorkflowGraphSync(scheduledRunV2, { initialState: 'idle', input: { transport: 'event', id: 'schedule.tick', payload: {} }, durableFacts: facts, isTransientState: state => state.startsWith('check_'), maxSteps: 8 });
|
|
29
|
+
return dispatchWorkflowGraphSync(scheduledRunV2, { initialState: 'idle', input: { transport: 'event', id: 'schedule.tick', payload: {} }, durableFacts: facts, graphName: 'scheduled-run', isTransientState: state => state.startsWith('check_'), maxSteps: 8 });
|
|
30
30
|
}
|
|
@@ -36,6 +36,7 @@ export function decideRelatedTaskAction(task, candidate, actionId) {
|
|
|
36
36
|
initialState: 'evaluate',
|
|
37
37
|
input: { transport: 'action', id: actionId, payload: {} },
|
|
38
38
|
durableFacts: relatedTaskActionFacts(task, candidate),
|
|
39
|
+
graphName: 'task-related-action',
|
|
39
40
|
isTransientState: state => state === 'check_other' || state === 'check_recovery',
|
|
40
41
|
maxSteps: 3,
|
|
41
42
|
});
|
|
@@ -18,7 +18,7 @@ export const troubleshooterV2 = new GraphRuntime(source, {
|
|
|
18
18
|
});
|
|
19
19
|
export function dispatchTroubleshooter(facts, onCreate) {
|
|
20
20
|
return dispatchWorkflowGraphSync(troubleshooterV2, {
|
|
21
|
-
initialState: 'idle', input: { transport: 'event', id: 'troubleshooter.reconcile', payload: {} }, durableFacts: facts,
|
|
21
|
+
initialState: 'idle', input: { transport: 'event', id: 'troubleshooter.reconcile', payload: {} }, durableFacts: facts, graphName: 'troubleshooter',
|
|
22
22
|
isTransientState: state => state.startsWith('check_'), maxSteps: 6,
|
|
23
23
|
executeAction: actionId => { if (actionId === 'troubleshooter.create')
|
|
24
24
|
onCreate();
|