@femtomc/mu-server 26.2.70 → 26.2.72
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/activities.d.ts +2 -0
- package/dist/api/activities.js +160 -0
- package/dist/api/config.d.ts +2 -0
- package/dist/api/config.js +45 -0
- package/dist/api/control_plane.d.ts +2 -0
- package/dist/api/control_plane.js +28 -0
- package/dist/api/cron.d.ts +2 -0
- package/dist/api/cron.js +182 -0
- package/dist/api/heartbeats.d.ts +2 -0
- package/dist/api/heartbeats.js +211 -0
- package/dist/api/identities.d.ts +2 -0
- package/dist/api/identities.js +103 -0
- package/dist/api/runs.d.ts +2 -0
- package/dist/api/runs.js +207 -0
- package/dist/cli.js +58 -3
- package/dist/config.d.ts +4 -21
- package/dist/config.js +24 -75
- package/dist/control_plane.d.ts +4 -2
- package/dist/control_plane.js +226 -25
- package/dist/control_plane_bootstrap_helpers.d.ts +2 -1
- package/dist/control_plane_bootstrap_helpers.js +11 -1
- package/dist/control_plane_contract.d.ts +57 -0
- package/dist/control_plane_contract.js +1 -1
- package/dist/control_plane_reload.d.ts +63 -0
- package/dist/control_plane_reload.js +525 -0
- package/dist/control_plane_run_queue_coordinator.d.ts +48 -0
- package/dist/control_plane_run_queue_coordinator.js +327 -0
- package/dist/control_plane_telegram_generation.js +0 -1
- package/dist/control_plane_wake_delivery.d.ts +50 -0
- package/dist/control_plane_wake_delivery.js +123 -0
- package/dist/index.d.ts +4 -1
- package/dist/index.js +2 -0
- package/dist/run_queue.d.ts +95 -0
- package/dist/run_queue.js +817 -0
- package/dist/run_supervisor.d.ts +20 -0
- package/dist/run_supervisor.js +25 -1
- package/dist/server.d.ts +5 -10
- package/dist/server.js +337 -528
- package/dist/server_program_orchestration.js +2 -0
- package/dist/server_routing.d.ts +3 -2
- package/dist/server_routing.js +28 -900
- package/package.json +7 -6
package/dist/control_plane.js
CHANGED
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
import { ControlPlaneCommandPipeline, ControlPlaneOutbox, ControlPlaneRuntime, DiscordControlPlaneAdapter, getControlPlanePaths, SlackControlPlaneAdapter, TelegramControlPlaneAdapterSpec, } from "@femtomc/mu-control-plane";
|
|
2
2
|
import { DEFAULT_MU_CONFIG } from "./config.js";
|
|
3
|
+
import { DEFAULT_INTER_ROOT_QUEUE_POLICY, normalizeInterRootQueuePolicy, } from "./control_plane_contract.js";
|
|
3
4
|
import { ControlPlaneRunSupervisor, } from "./run_supervisor.js";
|
|
4
|
-
import {
|
|
5
|
+
import { DurableRunQueue, queueStatesForRunStatusFilter, runSnapshotFromQueueSnapshot } from "./run_queue.js";
|
|
6
|
+
import { buildMessagingOperatorRuntime, createOutboxDrainLoop } from "./control_plane_bootstrap_helpers.js";
|
|
7
|
+
import { ControlPlaneRunQueueCoordinator } from "./control_plane_run_queue_coordinator.js";
|
|
5
8
|
import { enqueueRunEventOutbox } from "./control_plane_run_outbox.js";
|
|
9
|
+
import { buildWakeOutboundEnvelope, resolveWakeFanoutCapability, wakeDeliveryMetadataFromOutboxRecord, wakeDispatchReasonCode, wakeFanoutDedupeKey, } from "./control_plane_wake_delivery.js";
|
|
6
10
|
import { TelegramAdapterGenerationManager } from "./control_plane_telegram_generation.js";
|
|
7
11
|
function generationTags(generation, component) {
|
|
8
12
|
return {
|
|
@@ -12,6 +16,25 @@ function generationTags(generation, component) {
|
|
|
12
16
|
component,
|
|
13
17
|
};
|
|
14
18
|
}
|
|
19
|
+
const WAKE_OUTBOX_MAX_ATTEMPTS = 6;
|
|
20
|
+
function emptyNotifyOperatorsResult() {
|
|
21
|
+
return {
|
|
22
|
+
queued: 0,
|
|
23
|
+
duplicate: 0,
|
|
24
|
+
skipped: 0,
|
|
25
|
+
decisions: [],
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
function normalizeIssueId(value) {
|
|
29
|
+
if (!value) {
|
|
30
|
+
return null;
|
|
31
|
+
}
|
|
32
|
+
const trimmed = value.trim();
|
|
33
|
+
if (!/^mu-[a-z0-9][a-z0-9-]*$/i.test(trimmed)) {
|
|
34
|
+
return null;
|
|
35
|
+
}
|
|
36
|
+
return trimmed.toLowerCase();
|
|
37
|
+
}
|
|
15
38
|
export function detectAdapters(config) {
|
|
16
39
|
const adapters = [];
|
|
17
40
|
const slackSecret = config.adapters.slack.signing_secret;
|
|
@@ -121,6 +144,8 @@ export async function bootstrapControlPlane(opts) {
|
|
|
121
144
|
let pipeline = null;
|
|
122
145
|
let runSupervisor = null;
|
|
123
146
|
let outboxDrainLoop = null;
|
|
147
|
+
let wakeDeliveryObserver = opts.wakeDeliveryObserver ?? null;
|
|
148
|
+
const outboundDeliveryChannels = new Set(["telegram"]);
|
|
124
149
|
const adapterMap = new Map();
|
|
125
150
|
try {
|
|
126
151
|
await runtime.start();
|
|
@@ -136,12 +161,20 @@ export async function bootstrapControlPlane(opts) {
|
|
|
136
161
|
});
|
|
137
162
|
await outbox.load();
|
|
138
163
|
let scheduleOutboxDrainRef = null;
|
|
164
|
+
const runQueue = new DurableRunQueue({ repoRoot: opts.repoRoot });
|
|
165
|
+
const interRootQueuePolicy = normalizeInterRootQueuePolicy(opts.interRootQueuePolicy ?? DEFAULT_INTER_ROOT_QUEUE_POLICY);
|
|
166
|
+
const runQueueCoordinator = new ControlPlaneRunQueueCoordinator({
|
|
167
|
+
runQueue,
|
|
168
|
+
interRootQueuePolicy,
|
|
169
|
+
getRunSupervisor: () => runSupervisor,
|
|
170
|
+
});
|
|
139
171
|
runSupervisor = new ControlPlaneRunSupervisor({
|
|
140
172
|
repoRoot: opts.repoRoot,
|
|
141
173
|
heartbeatScheduler: opts.heartbeatScheduler,
|
|
142
174
|
heartbeatIntervalMs: opts.runSupervisorHeartbeatIntervalMs,
|
|
143
175
|
spawnProcess: opts.runSupervisorSpawnProcess,
|
|
144
176
|
onEvent: async (event) => {
|
|
177
|
+
await runQueueCoordinator.onRunEvent(event);
|
|
145
178
|
const outboxRecord = await enqueueRunEventOutbox({
|
|
146
179
|
outbox,
|
|
147
180
|
event,
|
|
@@ -152,6 +185,7 @@ export async function bootstrapControlPlane(opts) {
|
|
|
152
185
|
}
|
|
153
186
|
},
|
|
154
187
|
});
|
|
188
|
+
await runQueueCoordinator.scheduleReconcile("bootstrap");
|
|
155
189
|
pipeline = new ControlPlaneCommandPipeline({
|
|
156
190
|
runtime,
|
|
157
191
|
operator,
|
|
@@ -247,10 +281,7 @@ export async function bootstrapControlPlane(opts) {
|
|
|
247
281
|
}
|
|
248
282
|
if (record.target_type === "run start" || record.target_type === "run resume") {
|
|
249
283
|
try {
|
|
250
|
-
const launched = await
|
|
251
|
-
if (!launched) {
|
|
252
|
-
return null;
|
|
253
|
-
}
|
|
284
|
+
const launched = await runQueueCoordinator.launchQueuedRunFromCommand(record);
|
|
254
285
|
return {
|
|
255
286
|
terminalState: "completed",
|
|
256
287
|
result: {
|
|
@@ -274,6 +305,8 @@ export async function bootstrapControlPlane(opts) {
|
|
|
274
305
|
run_mode: launched.mode,
|
|
275
306
|
run_root_id: launched.root_issue_id,
|
|
276
307
|
run_source: launched.source,
|
|
308
|
+
queue_id: launched.queue_id ?? null,
|
|
309
|
+
queue_state: launched.queue_state ?? null,
|
|
277
310
|
},
|
|
278
311
|
},
|
|
279
312
|
],
|
|
@@ -282,7 +315,7 @@ export async function bootstrapControlPlane(opts) {
|
|
|
282
315
|
catch (err) {
|
|
283
316
|
return {
|
|
284
317
|
terminalState: "failed",
|
|
285
|
-
errorCode: err instanceof Error && err.message ? err.message : "
|
|
318
|
+
errorCode: err instanceof Error && err.message ? err.message : "run_queue_start_failed",
|
|
286
319
|
trace: {
|
|
287
320
|
cliCommandKind: record.target_type.replaceAll(" ", "_"),
|
|
288
321
|
runRootId: record.target_id,
|
|
@@ -291,9 +324,9 @@ export async function bootstrapControlPlane(opts) {
|
|
|
291
324
|
}
|
|
292
325
|
}
|
|
293
326
|
if (record.target_type === "run interrupt") {
|
|
294
|
-
const result =
|
|
327
|
+
const result = await runQueueCoordinator.interruptQueuedRun({
|
|
295
328
|
rootIssueId: record.target_id,
|
|
296
|
-
})
|
|
329
|
+
});
|
|
297
330
|
if (!result.ok) {
|
|
298
331
|
return {
|
|
299
332
|
terminalState: "failed",
|
|
@@ -412,6 +445,104 @@ export async function bootstrapControlPlane(opts) {
|
|
|
412
445
|
},
|
|
413
446
|
isActive: () => telegramManager.hasActiveGeneration(),
|
|
414
447
|
});
|
|
448
|
+
const notifyOperators = async (notifyOpts) => {
|
|
449
|
+
if (!pipeline) {
|
|
450
|
+
return emptyNotifyOperatorsResult();
|
|
451
|
+
}
|
|
452
|
+
const message = notifyOpts.message.trim();
|
|
453
|
+
const dedupeKey = notifyOpts.dedupeKey.trim();
|
|
454
|
+
if (!message || !dedupeKey) {
|
|
455
|
+
return emptyNotifyOperatorsResult();
|
|
456
|
+
}
|
|
457
|
+
const wakeSource = typeof notifyOpts.wake?.wakeSource === "string" ? notifyOpts.wake.wakeSource.trim() : "";
|
|
458
|
+
const wakeProgramId = typeof notifyOpts.wake?.programId === "string" ? notifyOpts.wake.programId.trim() : "";
|
|
459
|
+
const wakeSourceTsMsRaw = notifyOpts.wake?.sourceTsMs;
|
|
460
|
+
const wakeSourceTsMs = typeof wakeSourceTsMsRaw === "number" && Number.isFinite(wakeSourceTsMsRaw)
|
|
461
|
+
? Math.trunc(wakeSourceTsMsRaw)
|
|
462
|
+
: null;
|
|
463
|
+
const wakeId = typeof notifyOpts.wake?.wakeId === "string" && notifyOpts.wake.wakeId.trim().length > 0
|
|
464
|
+
? notifyOpts.wake.wakeId.trim()
|
|
465
|
+
: `wake-${(() => {
|
|
466
|
+
const hasher = new Bun.CryptoHasher("sha256");
|
|
467
|
+
hasher.update(`${dedupeKey}:${message}`);
|
|
468
|
+
return hasher.digest("hex").slice(0, 16);
|
|
469
|
+
})()}`;
|
|
470
|
+
const context = {
|
|
471
|
+
wakeId,
|
|
472
|
+
dedupeKey,
|
|
473
|
+
wakeSource: wakeSource || null,
|
|
474
|
+
programId: wakeProgramId || null,
|
|
475
|
+
sourceTsMs: wakeSourceTsMs,
|
|
476
|
+
};
|
|
477
|
+
const nowMs = Math.trunc(Date.now());
|
|
478
|
+
const telegramBotToken = telegramManager.activeBotToken();
|
|
479
|
+
const bindings = pipeline.identities
|
|
480
|
+
.listBindings({ includeInactive: false })
|
|
481
|
+
.filter((binding) => binding.scopes.includes("cp.ops.admin"));
|
|
482
|
+
const result = emptyNotifyOperatorsResult();
|
|
483
|
+
for (const binding of bindings) {
|
|
484
|
+
const bindingDedupeKey = wakeFanoutDedupeKey({
|
|
485
|
+
dedupeKey,
|
|
486
|
+
wakeId,
|
|
487
|
+
binding,
|
|
488
|
+
});
|
|
489
|
+
const capability = resolveWakeFanoutCapability({
|
|
490
|
+
binding,
|
|
491
|
+
isChannelDeliverySupported: (channel) => outboundDeliveryChannels.has(channel),
|
|
492
|
+
telegramBotToken,
|
|
493
|
+
});
|
|
494
|
+
if (!capability.ok) {
|
|
495
|
+
result.skipped += 1;
|
|
496
|
+
result.decisions.push({
|
|
497
|
+
state: "skipped",
|
|
498
|
+
reason_code: capability.reasonCode,
|
|
499
|
+
binding_id: binding.binding_id,
|
|
500
|
+
channel: binding.channel,
|
|
501
|
+
dedupe_key: bindingDedupeKey,
|
|
502
|
+
outbox_id: null,
|
|
503
|
+
});
|
|
504
|
+
continue;
|
|
505
|
+
}
|
|
506
|
+
const envelope = buildWakeOutboundEnvelope({
|
|
507
|
+
repoRoot: opts.repoRoot,
|
|
508
|
+
nowMs,
|
|
509
|
+
message,
|
|
510
|
+
binding,
|
|
511
|
+
context,
|
|
512
|
+
metadata: notifyOpts.metadata,
|
|
513
|
+
});
|
|
514
|
+
const enqueueDecision = await outbox.enqueue({
|
|
515
|
+
dedupeKey: bindingDedupeKey,
|
|
516
|
+
envelope,
|
|
517
|
+
nowMs,
|
|
518
|
+
maxAttempts: WAKE_OUTBOX_MAX_ATTEMPTS,
|
|
519
|
+
});
|
|
520
|
+
if (enqueueDecision.kind === "enqueued") {
|
|
521
|
+
result.queued += 1;
|
|
522
|
+
scheduleOutboxDrainRef?.();
|
|
523
|
+
result.decisions.push({
|
|
524
|
+
state: "queued",
|
|
525
|
+
reason_code: "outbox_enqueued",
|
|
526
|
+
binding_id: binding.binding_id,
|
|
527
|
+
channel: binding.channel,
|
|
528
|
+
dedupe_key: bindingDedupeKey,
|
|
529
|
+
outbox_id: enqueueDecision.record.outbox_id,
|
|
530
|
+
});
|
|
531
|
+
}
|
|
532
|
+
else {
|
|
533
|
+
result.duplicate += 1;
|
|
534
|
+
result.decisions.push({
|
|
535
|
+
state: "duplicate",
|
|
536
|
+
reason_code: "outbox_duplicate",
|
|
537
|
+
binding_id: binding.binding_id,
|
|
538
|
+
channel: binding.channel,
|
|
539
|
+
dedupe_key: bindingDedupeKey,
|
|
540
|
+
outbox_id: enqueueDecision.record.outbox_id,
|
|
541
|
+
});
|
|
542
|
+
}
|
|
543
|
+
}
|
|
544
|
+
return result;
|
|
545
|
+
};
|
|
415
546
|
const deliver = async (record) => {
|
|
416
547
|
const { envelope } = record;
|
|
417
548
|
if (envelope.channel === "telegram") {
|
|
@@ -454,7 +585,35 @@ export async function bootstrapControlPlane(opts) {
|
|
|
454
585
|
}
|
|
455
586
|
return undefined;
|
|
456
587
|
};
|
|
457
|
-
const outboxDrain = createOutboxDrainLoop({
|
|
588
|
+
const outboxDrain = createOutboxDrainLoop({
|
|
589
|
+
outbox,
|
|
590
|
+
deliver,
|
|
591
|
+
onOutcome: async (outcome) => {
|
|
592
|
+
if (!wakeDeliveryObserver) {
|
|
593
|
+
return;
|
|
594
|
+
}
|
|
595
|
+
const metadata = wakeDeliveryMetadataFromOutboxRecord(outcome.record);
|
|
596
|
+
if (!metadata) {
|
|
597
|
+
return;
|
|
598
|
+
}
|
|
599
|
+
const state = outcome.kind === "delivered" ? "delivered" : outcome.kind === "retried" ? "retried" : "dead_letter";
|
|
600
|
+
await wakeDeliveryObserver({
|
|
601
|
+
state,
|
|
602
|
+
reason_code: wakeDispatchReasonCode({
|
|
603
|
+
state,
|
|
604
|
+
lastError: outcome.record.last_error,
|
|
605
|
+
deadLetterReason: outcome.record.dead_letter_reason,
|
|
606
|
+
}),
|
|
607
|
+
wake_id: metadata.wakeId,
|
|
608
|
+
dedupe_key: metadata.wakeDedupeKey,
|
|
609
|
+
binding_id: metadata.bindingId,
|
|
610
|
+
channel: metadata.channel,
|
|
611
|
+
outbox_id: metadata.outboxId,
|
|
612
|
+
outbox_dedupe_key: metadata.outboxDedupeKey,
|
|
613
|
+
attempt_count: outcome.record.attempt_count,
|
|
614
|
+
});
|
|
615
|
+
},
|
|
616
|
+
});
|
|
458
617
|
const scheduleOutboxDrain = outboxDrain.scheduleOutboxDrain;
|
|
459
618
|
scheduleOutboxDrainRef = scheduleOutboxDrain;
|
|
460
619
|
outboxDrainLoop = outboxDrain;
|
|
@@ -472,6 +631,12 @@ export async function bootstrapControlPlane(opts) {
|
|
|
472
631
|
}
|
|
473
632
|
return result.response;
|
|
474
633
|
},
|
|
634
|
+
async notifyOperators(notifyOpts) {
|
|
635
|
+
return await notifyOperators(notifyOpts);
|
|
636
|
+
},
|
|
637
|
+
setWakeDeliveryObserver(observer) {
|
|
638
|
+
wakeDeliveryObserver = observer;
|
|
639
|
+
},
|
|
475
640
|
async reloadTelegramGeneration(reloadOpts) {
|
|
476
641
|
const result = await telegramManager.reload({
|
|
477
642
|
config: reloadOpts.config,
|
|
@@ -483,41 +648,74 @@ export async function bootstrapControlPlane(opts) {
|
|
|
483
648
|
return result;
|
|
484
649
|
},
|
|
485
650
|
async listRuns(opts = {}) {
|
|
486
|
-
|
|
651
|
+
const limit = Math.max(1, Math.min(500, Math.trunc(opts.limit ?? 100)));
|
|
652
|
+
const fallbackStatusFilter = queueStatesForRunStatusFilter(opts.status);
|
|
653
|
+
if (Array.isArray(fallbackStatusFilter) && fallbackStatusFilter.length === 0) {
|
|
654
|
+
return [];
|
|
655
|
+
}
|
|
656
|
+
const queued = await runQueue.listRunSnapshots({
|
|
487
657
|
status: opts.status,
|
|
488
|
-
limit
|
|
489
|
-
|
|
658
|
+
limit,
|
|
659
|
+
runtimeByJobId: runQueueCoordinator.runtimeSnapshotsByJobId(),
|
|
660
|
+
});
|
|
661
|
+
const seen = new Set(queued.map((run) => run.job_id));
|
|
662
|
+
const fallbackRuns = runSupervisor?.list({ limit: 500 }) ?? [];
|
|
663
|
+
for (const run of fallbackRuns) {
|
|
664
|
+
if (seen.has(run.job_id)) {
|
|
665
|
+
continue;
|
|
666
|
+
}
|
|
667
|
+
if (fallbackStatusFilter && fallbackStatusFilter.length > 0) {
|
|
668
|
+
const mapped = run.status === "completed"
|
|
669
|
+
? "done"
|
|
670
|
+
: run.status === "failed"
|
|
671
|
+
? "failed"
|
|
672
|
+
: run.status === "cancelled"
|
|
673
|
+
? "cancelled"
|
|
674
|
+
: "active";
|
|
675
|
+
if (!fallbackStatusFilter.includes(mapped)) {
|
|
676
|
+
continue;
|
|
677
|
+
}
|
|
678
|
+
}
|
|
679
|
+
queued.push(run);
|
|
680
|
+
seen.add(run.job_id);
|
|
681
|
+
}
|
|
682
|
+
return queued.slice(0, limit);
|
|
490
683
|
},
|
|
491
684
|
async getRun(idOrRoot) {
|
|
685
|
+
const queued = await runQueue.get(idOrRoot);
|
|
686
|
+
if (queued) {
|
|
687
|
+
const runtime = queued.job_id ? (runSupervisor?.get(queued.job_id) ?? null) : null;
|
|
688
|
+
return runSnapshotFromQueueSnapshot(queued, runtime);
|
|
689
|
+
}
|
|
492
690
|
return runSupervisor?.get(idOrRoot) ?? null;
|
|
493
691
|
},
|
|
494
692
|
async startRun(startOpts) {
|
|
495
|
-
|
|
693
|
+
return await runQueueCoordinator.launchQueuedRun({
|
|
694
|
+
mode: "run_start",
|
|
496
695
|
prompt: startOpts.prompt,
|
|
497
696
|
maxSteps: startOpts.maxSteps,
|
|
498
697
|
source: "api",
|
|
698
|
+
dedupeKey: `api:run_start:${crypto.randomUUID()}`,
|
|
499
699
|
});
|
|
500
|
-
if (!run) {
|
|
501
|
-
throw new Error("run_supervisor_unavailable");
|
|
502
|
-
}
|
|
503
|
-
return run;
|
|
504
700
|
},
|
|
505
701
|
async resumeRun(resumeOpts) {
|
|
506
|
-
const
|
|
507
|
-
|
|
702
|
+
const rootIssueId = normalizeIssueId(resumeOpts.rootIssueId);
|
|
703
|
+
if (!rootIssueId) {
|
|
704
|
+
throw new Error("run_resume_invalid_root_issue_id");
|
|
705
|
+
}
|
|
706
|
+
return await runQueueCoordinator.launchQueuedRun({
|
|
707
|
+
mode: "run_resume",
|
|
708
|
+
rootIssueId,
|
|
508
709
|
maxSteps: resumeOpts.maxSteps,
|
|
509
710
|
source: "api",
|
|
711
|
+
dedupeKey: `api:run_resume:${rootIssueId}:${crypto.randomUUID()}`,
|
|
510
712
|
});
|
|
511
|
-
if (!run) {
|
|
512
|
-
throw new Error("run_supervisor_unavailable");
|
|
513
|
-
}
|
|
514
|
-
return run;
|
|
515
713
|
},
|
|
516
714
|
async interruptRun(interruptOpts) {
|
|
517
|
-
return
|
|
715
|
+
return await runQueueCoordinator.interruptQueuedRun(interruptOpts);
|
|
518
716
|
},
|
|
519
717
|
async heartbeatRun(heartbeatOpts) {
|
|
520
|
-
return
|
|
718
|
+
return await runQueueCoordinator.heartbeatQueuedRun(heartbeatOpts);
|
|
521
719
|
},
|
|
522
720
|
async traceRun(traceOpts) {
|
|
523
721
|
return (await runSupervisor?.trace(traceOpts.idOrRoot, { limit: traceOpts.limit })) ?? null;
|
|
@@ -529,6 +727,8 @@ export async function bootstrapControlPlane(opts) {
|
|
|
529
727
|
return await pipeline.handleTerminalInbound(terminalOpts);
|
|
530
728
|
},
|
|
531
729
|
async stop() {
|
|
730
|
+
wakeDeliveryObserver = null;
|
|
731
|
+
runQueueCoordinator.stop();
|
|
532
732
|
if (outboxDrainLoop) {
|
|
533
733
|
outboxDrainLoop.stop();
|
|
534
734
|
outboxDrainLoop = null;
|
|
@@ -552,6 +752,7 @@ export async function bootstrapControlPlane(opts) {
|
|
|
552
752
|
};
|
|
553
753
|
}
|
|
554
754
|
catch (err) {
|
|
755
|
+
wakeDeliveryObserver = null;
|
|
555
756
|
if (outboxDrainLoop) {
|
|
556
757
|
outboxDrainLoop.stop();
|
|
557
758
|
outboxDrainLoop = null;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type MessagingOperatorBackend, MessagingOperatorRuntime } from "@femtomc/mu-agent";
|
|
2
|
-
import { ControlPlaneOutbox, type OutboxDeliveryHandlerResult, type OutboxRecord } from "@femtomc/mu-control-plane";
|
|
2
|
+
import { ControlPlaneOutbox, type OutboxDeliveryHandlerResult, type OutboxDispatchOutcome, type OutboxRecord } from "@femtomc/mu-control-plane";
|
|
3
3
|
import type { ControlPlaneConfig } from "./control_plane_contract.js";
|
|
4
4
|
export declare function buildMessagingOperatorRuntime(opts: {
|
|
5
5
|
repoRoot: string;
|
|
@@ -9,6 +9,7 @@ export declare function buildMessagingOperatorRuntime(opts: {
|
|
|
9
9
|
export declare function createOutboxDrainLoop(opts: {
|
|
10
10
|
outbox: ControlPlaneOutbox;
|
|
11
11
|
deliver: (record: OutboxRecord) => Promise<undefined | OutboxDeliveryHandlerResult>;
|
|
12
|
+
onOutcome?: (outcome: OutboxDispatchOutcome) => void | Promise<void>;
|
|
12
13
|
}): {
|
|
13
14
|
scheduleOutboxDrain: () => void;
|
|
14
15
|
stop: () => void;
|
|
@@ -40,7 +40,17 @@ export function createOutboxDrainLoop(opts) {
|
|
|
40
40
|
try {
|
|
41
41
|
do {
|
|
42
42
|
drainRequested = false;
|
|
43
|
-
await dispatcher.drainDue();
|
|
43
|
+
const outcomes = await dispatcher.drainDue();
|
|
44
|
+
if (opts.onOutcome) {
|
|
45
|
+
for (const outcome of outcomes) {
|
|
46
|
+
try {
|
|
47
|
+
await opts.onOutcome(outcome);
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
// Keep telemetry callbacks non-fatal.
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
44
54
|
} while (drainRequested && !stopped);
|
|
45
55
|
}
|
|
46
56
|
catch {
|
|
@@ -9,6 +9,14 @@ import type { MuConfig } from "./config.js";
|
|
|
9
9
|
* - Interface adapters in `control_plane.ts` implement these seams.
|
|
10
10
|
*/
|
|
11
11
|
export type ControlPlaneConfig = MuConfig["control_plane"];
|
|
12
|
+
/**
|
|
13
|
+
* Durable orchestration queue contract (default-on path).
|
|
14
|
+
*
|
|
15
|
+
* Scheduler policy/state-machine primitives are owned by `@femtomc/mu-orchestrator` and
|
|
16
|
+
* re-exported here so existing server/control-plane callers keep a stable import surface.
|
|
17
|
+
*/
|
|
18
|
+
export type { InterRootQueuePolicy, OrchestrationQueueState } from "@femtomc/mu-orchestrator";
|
|
19
|
+
export { DEFAULT_INTER_ROOT_QUEUE_POLICY, normalizeInterRootQueuePolicy, ORCHESTRATION_QUEUE_ALLOWED_TRANSITIONS, ORCHESTRATION_QUEUE_INVARIANTS, } from "@femtomc/mu-orchestrator";
|
|
12
20
|
export type ActiveAdapter = {
|
|
13
21
|
name: Channel;
|
|
14
22
|
route: string;
|
|
@@ -76,9 +84,50 @@ export type ControlPlaneSessionLifecycle = {
|
|
|
76
84
|
reload: () => Promise<ControlPlaneSessionMutationResult>;
|
|
77
85
|
update: () => Promise<ControlPlaneSessionMutationResult>;
|
|
78
86
|
};
|
|
87
|
+
export type WakeDeliveryState = "queued" | "duplicate" | "skipped" | "delivered" | "retried" | "dead_letter";
|
|
88
|
+
export type WakeNotifyDecision = {
|
|
89
|
+
state: "queued" | "duplicate" | "skipped";
|
|
90
|
+
reason_code: string;
|
|
91
|
+
binding_id: string;
|
|
92
|
+
channel: Channel;
|
|
93
|
+
dedupe_key: string;
|
|
94
|
+
outbox_id: string | null;
|
|
95
|
+
};
|
|
96
|
+
export type WakeNotifyContext = {
|
|
97
|
+
wakeId: string;
|
|
98
|
+
wakeSource?: string | null;
|
|
99
|
+
programId?: string | null;
|
|
100
|
+
sourceTsMs?: number | null;
|
|
101
|
+
};
|
|
102
|
+
export type NotifyOperatorsOpts = {
|
|
103
|
+
message: string;
|
|
104
|
+
dedupeKey: string;
|
|
105
|
+
wake?: WakeNotifyContext | null;
|
|
106
|
+
metadata?: Record<string, unknown>;
|
|
107
|
+
};
|
|
108
|
+
export type NotifyOperatorsResult = {
|
|
109
|
+
queued: number;
|
|
110
|
+
duplicate: number;
|
|
111
|
+
skipped: number;
|
|
112
|
+
decisions: WakeNotifyDecision[];
|
|
113
|
+
};
|
|
114
|
+
export type WakeDeliveryEvent = {
|
|
115
|
+
state: "delivered" | "retried" | "dead_letter";
|
|
116
|
+
reason_code: string;
|
|
117
|
+
wake_id: string;
|
|
118
|
+
dedupe_key: string;
|
|
119
|
+
binding_id: string;
|
|
120
|
+
channel: Channel;
|
|
121
|
+
outbox_id: string;
|
|
122
|
+
outbox_dedupe_key: string;
|
|
123
|
+
attempt_count: number;
|
|
124
|
+
};
|
|
125
|
+
export type WakeDeliveryObserver = (event: WakeDeliveryEvent) => void | Promise<void>;
|
|
79
126
|
export type ControlPlaneHandle = {
|
|
80
127
|
activeAdapters: ActiveAdapter[];
|
|
81
128
|
handleWebhook(path: string, req: Request): Promise<Response | null>;
|
|
129
|
+
notifyOperators?(opts: NotifyOperatorsOpts): Promise<NotifyOperatorsResult>;
|
|
130
|
+
setWakeDeliveryObserver?(observer: WakeDeliveryObserver | null): void;
|
|
82
131
|
reloadTelegramGeneration?(opts: {
|
|
83
132
|
config: ControlPlaneConfig;
|
|
84
133
|
reason: string;
|
|
@@ -88,10 +137,18 @@ export type ControlPlaneHandle = {
|
|
|
88
137
|
limit?: number;
|
|
89
138
|
}): Promise<ControlPlaneRunSnapshot[]>;
|
|
90
139
|
getRun?(idOrRoot: string): Promise<ControlPlaneRunSnapshot | null>;
|
|
140
|
+
/**
|
|
141
|
+
* Run lifecycle boundary: accepts start intent into the default queue/reconcile path.
|
|
142
|
+
* Compatibility adapters may dispatch immediately after enqueue, but must preserve queue invariants.
|
|
143
|
+
*/
|
|
91
144
|
startRun?(opts: {
|
|
92
145
|
prompt: string;
|
|
93
146
|
maxSteps?: number;
|
|
94
147
|
}): Promise<ControlPlaneRunSnapshot>;
|
|
148
|
+
/**
|
|
149
|
+
* Run lifecycle boundary: accepts resume intent into the default queue/reconcile path.
|
|
150
|
+
* No flag-based alternate path is allowed.
|
|
151
|
+
*/
|
|
95
152
|
resumeRun?(opts: {
|
|
96
153
|
rootIssueId: string;
|
|
97
154
|
maxSteps?: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export {};
|
|
1
|
+
export { DEFAULT_INTER_ROOT_QUEUE_POLICY, normalizeInterRootQueuePolicy, ORCHESTRATION_QUEUE_ALLOWED_TRANSITIONS, ORCHESTRATION_QUEUE_INVARIANTS, } from "@femtomc/mu-orchestrator";
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { GenerationTelemetryRecorder, type ReloadableGenerationIdentity, type ReloadLifecycleReason } from "@femtomc/mu-control-plane";
|
|
2
|
+
import type { ControlPlaneConfig, ControlPlaneHandle, TelegramGenerationReloadResult } from "./control_plane_contract.js";
|
|
3
|
+
import { ControlPlaneGenerationSupervisor } from "./generation_supervisor.js";
|
|
4
|
+
export type ControlPlaneSummary = {
|
|
5
|
+
active: boolean;
|
|
6
|
+
adapters: string[];
|
|
7
|
+
routes: Array<{
|
|
8
|
+
name: string;
|
|
9
|
+
route: string;
|
|
10
|
+
}>;
|
|
11
|
+
};
|
|
12
|
+
export type ControlPlaneReloadResult = {
|
|
13
|
+
ok: boolean;
|
|
14
|
+
reason: string;
|
|
15
|
+
previous_control_plane: ControlPlaneSummary;
|
|
16
|
+
control_plane: ControlPlaneSummary;
|
|
17
|
+
generation: {
|
|
18
|
+
attempt_id: string;
|
|
19
|
+
coalesced: boolean;
|
|
20
|
+
from_generation: ReloadableGenerationIdentity | null;
|
|
21
|
+
to_generation: ReloadableGenerationIdentity;
|
|
22
|
+
active_generation: ReloadableGenerationIdentity | null;
|
|
23
|
+
outcome: "success" | "failure";
|
|
24
|
+
};
|
|
25
|
+
telegram_generation?: TelegramGenerationReloadResult;
|
|
26
|
+
error?: string;
|
|
27
|
+
};
|
|
28
|
+
export type ControlPlaneReloader = (opts: {
|
|
29
|
+
repoRoot: string;
|
|
30
|
+
previous: ControlPlaneHandle | null;
|
|
31
|
+
config: ControlPlaneConfig;
|
|
32
|
+
generation: ReloadableGenerationIdentity;
|
|
33
|
+
}) => Promise<ControlPlaneHandle | null>;
|
|
34
|
+
export type ConfigReader = (repoRoot: string) => Promise<import("./config.js").MuConfig>;
|
|
35
|
+
export type ConfigWriter = (repoRoot: string, config: import("./config.js").MuConfig) => Promise<string>;
|
|
36
|
+
export declare function summarizeControlPlane(handle: ControlPlaneHandle | null): ControlPlaneSummary;
|
|
37
|
+
export type ReloadManagerDeps = {
|
|
38
|
+
repoRoot: string;
|
|
39
|
+
initialControlPlane: ControlPlaneHandle | null;
|
|
40
|
+
controlPlaneReloader: ControlPlaneReloader;
|
|
41
|
+
generationTelemetry: GenerationTelemetryRecorder;
|
|
42
|
+
loadConfigFromDisk: () => Promise<import("./config.js").MuConfig>;
|
|
43
|
+
};
|
|
44
|
+
export type ReloadManager = {
|
|
45
|
+
reloadControlPlane: (reason: ReloadLifecycleReason) => Promise<ControlPlaneReloadResult>;
|
|
46
|
+
getControlPlaneStatus: () => {
|
|
47
|
+
active: boolean;
|
|
48
|
+
adapters: string[];
|
|
49
|
+
routes: Array<{
|
|
50
|
+
name: string;
|
|
51
|
+
route: string;
|
|
52
|
+
}>;
|
|
53
|
+
generation: import("@femtomc/mu-control-plane").GenerationSupervisorSnapshot;
|
|
54
|
+
observability: {
|
|
55
|
+
counters: ReturnType<GenerationTelemetryRecorder["counters"]>;
|
|
56
|
+
};
|
|
57
|
+
};
|
|
58
|
+
getControlPlaneCurrent: () => ControlPlaneHandle | null;
|
|
59
|
+
setControlPlaneCurrent: (handle: ControlPlaneHandle | null) => void;
|
|
60
|
+
generationSupervisor: ControlPlaneGenerationSupervisor;
|
|
61
|
+
generationTelemetry: GenerationTelemetryRecorder;
|
|
62
|
+
};
|
|
63
|
+
export declare function createReloadManager(deps: ReloadManagerDeps): ReloadManager;
|