@voybio/ace-swarm 2.4.7 → 2.4.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/assets/logo/ace-logo.png +0 -0
- package/dist/ace-server-instructions.js +14 -2
- package/dist/cli.js +7 -2
- package/dist/job-scheduler.d.ts +11 -0
- package/dist/job-scheduler.js +132 -28
- package/dist/orchestrator-supervisor.d.ts +3 -0
- package/dist/orchestrator-supervisor.js +63 -11
- package/dist/server.d.ts +14 -3
- package/dist/server.js +14 -3
- package/dist/tools-files.js +31 -7
- package/dist/tools-handoff.d.ts +3 -2
- package/dist/tools-handoff.js +25 -19
- package/dist/tools-scheduler.js +1 -0
- package/docs/mcp-tool-reference.md +6 -1
- package/docs/troubleshooting.md +29 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
Binary file
|
|
@@ -31,6 +31,13 @@ const REFRESH_TOOLS = new Set([
|
|
|
31
31
|
// New mutation tools with non-standard prefixes must be added here explicitly.
|
|
32
32
|
const NAMED_MUTATION_TOOLS = new Set([
|
|
33
33
|
"rewrite_targets",
|
|
34
|
+
"apply_patch",
|
|
35
|
+
"astgrep_rewrite",
|
|
36
|
+
"reap_stale_jobs",
|
|
37
|
+
"heartbeat_job",
|
|
38
|
+
"generate_kanban_ui",
|
|
39
|
+
"codemunch_snapshot",
|
|
40
|
+
"codemunch_index",
|
|
34
41
|
]);
|
|
35
42
|
const MUTATION_PREFIXES = [
|
|
36
43
|
"ack_",
|
|
@@ -209,7 +216,7 @@ export function buildServerInstructions(workspaceRoot) {
|
|
|
209
216
|
coreFiles.length > 0
|
|
210
217
|
? `Ground truth for task/status/scope/evidence is store-backed under ${coreFiles.join(", ")} (canonical store: ${ACE_ROOT_REL}/agent-state/ace-state.ace).`
|
|
211
218
|
: `Ground truth is store-backed under ${ACE_ROOT_REL}/agent-state/* artifacts (canonical store: ${ACE_ROOT_REL}/agent-state/ace-state.ace), not chat history.`,
|
|
212
|
-
"`ace compact` and the automatic lifecycle sweep intentionally remove
|
|
219
|
+
"`ace compact` and the automatic lifecycle sweep intentionally remove loose agent-state/* files from disk after folding them into the store — this is expected, not data loss, and applies to every swept artifact (kanban, run-ledger, handoffs, playbook, tracker snapshots, etc.), not only TASK/STATUS/SCOPE/EVIDENCE_LOG. If a path is not on disk, call list_workspace on its directory to see the full store-backed listing (or `ace ls` from the CLI), then read it back with read_workspace_file (or `ace read <path>`); never recreate swept files.",
|
|
213
220
|
stateDirs.length > 0
|
|
214
221
|
? `Durable artifacts belong under ${stateDirs.map((entry) => `${entry}/`).join(", ")} and are persisted into the store; the loose copies are sweep inputs, not the source of truth.`
|
|
215
222
|
: undefined,
|
|
@@ -217,6 +224,9 @@ export function buildServerInstructions(workspaceRoot) {
|
|
|
217
224
|
"Use route_task and get_agent_instructions for role guidance; do not self-assign by guesswork.",
|
|
218
225
|
"After meaningful edits or state changes, call execute_gates and persist evidence/handoffs in ACE artifacts.",
|
|
219
226
|
"For delegated or long-running work, use create_handoff / ack_handoff and append_vericify_process_post instead of ad-hoc status notes.",
|
|
227
|
+
"For self-paced, evidence-gated multi-step objectives beyond this session, use start_goal / get_goal_status / resume_goal / stop_goal (or `ace goal` from the CLI).",
|
|
228
|
+
"Prefer the matching MCP tool over ad-hoc shell/chat tracking: git_status / git_diff / git_log / git_commit for git; enqueue_job / dispatch_jobs / heartbeat_job / get_lock_table for background jobs; snapshot_state / semantic_diff / drift_report / rewrite_targets for change intelligence.",
|
|
229
|
+
"skill_catalog_summary / skill_audit / problem_triage surface packaged skills; get_todo / update_todo / get_run_ledger is the human-facing todo surface; scan_workspace_delta / web_research_packet / codemunch_snapshot / codemunch_index cover workspace-wide search and indexing.",
|
|
220
230
|
`Current ACE digest: ${digest}.`,
|
|
221
231
|
autonomyHint ? `Runtime autonomy signals: ${autonomyHint}.` : undefined,
|
|
222
232
|
];
|
|
@@ -240,11 +250,13 @@ export function buildHostInstructionText(host, workspaceRoot) {
|
|
|
240
250
|
"2. Call `route_task` and `get_agent_instructions` before choosing a role.",
|
|
241
251
|
"3. Prefer ACE prompts/resources (`ace-orchestrator`, `get_task_pack`, `get_skill_instructions`) over relying on this file.",
|
|
242
252
|
"4. Read/write task state through MCP tools (`recall_context`, `read_workspace_file`, `write_workspace_file`, lifecycle append tools); do not create files under `agent-state/` directly.",
|
|
253
|
+
"5. For self-paced, evidence-gated multi-step objectives beyond this session, use `start_goal` / `get_goal_status` / `resume_goal` / `stop_goal` (or `ace goal` from the CLI).",
|
|
254
|
+
"6. Also prefer the matching MCP domain over ad-hoc equivalents for git, background jobs/scheduler, change intelligence, discovery/indexing, skills, and todo — see docs/mcp-tool-reference.md for the full per-domain tool catalog.",
|
|
243
255
|
"",
|
|
244
256
|
"Ground truth:",
|
|
245
257
|
"- ACE MCP tools are the authoritative interface for task state and lifecycle updates.",
|
|
246
258
|
"- `agent-state/ace-state.ace` is the canonical store path; visible `agent-state/*` files are compatibility projections or sweep inputs.",
|
|
247
|
-
"- `ace compact` and the automatic lifecycle sweep intentionally remove loose `agent-state/*` files from disk after folding them into the store — this is expected, not data loss. Do NOT recreate them; retrieve
|
|
259
|
+
"- `ace compact` and the automatic lifecycle sweep intentionally remove loose `agent-state/*` files from disk after folding them into the store — this is expected, not data loss, for any swept artifact. Do NOT recreate them; if a path isn't on disk, call `list_workspace` on its directory (or `ace ls`) to see the full store-backed listing, then retrieve it via `ace read <path>` or the `read_workspace_file` / `recall_context` MCP tools.",
|
|
248
260
|
"- `.agents/ACE/tasks/todo.md` is the human-facing todo surface.",
|
|
249
261
|
"- Hook context is store-backed in `agent-state/ace-state.ace` and consumed without extra workspace projections.",
|
|
250
262
|
"",
|
package/dist/cli.js
CHANGED
|
@@ -98,8 +98,13 @@ Options for turnkey:
|
|
|
98
98
|
Options for mcp:
|
|
99
99
|
--profile <host-min|full> Select the visible tool schema (default: full)
|
|
100
100
|
|
|
101
|
-
Options for playbook
|
|
102
|
-
--
|
|
101
|
+
Options for playbook:
|
|
102
|
+
list [--section <s>] [--status <s>] List playbook bullets (default subcommand)
|
|
103
|
+
show <id> Show one bullet as JSON
|
|
104
|
+
add --content <text> [--section <s>] Add a bullet
|
|
105
|
+
[--reason-code <code>] Scope the lesson to a matching failure reason code
|
|
106
|
+
tag <id> helpful|harmful Curate a bullet's helpful/harmful tag
|
|
107
|
+
gc Prune and quarantine curated bullets
|
|
103
108
|
|
|
104
109
|
Options for eval:
|
|
105
110
|
--model, --models <csv> Model name or comma-separated model names (required)
|
package/dist/job-scheduler.d.ts
CHANGED
|
@@ -103,8 +103,12 @@ export interface DispatchJobsInput {
|
|
|
103
103
|
}
|
|
104
104
|
export interface DispatchJobsResult {
|
|
105
105
|
ok: boolean;
|
|
106
|
+
reason_code?: "scheduler_lease_held";
|
|
107
|
+
error?: string;
|
|
106
108
|
owner: string;
|
|
107
109
|
lease_acquired: boolean;
|
|
110
|
+
/** Whether execution still retains ownership after this tick. */
|
|
111
|
+
lease_retained: boolean;
|
|
108
112
|
lease: SchedulerLeaseFile;
|
|
109
113
|
queue_path: string;
|
|
110
114
|
lock_path: string;
|
|
@@ -210,6 +214,13 @@ export declare function readSchedulerLease(): SchedulerLeaseFile | undefined;
|
|
|
210
214
|
*/
|
|
211
215
|
export declare const DEFAULT_HEARTBEAT_STALE_MS = 120000;
|
|
212
216
|
export declare function enqueueJob(input: EnqueueJobInput): Promise<EnqueueJobResult>;
|
|
217
|
+
/** Mirror handoff lifecycle without admitting or executing a job. */
|
|
218
|
+
export declare function syncHandoffLifecycleJob(input: EnqueueJobInput & {
|
|
219
|
+
job_id: string;
|
|
220
|
+
now_iso?: string;
|
|
221
|
+
}): Promise<EnqueueJobResult & {
|
|
222
|
+
enqueued: boolean;
|
|
223
|
+
}>;
|
|
213
224
|
export declare function acknowledgeJob(input: AckJobInput): Promise<AckJobResult>;
|
|
214
225
|
export declare function dispatchJobs(input?: DispatchJobsInput): Promise<DispatchJobsResult>;
|
|
215
226
|
export declare function dispatchJobNow(input: DispatchJobNowInput): Promise<DispatchJobNowResult>;
|
package/dist/job-scheduler.js
CHANGED
|
@@ -328,7 +328,7 @@ export function readSchedulerLease() {
|
|
|
328
328
|
const root = resolveWorkspaceRoot();
|
|
329
329
|
if (storeExistsSync(root)) {
|
|
330
330
|
const stored = readStoreJsonSync(root, "state/scheduler/lease");
|
|
331
|
-
if (stored)
|
|
331
|
+
if (stored !== undefined)
|
|
332
332
|
return stored ?? undefined;
|
|
333
333
|
}
|
|
334
334
|
const raw = safeRead(SCHEDULER_LEASE_REL);
|
|
@@ -336,28 +336,46 @@ export function readSchedulerLease() {
|
|
|
336
336
|
return undefined;
|
|
337
337
|
return parseLease(raw);
|
|
338
338
|
}
|
|
339
|
+
/** Serialize admission as well as persistence across processes. */
|
|
340
|
+
async function withSchedulerTransaction(fn) {
|
|
341
|
+
const storePath = getWorkspaceStorePath(resolveWorkspaceRoot());
|
|
342
|
+
return withFileLock(SCHEDULER_LOCK_REL, () => withStoreWriteCoordinator(storePath, fn, { operation_label: "scheduler_transaction" }));
|
|
343
|
+
}
|
|
344
|
+
function hasExecutionOwnership(queue, locks) {
|
|
345
|
+
return queue.jobs.some((job) => job.status === "running" ||
|
|
346
|
+
(job.status === "unknown_recovery" && job.reason_code === "handoff_execution_unverified")) ||
|
|
347
|
+
locks.locks.some((lock) => isJobLockActive(lock));
|
|
348
|
+
}
|
|
349
|
+
function leaseForAcquisition() {
|
|
350
|
+
// Called inside the transaction: legacy idle leases can be reclaimed immediately.
|
|
351
|
+
return hasExecutionOwnership(readJobQueue(), readJobLockTable()) ? readSchedulerLease() : undefined;
|
|
352
|
+
}
|
|
353
|
+
// Only called from withSchedulerTransaction. Do not reacquire the store coordinator.
|
|
339
354
|
async function persistSchedulerState(input) {
|
|
340
355
|
const root = resolveWorkspaceRoot();
|
|
356
|
+
const lease = hasExecutionOwnership(input.queue, input.locks) ? input.lease : undefined;
|
|
341
357
|
if (!storeExistsSync(root)) {
|
|
358
|
+
// Legacy workspaces share the same cross-process transaction lock. Do not
|
|
359
|
+
// create a partial store that would hide their handoffs, events, and todos.
|
|
342
360
|
safeWrite(JOB_QUEUE_REL, JSON.stringify({ ...input.queue, updated_at: nowIso() }, null, 2));
|
|
343
361
|
safeWrite(JOB_LOCK_TABLE_REL, JSON.stringify({ ...input.locks, updated_at: nowIso() }, null, 2));
|
|
344
|
-
safeWrite(SCHEDULER_LEASE_REL, JSON.stringify(
|
|
362
|
+
safeWrite(SCHEDULER_LEASE_REL, JSON.stringify(lease ?? null, null, 2));
|
|
345
363
|
return;
|
|
346
364
|
}
|
|
347
|
-
const
|
|
348
|
-
|
|
349
|
-
const
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
}
|
|
365
|
+
const store = await openStore(getWorkspaceStorePath(root));
|
|
366
|
+
try {
|
|
367
|
+
const repo = new SchedulerRepository(store);
|
|
368
|
+
await repo.replaceState({
|
|
369
|
+
...input,
|
|
370
|
+
lease,
|
|
371
|
+
});
|
|
372
|
+
await store.commit();
|
|
373
|
+
const projections = new ProjectionManager(store, root);
|
|
374
|
+
await projections.projectAfterCommit(["scheduler"]);
|
|
375
|
+
}
|
|
376
|
+
finally {
|
|
377
|
+
await store.close();
|
|
378
|
+
}
|
|
361
379
|
}
|
|
362
380
|
function compareJobs(a, b) {
|
|
363
381
|
const priorityCmp = PRIORITY_WEIGHT[a.priority] - PRIORITY_WEIGHT[b.priority];
|
|
@@ -607,7 +625,7 @@ function hasActiveLeaseForOwner(lease, owner, now) {
|
|
|
607
625
|
return expires.getTime() > now.getTime();
|
|
608
626
|
}
|
|
609
627
|
function ensureLeaseForOwner(owner, now, ttlSeconds = defaultLeaseTtlSeconds()) {
|
|
610
|
-
const leaseResult = acquireLease(
|
|
628
|
+
const leaseResult = acquireLease(leaseForAcquisition(), owner, ttlSeconds, now);
|
|
611
629
|
if (!leaseResult.acquired) {
|
|
612
630
|
return leaseResult;
|
|
613
631
|
}
|
|
@@ -703,6 +721,8 @@ function startJobNow(job, table, lease, now, buf, sourceTool) {
|
|
|
703
721
|
function reconcileRecoveryState(queue, table, now, recoveryMode, buf) {
|
|
704
722
|
let recovered = 0;
|
|
705
723
|
for (const job of queue.jobs) {
|
|
724
|
+
if (job.kind === "handoff.lifecycle")
|
|
725
|
+
continue;
|
|
706
726
|
if (job.status === "unknown_recovery") {
|
|
707
727
|
releaseLocksForRecoveredJob(table, job.job_id, "recovery_hold", now);
|
|
708
728
|
continue;
|
|
@@ -754,6 +774,8 @@ function reapStaleRunningJobs(queue, table, now, staleMs, buf) {
|
|
|
754
774
|
return [];
|
|
755
775
|
const reaped = [];
|
|
756
776
|
for (const job of queue.jobs) {
|
|
777
|
+
if (job.kind === "handoff.lifecycle")
|
|
778
|
+
continue;
|
|
757
779
|
if (job.status !== "running")
|
|
758
780
|
continue;
|
|
759
781
|
const activeLocks = table.locks.filter((lock) => lock.job_id === job.job_id && lock.status === "active");
|
|
@@ -801,7 +823,7 @@ function reapStaleRunningJobs(queue, table, now, staleMs, buf) {
|
|
|
801
823
|
}
|
|
802
824
|
export async function enqueueJob(input) {
|
|
803
825
|
const buf = newEventBuffer();
|
|
804
|
-
const result = await
|
|
826
|
+
const result = await withSchedulerTransaction(async () => {
|
|
805
827
|
if (!input.kind || input.kind.trim().length === 0) {
|
|
806
828
|
const queue = readJobQueue();
|
|
807
829
|
return {
|
|
@@ -900,9 +922,73 @@ export async function enqueueJob(input) {
|
|
|
900
922
|
await flushTransitionEvents(buf);
|
|
901
923
|
return result;
|
|
902
924
|
}
|
|
925
|
+
/** Mirror handoff lifecycle without admitting or executing a job. */
|
|
926
|
+
export async function syncHandoffLifecycleJob(input) {
|
|
927
|
+
return withSchedulerTransaction(async () => {
|
|
928
|
+
const queue = readJobQueue();
|
|
929
|
+
const locks = readJobLockTable();
|
|
930
|
+
const now = parseDate(input.now_iso) ?? new Date();
|
|
931
|
+
let job = queue.jobs.find((row) => row.job_id === input.job_id);
|
|
932
|
+
if (job && (job.kind !== "handoff.lifecycle" || job.payload.handoff_id !== input.payload?.handoff_id)) {
|
|
933
|
+
return {
|
|
934
|
+
ok: false,
|
|
935
|
+
enqueued: false,
|
|
936
|
+
path: getJobQueuePath(),
|
|
937
|
+
queue,
|
|
938
|
+
nack: { reason_code: "duplicate_job_id", reason: "Job belongs to another lifecycle or execution." },
|
|
939
|
+
};
|
|
940
|
+
}
|
|
941
|
+
const enqueued = !job;
|
|
942
|
+
if (!job) {
|
|
943
|
+
job = {
|
|
944
|
+
job_id: input.job_id,
|
|
945
|
+
kind: "handoff.lifecycle",
|
|
946
|
+
payload: input.payload ?? {},
|
|
947
|
+
status: "blocked",
|
|
948
|
+
priority: toPriority(input.priority),
|
|
949
|
+
dependencies: input.dependencies ?? [],
|
|
950
|
+
resource_requirements: normalizeResources(input.resource_requirements),
|
|
951
|
+
attempt: 0,
|
|
952
|
+
max_attempts: 3,
|
|
953
|
+
backoff_ms: 5000,
|
|
954
|
+
created_at: nowIso(now),
|
|
955
|
+
accepted_at: nowIso(now),
|
|
956
|
+
};
|
|
957
|
+
queue.jobs.push(job);
|
|
958
|
+
}
|
|
959
|
+
job.payload = { ...job.payload, ...input.payload };
|
|
960
|
+
if (job.payload.status === "completed") {
|
|
961
|
+
// The canonical handoff completion is the receipt resolving legacy bookkeeping.
|
|
962
|
+
job.status = "done";
|
|
963
|
+
job.completed_at = job.completed_at ?? nowIso(now);
|
|
964
|
+
job.reason_code = undefined;
|
|
965
|
+
job.error = undefined;
|
|
966
|
+
releaseLocksForRecoveredJob(locks, job.job_id, "handoff_completed", now);
|
|
967
|
+
}
|
|
968
|
+
else if (job.status === "running" || job.status === "unknown_recovery" || hasActiveLockForJob(locks.locks, job.job_id)) {
|
|
969
|
+
// Do not infer worker death from a legacy synthetic job's kind.
|
|
970
|
+
job.status = "unknown_recovery";
|
|
971
|
+
job.reason_code = "handoff_execution_unverified";
|
|
972
|
+
job.error = "Legacy handoff execution requires a completion receipt; ownership and locks are preserved.";
|
|
973
|
+
}
|
|
974
|
+
else if (!TERMINAL_STATUSES.has(job.status)) {
|
|
975
|
+
job.status = "blocked";
|
|
976
|
+
job.reason_code = "handoff_metadata_only";
|
|
977
|
+
job.error = "Lifecycle metadata is synchronized by the handoff registry, not dispatched.";
|
|
978
|
+
}
|
|
979
|
+
await persistSchedulerState({ queue, locks, lease: readSchedulerLease() });
|
|
980
|
+
return {
|
|
981
|
+
ok: true,
|
|
982
|
+
enqueued,
|
|
983
|
+
path: getJobQueuePath(),
|
|
984
|
+
queue,
|
|
985
|
+
ack: { job_id: job.job_id, status: job.status },
|
|
986
|
+
};
|
|
987
|
+
});
|
|
988
|
+
}
|
|
903
989
|
export async function acknowledgeJob(input) {
|
|
904
990
|
const buf = newEventBuffer();
|
|
905
|
-
const result = await
|
|
991
|
+
const result = await withSchedulerTransaction(async () => {
|
|
906
992
|
const queue = readJobQueue();
|
|
907
993
|
const job = queue.jobs.find((row) => row.job_id === input.job_id);
|
|
908
994
|
if (!job) {
|
|
@@ -984,20 +1070,23 @@ export async function acknowledgeJob(input) {
|
|
|
984
1070
|
}
|
|
985
1071
|
export async function dispatchJobs(input = {}) {
|
|
986
1072
|
const buf = newEventBuffer();
|
|
987
|
-
const result = await
|
|
1073
|
+
const result = await withSchedulerTransaction(async () => {
|
|
988
1074
|
const owner = deriveOwner(input.owner);
|
|
989
1075
|
const now = parseDate(input.now_iso) ?? new Date();
|
|
990
1076
|
const table = readJobLockTable();
|
|
991
1077
|
const leaseTtl = typeof input.lease_ttl_seconds === "number" && input.lease_ttl_seconds > 0
|
|
992
1078
|
? Math.floor(input.lease_ttl_seconds)
|
|
993
1079
|
: defaultLeaseTtlSeconds();
|
|
994
|
-
const leaseResult = acquireLease(
|
|
1080
|
+
const leaseResult = acquireLease(leaseForAcquisition(), owner, leaseTtl, now);
|
|
995
1081
|
if (!leaseResult.acquired) {
|
|
996
1082
|
const queue = readJobQueue();
|
|
997
1083
|
return {
|
|
998
1084
|
ok: false,
|
|
1085
|
+
reason_code: "scheduler_lease_held",
|
|
1086
|
+
error: `Scheduler lease is held by ${leaseResult.lease.owner} until ${leaseResult.lease.expires_at}; dispatch requested by ${owner}.`,
|
|
999
1087
|
owner,
|
|
1000
1088
|
lease_acquired: false,
|
|
1089
|
+
lease_retained: true,
|
|
1001
1090
|
lease: leaseResult.lease,
|
|
1002
1091
|
queue_path: getJobQueuePath(),
|
|
1003
1092
|
lock_path: getJobLockTablePath(),
|
|
@@ -1006,7 +1095,7 @@ export async function dispatchJobs(input = {}) {
|
|
|
1006
1095
|
summary: {
|
|
1007
1096
|
total_jobs: queue.jobs.length,
|
|
1008
1097
|
blocked_jobs: queue.jobs.filter((job) => job.status === "blocked").length,
|
|
1009
|
-
ready_jobs: queue.jobs.filter((job) => job.status === "ready").length,
|
|
1098
|
+
ready_jobs: queue.jobs.filter((job) => job.status === "ready" && job.kind !== "handoff.lifecycle").length,
|
|
1010
1099
|
running_jobs: queue.jobs.filter((job) => job.status === "running").length,
|
|
1011
1100
|
started_jobs: 0,
|
|
1012
1101
|
recovered_jobs: 0,
|
|
@@ -1029,6 +1118,8 @@ export async function dispatchJobs(input = {}) {
|
|
|
1029
1118
|
const reapedJobIds = reapStaleRunningJobs(queue, table, now, staleMs, buf);
|
|
1030
1119
|
const reapedJobs = reapedJobIds.length;
|
|
1031
1120
|
for (const job of queue.jobs) {
|
|
1121
|
+
if (job.kind === "handoff.lifecycle")
|
|
1122
|
+
continue;
|
|
1032
1123
|
if (TERMINAL_STATUSES.has(job.status))
|
|
1033
1124
|
continue;
|
|
1034
1125
|
if (job.status === "running" || job.status === "unknown_recovery")
|
|
@@ -1051,7 +1142,7 @@ export async function dispatchJobs(input = {}) {
|
|
|
1051
1142
|
}
|
|
1052
1143
|
const occupancy = buildOccupancy(table.locks);
|
|
1053
1144
|
const readyJobs = queue.jobs
|
|
1054
|
-
.filter((job) => job.status === "ready")
|
|
1145
|
+
.filter((job) => job.status === "ready" && job.kind !== "handoff.lifecycle")
|
|
1055
1146
|
.sort(compareJobs);
|
|
1056
1147
|
let startedJobs = 0;
|
|
1057
1148
|
for (const job of readyJobs) {
|
|
@@ -1080,7 +1171,7 @@ export async function dispatchJobs(input = {}) {
|
|
|
1080
1171
|
const summary = {
|
|
1081
1172
|
total_jobs: queue.jobs.length,
|
|
1082
1173
|
blocked_jobs: queue.jobs.filter((job) => job.status === "blocked").length,
|
|
1083
|
-
ready_jobs: queue.jobs.filter((job) => job.status === "ready").length,
|
|
1174
|
+
ready_jobs: queue.jobs.filter((job) => job.status === "ready" && job.kind !== "handoff.lifecycle").length,
|
|
1084
1175
|
running_jobs: queue.jobs.filter((job) => job.status === "running").length,
|
|
1085
1176
|
started_jobs: startedJobs,
|
|
1086
1177
|
recovered_jobs: recoveredJobs,
|
|
@@ -1093,6 +1184,7 @@ export async function dispatchJobs(input = {}) {
|
|
|
1093
1184
|
payload: {
|
|
1094
1185
|
owner,
|
|
1095
1186
|
lease_acquired: true,
|
|
1187
|
+
lease_retained: hasExecutionOwnership(queue, table),
|
|
1096
1188
|
lease_id: refreshedLease.lease_id,
|
|
1097
1189
|
recovery_mode: leaseResult.recovery_mode,
|
|
1098
1190
|
...summary,
|
|
@@ -1107,6 +1199,7 @@ export async function dispatchJobs(input = {}) {
|
|
|
1107
1199
|
owner,
|
|
1108
1200
|
lease_id: refreshedLease.lease_id,
|
|
1109
1201
|
lease_acquired: true,
|
|
1202
|
+
lease_retained: hasExecutionOwnership(queue, table),
|
|
1110
1203
|
recovery_mode: leaseResult.recovery_mode,
|
|
1111
1204
|
summary,
|
|
1112
1205
|
},
|
|
@@ -1115,6 +1208,7 @@ export async function dispatchJobs(input = {}) {
|
|
|
1115
1208
|
ok: true,
|
|
1116
1209
|
owner,
|
|
1117
1210
|
lease_acquired: true,
|
|
1211
|
+
lease_retained: hasExecutionOwnership(queue, table),
|
|
1118
1212
|
lease: refreshedLease,
|
|
1119
1213
|
queue_path: getJobQueuePath(),
|
|
1120
1214
|
lock_path: getJobLockTablePath(),
|
|
@@ -1128,7 +1222,7 @@ export async function dispatchJobs(input = {}) {
|
|
|
1128
1222
|
}
|
|
1129
1223
|
export async function dispatchJobNow(input) {
|
|
1130
1224
|
const buf = newEventBuffer();
|
|
1131
|
-
const result = await
|
|
1225
|
+
const result = await withSchedulerTransaction(async () => {
|
|
1132
1226
|
const owner = deriveOwner(input.owner);
|
|
1133
1227
|
const now = parseDate(input.now_iso) ?? new Date();
|
|
1134
1228
|
const leaseResult = ensureLeaseForOwner(owner, now);
|
|
@@ -1155,6 +1249,16 @@ export async function dispatchJobNow(input) {
|
|
|
1155
1249
|
error: `Unknown job_id: ${input.job_id}`,
|
|
1156
1250
|
};
|
|
1157
1251
|
}
|
|
1252
|
+
if (job.kind === "handoff.lifecycle") {
|
|
1253
|
+
return {
|
|
1254
|
+
ok: false,
|
|
1255
|
+
queue_path: getJobQueuePath(),
|
|
1256
|
+
lock_path: getJobLockTablePath(),
|
|
1257
|
+
queue,
|
|
1258
|
+
locks: table,
|
|
1259
|
+
error: "Handoff lifecycle jobs are metadata; synchronize the handoff instead of dispatching it.",
|
|
1260
|
+
};
|
|
1261
|
+
}
|
|
1158
1262
|
if (job.status === "unknown_recovery") {
|
|
1159
1263
|
return {
|
|
1160
1264
|
ok: false,
|
|
@@ -1253,7 +1357,7 @@ export async function dispatchJobNow(input) {
|
|
|
1253
1357
|
}
|
|
1254
1358
|
export async function completeJob(input) {
|
|
1255
1359
|
const buf = newEventBuffer();
|
|
1256
|
-
const result = await
|
|
1360
|
+
const result = await withSchedulerTransaction(async () => {
|
|
1257
1361
|
const owner = deriveOwner(input.owner);
|
|
1258
1362
|
const now = parseDate(input.now_iso) ?? new Date();
|
|
1259
1363
|
const leaseResult = ensureLeaseForOwner(owner, now);
|
|
@@ -1394,7 +1498,7 @@ export async function completeJob(input) {
|
|
|
1394
1498
|
* heartbeat reliably signals a dead worker.
|
|
1395
1499
|
*/
|
|
1396
1500
|
export async function heartbeatJob(input) {
|
|
1397
|
-
return
|
|
1501
|
+
return withSchedulerTransaction(async () => {
|
|
1398
1502
|
const owner = deriveOwner(input.owner);
|
|
1399
1503
|
const now = parseDate(input.now_iso) ?? new Date();
|
|
1400
1504
|
const leaseTtl = typeof input.lease_ttl_seconds === "number" && input.lease_ttl_seconds > 0
|
|
@@ -1464,7 +1568,7 @@ export async function heartbeatJob(input) {
|
|
|
1464
1568
|
*/
|
|
1465
1569
|
export async function reapStaleRunningJobsTick(input = {}) {
|
|
1466
1570
|
const buf = newEventBuffer();
|
|
1467
|
-
const result = await
|
|
1571
|
+
const result = await withSchedulerTransaction(async () => {
|
|
1468
1572
|
const now = parseDate(input.now_iso) ?? new Date();
|
|
1469
1573
|
const staleMs = resolveStaleMs(input.heartbeat_stale_seconds);
|
|
1470
1574
|
const queue = readJobQueue();
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type DispatchJobsResult } from "./job-scheduler.js";
|
|
1
2
|
import type { BridgeResult } from "./model-bridge.js";
|
|
2
3
|
import type { LocalModelRuntimeRepository } from "./store/repositories/local-model-runtime-repository.js";
|
|
3
4
|
export type TaskStepStatus = "planned" | "running" | "done" | "failed" | "skipped" | "blocked";
|
|
@@ -151,6 +152,8 @@ export declare function amendTaskPlanWithTransition(plan: TaskPlan, amendment: T
|
|
|
151
152
|
export declare function promoteParallelGroupToScheduler(plan: TaskPlan, parallelGroup: string, owner?: string): Promise<{
|
|
152
153
|
plan: TaskPlan;
|
|
153
154
|
job_ids: string[];
|
|
155
|
+
dispatch?: DispatchJobsResult;
|
|
156
|
+
blocked_reason?: string;
|
|
154
157
|
}>;
|
|
155
158
|
export declare function superviseTaskPlan(inputPlan: TaskPlan, hooks: SupervisorHooks): Promise<SupervisorRunResult>;
|
|
156
159
|
//# sourceMappingURL=orchestrator-supervisor.d.ts.map
|
|
@@ -281,7 +281,10 @@ export async function promoteParallelGroupToScheduler(plan, parallelGroup, owner
|
|
|
281
281
|
const jobIds = [];
|
|
282
282
|
const nextSteps = cloneSteps(plan.steps);
|
|
283
283
|
for (const step of readySteps) {
|
|
284
|
+
// Stable identity also makes retrying the original plan safe after a refused tick.
|
|
285
|
+
const requestedJobId = step.job_id ?? `PLANJOB-${plan.plan_id}-${step.step_id}`;
|
|
284
286
|
const enqueued = await enqueueJob({
|
|
287
|
+
job_id: requestedJobId,
|
|
285
288
|
kind: step.role,
|
|
286
289
|
payload: {
|
|
287
290
|
task: step.task,
|
|
@@ -295,7 +298,11 @@ export async function promoteParallelGroupToScheduler(plan, parallelGroup, owner
|
|
|
295
298
|
ref: dependency,
|
|
296
299
|
})) ?? [],
|
|
297
300
|
});
|
|
298
|
-
const
|
|
301
|
+
const existingJob = enqueued.nack?.reason_code === "duplicate_job_id"
|
|
302
|
+
? enqueued.queue.jobs.find((job) => job.job_id === requestedJobId &&
|
|
303
|
+
job.payload.plan_id === plan.plan_id && job.payload.step_id === step.step_id)
|
|
304
|
+
: undefined;
|
|
305
|
+
const jobId = enqueued.ack?.job_id ?? existingJob?.job_id;
|
|
299
306
|
if (!jobId)
|
|
300
307
|
continue;
|
|
301
308
|
jobIds.push(jobId);
|
|
@@ -303,36 +310,70 @@ export async function promoteParallelGroupToScheduler(plan, parallelGroup, owner
|
|
|
303
310
|
if (stepIndex >= 0) {
|
|
304
311
|
nextSteps[stepIndex] = {
|
|
305
312
|
...nextSteps[stepIndex],
|
|
306
|
-
status: "running",
|
|
307
313
|
job_id: jobId,
|
|
308
314
|
};
|
|
309
315
|
}
|
|
310
316
|
}
|
|
317
|
+
let dispatch;
|
|
318
|
+
let blockedReason;
|
|
311
319
|
if (jobIds.length > 0) {
|
|
312
320
|
const refs = deriveWorkspaceVericifyRunRef({
|
|
313
321
|
session_id: plan.plan_id,
|
|
314
322
|
role: "orchestrator",
|
|
315
323
|
});
|
|
316
|
-
await dispatchJobs({ owner });
|
|
324
|
+
dispatch = await dispatchJobs({ owner });
|
|
325
|
+
blockedReason = dispatch.error;
|
|
326
|
+
const promotedIds = new Set(jobIds);
|
|
327
|
+
for (const step of nextSteps) {
|
|
328
|
+
if (!step.job_id || !promotedIds.has(step.job_id))
|
|
329
|
+
continue;
|
|
330
|
+
const job = dispatch.queue.jobs.find((candidate) => candidate.job_id === step.job_id);
|
|
331
|
+
// A recorded assignment is not proof of dispatch. Keep queued steps retryable.
|
|
332
|
+
switch (job?.status) {
|
|
333
|
+
case "running":
|
|
334
|
+
step.status = "running";
|
|
335
|
+
break;
|
|
336
|
+
case "done":
|
|
337
|
+
step.status = "done";
|
|
338
|
+
break;
|
|
339
|
+
case "failed_terminal":
|
|
340
|
+
step.status = "failed";
|
|
341
|
+
break;
|
|
342
|
+
case "canceled":
|
|
343
|
+
step.status = "skipped";
|
|
344
|
+
break;
|
|
345
|
+
case "unknown_recovery":
|
|
346
|
+
step.status = "blocked";
|
|
347
|
+
break;
|
|
348
|
+
default: step.status = "planned";
|
|
349
|
+
}
|
|
350
|
+
step.blocked_reason = step.status === "planned" || step.status === "blocked"
|
|
351
|
+
? dispatch.error ?? job?.error
|
|
352
|
+
: undefined;
|
|
353
|
+
}
|
|
317
354
|
await appendVericifyProcessPostSafe({
|
|
318
355
|
run_id: refs.run_id,
|
|
319
356
|
branch_id: refs.branch_id,
|
|
320
357
|
lane_id: refs.lane_id,
|
|
321
358
|
agent_id: "agent-orchestrator",
|
|
322
|
-
kind: "progress",
|
|
323
|
-
summary:
|
|
359
|
+
kind: blockedReason ? "blocker" : "progress",
|
|
360
|
+
summary: blockedReason
|
|
361
|
+
? `Parallel group '${parallelGroup}' queued; ${blockedReason}`
|
|
362
|
+
: `Promoted parallel group '${parallelGroup}' to scheduler: ${jobIds.join(", ")}; running=${nextSteps.filter((step) => step.job_id && promotedIds.has(step.job_id) && step.status === "running").length}`,
|
|
324
363
|
tool_refs: ["enqueue_job", "dispatch_jobs"],
|
|
325
364
|
evidence_refs: jobIds,
|
|
326
365
|
});
|
|
327
366
|
}
|
|
367
|
+
const promotedPlan = withDerivedStatus({
|
|
368
|
+
...plan,
|
|
369
|
+
execution_mode: "scheduled",
|
|
370
|
+
steps: nextSteps,
|
|
371
|
+
});
|
|
328
372
|
return {
|
|
329
|
-
plan:
|
|
330
|
-
...plan,
|
|
331
|
-
execution_mode: "scheduled",
|
|
332
|
-
steps: nextSteps,
|
|
333
|
-
status: "executing",
|
|
334
|
-
}),
|
|
373
|
+
plan: blockedReason ? { ...promotedPlan, status: "blocked" } : promotedPlan,
|
|
335
374
|
job_ids: jobIds,
|
|
375
|
+
dispatch,
|
|
376
|
+
blocked_reason: blockedReason,
|
|
336
377
|
};
|
|
337
378
|
}
|
|
338
379
|
export async function superviseTaskPlan(inputPlan, hooks) {
|
|
@@ -368,6 +409,17 @@ export async function superviseTaskPlan(inputPlan, hooks) {
|
|
|
368
409
|
const promoted = await promoteParallelGroupToScheduler(plan, readyGroup);
|
|
369
410
|
jobIds.push(...promoted.job_ids);
|
|
370
411
|
plan = promoted.plan;
|
|
412
|
+
if (promoted.blocked_reason) {
|
|
413
|
+
await hooks.emitStatusEvent?.({ status: "blocked", summary: promoted.blocked_reason });
|
|
414
|
+
await hooks.contextSnapshot?.(plan);
|
|
415
|
+
return {
|
|
416
|
+
plan,
|
|
417
|
+
handoff_ids: handoffIds,
|
|
418
|
+
job_ids: jobIds,
|
|
419
|
+
circuit_opened: false,
|
|
420
|
+
blocked_reason: promoted.blocked_reason,
|
|
421
|
+
};
|
|
422
|
+
}
|
|
371
423
|
break;
|
|
372
424
|
}
|
|
373
425
|
}
|
package/dist/server.d.ts
CHANGED
|
@@ -11,9 +11,20 @@ export declare function createAceServer(options?: CreateAceServerOptions): McpSe
|
|
|
11
11
|
export type AceHostProfile = "host-min" | "full";
|
|
12
12
|
/**
|
|
13
13
|
* Curated premium-host surface (report WS-G). ~15–25 high-value tools instead of the
|
|
14
|
-
* full tool surface, cutting the per-session schema tax.
|
|
15
|
-
*
|
|
16
|
-
*
|
|
14
|
+
* full tool surface, cutting the per-session schema tax. `run_orchestrator` only reaches an
|
|
15
|
+
* omitted tool if the model-bridge child it spawns happens to have that tool in its own MCP
|
|
16
|
+
* session — it is a role/plan dispatcher, not a generic proxy for any omitted tool by name.
|
|
17
|
+
* The reliable way to reach anything below is to start with `--profile full`.
|
|
18
|
+
*
|
|
19
|
+
* Whole domains are intentionally absent, not just individual tools: goal-engine
|
|
20
|
+
* (start_goal/get_goal_status/resume_goal/stop_goal — until WS-F ships), git
|
|
21
|
+
* (git_status/git_diff/git_log/git_commit), scheduler (enqueue_job/dispatch_jobs/
|
|
22
|
+
* heartbeat_job/reap_stale_jobs/get_lock_table/…), drift/change-intelligence
|
|
23
|
+
* (snapshot_state/semantic_diff/drift_report/rewrite_targets/list_snapshots), discovery
|
|
24
|
+
* (scan_workspace_delta/web_research_packet/codemunch_snapshot/codemunch_index/
|
|
25
|
+
* mcp_ingestion_probe/refresh_astgrep_index), skills (skill_catalog_summary/skill_audit/
|
|
26
|
+
* problem_triage), and todo (get_todo/update_todo/get_run_ledger) all have zero
|
|
27
|
+
* representation below.
|
|
17
28
|
*/
|
|
18
29
|
export declare const HOST_MIN_TOOL_PROFILE: readonly string[];
|
|
19
30
|
export declare function resolveHostProfileAllowlist(profile: AceHostProfile): readonly string[] | undefined;
|
package/dist/server.js
CHANGED
|
@@ -94,9 +94,20 @@ async function verifyMcpStoreStartup(workspaceRoot, logStartup) {
|
|
|
94
94
|
}
|
|
95
95
|
/**
|
|
96
96
|
* Curated premium-host surface (report WS-G). ~15–25 high-value tools instead of the
|
|
97
|
-
* full tool surface, cutting the per-session schema tax.
|
|
98
|
-
*
|
|
99
|
-
*
|
|
97
|
+
* full tool surface, cutting the per-session schema tax. `run_orchestrator` only reaches an
|
|
98
|
+
* omitted tool if the model-bridge child it spawns happens to have that tool in its own MCP
|
|
99
|
+
* session — it is a role/plan dispatcher, not a generic proxy for any omitted tool by name.
|
|
100
|
+
* The reliable way to reach anything below is to start with `--profile full`.
|
|
101
|
+
*
|
|
102
|
+
* Whole domains are intentionally absent, not just individual tools: goal-engine
|
|
103
|
+
* (start_goal/get_goal_status/resume_goal/stop_goal — until WS-F ships), git
|
|
104
|
+
* (git_status/git_diff/git_log/git_commit), scheduler (enqueue_job/dispatch_jobs/
|
|
105
|
+
* heartbeat_job/reap_stale_jobs/get_lock_table/…), drift/change-intelligence
|
|
106
|
+
* (snapshot_state/semantic_diff/drift_report/rewrite_targets/list_snapshots), discovery
|
|
107
|
+
* (scan_workspace_delta/web_research_packet/codemunch_snapshot/codemunch_index/
|
|
108
|
+
* mcp_ingestion_probe/refresh_astgrep_index), skills (skill_catalog_summary/skill_audit/
|
|
109
|
+
* problem_triage), and todo (get_todo/update_todo/get_run_ledger) all have zero
|
|
110
|
+
* representation below.
|
|
100
111
|
*/
|
|
101
112
|
export const HOST_MIN_TOOL_PROFILE = [
|
|
102
113
|
"recall_context",
|
package/dist/tools-files.js
CHANGED
|
@@ -17,7 +17,8 @@ import { safeEditFile, diffContents, applyPatch } from "./safe-edit.js";
|
|
|
17
17
|
import { detectAstgrepCommand, flattenMetaVariables, locateAstgrepMatches, runAstgrepQuery, } from "./astgrep-index.js";
|
|
18
18
|
import { readRuntimeProfile, resolveEffectiveSurgicalReadBudget, validateRuntimeProfileContent, } from "./runtime-profile.js";
|
|
19
19
|
import { withLocalModelRuntimeRepository } from "./store/repositories/local-model-runtime-repository.js";
|
|
20
|
-
import { listStoreKeysSync } from "./store/store-snapshot.js";
|
|
20
|
+
import { getWorkspaceStorePath, listStoreKeysSync } from "./store/store-snapshot.js";
|
|
21
|
+
import { listStoreEntries } from "./store/store-cli.js";
|
|
21
22
|
function reasonCodeFromError(error) {
|
|
22
23
|
return typeof error === "object" && error !== null && "reason_code" in error
|
|
23
24
|
? String(error.reason_code)
|
|
@@ -1092,7 +1093,7 @@ export function registerFileTools(server) {
|
|
|
1092
1093
|
],
|
|
1093
1094
|
};
|
|
1094
1095
|
});
|
|
1095
|
-
server.tool("list_workspace", "List files/directories in a workspace path", {
|
|
1096
|
+
server.tool("list_workspace", "List files/directories in a workspace path, plus any ACEPACK store-only artifacts under it (content the lifecycle sweep folded into the store and removed from disk). Store-only entries are flat logical paths, not directory entries; read them back with read_workspace_file.", {
|
|
1096
1097
|
path: z
|
|
1097
1098
|
.string()
|
|
1098
1099
|
.optional()
|
|
@@ -1110,24 +1111,47 @@ export function registerFileTools(server) {
|
|
|
1110
1111
|
],
|
|
1111
1112
|
};
|
|
1112
1113
|
}
|
|
1114
|
+
const diskNames = new Set();
|
|
1115
|
+
let diskSection;
|
|
1116
|
+
let diskError;
|
|
1113
1117
|
try {
|
|
1114
|
-
const
|
|
1118
|
+
const diskEntries = readdirSync(dir, { withFileTypes: true });
|
|
1119
|
+
for (const entry of diskEntries)
|
|
1120
|
+
diskNames.add(entry.name);
|
|
1121
|
+
diskSection = diskEntries
|
|
1115
1122
|
.map((entry) => `${entry.isDirectory() ? "[DIR]" : "[FILE]"} ${entry.name}`)
|
|
1116
1123
|
.join("\n");
|
|
1117
|
-
return {
|
|
1118
|
-
content: [{ type: "text", text: `# ${dir}\n\n${entries}` }],
|
|
1119
|
-
};
|
|
1120
1124
|
}
|
|
1121
1125
|
catch (err) {
|
|
1126
|
+
diskError = String(err);
|
|
1127
|
+
}
|
|
1128
|
+
const storePrefix = normalizeRelPath(rel);
|
|
1129
|
+
let storeSection;
|
|
1130
|
+
if (existsSync(getWorkspaceStorePath(WORKSPACE_ROOT))) {
|
|
1131
|
+
const storeOnly = listStoreEntries(WORKSPACE_ROOT, storePrefix).filter((entry) => !diskNames.has(entry.path.slice(storePrefix.length).replace(/^\//, "").split("/")[0]));
|
|
1132
|
+
if (storeOnly.length > 0) {
|
|
1133
|
+
storeSection = storeOnly
|
|
1134
|
+
.map((entry) => `[STORE] ${entry.path} (${entry.bytes}B)`)
|
|
1135
|
+
.join("\n");
|
|
1136
|
+
}
|
|
1137
|
+
}
|
|
1138
|
+
if (diskError && !storeSection) {
|
|
1122
1139
|
return {
|
|
1123
1140
|
content: [
|
|
1124
1141
|
{
|
|
1125
1142
|
type: "text",
|
|
1126
|
-
text: `❌ Could not list ${dir}\n${
|
|
1143
|
+
text: `❌ Could not list ${dir}\n${diskError}`,
|
|
1127
1144
|
},
|
|
1128
1145
|
],
|
|
1129
1146
|
};
|
|
1130
1147
|
}
|
|
1148
|
+
const sections = [
|
|
1149
|
+
diskError ? `(no such disk directory: ${diskError})` : diskSection || "(empty)",
|
|
1150
|
+
storeSection ? `\nStore-only (not on disk):\n${storeSection}` : undefined,
|
|
1151
|
+
].filter(Boolean);
|
|
1152
|
+
return {
|
|
1153
|
+
content: [{ type: "text", text: `# ${dir}\n\n${sections.join("\n")}` }],
|
|
1154
|
+
};
|
|
1131
1155
|
});
|
|
1132
1156
|
// ── Safe-edit (new P0 tool) ───────────────────────────────────────
|
|
1133
1157
|
server.tool("safe_edit_file", "Edit a file using copy→validate→swap pattern. Automatically rolls back if validation or tests fail. Prefer apply_patch for small targeted edits. Cost: heavy for large files.", {
|
package/dist/tools-handoff.d.ts
CHANGED
|
@@ -3,13 +3,14 @@
|
|
|
3
3
|
*/
|
|
4
4
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
5
5
|
import { HandoffRecord } from "./handoff-registry.js";
|
|
6
|
-
import { DispatchJobsResult } from "./job-scheduler.js";
|
|
7
6
|
export interface HandoffSchedulerSyncResult {
|
|
8
7
|
ok: boolean;
|
|
9
8
|
job_id: string;
|
|
10
9
|
enqueued: boolean;
|
|
11
10
|
enqueue_reason?: string;
|
|
12
|
-
|
|
11
|
+
dispatch_performed: false;
|
|
12
|
+
reconciliation_required: boolean;
|
|
13
|
+
reconciliation_reason?: string;
|
|
13
14
|
}
|
|
14
15
|
export interface HandoffSchedulerBackfillResult {
|
|
15
16
|
scanned: number;
|
package/dist/tools-handoff.js
CHANGED
|
@@ -10,9 +10,8 @@ import { validateHandoffPayload, validateSwarmHandoffPayload, lintHandoffPayload
|
|
|
10
10
|
import { acknowledgeHandoffSafe, getHandoffRegistryPath, readHandoffRegistry, registerHandoffSafe, } from "./handoff-registry.js";
|
|
11
11
|
import { appendRunLedgerEntrySafe } from "./run-ledger.js";
|
|
12
12
|
import { refreshKanbanArtifacts } from "./kanban.js";
|
|
13
|
-
import {
|
|
13
|
+
import { readJobQueue, syncHandoffLifecycleJob } from "./job-scheduler.js";
|
|
14
14
|
import { appendStatusEventSafe } from "./status-events.js";
|
|
15
|
-
const DEFAULT_SCHEDULER_OWNER = "capability-ops";
|
|
16
15
|
function toSchedulerPriority(input) {
|
|
17
16
|
if (input === "P0" || input === "P1" || input === "P2" || input === "P3") {
|
|
18
17
|
return input;
|
|
@@ -62,9 +61,9 @@ async function recordHandoffSchedulerSyncEffects(record, sync) {
|
|
|
62
61
|
}
|
|
63
62
|
}
|
|
64
63
|
export async function syncHandoffIntoScheduler(record, options = {}) {
|
|
65
|
-
const owner = options.owner?.trim() || DEFAULT_SCHEDULER_OWNER;
|
|
66
64
|
const jobId = handoffSchedulerJobId(record.handoff_id);
|
|
67
|
-
const enqueue = await
|
|
65
|
+
const enqueue = await syncHandoffLifecycleJob({
|
|
66
|
+
now_iso: options.now_iso,
|
|
68
67
|
job_id: jobId,
|
|
69
68
|
kind: "handoff.lifecycle",
|
|
70
69
|
priority: toSchedulerPriority(record.priority),
|
|
@@ -85,28 +84,30 @@ export async function syncHandoffIntoScheduler(record, options = {}) {
|
|
|
85
84
|
],
|
|
86
85
|
resource_requirements: buildHandoffResources(record),
|
|
87
86
|
});
|
|
88
|
-
const enqueued = enqueue.
|
|
87
|
+
const enqueued = enqueue.enqueued;
|
|
89
88
|
const enqueueReason = enqueue.ok || !enqueue.nack
|
|
90
89
|
? undefined
|
|
91
90
|
: `${enqueue.nack.reason_code}: ${enqueue.nack.reason}`;
|
|
92
|
-
const tick = await dispatchJobs({
|
|
93
|
-
owner,
|
|
94
|
-
...(options.now_iso ? { now_iso: options.now_iso } : {}),
|
|
95
|
-
});
|
|
96
91
|
const syncResult = {
|
|
97
|
-
ok: enqueue.ok
|
|
92
|
+
ok: enqueue.ok,
|
|
98
93
|
job_id: jobId,
|
|
99
94
|
enqueued,
|
|
100
95
|
enqueue_reason: enqueueReason,
|
|
101
|
-
|
|
96
|
+
dispatch_performed: false,
|
|
97
|
+
reconciliation_required: enqueue.ack?.status === "unknown_recovery",
|
|
98
|
+
reconciliation_reason: enqueue.ack?.status === "unknown_recovery"
|
|
99
|
+
? enqueue.queue.jobs.find((job) => job.job_id === jobId)?.error
|
|
100
|
+
: undefined,
|
|
102
101
|
};
|
|
103
102
|
await recordHandoffSchedulerSyncEffects(record, syncResult);
|
|
104
103
|
return syncResult;
|
|
105
104
|
}
|
|
106
105
|
export async function backfillHandoffsIntoScheduler(options = {}) {
|
|
107
106
|
const registry = readHandoffRegistry();
|
|
107
|
+
const existing = new Set(readJobQueue().jobs.filter((job) => job.kind === "handoff.lifecycle")
|
|
108
|
+
.map((job) => job.payload.handoff_id));
|
|
108
109
|
const active = Object.values(registry.handoffs)
|
|
109
|
-
.filter((record) => record.status !== "completed")
|
|
110
|
+
.filter((record) => record.status !== "completed" || existing.has(record.handoff_id))
|
|
110
111
|
.sort((a, b) => Date.parse(a.created_at) - Date.parse(b.created_at));
|
|
111
112
|
const result = {
|
|
112
113
|
scanned: active.length,
|
|
@@ -117,16 +118,15 @@ export async function backfillHandoffsIntoScheduler(options = {}) {
|
|
|
117
118
|
for (const record of active) {
|
|
118
119
|
try {
|
|
119
120
|
const sync = await syncHandoffIntoScheduler(record, options);
|
|
121
|
+
if (sync.enqueued)
|
|
122
|
+
result.enqueued += 1;
|
|
120
123
|
if (sync.ok) {
|
|
121
124
|
result.synced += 1;
|
|
122
|
-
if (sync.enqueued) {
|
|
123
|
-
result.enqueued += 1;
|
|
124
|
-
}
|
|
125
125
|
}
|
|
126
|
-
|
|
126
|
+
if (!sync.ok || sync.reconciliation_required) {
|
|
127
127
|
result.failures.push({
|
|
128
128
|
handoff_id: record.handoff_id,
|
|
129
|
-
error: sync.enqueue_reason ?? "scheduler sync returned not-ok",
|
|
129
|
+
error: sync.reconciliation_reason ?? sync.enqueue_reason ?? "scheduler sync returned not-ok",
|
|
130
130
|
});
|
|
131
131
|
}
|
|
132
132
|
}
|
|
@@ -282,7 +282,10 @@ export function registerHandoffTools(server) {
|
|
|
282
282
|
`Handoff registry: ${registry.path}`,
|
|
283
283
|
`Run ledger: ${runLedger.path} (${runLedger.entry.id})`,
|
|
284
284
|
`Scheduler job: ${schedulerSync.job_id} (${schedulerSync.enqueued ? "enqueued" : "already_exists"})`,
|
|
285
|
-
|
|
285
|
+
"Scheduler: handoff metadata synchronized; no execution dispatch",
|
|
286
|
+
schedulerSync.reconciliation_required
|
|
287
|
+
? `Scheduler reconciliation required: ${schedulerSync.reconciliation_reason}`
|
|
288
|
+
: "",
|
|
286
289
|
schedulerSync.enqueue_reason
|
|
287
290
|
? `Scheduler enqueue detail: ${schedulerSync.enqueue_reason}`
|
|
288
291
|
: "",
|
|
@@ -466,7 +469,10 @@ export function registerHandoffTools(server) {
|
|
|
466
469
|
? `scheduler-job: ${schedulerSync.job_id} (${schedulerSync.enqueued ? "enqueued" : "already_exists"})`
|
|
467
470
|
: "",
|
|
468
471
|
schedulerSync
|
|
469
|
-
?
|
|
472
|
+
? "scheduler: handoff metadata synchronized; no execution dispatch"
|
|
473
|
+
: "",
|
|
474
|
+
schedulerSync?.reconciliation_required
|
|
475
|
+
? `scheduler-reconciliation-required: ${schedulerSync.reconciliation_reason}`
|
|
470
476
|
: "",
|
|
471
477
|
schedulerSync?.enqueue_reason
|
|
472
478
|
? `scheduler-enqueue-detail: ${schedulerSync.enqueue_reason}`
|
package/dist/tools-scheduler.js
CHANGED
|
@@ -144,6 +144,7 @@ export function registerSchedulerTools(server) {
|
|
|
144
144
|
result.ok ? "✅ Scheduler dispatch completed" : "⚠️ Scheduler lease not acquired",
|
|
145
145
|
`Owner: ${result.owner}`,
|
|
146
146
|
`Lease acquired: ${result.lease_acquired}`,
|
|
147
|
+
`Lease retained for execution: ${result.lease_retained}`,
|
|
147
148
|
`Queue: ${result.queue_path}`,
|
|
148
149
|
`Locks: ${result.lock_path}`,
|
|
149
150
|
asJsonBlock(result),
|
|
@@ -78,11 +78,16 @@ create_handoff {
|
|
|
78
78
|
| `get_lock_table` | Read active resource locks |
|
|
79
79
|
| `get_scheduler_lease` | Read the current scheduler lease |
|
|
80
80
|
|
|
81
|
+
Dispatch reserves resources and records running work; a worker executes the job
|
|
82
|
+
body. `handoff.lifecycle` is reserved for metadata synchronization and is never
|
|
83
|
+
dispatched. Empty ticks release ownership; `lease_retained` in the dispatch result
|
|
84
|
+
indicates whether execution still holds the lease afterward.
|
|
85
|
+
|
|
81
86
|
Example — queuing a blocked job:
|
|
82
87
|
|
|
83
88
|
```text
|
|
84
89
|
enqueue_job {
|
|
85
|
-
"kind": "
|
|
90
|
+
"kind": "example.analysis",
|
|
86
91
|
"priority": "P1",
|
|
87
92
|
"dependencies": [{ "kind": "handoff", "ref": "HND-20260710-abc12345", "expected": "accepted" }],
|
|
88
93
|
"resource_requirements": ["agent-state/handoff-registry.json"],
|
package/docs/troubleshooting.md
CHANGED
|
@@ -34,6 +34,35 @@ Store-first is normal. `ACE_STORE_FIRST=0` is a legacy diagnostic escape hatch.
|
|
|
34
34
|
|
|
35
35
|
Run `ace fsck --json`. A `.corrupt-*` file is a preserved copy, not disposable litter. `ace repair` re-bakes known assets and projections; it does not reconstruct arbitrary corrupt bytes. Restore from a verified backup when structural checks cannot be repaired safely.
|
|
36
36
|
|
|
37
|
+
## Scheduler
|
|
38
|
+
|
|
39
|
+
### Jobs are recorded but dispatch reports a held lease
|
|
40
|
+
|
|
41
|
+
Enqueue and dispatch are separate operations. `get_scheduler_lease` identifies the
|
|
42
|
+
current owner and expiry; `dispatch_jobs` returns `scheduler_lease_held` when
|
|
43
|
+
another owner holds an unexpired lease protecting running work or active locks.
|
|
44
|
+
An idle lease is reclaimed during dispatch without waiting for expiry. Empty
|
|
45
|
+
or dependency-blocked ticks retain no lease, and completing the last running job
|
|
46
|
+
releases it. `lease_acquired` describes admission for the tick; `lease_retained`
|
|
47
|
+
says whether execution still holds ownership afterward. The returned `lease` is
|
|
48
|
+
the admission receipt, even if it was released at the end of the tick.
|
|
49
|
+
|
|
50
|
+
Handoff creation, acknowledgment, and startup backfill synchronize lifecycle
|
|
51
|
+
metadata without execution dispatch. `handoff.lifecycle` records cannot execute
|
|
52
|
+
through either dispatch tool. Legacy records that appeared to be running are
|
|
53
|
+
parked in `unknown_recovery` with their existing locks preserved until the handoff
|
|
54
|
+
completion receipt resolves them. Backfill also reconciles completed handoffs
|
|
55
|
+
that already have scheduler records; it preserves IDs and history.
|
|
56
|
+
|
|
57
|
+
Parallel-group promotion leaves undispatched steps planned and retains their job
|
|
58
|
+
IDs. Retrying promotion reuses the same jobs, including when retrying the original
|
|
59
|
+
plan. Resource contention can leave a step queued after successful lease acquisition.
|
|
60
|
+
|
|
61
|
+
Running work still requires its owning scheduler. Do not adopt another owner's
|
|
62
|
+
identity or delete locks to bypass contention. Check host-agent execution receipts
|
|
63
|
+
before retrying work already launched outside ACE. Ordinary watchdog recovery
|
|
64
|
+
leaves stale jobs parked; acquiring a released lease does not resume those jobs.
|
|
65
|
+
|
|
37
66
|
## Providers
|
|
38
67
|
|
|
39
68
|
### Ollama unreachable or model missing
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@voybio/ace-swarm",
|
|
3
|
-
"version": "2.4.
|
|
3
|
+
"version": "2.4.9",
|
|
4
4
|
"description": "ACE Framework MCP server and CLI — single-file ACEPACK state, local-model serving, agent orchestration, and host compliance enforcement.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|