@the-open-engine/zeroshot 6.25.0 → 6.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/index.js +102 -74
- package/package.json +4 -2
- package/src/agent/provider-session.js +18 -4
- package/src/hosted-target/bounds.ts +6 -0
- package/src/hosted-target/errors.ts +90 -0
- package/src/hosted-target/index.ts +44 -0
- package/src/hosted-target/response-validation.ts +86 -0
- package/src/hosted-target/retry.ts +46 -0
- package/src/hosted-target/target-adapter.ts +10 -0
- package/src/hosted-target/types.ts +58 -0
- package/src/hosted-target/zero-cloud-v1-adapter.ts +386 -0
- package/src/omp-session-limits.js +25 -1
- package/src/omp-session-partition.js +159 -32
- package/src/omp-session-verifier.js +251 -87
- package/task-lib/commands/clean.js +80 -46
- package/task-lib/commands/kill.js +21 -0
- package/task-lib/commands/resume.js +20 -0
- package/task-lib/commands/run.js +17 -2
- package/task-lib/omp-session-cleanup.js +46 -9
- package/task-lib/omp-session-ownership-schema.js +21 -15
- package/task-lib/omp-session-ownership.js +66 -31
- package/task-lib/rpc-watcher.js +51 -15
- package/task-lib/runner.js +51 -4
- package/task-lib/store.js +67 -72
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { unlinkSync, existsSync } from 'fs';
|
|
2
2
|
import chalk from 'chalk';
|
|
3
|
-
import { loadTasks,
|
|
3
|
+
import { clearTaskCommandCleanup, loadTasks, removeTaskIfUnchanged } from '../store.js';
|
|
4
4
|
import { createCommandSpecCleanup } from '../command-spec-cleanup.js';
|
|
5
5
|
import { cleanupOmpSessionPartitionForTask } from '../omp-session-cleanup.js';
|
|
6
6
|
|
|
@@ -8,13 +8,82 @@ import { cleanupOmpSessionPartitionForTask } from '../omp-session-cleanup.js';
|
|
|
8
8
|
* Delete a task's OMP session partition directory as part of removing its row. Every ownership
|
|
9
9
|
* state is cleaned here, including `provisional`: the row is going away, so leaving its partition
|
|
10
10
|
* behind would orphan a directory nothing can ever reclaim. The shared OMP CAS blob root is never
|
|
11
|
-
* touched. An unsafe/unresolvable path
|
|
12
|
-
*
|
|
11
|
+
* touched. An unsafe/unresolvable path — or an ownership record that exists but cannot be read —
|
|
12
|
+
* preserves the record, and therefore the whole task row, with an actionable warning.
|
|
13
13
|
*/
|
|
14
14
|
export function cleanUpOmpSessionPartition(task, warn) {
|
|
15
15
|
return cleanupOmpSessionPartitionForTask(task, warn);
|
|
16
16
|
}
|
|
17
17
|
|
|
18
|
+
/**
|
|
19
|
+
* The live-task retention boundary, evaluated before *any* cleanup side effect.
|
|
20
|
+
*
|
|
21
|
+
* A running task owns everything the row points at: its OMP session partition is the working
|
|
22
|
+
* directory of a live provider process, and its command-cleanup receipt names paths that process
|
|
23
|
+
* is still using. This used to be checked only inside the `commandCleanup` branch — i.e. after the
|
|
24
|
+
* OMP partition had already been staged and recursively deleted — so `clean --all` could destroy a
|
|
25
|
+
* live session's transcript for any task that happened not to carry a cleanup receipt.
|
|
26
|
+
*/
|
|
27
|
+
function isLiveTask(task) {
|
|
28
|
+
return task.status === 'running';
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Remove one task row that `clean` selected, in the only order that is safe:
|
|
33
|
+
* live-task check, then OMP partition, then command cleanup, then log file, then an owner-fenced
|
|
34
|
+
* row delete.
|
|
35
|
+
*
|
|
36
|
+
* `task` is a snapshot from the caller's single `loadTasks()`, so the delete is conditional on the
|
|
37
|
+
* row still matching it (see removeTaskIfUnchanged). A watcher update, a kill, or a resume's
|
|
38
|
+
* ownership transfer landing mid-cleanup leaves the row in place rather than being reverted by a
|
|
39
|
+
* whole-table rewrite.
|
|
40
|
+
*
|
|
41
|
+
* @returns {{removed: boolean, reason: string|null}} `reason` is a short retention label
|
|
42
|
+
*/
|
|
43
|
+
export function removeCleanedTask(task, { warn }) {
|
|
44
|
+
if (isLiveTask(task)) {
|
|
45
|
+
return { removed: false, reason: 'running' };
|
|
46
|
+
}
|
|
47
|
+
if (!cleanUpOmpSessionPartition(task, warn)) {
|
|
48
|
+
return { removed: false, reason: 'OMP partition cleanup pending' };
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
let cleanupCleared = false;
|
|
52
|
+
if (task.commandCleanup) {
|
|
53
|
+
let recovered = false;
|
|
54
|
+
try {
|
|
55
|
+
const cleanup = createCommandSpecCleanup(task.commandCleanup, (cleanupPath, error) => {
|
|
56
|
+
warn(`failed to clean up ${cleanupPath}: ${error.message}`);
|
|
57
|
+
});
|
|
58
|
+
recovered = cleanup.runSync();
|
|
59
|
+
} catch (error) {
|
|
60
|
+
warn(`failed to validate cleanup for task ${task.id}: ${error.message}`);
|
|
61
|
+
}
|
|
62
|
+
if (!recovered) {
|
|
63
|
+
return { removed: false, reason: 'command cleanup pending' };
|
|
64
|
+
}
|
|
65
|
+
cleanupCleared = true;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
if (task.logFile && existsSync(task.logFile)) {
|
|
69
|
+
unlinkSync(task.logFile);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
if (
|
|
73
|
+
!removeTaskIfUnchanged(task.id, {
|
|
74
|
+
status: task.status,
|
|
75
|
+
ompSessionOwnership: task.ompSessionOwnership ?? null,
|
|
76
|
+
})
|
|
77
|
+
) {
|
|
78
|
+
// The row moved on under us. Its side effects are already done, so record the one piece of
|
|
79
|
+
// durable state that would otherwise be retried forever, using a single-column write that
|
|
80
|
+
// cannot clobber whatever the concurrent writer just persisted.
|
|
81
|
+
if (cleanupCleared) clearTaskCommandCleanup(task.id, task.commandCleanup);
|
|
82
|
+
return { removed: false, reason: 'the row changed while it was being cleaned' };
|
|
83
|
+
}
|
|
84
|
+
return { removed: true, reason: null };
|
|
85
|
+
}
|
|
86
|
+
|
|
18
87
|
export function cleanTasks(options = {}) {
|
|
19
88
|
const tasks = loadTasks();
|
|
20
89
|
const taskList = Object.values(tasks);
|
|
@@ -48,56 +117,21 @@ export function cleanTasks(options = {}) {
|
|
|
48
117
|
console.log(chalk.dim(`Removing ${toRemove.length} task(s)...\n`));
|
|
49
118
|
|
|
50
119
|
for (const task of toRemove) {
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
) {
|
|
120
|
+
const { removed, reason } = removeCleanedTask(task, {
|
|
121
|
+
warn: (message) => console.log(chalk.yellow(`Warning: ${message}`)),
|
|
122
|
+
});
|
|
123
|
+
if (!removed) {
|
|
56
124
|
cleanupFailed = true;
|
|
57
|
-
console.log(
|
|
58
|
-
chalk.yellow(` Retained: ${task.id} [${task.status}] (OMP partition cleanup pending)`)
|
|
59
|
-
);
|
|
125
|
+
console.log(chalk.yellow(` Retained: ${task.id} [${task.status}] (${reason})`));
|
|
60
126
|
continue;
|
|
61
127
|
}
|
|
62
|
-
if (task.commandCleanup) {
|
|
63
|
-
if (task.status === 'running') {
|
|
64
|
-
cleanupFailed = true;
|
|
65
|
-
console.log(
|
|
66
|
-
chalk.yellow(` Retained: ${task.id} [running] (live command cleanup ownership)`)
|
|
67
|
-
);
|
|
68
|
-
continue;
|
|
69
|
-
}
|
|
70
|
-
let recovered = false;
|
|
71
|
-
try {
|
|
72
|
-
const cleanup = createCommandSpecCleanup(task.commandCleanup, (cleanupPath, error) => {
|
|
73
|
-
console.log(chalk.yellow(`Warning: failed to clean up ${cleanupPath}: ${error.message}`));
|
|
74
|
-
});
|
|
75
|
-
recovered = cleanup.runSync();
|
|
76
|
-
} catch (error) {
|
|
77
|
-
console.log(
|
|
78
|
-
chalk.yellow(`Warning: failed to validate cleanup for task ${task.id}: ${error.message}`)
|
|
79
|
-
);
|
|
80
|
-
}
|
|
81
|
-
if (!recovered) {
|
|
82
|
-
cleanupFailed = true;
|
|
83
|
-
console.log(
|
|
84
|
-
chalk.yellow(` Retained: ${task.id} [${task.status}] (command cleanup pending)`)
|
|
85
|
-
);
|
|
86
|
-
continue;
|
|
87
|
-
}
|
|
88
|
-
task.commandCleanup = null;
|
|
89
|
-
}
|
|
90
|
-
if (task.logFile && existsSync(task.logFile)) {
|
|
91
|
-
unlinkSync(task.logFile);
|
|
92
|
-
}
|
|
93
|
-
|
|
94
128
|
console.log(chalk.dim(` Removed: ${task.id} [${task.status}]`));
|
|
95
|
-
delete tasks[task.id];
|
|
96
129
|
removedCount++;
|
|
97
130
|
}
|
|
98
131
|
|
|
99
|
-
|
|
100
|
-
|
|
132
|
+
// No whole-table rewrite here by design. Rows are deleted individually above, each fenced on the
|
|
133
|
+
// snapshot it was validated against, so a concurrent watcher/kill/ownership-transfer write is
|
|
134
|
+
// never reverted by cleanup finishing after it.
|
|
101
135
|
console.log(chalk.green(`\n✓ Cleaned ${removedCount} task(s)`));
|
|
102
136
|
if (cleanupFailed) process.exitCode = 1;
|
|
103
137
|
}
|
|
@@ -2,6 +2,25 @@ import chalk from 'chalk';
|
|
|
2
2
|
import { getTask, requestTaskCancellation, updateTask } from '../store.js';
|
|
3
3
|
import { createCommandSpecCleanup } from '../command-spec-cleanup.js';
|
|
4
4
|
import { terminateProcess } from '../process-termination.js';
|
|
5
|
+
import { retireOmpOwnershipAtTerminalBoundary } from '../omp-session-ownership.js';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Retire the task's OMP session ownership at a confirmed terminal boundary (killed / stale).
|
|
9
|
+
*
|
|
10
|
+
* A killed task's provisional partition claim would otherwise outlive the process that made it: no
|
|
11
|
+
* watcher is left to reach `finalizeOmpOwnership`, and cleanup refuses to reclaim a partition any
|
|
12
|
+
* row still claims provisionally, so the directory would be unreclaimable forever. Runs *before*
|
|
13
|
+
* the terminal status write so no window exists where the row is terminal but still claiming.
|
|
14
|
+
*/
|
|
15
|
+
function retireOmpOwnershipForKilledTask(taskId) {
|
|
16
|
+
retireOmpOwnershipAtTerminalBoundary(taskId, (error) => {
|
|
17
|
+
console.log(
|
|
18
|
+
chalk.yellow(
|
|
19
|
+
`Warning: failed to retire the OMP session ownership of task ${taskId}: ${error.message}`
|
|
20
|
+
)
|
|
21
|
+
);
|
|
22
|
+
});
|
|
23
|
+
}
|
|
5
24
|
|
|
6
25
|
async function cleanupTerminatedTask(task) {
|
|
7
26
|
if (!task.commandCleanup) return {};
|
|
@@ -125,6 +144,7 @@ export async function killTaskCommand(taskId, options = {}) {
|
|
|
125
144
|
}
|
|
126
145
|
console.log(chalk.yellow('Process already dead, updating status...'));
|
|
127
146
|
const cleanupUpdate = await cleanupTerminatedTask(task);
|
|
147
|
+
retireOmpOwnershipForKilledTask(taskId);
|
|
128
148
|
updateTask(taskId, {
|
|
129
149
|
status: 'stale',
|
|
130
150
|
pid: null,
|
|
@@ -147,6 +167,7 @@ export async function killTaskCommand(taskId, options = {}) {
|
|
|
147
167
|
if (result.degraded) {
|
|
148
168
|
console.log(chalk.yellow(`Warning: ${result.degradedReason}`));
|
|
149
169
|
}
|
|
170
|
+
retireOmpOwnershipForKilledTask(taskId);
|
|
150
171
|
updateTask(taskId, {
|
|
151
172
|
status: 'killed',
|
|
152
173
|
pid: null,
|
|
@@ -13,12 +13,29 @@ const { providerSupportsCapability } = require('../../lib/provider-names.js');
|
|
|
13
13
|
* including the partition identity, which the cluster path can only learn from the row itself.
|
|
14
14
|
* `state === 'committed'` is required: a provisional or cleanup-required record never durably
|
|
15
15
|
* proved a resumable session, so anything less fails closed to a fresh context.
|
|
16
|
+
*
|
|
17
|
+
* This surface is standalone-only, and refuses a `cluster-agent` owner outright.
|
|
18
|
+
*
|
|
19
|
+
* A cluster-agent lineage belongs to a live agent generation: the committed record is the tail of
|
|
20
|
+
* an `agentId`/`clusterId`/iteration chain whose next turn is spawned by that agent process, and
|
|
21
|
+
* only that process can reach the post-hook boundary where a cluster-agent owner may be committed
|
|
22
|
+
* (agent-lifecycle.js#finalizeProviderSessionAfterCommit). Handing those ids to a detached
|
|
23
|
+
* `zeroshot task resume` would transfer the whole lineage onto a row no parent agent knows about
|
|
24
|
+
* or can ever commit: the prior owner's record is cleared by the transfer, the resumed row stays
|
|
25
|
+
* provisional to the end of time, and the partition becomes unreclaimable while the agent that
|
|
26
|
+
* *should* own the continuation silently falls back to a fresh context. Refusing before spawn is
|
|
27
|
+
* what keeps that lineage intact and resumable by its real owner.
|
|
16
28
|
*/
|
|
17
29
|
function buildOmpResumeTaskOptions(task) {
|
|
18
30
|
const ownership = validateOwnedByTask(task.ompSessionOwnership, task.id);
|
|
19
31
|
if (!ownership) {
|
|
20
32
|
throw new Error(`Task ${task.id} has no valid OMP session ownership record; refusing resume.`);
|
|
21
33
|
}
|
|
34
|
+
if (ownership.owner.kind !== 'standalone') {
|
|
35
|
+
throw new Error(
|
|
36
|
+
`Task ${task.id} OMP session is owned by cluster agent ${ownership.owner.clusterId}/${ownership.owner.agentId}; manual resume is standalone-only. Let the owning agent continue it.`
|
|
37
|
+
);
|
|
38
|
+
}
|
|
22
39
|
if (ownership.state !== 'committed' || !ownership.session || !ownership.partitionIdentity) {
|
|
23
40
|
throw new Error(
|
|
24
41
|
`Task ${task.id} OMP session ownership is '${ownership.state}', not a committed resumable session; refusing resume.`
|
|
@@ -28,6 +45,9 @@ function buildOmpResumeTaskOptions(task) {
|
|
|
28
45
|
cwd: ownership.canonicalWorkspace,
|
|
29
46
|
provider: task.provider,
|
|
30
47
|
storageRoot: ownership.storageRoot,
|
|
48
|
+
// A standalone owner carries null cluster/agent ids by schema, so the resumed row is
|
|
49
|
+
// standalone too. They are passed explicitly rather than omitted so this stays an exact
|
|
50
|
+
// lineage copy of the record above, not an inference.
|
|
31
51
|
clusterId: ownership.owner.clusterId,
|
|
32
52
|
agentId: ownership.owner.agentId,
|
|
33
53
|
ompResume: {
|
package/task-lib/commands/run.js
CHANGED
|
@@ -23,13 +23,28 @@ const OMP_RESUME_ALLOWED_FIELDS = new Set([
|
|
|
23
23
|
...OMP_RESUME_OPTIONAL_IDENTITY_FIELDS,
|
|
24
24
|
]);
|
|
25
25
|
|
|
26
|
+
// Issue #866 fixes device/inode as *canonical unsigned decimal strings*. This descriptor arrives
|
|
27
|
+
// over argv from another process, so the type is checked, never coerced: `String(value.device)`
|
|
28
|
+
// would have accepted the JSON number 42, `new String('42')`, `['42']`, and anything else with a
|
|
29
|
+
// matching toString, then silently canonicalized it into a string the persisted record never
|
|
30
|
+
// contained. A descriptor that does not already carry the canonical form is a descriptor built by
|
|
31
|
+
// something other than this codebase's writer, and it fails closed rather than being repaired.
|
|
32
|
+
const CANONICAL_DECIMAL = /^(0|[1-9][0-9]*)$/;
|
|
33
|
+
const IDENTITY_KEYS = new Set(['device', 'inode']);
|
|
34
|
+
|
|
35
|
+
function isCanonicalDecimalString(value) {
|
|
36
|
+
return typeof value === 'string' && CANONICAL_DECIMAL.test(value);
|
|
37
|
+
}
|
|
38
|
+
|
|
26
39
|
function isIdentityShape(value) {
|
|
27
40
|
return (
|
|
28
41
|
value !== null &&
|
|
29
42
|
typeof value === 'object' &&
|
|
30
43
|
!Array.isArray(value) &&
|
|
31
|
-
|
|
32
|
-
|
|
44
|
+
Object.keys(value).length === IDENTITY_KEYS.size &&
|
|
45
|
+
Object.keys(value).every((key) => IDENTITY_KEYS.has(key)) &&
|
|
46
|
+
isCanonicalDecimalString(value.device) &&
|
|
47
|
+
isCanonicalDecimalString(value.inode)
|
|
33
48
|
);
|
|
34
49
|
}
|
|
35
50
|
|
|
@@ -18,7 +18,12 @@
|
|
|
18
18
|
//
|
|
19
19
|
// An unsafe or unresolvable path preserves the owner record with an actionable warning instead of
|
|
20
20
|
// deleting, so the operator can inspect it and the cleanup stays durably retryable.
|
|
21
|
-
import {
|
|
21
|
+
import {
|
|
22
|
+
loadTasks,
|
|
23
|
+
updateTask,
|
|
24
|
+
getTaskStoreDatabase,
|
|
25
|
+
hasUnreadableOmpSessionOwnership,
|
|
26
|
+
} from './store.js';
|
|
22
27
|
import { findAuthoritativeOwnersForPartition } from './omp-session-ownership.js';
|
|
23
28
|
import {
|
|
24
29
|
serializeOmpSessionOwnership,
|
|
@@ -52,10 +57,17 @@ const {
|
|
|
52
57
|
* writes this column only through `serializeOmpSessionOwnership`, whose output is canonical per
|
|
53
58
|
* record, so comparing the stored bytes is an exact "still the same record" test.
|
|
54
59
|
*
|
|
55
|
-
* The recursive removal runs *after* the fence is released
|
|
56
|
-
*
|
|
57
|
-
* arbitrarily large `rm -r` would stall every other
|
|
60
|
+
* The recursive removal runs *after* the fence is released. By then the tree only answers to its
|
|
61
|
+
* deterministic owner-bound staging name, and every retry revalidates that name and identity.
|
|
62
|
+
* Holding a write lock across an arbitrarily large `rm -r` would stall every other store writer.
|
|
58
63
|
*/
|
|
64
|
+
function describeBlockingOwner(owner) {
|
|
65
|
+
if (owner.unknown) {
|
|
66
|
+
return `${owner.taskId} (ownership record is unreadable or invalid; inspect or repair that task row)`;
|
|
67
|
+
}
|
|
68
|
+
return `${owner.taskId} (${owner.state})`;
|
|
69
|
+
}
|
|
70
|
+
|
|
59
71
|
function stageUnderOwnerFence(ownership, taskId) {
|
|
60
72
|
const database = getTaskStoreDatabase();
|
|
61
73
|
const expectedRecord = serializeOmpSessionOwnership(ownership);
|
|
@@ -76,7 +88,7 @@ function stageUnderOwnerFence(ownership, taskId) {
|
|
|
76
88
|
return {
|
|
77
89
|
staged: false,
|
|
78
90
|
deleted: false,
|
|
79
|
-
reason: `it is still claimed by ${owners.map(
|
|
91
|
+
reason: `it is still claimed by ${owners.map(describeBlockingOwner).join(', ')}`,
|
|
80
92
|
};
|
|
81
93
|
}
|
|
82
94
|
return stageOmpSessionPartitionForDeletion(ownership);
|
|
@@ -96,6 +108,17 @@ function stageUnderOwnerFence(ownership, taskId) {
|
|
|
96
108
|
* successful delete — required on surfaces (cluster clear) where the row itself survives.
|
|
97
109
|
*/
|
|
98
110
|
export function cleanupOmpSessionPartitionForTask(task, warn, { clearRecord = false } = {}) {
|
|
111
|
+
// A SQL-NULL ownership column is exact truth that this task never allocated a partition: there
|
|
112
|
+
// is nothing to clean and the row is free to go. An *unreadable* column is the opposite — some
|
|
113
|
+
// partition may exist that only this row still points at — so the row and its evidence are
|
|
114
|
+
// retained for an operator instead of being deleted into an orphan. The malformed bytes are
|
|
115
|
+
// never parsed, canonicalized, or otherwise acted on.
|
|
116
|
+
if (hasUnreadableOmpSessionOwnership(task)) {
|
|
117
|
+
warn(
|
|
118
|
+
`Task ${task.id}: OMP session ownership record is present but unreadable; retaining the task row and its record for inspection. Nothing was deleted, and any partition it named must be reclaimed manually.`
|
|
119
|
+
);
|
|
120
|
+
return false;
|
|
121
|
+
}
|
|
99
122
|
if (!task?.ompSessionOwnership) return true;
|
|
100
123
|
const ownership = validateOwnedByTask(task.ompSessionOwnership, task.id);
|
|
101
124
|
if (!ownership) {
|
|
@@ -119,7 +142,7 @@ export function cleanupOmpSessionPartitionForTask(task, warn, { clearRecord = fa
|
|
|
119
142
|
return false;
|
|
120
143
|
}
|
|
121
144
|
|
|
122
|
-
const { deleted, reason } = removeStagedOmpSessionPartition(staged.stagingPath);
|
|
145
|
+
const { deleted, reason } = removeStagedOmpSessionPartition(staged.stagingPath, ownership);
|
|
123
146
|
if (!deleted) {
|
|
124
147
|
warn(`Task ${task.id}: retained OMP session partition ${ownership.partitionId} (${reason}).`);
|
|
125
148
|
return false;
|
|
@@ -139,14 +162,28 @@ function finishCleanup(task, clearRecord) {
|
|
|
139
162
|
* the cluster's own `storageDir`, so this is what makes cluster clear (and therefore purge)
|
|
140
163
|
* actually reclaim them; the task rows themselves survive and have their ownership cleared.
|
|
141
164
|
*
|
|
142
|
-
*
|
|
165
|
+
* A row whose ownership column is present but unreadable cannot be attributed to a cluster at all
|
|
166
|
+
* — the owner tuple is exactly what is unreadable — so it is reported separately (`unreadable`)
|
|
167
|
+
* rather than silently skipped. Cluster clear keeps task rows, so the evidence survives either way;
|
|
168
|
+
* the warning is what tells the operator a partition may need reclaiming by hand.
|
|
169
|
+
*
|
|
170
|
+
* @returns {{deleted: string[], retained: string[], unreadable: string[]}} partition ids, plus the
|
|
171
|
+
* task ids whose ownership record could not be read
|
|
143
172
|
*/
|
|
144
173
|
export function cleanupOmpSessionPartitionsForCluster(clusterId, warn) {
|
|
145
174
|
const deleted = [];
|
|
146
175
|
const retained = [];
|
|
147
|
-
|
|
176
|
+
const unreadable = [];
|
|
177
|
+
if (!clusterId) return { deleted, retained, unreadable };
|
|
148
178
|
|
|
149
179
|
for (const task of Object.values(loadTasks())) {
|
|
180
|
+
if (hasUnreadableOmpSessionOwnership(task)) {
|
|
181
|
+
unreadable.push(task.id);
|
|
182
|
+
warn(
|
|
183
|
+
`Task ${task.id}: OMP session ownership record is present but unreadable, so it cannot be attributed to a cluster; the row and its record are retained for inspection.`
|
|
184
|
+
);
|
|
185
|
+
continue;
|
|
186
|
+
}
|
|
150
187
|
const ownership = validateOwnedByTask(task?.ompSessionOwnership ?? null, task?.id);
|
|
151
188
|
if (!ownership || ownership.owner.kind !== 'cluster-agent') continue;
|
|
152
189
|
if (ownership.owner.clusterId !== clusterId) continue;
|
|
@@ -156,5 +193,5 @@ export function cleanupOmpSessionPartitionsForCluster(clusterId, warn) {
|
|
|
156
193
|
retained.push(ownership.partitionId);
|
|
157
194
|
}
|
|
158
195
|
}
|
|
159
|
-
return { deleted, retained };
|
|
196
|
+
return { deleted, retained, unreadable };
|
|
160
197
|
}
|
|
@@ -8,7 +8,6 @@
|
|
|
8
8
|
// `owner`, `session`, or either identity) rejects the whole record, and every known key is
|
|
9
9
|
// re-derived into a canonical form on the way out. A record that does not validate is never
|
|
10
10
|
// partially trusted — callers fail closed to a fresh context.
|
|
11
|
-
import { createHash } from 'crypto';
|
|
12
11
|
import { isAbsolute, resolve as resolvePath } from 'path';
|
|
13
12
|
import { createRequire } from 'module';
|
|
14
13
|
|
|
@@ -16,11 +15,7 @@ const require = createRequire(import.meta.url);
|
|
|
16
15
|
const { PARTITION_ID_PATTERN, partitionPathFor } = require('../src/omp-session-partition.js');
|
|
17
16
|
|
|
18
17
|
export const OMP_OWNERSHIP_SCHEMA_VERSION = 1;
|
|
19
|
-
export const OMP_OWNERSHIP_STATES = Object.freeze([
|
|
20
|
-
'provisional',
|
|
21
|
-
'committed',
|
|
22
|
-
'cleanup-required',
|
|
23
|
-
]);
|
|
18
|
+
export const OMP_OWNERSHIP_STATES = Object.freeze(['provisional', 'committed', 'cleanup-required']);
|
|
24
19
|
|
|
25
20
|
const SHA256_PATTERN = /^sha256:[a-f0-9]{64}$/u;
|
|
26
21
|
const DECIMAL_PATTERN = /^(0|[1-9][0-9]*)$/u;
|
|
@@ -89,14 +84,6 @@ export function canonicalOwnerUid() {
|
|
|
89
84
|
return typeof process.getuid === 'function' ? String(process.getuid()) : '0';
|
|
90
85
|
}
|
|
91
86
|
|
|
92
|
-
/** sha256 over the UTF-8 bytes of a stable JSON encoding (sorted keys) of `fields`. */
|
|
93
|
-
export function computeExecutionFingerprint(fields) {
|
|
94
|
-
const sortedKeys = Object.keys(fields).sort();
|
|
95
|
-
const stable = {};
|
|
96
|
-
for (const key of sortedKeys) stable[key] = fields[key];
|
|
97
|
-
return `sha256:${createHash('sha256').update(JSON.stringify(stable), 'utf8').digest('hex')}`;
|
|
98
|
-
}
|
|
99
|
-
|
|
100
87
|
function normalizeOwner(owner) {
|
|
101
88
|
if (!isPlainObject(owner) || !hasOnlyKeys(owner, OWNER_KEYS)) return null;
|
|
102
89
|
if (!OWNER_KINDS.has(owner.kind)) return null;
|
|
@@ -179,7 +166,9 @@ export function validateOmpSessionOwnership(value) {
|
|
|
179
166
|
// No partially populated pairs, in any state: an observation of the materialized session is
|
|
180
167
|
// either complete (both the partition identity and the full session tuple) or absent.
|
|
181
168
|
if (hasPartitionIdentity !== hasSession) return null;
|
|
182
|
-
const partitionIdentity = hasPartitionIdentity
|
|
169
|
+
const partitionIdentity = hasPartitionIdentity
|
|
170
|
+
? normalizeIdentity(value.partitionIdentity)
|
|
171
|
+
: null;
|
|
183
172
|
const session = hasSession ? normalizeSession(value.session) : null;
|
|
184
173
|
if (hasPartitionIdentity && (!partitionIdentity || !session)) return null;
|
|
185
174
|
// `committed` is the only state that asserts a resumable session, so it is the only state that
|
|
@@ -252,6 +241,23 @@ export function parseOmpSessionOwnership(raw) {
|
|
|
252
241
|
return validateOmpSessionOwnership(parsed);
|
|
253
242
|
}
|
|
254
243
|
|
|
244
|
+
/**
|
|
245
|
+
* Closed raw-column inspection seam for a stored `omp_session_ownership` value.
|
|
246
|
+
*
|
|
247
|
+
* `parseOmpSessionOwnership` collapses *both* "SQL NULL" and "non-null but unreadable" to `null`,
|
|
248
|
+
* and those two mean opposite things to cleanup: SQL NULL is exact truth that there is nothing to
|
|
249
|
+
* clean, while unreadable bytes are evidence that a partition may exist whose owner record we can
|
|
250
|
+
* no longer interpret. Deleting such a row would orphan that partition permanently, so every
|
|
251
|
+
* cleanup surface needs to tell them apart.
|
|
252
|
+
*
|
|
253
|
+
* The seam is deliberately closed: it reports only `{present, valid}` and never hands the raw text
|
|
254
|
+
* back, so nothing downstream can be tempted to parse, canonicalize, or act on malformed JSON.
|
|
255
|
+
*/
|
|
256
|
+
export function inspectStoredOmpSessionOwnership(raw) {
|
|
257
|
+
if (raw === null || raw === undefined) return { present: false, valid: false };
|
|
258
|
+
return { present: true, valid: parseOmpSessionOwnership(raw) !== null };
|
|
259
|
+
}
|
|
260
|
+
|
|
255
261
|
export function serializeOmpSessionOwnership(value) {
|
|
256
262
|
if (!value) return null;
|
|
257
263
|
const validated = validateOmpSessionOwnership(value);
|
|
@@ -16,12 +16,14 @@
|
|
|
16
16
|
//
|
|
17
17
|
// A resumed turn additionally performs `transferOmpSessionOwnership` before its prompt is written:
|
|
18
18
|
// in one transaction the prior committed owner's record is cleared and its lineage (partition
|
|
19
|
-
// identity + session tuple) is moved onto the resumed task's still-`provisional` row.
|
|
20
|
-
// exactly one *committed* owner — and keeps "committed" meaning what it means everywhere else,
|
|
21
|
-
// this turn's own success boundary has passed — instead of publishing a half-finished
|
|
22
|
-
// continuation as resumable.
|
|
19
|
+
// identity + session tuple) is moved onto the resumed task's still-`provisional` row.
|
|
23
20
|
//
|
|
24
|
-
//
|
|
21
|
+
// That means a partition has at most one *committed* owner, and for the whole span of a resumed
|
|
22
|
+
// turn it has NONE: the authoritative live claimant is `provisional` by design, because
|
|
23
|
+
// "committed" means this row's own success boundary has already passed. Publishing a half-finished
|
|
24
|
+
// continuation as resumable is exactly what that ordering prevents.
|
|
25
|
+
//
|
|
26
|
+
// It also does NOT keep exactly one row *referencing* the partition. A resumed row is inserted (and
|
|
25
27
|
// therefore already names the partition) before its transfer runs, so two competing resumes of the
|
|
26
28
|
// same committed session put three rows on one partition: the prior owner plus both candidates.
|
|
27
29
|
// Only one transfer can win; the loser fails closed and is retired to `cleanup-required` holding a
|
|
@@ -34,14 +36,12 @@ import { statSync } from 'fs';
|
|
|
34
36
|
import { getTask, getTaskStoreDatabase } from './store.js';
|
|
35
37
|
import {
|
|
36
38
|
buildProvisionalOwnership,
|
|
37
|
-
|
|
39
|
+
parseOmpSessionOwnership,
|
|
38
40
|
serializeOmpSessionOwnership,
|
|
39
41
|
validateOmpSessionOwnership,
|
|
40
42
|
validateOwnedByTask,
|
|
41
43
|
} from './omp-session-ownership-schema.js';
|
|
42
44
|
|
|
43
|
-
export { computeExecutionFingerprint };
|
|
44
|
-
|
|
45
45
|
function statIdentity(targetPath) {
|
|
46
46
|
const stat = statSync(targetPath);
|
|
47
47
|
return { device: String(stat.dev), inode: String(stat.ino) };
|
|
@@ -49,12 +49,7 @@ function statIdentity(targetPath) {
|
|
|
49
49
|
|
|
50
50
|
/** Pure builder for the initial provisional record; embed the result in the task row passed to
|
|
51
51
|
* addTask() so the SQL row is durable before the partition directory is created on disk. */
|
|
52
|
-
export function writeProvisionalOwnership({
|
|
53
|
-
partitionId,
|
|
54
|
-
storageRoot,
|
|
55
|
-
canonicalWorkspace,
|
|
56
|
-
owner,
|
|
57
|
-
}) {
|
|
52
|
+
export function writeProvisionalOwnership({ partitionId, storageRoot, canonicalWorkspace, owner }) {
|
|
58
53
|
return buildProvisionalOwnership({
|
|
59
54
|
partitionId,
|
|
60
55
|
storageRoot,
|
|
@@ -185,6 +180,35 @@ export function markCleanupRequired(taskId) {
|
|
|
185
180
|
return casOwnership(taskId, current, updated) ? updated : readOwnership(taskId);
|
|
186
181
|
}
|
|
187
182
|
|
|
183
|
+
/**
|
|
184
|
+
* Retire a task's OMP ownership from a *confirmed durable task boundary* — a failed, cancelled,
|
|
185
|
+
* stale, or killed transition — without ever letting that retirement break the boundary itself.
|
|
186
|
+
*
|
|
187
|
+
* Every caller here is already committing a terminal task-status write (task-lib/runner.js's
|
|
188
|
+
* row-before-directory failure, `zeroshot task kill`, `zeroshot clear`, `zeroshot kill-all`), so
|
|
189
|
+
* this must not throw: an unreachable or locked task store must not turn "the task is killed" into
|
|
190
|
+
* an unhandled rejection. It is idempotent and safe to re-enter — `markCleanupRequired` no-ops once
|
|
191
|
+
* the record has left `provisional`, so a retried kill or a crash-recovery replay converges.
|
|
192
|
+
*
|
|
193
|
+
* The decision is derived from the boundary, never from whether the partition directory happens to
|
|
194
|
+
* exist: a fresh partition can be mid-materialization at exactly this moment, and file presence
|
|
195
|
+
* would answer the wrong question.
|
|
196
|
+
*
|
|
197
|
+
* @param {string} taskId
|
|
198
|
+
* @param {(error: Error) => void} [onError] receives an unexpected store failure for logging
|
|
199
|
+
* @returns {boolean} true when the retirement ran without error (including no-op cases)
|
|
200
|
+
*/
|
|
201
|
+
export function retireOmpOwnershipAtTerminalBoundary(taskId, onError) {
|
|
202
|
+
if (typeof taskId !== 'string' || taskId.length === 0) return true;
|
|
203
|
+
try {
|
|
204
|
+
markCleanupRequired(taskId);
|
|
205
|
+
return true;
|
|
206
|
+
} catch (error) {
|
|
207
|
+
onError?.(error instanceof Error ? error : new Error(String(error)));
|
|
208
|
+
return false;
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
188
212
|
/**
|
|
189
213
|
* The ownership states that constitute a live claim on a partition.
|
|
190
214
|
*
|
|
@@ -201,28 +225,34 @@ export const AUTHORITATIVE_OWNERSHIP_STATES = Object.freeze(['provisional', 'com
|
|
|
201
225
|
|
|
202
226
|
const AUTHORITATIVE_STATES = new Set(AUTHORITATIVE_OWNERSHIP_STATES);
|
|
203
227
|
|
|
204
|
-
/** Every row other than `excludeTaskId` whose
|
|
205
|
-
*
|
|
206
|
-
*
|
|
228
|
+
/** Every row other than `excludeTaskId` whose own valid record names this partition, plus every
|
|
229
|
+
* unreadable or invalid non-null row as global unknown authoritative evidence. A malformed record
|
|
230
|
+
* cannot safely prove which partition it names, so cleanup must assume it may name the partition
|
|
231
|
+
* being considered; skipping it could orphan the only directory that damaged row still points at.
|
|
232
|
+
*
|
|
233
|
+
* The partition filter is applied in JS after validation rather than in SQL. SQLite's
|
|
234
|
+
* `json_extract()` raises "malformed JSON" for a column whose bytes are not valid JSON. Reading the
|
|
235
|
+
* column as opaque text both avoids that failure and lets cleanup represent corruption explicitly
|
|
236
|
+
* instead of treating unknown evidence as absence. */
|
|
207
237
|
function ownersForPartition(partitionId, excludeTaskId, database) {
|
|
208
238
|
const rows = database
|
|
209
239
|
.prepare(
|
|
210
|
-
`SELECT id, omp_session_ownership FROM tasks
|
|
211
|
-
WHERE omp_session_ownership IS NOT NULL
|
|
212
|
-
AND json_extract(omp_session_ownership, '$.partitionId') = ?`
|
|
240
|
+
`SELECT id, omp_session_ownership AS record FROM tasks
|
|
241
|
+
WHERE omp_session_ownership IS NOT NULL`
|
|
213
242
|
)
|
|
214
|
-
.all(
|
|
243
|
+
.all();
|
|
215
244
|
const owners = [];
|
|
216
245
|
for (const row of rows) {
|
|
217
246
|
if (row.id === excludeTaskId) continue;
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
247
|
+
const record = parseOmpSessionOwnership(row.record);
|
|
248
|
+
const owned = record && validateOwnedByTask(record, row.id);
|
|
249
|
+
if (!owned) {
|
|
250
|
+
owners.push({ taskId: row.id, state: null, unknown: true });
|
|
222
251
|
continue;
|
|
223
252
|
}
|
|
224
|
-
|
|
225
|
-
|
|
253
|
+
if (owned.partitionId === partitionId) {
|
|
254
|
+
owners.push({ taskId: row.id, state: owned.state });
|
|
255
|
+
}
|
|
226
256
|
}
|
|
227
257
|
return owners;
|
|
228
258
|
}
|
|
@@ -258,8 +288,8 @@ export function findAuthoritativeOwnersForPartition(
|
|
|
258
288
|
excludeTaskId = null,
|
|
259
289
|
database = getTaskStoreDatabase()
|
|
260
290
|
) {
|
|
261
|
-
return ownersForPartition(partitionId, excludeTaskId, database).filter(
|
|
262
|
-
AUTHORITATIVE_STATES.has(owner.state)
|
|
291
|
+
return ownersForPartition(partitionId, excludeTaskId, database).filter(
|
|
292
|
+
(owner) => owner.unknown || AUTHORITATIVE_STATES.has(owner.state)
|
|
263
293
|
);
|
|
264
294
|
}
|
|
265
295
|
|
|
@@ -270,8 +300,13 @@ export function findAuthoritativeOwnersForPartition(
|
|
|
270
300
|
* Both sides are fenced on their exact current JSON inside one transaction, so the outcome is all
|
|
271
301
|
* or nothing: either the prior owner's record is cleared *and* the resumed row carries the
|
|
272
302
|
* inherited lineage, or nothing changed and the caller must fail the turn closed. There is never a
|
|
273
|
-
* window in which two rows are committed owners of the same partition
|
|
274
|
-
*
|
|
303
|
+
* window in which two rows are *committed* owners of the same partition.
|
|
304
|
+
*
|
|
305
|
+
* There is, however, a long window in which *no* row is committed: from the instant this transfer
|
|
306
|
+
* applies until the resumed turn reaches its own success boundary, the authoritative claimant is
|
|
307
|
+
* this `provisional` row. That is the intended steady state of a resumed turn, not a gap — which
|
|
308
|
+
* is why every partition fence is over the authoritative states (see
|
|
309
|
+
* findAuthoritativeOwnersForPartition) and not over the committed rows alone.
|
|
275
310
|
*
|
|
276
311
|
* Returns the transferred record, or null when the transfer did not apply (prior owner already
|
|
277
312
|
* moved, resumed row already advanced, lineage mismatch).
|