@ran-sh/dsh-crew 1.0.2 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/README.zh.md +4 -0
- package/codex/AGENTS.md +101 -92
- package/docs/ui-surfaces.md +48 -21
- package/lib/client.js +235 -34
- package/official-web-bridge/lib/client.js +368 -4532
- package/official-web-bridge/package.json +1 -2
- package/package.json +130 -67
- package/scripts/build-client.mjs +30 -15
- package/scripts/remove-legacy-official-bridge.ps1 +89 -0
- package/scripts/setup.mjs +152 -140
- package/scripts/verify-npm-install.mjs +311 -310
- package/scripts/verify-official-bridge-e2e.mjs +12 -1
- package/src/client/host-readiness.mjs +62 -59
- package/src/client/index.tsx +117 -24
- package/src/client/quick-entry.tsx +10 -0
- package/src/client/quick-panel.tsx +295 -0
- package/src/client/surface-detection.mjs +43 -30
- package/src/credential-reference.mjs +38 -0
- package/src/dsh-cli-runtime.mjs +500 -40
- package/src/dsh-cohort.mjs +20 -0
- package/src/hub/index.mjs +1602 -1016
- package/src/install/npx-lifecycle.mjs +1983 -468
- package/src/install/official-web.mjs +36 -73
- package/src/jobs.mjs +20 -24
- package/src/model-catalog.mjs +8 -1
- package/src/official-web-bridge.mjs +329 -249
- package/src/provider-delete-adapters.mjs +165 -17
- package/src/provider-inventory.mjs +17 -3
- package/src/provider-layer-migration-adapters.mjs +759 -0
- package/src/provider-layer-migration.mjs +198 -0
- package/src/provider-lifecycle-state.mjs +4 -1
- package/src/provider-profile-store.mjs +193 -4
- package/src/provider-settings-store.mjs +82 -6
- package/src/provider-store-lock.mjs +67 -0
- package/src/runtime-identity.mjs +24 -1
- package/src/supervisor/restart-request.mjs +202 -0
- package/windows/start-dsh-crew.ps1 +783 -370
- package/worker.cordis.yml +32 -46
- package/zcode/AGENTS.md +35 -26
package/src/hub/index.mjs
CHANGED
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
// Web UI session list), exposes a loopback jobs API for the CC/Codex MCP shim,
|
|
4
4
|
// and serves the one-click installer endpoints for the settings page.
|
|
5
5
|
|
|
6
|
-
import { createHash, randomUUID } from 'node:crypto';
|
|
7
|
-
import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
|
|
8
|
-
import { dirname, join, isAbsolute, resolve as resolvePath } from 'node:path';
|
|
6
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
7
|
+
import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
|
|
8
|
+
import { dirname, join, isAbsolute, resolve as resolvePath } from 'node:path';
|
|
9
9
|
import { homedir } from 'node:os';
|
|
10
10
|
import { createShardWriter, readMergedStatus } from '../status-shard.mjs';
|
|
11
11
|
import {
|
|
@@ -30,21 +30,23 @@ import { addContextReferences, buildWorkspaceTask, isSafeBranchName, loadWorkspa
|
|
|
30
30
|
import { buildExtensionContract } from '../extension-contract.mjs';
|
|
31
31
|
import { cleanupIsolatedWorkspace, createIsolatedWorkspace } from '../workspace-isolation.mjs';
|
|
32
32
|
import { assessWorkspaceReadiness } from '../workspace-readiness.mjs';
|
|
33
|
-
import { buildHubExecutionRows } from '../config-readiness.mjs';
|
|
34
|
-
import { localRequestCore, originLoopback } from '../local-request-guard.mjs';
|
|
35
|
-
import { raceWaiters } from '../removable-waiter.mjs';
|
|
36
|
-
import { buildProviderInventory } from '../provider-inventory.mjs';
|
|
37
|
-
import {
|
|
38
|
-
import {
|
|
39
|
-
import {
|
|
40
|
-
import {
|
|
41
|
-
import {
|
|
42
|
-
import {
|
|
43
|
-
import {
|
|
44
|
-
import {
|
|
45
|
-
import {
|
|
46
|
-
import {
|
|
47
|
-
import {
|
|
33
|
+
import { buildHubExecutionRows } from '../config-readiness.mjs';
|
|
34
|
+
import { localRequestCore, originLoopback } from '../local-request-guard.mjs';
|
|
35
|
+
import { raceWaiters } from '../removable-waiter.mjs';
|
|
36
|
+
import { buildProviderInventory } from '../provider-inventory.mjs';
|
|
37
|
+
import { buildProviderLayerMigrationPlan, executeProviderLayerMigration } from '../provider-layer-migration.mjs';
|
|
38
|
+
import { inspectProviderProfile, readProviderDeclarations, readProviderMaterialization } from '../provider-profile-store.mjs';
|
|
39
|
+
import { inspectProviderSettings, readHarnessDefault, readProviderSettingsDeclarations, readProviderSettingsMaterialization } from '../provider-settings-store.mjs';
|
|
40
|
+
import { normalizeProviderLifecycleState } from '../provider-lifecycle-state.mjs';
|
|
41
|
+
import { createProviderHealthStore } from '../provider-health.mjs';
|
|
42
|
+
import { planProviderDelete } from '../provider-lifecycle.mjs';
|
|
43
|
+
import { createProviderDeleteFileHooks, isRecoverableProviderDeleteBackup, readProviderDeleteManifestFile } from '../provider-delete-adapters.mjs';
|
|
44
|
+
import { buildCredentialReferenceInventory } from '../credential-reference-inventory.mjs';
|
|
45
|
+
import { executeCredentialPurge, planCredentialPurge } from '../credential-lifecycle.mjs';
|
|
46
|
+
import { createCredentialPurgeFileHooks } from '../credential-purge-adapters.mjs';
|
|
47
|
+
import { normalizeCredentialPurgeState, recordCredentialPurgeOutcome } from '../credential-purge-state.mjs';
|
|
48
|
+
import { buildRuntimeReadinessSnapshot } from '../runtime-readiness-snapshot.mjs';
|
|
49
|
+
import { createProviderLayerMigrationFileHooks, readProviderLayerMigrationTransactions } from '../provider-layer-migration-adapters.mjs';
|
|
48
50
|
|
|
49
51
|
// policy.mjs is pure (no @deepseek-ai imports, no ctx access), so importing it
|
|
50
52
|
// here is safe for the profile-realm discipline: it never pulls in package
|
|
@@ -101,296 +103,351 @@ function seedLangFromHost(ctx) {
|
|
|
101
103
|
import { setLang } from '../i18n.mjs';
|
|
102
104
|
|
|
103
105
|
const ROUTE_BASE = '/_dsh/dsh-crew';
|
|
106
|
+
// Quick-config allowlist: the ONLY config keys writable from the official
|
|
107
|
+
// 3080 quick-controls surface. The full /config endpoint stays the single
|
|
108
|
+
// canonical authority; quick-config is a projection, never a second store.
|
|
109
|
+
export const QUICK_CONFIG_KEYS = Object.freeze([
|
|
110
|
+
'subagents_enabled',
|
|
111
|
+
'flash_model_priority',
|
|
112
|
+
'pro_model_priority',
|
|
113
|
+
'vision_enabled',
|
|
114
|
+
'imagegen_enabled',
|
|
115
|
+
'vision_provider',
|
|
116
|
+
'imagegen_provider',
|
|
117
|
+
]);
|
|
104
118
|
const LEGACY_TIER_MODELS = { flash: 'deepseek-v4-flash', pro: 'deepseek-v4-pro' };
|
|
105
119
|
// Local copy (the hub must not import jobs.mjs, which pulls the DSH SDK into
|
|
106
120
|
// the profile realm): a valid dispatch role set.
|
|
107
121
|
const ROLES = { worker: true, reviewer: true };
|
|
108
|
-
const CONFIG_DIR = join(homedir(), '.config', 'dsh-crew');
|
|
109
|
-
const CREDENTIAL_PURGE_STATE_FILE = join(CONFIG_DIR, 'credential-purge-lifecycle.json');
|
|
110
|
-
const RECOVERY_ACTION_PATTERN = /^[a-f0-9]{32}$/u;
|
|
111
|
-
|
|
112
|
-
function recoveryActionId(name) {
|
|
113
|
-
return createHash('sha256').update(String(name)).digest('hex').slice(0, 32);
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
function isRecoveryControlEntry(name) {
|
|
117
|
-
return name === '.quarantine'
|
|
118
|
-
|| name === '.delete.lock'
|
|
119
|
-
|| name === '.delete.reclaim.lock'
|
|
120
|
-
|| name === '.delete.recovery.lock'
|
|
121
|
-
|| name === '.delete.recovery.lock.claim'
|
|
122
|
-
|| /^\.delete\.lock\.[0-9a-f-]+\.(?:active|staging|retired)$/iu.test(name)
|
|
123
|
-
|| /^\.delete\.recovery\.lock\.[0-9a-f-]+\.stale$/iu.test(name);
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
function readProviderLifecycleStateStatus() {
|
|
127
|
-
const file = join(CONFIG_DIR, 'provider-lifecycle.json');
|
|
128
|
-
if (!existsSync(file)) return { ok: true, state: normalizeProviderLifecycleState() };
|
|
129
|
-
try {
|
|
130
|
-
return { ok: true, state: normalizeProviderLifecycleState(JSON.parse(readFileSync(file, 'utf8'))) };
|
|
131
|
-
} catch {
|
|
132
|
-
return { ok: false, code: 'PROVIDER_LIFECYCLE_UNAVAILABLE', state: normalizeProviderLifecycleState() };
|
|
133
|
-
}
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
function readProviderLifecycleState() {
|
|
137
|
-
return readProviderLifecycleStateStatus().state;
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
function readCredentialPurgeState() {
|
|
141
|
-
try { return normalizeCredentialPurgeState(JSON.parse(readFileSync(CREDENTIAL_PURGE_STATE_FILE, 'utf8'))); } catch { return normalizeCredentialPurgeState(); }
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
export function readProviderRecoveryTransactions(root = join(CONFIG_DIR, 'provider-backups')) {
|
|
145
|
-
if (!existsSync(root)) return [];
|
|
146
|
-
try {
|
|
147
|
-
const managedRoot = dirname(resolvePath(root));
|
|
148
|
-
const paths = {
|
|
149
|
-
profile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
150
|
-
settings: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
151
|
-
config: join(CONFIG_DIR, 'config.json'),
|
|
152
|
-
lifecycle: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
153
|
-
};
|
|
154
|
-
const transactions = readdirSync(root, { withFileTypes: true }).filter((entry) => !isRecoveryControlEntry(entry.name)).flatMap((entry) => {
|
|
155
|
-
const transactionRoot = join(root, entry.name);
|
|
156
|
-
const manifestFile = join(root, entry.name, 'manifest.json');
|
|
157
|
-
const actionId = recoveryActionId(entry.name);
|
|
158
|
-
if (!entry.isDirectory()) {
|
|
159
|
-
let updatedAt;
|
|
160
|
-
try { updatedAt = statSync(transactionRoot).mtime.toISOString(); } catch { updatedAt = new Date(0).toISOString(); }
|
|
161
|
-
return [{ storage_id: entry.name, action_id: actionId, transaction_id: null, provider_id: null, phase: 'RECOVERY_UNRESOLVED', updated_at: updatedAt, recoverable: false, unresolved: true }];
|
|
162
|
-
}
|
|
163
|
-
try {
|
|
164
|
-
const manifest = readProviderDeleteManifestFile(manifestFile, managedRoot);
|
|
165
|
-
const transactionId = typeof manifest?.plan?.plan_id === 'string' ? manifest.plan.plan_id : null;
|
|
166
|
-
const providerId = typeof manifest?.provider_id === 'string' ? manifest.provider_id : null;
|
|
167
|
-
const phase = typeof manifest?.phase_journal?.phase === 'string' && manifest.phase_journal.phase.trim()
|
|
168
|
-
? manifest.phase_journal.phase.trim() : 'RECOVERY_INCOMPLETE';
|
|
169
|
-
const updatedAt = typeof manifest?.phase_journal?.updated_at === 'string' && manifest.phase_journal.updated_at.trim()
|
|
170
|
-
? manifest.phase_journal.updated_at : statSync(manifestFile).mtime.toISOString();
|
|
171
|
-
const hasPendingMutation = manifest?.mutation_journal && typeof manifest.mutation_journal === 'object'
|
|
172
|
-
&& Object.keys(manifest.mutation_journal).length > 0;
|
|
173
|
-
const recoverable = transactionId != null && providerId != null && isRecoverableProviderDeleteBackup(manifest, {
|
|
174
|
-
root: transactionRoot,
|
|
175
|
-
managedRoot,
|
|
176
|
-
paths,
|
|
177
|
-
expectedProviderId: providerId,
|
|
178
|
-
}) === true;
|
|
179
|
-
if (['ROLLED_BACK', 'VERIFIED'].includes(phase) && !hasPendingMutation && recoverable) return [];
|
|
180
|
-
return [{ storage_id: entry.name, action_id: actionId, transaction_id: transactionId ?? entry.name, provider_id: providerId, phase, updated_at: updatedAt, recoverable, unresolved: recoverable !== true }];
|
|
181
|
-
} catch {
|
|
182
|
-
let updatedAt;
|
|
183
|
-
try { updatedAt = statSync(transactionRoot).mtime.toISOString(); } catch { updatedAt = new Date(0).toISOString(); }
|
|
184
|
-
return [{
|
|
185
|
-
storage_id: entry.name,
|
|
186
|
-
action_id: actionId,
|
|
187
|
-
transaction_id: null,
|
|
188
|
-
provider_id: null,
|
|
189
|
-
phase: 'RECOVERY_UNRESOLVED',
|
|
190
|
-
updated_at: updatedAt,
|
|
191
|
-
recoverable: false,
|
|
192
|
-
unresolved: true,
|
|
193
|
-
}];
|
|
194
|
-
}
|
|
195
|
-
});
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
}
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
}
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
let
|
|
282
|
-
|
|
283
|
-
try {
|
|
284
|
-
declarationEvidence.sources.
|
|
285
|
-
if (existsSync(
|
|
286
|
-
const source = readFileSync(
|
|
287
|
-
const parsed =
|
|
288
|
-
if (parsed.ok)
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
else
|
|
293
|
-
}
|
|
294
|
-
} catch {
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
};
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
const
|
|
318
|
-
|
|
319
|
-
const
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
122
|
+
const CONFIG_DIR = join(homedir(), '.config', 'dsh-crew');
|
|
123
|
+
const CREDENTIAL_PURGE_STATE_FILE = join(CONFIG_DIR, 'credential-purge-lifecycle.json');
|
|
124
|
+
const RECOVERY_ACTION_PATTERN = /^[a-f0-9]{32}$/u;
|
|
125
|
+
|
|
126
|
+
function recoveryActionId(name) {
|
|
127
|
+
return createHash('sha256').update(String(name)).digest('hex').slice(0, 32);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function isRecoveryControlEntry(name) {
|
|
131
|
+
return name === '.quarantine'
|
|
132
|
+
|| name === '.delete.lock'
|
|
133
|
+
|| name === '.delete.reclaim.lock'
|
|
134
|
+
|| name === '.delete.recovery.lock'
|
|
135
|
+
|| name === '.delete.recovery.lock.claim'
|
|
136
|
+
|| /^\.delete\.lock\.[0-9a-f-]+\.(?:active|staging|retired)$/iu.test(name)
|
|
137
|
+
|| /^\.delete\.recovery\.lock\.[0-9a-f-]+\.stale$/iu.test(name);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function readProviderLifecycleStateStatus() {
|
|
141
|
+
const file = join(CONFIG_DIR, 'provider-lifecycle.json');
|
|
142
|
+
if (!existsSync(file)) return { ok: true, state: normalizeProviderLifecycleState() };
|
|
143
|
+
try {
|
|
144
|
+
return { ok: true, state: normalizeProviderLifecycleState(JSON.parse(readFileSync(file, 'utf8'))) };
|
|
145
|
+
} catch {
|
|
146
|
+
return { ok: false, code: 'PROVIDER_LIFECYCLE_UNAVAILABLE', state: normalizeProviderLifecycleState() };
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function readProviderLifecycleState() {
|
|
151
|
+
return readProviderLifecycleStateStatus().state;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
function readCredentialPurgeState() {
|
|
155
|
+
try { return normalizeCredentialPurgeState(JSON.parse(readFileSync(CREDENTIAL_PURGE_STATE_FILE, 'utf8'))); } catch { return normalizeCredentialPurgeState(); }
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
export function readProviderRecoveryTransactions(root = join(CONFIG_DIR, 'provider-backups')) {
|
|
159
|
+
if (!existsSync(root)) return [];
|
|
160
|
+
try {
|
|
161
|
+
const managedRoot = dirname(resolvePath(root));
|
|
162
|
+
const paths = {
|
|
163
|
+
profile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
164
|
+
settings: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
165
|
+
config: join(CONFIG_DIR, 'config.json'),
|
|
166
|
+
lifecycle: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
167
|
+
};
|
|
168
|
+
const transactions = readdirSync(root, { withFileTypes: true }).filter((entry) => !isRecoveryControlEntry(entry.name)).flatMap((entry) => {
|
|
169
|
+
const transactionRoot = join(root, entry.name);
|
|
170
|
+
const manifestFile = join(root, entry.name, 'manifest.json');
|
|
171
|
+
const actionId = recoveryActionId(entry.name);
|
|
172
|
+
if (!entry.isDirectory()) {
|
|
173
|
+
let updatedAt;
|
|
174
|
+
try { updatedAt = statSync(transactionRoot).mtime.toISOString(); } catch { updatedAt = new Date(0).toISOString(); }
|
|
175
|
+
return [{ storage_id: entry.name, action_id: actionId, transaction_id: null, provider_id: null, phase: 'RECOVERY_UNRESOLVED', updated_at: updatedAt, recoverable: false, unresolved: true }];
|
|
176
|
+
}
|
|
177
|
+
try {
|
|
178
|
+
const manifest = readProviderDeleteManifestFile(manifestFile, managedRoot);
|
|
179
|
+
const transactionId = typeof manifest?.plan?.plan_id === 'string' ? manifest.plan.plan_id : null;
|
|
180
|
+
const providerId = typeof manifest?.provider_id === 'string' ? manifest.provider_id : null;
|
|
181
|
+
const phase = typeof manifest?.phase_journal?.phase === 'string' && manifest.phase_journal.phase.trim()
|
|
182
|
+
? manifest.phase_journal.phase.trim() : 'RECOVERY_INCOMPLETE';
|
|
183
|
+
const updatedAt = typeof manifest?.phase_journal?.updated_at === 'string' && manifest.phase_journal.updated_at.trim()
|
|
184
|
+
? manifest.phase_journal.updated_at : statSync(manifestFile).mtime.toISOString();
|
|
185
|
+
const hasPendingMutation = manifest?.mutation_journal && typeof manifest.mutation_journal === 'object'
|
|
186
|
+
&& Object.keys(manifest.mutation_journal).length > 0;
|
|
187
|
+
const recoverable = transactionId != null && providerId != null && isRecoverableProviderDeleteBackup(manifest, {
|
|
188
|
+
root: transactionRoot,
|
|
189
|
+
managedRoot,
|
|
190
|
+
paths,
|
|
191
|
+
expectedProviderId: providerId,
|
|
192
|
+
}) === true;
|
|
193
|
+
if (['ROLLED_BACK', 'VERIFIED'].includes(phase) && !hasPendingMutation && recoverable) return [];
|
|
194
|
+
return [{ storage_id: entry.name, action_id: actionId, transaction_id: transactionId ?? entry.name, provider_id: providerId, phase, updated_at: updatedAt, recoverable, unresolved: recoverable !== true }];
|
|
195
|
+
} catch {
|
|
196
|
+
let updatedAt;
|
|
197
|
+
try { updatedAt = statSync(transactionRoot).mtime.toISOString(); } catch { updatedAt = new Date(0).toISOString(); }
|
|
198
|
+
return [{
|
|
199
|
+
storage_id: entry.name,
|
|
200
|
+
action_id: actionId,
|
|
201
|
+
transaction_id: null,
|
|
202
|
+
provider_id: null,
|
|
203
|
+
phase: 'RECOVERY_UNRESOLVED',
|
|
204
|
+
updated_at: updatedAt,
|
|
205
|
+
recoverable: false,
|
|
206
|
+
unresolved: true,
|
|
207
|
+
}];
|
|
208
|
+
}
|
|
209
|
+
});
|
|
210
|
+
// Recovery is a safety gate, not a bounded history view. Never discard an
|
|
211
|
+
// older unresolved transaction merely because newer entries exist.
|
|
212
|
+
return transactions.sort((a, b) => a.updated_at.localeCompare(b.updated_at));
|
|
213
|
+
} catch {
|
|
214
|
+
let updatedAt;
|
|
215
|
+
try { updatedAt = statSync(root).mtime.toISOString(); } catch { updatedAt = new Date(0).toISOString(); }
|
|
216
|
+
return [{ storage_id: null, action_id: null, transaction_id: null, provider_id: null, phase: 'RECOVERY_UNRESOLVED', updated_at: updatedAt, recoverable: false, unresolved: true }];
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
function writeCredentialPurgeState(state) {
|
|
221
|
+
mkdirSync(CONFIG_DIR, { recursive: true });
|
|
222
|
+
const temp = `${CREDENTIAL_PURGE_STATE_FILE}.${process.pid}.${Date.now()}.tmp`;
|
|
223
|
+
try {
|
|
224
|
+
writeFileSync(temp, JSON.stringify(normalizeCredentialPurgeState(state), null, 2) + '\n', 'utf8');
|
|
225
|
+
renameSync(temp, CREDENTIAL_PURGE_STATE_FILE);
|
|
226
|
+
} catch (error) {
|
|
227
|
+
try { rmSync(temp, { force: true }); } catch {}
|
|
228
|
+
throw error;
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
function providerInventoryPolicy(config) {
|
|
233
|
+
return {
|
|
234
|
+
worker: config?.worker?.model_policy ?? {
|
|
235
|
+
priority: config?.flash_model_priority,
|
|
236
|
+
escalation_priority: config?.pro_model_priority,
|
|
237
|
+
},
|
|
238
|
+
reviewer: config?.review?.model_policy ?? {
|
|
239
|
+
priority: config?.pro_model_priority,
|
|
240
|
+
escalation_priority: [],
|
|
241
|
+
},
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
export function hasCompleteProviderCatalogEvidence(evidence) {
|
|
246
|
+
return evidence?.ok === true && evidence.partial !== true;
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
export function hasCompleteProviderDeclarationEvidence(evidence) {
|
|
250
|
+
return evidence?.ok === true
|
|
251
|
+
&& Object.values(evidence.sources ?? {}).every((source) => source?.present !== true || !source.code);
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
export function hasAvailableProviderLifecycleEvidence(evidence) {
|
|
255
|
+
return evidence?.ok === true;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
export function hasPendingProviderRecoveryTransactions(inventory) {
|
|
259
|
+
return Array.isArray(inventory?.recovery_transactions) && inventory.recovery_transactions.length > 0;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
export function hasProviderRuntimeRestartEvidence(beforeRuntimeId, currentRuntimeId) {
|
|
263
|
+
return typeof beforeRuntimeId === 'string' && beforeRuntimeId.trim() !== ''
|
|
264
|
+
&& typeof currentRuntimeId === 'string' && currentRuntimeId.trim() !== ''
|
|
265
|
+
&& beforeRuntimeId !== currentRuntimeId;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/** Preserve profile/settings provenance when preparing the migration planner. */
|
|
269
|
+
export function buildProviderMigrationDeclarations(profileDeclarations = [], settingsDeclarations = []) {
|
|
270
|
+
return [
|
|
271
|
+
...(Array.isArray(profileDeclarations) ? profileDeclarations : []),
|
|
272
|
+
...(Array.isArray(settingsDeclarations) ? settingsDeclarations : []),
|
|
273
|
+
];
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
async function readProviderInventorySnapshot(hub, ctx, config) {
|
|
277
|
+
let catalog = { providers: [], harness_default: null };
|
|
278
|
+
let catalogEvidence = { ok: false, code: 'MODEL_CATALOG_UNAVAILABLE' };
|
|
279
|
+
try {
|
|
280
|
+
catalog = await readHarnessModelCatalog({
|
|
281
|
+
llm: ctx.llm ?? ctx.get('llm'),
|
|
282
|
+
getCurrentSelection: () => ctx.get('agentDefaultModel')?.currentSelection?.(),
|
|
283
|
+
});
|
|
284
|
+
catalogEvidence = {
|
|
285
|
+
ok: catalog.partial !== true,
|
|
286
|
+
...(catalog.partial === true ? { code: 'PROVIDER_CATALOG_INCOMPLETE' } : {}),
|
|
287
|
+
provider_count: Number.isInteger(catalog.provider_count) ? catalog.provider_count : catalog.providers.length,
|
|
288
|
+
refreshed_at: typeof catalog.refreshed_at === 'string' ? catalog.refreshed_at : null,
|
|
289
|
+
partial: catalog.partial === true,
|
|
290
|
+
};
|
|
291
|
+
} catch {}
|
|
292
|
+
const profileFile = join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml');
|
|
293
|
+
let declarations = [];
|
|
294
|
+
let profileDeclarations = [];
|
|
295
|
+
let settingsDeclarations = [];
|
|
296
|
+
const declarationEvidence = { ok: true, sources: {} };
|
|
297
|
+
try {
|
|
298
|
+
declarationEvidence.sources.profile = { present: existsSync(profileFile) };
|
|
299
|
+
if (existsSync(profileFile)) {
|
|
300
|
+
const source = readFileSync(profileFile, 'utf8');
|
|
301
|
+
const parsed = readProviderDeclarations(source, { file: 'harness/profiles/dsh-crew/cordis.patch.yml' });
|
|
302
|
+
if (parsed.ok) {
|
|
303
|
+
profileDeclarations = parsed.declarations;
|
|
304
|
+
declarations = profileDeclarations;
|
|
305
|
+
}
|
|
306
|
+
else { declarationEvidence.ok = false; declarationEvidence.sources.profile.code = parsed.code; }
|
|
307
|
+
}
|
|
308
|
+
} catch { declarationEvidence.ok = false; declarationEvidence.sources.profile = { present: true, code: 'PROVIDER_SOURCE_UNAVAILABLE' }; }
|
|
309
|
+
const settingsFile = join(CONFIG_DIR, 'harness', 'settings.yaml');
|
|
310
|
+
let settingsDefault = null;
|
|
311
|
+
let defaultEvidence = { ok: true, source: 'none' };
|
|
312
|
+
try {
|
|
313
|
+
declarationEvidence.sources.settings = { present: existsSync(settingsFile) };
|
|
314
|
+
if (existsSync(settingsFile)) {
|
|
315
|
+
const source = readFileSync(settingsFile, 'utf8');
|
|
316
|
+
const parsed = readProviderSettingsDeclarations(source, { file: 'harness/settings.yaml' });
|
|
317
|
+
if (parsed.ok) {
|
|
318
|
+
settingsDeclarations = parsed.declarations;
|
|
319
|
+
declarations = [...declarations, ...settingsDeclarations];
|
|
320
|
+
}
|
|
321
|
+
else { declarationEvidence.ok = false; declarationEvidence.sources.settings.code = parsed.code; }
|
|
322
|
+
const parsedDefault = readHarnessDefault(source);
|
|
323
|
+
if (parsedDefault.ok) settingsDefault = parsedDefault;
|
|
324
|
+
else defaultEvidence = { ok: false, code: 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' };
|
|
325
|
+
}
|
|
326
|
+
} catch {
|
|
327
|
+
declarationEvidence.ok = false;
|
|
328
|
+
declarationEvidence.sources.settings = { present: true, code: 'PROVIDER_SOURCE_UNAVAILABLE' };
|
|
329
|
+
defaultEvidence = { ok: false, code: 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' };
|
|
330
|
+
}
|
|
331
|
+
const liveDefault = catalog.harness_default && typeof catalog.harness_default.provider === 'string' && typeof catalog.harness_default.model === 'string'
|
|
332
|
+
? { provider: catalog.harness_default.provider, model: catalog.harness_default.model } : null;
|
|
333
|
+
const persistedDefault = settingsDefault ? { provider: settingsDefault.provider, model: settingsDefault.model } : null;
|
|
334
|
+
if (liveDefault || persistedDefault) {
|
|
335
|
+
if (!liveDefault || !persistedDefault) defaultEvidence = { ok: false, code: 'PROVIDER_DEFAULT_AUTHORITY_MISMATCH' };
|
|
336
|
+
else if (liveDefault.provider !== persistedDefault.provider || liveDefault.model !== persistedDefault.model) defaultEvidence = { ok: false, code: 'PROVIDER_DEFAULT_AUTHORITY_MISMATCH' };
|
|
337
|
+
}
|
|
338
|
+
// The persisted agent-default-model is the mutation authority. Keep it in
|
|
339
|
+
// the inventory even when the live selection is stale; the evidence gate
|
|
340
|
+
// above prevents destructive operations until both sources agree.
|
|
341
|
+
if (persistedDefault) {
|
|
342
|
+
catalog = {
|
|
343
|
+
...catalog,
|
|
344
|
+
harness_default: persistedDefault,
|
|
345
|
+
harness_default_authority: { kind: 'harness-settings', locator: settingsDefault.locator },
|
|
346
|
+
};
|
|
347
|
+
}
|
|
348
|
+
const lifecycleEvidence = readProviderLifecycleStateStatus();
|
|
349
|
+
const lifecycle = lifecycleEvidence.state;
|
|
350
|
+
const recoveryTransactions = [
|
|
351
|
+
...readProviderRecoveryTransactions(),
|
|
352
|
+
...readProviderLayerMigrationTransactions(join(CONFIG_DIR, 'provider-migration-backups')),
|
|
353
|
+
];
|
|
354
|
+
const credentialPurge = readCredentialPurgeState();
|
|
355
|
+
const inventory = buildProviderInventory({
|
|
356
|
+
catalog,
|
|
357
|
+
declarations,
|
|
358
|
+
policy: providerInventoryPolicy(config),
|
|
359
|
+
tombstones: lifecycle.tombstones,
|
|
360
|
+
additionalProviderIds: [
|
|
361
|
+
...Object.keys(lifecycle.tombstones ?? {}),
|
|
362
|
+
...recoveryTransactions.map((entry) => entry.provider_id).filter((id) => typeof id === 'string' && id.trim()),
|
|
363
|
+
],
|
|
364
|
+
activeJobs: hub.list(),
|
|
365
|
+
});
|
|
366
|
+
// Keep each authoritative source's credential reference with its own
|
|
367
|
+
// declaration. Reconstructing from the merged inventory would silently
|
|
368
|
+
// attach the first key to every layer when profile/settings differ.
|
|
369
|
+
const migrationDeclarations = buildProviderMigrationDeclarations(profileDeclarations, settingsDeclarations);
|
|
370
|
+
const routingReferences = [
|
|
371
|
+
...(config.flash_model_priority ?? []), ...(config.pro_model_priority ?? []),
|
|
372
|
+
...(config.worker?.model_policy?.priority ?? []), ...(config.worker?.model_policy?.escalation_priority ?? []),
|
|
373
|
+
...(config.review?.model_policy?.priority ?? []), ...(config.review?.model_policy?.escalation_priority ?? []),
|
|
374
|
+
];
|
|
375
|
+
const migration = buildProviderLayerMigrationPlan({
|
|
376
|
+
declarations: migrationDeclarations,
|
|
377
|
+
catalogProviders: catalog.providers,
|
|
378
|
+
harnessDefault: catalog.harness_default,
|
|
379
|
+
routingReferences,
|
|
380
|
+
tombstones: lifecycle.tombstones,
|
|
381
|
+
recoveryTransactions,
|
|
382
|
+
catalogEvidence,
|
|
383
|
+
declarationEvidence,
|
|
384
|
+
lifecycleEvidence,
|
|
385
|
+
defaultEvidence,
|
|
386
|
+
});
|
|
387
|
+
return {
|
|
388
|
+
...inventory,
|
|
389
|
+
migration,
|
|
390
|
+
catalog_evidence: catalogEvidence,
|
|
391
|
+
declaration_evidence: declarationEvidence,
|
|
392
|
+
default_evidence: defaultEvidence,
|
|
393
|
+
lifecycle_evidence: lifecycleEvidence.ok ? { ok: true } : { ok: false, code: lifecycleEvidence.code },
|
|
394
|
+
credential_history_refs: Object.values(lifecycle.transactions ?? {})
|
|
395
|
+
.flatMap((transaction) => Array.isArray(transaction?.credential_refs) ? transaction.credential_refs : []),
|
|
396
|
+
credential_purged_refs: Object.keys(credentialPurge.purged),
|
|
397
|
+
credential_purge_unverified: Object.entries(credentialPurge.unverified).map(([referenceId, record]) => ({
|
|
398
|
+
reference_id: referenceId,
|
|
399
|
+
transaction_id: record.transaction_id,
|
|
400
|
+
state: record.state,
|
|
401
|
+
})),
|
|
402
|
+
lifecycle_transactions: Object.entries(lifecycle.transactions ?? {}).slice(-64).map(([transactionId, transaction]) => ({
|
|
403
|
+
transaction_id: transactionId,
|
|
404
|
+
provider_id: transaction.provider_id,
|
|
405
|
+
state: transaction.state,
|
|
406
|
+
...(transaction.updated_at ? { updated_at: transaction.updated_at } : {}),
|
|
407
|
+
...(transaction.expected_revision ? { expected_revision: transaction.expected_revision } : {}),
|
|
408
|
+
})),
|
|
409
|
+
recovery_transactions: recoveryTransactions,
|
|
410
|
+
};
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
function readProviderSourceRevisions() {
|
|
414
|
+
const profileFile = join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml');
|
|
415
|
+
const settingsFile = join(CONFIG_DIR, 'harness', 'settings.yaml');
|
|
416
|
+
const read = (file, inspect) => {
|
|
417
|
+
try {
|
|
418
|
+
if (!existsSync(file)) return { ok: false, code: 'PROVIDER_SOURCE_MISSING', revision: null };
|
|
419
|
+
const parsed = inspect(readFileSync(file, 'utf8'));
|
|
420
|
+
return parsed.ok ? { ok: true, revision: parsed.revision } : { ok: false, code: parsed.code, revision: parsed.revision ?? null };
|
|
421
|
+
} catch { return { ok: false, code: 'PROVIDER_SOURCE_UNAVAILABLE', revision: null }; }
|
|
422
|
+
};
|
|
423
|
+
return { profile: read(profileFile, inspectProviderProfile), settings: read(settingsFile, inspectProviderSettings) };
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
export function hubCanonicalEvents(job = {}) {
|
|
370
427
|
if (!job.id) return [];
|
|
371
428
|
const role = job.role === 'reviewer' ? 'reviewer' : 'worker';
|
|
372
429
|
const atStart = job.startedAt ?? null;
|
|
373
430
|
const atEnd = job.endedAt ?? atStart;
|
|
374
|
-
const definitions = [
|
|
375
|
-
['job.created', atStart, { client_job_id: job.client_job_id ?? null }],
|
|
376
|
-
['job.started', atStart, {}],
|
|
377
|
-
['runtime.bound', atStart, {
|
|
378
|
-
execution_plane: job.execution_context?.execution_plane ?? null,
|
|
379
|
-
profile: job.execution_context?.profile ?? null,
|
|
380
|
-
listen_port: Number.isInteger(job.execution_context?.listen_port) ? job.execution_context.listen_port : null,
|
|
381
|
-
runtime_id: job.execution_context?.runtime_id ?? null,
|
|
382
|
-
}],
|
|
383
|
-
['model.selected', atStart, { provider: job.provider ?? null, model: job.model ?? null, source: job.selection_source ?? null }],
|
|
384
|
-
['model.admitted', atStart, {
|
|
385
|
-
provider: job.provider ?? null,
|
|
386
|
-
model: job.model ?? null,
|
|
387
|
-
health_state: job.health_state ?? 'unprobed',
|
|
388
|
-
reason_code: job.health_reason_code ?? null,
|
|
389
|
-
observed_at: Number.isFinite(job.health_observed_at) ? job.health_observed_at : null,
|
|
390
|
-
}],
|
|
391
|
-
[role === 'reviewer' ? 'review.started' : 'worker.started', atStart, { run_id: job.id }],
|
|
392
|
-
['agent.created', atStart, { run_id: job.id }],
|
|
393
|
-
];
|
|
431
|
+
const definitions = [
|
|
432
|
+
['job.created', atStart, { client_job_id: job.client_job_id ?? null }],
|
|
433
|
+
['job.started', atStart, {}],
|
|
434
|
+
['runtime.bound', atStart, {
|
|
435
|
+
execution_plane: job.execution_context?.execution_plane ?? null,
|
|
436
|
+
profile: job.execution_context?.profile ?? null,
|
|
437
|
+
listen_port: Number.isInteger(job.execution_context?.listen_port) ? job.execution_context.listen_port : null,
|
|
438
|
+
runtime_id: job.execution_context?.runtime_id ?? null,
|
|
439
|
+
}],
|
|
440
|
+
['model.selected', atStart, { provider: job.provider ?? null, model: job.model ?? null, source: job.selection_source ?? null }],
|
|
441
|
+
['model.admitted', atStart, {
|
|
442
|
+
provider: job.provider ?? null,
|
|
443
|
+
model: job.model ?? null,
|
|
444
|
+
health_state: job.health_state ?? 'unprobed',
|
|
445
|
+
reason_code: job.health_reason_code ?? null,
|
|
446
|
+
observed_at: Number.isFinite(job.health_observed_at) ? job.health_observed_at : null,
|
|
447
|
+
}],
|
|
448
|
+
[role === 'reviewer' ? 'review.started' : 'worker.started', atStart, { run_id: job.id }],
|
|
449
|
+
['agent.created', atStart, { run_id: job.id }],
|
|
450
|
+
];
|
|
394
451
|
if (job.status !== 'running') {
|
|
395
452
|
definitions.push([
|
|
396
453
|
role === 'reviewer' ? 'review.completed' : 'worker.completed',
|
|
@@ -410,7 +467,7 @@ export function hubCanonicalEvents(job = {}) {
|
|
|
410
467
|
|
|
411
468
|
// ---------- job registry ----------
|
|
412
469
|
|
|
413
|
-
export function applyHubWorkspaceEvidence({ outcome, workspaceDiff, allowNoChanges = false, isolation = 'shared', role = 'worker' } = {}) {
|
|
470
|
+
export function applyHubWorkspaceEvidence({ outcome, workspaceDiff, allowNoChanges = false, isolation = 'shared', role = 'worker' } = {}) {
|
|
414
471
|
const changes = workspaceDiff?.changes ?? {};
|
|
415
472
|
const hasChanges = ['modified', 'deleted', 'renamed', 'untracked']
|
|
416
473
|
.some((key) => Array.isArray(changes[key]) && changes[key].length > 0);
|
|
@@ -420,71 +477,71 @@ export function applyHubWorkspaceEvidence({ outcome, workspaceDiff, allowNoChang
|
|
|
420
477
|
hasChanges,
|
|
421
478
|
allowNoChanges: allowNoChanges === true && isolation === 'worktree',
|
|
422
479
|
requireNoChangeAuthorization: role === 'worker',
|
|
423
|
-
});
|
|
424
|
-
}
|
|
425
|
-
|
|
426
|
-
/**
|
|
427
|
-
* Build the explicit provider callability probe used by the 3210 lifecycle
|
|
428
|
-
* route. It performs one token-bounded Harness stream through the same llm
|
|
429
|
-
* runtime used by WorkerRegistry; no credential values are read or returned.
|
|
430
|
-
*/
|
|
431
|
-
export function createProviderProbe(ctx) {
|
|
432
|
-
const llm = ctx?.llm ?? ctx?.get?.('llm');
|
|
433
|
-
if (!llm || typeof llm.stream !== 'function') return null;
|
|
434
|
-
return async ({ provider, model, signal }) => {
|
|
435
|
-
try {
|
|
436
|
-
const stream = llm.stream({
|
|
437
|
-
provider,
|
|
438
|
-
model,
|
|
439
|
-
messages: [userMessage('dsh-crew provider probe')],
|
|
440
|
-
maxTokens: 1,
|
|
441
|
-
temperature: 0,
|
|
442
|
-
signal,
|
|
443
|
-
});
|
|
444
|
-
for await (const chunk of stream) {
|
|
445
|
-
if (chunk?.type !== 'finish') continue;
|
|
446
|
-
const reason = chunk.reason;
|
|
447
|
-
if (reason?.kind === 'error' || reason?.kind === 'aborted') {
|
|
448
|
-
return { ok: false, error: reason.failure ?? { code: `PROVIDER_PROBE_${String(reason.kind).toUpperCase()}` } };
|
|
449
|
-
}
|
|
450
|
-
return { ok: true };
|
|
451
|
-
}
|
|
452
|
-
return { ok: false, error: { code: 'PROVIDER_PROBE_NO_FINISH' } };
|
|
453
|
-
} catch (error) {
|
|
454
|
-
return { ok: false, error };
|
|
455
|
-
}
|
|
456
|
-
};
|
|
457
|
-
}
|
|
458
|
-
|
|
459
|
-
/** Pick the first configured model for a provider, falling back to its live catalog. */
|
|
460
|
-
export function selectProviderProbeModel({ providerId, record, config } = {}) {
|
|
461
|
-
const advertised = new Set(Array.isArray(record?.models) ? record.models.filter((model) => typeof model === 'string') : []);
|
|
462
|
-
if (advertised.size === 0) return null;
|
|
463
|
-
const priorities = [
|
|
464
|
-
...(Array.isArray(config?.worker?.model_policy?.priority) ? config.worker.model_policy.priority : []),
|
|
465
|
-
...(Array.isArray(config?.flash_model_priority) ? config.flash_model_priority : []),
|
|
466
|
-
...(Array.isArray(config?.review?.model_policy?.priority) ? config.review.model_policy.priority : []),
|
|
467
|
-
...(Array.isArray(config?.pro_model_priority) ? config.pro_model_priority : []),
|
|
468
|
-
];
|
|
469
|
-
for (const ref of priorities) {
|
|
470
|
-
if (ref?.provider === providerId && advertised.has(ref.model)) return ref.model;
|
|
471
|
-
}
|
|
472
|
-
return [...advertised][0];
|
|
473
|
-
}
|
|
474
|
-
|
|
475
|
-
// Exported for the unit tests (test/hub-windows.test.mjs); instantiation
|
|
480
|
+
});
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
/**
|
|
484
|
+
* Build the explicit provider callability probe used by the 3210 lifecycle
|
|
485
|
+
* route. It performs one token-bounded Harness stream through the same llm
|
|
486
|
+
* runtime used by WorkerRegistry; no credential values are read or returned.
|
|
487
|
+
*/
|
|
488
|
+
export function createProviderProbe(ctx) {
|
|
489
|
+
const llm = ctx?.llm ?? ctx?.get?.('llm');
|
|
490
|
+
if (!llm || typeof llm.stream !== 'function') return null;
|
|
491
|
+
return async ({ provider, model, signal }) => {
|
|
492
|
+
try {
|
|
493
|
+
const stream = llm.stream({
|
|
494
|
+
provider,
|
|
495
|
+
model,
|
|
496
|
+
messages: [userMessage('dsh-crew provider probe')],
|
|
497
|
+
maxTokens: 1,
|
|
498
|
+
temperature: 0,
|
|
499
|
+
signal,
|
|
500
|
+
});
|
|
501
|
+
for await (const chunk of stream) {
|
|
502
|
+
if (chunk?.type !== 'finish') continue;
|
|
503
|
+
const reason = chunk.reason;
|
|
504
|
+
if (reason?.kind === 'error' || reason?.kind === 'aborted') {
|
|
505
|
+
return { ok: false, error: reason.failure ?? { code: `PROVIDER_PROBE_${String(reason.kind).toUpperCase()}` } };
|
|
506
|
+
}
|
|
507
|
+
return { ok: true };
|
|
508
|
+
}
|
|
509
|
+
return { ok: false, error: { code: 'PROVIDER_PROBE_NO_FINISH' } };
|
|
510
|
+
} catch (error) {
|
|
511
|
+
return { ok: false, error };
|
|
512
|
+
}
|
|
513
|
+
};
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
/** Pick the first configured model for a provider, falling back to its live catalog. */
|
|
517
|
+
export function selectProviderProbeModel({ providerId, record, config } = {}) {
|
|
518
|
+
const advertised = new Set(Array.isArray(record?.models) ? record.models.filter((model) => typeof model === 'string') : []);
|
|
519
|
+
if (advertised.size === 0) return null;
|
|
520
|
+
const priorities = [
|
|
521
|
+
...(Array.isArray(config?.worker?.model_policy?.priority) ? config.worker.model_policy.priority : []),
|
|
522
|
+
...(Array.isArray(config?.flash_model_priority) ? config.flash_model_priority : []),
|
|
523
|
+
...(Array.isArray(config?.review?.model_policy?.priority) ? config.review.model_policy.priority : []),
|
|
524
|
+
...(Array.isArray(config?.pro_model_priority) ? config.pro_model_priority : []),
|
|
525
|
+
];
|
|
526
|
+
for (const ref of priorities) {
|
|
527
|
+
if (ref?.provider === providerId && advertised.has(ref.model)) return ref.model;
|
|
528
|
+
}
|
|
529
|
+
return [...advertised][0];
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
// Exported for the unit tests (test/hub-windows.test.mjs); instantiation
|
|
476
533
|
// needs only a duck-typed ctx, so spawn()'s path guard is testable without a
|
|
477
534
|
// live DSH host.
|
|
478
|
-
export class WorkerRegistry { constructor(ctx) {
|
|
479
|
-
this.ctx = ctx;
|
|
480
|
-
this.jobs = new Map();
|
|
481
|
-
this.providerMutations = new Set();
|
|
482
|
-
this.providerMutationEpochs = new Map();
|
|
483
|
-
this.providerLeases = new Map();
|
|
484
|
-
this.nextId = 1;
|
|
485
|
-
this.shard = createShardWriter('hub');
|
|
486
|
-
this.healthStore = createProviderHealthStore();
|
|
487
|
-
}
|
|
535
|
+
export class WorkerRegistry { constructor(ctx) {
|
|
536
|
+
this.ctx = ctx;
|
|
537
|
+
this.jobs = new Map();
|
|
538
|
+
this.providerMutations = new Set();
|
|
539
|
+
this.providerMutationEpochs = new Map();
|
|
540
|
+
this.providerLeases = new Map();
|
|
541
|
+
this.nextId = 1;
|
|
542
|
+
this.shard = createShardWriter('hub');
|
|
543
|
+
this.healthStore = createProviderHealthStore();
|
|
544
|
+
}
|
|
488
545
|
|
|
489
546
|
view(job, withResult = false) {
|
|
490
547
|
const v = {
|
|
@@ -504,13 +561,13 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
504
561
|
task_status: job.outcome?.task_status ?? null,
|
|
505
562
|
workspace_evidence_ok: job.outcome?.workspace_evidence_ok ?? null,
|
|
506
563
|
review_verdict: job.review?.verdict ?? null,
|
|
507
|
-
workspace_diff_available: !!job.workspaceDiff && job.workspaceDiff.kind === 'git',
|
|
508
|
-
workspace_retained: job.workspace_retained === true,
|
|
509
|
-
cleanup_warning: job.cleanup_warning ?? null,
|
|
510
|
-
profile_id: job.profile_id ?? null,
|
|
511
|
-
workspace_context: job.workspace_context ?? null,
|
|
512
|
-
execution_context: job.execution_context ?? null,
|
|
513
|
-
event_cursor: hubCanonicalEvents(job).at(-1)?.sequence ?? 0,
|
|
564
|
+
workspace_diff_available: !!job.workspaceDiff && job.workspaceDiff.kind === 'git',
|
|
565
|
+
workspace_retained: job.workspace_retained === true,
|
|
566
|
+
cleanup_warning: job.cleanup_warning ?? null,
|
|
567
|
+
profile_id: job.profile_id ?? null,
|
|
568
|
+
workspace_context: job.workspace_context ?? null,
|
|
569
|
+
execution_context: job.execution_context ?? null,
|
|
570
|
+
event_cursor: hubCanonicalEvents(job).at(-1)?.sequence ?? 0,
|
|
514
571
|
};
|
|
515
572
|
if (withResult) {
|
|
516
573
|
v.result = job.result; v.error = job.error; v.stopReason = job.stopReason;
|
|
@@ -526,45 +583,51 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
526
583
|
return v;
|
|
527
584
|
}
|
|
528
585
|
|
|
529
|
-
list() {
|
|
530
|
-
return [...this.jobs.values()].map((job) => this.view(job));
|
|
531
|
-
}
|
|
532
|
-
|
|
533
|
-
beginProviderMutation(providerId) {
|
|
534
|
-
if (typeof providerId !== 'string' || !providerId.trim() || this.providerMutations.has(providerId)) return false;
|
|
535
|
-
this.providerMutationEpochs.set(providerId, (this.providerMutationEpochs.get(providerId) ?? 0) + 1);
|
|
536
|
-
this.providerMutations.add(providerId);
|
|
537
|
-
return true;
|
|
538
|
-
}
|
|
539
|
-
|
|
540
|
-
endProviderMutation(providerId) {
|
|
541
|
-
this.providerMutations.delete(providerId);
|
|
542
|
-
}
|
|
543
|
-
|
|
544
|
-
getProviderMutationEpoch(providerId) {
|
|
545
|
-
return this.providerMutationEpochs.get(providerId) ?? 0;
|
|
546
|
-
}
|
|
547
|
-
|
|
548
|
-
assertProviderNotMutating(providerId, expectedEpoch = undefined) {
|
|
549
|
-
if (this.providerMutations.has(providerId)
|
|
550
|
-
|| (expectedEpoch !== undefined && this.getProviderMutationEpoch(providerId) !== expectedEpoch)) {
|
|
551
|
-
throw Object.assign(new Error('provider mutation is active'), { code: 'PROVIDER_DELETE_BUSY' });
|
|
552
|
-
}
|
|
553
|
-
}
|
|
554
|
-
|
|
555
|
-
acquireProviderLease(providerId) {
|
|
556
|
-
this.providerLeases.set(providerId, (this.providerLeases.get(providerId) ?? 0) + 1);
|
|
557
|
-
}
|
|
558
|
-
|
|
559
|
-
releaseProviderLease(providerId) {
|
|
560
|
-
const count = this.providerLeases.get(providerId) ?? 0;
|
|
561
|
-
if (count <= 1) this.providerLeases.delete(providerId);
|
|
562
|
-
else this.providerLeases.set(providerId, count - 1);
|
|
563
|
-
}
|
|
564
|
-
|
|
565
|
-
hasProviderLease(providerId) {
|
|
566
|
-
return (this.providerLeases.get(providerId) ?? 0) > 0;
|
|
567
|
-
}
|
|
586
|
+
list() {
|
|
587
|
+
return [...this.jobs.values()].map((job) => this.view(job));
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
beginProviderMutation(providerId) {
|
|
591
|
+
if (typeof providerId !== 'string' || !providerId.trim() || this.providerMutations.has(providerId)) return false;
|
|
592
|
+
this.providerMutationEpochs.set(providerId, (this.providerMutationEpochs.get(providerId) ?? 0) + 1);
|
|
593
|
+
this.providerMutations.add(providerId);
|
|
594
|
+
return true;
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
endProviderMutation(providerId) {
|
|
598
|
+
this.providerMutations.delete(providerId);
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
getProviderMutationEpoch(providerId) {
|
|
602
|
+
return this.providerMutationEpochs.get(providerId) ?? 0;
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
assertProviderNotMutating(providerId, expectedEpoch = undefined) {
|
|
606
|
+
if (this.providerMutations.has(providerId)
|
|
607
|
+
|| (expectedEpoch !== undefined && this.getProviderMutationEpoch(providerId) !== expectedEpoch)) {
|
|
608
|
+
throw Object.assign(new Error('provider mutation is active'), { code: 'PROVIDER_DELETE_BUSY' });
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
acquireProviderLease(providerId) {
|
|
613
|
+
this.providerLeases.set(providerId, (this.providerLeases.get(providerId) ?? 0) + 1);
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
releaseProviderLease(providerId) {
|
|
617
|
+
const count = this.providerLeases.get(providerId) ?? 0;
|
|
618
|
+
if (count <= 1) this.providerLeases.delete(providerId);
|
|
619
|
+
else this.providerLeases.set(providerId, count - 1);
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
hasProviderLease(providerId) {
|
|
623
|
+
return (this.providerLeases.get(providerId) ?? 0) > 0;
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
// Profile/settings are shared stores. A migration or delete of provider A
|
|
627
|
+
// must fence a concurrent lifecycle write for provider B in this Hub.
|
|
628
|
+
beginProviderStoreMutation() { return this.beginProviderMutation('__dsh-provider-store__'); }
|
|
629
|
+
endProviderStoreMutation() { this.endProviderMutation('__dsh-provider-store__'); }
|
|
630
|
+
hasProviderStoreMutation() { return this.providerMutations.has('__dsh-provider-store__'); }
|
|
568
631
|
|
|
569
632
|
publish() {
|
|
570
633
|
this.shard.publish([...this.jobs.values()].map((j) => this.view(j)));
|
|
@@ -578,7 +641,7 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
578
641
|
* `role` (worker | reviewer) records who does the work; `tier` remains the
|
|
579
642
|
* legacy model-class slot. Reviewer-role jobs always use the pro slot.
|
|
580
643
|
*/
|
|
581
|
-
async spawn({ task, tier = 'flash', role, attempt = 0, effort = 'max', cwd, source = 'api', preset, delivery = 'coding', client_job_id, requested_isolation, workspace_branch, timeout_seconds, profile_id, workspace_context, allow_no_changes, ingress = 'direct-3210' }) {
|
|
644
|
+
async spawn({ task, tier = 'flash', role, attempt = 0, effort = 'max', cwd, source = 'api', preset, delivery = 'coding', client_job_id, requested_isolation, workspace_branch, timeout_seconds, profile_id, workspace_context, allow_no_changes, ingress = 'direct-3210' }) {
|
|
582
645
|
// role is only honored when the caller explicitly names it; a legacy
|
|
583
646
|
// tier-only spawn (role === undefined) keeps the exact v0.1 resolution.
|
|
584
647
|
const hasRole = role === 'worker' || role === 'reviewer';
|
|
@@ -597,9 +660,9 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
597
660
|
// Provider routing: follow-dsh reads the DSH Models selection live; the
|
|
598
661
|
// default deepseek-official keeps legacy setups unchanged. Re-resolved on
|
|
599
662
|
// every spawn so a Models change takes effect on the next worker.
|
|
600
|
-
const cfg = normalizeGlobalConfig(this.getConfig?.() ?? {});
|
|
601
|
-
const lifecycleState = readProviderLifecycleState();
|
|
602
|
-
const healthGate = cfg.health_gate ?? cfg.worker?.model_policy?.health_gate;
|
|
663
|
+
const cfg = normalizeGlobalConfig(this.getConfig?.() ?? {});
|
|
664
|
+
const lifecycleState = readProviderLifecycleState();
|
|
665
|
+
const healthGate = cfg.health_gate ?? cfg.worker?.model_policy?.health_gate;
|
|
603
666
|
const workerProviderMode = normalizeWorkerProviderMode(cfg.worker_provider_mode);
|
|
604
667
|
const getCurrentSelection = () => this.ctx.get('agentDefaultModel')?.currentSelection?.();
|
|
605
668
|
let selection;
|
|
@@ -640,30 +703,30 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
640
703
|
// tier-only spawns resolve through the tier resolver exactly as before.
|
|
641
704
|
if (hasRole) {
|
|
642
705
|
const policy = resolveModelPolicy(cfg, effRole, { attempt });
|
|
643
|
-
selection = resolveModel({
|
|
644
|
-
role: effRole,
|
|
645
|
-
attempt,
|
|
646
|
-
policy,
|
|
647
|
-
catalog,
|
|
648
|
-
harnessDefault: catalog.harness_default ?? getCurrentSelection(),
|
|
649
|
-
healthStore: this.healthStore,
|
|
650
|
-
healthGate: policy.health_gate ?? healthGate,
|
|
651
|
-
allowFallback: cfg.allow_fallback !== false,
|
|
652
|
-
tombstones: lifecycleState.tombstones,
|
|
653
|
-
});
|
|
706
|
+
selection = resolveModel({
|
|
707
|
+
role: effRole,
|
|
708
|
+
attempt,
|
|
709
|
+
policy,
|
|
710
|
+
catalog,
|
|
711
|
+
harnessDefault: catalog.harness_default ?? getCurrentSelection(),
|
|
712
|
+
healthStore: this.healthStore,
|
|
713
|
+
healthGate: policy.health_gate ?? healthGate,
|
|
714
|
+
allowFallback: cfg.allow_fallback !== false,
|
|
715
|
+
tombstones: lifecycleState.tombstones,
|
|
716
|
+
});
|
|
654
717
|
} else {
|
|
655
718
|
selection = resolveWorkerModel({
|
|
656
719
|
tier: effTier,
|
|
657
720
|
priority: cfg[`${effTier}_model_priority`],
|
|
658
721
|
priorityConfigured: cfg[`${effTier}_model_priority_configured`],
|
|
659
722
|
fallback: cfg[`${effTier}_model_fallback`],
|
|
660
|
-
catalog,
|
|
661
|
-
harnessDefault: catalog.harness_default ?? getCurrentSelection(),
|
|
662
|
-
healthStore: this.healthStore,
|
|
663
|
-
healthGate,
|
|
664
|
-
allowFallback: cfg.allow_fallback !== false,
|
|
665
|
-
tombstones: lifecycleState.tombstones,
|
|
666
|
-
traceContext: {
|
|
723
|
+
catalog,
|
|
724
|
+
harnessDefault: catalog.harness_default ?? getCurrentSelection(),
|
|
725
|
+
healthStore: this.healthStore,
|
|
726
|
+
healthGate,
|
|
727
|
+
allowFallback: cfg.allow_fallback !== false,
|
|
728
|
+
tombstones: lifecycleState.tombstones,
|
|
729
|
+
traceContext: {
|
|
667
730
|
role: effRole,
|
|
668
731
|
logicalAttempt: attempt,
|
|
669
732
|
modelClassHint: effTier,
|
|
@@ -671,11 +734,20 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
671
734
|
candidateSet: attempt > 0 ? 'escalation' : 'primary',
|
|
672
735
|
},
|
|
673
736
|
});
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
}
|
|
677
|
-
const
|
|
678
|
-
|
|
737
|
+
}
|
|
738
|
+
if (!selection.ok) throw Object.assign(new Error(selection.message), { policyCode: selection.code });
|
|
739
|
+
}
|
|
740
|
+
const pendingLayerMigration = readProviderLayerMigrationTransactions(join(CONFIG_DIR, 'provider-migration-backups'))
|
|
741
|
+
.find((entry) => entry.unresolved === true || entry.provider_id === selection.provider);
|
|
742
|
+
if (pendingLayerMigration) throw Object.assign(new Error('provider layer migration is pending verification'), { code: 'PROVIDER_MIGRATION_PENDING' });
|
|
743
|
+
// A process restart clears the in-memory mutation fence, so persisted
|
|
744
|
+
// provider-delete transactions must also gate admission. Unknown-owner
|
|
745
|
+
// recovery entries fence globally; known entries fence their provider.
|
|
746
|
+
const pendingProviderDelete = readProviderRecoveryTransactions()
|
|
747
|
+
.find((entry) => entry.unresolved === true || entry.provider_id === selection.provider);
|
|
748
|
+
if (pendingProviderDelete) throw Object.assign(new Error('provider delete recovery is pending'), { code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
749
|
+
const providerMutationEpoch = this.getProviderMutationEpoch(selection.provider);
|
|
750
|
+
this.assertProviderNotMutating(selection.provider, providerMutationEpoch);
|
|
679
751
|
|
|
680
752
|
// The worker always gets the auditable Delivery Contract appended (unless
|
|
681
753
|
// it already carries one), so its final message follows ## Diff / ## Tests
|
|
@@ -707,19 +779,19 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
707
779
|
id, client_job_id: client_job_id ?? null, sessionId, role: jobRole, attempt, tier: effTier, provider: selection.provider, model: selection.model,
|
|
708
780
|
selection_source: selection.source, selection_trace: selection.selection_trace ?? null,
|
|
709
781
|
effort, reasoning_effort: selection.reasoningEffort,
|
|
710
|
-
task, source, cwd: executionCwd, requested_cwd: cwd,
|
|
711
|
-
isolation: isolatedWorkspace ? 'worktree' : 'shared',
|
|
712
|
-
execution_context: (() => {
|
|
713
|
-
const runtime = getHubRuntimeIdentity();
|
|
714
|
-
return {
|
|
715
|
-
execution_plane: runtime.execution_plane,
|
|
716
|
-
profile: runtime.profile,
|
|
717
|
-
listen_port: runtime.listen_port,
|
|
718
|
-
runtime_id: runtime.runtime_id,
|
|
719
|
-
ingress: ingress === 'official-3080' ? 'official-3080' : 'direct-3210',
|
|
720
|
-
};
|
|
721
|
-
})(),
|
|
722
|
-
allow_no_changes: allow_no_changes === true,
|
|
782
|
+
task, source, cwd: executionCwd, requested_cwd: cwd,
|
|
783
|
+
isolation: isolatedWorkspace ? 'worktree' : 'shared',
|
|
784
|
+
execution_context: (() => {
|
|
785
|
+
const runtime = getHubRuntimeIdentity();
|
|
786
|
+
return {
|
|
787
|
+
execution_plane: runtime.execution_plane,
|
|
788
|
+
profile: runtime.profile,
|
|
789
|
+
listen_port: runtime.listen_port,
|
|
790
|
+
runtime_id: runtime.runtime_id,
|
|
791
|
+
ingress: ingress === 'official-3080' ? 'official-3080' : 'direct-3210',
|
|
792
|
+
};
|
|
793
|
+
})(),
|
|
794
|
+
allow_no_changes: allow_no_changes === true,
|
|
723
795
|
workspace_branch: workspace_branch ?? null, isolatedWorkspace,
|
|
724
796
|
profile_id: profile_id ?? null, workspace_context: workspace_context ?? null,
|
|
725
797
|
prompt: workerPrompt, delivery: delivery === 'review' ? 'review' : 'coding',
|
|
@@ -734,48 +806,48 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
734
806
|
// Abort latch: cancel()/timeout set this the moment the job becomes
|
|
735
807
|
// terminal, even while agents.create() is still pending, so a handle
|
|
736
808
|
// acquired after that point is disposed instead of being driven.
|
|
737
|
-
abortRequested: false,
|
|
738
|
-
abortPromise: null,
|
|
739
|
-
resolveAbort: null,
|
|
740
|
-
providerLease: false,
|
|
741
|
-
providerLeaseReleaseBlocked: false,
|
|
742
|
-
lateHandleCreated: false,
|
|
743
|
-
lateCreateCleanupPromise: null,
|
|
809
|
+
abortRequested: false,
|
|
810
|
+
abortPromise: null,
|
|
811
|
+
resolveAbort: null,
|
|
812
|
+
providerLease: false,
|
|
813
|
+
providerLeaseReleaseBlocked: false,
|
|
814
|
+
lateHandleCreated: false,
|
|
815
|
+
lateCreateCleanupPromise: null,
|
|
744
816
|
};
|
|
745
|
-
job.abortPromise = new Promise((resolve) => { job.resolveAbort = resolve; });
|
|
817
|
+
job.abortPromise = new Promise((resolve) => { job.resolveAbort = resolve; });
|
|
746
818
|
const disposeJobHandle = async () => {
|
|
747
819
|
if (!job.handle) return;
|
|
748
|
-
if (!job.handle_dispose_promise) {
|
|
749
|
-
try {
|
|
750
|
-
job.handle_dispose_promise = Promise.resolve(job.handle.dispose());
|
|
751
|
-
} catch (error) {
|
|
752
|
-
job.handle_dispose_promise = Promise.reject(error);
|
|
753
|
-
}
|
|
754
|
-
}
|
|
755
|
-
try {
|
|
756
|
-
await job.handle_dispose_promise;
|
|
757
|
-
} catch (error) {
|
|
758
|
-
// A failed disposer is not proof that the provider is idle. Keep the
|
|
759
|
-
// lease fenced even when the job is already terminal; the operator
|
|
760
|
-
// must restart/reclaim the owned runtime before deletion can proceed.
|
|
761
|
-
job.providerLeaseReleaseBlocked = true;
|
|
762
|
-
throw error;
|
|
763
|
-
}
|
|
820
|
+
if (!job.handle_dispose_promise) {
|
|
821
|
+
try {
|
|
822
|
+
job.handle_dispose_promise = Promise.resolve(job.handle.dispose());
|
|
823
|
+
} catch (error) {
|
|
824
|
+
job.handle_dispose_promise = Promise.reject(error);
|
|
825
|
+
}
|
|
826
|
+
}
|
|
827
|
+
try {
|
|
828
|
+
await job.handle_dispose_promise;
|
|
829
|
+
} catch (error) {
|
|
830
|
+
// A failed disposer is not proof that the provider is idle. Keep the
|
|
831
|
+
// lease fenced even when the job is already terminal; the operator
|
|
832
|
+
// must restart/reclaim the owned runtime before deletion can proceed.
|
|
833
|
+
job.providerLeaseReleaseBlocked = true;
|
|
834
|
+
throw error;
|
|
835
|
+
}
|
|
764
836
|
};
|
|
765
837
|
job.disposeHandle = disposeJobHandle;
|
|
766
838
|
// Read-only pre-run snapshot (async, never blocks dispatch): the audit
|
|
767
839
|
// only needs the before-state by the time the worker finishes. Non-repos
|
|
768
840
|
// degrade to { kind:'no-git' } instead of failing the job.
|
|
769
|
-
job.baseline = await captureWorkspaceBaseline({ cwd: executionCwd }).catch(() => ({ kind: 'no-git', reason: NOT_A_GIT_REPOSITORY, error: 'workspace audit failed' }));
|
|
770
|
-
try {
|
|
771
|
-
this.assertProviderNotMutating(selection.provider, providerMutationEpoch);
|
|
772
|
-
this.acquireProviderLease(selection.provider);
|
|
773
|
-
job.providerLease = true;
|
|
774
|
-
} catch (error) {
|
|
775
|
-
if (isolatedWorkspace) await cleanupIsolatedWorkspace({ repoRoot: isolatedWorkspace.repoRoot, worktreePath: isolatedWorkspace.worktreePath }).catch(() => {});
|
|
776
|
-
throw error;
|
|
777
|
-
}
|
|
778
|
-
this.jobs.set(id, job);
|
|
841
|
+
job.baseline = await captureWorkspaceBaseline({ cwd: executionCwd }).catch(() => ({ kind: 'no-git', reason: NOT_A_GIT_REPOSITORY, error: 'workspace audit failed' }));
|
|
842
|
+
try {
|
|
843
|
+
this.assertProviderNotMutating(selection.provider, providerMutationEpoch);
|
|
844
|
+
this.acquireProviderLease(selection.provider);
|
|
845
|
+
job.providerLease = true;
|
|
846
|
+
} catch (error) {
|
|
847
|
+
if (isolatedWorkspace) await cleanupIsolatedWorkspace({ repoRoot: isolatedWorkspace.repoRoot, worktreePath: isolatedWorkspace.worktreePath }).catch(() => {});
|
|
848
|
+
throw error;
|
|
849
|
+
}
|
|
850
|
+
this.jobs.set(id, job);
|
|
779
851
|
|
|
780
852
|
const onEvent = (session, event) => {
|
|
781
853
|
switch (event.type) {
|
|
@@ -806,7 +878,7 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
806
878
|
};
|
|
807
879
|
|
|
808
880
|
const run = async () => {
|
|
809
|
-
const createPromise = this.ctx.agents.create({
|
|
881
|
+
const createPromise = this.ctx.agents.create({
|
|
810
882
|
sessionId,
|
|
811
883
|
meta: { cwd: executionCwd, ...(presetId === undefined ? {} : { agentPreset: presetId }) },
|
|
812
884
|
agentOptions: { provider: selection.provider, model: selection.model },
|
|
@@ -820,23 +892,23 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
820
892
|
});
|
|
821
893
|
},
|
|
822
894
|
});
|
|
823
|
-
const created = await Promise.race([
|
|
824
|
-
createPromise.then((handle) => ({ handle })),
|
|
825
|
-
job.abortPromise.then(() => ({ aborted: true })),
|
|
826
|
-
]);
|
|
827
|
-
if (created.aborted) {
|
|
828
|
-
job.lateCreateCleanupPromise = createPromise.then(async (handle) => {
|
|
829
|
-
if (!handle) return;
|
|
830
|
-
job.lateHandleCreated = true;
|
|
831
|
-
job.handle = handle;
|
|
832
|
-
if (job.status !== 'running' || job.abortRequested) await job.disposeHandle();
|
|
833
|
-
}).catch((error) => {
|
|
834
|
-
job.cleanup_warning = `late agent creation failed: ${error?.message ?? String(error)}`;
|
|
835
|
-
throw error;
|
|
836
|
-
});
|
|
837
|
-
return;
|
|
838
|
-
}
|
|
839
|
-
const handle = created.handle;
|
|
895
|
+
const created = await Promise.race([
|
|
896
|
+
createPromise.then((handle) => ({ handle })),
|
|
897
|
+
job.abortPromise.then(() => ({ aborted: true })),
|
|
898
|
+
]);
|
|
899
|
+
if (created.aborted) {
|
|
900
|
+
job.lateCreateCleanupPromise = createPromise.then(async (handle) => {
|
|
901
|
+
if (!handle) return;
|
|
902
|
+
job.lateHandleCreated = true;
|
|
903
|
+
job.handle = handle;
|
|
904
|
+
if (job.status !== 'running' || job.abortRequested) await job.disposeHandle();
|
|
905
|
+
}).catch((error) => {
|
|
906
|
+
job.cleanup_warning = `late agent creation failed: ${error?.message ?? String(error)}`;
|
|
907
|
+
throw error;
|
|
908
|
+
});
|
|
909
|
+
return;
|
|
910
|
+
}
|
|
911
|
+
const handle = created.handle;
|
|
840
912
|
job.handle = handle;
|
|
841
913
|
// The job may have become terminal (cancel/timeout) while create() was
|
|
842
914
|
// pending; the old disposer was a no-op in that window. Dispose the
|
|
@@ -863,7 +935,7 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
863
935
|
// Cancel/timeout may have become terminal while waiting for the initial
|
|
864
936
|
// idle boundary; do not deliver the prompt to a job that is no longer
|
|
865
937
|
// running. The shared finally block still disposes the handle exactly once.
|
|
866
|
-
if (job.status !== 'running' || job.abortRequested) return;
|
|
938
|
+
if (job.status !== 'running' || job.abortRequested) return;
|
|
867
939
|
handle.agent.followup(userMessage(job.prompt));
|
|
868
940
|
await handle.agent.whenIdle();
|
|
869
941
|
job.result = job.lastAssistantText ?? '';
|
|
@@ -878,11 +950,11 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
878
950
|
if (job.status !== 'running') return;
|
|
879
951
|
job.abortRequested = true;
|
|
880
952
|
job.status = 'failed';
|
|
881
|
-
job.phase = JOB_PHASES.FAILED;
|
|
953
|
+
job.phase = JOB_PHASES.FAILED;
|
|
882
954
|
job.error = `job timed out after ${timeout_seconds}s`;
|
|
883
955
|
job.error_code = 'JOB_TIMEOUT';
|
|
884
956
|
job.endedAt = new Date().toISOString();
|
|
885
|
-
job.resolveAbort?.();
|
|
957
|
+
job.resolveAbort?.();
|
|
886
958
|
this.publish();
|
|
887
959
|
for (const waiter of job.waiters.splice(0)) waiter();
|
|
888
960
|
try { await disposeJobHandle(); } catch (error) {
|
|
@@ -902,23 +974,23 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
902
974
|
job.endedAt = new Date().toISOString();
|
|
903
975
|
job.currentTool = null;
|
|
904
976
|
let handleCleanupWarning = job.cleanup_warning ?? null;
|
|
905
|
-
try { await disposeJobHandle(); } catch (error) {
|
|
906
|
-
handleCleanupWarning = `agent cleanup failed: ${error?.message ?? String(error)}`;
|
|
907
|
-
}
|
|
908
|
-
if (job.providerLease) {
|
|
909
|
-
const releaseLease = () => {
|
|
910
|
-
if (!job.providerLease) return;
|
|
911
|
-
this.releaseProviderLease(job.provider);
|
|
912
|
-
job.providerLease = false;
|
|
913
|
-
};
|
|
914
|
-
if (job.lateCreateCleanupPromise) {
|
|
915
|
-
job.lateCreateCleanupPromise.then(releaseLease).catch(() => {
|
|
916
|
-
if (!job.lateHandleCreated && !job.providerLeaseReleaseBlocked) releaseLease();
|
|
917
|
-
// A late-created handle that failed disposal remains fenced by
|
|
918
|
-
// the provider lease until an operator restarts/reclaims it.
|
|
919
|
-
});
|
|
920
|
-
} else if (!job.providerLeaseReleaseBlocked) releaseLease();
|
|
921
|
-
}
|
|
977
|
+
try { await disposeJobHandle(); } catch (error) {
|
|
978
|
+
handleCleanupWarning = `agent cleanup failed: ${error?.message ?? String(error)}`;
|
|
979
|
+
}
|
|
980
|
+
if (job.providerLease) {
|
|
981
|
+
const releaseLease = () => {
|
|
982
|
+
if (!job.providerLease) return;
|
|
983
|
+
this.releaseProviderLease(job.provider);
|
|
984
|
+
job.providerLease = false;
|
|
985
|
+
};
|
|
986
|
+
if (job.lateCreateCleanupPromise) {
|
|
987
|
+
job.lateCreateCleanupPromise.then(releaseLease).catch(() => {
|
|
988
|
+
if (!job.lateHandleCreated && !job.providerLeaseReleaseBlocked) releaseLease();
|
|
989
|
+
// A late-created handle that failed disposal remains fenced by
|
|
990
|
+
// the provider lease until an operator restarts/reclaims it.
|
|
991
|
+
});
|
|
992
|
+
} else if (!job.providerLeaseReleaseBlocked) releaseLease();
|
|
993
|
+
}
|
|
922
994
|
// Delivery completeness is separate from execution status: a job can
|
|
923
995
|
// be done yet fail to report Diff/Tests/Risks (or Review sections for
|
|
924
996
|
// an automatic review). Parse whatever final message the worker
|
|
@@ -954,23 +1026,23 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
954
1026
|
job.workspaceDiff = job.baseline.kind === 'git'
|
|
955
1027
|
? await captureWorkspaceDiff({ cwd: executionCwd, baseline: job.baseline }).catch(() => ({ kind: 'no-git', reason: NOT_A_GIT_REPOSITORY, error: 'workspace diff failed' }))
|
|
956
1028
|
: job.baseline;
|
|
957
|
-
job.outcome = applyHubWorkspaceEvidence({
|
|
958
|
-
outcome: job.outcome,
|
|
959
|
-
workspaceDiff: job.workspaceDiff,
|
|
960
|
-
allowNoChanges: job.allow_no_changes,
|
|
961
|
-
isolation: job.isolation,
|
|
962
|
-
role: job.role,
|
|
963
|
-
});
|
|
964
|
-
// Record only sanitized callability evidence. Cancelled jobs are
|
|
965
|
-
// operator actions, not provider health signals.
|
|
966
|
-
if (job.status !== 'cancelled') {
|
|
967
|
-
this.healthStore.record(job.provider, job.model, {
|
|
968
|
-
ok: job.status === 'done',
|
|
969
|
-
observed_at: Date.parse(job.endedAt ?? '') || Date.now(),
|
|
970
|
-
error: { code: job.error_code, message: job.error },
|
|
971
|
-
});
|
|
972
|
-
}
|
|
973
|
-
if (job.role === 'reviewer' && job.review && job.workspaceDiff?.kind === 'git') {
|
|
1029
|
+
job.outcome = applyHubWorkspaceEvidence({
|
|
1030
|
+
outcome: job.outcome,
|
|
1031
|
+
workspaceDiff: job.workspaceDiff,
|
|
1032
|
+
allowNoChanges: job.allow_no_changes,
|
|
1033
|
+
isolation: job.isolation,
|
|
1034
|
+
role: job.role,
|
|
1035
|
+
});
|
|
1036
|
+
// Record only sanitized callability evidence. Cancelled jobs are
|
|
1037
|
+
// operator actions, not provider health signals.
|
|
1038
|
+
if (job.status !== 'cancelled') {
|
|
1039
|
+
this.healthStore.record(job.provider, job.model, {
|
|
1040
|
+
ok: job.status === 'done',
|
|
1041
|
+
observed_at: Date.parse(job.endedAt ?? '') || Date.now(),
|
|
1042
|
+
error: { code: job.error_code, message: job.error },
|
|
1043
|
+
});
|
|
1044
|
+
}
|
|
1045
|
+
if (job.role === 'reviewer' && job.review && job.workspaceDiff?.kind === 'git') {
|
|
974
1046
|
const changes = job.workspaceDiff.changes ?? {};
|
|
975
1047
|
const mutated = ['modified', 'deleted', 'renamed', 'untracked'].some((key) => Array.isArray(changes[key]) && changes[key].length > 0);
|
|
976
1048
|
if (mutated) {
|
|
@@ -1011,8 +1083,8 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
1011
1083
|
job.status = 'cancelled';
|
|
1012
1084
|
job.phase = JOB_PHASES.CANCELLED;
|
|
1013
1085
|
job.error = 'cancelled by request';
|
|
1014
|
-
job.resolveAbort?.();
|
|
1015
|
-
for (const waiter of job.waiters.splice(0)) waiter();
|
|
1086
|
+
job.resolveAbort?.();
|
|
1087
|
+
for (const waiter of job.waiters.splice(0)) waiter();
|
|
1016
1088
|
try { await job.disposeHandle?.(); } catch (error) {
|
|
1017
1089
|
job.cleanup_warning = `agent cleanup failed: ${error?.message ?? String(error)}`;
|
|
1018
1090
|
}
|
|
@@ -1176,10 +1248,10 @@ export function resolveHubSpawnPayload(payload, getConfig = () => ({}), dependen
|
|
|
1176
1248
|
|
|
1177
1249
|
// ---------- plugin entry ----------
|
|
1178
1250
|
|
|
1179
|
-
export async function apply(ctx) {
|
|
1180
|
-
seedLangFromHost(ctx);
|
|
1181
|
-
const hub = new WorkerRegistry(ctx);
|
|
1182
|
-
const builtInProviderProbe = createProviderProbe(ctx);
|
|
1251
|
+
export async function apply(ctx) {
|
|
1252
|
+
seedLangFromHost(ctx);
|
|
1253
|
+
const hub = new WorkerRegistry(ctx);
|
|
1254
|
+
const builtInProviderProbe = createProviderProbe(ctx);
|
|
1183
1255
|
try {
|
|
1184
1256
|
const { readGlobalConfig } = await import('../install/install.mjs');
|
|
1185
1257
|
hub.getConfig = () => readGlobalConfig();
|
|
@@ -1189,18 +1261,25 @@ export async function apply(ctx) {
|
|
|
1189
1261
|
// enforced its own), so the check uses the global config only. The route
|
|
1190
1262
|
// calls resolveHubSpawnPayload (above) to stamp the effective tier onto
|
|
1191
1263
|
// the spawn payload.
|
|
1192
|
-
const disposers = [];
|
|
1193
|
-
const PROVIDER_DELETE_PLAN_TTL_MS = 10 * 60 * 1000;
|
|
1194
|
-
const providerDeletePlans = new Map();
|
|
1195
|
-
const
|
|
1196
|
-
const
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1264
|
+
const disposers = [];
|
|
1265
|
+
const PROVIDER_DELETE_PLAN_TTL_MS = 10 * 60 * 1000;
|
|
1266
|
+
const providerDeletePlans = new Map();
|
|
1267
|
+
const providerMigrationPlans = new Map();
|
|
1268
|
+
const providerMigrationOwners = new Map();
|
|
1269
|
+
const providerStoreMutationOwners = new Map();
|
|
1270
|
+
const credentialPurgePlans = new Map();
|
|
1271
|
+
const rememberProviderDeletePlan = (plan) => {
|
|
1272
|
+
providerDeletePlans.set(plan.plan_id, { plan, created_at: Date.now() });
|
|
1273
|
+
while (providerDeletePlans.size > 32) providerDeletePlans.delete(providerDeletePlans.keys().next().value);
|
|
1274
|
+
};
|
|
1275
|
+
const rememberProviderMigrationPlan = (plan) => {
|
|
1276
|
+
providerMigrationPlans.set(plan.plan_id, { plan, created_at: Date.now() });
|
|
1277
|
+
while (providerMigrationPlans.size > 32) providerMigrationPlans.delete(providerMigrationPlans.keys().next().value);
|
|
1278
|
+
};
|
|
1279
|
+
const rememberCredentialPurgePlan = (plan) => {
|
|
1280
|
+
credentialPurgePlans.set(plan.plan_id, { plan, created_at: Date.now() });
|
|
1281
|
+
while (credentialPurgePlans.size > 32) credentialPurgePlans.delete(credentialPurgePlans.keys().next().value);
|
|
1282
|
+
};
|
|
1204
1283
|
|
|
1205
1284
|
// Multimodal bridge: register describe_image / generate_image for the DS
|
|
1206
1285
|
// model. Config is read per call so settings-page edits apply live; the
|
|
@@ -1289,8 +1368,8 @@ export async function apply(ctx) {
|
|
|
1289
1368
|
// as its raw default (pro-only + no tier used to spawn flash).
|
|
1290
1369
|
const resolved = resolveHubSpawnPayload(payload, () => hub.getConfig?.() ?? {});
|
|
1291
1370
|
if (!resolved.ok) return sendJson(res, 400, { ok: false, error: resolved.error, code: resolved.code });
|
|
1292
|
-
const ingress = req.headers?.['x-dsh-crew-ingress'] === 'official-3080' ? 'official-3080' : 'direct-3210';
|
|
1293
|
-
const job = await hub.spawn({ ...resolved.payload, ingress });
|
|
1371
|
+
const ingress = req.headers?.['x-dsh-crew-ingress'] === 'official-3080' ? 'official-3080' : 'direct-3210';
|
|
1372
|
+
const job = await hub.spawn({ ...resolved.payload, ingress });
|
|
1294
1373
|
return sendJson(res, 200, { ok: true, job: hub.view(job) });
|
|
1295
1374
|
}
|
|
1296
1375
|
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'cancel') {
|
|
@@ -1370,25 +1449,25 @@ export async function apply(ctx) {
|
|
|
1370
1449
|
const workspaceReadiness = requestedWorkspaceId && !requestedContext
|
|
1371
1450
|
? { status: 'UNAVAILABLE', reason_code: 'WORKSPACE_CONTEXT_NOT_FOUND' }
|
|
1372
1451
|
: await assessWorkspaceReadiness({ cwd: requestedContext?.repo_root ?? null });
|
|
1373
|
-
const liveJobs = typeof hub.list === 'function' ? hub.list() : [];
|
|
1374
|
-
const executionRows = buildHubExecutionRows(liveJobs);
|
|
1375
|
-
const readinessMatrix = { rows: [
|
|
1376
|
-
{ id: 'hub_compatibility', status: 'PASS', reason_code: 'LIVE_CHECK_PASSED' },
|
|
1377
|
-
catalogStatus,
|
|
1378
|
-
...executionRows,
|
|
1379
|
-
] };
|
|
1380
|
-
const readinessSnapshot = buildRuntimeReadinessSnapshot({
|
|
1381
|
-
runtime: getHubRuntimeIdentity(),
|
|
1382
|
-
readinessMatrix,
|
|
1383
|
-
selections: { worker: liveJobs.find((job) => job?.role === 'worker') ?? null, reviewer: liveJobs.find((job) => job?.role === 'reviewer') ?? null },
|
|
1384
|
-
health: hub.healthStore.list(),
|
|
1385
|
-
jobs: liveJobs,
|
|
1386
|
-
workspace: workspaceReadiness,
|
|
1387
|
-
});
|
|
1388
|
-
const contract = buildExtensionContract({
|
|
1389
|
-
config,
|
|
1390
|
-
readinessMatrix,
|
|
1391
|
-
readinessSnapshot,
|
|
1452
|
+
const liveJobs = typeof hub.list === 'function' ? hub.list() : [];
|
|
1453
|
+
const executionRows = buildHubExecutionRows(liveJobs);
|
|
1454
|
+
const readinessMatrix = { rows: [
|
|
1455
|
+
{ id: 'hub_compatibility', status: 'PASS', reason_code: 'LIVE_CHECK_PASSED' },
|
|
1456
|
+
catalogStatus,
|
|
1457
|
+
...executionRows,
|
|
1458
|
+
] };
|
|
1459
|
+
const readinessSnapshot = buildRuntimeReadinessSnapshot({
|
|
1460
|
+
runtime: getHubRuntimeIdentity(),
|
|
1461
|
+
readinessMatrix,
|
|
1462
|
+
selections: { worker: liveJobs.find((job) => job?.role === 'worker') ?? null, reviewer: liveJobs.find((job) => job?.role === 'reviewer') ?? null },
|
|
1463
|
+
health: hub.healthStore.list(),
|
|
1464
|
+
jobs: liveJobs,
|
|
1465
|
+
workspace: workspaceReadiness,
|
|
1466
|
+
});
|
|
1467
|
+
const contract = buildExtensionContract({
|
|
1468
|
+
config,
|
|
1469
|
+
readinessMatrix,
|
|
1470
|
+
readinessSnapshot,
|
|
1392
1471
|
workspace: workspaceReadiness,
|
|
1393
1472
|
profiles,
|
|
1394
1473
|
runtime: getHubRuntimeIdentity(),
|
|
@@ -1415,6 +1494,111 @@ export async function apply(ctx) {
|
|
|
1415
1494
|
},
|
|
1416
1495
|
}));
|
|
1417
1496
|
|
|
1497
|
+
// Quick-config: the narrow, capability-projected write surface used by
|
|
1498
|
+
// the official 3080 quick-controls panel. Only the user-facing toggles
|
|
1499
|
+
// and model-priority lists are writable here; the FULL config endpoint
|
|
1500
|
+
// remains the single authority for everything else. This is a projection
|
|
1501
|
+
// over writeGlobalConfig, never a second config store.
|
|
1502
|
+
const projectQuickConfig = (config) => Object.fromEntries(
|
|
1503
|
+
[...QUICK_CONFIG_KEYS].map((key) => [key, config[key]]),
|
|
1504
|
+
);
|
|
1505
|
+
disposers.push(webServer.register({
|
|
1506
|
+
kind: 'exact', path: `${ROUTE_BASE}/quick-config`,
|
|
1507
|
+
handler: async (req, res) => {
|
|
1508
|
+
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
1509
|
+
try {
|
|
1510
|
+
const { readGlobalConfig, writeGlobalConfig } = await import(`../install/install.mjs?t=${Date.now()}`);
|
|
1511
|
+
if (req.method === 'GET') {
|
|
1512
|
+
return sendJson(res, 200, { ok: true, config: projectQuickConfig(readGlobalConfig()) });
|
|
1513
|
+
}
|
|
1514
|
+
if (req.method === 'POST') {
|
|
1515
|
+
const body = await readBody(req);
|
|
1516
|
+
const patch = {};
|
|
1517
|
+
for (const [key, value] of Object.entries(body ?? {})) {
|
|
1518
|
+
if (!QUICK_CONFIG_KEYS.includes(key)) {
|
|
1519
|
+
return sendJson(res, 403, { ok: false, code: 'QUICK_CONFIG_KEY_FORBIDDEN', error: `key not writable from the quick surface: ${key}` });
|
|
1520
|
+
}
|
|
1521
|
+
patch[key] = value;
|
|
1522
|
+
}
|
|
1523
|
+
const next = writeGlobalConfig(patch);
|
|
1524
|
+
return sendJson(res, 200, { ok: true, config: projectQuickConfig(next) });
|
|
1525
|
+
}
|
|
1526
|
+
return sendJson(res, 405, { ok: false, error: 'GET or POST' }, { allow: 'GET, POST' });
|
|
1527
|
+
} catch (err) {
|
|
1528
|
+
return sendJson(res, 500, { ok: false, error: err?.message ?? String(err) });
|
|
1529
|
+
}
|
|
1530
|
+
},
|
|
1531
|
+
}));
|
|
1532
|
+
|
|
1533
|
+
disposers.push(webServer.register({
|
|
1534
|
+
kind: 'exact', path: `${ROUTE_BASE}/quick-status`,
|
|
1535
|
+
handler: async (req, res) => {
|
|
1536
|
+
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
1537
|
+
if (req.method !== 'GET') return sendJson(res, 405, { ok: false, error: 'GET only' }, { allow: 'GET' });
|
|
1538
|
+
try {
|
|
1539
|
+
const { readGlobalConfig } = await import(`../install/install.mjs?t=${Date.now()}`);
|
|
1540
|
+
const runtime = getHubRuntimeIdentity();
|
|
1541
|
+
return sendJson(res, 200, {
|
|
1542
|
+
ok: true,
|
|
1543
|
+
ready: true,
|
|
1544
|
+
runtime: { runtime_version: runtime.runtime_version, dsh_version: runtime.dsh_version ?? null, runtime_id: runtime.runtime_id },
|
|
1545
|
+
config: projectQuickConfig(readGlobalConfig()),
|
|
1546
|
+
});
|
|
1547
|
+
} catch (err) {
|
|
1548
|
+
return sendJson(res, 500, { ok: false, error: err?.message ?? String(err) });
|
|
1549
|
+
}
|
|
1550
|
+
},
|
|
1551
|
+
}));
|
|
1552
|
+
|
|
1553
|
+
disposers.push(webServer.register({
|
|
1554
|
+
kind: 'exact', path: `${ROUTE_BASE}/runtime/restart-request`,
|
|
1555
|
+
handler: async (req, res) => {
|
|
1556
|
+
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
1557
|
+
if (req.method !== 'POST') return sendJson(res, 405, { ok: false, error: 'POST only' }, { allow: 'POST' });
|
|
1558
|
+
try {
|
|
1559
|
+
const { createRestartRequest, readSupervisorHeartbeat, supervisorStateRoot } = await import(`../supervisor/restart-request.mjs?t=${Date.now()}`);
|
|
1560
|
+
const appRoot = CONFIG_DIR;
|
|
1561
|
+
// The restart-request helpers take the Crew app root and append
|
|
1562
|
+
// supervisor/ internally — do NOT pre-compose the path here.
|
|
1563
|
+
const heartbeat = readSupervisorHeartbeat(appRoot);
|
|
1564
|
+
if (!heartbeat) {
|
|
1565
|
+
return sendJson(res, 503, { ok: false, code: 'CREW_SUPERVISOR_UNAVAILABLE', error: 'no fresh supervisor heartbeat; restart cannot be executed' });
|
|
1566
|
+
}
|
|
1567
|
+
let reason = null;
|
|
1568
|
+
try {
|
|
1569
|
+
const body = await readBody(req);
|
|
1570
|
+
reason = typeof body?.reason === 'string' && body.reason.length > 0 ? body.reason.slice(0, 200) : null;
|
|
1571
|
+
} catch { /* body optional */ }
|
|
1572
|
+
const created = createRestartRequest({ appRoot, runtimeIdentity: getHubRuntimeIdentity(), reason });
|
|
1573
|
+
if (!created.ok) return sendJson(res, 400, { ok: false, code: created.code, error: created.error });
|
|
1574
|
+
return sendJson(res, 202, { ok: true, state: 'RESTART_REQUESTED', request_id: created.request.request_id, expires_at: created.request.expires_at });
|
|
1575
|
+
} catch (err) {
|
|
1576
|
+
return sendJson(res, 500, { ok: false, error: err?.message ?? String(err) });
|
|
1577
|
+
}
|
|
1578
|
+
},
|
|
1579
|
+
}));
|
|
1580
|
+
|
|
1581
|
+
disposers.push(webServer.register({
|
|
1582
|
+
kind: 'exact', path: `${ROUTE_BASE}/runtime/restart-status`,
|
|
1583
|
+
handler: async (req, res) => {
|
|
1584
|
+
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
1585
|
+
if (req.method !== 'GET') return sendJson(res, 405, { ok: false, error: 'GET only' }, { allow: 'GET' });
|
|
1586
|
+
try {
|
|
1587
|
+
const { readRestartResult, readRestartRequest, supervisorStateRoot } = await import(`../supervisor/restart-request.mjs?t=${Date.now()}`);
|
|
1588
|
+
const appRoot = CONFIG_DIR;
|
|
1589
|
+
const requestId = new URL(req.url, 'http://localhost').searchParams.get('id');
|
|
1590
|
+
if (!requestId || !/^[0-9a-f-]{36}$/i.test(requestId)) return sendJson(res, 400, { ok: false, error: 'request id required' });
|
|
1591
|
+
const result = readRestartResult(appRoot, requestId);
|
|
1592
|
+
if (result) return sendJson(res, 200, { ok: true, state: result.state, request_id: result.request_id, detail: result.detail ?? null });
|
|
1593
|
+
const pending = readRestartRequest(appRoot, requestId);
|
|
1594
|
+
if (pending) return sendJson(res, 200, { ok: true, state: 'RESTART_REQUESTED', request_id: pending.request_id });
|
|
1595
|
+
return sendJson(res, 404, { ok: false, error: 'unknown restart request' });
|
|
1596
|
+
} catch (err) {
|
|
1597
|
+
return sendJson(res, 500, { ok: false, error: err?.message ?? String(err) });
|
|
1598
|
+
}
|
|
1599
|
+
},
|
|
1600
|
+
}));
|
|
1601
|
+
|
|
1418
1602
|
disposers.push(webServer.register({
|
|
1419
1603
|
kind: 'exact', path: `${ROUTE_BASE}/presets`,
|
|
1420
1604
|
handler: async (req, res) => {
|
|
@@ -1434,15 +1618,15 @@ export async function apply(ctx) {
|
|
|
1434
1618
|
},
|
|
1435
1619
|
}));
|
|
1436
1620
|
|
|
1437
|
-
disposers.push(webServer.register({
|
|
1438
|
-
kind: 'exact', path: `${ROUTE_BASE}/provider`,
|
|
1439
|
-
handler: async (req, res) => {
|
|
1440
|
-
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
1441
|
-
try {
|
|
1442
|
-
const config = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
1443
|
-
const lifecycleState = readProviderLifecycleState();
|
|
1444
|
-
const healthGate = config.health_gate ?? config.worker?.model_policy?.health_gate;
|
|
1445
|
-
const mode = normalizeWorkerProviderMode(config.worker_provider_mode);
|
|
1621
|
+
disposers.push(webServer.register({
|
|
1622
|
+
kind: 'exact', path: `${ROUTE_BASE}/provider`,
|
|
1623
|
+
handler: async (req, res) => {
|
|
1624
|
+
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
1625
|
+
try {
|
|
1626
|
+
const config = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
1627
|
+
const lifecycleState = readProviderLifecycleState();
|
|
1628
|
+
const healthGate = config.health_gate ?? config.worker?.model_policy?.health_gate;
|
|
1629
|
+
const mode = normalizeWorkerProviderMode(config.worker_provider_mode);
|
|
1446
1630
|
let selections;
|
|
1447
1631
|
if (mode === 'deepseek-official') {
|
|
1448
1632
|
selections = {
|
|
@@ -1460,14 +1644,14 @@ export async function apply(ctx) {
|
|
|
1460
1644
|
tier,
|
|
1461
1645
|
priority: config[`${tier}_model_priority`],
|
|
1462
1646
|
priorityConfigured: config[`${tier}_model_priority_configured`],
|
|
1463
|
-
fallback: config[`${tier}_model_fallback`],
|
|
1464
|
-
catalog,
|
|
1465
|
-
harnessDefault: catalog.harness_default,
|
|
1466
|
-
healthStore: hub.healthStore,
|
|
1467
|
-
healthGate,
|
|
1468
|
-
allowFallback: config.allow_fallback !== false,
|
|
1469
|
-
tombstones: lifecycleState.tombstones,
|
|
1470
|
-
});
|
|
1647
|
+
fallback: config[`${tier}_model_fallback`],
|
|
1648
|
+
catalog,
|
|
1649
|
+
harnessDefault: catalog.harness_default,
|
|
1650
|
+
healthStore: hub.healthStore,
|
|
1651
|
+
healthGate,
|
|
1652
|
+
allowFallback: config.allow_fallback !== false,
|
|
1653
|
+
tombstones: lifecycleState.tombstones,
|
|
1654
|
+
});
|
|
1471
1655
|
selections[tier] = selected.ok
|
|
1472
1656
|
? { provider: selected.provider, model: selected.model, source: selected.source }
|
|
1473
1657
|
: { code: selected.code, error: selected.message };
|
|
@@ -1482,427 +1666,829 @@ export async function apply(ctx) {
|
|
|
1482
1666
|
} catch (err) {
|
|
1483
1667
|
return sendJson(res, 503, { ok: false, code: 'MODEL_CATALOG_UNAVAILABLE', error: 'Unable to resolve Harness worker models.' });
|
|
1484
1668
|
}
|
|
1485
|
-
},
|
|
1486
|
-
}));
|
|
1487
|
-
|
|
1488
|
-
// Secret-free Harness provider inventory. This is deliberately separate
|
|
1489
|
-
// from /models (catalog) and /provider (effective routing): it reports
|
|
1490
|
-
// declaration provenance, ownership, lifecycle state and policy refs.
|
|
1491
|
-
disposers.push(webServer.register({
|
|
1492
|
-
kind: 'exact', path: `${ROUTE_BASE}/providers`,
|
|
1493
|
-
handler: async (req, res) => {
|
|
1494
|
-
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
1495
|
-
if (req.method !== 'GET') return sendJson(res, 405, { ok: false, error: 'GET only' }, { allow: 'GET' });
|
|
1496
|
-
try {
|
|
1497
|
-
const config = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
1498
|
-
const inventory = await readProviderInventorySnapshot(hub, ctx, config);
|
|
1499
|
-
return sendJson(res, 200, {
|
|
1500
|
-
ok: true,
|
|
1501
|
-
...inventory,
|
|
1502
|
-
runtime: getHubRuntimeIdentity(),
|
|
1503
|
-
});
|
|
1504
|
-
} catch {
|
|
1505
|
-
return sendJson(res, 503, { ok: false, code: 'PROVIDER_INVENTORY_UNAVAILABLE', error: 'Unable to read Harness provider inventory.' });
|
|
1506
|
-
}
|
|
1507
|
-
},
|
|
1508
|
-
}));
|
|
1509
|
-
|
|
1510
|
-
// Credential references are a separate, secret-free inventory. Provider
|
|
1511
|
-
// deletion can report these references, but never implies their purge.
|
|
1512
|
-
disposers.push(webServer.register({
|
|
1513
|
-
kind: 'exact', path: `${ROUTE_BASE}/credential-references`,
|
|
1514
|
-
handler: async (req, res) => {
|
|
1515
|
-
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
1516
|
-
if (req.method !== 'GET') return sendJson(res, 405, { ok: false, error: 'GET only' }, { allow: 'GET' });
|
|
1517
|
-
try {
|
|
1518
|
-
const config = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
1519
|
-
const snapshot = await readProviderInventorySnapshot(hub, ctx, config);
|
|
1520
|
-
const credentials = buildCredentialReferenceInventory({ providers: snapshot.records, additional_refs: snapshot.credential_history_refs, purged_refs: snapshot.credential_purged_refs });
|
|
1521
|
-
return sendJson(res, 200, { ok: true, ...credentials, unverified_purges: snapshot.credential_purge_unverified, runtime: getHubRuntimeIdentity() });
|
|
1522
|
-
} catch {
|
|
1523
|
-
return sendJson(res, 503, { ok: false, code: 'CREDENTIAL_REFERENCE_INVENTORY_UNAVAILABLE' });
|
|
1524
|
-
}
|
|
1525
|
-
},
|
|
1526
|
-
}));
|
|
1527
|
-
|
|
1528
|
-
disposers.push(webServer.register({
|
|
1529
|
-
kind: 'prefix', path: `${ROUTE_BASE}/credential-references`,
|
|
1530
|
-
handler: async (req, res) => {
|
|
1531
|
-
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
1532
|
-
let url;
|
|
1533
|
-
try { url = new URL(req.url, 'http://localhost'); } catch { return sendJson(res, 400, { ok: false, code: 'CREDENTIAL_PURGE_PLAN_INVALID' }); }
|
|
1534
|
-
const parts = url.pathname.slice(`${ROUTE_BASE}/credential-references`.length).split('/').filter(Boolean);
|
|
1535
|
-
try {
|
|
1536
|
-
const referenceId = decodeURIComponent(parts[0] ?? '');
|
|
1537
|
-
const body = await readBody(req);
|
|
1538
|
-
const config = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
1539
|
-
const snapshot = await readProviderInventorySnapshot(hub, ctx, config);
|
|
1540
|
-
const credentials = buildCredentialReferenceInventory({ providers: snapshot.records, additional_refs: snapshot.credential_history_refs, purged_refs: snapshot.credential_purged_refs });
|
|
1541
|
-
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'purge-plan') {
|
|
1542
|
-
const planned = planCredentialPurge({ inventory: credentials, referenceId, expectedRevision: body?.expected_revision });
|
|
1543
|
-
if (!planned.ok) return sendJson(res, 409, { ok: false, code: planned.code });
|
|
1544
|
-
rememberCredentialPurgePlan(planned.plan);
|
|
1545
|
-
return sendJson(res, 200, { ok: true, plan: planned.plan });
|
|
1546
|
-
}
|
|
1547
|
-
if (req.method === 'DELETE' && parts.length === 1) {
|
|
1548
|
-
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'CREDENTIAL_PURGE_CONFIRM_REQUIRED' });
|
|
1549
|
-
const stored = credentialPurgePlans.get(body?.plan_id);
|
|
1550
|
-
if (!stored || Date.now() - stored.created_at > PROVIDER_DELETE_PLAN_TTL_MS || stored.plan.reference_id !== referenceId) {
|
|
1551
|
-
credentialPurgePlans.delete(body?.plan_id);
|
|
1552
|
-
return sendJson(res, 409, { ok: false, code: 'CREDENTIAL_PURGE_PLAN_EXPIRED' });
|
|
1553
|
-
}
|
|
1554
|
-
const current = planCredentialPurge({ inventory: credentials, referenceId, expectedRevision: stored.plan.expected_revision });
|
|
1555
|
-
if (!current.ok) return sendJson(res, 409, { ok: false, code: current.code });
|
|
1556
|
-
if (current.plan.expected_revision !== stored.plan.expected_revision) return sendJson(res, 409, { ok: false, code: 'CREDENTIAL_REFERENCE_CHANGED' });
|
|
1557
|
-
credentialPurgePlans.delete(body.plan_id);
|
|
1558
|
-
const adapter = createCredentialPurgeFileHooks({
|
|
1559
|
-
credentialsFile: join(CONFIG_DIR, 'harness', '.credentials.yaml'),
|
|
1560
|
-
crewHome: join(CONFIG_DIR, 'harness'),
|
|
1561
|
-
});
|
|
1562
|
-
const result = await executeCredentialPurge(current.plan, {
|
|
1563
|
-
recheck: async (id) => {
|
|
1564
|
-
const freshSnapshot = await readProviderInventorySnapshot(hub, ctx, config);
|
|
1565
|
-
const freshCredentials = buildCredentialReferenceInventory({ providers: freshSnapshot.records, additional_refs: freshSnapshot.credential_history_refs, purged_refs: freshSnapshot.credential_purged_refs });
|
|
1566
|
-
const freshPlan = planCredentialPurge({ inventory: freshCredentials, referenceId: id });
|
|
1567
|
-
if (!freshPlan.ok) return { ok: false };
|
|
1568
|
-
const freshRecord = freshCredentials.records.find((entry) => entry.reference_id === id);
|
|
1569
|
-
return {
|
|
1570
|
-
ok: true,
|
|
1571
|
-
revision: freshPlan.plan.expected_revision,
|
|
1572
|
-
orphan: freshRecord?.orphan === true,
|
|
1573
|
-
ownership: freshRecord?.ownership,
|
|
1574
|
-
purge_capability: freshRecord?.purge_capability,
|
|
1575
|
-
};
|
|
1576
|
-
},
|
|
1577
|
-
...(adapter.ok ? { purge: adapter.purge, verify: adapter.verify } : {}),
|
|
1578
|
-
});
|
|
1579
|
-
if (result.state === 'PURGED' || result.state === 'VERIFIED') {
|
|
1580
|
-
try {
|
|
1581
|
-
writeCredentialPurgeState(recordCredentialPurgeOutcome(readCredentialPurgeState(), current.plan, result));
|
|
1582
|
-
} catch {
|
|
1583
|
-
return sendJson(res, 503, { ok: false, code: 'CREDENTIAL_PURGE_AUDIT_FAILED', state: result.state, reference_id: referenceId });
|
|
1584
|
-
}
|
|
1585
|
-
}
|
|
1586
|
-
return sendJson(res, result.ok ? 200 : 503, result);
|
|
1587
|
-
}
|
|
1588
|
-
return sendJson(res, 404, { ok: false, code: 'CREDENTIAL_REFERENCE_ENDPOINT_NOT_FOUND' });
|
|
1589
|
-
} catch (error) {
|
|
1590
|
-
return sendJson(res, 400, { ok: false, code: boundedMachineCodeFromError(error) ?? 'CREDENTIAL_PURGE_PLAN_INVALID' });
|
|
1591
|
-
}
|
|
1592
|
-
},
|
|
1593
|
-
}));
|
|
1594
|
-
|
|
1595
|
-
// Deletion is two-phase: this endpoint only computes a revision-bound,
|
|
1596
|
-
// secret-free impact plan. A later mutation endpoint must present that
|
|
1597
|
-
// plan and perform restart/verification through an owned supervisor.
|
|
1598
|
-
disposers.push(webServer.register({
|
|
1599
|
-
kind: 'prefix', path: `${ROUTE_BASE}/providers`,
|
|
1600
|
-
handler: async (req, res) => {
|
|
1601
|
-
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
1602
|
-
let url;
|
|
1603
|
-
try { url = new URL(req.url, 'http://localhost'); } catch { return sendJson(res, 400, { ok: false, code: 'PROVIDER_PLAN_INVALID' }); }
|
|
1604
|
-
const parts = url.pathname.slice(`${ROUTE_BASE}/providers`.length).split('/').filter(Boolean);
|
|
1605
|
-
try {
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
if (
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
}
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
);
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
}
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
}
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1669
|
+
},
|
|
1670
|
+
}));
|
|
1671
|
+
|
|
1672
|
+
// Secret-free Harness provider inventory. This is deliberately separate
|
|
1673
|
+
// from /models (catalog) and /provider (effective routing): it reports
|
|
1674
|
+
// declaration provenance, ownership, lifecycle state and policy refs.
|
|
1675
|
+
disposers.push(webServer.register({
|
|
1676
|
+
kind: 'exact', path: `${ROUTE_BASE}/providers`,
|
|
1677
|
+
handler: async (req, res) => {
|
|
1678
|
+
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
1679
|
+
if (req.method !== 'GET') return sendJson(res, 405, { ok: false, error: 'GET only' }, { allow: 'GET' });
|
|
1680
|
+
try {
|
|
1681
|
+
const config = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
1682
|
+
const inventory = await readProviderInventorySnapshot(hub, ctx, config);
|
|
1683
|
+
return sendJson(res, 200, {
|
|
1684
|
+
ok: true,
|
|
1685
|
+
...inventory,
|
|
1686
|
+
runtime: getHubRuntimeIdentity(),
|
|
1687
|
+
});
|
|
1688
|
+
} catch {
|
|
1689
|
+
return sendJson(res, 503, { ok: false, code: 'PROVIDER_INVENTORY_UNAVAILABLE', error: 'Unable to read Harness provider inventory.' });
|
|
1690
|
+
}
|
|
1691
|
+
},
|
|
1692
|
+
}));
|
|
1693
|
+
|
|
1694
|
+
// Credential references are a separate, secret-free inventory. Provider
|
|
1695
|
+
// deletion can report these references, but never implies their purge.
|
|
1696
|
+
disposers.push(webServer.register({
|
|
1697
|
+
kind: 'exact', path: `${ROUTE_BASE}/credential-references`,
|
|
1698
|
+
handler: async (req, res) => {
|
|
1699
|
+
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
1700
|
+
if (req.method !== 'GET') return sendJson(res, 405, { ok: false, error: 'GET only' }, { allow: 'GET' });
|
|
1701
|
+
try {
|
|
1702
|
+
const config = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
1703
|
+
const snapshot = await readProviderInventorySnapshot(hub, ctx, config);
|
|
1704
|
+
const credentials = buildCredentialReferenceInventory({ providers: snapshot.records, additional_refs: snapshot.credential_history_refs, purged_refs: snapshot.credential_purged_refs });
|
|
1705
|
+
return sendJson(res, 200, { ok: true, ...credentials, unverified_purges: snapshot.credential_purge_unverified, runtime: getHubRuntimeIdentity() });
|
|
1706
|
+
} catch {
|
|
1707
|
+
return sendJson(res, 503, { ok: false, code: 'CREDENTIAL_REFERENCE_INVENTORY_UNAVAILABLE' });
|
|
1708
|
+
}
|
|
1709
|
+
},
|
|
1710
|
+
}));
|
|
1711
|
+
|
|
1712
|
+
disposers.push(webServer.register({
|
|
1713
|
+
kind: 'prefix', path: `${ROUTE_BASE}/credential-references`,
|
|
1714
|
+
handler: async (req, res) => {
|
|
1715
|
+
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
1716
|
+
let url;
|
|
1717
|
+
try { url = new URL(req.url, 'http://localhost'); } catch { return sendJson(res, 400, { ok: false, code: 'CREDENTIAL_PURGE_PLAN_INVALID' }); }
|
|
1718
|
+
const parts = url.pathname.slice(`${ROUTE_BASE}/credential-references`.length).split('/').filter(Boolean);
|
|
1719
|
+
try {
|
|
1720
|
+
const referenceId = decodeURIComponent(parts[0] ?? '');
|
|
1721
|
+
const body = await readBody(req);
|
|
1722
|
+
const config = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
1723
|
+
const snapshot = await readProviderInventorySnapshot(hub, ctx, config);
|
|
1724
|
+
const credentials = buildCredentialReferenceInventory({ providers: snapshot.records, additional_refs: snapshot.credential_history_refs, purged_refs: snapshot.credential_purged_refs });
|
|
1725
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'purge-plan') {
|
|
1726
|
+
const planned = planCredentialPurge({ inventory: credentials, referenceId, expectedRevision: body?.expected_revision });
|
|
1727
|
+
if (!planned.ok) return sendJson(res, 409, { ok: false, code: planned.code });
|
|
1728
|
+
rememberCredentialPurgePlan(planned.plan);
|
|
1729
|
+
return sendJson(res, 200, { ok: true, plan: planned.plan });
|
|
1730
|
+
}
|
|
1731
|
+
if (req.method === 'DELETE' && parts.length === 1) {
|
|
1732
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'CREDENTIAL_PURGE_CONFIRM_REQUIRED' });
|
|
1733
|
+
const stored = credentialPurgePlans.get(body?.plan_id);
|
|
1734
|
+
if (!stored || Date.now() - stored.created_at > PROVIDER_DELETE_PLAN_TTL_MS || stored.plan.reference_id !== referenceId) {
|
|
1735
|
+
credentialPurgePlans.delete(body?.plan_id);
|
|
1736
|
+
return sendJson(res, 409, { ok: false, code: 'CREDENTIAL_PURGE_PLAN_EXPIRED' });
|
|
1737
|
+
}
|
|
1738
|
+
const current = planCredentialPurge({ inventory: credentials, referenceId, expectedRevision: stored.plan.expected_revision });
|
|
1739
|
+
if (!current.ok) return sendJson(res, 409, { ok: false, code: current.code });
|
|
1740
|
+
if (current.plan.expected_revision !== stored.plan.expected_revision) return sendJson(res, 409, { ok: false, code: 'CREDENTIAL_REFERENCE_CHANGED' });
|
|
1741
|
+
credentialPurgePlans.delete(body.plan_id);
|
|
1742
|
+
const adapter = createCredentialPurgeFileHooks({
|
|
1743
|
+
credentialsFile: join(CONFIG_DIR, 'harness', '.credentials.yaml'),
|
|
1744
|
+
crewHome: join(CONFIG_DIR, 'harness'),
|
|
1745
|
+
});
|
|
1746
|
+
const result = await executeCredentialPurge(current.plan, {
|
|
1747
|
+
recheck: async (id) => {
|
|
1748
|
+
const freshSnapshot = await readProviderInventorySnapshot(hub, ctx, config);
|
|
1749
|
+
const freshCredentials = buildCredentialReferenceInventory({ providers: freshSnapshot.records, additional_refs: freshSnapshot.credential_history_refs, purged_refs: freshSnapshot.credential_purged_refs });
|
|
1750
|
+
const freshPlan = planCredentialPurge({ inventory: freshCredentials, referenceId: id });
|
|
1751
|
+
if (!freshPlan.ok) return { ok: false };
|
|
1752
|
+
const freshRecord = freshCredentials.records.find((entry) => entry.reference_id === id);
|
|
1753
|
+
return {
|
|
1754
|
+
ok: true,
|
|
1755
|
+
revision: freshPlan.plan.expected_revision,
|
|
1756
|
+
orphan: freshRecord?.orphan === true,
|
|
1757
|
+
ownership: freshRecord?.ownership,
|
|
1758
|
+
purge_capability: freshRecord?.purge_capability,
|
|
1759
|
+
};
|
|
1760
|
+
},
|
|
1761
|
+
...(adapter.ok ? { purge: adapter.purge, verify: adapter.verify } : {}),
|
|
1762
|
+
});
|
|
1763
|
+
if (result.state === 'PURGED' || result.state === 'VERIFIED') {
|
|
1764
|
+
try {
|
|
1765
|
+
writeCredentialPurgeState(recordCredentialPurgeOutcome(readCredentialPurgeState(), current.plan, result));
|
|
1766
|
+
} catch {
|
|
1767
|
+
return sendJson(res, 503, { ok: false, code: 'CREDENTIAL_PURGE_AUDIT_FAILED', state: result.state, reference_id: referenceId });
|
|
1768
|
+
}
|
|
1769
|
+
}
|
|
1770
|
+
return sendJson(res, result.ok ? 200 : 503, result);
|
|
1771
|
+
}
|
|
1772
|
+
return sendJson(res, 404, { ok: false, code: 'CREDENTIAL_REFERENCE_ENDPOINT_NOT_FOUND' });
|
|
1773
|
+
} catch (error) {
|
|
1774
|
+
return sendJson(res, 400, { ok: false, code: boundedMachineCodeFromError(error) ?? 'CREDENTIAL_PURGE_PLAN_INVALID' });
|
|
1775
|
+
}
|
|
1776
|
+
},
|
|
1777
|
+
}));
|
|
1778
|
+
|
|
1779
|
+
// Deletion is two-phase: this endpoint only computes a revision-bound,
|
|
1780
|
+
// secret-free impact plan. A later mutation endpoint must present that
|
|
1781
|
+
// plan and perform restart/verification through an owned supervisor.
|
|
1782
|
+
disposers.push(webServer.register({
|
|
1783
|
+
kind: 'prefix', path: `${ROUTE_BASE}/providers`,
|
|
1784
|
+
handler: async (req, res) => {
|
|
1785
|
+
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
1786
|
+
let url;
|
|
1787
|
+
try { url = new URL(req.url, 'http://localhost'); } catch { return sendJson(res, 400, { ok: false, code: 'PROVIDER_PLAN_INVALID' }); }
|
|
1788
|
+
const parts = url.pathname.slice(`${ROUTE_BASE}/providers`.length).split('/').filter(Boolean);
|
|
1789
|
+
try {
|
|
1790
|
+
if (req.method === 'GET' && parts.length === 1 && parts[0] === 'migration-status') {
|
|
1791
|
+
const config = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
1792
|
+
const inventory = await readProviderInventorySnapshot(hub, ctx, config);
|
|
1793
|
+
return sendJson(res, 200, { ok: true, ...inventory.migration, runtime: getHubRuntimeIdentity() });
|
|
1794
|
+
}
|
|
1795
|
+
const providerId = decodeURIComponent(parts[0]);
|
|
1796
|
+
const body = await readBody(req);
|
|
1797
|
+
if (req.method === 'POST' && parts[0] === '_recovery' && parts[1] === 'lock' && parts.length === 2) {
|
|
1798
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_CONFIRM_REQUIRED' });
|
|
1799
|
+
const hooks = createProviderDeleteFileHooks({
|
|
1800
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1801
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1802
|
+
configFile: join(CONFIG_DIR, 'config.json'),
|
|
1803
|
+
lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
1804
|
+
backupDir: join(CONFIG_DIR, 'provider-backups'),
|
|
1805
|
+
sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
|
|
1806
|
+
});
|
|
1807
|
+
const recovered = await hooks.recoverLock({ force: body?.force === true });
|
|
1808
|
+
const migrationHooks = createProviderLayerMigrationFileHooks({
|
|
1809
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1810
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1811
|
+
backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
|
|
1812
|
+
});
|
|
1813
|
+
const migrationRecovered = await migrationHooks.recoverLock({ confirm: true });
|
|
1814
|
+
const ok = recovered.ok === true && migrationRecovered.ok === true;
|
|
1815
|
+
return sendJson(res, ok ? 200 : 409, { ok, delete_lock: recovered, migration_lock: migrationRecovered });
|
|
1816
|
+
}
|
|
1817
|
+
if (req.method === 'POST' && parts[0] === '_recovery' && parts[1] === 'quarantine' && parts.length === 2) {
|
|
1818
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_QUARANTINE_CONFIRM_REQUIRED' });
|
|
1819
|
+
const actionId = typeof body?.action_id === 'string' && RECOVERY_ACTION_PATTERN.test(body.action_id)
|
|
1820
|
+
? body.action_id : null;
|
|
1821
|
+
if (!actionId) return sendJson(res, 400, { ok: false, code: 'PROVIDER_DELETE_PLAN_INVALID' });
|
|
1822
|
+
const recovery = [
|
|
1823
|
+
...readProviderRecoveryTransactions(),
|
|
1824
|
+
...readProviderLayerMigrationTransactions(join(CONFIG_DIR, 'provider-migration-backups')),
|
|
1825
|
+
].find((entry) => entry.action_id === actionId);
|
|
1826
|
+
if (!recovery) return sendJson(res, 404, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_NOT_FOUND' });
|
|
1827
|
+
if (recovery.recoverable === true) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_RECOVERABLE' });
|
|
1828
|
+
if (typeof recovery.storage_id !== 'string') return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_QUARANTINE_FAILED' });
|
|
1829
|
+
if (recovery.source === 'provider-layer-migration') {
|
|
1830
|
+
try {
|
|
1831
|
+
const migrationHooks = createProviderLayerMigrationFileHooks({
|
|
1832
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1833
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1834
|
+
backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
|
|
1835
|
+
});
|
|
1836
|
+
const quarantined = await migrationHooks.quarantine(actionId);
|
|
1837
|
+
return sendJson(res, 200, { ...quarantined, transaction_id: recovery.transaction_id ?? null });
|
|
1838
|
+
} catch (error) {
|
|
1839
|
+
const code = boundedMachineCodeFromError(error) ?? 'PROVIDER_MIGRATION_RECOVERY_QUARANTINE_FAILED';
|
|
1840
|
+
return sendJson(res, 409, { ok: false, code });
|
|
1841
|
+
}
|
|
1842
|
+
}
|
|
1843
|
+
try {
|
|
1844
|
+
const hooks = createProviderDeleteFileHooks({
|
|
1845
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1846
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1847
|
+
configFile: join(CONFIG_DIR, 'config.json'),
|
|
1848
|
+
lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
1849
|
+
backupDir: join(CONFIG_DIR, 'provider-backups'),
|
|
1850
|
+
sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
|
|
1851
|
+
afterLockAcquired: () => {
|
|
1852
|
+
const current = readProviderRecoveryTransactions().find((entry) => entry.action_id === actionId);
|
|
1853
|
+
if (!current || current.recoverable === true || current.storage_id !== recovery.storage_id) {
|
|
1854
|
+
throw Object.assign(new Error('provider recovery entry changed before quarantine'), { code: 'PROVIDER_DELETE_STATE_CHANGED' });
|
|
1855
|
+
}
|
|
1856
|
+
},
|
|
1857
|
+
});
|
|
1858
|
+
const quarantined = await hooks.quarantine(recovery.storage_id);
|
|
1859
|
+
return sendJson(res, 200, { ...quarantined, action_id: actionId, transaction_id: recovery.transaction_id ?? null });
|
|
1860
|
+
} catch (error) {
|
|
1861
|
+
const code = boundedMachineCodeFromError(error) ?? 'PROVIDER_DELETE_RECOVERY_QUARANTINE_FAILED';
|
|
1862
|
+
return sendJson(res, 409, { ok: false, code });
|
|
1863
|
+
}
|
|
1864
|
+
}
|
|
1865
|
+
const config = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
1866
|
+
const inventory = await readProviderInventorySnapshot(hub, ctx, config);
|
|
1867
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'migrate-plan') {
|
|
1868
|
+
if (!hasCompleteProviderCatalogEvidence(inventory.catalog_evidence)) return sendJson(res, 503, { ok: false, code: 'MODEL_CATALOG_UNAVAILABLE' });
|
|
1869
|
+
if (!hasCompleteProviderDeclarationEvidence(inventory.declaration_evidence)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_SOURCE_UNRESOLVED' });
|
|
1870
|
+
if (!hasAvailableProviderLifecycleEvidence(inventory.lifecycle_evidence)) return sendJson(res, 503, { ok: false, code: 'PROVIDER_LIFECYCLE_UNAVAILABLE' });
|
|
1871
|
+
if (inventory.default_evidence?.ok !== true) return sendJson(res, 409, { ok: false, code: inventory.default_evidence?.code ?? 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' });
|
|
1872
|
+
if (hasPendingProviderRecoveryTransactions(inventory)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
1873
|
+
const entry = inventory.migration.providers.find((candidate) => candidate.provider_id === providerId);
|
|
1874
|
+
if (!entry) return sendJson(res, 404, { ok: false, code: 'PROVIDER_MIGRATION_NOT_REQUIRED' });
|
|
1875
|
+
if (entry.action === 'blocked' || entry.action === 'collision-review') return sendJson(res, 409, { ok: false, code: entry.blocked_reason ?? 'PROVIDER_MIGRATION_BLOCKED' });
|
|
1876
|
+
const revisions = readProviderSourceRevisions();
|
|
1877
|
+
if (!revisions.profile.ok) return sendJson(res, 409, { ok: false, code: revisions.profile.code ?? 'PROVIDER_PROFILE_CHANGED' });
|
|
1878
|
+
if ((entry.action === 'promote-existing-user' || entry.harness_default) && !revisions.settings.ok) return sendJson(res, 409, { ok: false, code: revisions.settings.code ?? 'PROVIDER_SETTINGS_CHANGED' });
|
|
1879
|
+
const profileFile = join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml');
|
|
1880
|
+
const parsedMaterialization = readProviderMaterialization(readFileSync(profileFile, 'utf8'), { providerId, file: 'harness/profiles/dsh-crew/cordis.patch.yml' });
|
|
1881
|
+
if (!parsedMaterialization.ok) return sendJson(res, 409, { ok: false, code: parsedMaterialization.code });
|
|
1882
|
+
const materialization = parsedMaterialization.provider;
|
|
1883
|
+
let userMaterializationBefore;
|
|
1884
|
+
if (entry.action === 'promote-existing-user') {
|
|
1885
|
+
const settingsFile = join(CONFIG_DIR, 'harness', 'settings.yaml');
|
|
1886
|
+
if (!existsSync(settingsFile)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_SETTINGS_CHANGED' });
|
|
1887
|
+
const parsedUserMaterialization = readProviderSettingsMaterialization(readFileSync(settingsFile, 'utf8'), { providerId, file: 'harness/settings.yaml' });
|
|
1888
|
+
if (!parsedUserMaterialization.ok) return sendJson(res, 409, { ok: false, code: parsedUserMaterialization.code });
|
|
1889
|
+
userMaterializationBefore = parsedUserMaterialization.provider;
|
|
1890
|
+
}
|
|
1891
|
+
const plan = {
|
|
1892
|
+
schema_version: 1,
|
|
1893
|
+
kind: 'provider-layer-migration',
|
|
1894
|
+
plan_id: randomUUID(),
|
|
1895
|
+
provider_id: providerId,
|
|
1896
|
+
action: entry.action,
|
|
1897
|
+
source: entry.source,
|
|
1898
|
+
routing_references: entry.referenced_by,
|
|
1899
|
+
materialization: { provider: materialization },
|
|
1900
|
+
...(userMaterializationBefore ? { user_materialization_before: userMaterializationBefore } : {}),
|
|
1901
|
+
expected_revisions: {
|
|
1902
|
+
profile: revisions.profile.revision,
|
|
1903
|
+
settings: revisions.settings.ok ? revisions.settings.revision : null,
|
|
1904
|
+
},
|
|
1905
|
+
harness_default_before: inventory.harness_default ?? null,
|
|
1906
|
+
runtime_id_before: getHubRuntimeIdentity().runtime_id,
|
|
1907
|
+
restart_required: true,
|
|
1908
|
+
};
|
|
1909
|
+
rememberProviderMigrationPlan(plan);
|
|
1910
|
+
return sendJson(res, 200, { ok: true, plan });
|
|
1911
|
+
}
|
|
1912
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'migrate') {
|
|
1913
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_MIGRATION_CONFIRM_REQUIRED' });
|
|
1914
|
+
const stored = providerMigrationPlans.get(body?.plan_id);
|
|
1915
|
+
if (!stored || Date.now() - stored.created_at > PROVIDER_DELETE_PLAN_TTL_MS || stored.plan.provider_id !== providerId) {
|
|
1916
|
+
providerMigrationPlans.delete(body?.plan_id);
|
|
1917
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_PLAN_EXPIRED' });
|
|
1918
|
+
}
|
|
1919
|
+
const entry = inventory.migration.providers.find((candidate) => candidate.provider_id === providerId);
|
|
1920
|
+
if (!entry || entry.action !== stored.plan.action || entry.action === 'blocked' || entry.action === 'collision-review') return sendJson(res, 409, { ok: false, code: entry?.blocked_reason ?? 'PROVIDER_MIGRATION_PLAN_CHANGED' });
|
|
1921
|
+
if (inventory.default_evidence?.ok !== true) return sendJson(res, 409, { ok: false, code: inventory.default_evidence?.code ?? 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' });
|
|
1922
|
+
const revisions = readProviderSourceRevisions();
|
|
1923
|
+
const expected = stored.plan.expected_revisions ?? {};
|
|
1924
|
+
if (revisions.profile.revision !== expected.profile || (revisions.settings.ok ? revisions.settings.revision : null) !== expected.settings) {
|
|
1925
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_SOURCE_CHANGED' });
|
|
1926
|
+
}
|
|
1927
|
+
const hooks = createProviderLayerMigrationFileHooks({
|
|
1928
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1929
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1930
|
+
backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
|
|
1931
|
+
});
|
|
1932
|
+
let result;
|
|
1933
|
+
if (hub.hasProviderLease(providerId)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_IN_USE' });
|
|
1934
|
+
if (!hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
1935
|
+
if (!hub.beginProviderMutation(providerId)) {
|
|
1936
|
+
hub.endProviderStoreMutation();
|
|
1937
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
1938
|
+
}
|
|
1939
|
+
let retainProviderMutation = false;
|
|
1940
|
+
let retainProviderStoreMutation = false;
|
|
1941
|
+
try {
|
|
1942
|
+
await hooks.acquireLock();
|
|
1943
|
+
// The plan was created from a pre-lock snapshot. Re-read every
|
|
1944
|
+
// authoritative source after taking the shared store lock so a
|
|
1945
|
+
// concurrent catalog/default/recovery change cannot be hidden
|
|
1946
|
+
// behind unchanged profile/settings revisions.
|
|
1947
|
+
const lockedConfig = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
1948
|
+
const lockedInventory = await readProviderInventorySnapshot(hub, ctx, lockedConfig);
|
|
1949
|
+
const lockedEntry = lockedInventory.migration.providers.find((candidate) => candidate.provider_id === providerId);
|
|
1950
|
+
const expectedRouting = Array.isArray(stored.plan.routing_references) ? stored.plan.routing_references : [];
|
|
1951
|
+
const sameRouting = JSON.stringify(lockedEntry?.referenced_by ?? []) === JSON.stringify(expectedRouting);
|
|
1952
|
+
const sameDefault = JSON.stringify(lockedInventory.harness_default ?? null) === JSON.stringify(stored.plan.harness_default_before ?? null);
|
|
1953
|
+
const profileFile = join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml');
|
|
1954
|
+
const lockedMaterialization = existsSync(profileFile)
|
|
1955
|
+
? readProviderMaterialization(readFileSync(profileFile, 'utf8'), { providerId, file: 'harness/profiles/dsh-crew/cordis.patch.yml' })
|
|
1956
|
+
: { ok: false };
|
|
1957
|
+
const sameMaterialization = lockedMaterialization.ok === true
|
|
1958
|
+
&& JSON.stringify(lockedMaterialization.provider) === JSON.stringify(stored.plan.materialization?.provider ?? null);
|
|
1959
|
+
const settingsFile = join(CONFIG_DIR, 'harness', 'settings.yaml');
|
|
1960
|
+
const lockedUserMaterialization = stored.plan.action === 'promote-existing-user' && existsSync(settingsFile)
|
|
1961
|
+
? readProviderSettingsMaterialization(readFileSync(settingsFile, 'utf8'), { providerId, file: 'harness/settings.yaml' })
|
|
1962
|
+
: { ok: stored.plan.action !== 'promote-existing-user' };
|
|
1963
|
+
const sameUserMaterialization = stored.plan.action !== 'promote-existing-user'
|
|
1964
|
+
|| (lockedUserMaterialization.ok === true && JSON.stringify(lockedUserMaterialization.provider) === JSON.stringify(stored.plan.user_materialization_before ?? null));
|
|
1965
|
+
const lockedRevisions = readProviderSourceRevisions();
|
|
1966
|
+
const expectedLocked = stored.plan.expected_revisions ?? {};
|
|
1967
|
+
const sameRevisions = lockedRevisions.profile.revision === expectedLocked.profile
|
|
1968
|
+
&& (lockedRevisions.settings.ok ? lockedRevisions.settings.revision : null) === expectedLocked.settings;
|
|
1969
|
+
if (hasPendingProviderRecoveryTransactions(lockedInventory)) {
|
|
1970
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
1971
|
+
}
|
|
1972
|
+
if (!lockedEntry || lockedEntry.action !== stored.plan.action || lockedEntry.collision !== null
|
|
1973
|
+
|| !sameRouting || !sameDefault || !sameMaterialization || !sameUserMaterialization || !sameRevisions
|
|
1974
|
+
|| lockedInventory.default_evidence?.ok !== true
|
|
1975
|
+
|| !hasCompleteProviderCatalogEvidence(lockedInventory.catalog_evidence)
|
|
1976
|
+
|| !hasCompleteProviderDeclarationEvidence(lockedInventory.declaration_evidence)
|
|
1977
|
+
|| !hasAvailableProviderLifecycleEvidence(lockedInventory.lifecycle_evidence)) {
|
|
1978
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_PLAN_CHANGED' });
|
|
1979
|
+
}
|
|
1980
|
+
if (hub.hasProviderLease(providerId)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_IN_USE' });
|
|
1981
|
+
result = await executeProviderLayerMigration(stored.plan, hooks, { confirm: true, deferRestart: true });
|
|
1982
|
+
if (result.ok && result.state === 'RESTART_PENDING') await hooks.setRestartRuntimeBaseline(getHubRuntimeIdentity().runtime_id);
|
|
1983
|
+
providerMigrationPlans.delete(body.plan_id);
|
|
1984
|
+
retainProviderMutation = result.state === 'RESTART_PENDING';
|
|
1985
|
+
if (retainProviderMutation) {
|
|
1986
|
+
providerMigrationOwners.set(providerId, stored.plan.plan_id);
|
|
1987
|
+
providerStoreMutationOwners.set(stored.plan.plan_id, providerId);
|
|
1988
|
+
retainProviderStoreMutation = true;
|
|
1989
|
+
}
|
|
1990
|
+
return sendJson(res, result.ok ? 202 : 409, { ok: result.ok, restart_required: result.restart_required === true, result });
|
|
1991
|
+
} finally {
|
|
1992
|
+
await hooks.release();
|
|
1993
|
+
if (!retainProviderMutation) hub.endProviderMutation(providerId);
|
|
1994
|
+
if (!retainProviderStoreMutation) hub.endProviderStoreMutation();
|
|
1995
|
+
}
|
|
1996
|
+
}
|
|
1997
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'verify-migration') {
|
|
1998
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_MIGRATION_VERIFY_CONFIRM_REQUIRED' });
|
|
1999
|
+
const transactionId = body?.transaction_id;
|
|
2000
|
+
const hooks = createProviderLayerMigrationFileHooks({
|
|
2001
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
2002
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
2003
|
+
backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
|
|
2004
|
+
existingMigrationId: transactionId,
|
|
2005
|
+
});
|
|
2006
|
+
let acquiredStoreFence = false;
|
|
2007
|
+
try {
|
|
2008
|
+
await hooks.acquireLock();
|
|
2009
|
+
await hooks.backup({});
|
|
2010
|
+
const plan = hooks.backupPlan();
|
|
2011
|
+
if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_PLAN_PROVIDER_MISMATCH' });
|
|
2012
|
+
const owner = providerMigrationOwners.get(providerId);
|
|
2013
|
+
if (providerStoreMutationOwners.get(plan.plan_id) !== providerId || !hub.hasProviderStoreMutation()) {
|
|
2014
|
+
if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
2015
|
+
providerStoreMutationOwners.set(plan.plan_id, providerId);
|
|
2016
|
+
acquiredStoreFence = true;
|
|
2017
|
+
}
|
|
2018
|
+
if (owner !== plan.plan_id) {
|
|
2019
|
+
if (hub.hasProviderLease(providerId) || !hub.beginProviderMutation(providerId)) {
|
|
2020
|
+
if (acquiredStoreFence && providerStoreMutationOwners.get(plan.plan_id) === providerId) { providerStoreMutationOwners.delete(plan.plan_id); hub.endProviderStoreMutation(); }
|
|
2021
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
2022
|
+
}
|
|
2023
|
+
providerMigrationOwners.set(providerId, plan.plan_id);
|
|
2024
|
+
}
|
|
2025
|
+
const verification = await hooks.verify(plan);
|
|
2026
|
+
const live = await readProviderInventorySnapshot(hub, ctx, config);
|
|
2027
|
+
verification.catalogEvidence = live.catalog_evidence;
|
|
2028
|
+
verification.defaultEvidence = live.default_evidence;
|
|
2029
|
+
verification.defaultPreserved = live.default_evidence?.ok === true
|
|
2030
|
+
&& JSON.stringify(live.harness_default ?? null) === JSON.stringify(plan.harness_default_before ?? null);
|
|
2031
|
+
verification.liveNativeRemovable = hasCompleteProviderCatalogEvidence(live.catalog_evidence)
|
|
2032
|
+
&& live.records.some((record) => record.id === providerId && record.lifecycle?.catalogued === true && record.native_removable === true);
|
|
2033
|
+
verification.runtimeRestarted = hasProviderRuntimeRestartEvidence(plan.runtime_id_before_restart ?? plan.runtime_id_before, getHubRuntimeIdentity().runtime_id);
|
|
2034
|
+
if (verification.nativeRemovable !== true || verification.liveNativeRemovable !== true || verification.defaultPreserved !== true || verification.runtimeRestarted !== true) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_VERIFY_FAILED', verification });
|
|
2035
|
+
await hooks.finalizeVerified();
|
|
2036
|
+
if (providerMigrationOwners.get(providerId) === plan.plan_id) {
|
|
2037
|
+
providerMigrationOwners.delete(providerId);
|
|
2038
|
+
hub.endProviderMutation(providerId);
|
|
2039
|
+
}
|
|
2040
|
+
if (providerStoreMutationOwners.get(plan.plan_id) === providerId) {
|
|
2041
|
+
providerStoreMutationOwners.delete(plan.plan_id);
|
|
2042
|
+
hub.endProviderStoreMutation();
|
|
2043
|
+
}
|
|
2044
|
+
return sendJson(res, 200, { ok: true, state: 'VERIFIED', transaction_id: plan.plan_id, provider_id: providerId, verification });
|
|
2045
|
+
} finally {
|
|
2046
|
+
await hooks.release();
|
|
2047
|
+
// Keep the transaction-owned fence on verification failure so
|
|
2048
|
+
// an incomplete migration cannot race a new Worker; the caller
|
|
2049
|
+
// must explicitly retry verification or roll it back.
|
|
2050
|
+
}
|
|
2051
|
+
}
|
|
2052
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'rollback-migration') {
|
|
2053
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_MIGRATION_ROLLBACK_CONFIRM_REQUIRED' });
|
|
2054
|
+
const hooks = createProviderLayerMigrationFileHooks({
|
|
2055
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
2056
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
2057
|
+
backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
|
|
2058
|
+
existingMigrationId: body?.transaction_id,
|
|
2059
|
+
});
|
|
2060
|
+
let acquiredMigrationFence = false;
|
|
2061
|
+
let retainProviderMutation = false;
|
|
2062
|
+
let acquiredStoreFence = false;
|
|
2063
|
+
try {
|
|
2064
|
+
await hooks.acquireLock();
|
|
2065
|
+
await hooks.backup({});
|
|
2066
|
+
const plan = hooks.backupPlan();
|
|
2067
|
+
if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_PLAN_PROVIDER_MISMATCH' });
|
|
2068
|
+
const owner = providerMigrationOwners.get(providerId);
|
|
2069
|
+
if (providerStoreMutationOwners.get(plan.plan_id) !== providerId || !hub.hasProviderStoreMutation()) {
|
|
2070
|
+
if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
2071
|
+
providerStoreMutationOwners.set(plan.plan_id, providerId);
|
|
2072
|
+
acquiredStoreFence = true;
|
|
2073
|
+
}
|
|
2074
|
+
if (owner !== plan.plan_id) {
|
|
2075
|
+
if (hub.hasProviderLease(providerId) || !hub.beginProviderMutation(providerId)) {
|
|
2076
|
+
if (acquiredStoreFence && providerStoreMutationOwners.get(plan.plan_id) === providerId) { providerStoreMutationOwners.delete(plan.plan_id); hub.endProviderStoreMutation(); }
|
|
2077
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
2078
|
+
}
|
|
2079
|
+
providerMigrationOwners.set(providerId, plan.plan_id);
|
|
2080
|
+
acquiredMigrationFence = true;
|
|
2081
|
+
retainProviderMutation = true;
|
|
2082
|
+
}
|
|
2083
|
+
const rollback = await hooks.rollback();
|
|
2084
|
+
await hooks.setRollbackRuntimeBaseline(getHubRuntimeIdentity().runtime_id);
|
|
2085
|
+
providerMigrationOwners.set(providerId, plan.plan_id);
|
|
2086
|
+
retainProviderMutation = true;
|
|
2087
|
+
return sendJson(res, 202, { ok: true, restart_required: true, transaction_id: plan.plan_id, provider_id: providerId, state: 'ROLLBACK_RESTART_PENDING' });
|
|
2088
|
+
} finally {
|
|
2089
|
+
await hooks.release();
|
|
2090
|
+
if (acquiredMigrationFence && !retainProviderMutation && providerMigrationOwners.get(providerId) === body?.transaction_id) {
|
|
2091
|
+
providerMigrationOwners.delete(providerId);
|
|
2092
|
+
hub.endProviderMutation(providerId);
|
|
2093
|
+
}
|
|
2094
|
+
}
|
|
2095
|
+
}
|
|
2096
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'verify-rollback-migration') {
|
|
2097
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_MIGRATION_ROLLBACK_VERIFY_CONFIRM_REQUIRED' });
|
|
2098
|
+
const transactionId = body?.transaction_id;
|
|
2099
|
+
const hooks = createProviderLayerMigrationFileHooks({
|
|
2100
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
2101
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
2102
|
+
backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
|
|
2103
|
+
existingMigrationId: transactionId,
|
|
2104
|
+
});
|
|
2105
|
+
let acquiredStoreFence = false;
|
|
2106
|
+
try {
|
|
2107
|
+
await hooks.acquireLock();
|
|
2108
|
+
await hooks.backup({});
|
|
2109
|
+
const plan = hooks.backupPlan();
|
|
2110
|
+
if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_PLAN_PROVIDER_MISMATCH' });
|
|
2111
|
+
const owner = providerMigrationOwners.get(providerId);
|
|
2112
|
+
if (providerStoreMutationOwners.get(plan.plan_id) !== providerId || !hub.hasProviderStoreMutation()) {
|
|
2113
|
+
if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
2114
|
+
providerStoreMutationOwners.set(plan.plan_id, providerId);
|
|
2115
|
+
acquiredStoreFence = true;
|
|
2116
|
+
}
|
|
2117
|
+
if (owner !== plan.plan_id) {
|
|
2118
|
+
if (hub.hasProviderLease(providerId) || !hub.beginProviderMutation(providerId)) {
|
|
2119
|
+
if (acquiredStoreFence && providerStoreMutationOwners.get(plan.plan_id) === providerId) { providerStoreMutationOwners.delete(plan.plan_id); hub.endProviderStoreMutation(); }
|
|
2120
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
2121
|
+
}
|
|
2122
|
+
providerMigrationOwners.set(providerId, plan.plan_id);
|
|
2123
|
+
}
|
|
2124
|
+
const verification = await hooks.verifyRollback(plan);
|
|
2125
|
+
const live = await readProviderInventorySnapshot(hub, ctx, config);
|
|
2126
|
+
verification.catalogEvidence = live.catalog_evidence;
|
|
2127
|
+
verification.defaultEvidence = live.default_evidence;
|
|
2128
|
+
verification.defaultPreserved = live.default_evidence?.ok === true
|
|
2129
|
+
&& JSON.stringify(live.harness_default ?? null) === JSON.stringify(plan.harness_default_before ?? null);
|
|
2130
|
+
verification.liveRestored = hasCompleteProviderCatalogEvidence(live.catalog_evidence)
|
|
2131
|
+
&& live.records.some((record) => record.id === providerId && record.lifecycle?.catalogued === true && record.native_removable === false);
|
|
2132
|
+
verification.runtimeRestarted = hasProviderRuntimeRestartEvidence(plan.rollback_runtime_id_before, getHubRuntimeIdentity().runtime_id);
|
|
2133
|
+
if (verification.ok !== true || verification.liveRestored !== true || verification.defaultPreserved !== true || verification.runtimeRestarted !== true) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_ROLLBACK_VERIFY_FAILED', verification });
|
|
2134
|
+
await hooks.finalizeRollback();
|
|
2135
|
+
if (providerMigrationOwners.get(providerId) === plan.plan_id) {
|
|
2136
|
+
providerMigrationOwners.delete(providerId);
|
|
2137
|
+
hub.endProviderMutation(providerId);
|
|
2138
|
+
}
|
|
2139
|
+
if (providerStoreMutationOwners.get(plan.plan_id) === providerId) {
|
|
2140
|
+
providerStoreMutationOwners.delete(plan.plan_id);
|
|
2141
|
+
hub.endProviderStoreMutation();
|
|
2142
|
+
}
|
|
2143
|
+
return sendJson(res, 200, { ok: true, state: 'ROLLED_BACK', transaction_id: plan.plan_id, provider_id: providerId, verification });
|
|
2144
|
+
} finally { await hooks.release(); }
|
|
2145
|
+
}
|
|
2146
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'delete-plan') {
|
|
2147
|
+
if (providerId !== 'deepseek-official' && hasPendingProviderRecoveryTransactions(inventory)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
2148
|
+
if (providerId !== 'deepseek-official' && !hasCompleteProviderCatalogEvidence(inventory.catalog_evidence)) return sendJson(res, 503, { ok: false, code: 'MODEL_CATALOG_UNAVAILABLE' });
|
|
2149
|
+
if (providerId !== 'deepseek-official' && !hasCompleteProviderDeclarationEvidence(inventory.declaration_evidence)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_SOURCE_UNRESOLVED' });
|
|
2150
|
+
if (providerId !== 'deepseek-official' && !hasAvailableProviderLifecycleEvidence(inventory.lifecycle_evidence)) return sendJson(res, 503, { ok: false, code: 'PROVIDER_LIFECYCLE_UNAVAILABLE' });
|
|
2151
|
+
if (providerId !== 'deepseek-official' && inventory.default_evidence?.ok !== true) return sendJson(res, 409, { ok: false, code: inventory.default_evidence?.code ?? 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' });
|
|
2152
|
+
const revisions = readProviderSourceRevisions();
|
|
2153
|
+
if (!revisions.profile.ok && !revisions.settings.ok) return sendJson(res, 409, { ok: false, code: revisions.profile.code ?? revisions.settings.code });
|
|
2154
|
+
const expectedRevision = (revisions.profile.ok ? revisions.profile.revision : null) ?? revisions.settings.revision;
|
|
2155
|
+
const planned = planProviderDelete({
|
|
2156
|
+
providerId,
|
|
2157
|
+
inventory,
|
|
2158
|
+
activeJobs: hub.list(),
|
|
2159
|
+
replacementDefault: body?.replacement_default,
|
|
2160
|
+
expectedRevision,
|
|
2161
|
+
expectedRevisions: {
|
|
2162
|
+
profile: revisions.profile.ok ? revisions.profile.revision : null,
|
|
2163
|
+
settings: revisions.settings.ok ? revisions.settings.revision : null,
|
|
2164
|
+
},
|
|
2165
|
+
});
|
|
2166
|
+
if (!planned.ok) return sendJson(res, 409, { ok: false, code: planned.code });
|
|
2167
|
+
const plan = { ...planned.plan, runtime_id_before: getHubRuntimeIdentity().runtime_id };
|
|
2168
|
+
rememberProviderDeletePlan(plan);
|
|
2169
|
+
return sendJson(res, 200, { ok: true, profile_revision: expectedRevision, plan });
|
|
2170
|
+
}
|
|
2171
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'probe') {
|
|
2172
|
+
const record = inventory.records.find((entry) => entry.id === providerId);
|
|
2173
|
+
if (!record) return sendJson(res, 404, { ok: false, code: 'PROVIDER_NOT_FOUND' });
|
|
2174
|
+
if (record.desired_state === 'absent') return sendJson(res, 409, { ok: false, code: 'PROVIDER_TOMBSTONED' });
|
|
2175
|
+
const model = selectProviderProbeModel({ providerId, record, config });
|
|
2176
|
+
if (!model) return sendJson(res, 409, { ok: false, code: 'PROVIDER_PROBE_MODEL_UNAVAILABLE' });
|
|
2177
|
+
let explicitProviderProbe;
|
|
2178
|
+
try { explicitProviderProbe = ctx?.providerProbe; } catch { explicitProviderProbe = undefined; }
|
|
2179
|
+
const providerProbe = typeof explicitProviderProbe === 'function' ? explicitProviderProbe : builtInProviderProbe;
|
|
2180
|
+
if (typeof providerProbe !== 'function') return sendJson(res, 503, { ok: false, code: 'PROVIDER_PROBE_UNAVAILABLE' });
|
|
2181
|
+
let observed;
|
|
2182
|
+
let timer;
|
|
2183
|
+
try {
|
|
2184
|
+
const timeout = new Promise((resolve) => { timer = setTimeout(() => resolve({ error: { name: 'TimeoutError' } }), 15_000); });
|
|
2185
|
+
observed = await Promise.race([
|
|
2186
|
+
Promise.resolve().then(() => providerProbe({ provider: providerId, model, signal: AbortSignal.timeout(15_000) })),
|
|
2187
|
+
timeout,
|
|
2188
|
+
]);
|
|
2189
|
+
} catch (error) {
|
|
2190
|
+
// Keep probe transport failures as bounded health evidence so a
|
|
2191
|
+
// credential/quota/timeout result is visible to routing policy;
|
|
2192
|
+
// never copy the raw provider error into the response.
|
|
2193
|
+
observed = { error };
|
|
2194
|
+
} finally {
|
|
2195
|
+
clearTimeout(timer);
|
|
2196
|
+
}
|
|
2197
|
+
const health = hub.healthStore.record(providerId, model, observed?.ok === true ? { ok: true } : { error: observed?.error ?? observed });
|
|
2198
|
+
return sendJson(res, 200, { ok: true, provider_id: providerId, model, health });
|
|
2199
|
+
}
|
|
2200
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'verify-delete') {
|
|
2201
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_DELETE_VERIFY_CONFIRM_REQUIRED' });
|
|
2202
|
+
const { transaction_id: transactionId } = body ?? {};
|
|
2203
|
+
const hooks = createProviderDeleteFileHooks({
|
|
2204
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
2205
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
2206
|
+
configFile: join(CONFIG_DIR, 'config.json'),
|
|
2207
|
+
lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
2208
|
+
backupDir: join(CONFIG_DIR, 'provider-backups'),
|
|
2209
|
+
sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
|
|
2210
|
+
existingBackupId: transactionId,
|
|
2211
|
+
expectedProviderId: providerId,
|
|
2212
|
+
});
|
|
2213
|
+
const plan = hooks.backupPlan();
|
|
2214
|
+
if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_PLAN_PROVIDER_MISMATCH' });
|
|
2215
|
+
await hooks.acquireLock();
|
|
2216
|
+
try {
|
|
2217
|
+
if (providerStoreMutationOwners.get(transactionId) !== providerId || !hub.hasProviderStoreMutation()) {
|
|
2218
|
+
if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_BUSY' });
|
|
2219
|
+
providerStoreMutationOwners.set(transactionId, providerId);
|
|
2220
|
+
}
|
|
2221
|
+
const verification = await hooks.verify(plan);
|
|
2222
|
+
const live = await readProviderInventorySnapshot(hub, ctx, config);
|
|
2223
|
+
verification.catalog_evidence = live.catalog_evidence;
|
|
2224
|
+
verification.default_evidence = live.default_evidence;
|
|
2225
|
+
verification.catalogAbsent = hasCompleteProviderCatalogEvidence(live.catalog_evidence)
|
|
2226
|
+
&& !live.records.some((entry) => entry.id === providerId && entry.lifecycle?.catalogued === true);
|
|
2227
|
+
verification.harnessDefault = plan.was_harness_default !== true || (
|
|
2228
|
+
live.harness_default?.provider === plan.replacement_default
|
|
2229
|
+
&& live.harness_default?.model === plan.replacement_default_model
|
|
2230
|
+
);
|
|
2231
|
+
verification.runtimeRestarted = hasProviderRuntimeRestartEvidence(plan.delete_runtime_id_before_restart, getHubRuntimeIdentity().runtime_id);
|
|
2232
|
+
if (verification?.providerAbsent !== true || !hasCompleteProviderCatalogEvidence(verification?.catalog_evidence) || verification?.default_evidence?.ok !== true || verification?.catalogAbsent !== true || verification?.harnessDefault !== true || verification?.runtimeRestarted !== true || verification?.routingClear !== true || verification?.tombstonePresent !== true) {
|
|
2233
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_VERIFY_FAILED', verification });
|
|
2234
|
+
}
|
|
2235
|
+
await hooks.recordTransaction({ transaction_id: transactionId, provider_id: providerId, state: 'VERIFIED' }, plan);
|
|
2236
|
+
hub.endProviderMutation(providerId);
|
|
2237
|
+
if (providerStoreMutationOwners.get(transactionId) === providerId) {
|
|
2238
|
+
providerStoreMutationOwners.delete(transactionId);
|
|
2239
|
+
hub.endProviderStoreMutation();
|
|
2240
|
+
}
|
|
2241
|
+
return sendJson(res, 200, { ok: true, transaction_id: transactionId, provider_id: providerId, state: 'VERIFIED', verification });
|
|
2242
|
+
} finally {
|
|
2243
|
+
await hooks.release();
|
|
2244
|
+
}
|
|
2245
|
+
}
|
|
2246
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'rollback') {
|
|
2247
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_ROLLBACK_CONFIRM_REQUIRED' });
|
|
2248
|
+
const { transaction_id: transactionId } = body ?? {};
|
|
2249
|
+
const hooks = createProviderDeleteFileHooks({
|
|
2250
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
2251
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
2252
|
+
configFile: join(CONFIG_DIR, 'config.json'),
|
|
2253
|
+
lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
2254
|
+
backupDir: join(CONFIG_DIR, 'provider-backups'),
|
|
2255
|
+
sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
|
|
2256
|
+
existingBackupId: transactionId,
|
|
2257
|
+
expectedProviderId: providerId,
|
|
2258
|
+
});
|
|
2259
|
+
await hooks.acquireLock();
|
|
2260
|
+
let acquiredStoreMutation = false;
|
|
2261
|
+
let retainStoreMutation = false;
|
|
2262
|
+
try {
|
|
2263
|
+
const plan = hooks.backupPlan();
|
|
2264
|
+
if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_PLAN_PROVIDER_MISMATCH' });
|
|
2265
|
+
if (providerStoreMutationOwners.get(transactionId) !== providerId || !hub.hasProviderStoreMutation()) {
|
|
2266
|
+
if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_BUSY' });
|
|
2267
|
+
providerStoreMutationOwners.set(transactionId, providerId);
|
|
2268
|
+
acquiredStoreMutation = true;
|
|
2269
|
+
}
|
|
2270
|
+
await hooks.rollback(plan);
|
|
2271
|
+
await hooks.setRuntimeBaseline(getHubRuntimeIdentity().runtime_id, 'rollback');
|
|
2272
|
+
const rollbackPlan = hooks.backupPlan() ?? plan;
|
|
2273
|
+
try {
|
|
2274
|
+
await hooks.recordTransaction({ transaction_id: transactionId, provider_id: providerId, state: 'ROLLBACK_PENDING' }, rollbackPlan);
|
|
2275
|
+
} catch (error) {
|
|
2276
|
+
return sendJson(res, 409, { ok: false, code: boundedMachineCodeFromError(error) ?? 'PROVIDER_LIFECYCLE_RECORD_FAILED' });
|
|
2277
|
+
}
|
|
2278
|
+
retainStoreMutation = true;
|
|
2279
|
+
return sendJson(res, 202, { ok: true, restart_required: true, transaction_id: transactionId, provider_id: providerId, state: 'ROLLBACK_PENDING' });
|
|
2280
|
+
} finally {
|
|
2281
|
+
await hooks.release();
|
|
2282
|
+
if (acquiredStoreMutation && !retainStoreMutation && providerStoreMutationOwners.get(transactionId) === providerId) {
|
|
2283
|
+
providerStoreMutationOwners.delete(transactionId);
|
|
2284
|
+
hub.endProviderStoreMutation();
|
|
2285
|
+
}
|
|
2286
|
+
}
|
|
2287
|
+
}
|
|
2288
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'verify-rollback') {
|
|
2289
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_ROLLBACK_VERIFY_CONFIRM_REQUIRED' });
|
|
2290
|
+
const { transaction_id: transactionId } = body ?? {};
|
|
2291
|
+
const hooks = createProviderDeleteFileHooks({
|
|
2292
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
2293
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
2294
|
+
configFile: join(CONFIG_DIR, 'config.json'),
|
|
2295
|
+
lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
2296
|
+
backupDir: join(CONFIG_DIR, 'provider-backups'),
|
|
2297
|
+
sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
|
|
2298
|
+
existingBackupId: transactionId,
|
|
2299
|
+
expectedProviderId: providerId,
|
|
2300
|
+
});
|
|
2301
|
+
const plan = hooks.backupPlan();
|
|
2302
|
+
if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_PLAN_PROVIDER_MISMATCH' });
|
|
2303
|
+
await hooks.acquireLock();
|
|
2304
|
+
try {
|
|
2305
|
+
if (providerStoreMutationOwners.get(transactionId) !== providerId || !hub.hasProviderStoreMutation()) {
|
|
2306
|
+
if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_BUSY' });
|
|
2307
|
+
providerStoreMutationOwners.set(transactionId, providerId);
|
|
2308
|
+
}
|
|
2309
|
+
const verified = await hooks.verifyRollback(plan);
|
|
2310
|
+
const live = await readProviderInventorySnapshot(hub, ctx, config);
|
|
2311
|
+
verified.catalog_evidence = live.catalog_evidence;
|
|
2312
|
+
verified.default_evidence = live.default_evidence;
|
|
2313
|
+
verified.catalogPresent = hasCompleteProviderCatalogEvidence(live.catalog_evidence)
|
|
2314
|
+
&& live.records.some((entry) => entry.id === providerId && entry.lifecycle?.catalogued === true);
|
|
2315
|
+
verified.harnessDefaultRestored = plan.was_harness_default !== true || (
|
|
2316
|
+
live.harness_default?.provider === plan.harness_default_before?.provider
|
|
2317
|
+
&& live.harness_default?.model === plan.harness_default_before?.model
|
|
2318
|
+
);
|
|
2319
|
+
verified.runtimeRestarted = hasProviderRuntimeRestartEvidence(plan.rollback_runtime_id_before_restart, getHubRuntimeIdentity().runtime_id);
|
|
2320
|
+
if (verified?.ok !== true || !hasCompleteProviderCatalogEvidence(verified?.catalog_evidence) || verified?.default_evidence?.ok !== true || verified?.catalogPresent !== true || verified?.harnessDefaultRestored !== true || verified?.runtimeRestarted !== true) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_ROLLBACK_VERIFY_FAILED', verification: verified });
|
|
2321
|
+
await hooks.recordTransaction({ transaction_id: transactionId, provider_id: providerId, state: 'ROLLED_BACK' }, plan);
|
|
2322
|
+
hub.endProviderMutation(providerId);
|
|
2323
|
+
if (providerStoreMutationOwners.get(transactionId) === providerId) {
|
|
2324
|
+
providerStoreMutationOwners.delete(transactionId);
|
|
2325
|
+
hub.endProviderStoreMutation();
|
|
2326
|
+
}
|
|
2327
|
+
return sendJson(res, 200, { ok: true, transaction_id: transactionId, provider_id: providerId, state: 'ROLLED_BACK' });
|
|
2328
|
+
} finally {
|
|
2329
|
+
await hooks.release();
|
|
2330
|
+
}
|
|
2331
|
+
}
|
|
2332
|
+
if (req.method === 'DELETE' && parts.length === 1) {
|
|
2333
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_DELETE_CONFIRM_REQUIRED' });
|
|
2334
|
+
if (body?.purge_orphan_credentials === true) {
|
|
2335
|
+
return sendJson(res, 400, { ok: false, code: 'CREDENTIAL_PURGE_REQUIRES_EXPLICIT_CONFIRMATION' });
|
|
2336
|
+
}
|
|
2337
|
+
const stored = providerDeletePlans.get(body?.plan_id);
|
|
2338
|
+
if (!stored || Date.now() - stored.created_at > PROVIDER_DELETE_PLAN_TTL_MS || stored.plan.provider_id !== providerId) {
|
|
2339
|
+
providerDeletePlans.delete(body?.plan_id);
|
|
2340
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_PLAN_EXPIRED' });
|
|
2341
|
+
}
|
|
2342
|
+
const revisions = readProviderSourceRevisions();
|
|
2343
|
+
if (!revisions.profile.ok && !revisions.settings.ok) return sendJson(res, 409, { ok: false, code: revisions.profile.code ?? revisions.settings.code });
|
|
2344
|
+
if (providerId !== 'deepseek-official' && !hasCompleteProviderCatalogEvidence(inventory.catalog_evidence)) return sendJson(res, 503, { ok: false, code: 'MODEL_CATALOG_UNAVAILABLE' });
|
|
2345
|
+
if (providerId !== 'deepseek-official' && !hasCompleteProviderDeclarationEvidence(inventory.declaration_evidence)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_SOURCE_UNRESOLVED' });
|
|
2346
|
+
if (providerId !== 'deepseek-official' && !hasAvailableProviderLifecycleEvidence(inventory.lifecycle_evidence)) return sendJson(res, 503, { ok: false, code: 'PROVIDER_LIFECYCLE_UNAVAILABLE' });
|
|
2347
|
+
if (providerId !== 'deepseek-official' && inventory.default_evidence?.ok !== true) return sendJson(res, 409, { ok: false, code: inventory.default_evidence?.code ?? 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' });
|
|
2348
|
+
const storedExpected = stored.plan.expected_revisions;
|
|
2349
|
+
const revisionsMatch = storedExpected
|
|
2350
|
+
? (storedExpected.profile === (revisions.profile.ok ? revisions.profile.revision : null)
|
|
2351
|
+
&& storedExpected.settings === (revisions.settings.ok ? revisions.settings.revision : null))
|
|
2352
|
+
: (body?.expected_revision === stored.plan.expected_revision && (revisions.profile.revision ?? revisions.settings.revision) === stored.plan.expected_revision);
|
|
2353
|
+
if (!revisionsMatch || body?.expected_revision !== stored.plan.expected_revision) {
|
|
2354
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_PROFILE_CHANGED' });
|
|
2355
|
+
}
|
|
2356
|
+
if (providerId !== 'deepseek-official' && hasPendingProviderRecoveryTransactions(inventory)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
2357
|
+
const refreshed = planProviderDelete({
|
|
2358
|
+
providerId,
|
|
2359
|
+
inventory,
|
|
2360
|
+
activeJobs: hub.list(),
|
|
2361
|
+
replacementDefault: stored.plan.replacement_default,
|
|
2362
|
+
expectedRevision: stored.plan.expected_revision,
|
|
2363
|
+
expectedRevisions: stored.plan.expected_revisions,
|
|
2364
|
+
});
|
|
2365
|
+
if (!refreshed.ok) return sendJson(res, 409, { ok: false, code: refreshed.code });
|
|
2366
|
+
if (refreshed.plan.replacement_default_model !== stored.plan.replacement_default_model) {
|
|
2367
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_PLAN_CHANGED' });
|
|
2368
|
+
}
|
|
2369
|
+
let executionPlan = {
|
|
2370
|
+
...refreshed.plan,
|
|
2371
|
+
plan_id: stored.plan.plan_id,
|
|
2372
|
+
expected_revision: stored.plan.expected_revision,
|
|
2373
|
+
expected_revisions: stored.plan.expected_revisions,
|
|
2374
|
+
runtime_id_before: stored.plan.runtime_id_before,
|
|
2375
|
+
};
|
|
2376
|
+
if (!hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_BUSY' });
|
|
2377
|
+
if (!hub.beginProviderMutation(providerId)) {
|
|
2378
|
+
hub.endProviderStoreMutation();
|
|
2379
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_BUSY' });
|
|
2380
|
+
}
|
|
2381
|
+
let hooks = null;
|
|
2382
|
+
let lockAcquired = false;
|
|
2383
|
+
let retainProviderMutation = false;
|
|
2384
|
+
let retainProviderStoreMutation = false;
|
|
2385
|
+
try {
|
|
2386
|
+
hooks = createProviderDeleteFileHooks({
|
|
2387
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
2388
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
2389
|
+
configFile: join(CONFIG_DIR, 'config.json'),
|
|
2390
|
+
lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
2391
|
+
backupDir: join(CONFIG_DIR, 'provider-backups'),
|
|
2392
|
+
sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
|
|
2393
|
+
runtimeIdProvider: () => getHubRuntimeIdentity().runtime_id,
|
|
2394
|
+
afterLockAcquired: () => {
|
|
2395
|
+
const pendingRecovery = [
|
|
2396
|
+
...readProviderRecoveryTransactions(),
|
|
2397
|
+
...readProviderLayerMigrationTransactions(join(CONFIG_DIR, 'provider-migration-backups')),
|
|
2398
|
+
];
|
|
2399
|
+
if (pendingRecovery.length > 0) {
|
|
2400
|
+
throw Object.assign(new Error('provider recovery transaction is pending'), { code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
2401
|
+
}
|
|
2402
|
+
if (hub.hasProviderLease(providerId)) {
|
|
2403
|
+
throw Object.assign(new Error('provider is in use'), { code: 'PROVIDER_IN_USE' });
|
|
2404
|
+
}
|
|
2405
|
+
},
|
|
2406
|
+
});
|
|
2407
|
+
// Hold the real shared store lock while rebuilding the live
|
|
2408
|
+
// inventory and delete plan. The pre-lock plan is only a
|
|
2409
|
+
// proposal; catalog/default/ownership/recovery facts must be
|
|
2410
|
+
// authoritative at the mutation boundary.
|
|
2411
|
+
await hooks.acquireLock();
|
|
2412
|
+
lockAcquired = true;
|
|
2413
|
+
const lockedConfig = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
2414
|
+
const lockedInventory = await readProviderInventorySnapshot(hub, ctx, lockedConfig);
|
|
2415
|
+
const lockedRevisions = readProviderSourceRevisions();
|
|
2416
|
+
const lockedExpected = stored.plan.expected_revisions;
|
|
2417
|
+
const lockedRevisionsMatch = lockedExpected
|
|
2418
|
+
? lockedExpected.profile === (lockedRevisions.profile.ok ? lockedRevisions.profile.revision : null)
|
|
2419
|
+
&& lockedExpected.settings === (lockedRevisions.settings.ok ? lockedRevisions.settings.revision : null)
|
|
2420
|
+
: false;
|
|
2421
|
+
if (providerId !== 'deepseek-official' && !hasCompleteProviderCatalogEvidence(lockedInventory.catalog_evidence)) throw Object.assign(new Error('provider catalog unavailable after lock'), { code: 'MODEL_CATALOG_UNAVAILABLE' });
|
|
2422
|
+
if (providerId !== 'deepseek-official' && !hasCompleteProviderDeclarationEvidence(lockedInventory.declaration_evidence)) throw Object.assign(new Error('provider declarations unavailable after lock'), { code: 'PROVIDER_DELETE_SOURCE_UNRESOLVED' });
|
|
2423
|
+
if (providerId !== 'deepseek-official' && !hasAvailableProviderLifecycleEvidence(lockedInventory.lifecycle_evidence)) throw Object.assign(new Error('provider lifecycle unavailable after lock'), { code: 'PROVIDER_LIFECYCLE_UNAVAILABLE' });
|
|
2424
|
+
if (providerId !== 'deepseek-official' && lockedInventory.default_evidence?.ok !== true) throw Object.assign(new Error('provider default authority changed after lock'), { code: lockedInventory.default_evidence?.code ?? 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' });
|
|
2425
|
+
if (providerId !== 'deepseek-official' && hasPendingProviderRecoveryTransactions(lockedInventory)) throw Object.assign(new Error('provider recovery transaction is pending'), { code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
2426
|
+
if (!lockedRevisionsMatch) throw Object.assign(new Error('provider source changed after lock'), { code: 'PROVIDER_PROFILE_CHANGED' });
|
|
2427
|
+
if (hub.hasProviderLease(providerId)) throw Object.assign(new Error('provider is in use'), { code: 'PROVIDER_IN_USE' });
|
|
2428
|
+
const lockedPlan = planProviderDelete({
|
|
2429
|
+
providerId,
|
|
2430
|
+
inventory: lockedInventory,
|
|
2431
|
+
activeJobs: hub.list(),
|
|
2432
|
+
replacementDefault: stored.plan.replacement_default,
|
|
2433
|
+
expectedRevision: stored.plan.expected_revision,
|
|
2434
|
+
expectedRevisions: stored.plan.expected_revisions,
|
|
2435
|
+
});
|
|
2436
|
+
if (!lockedPlan.ok) throw Object.assign(new Error('provider delete plan changed after lock'), { code: lockedPlan.code });
|
|
2437
|
+
const sameLockedPlan = lockedPlan.plan.replacement_default_model === stored.plan.replacement_default_model
|
|
2438
|
+
&& lockedPlan.plan.was_harness_default === stored.plan.was_harness_default
|
|
2439
|
+
&& JSON.stringify(lockedPlan.plan.declaration_authorities) === JSON.stringify(stored.plan.declaration_authorities)
|
|
2440
|
+
&& JSON.stringify(lockedPlan.plan.harness_default_before ?? null) === JSON.stringify(stored.plan.harness_default_before ?? null)
|
|
2441
|
+
&& JSON.stringify(lockedPlan.plan.credential_refs) === JSON.stringify(stored.plan.credential_refs);
|
|
2442
|
+
if (!sameLockedPlan) throw Object.assign(new Error('provider delete plan semantics changed after lock'), { code: 'PROVIDER_DELETE_PLAN_CHANGED' });
|
|
2443
|
+
executionPlan = {
|
|
2444
|
+
...lockedPlan.plan,
|
|
2445
|
+
plan_id: stored.plan.plan_id,
|
|
2446
|
+
expected_revision: stored.plan.expected_revision,
|
|
2447
|
+
expected_revisions: stored.plan.expected_revisions,
|
|
2448
|
+
runtime_id_before: stored.plan.runtime_id_before,
|
|
2449
|
+
};
|
|
2450
|
+
providerDeletePlans.delete(body.plan_id);
|
|
2451
|
+
const { executeProviderDelete } = await import('../provider-lifecycle.mjs');
|
|
2452
|
+
const result = await executeProviderDelete(executionPlan, hooks, { deferRestart: true });
|
|
2453
|
+
retainProviderMutation = result.state === 'RESTART_PENDING';
|
|
2454
|
+
if (retainProviderMutation) {
|
|
2455
|
+
providerStoreMutationOwners.set(executionPlan.plan_id, providerId);
|
|
2456
|
+
retainProviderStoreMutation = true;
|
|
2457
|
+
}
|
|
2458
|
+
return sendJson(res, result.state === 'RESTART_PENDING' ? 202 : 409, { ok: result.state === 'RESTART_PENDING', restart_required: true, result });
|
|
2459
|
+
} finally {
|
|
2460
|
+
if (lockAcquired && hooks) await hooks.release().catch(() => {});
|
|
2461
|
+
if (!retainProviderMutation) hub.endProviderMutation(providerId);
|
|
2462
|
+
if (!retainProviderStoreMutation) hub.endProviderStoreMutation();
|
|
2463
|
+
}
|
|
2464
|
+
}
|
|
2465
|
+
return sendJson(res, 404, { ok: false, error: 'unknown providers endpoint' });
|
|
2466
|
+
} catch (err) {
|
|
2467
|
+
const isProbe = req.method === 'POST' && parts.length === 2 && parts[1] === 'probe';
|
|
2468
|
+
const code = boundedMachineCodeFromError(err) ?? (isProbe ? 'PROVIDER_PROBE_FAILED' : 'PROVIDER_PLAN_INVALID');
|
|
2469
|
+
const status = code === 'PROVIDER_DELETE_RESTART_SUPERVISOR_UNAVAILABLE' || code.startsWith('PROVIDER_PROBE') ? 503
|
|
2470
|
+
: code.startsWith('PROVIDER_') || code.startsWith('CREDENTIAL_') ? 409 : 400;
|
|
2471
|
+
return sendJson(res, status, { ok: false, code });
|
|
2472
|
+
}
|
|
2473
|
+
},
|
|
2474
|
+
}));
|
|
2475
|
+
|
|
2476
|
+
disposers.push(webServer.register({
|
|
2477
|
+
kind: 'exact', path: `${ROUTE_BASE}/provider-health`,
|
|
2478
|
+
handler: async (req, res) => {
|
|
2479
|
+
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
2480
|
+
if (req.method !== 'GET') return sendJson(res, 405, { ok: false, error: 'GET only' }, { allow: 'GET' });
|
|
2481
|
+
return sendJson(res, 200, {
|
|
2482
|
+
ok: true,
|
|
2483
|
+
schema_version: 1,
|
|
2484
|
+
health: hub.healthStore.list(),
|
|
2485
|
+
runtime: getHubRuntimeIdentity(),
|
|
2486
|
+
});
|
|
2487
|
+
},
|
|
2488
|
+
}));
|
|
2489
|
+
|
|
2490
|
+
disposers.push(webServer.register({
|
|
2491
|
+
kind: 'exact', path: `${ROUTE_BASE}/models`,
|
|
1906
2492
|
handler: async (req, res) => {
|
|
1907
2493
|
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
1908
2494
|
if (req.method !== 'GET' && req.method !== 'POST') return sendJson(res, 405, { ok: false, error: 'GET or POST' }, { allow: 'GET, POST' });
|