@ran-sh/dsh-crew 1.0.3 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/codex/AGENTS.md +101 -92
- package/docs/ui-surfaces.md +48 -21
- package/lib/client.js +68 -44
- package/official-web-bridge/lib/client.js +368 -4709
- package/official-web-bridge/package.json +1 -2
- package/package.json +130 -67
- package/scripts/build-client.mjs +30 -15
- package/scripts/remove-legacy-official-bridge.ps1 +89 -0
- package/scripts/setup.mjs +152 -140
- package/scripts/verify-npm-install.mjs +311 -310
- package/scripts/verify-official-bridge-e2e.mjs +12 -1
- package/src/client/host-readiness.mjs +62 -59
- package/src/client/index.tsx +35 -30
- package/src/client/quick-entry.tsx +10 -0
- package/src/client/quick-panel.tsx +295 -0
- package/src/client/surface-detection.mjs +43 -30
- package/src/dsh-cli-runtime.mjs +500 -40
- package/src/dsh-cohort.mjs +20 -0
- package/src/hub/index.mjs +1602 -1485
- package/src/install/npx-lifecycle.mjs +1568 -111
- package/src/install/official-web.mjs +36 -73
- package/src/jobs.mjs +20 -24
- package/src/official-web-bridge.mjs +329 -249
- package/src/runtime-identity.mjs +23 -1
- package/src/supervisor/restart-request.mjs +202 -0
- package/windows/start-dsh-crew.ps1 +783 -370
- package/worker.cordis.yml +32 -46
- package/zcode/AGENTS.md +35 -26
package/src/hub/index.mjs
CHANGED
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
// Web UI session list), exposes a loopback jobs API for the CC/Codex MCP shim,
|
|
4
4
|
// and serves the one-click installer endpoints for the settings page.
|
|
5
5
|
|
|
6
|
-
import { createHash, randomUUID } from 'node:crypto';
|
|
7
|
-
import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
|
|
8
|
-
import { dirname, join, isAbsolute, resolve as resolvePath } from 'node:path';
|
|
6
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
7
|
+
import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
|
|
8
|
+
import { dirname, join, isAbsolute, resolve as resolvePath } from 'node:path';
|
|
9
9
|
import { homedir } from 'node:os';
|
|
10
10
|
import { createShardWriter, readMergedStatus } from '../status-shard.mjs';
|
|
11
11
|
import {
|
|
@@ -30,23 +30,23 @@ import { addContextReferences, buildWorkspaceTask, isSafeBranchName, loadWorkspa
|
|
|
30
30
|
import { buildExtensionContract } from '../extension-contract.mjs';
|
|
31
31
|
import { cleanupIsolatedWorkspace, createIsolatedWorkspace } from '../workspace-isolation.mjs';
|
|
32
32
|
import { assessWorkspaceReadiness } from '../workspace-readiness.mjs';
|
|
33
|
-
import { buildHubExecutionRows } from '../config-readiness.mjs';
|
|
34
|
-
import { localRequestCore, originLoopback } from '../local-request-guard.mjs';
|
|
35
|
-
import { raceWaiters } from '../removable-waiter.mjs';
|
|
36
|
-
import { buildProviderInventory } from '../provider-inventory.mjs';
|
|
37
|
-
import { buildProviderLayerMigrationPlan, executeProviderLayerMigration } from '../provider-layer-migration.mjs';
|
|
38
|
-
import { inspectProviderProfile, readProviderDeclarations, readProviderMaterialization } from '../provider-profile-store.mjs';
|
|
39
|
-
import { inspectProviderSettings, readHarnessDefault, readProviderSettingsDeclarations, readProviderSettingsMaterialization } from '../provider-settings-store.mjs';
|
|
40
|
-
import { normalizeProviderLifecycleState } from '../provider-lifecycle-state.mjs';
|
|
41
|
-
import { createProviderHealthStore } from '../provider-health.mjs';
|
|
42
|
-
import { planProviderDelete } from '../provider-lifecycle.mjs';
|
|
43
|
-
import { createProviderDeleteFileHooks, isRecoverableProviderDeleteBackup, readProviderDeleteManifestFile } from '../provider-delete-adapters.mjs';
|
|
44
|
-
import { buildCredentialReferenceInventory } from '../credential-reference-inventory.mjs';
|
|
45
|
-
import { executeCredentialPurge, planCredentialPurge } from '../credential-lifecycle.mjs';
|
|
46
|
-
import { createCredentialPurgeFileHooks } from '../credential-purge-adapters.mjs';
|
|
47
|
-
import { normalizeCredentialPurgeState, recordCredentialPurgeOutcome } from '../credential-purge-state.mjs';
|
|
48
|
-
import { buildRuntimeReadinessSnapshot } from '../runtime-readiness-snapshot.mjs';
|
|
49
|
-
import { createProviderLayerMigrationFileHooks, readProviderLayerMigrationTransactions } from '../provider-layer-migration-adapters.mjs';
|
|
33
|
+
import { buildHubExecutionRows } from '../config-readiness.mjs';
|
|
34
|
+
import { localRequestCore, originLoopback } from '../local-request-guard.mjs';
|
|
35
|
+
import { raceWaiters } from '../removable-waiter.mjs';
|
|
36
|
+
import { buildProviderInventory } from '../provider-inventory.mjs';
|
|
37
|
+
import { buildProviderLayerMigrationPlan, executeProviderLayerMigration } from '../provider-layer-migration.mjs';
|
|
38
|
+
import { inspectProviderProfile, readProviderDeclarations, readProviderMaterialization } from '../provider-profile-store.mjs';
|
|
39
|
+
import { inspectProviderSettings, readHarnessDefault, readProviderSettingsDeclarations, readProviderSettingsMaterialization } from '../provider-settings-store.mjs';
|
|
40
|
+
import { normalizeProviderLifecycleState } from '../provider-lifecycle-state.mjs';
|
|
41
|
+
import { createProviderHealthStore } from '../provider-health.mjs';
|
|
42
|
+
import { planProviderDelete } from '../provider-lifecycle.mjs';
|
|
43
|
+
import { createProviderDeleteFileHooks, isRecoverableProviderDeleteBackup, readProviderDeleteManifestFile } from '../provider-delete-adapters.mjs';
|
|
44
|
+
import { buildCredentialReferenceInventory } from '../credential-reference-inventory.mjs';
|
|
45
|
+
import { executeCredentialPurge, planCredentialPurge } from '../credential-lifecycle.mjs';
|
|
46
|
+
import { createCredentialPurgeFileHooks } from '../credential-purge-adapters.mjs';
|
|
47
|
+
import { normalizeCredentialPurgeState, recordCredentialPurgeOutcome } from '../credential-purge-state.mjs';
|
|
48
|
+
import { buildRuntimeReadinessSnapshot } from '../runtime-readiness-snapshot.mjs';
|
|
49
|
+
import { createProviderLayerMigrationFileHooks, readProviderLayerMigrationTransactions } from '../provider-layer-migration-adapters.mjs';
|
|
50
50
|
|
|
51
51
|
// policy.mjs is pure (no @deepseek-ai imports, no ctx access), so importing it
|
|
52
52
|
// here is safe for the profile-realm discipline: it never pulls in package
|
|
@@ -103,339 +103,351 @@ function seedLangFromHost(ctx) {
|
|
|
103
103
|
import { setLang } from '../i18n.mjs';
|
|
104
104
|
|
|
105
105
|
const ROUTE_BASE = '/_dsh/dsh-crew';
|
|
106
|
+
// Quick-config allowlist: the ONLY config keys writable from the official
|
|
107
|
+
// 3080 quick-controls surface. The full /config endpoint stays the single
|
|
108
|
+
// canonical authority; quick-config is a projection, never a second store.
|
|
109
|
+
export const QUICK_CONFIG_KEYS = Object.freeze([
|
|
110
|
+
'subagents_enabled',
|
|
111
|
+
'flash_model_priority',
|
|
112
|
+
'pro_model_priority',
|
|
113
|
+
'vision_enabled',
|
|
114
|
+
'imagegen_enabled',
|
|
115
|
+
'vision_provider',
|
|
116
|
+
'imagegen_provider',
|
|
117
|
+
]);
|
|
106
118
|
const LEGACY_TIER_MODELS = { flash: 'deepseek-v4-flash', pro: 'deepseek-v4-pro' };
|
|
107
119
|
// Local copy (the hub must not import jobs.mjs, which pulls the DSH SDK into
|
|
108
120
|
// the profile realm): a valid dispatch role set.
|
|
109
121
|
const ROLES = { worker: true, reviewer: true };
|
|
110
|
-
const CONFIG_DIR = join(homedir(), '.config', 'dsh-crew');
|
|
111
|
-
const CREDENTIAL_PURGE_STATE_FILE = join(CONFIG_DIR, 'credential-purge-lifecycle.json');
|
|
112
|
-
const RECOVERY_ACTION_PATTERN = /^[a-f0-9]{32}$/u;
|
|
113
|
-
|
|
114
|
-
function recoveryActionId(name) {
|
|
115
|
-
return createHash('sha256').update(String(name)).digest('hex').slice(0, 32);
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
function isRecoveryControlEntry(name) {
|
|
119
|
-
return name === '.quarantine'
|
|
120
|
-
|| name === '.delete.lock'
|
|
121
|
-
|| name === '.delete.reclaim.lock'
|
|
122
|
-
|| name === '.delete.recovery.lock'
|
|
123
|
-
|| name === '.delete.recovery.lock.claim'
|
|
124
|
-
|| /^\.delete\.lock\.[0-9a-f-]+\.(?:active|staging|retired)$/iu.test(name)
|
|
125
|
-
|| /^\.delete\.recovery\.lock\.[0-9a-f-]+\.stale$/iu.test(name);
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
function readProviderLifecycleStateStatus() {
|
|
129
|
-
const file = join(CONFIG_DIR, 'provider-lifecycle.json');
|
|
130
|
-
if (!existsSync(file)) return { ok: true, state: normalizeProviderLifecycleState() };
|
|
131
|
-
try {
|
|
132
|
-
return { ok: true, state: normalizeProviderLifecycleState(JSON.parse(readFileSync(file, 'utf8'))) };
|
|
133
|
-
} catch {
|
|
134
|
-
return { ok: false, code: 'PROVIDER_LIFECYCLE_UNAVAILABLE', state: normalizeProviderLifecycleState() };
|
|
135
|
-
}
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
function readProviderLifecycleState() {
|
|
139
|
-
return readProviderLifecycleStateStatus().state;
|
|
140
|
-
}
|
|
141
|
-
|
|
142
|
-
function readCredentialPurgeState() {
|
|
143
|
-
try { return normalizeCredentialPurgeState(JSON.parse(readFileSync(CREDENTIAL_PURGE_STATE_FILE, 'utf8'))); } catch { return normalizeCredentialPurgeState(); }
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
export function readProviderRecoveryTransactions(root = join(CONFIG_DIR, 'provider-backups')) {
|
|
147
|
-
if (!existsSync(root)) return [];
|
|
148
|
-
try {
|
|
149
|
-
const managedRoot = dirname(resolvePath(root));
|
|
150
|
-
const paths = {
|
|
151
|
-
profile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
152
|
-
settings: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
153
|
-
config: join(CONFIG_DIR, 'config.json'),
|
|
154
|
-
lifecycle: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
155
|
-
};
|
|
156
|
-
const transactions = readdirSync(root, { withFileTypes: true }).filter((entry) => !isRecoveryControlEntry(entry.name)).flatMap((entry) => {
|
|
157
|
-
const transactionRoot = join(root, entry.name);
|
|
158
|
-
const manifestFile = join(root, entry.name, 'manifest.json');
|
|
159
|
-
const actionId = recoveryActionId(entry.name);
|
|
160
|
-
if (!entry.isDirectory()) {
|
|
161
|
-
let updatedAt;
|
|
162
|
-
try { updatedAt = statSync(transactionRoot).mtime.toISOString(); } catch { updatedAt = new Date(0).toISOString(); }
|
|
163
|
-
return [{ storage_id: entry.name, action_id: actionId, transaction_id: null, provider_id: null, phase: 'RECOVERY_UNRESOLVED', updated_at: updatedAt, recoverable: false, unresolved: true }];
|
|
164
|
-
}
|
|
165
|
-
try {
|
|
166
|
-
const manifest = readProviderDeleteManifestFile(manifestFile, managedRoot);
|
|
167
|
-
const transactionId = typeof manifest?.plan?.plan_id === 'string' ? manifest.plan.plan_id : null;
|
|
168
|
-
const providerId = typeof manifest?.provider_id === 'string' ? manifest.provider_id : null;
|
|
169
|
-
const phase = typeof manifest?.phase_journal?.phase === 'string' && manifest.phase_journal.phase.trim()
|
|
170
|
-
? manifest.phase_journal.phase.trim() : 'RECOVERY_INCOMPLETE';
|
|
171
|
-
const updatedAt = typeof manifest?.phase_journal?.updated_at === 'string' && manifest.phase_journal.updated_at.trim()
|
|
172
|
-
? manifest.phase_journal.updated_at : statSync(manifestFile).mtime.toISOString();
|
|
173
|
-
const hasPendingMutation = manifest?.mutation_journal && typeof manifest.mutation_journal === 'object'
|
|
174
|
-
&& Object.keys(manifest.mutation_journal).length > 0;
|
|
175
|
-
const recoverable = transactionId != null && providerId != null && isRecoverableProviderDeleteBackup(manifest, {
|
|
176
|
-
root: transactionRoot,
|
|
177
|
-
managedRoot,
|
|
178
|
-
paths,
|
|
179
|
-
expectedProviderId: providerId,
|
|
180
|
-
}) === true;
|
|
181
|
-
if (['ROLLED_BACK', 'VERIFIED'].includes(phase) && !hasPendingMutation && recoverable) return [];
|
|
182
|
-
return [{ storage_id: entry.name, action_id: actionId, transaction_id: transactionId ?? entry.name, provider_id: providerId, phase, updated_at: updatedAt, recoverable, unresolved: recoverable !== true }];
|
|
183
|
-
} catch {
|
|
184
|
-
let updatedAt;
|
|
185
|
-
try { updatedAt = statSync(transactionRoot).mtime.toISOString(); } catch { updatedAt = new Date(0).toISOString(); }
|
|
186
|
-
return [{
|
|
187
|
-
storage_id: entry.name,
|
|
188
|
-
action_id: actionId,
|
|
189
|
-
transaction_id: null,
|
|
190
|
-
provider_id: null,
|
|
191
|
-
phase: 'RECOVERY_UNRESOLVED',
|
|
192
|
-
updated_at: updatedAt,
|
|
193
|
-
recoverable: false,
|
|
194
|
-
unresolved: true,
|
|
195
|
-
}];
|
|
196
|
-
}
|
|
197
|
-
});
|
|
198
|
-
// Recovery is a safety gate, not a bounded history view. Never discard an
|
|
199
|
-
// older unresolved transaction merely because newer entries exist.
|
|
200
|
-
return transactions.sort((a, b) => a.updated_at.localeCompare(b.updated_at));
|
|
201
|
-
} catch {
|
|
202
|
-
let updatedAt;
|
|
203
|
-
try { updatedAt = statSync(root).mtime.toISOString(); } catch { updatedAt = new Date(0).toISOString(); }
|
|
204
|
-
return [{ storage_id: null, action_id: null, transaction_id: null, provider_id: null, phase: 'RECOVERY_UNRESOLVED', updated_at: updatedAt, recoverable: false, unresolved: true }];
|
|
205
|
-
}
|
|
206
|
-
}
|
|
207
|
-
|
|
208
|
-
function writeCredentialPurgeState(state) {
|
|
209
|
-
mkdirSync(CONFIG_DIR, { recursive: true });
|
|
210
|
-
const temp = `${CREDENTIAL_PURGE_STATE_FILE}.${process.pid}.${Date.now()}.tmp`;
|
|
211
|
-
try {
|
|
212
|
-
writeFileSync(temp, JSON.stringify(normalizeCredentialPurgeState(state), null, 2) + '\n', 'utf8');
|
|
213
|
-
renameSync(temp, CREDENTIAL_PURGE_STATE_FILE);
|
|
214
|
-
} catch (error) {
|
|
215
|
-
try { rmSync(temp, { force: true }); } catch {}
|
|
216
|
-
throw error;
|
|
217
|
-
}
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
function providerInventoryPolicy(config) {
|
|
221
|
-
return {
|
|
222
|
-
worker: config?.worker?.model_policy ?? {
|
|
223
|
-
priority: config?.flash_model_priority,
|
|
224
|
-
escalation_priority: config?.pro_model_priority,
|
|
225
|
-
},
|
|
226
|
-
reviewer: config?.review?.model_policy ?? {
|
|
227
|
-
priority: config?.pro_model_priority,
|
|
228
|
-
escalation_priority: [],
|
|
229
|
-
},
|
|
230
|
-
};
|
|
231
|
-
}
|
|
232
|
-
|
|
233
|
-
export function hasCompleteProviderCatalogEvidence(evidence) {
|
|
234
|
-
return evidence?.ok === true && evidence.partial !== true;
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
export function hasCompleteProviderDeclarationEvidence(evidence) {
|
|
238
|
-
return evidence?.ok === true
|
|
239
|
-
&& Object.values(evidence.sources ?? {}).every((source) => source?.present !== true || !source.code);
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
export function hasAvailableProviderLifecycleEvidence(evidence) {
|
|
243
|
-
return evidence?.ok === true;
|
|
244
|
-
}
|
|
245
|
-
|
|
246
|
-
export function hasPendingProviderRecoveryTransactions(inventory) {
|
|
247
|
-
return Array.isArray(inventory?.recovery_transactions) && inventory.recovery_transactions.length > 0;
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
export function hasProviderRuntimeRestartEvidence(beforeRuntimeId, currentRuntimeId) {
|
|
251
|
-
return typeof beforeRuntimeId === 'string' && beforeRuntimeId.trim() !== ''
|
|
252
|
-
&& typeof currentRuntimeId === 'string' && currentRuntimeId.trim() !== ''
|
|
253
|
-
&& beforeRuntimeId !== currentRuntimeId;
|
|
254
|
-
}
|
|
255
|
-
|
|
256
|
-
/** Preserve profile/settings provenance when preparing the migration planner. */
|
|
257
|
-
export function buildProviderMigrationDeclarations(profileDeclarations = [], settingsDeclarations = []) {
|
|
258
|
-
return [
|
|
259
|
-
...(Array.isArray(profileDeclarations) ? profileDeclarations : []),
|
|
260
|
-
...(Array.isArray(settingsDeclarations) ? settingsDeclarations : []),
|
|
261
|
-
];
|
|
262
|
-
}
|
|
263
|
-
|
|
264
|
-
async function readProviderInventorySnapshot(hub, ctx, config) {
|
|
265
|
-
let catalog = { providers: [], harness_default: null };
|
|
266
|
-
let catalogEvidence = { ok: false, code: 'MODEL_CATALOG_UNAVAILABLE' };
|
|
267
|
-
try {
|
|
268
|
-
catalog = await readHarnessModelCatalog({
|
|
269
|
-
llm: ctx.llm ?? ctx.get('llm'),
|
|
270
|
-
getCurrentSelection: () => ctx.get('agentDefaultModel')?.currentSelection?.(),
|
|
271
|
-
});
|
|
272
|
-
catalogEvidence = {
|
|
273
|
-
ok: catalog.partial !== true,
|
|
274
|
-
...(catalog.partial === true ? { code: 'PROVIDER_CATALOG_INCOMPLETE' } : {}),
|
|
275
|
-
provider_count: Number.isInteger(catalog.provider_count) ? catalog.provider_count : catalog.providers.length,
|
|
276
|
-
refreshed_at: typeof catalog.refreshed_at === 'string' ? catalog.refreshed_at : null,
|
|
277
|
-
partial: catalog.partial === true,
|
|
278
|
-
};
|
|
279
|
-
} catch {}
|
|
280
|
-
const profileFile = join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml');
|
|
281
|
-
let declarations = [];
|
|
282
|
-
let profileDeclarations = [];
|
|
283
|
-
let settingsDeclarations = [];
|
|
284
|
-
const declarationEvidence = { ok: true, sources: {} };
|
|
285
|
-
try {
|
|
286
|
-
declarationEvidence.sources.profile = { present: existsSync(profileFile) };
|
|
287
|
-
if (existsSync(profileFile)) {
|
|
288
|
-
const source = readFileSync(profileFile, 'utf8');
|
|
289
|
-
const parsed = readProviderDeclarations(source, { file: 'harness/profiles/dsh-crew/cordis.patch.yml' });
|
|
290
|
-
if (parsed.ok) {
|
|
291
|
-
profileDeclarations = parsed.declarations;
|
|
292
|
-
declarations = profileDeclarations;
|
|
293
|
-
}
|
|
294
|
-
else { declarationEvidence.ok = false; declarationEvidence.sources.profile.code = parsed.code; }
|
|
295
|
-
}
|
|
296
|
-
} catch { declarationEvidence.ok = false; declarationEvidence.sources.profile = { present: true, code: 'PROVIDER_SOURCE_UNAVAILABLE' }; }
|
|
297
|
-
const settingsFile = join(CONFIG_DIR, 'harness', 'settings.yaml');
|
|
298
|
-
let settingsDefault = null;
|
|
299
|
-
let defaultEvidence = { ok: true, source: 'none' };
|
|
300
|
-
try {
|
|
301
|
-
declarationEvidence.sources.settings = { present: existsSync(settingsFile) };
|
|
302
|
-
if (existsSync(settingsFile)) {
|
|
303
|
-
const source = readFileSync(settingsFile, 'utf8');
|
|
304
|
-
const parsed = readProviderSettingsDeclarations(source, { file: 'harness/settings.yaml' });
|
|
305
|
-
if (parsed.ok) {
|
|
306
|
-
settingsDeclarations = parsed.declarations;
|
|
307
|
-
declarations = [...declarations, ...settingsDeclarations];
|
|
308
|
-
}
|
|
309
|
-
else { declarationEvidence.ok = false; declarationEvidence.sources.settings.code = parsed.code; }
|
|
310
|
-
const parsedDefault = readHarnessDefault(source);
|
|
311
|
-
if (parsedDefault.ok) settingsDefault = parsedDefault;
|
|
312
|
-
else defaultEvidence = { ok: false, code: 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' };
|
|
313
|
-
}
|
|
314
|
-
} catch {
|
|
315
|
-
declarationEvidence.ok = false;
|
|
316
|
-
declarationEvidence.sources.settings = { present: true, code: 'PROVIDER_SOURCE_UNAVAILABLE' };
|
|
317
|
-
defaultEvidence = { ok: false, code: 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' };
|
|
318
|
-
}
|
|
319
|
-
const liveDefault = catalog.harness_default && typeof catalog.harness_default.provider === 'string' && typeof catalog.harness_default.model === 'string'
|
|
320
|
-
? { provider: catalog.harness_default.provider, model: catalog.harness_default.model } : null;
|
|
321
|
-
const persistedDefault = settingsDefault ? { provider: settingsDefault.provider, model: settingsDefault.model } : null;
|
|
322
|
-
if (liveDefault || persistedDefault) {
|
|
323
|
-
if (!liveDefault || !persistedDefault) defaultEvidence = { ok: false, code: 'PROVIDER_DEFAULT_AUTHORITY_MISMATCH' };
|
|
324
|
-
else if (liveDefault.provider !== persistedDefault.provider || liveDefault.model !== persistedDefault.model) defaultEvidence = { ok: false, code: 'PROVIDER_DEFAULT_AUTHORITY_MISMATCH' };
|
|
325
|
-
}
|
|
326
|
-
// The persisted agent-default-model is the mutation authority. Keep it in
|
|
327
|
-
// the inventory even when the live selection is stale; the evidence gate
|
|
328
|
-
// above prevents destructive operations until both sources agree.
|
|
329
|
-
if (persistedDefault) {
|
|
330
|
-
catalog = {
|
|
331
|
-
...catalog,
|
|
332
|
-
harness_default: persistedDefault,
|
|
333
|
-
harness_default_authority: { kind: 'harness-settings', locator: settingsDefault.locator },
|
|
334
|
-
};
|
|
335
|
-
}
|
|
336
|
-
const lifecycleEvidence = readProviderLifecycleStateStatus();
|
|
337
|
-
const lifecycle = lifecycleEvidence.state;
|
|
338
|
-
const recoveryTransactions = [
|
|
339
|
-
...readProviderRecoveryTransactions(),
|
|
340
|
-
...readProviderLayerMigrationTransactions(join(CONFIG_DIR, 'provider-migration-backups')),
|
|
341
|
-
];
|
|
342
|
-
const credentialPurge = readCredentialPurgeState();
|
|
343
|
-
const inventory = buildProviderInventory({
|
|
344
|
-
catalog,
|
|
345
|
-
declarations,
|
|
346
|
-
policy: providerInventoryPolicy(config),
|
|
347
|
-
tombstones: lifecycle.tombstones,
|
|
348
|
-
additionalProviderIds: [
|
|
349
|
-
...Object.keys(lifecycle.tombstones ?? {}),
|
|
350
|
-
...recoveryTransactions.map((entry) => entry.provider_id).filter((id) => typeof id === 'string' && id.trim()),
|
|
351
|
-
],
|
|
352
|
-
activeJobs: hub.list(),
|
|
353
|
-
});
|
|
354
|
-
// Keep each authoritative source's credential reference with its own
|
|
355
|
-
// declaration. Reconstructing from the merged inventory would silently
|
|
356
|
-
// attach the first key to every layer when profile/settings differ.
|
|
357
|
-
const migrationDeclarations = buildProviderMigrationDeclarations(profileDeclarations, settingsDeclarations);
|
|
358
|
-
const routingReferences = [
|
|
359
|
-
...(config.flash_model_priority ?? []), ...(config.pro_model_priority ?? []),
|
|
360
|
-
...(config.worker?.model_policy?.priority ?? []), ...(config.worker?.model_policy?.escalation_priority ?? []),
|
|
361
|
-
...(config.review?.model_policy?.priority ?? []), ...(config.review?.model_policy?.escalation_priority ?? []),
|
|
362
|
-
];
|
|
363
|
-
const migration = buildProviderLayerMigrationPlan({
|
|
364
|
-
declarations: migrationDeclarations,
|
|
365
|
-
catalogProviders: catalog.providers,
|
|
366
|
-
harnessDefault: catalog.harness_default,
|
|
367
|
-
routingReferences,
|
|
368
|
-
tombstones: lifecycle.tombstones,
|
|
369
|
-
recoveryTransactions,
|
|
370
|
-
catalogEvidence,
|
|
371
|
-
declarationEvidence,
|
|
372
|
-
lifecycleEvidence,
|
|
373
|
-
defaultEvidence,
|
|
374
|
-
});
|
|
375
|
-
return {
|
|
376
|
-
...inventory,
|
|
377
|
-
migration,
|
|
378
|
-
catalog_evidence: catalogEvidence,
|
|
379
|
-
declaration_evidence: declarationEvidence,
|
|
380
|
-
default_evidence: defaultEvidence,
|
|
381
|
-
lifecycle_evidence: lifecycleEvidence.ok ? { ok: true } : { ok: false, code: lifecycleEvidence.code },
|
|
382
|
-
credential_history_refs: Object.values(lifecycle.transactions ?? {})
|
|
383
|
-
.flatMap((transaction) => Array.isArray(transaction?.credential_refs) ? transaction.credential_refs : []),
|
|
384
|
-
credential_purged_refs: Object.keys(credentialPurge.purged),
|
|
385
|
-
credential_purge_unverified: Object.entries(credentialPurge.unverified).map(([referenceId, record]) => ({
|
|
386
|
-
reference_id: referenceId,
|
|
387
|
-
transaction_id: record.transaction_id,
|
|
388
|
-
state: record.state,
|
|
389
|
-
})),
|
|
390
|
-
lifecycle_transactions: Object.entries(lifecycle.transactions ?? {}).slice(-64).map(([transactionId, transaction]) => ({
|
|
391
|
-
transaction_id: transactionId,
|
|
392
|
-
provider_id: transaction.provider_id,
|
|
393
|
-
state: transaction.state,
|
|
394
|
-
...(transaction.updated_at ? { updated_at: transaction.updated_at } : {}),
|
|
395
|
-
...(transaction.expected_revision ? { expected_revision: transaction.expected_revision } : {}),
|
|
396
|
-
})),
|
|
397
|
-
recovery_transactions: recoveryTransactions,
|
|
398
|
-
};
|
|
399
|
-
}
|
|
400
|
-
|
|
401
|
-
function readProviderSourceRevisions() {
|
|
402
|
-
const profileFile = join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml');
|
|
403
|
-
const settingsFile = join(CONFIG_DIR, 'harness', 'settings.yaml');
|
|
404
|
-
const read = (file, inspect) => {
|
|
405
|
-
try {
|
|
406
|
-
if (!existsSync(file)) return { ok: false, code: 'PROVIDER_SOURCE_MISSING', revision: null };
|
|
407
|
-
const parsed = inspect(readFileSync(file, 'utf8'));
|
|
408
|
-
return parsed.ok ? { ok: true, revision: parsed.revision } : { ok: false, code: parsed.code, revision: parsed.revision ?? null };
|
|
409
|
-
} catch { return { ok: false, code: 'PROVIDER_SOURCE_UNAVAILABLE', revision: null }; }
|
|
410
|
-
};
|
|
411
|
-
return { profile: read(profileFile, inspectProviderProfile), settings: read(settingsFile, inspectProviderSettings) };
|
|
412
|
-
}
|
|
413
|
-
|
|
414
|
-
export function hubCanonicalEvents(job = {}) {
|
|
122
|
+
const CONFIG_DIR = join(homedir(), '.config', 'dsh-crew');
|
|
123
|
+
const CREDENTIAL_PURGE_STATE_FILE = join(CONFIG_DIR, 'credential-purge-lifecycle.json');
|
|
124
|
+
const RECOVERY_ACTION_PATTERN = /^[a-f0-9]{32}$/u;
|
|
125
|
+
|
|
126
|
+
function recoveryActionId(name) {
|
|
127
|
+
return createHash('sha256').update(String(name)).digest('hex').slice(0, 32);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function isRecoveryControlEntry(name) {
|
|
131
|
+
return name === '.quarantine'
|
|
132
|
+
|| name === '.delete.lock'
|
|
133
|
+
|| name === '.delete.reclaim.lock'
|
|
134
|
+
|| name === '.delete.recovery.lock'
|
|
135
|
+
|| name === '.delete.recovery.lock.claim'
|
|
136
|
+
|| /^\.delete\.lock\.[0-9a-f-]+\.(?:active|staging|retired)$/iu.test(name)
|
|
137
|
+
|| /^\.delete\.recovery\.lock\.[0-9a-f-]+\.stale$/iu.test(name);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function readProviderLifecycleStateStatus() {
|
|
141
|
+
const file = join(CONFIG_DIR, 'provider-lifecycle.json');
|
|
142
|
+
if (!existsSync(file)) return { ok: true, state: normalizeProviderLifecycleState() };
|
|
143
|
+
try {
|
|
144
|
+
return { ok: true, state: normalizeProviderLifecycleState(JSON.parse(readFileSync(file, 'utf8'))) };
|
|
145
|
+
} catch {
|
|
146
|
+
return { ok: false, code: 'PROVIDER_LIFECYCLE_UNAVAILABLE', state: normalizeProviderLifecycleState() };
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function readProviderLifecycleState() {
|
|
151
|
+
return readProviderLifecycleStateStatus().state;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
function readCredentialPurgeState() {
|
|
155
|
+
try { return normalizeCredentialPurgeState(JSON.parse(readFileSync(CREDENTIAL_PURGE_STATE_FILE, 'utf8'))); } catch { return normalizeCredentialPurgeState(); }
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
export function readProviderRecoveryTransactions(root = join(CONFIG_DIR, 'provider-backups')) {
|
|
159
|
+
if (!existsSync(root)) return [];
|
|
160
|
+
try {
|
|
161
|
+
const managedRoot = dirname(resolvePath(root));
|
|
162
|
+
const paths = {
|
|
163
|
+
profile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
164
|
+
settings: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
165
|
+
config: join(CONFIG_DIR, 'config.json'),
|
|
166
|
+
lifecycle: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
167
|
+
};
|
|
168
|
+
const transactions = readdirSync(root, { withFileTypes: true }).filter((entry) => !isRecoveryControlEntry(entry.name)).flatMap((entry) => {
|
|
169
|
+
const transactionRoot = join(root, entry.name);
|
|
170
|
+
const manifestFile = join(root, entry.name, 'manifest.json');
|
|
171
|
+
const actionId = recoveryActionId(entry.name);
|
|
172
|
+
if (!entry.isDirectory()) {
|
|
173
|
+
let updatedAt;
|
|
174
|
+
try { updatedAt = statSync(transactionRoot).mtime.toISOString(); } catch { updatedAt = new Date(0).toISOString(); }
|
|
175
|
+
return [{ storage_id: entry.name, action_id: actionId, transaction_id: null, provider_id: null, phase: 'RECOVERY_UNRESOLVED', updated_at: updatedAt, recoverable: false, unresolved: true }];
|
|
176
|
+
}
|
|
177
|
+
try {
|
|
178
|
+
const manifest = readProviderDeleteManifestFile(manifestFile, managedRoot);
|
|
179
|
+
const transactionId = typeof manifest?.plan?.plan_id === 'string' ? manifest.plan.plan_id : null;
|
|
180
|
+
const providerId = typeof manifest?.provider_id === 'string' ? manifest.provider_id : null;
|
|
181
|
+
const phase = typeof manifest?.phase_journal?.phase === 'string' && manifest.phase_journal.phase.trim()
|
|
182
|
+
? manifest.phase_journal.phase.trim() : 'RECOVERY_INCOMPLETE';
|
|
183
|
+
const updatedAt = typeof manifest?.phase_journal?.updated_at === 'string' && manifest.phase_journal.updated_at.trim()
|
|
184
|
+
? manifest.phase_journal.updated_at : statSync(manifestFile).mtime.toISOString();
|
|
185
|
+
const hasPendingMutation = manifest?.mutation_journal && typeof manifest.mutation_journal === 'object'
|
|
186
|
+
&& Object.keys(manifest.mutation_journal).length > 0;
|
|
187
|
+
const recoverable = transactionId != null && providerId != null && isRecoverableProviderDeleteBackup(manifest, {
|
|
188
|
+
root: transactionRoot,
|
|
189
|
+
managedRoot,
|
|
190
|
+
paths,
|
|
191
|
+
expectedProviderId: providerId,
|
|
192
|
+
}) === true;
|
|
193
|
+
if (['ROLLED_BACK', 'VERIFIED'].includes(phase) && !hasPendingMutation && recoverable) return [];
|
|
194
|
+
return [{ storage_id: entry.name, action_id: actionId, transaction_id: transactionId ?? entry.name, provider_id: providerId, phase, updated_at: updatedAt, recoverable, unresolved: recoverable !== true }];
|
|
195
|
+
} catch {
|
|
196
|
+
let updatedAt;
|
|
197
|
+
try { updatedAt = statSync(transactionRoot).mtime.toISOString(); } catch { updatedAt = new Date(0).toISOString(); }
|
|
198
|
+
return [{
|
|
199
|
+
storage_id: entry.name,
|
|
200
|
+
action_id: actionId,
|
|
201
|
+
transaction_id: null,
|
|
202
|
+
provider_id: null,
|
|
203
|
+
phase: 'RECOVERY_UNRESOLVED',
|
|
204
|
+
updated_at: updatedAt,
|
|
205
|
+
recoverable: false,
|
|
206
|
+
unresolved: true,
|
|
207
|
+
}];
|
|
208
|
+
}
|
|
209
|
+
});
|
|
210
|
+
// Recovery is a safety gate, not a bounded history view. Never discard an
|
|
211
|
+
// older unresolved transaction merely because newer entries exist.
|
|
212
|
+
return transactions.sort((a, b) => a.updated_at.localeCompare(b.updated_at));
|
|
213
|
+
} catch {
|
|
214
|
+
let updatedAt;
|
|
215
|
+
try { updatedAt = statSync(root).mtime.toISOString(); } catch { updatedAt = new Date(0).toISOString(); }
|
|
216
|
+
return [{ storage_id: null, action_id: null, transaction_id: null, provider_id: null, phase: 'RECOVERY_UNRESOLVED', updated_at: updatedAt, recoverable: false, unresolved: true }];
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
function writeCredentialPurgeState(state) {
|
|
221
|
+
mkdirSync(CONFIG_DIR, { recursive: true });
|
|
222
|
+
const temp = `${CREDENTIAL_PURGE_STATE_FILE}.${process.pid}.${Date.now()}.tmp`;
|
|
223
|
+
try {
|
|
224
|
+
writeFileSync(temp, JSON.stringify(normalizeCredentialPurgeState(state), null, 2) + '\n', 'utf8');
|
|
225
|
+
renameSync(temp, CREDENTIAL_PURGE_STATE_FILE);
|
|
226
|
+
} catch (error) {
|
|
227
|
+
try { rmSync(temp, { force: true }); } catch {}
|
|
228
|
+
throw error;
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
function providerInventoryPolicy(config) {
|
|
233
|
+
return {
|
|
234
|
+
worker: config?.worker?.model_policy ?? {
|
|
235
|
+
priority: config?.flash_model_priority,
|
|
236
|
+
escalation_priority: config?.pro_model_priority,
|
|
237
|
+
},
|
|
238
|
+
reviewer: config?.review?.model_policy ?? {
|
|
239
|
+
priority: config?.pro_model_priority,
|
|
240
|
+
escalation_priority: [],
|
|
241
|
+
},
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
export function hasCompleteProviderCatalogEvidence(evidence) {
|
|
246
|
+
return evidence?.ok === true && evidence.partial !== true;
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
export function hasCompleteProviderDeclarationEvidence(evidence) {
|
|
250
|
+
return evidence?.ok === true
|
|
251
|
+
&& Object.values(evidence.sources ?? {}).every((source) => source?.present !== true || !source.code);
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
export function hasAvailableProviderLifecycleEvidence(evidence) {
|
|
255
|
+
return evidence?.ok === true;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
export function hasPendingProviderRecoveryTransactions(inventory) {
|
|
259
|
+
return Array.isArray(inventory?.recovery_transactions) && inventory.recovery_transactions.length > 0;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
export function hasProviderRuntimeRestartEvidence(beforeRuntimeId, currentRuntimeId) {
|
|
263
|
+
return typeof beforeRuntimeId === 'string' && beforeRuntimeId.trim() !== ''
|
|
264
|
+
&& typeof currentRuntimeId === 'string' && currentRuntimeId.trim() !== ''
|
|
265
|
+
&& beforeRuntimeId !== currentRuntimeId;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/** Preserve profile/settings provenance when preparing the migration planner. */
|
|
269
|
+
export function buildProviderMigrationDeclarations(profileDeclarations = [], settingsDeclarations = []) {
|
|
270
|
+
return [
|
|
271
|
+
...(Array.isArray(profileDeclarations) ? profileDeclarations : []),
|
|
272
|
+
...(Array.isArray(settingsDeclarations) ? settingsDeclarations : []),
|
|
273
|
+
];
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
async function readProviderInventorySnapshot(hub, ctx, config) {
|
|
277
|
+
let catalog = { providers: [], harness_default: null };
|
|
278
|
+
let catalogEvidence = { ok: false, code: 'MODEL_CATALOG_UNAVAILABLE' };
|
|
279
|
+
try {
|
|
280
|
+
catalog = await readHarnessModelCatalog({
|
|
281
|
+
llm: ctx.llm ?? ctx.get('llm'),
|
|
282
|
+
getCurrentSelection: () => ctx.get('agentDefaultModel')?.currentSelection?.(),
|
|
283
|
+
});
|
|
284
|
+
catalogEvidence = {
|
|
285
|
+
ok: catalog.partial !== true,
|
|
286
|
+
...(catalog.partial === true ? { code: 'PROVIDER_CATALOG_INCOMPLETE' } : {}),
|
|
287
|
+
provider_count: Number.isInteger(catalog.provider_count) ? catalog.provider_count : catalog.providers.length,
|
|
288
|
+
refreshed_at: typeof catalog.refreshed_at === 'string' ? catalog.refreshed_at : null,
|
|
289
|
+
partial: catalog.partial === true,
|
|
290
|
+
};
|
|
291
|
+
} catch {}
|
|
292
|
+
const profileFile = join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml');
|
|
293
|
+
let declarations = [];
|
|
294
|
+
let profileDeclarations = [];
|
|
295
|
+
let settingsDeclarations = [];
|
|
296
|
+
const declarationEvidence = { ok: true, sources: {} };
|
|
297
|
+
try {
|
|
298
|
+
declarationEvidence.sources.profile = { present: existsSync(profileFile) };
|
|
299
|
+
if (existsSync(profileFile)) {
|
|
300
|
+
const source = readFileSync(profileFile, 'utf8');
|
|
301
|
+
const parsed = readProviderDeclarations(source, { file: 'harness/profiles/dsh-crew/cordis.patch.yml' });
|
|
302
|
+
if (parsed.ok) {
|
|
303
|
+
profileDeclarations = parsed.declarations;
|
|
304
|
+
declarations = profileDeclarations;
|
|
305
|
+
}
|
|
306
|
+
else { declarationEvidence.ok = false; declarationEvidence.sources.profile.code = parsed.code; }
|
|
307
|
+
}
|
|
308
|
+
} catch { declarationEvidence.ok = false; declarationEvidence.sources.profile = { present: true, code: 'PROVIDER_SOURCE_UNAVAILABLE' }; }
|
|
309
|
+
const settingsFile = join(CONFIG_DIR, 'harness', 'settings.yaml');
|
|
310
|
+
let settingsDefault = null;
|
|
311
|
+
let defaultEvidence = { ok: true, source: 'none' };
|
|
312
|
+
try {
|
|
313
|
+
declarationEvidence.sources.settings = { present: existsSync(settingsFile) };
|
|
314
|
+
if (existsSync(settingsFile)) {
|
|
315
|
+
const source = readFileSync(settingsFile, 'utf8');
|
|
316
|
+
const parsed = readProviderSettingsDeclarations(source, { file: 'harness/settings.yaml' });
|
|
317
|
+
if (parsed.ok) {
|
|
318
|
+
settingsDeclarations = parsed.declarations;
|
|
319
|
+
declarations = [...declarations, ...settingsDeclarations];
|
|
320
|
+
}
|
|
321
|
+
else { declarationEvidence.ok = false; declarationEvidence.sources.settings.code = parsed.code; }
|
|
322
|
+
const parsedDefault = readHarnessDefault(source);
|
|
323
|
+
if (parsedDefault.ok) settingsDefault = parsedDefault;
|
|
324
|
+
else defaultEvidence = { ok: false, code: 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' };
|
|
325
|
+
}
|
|
326
|
+
} catch {
|
|
327
|
+
declarationEvidence.ok = false;
|
|
328
|
+
declarationEvidence.sources.settings = { present: true, code: 'PROVIDER_SOURCE_UNAVAILABLE' };
|
|
329
|
+
defaultEvidence = { ok: false, code: 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' };
|
|
330
|
+
}
|
|
331
|
+
const liveDefault = catalog.harness_default && typeof catalog.harness_default.provider === 'string' && typeof catalog.harness_default.model === 'string'
|
|
332
|
+
? { provider: catalog.harness_default.provider, model: catalog.harness_default.model } : null;
|
|
333
|
+
const persistedDefault = settingsDefault ? { provider: settingsDefault.provider, model: settingsDefault.model } : null;
|
|
334
|
+
if (liveDefault || persistedDefault) {
|
|
335
|
+
if (!liveDefault || !persistedDefault) defaultEvidence = { ok: false, code: 'PROVIDER_DEFAULT_AUTHORITY_MISMATCH' };
|
|
336
|
+
else if (liveDefault.provider !== persistedDefault.provider || liveDefault.model !== persistedDefault.model) defaultEvidence = { ok: false, code: 'PROVIDER_DEFAULT_AUTHORITY_MISMATCH' };
|
|
337
|
+
}
|
|
338
|
+
// The persisted agent-default-model is the mutation authority. Keep it in
|
|
339
|
+
// the inventory even when the live selection is stale; the evidence gate
|
|
340
|
+
// above prevents destructive operations until both sources agree.
|
|
341
|
+
if (persistedDefault) {
|
|
342
|
+
catalog = {
|
|
343
|
+
...catalog,
|
|
344
|
+
harness_default: persistedDefault,
|
|
345
|
+
harness_default_authority: { kind: 'harness-settings', locator: settingsDefault.locator },
|
|
346
|
+
};
|
|
347
|
+
}
|
|
348
|
+
const lifecycleEvidence = readProviderLifecycleStateStatus();
|
|
349
|
+
const lifecycle = lifecycleEvidence.state;
|
|
350
|
+
const recoveryTransactions = [
|
|
351
|
+
...readProviderRecoveryTransactions(),
|
|
352
|
+
...readProviderLayerMigrationTransactions(join(CONFIG_DIR, 'provider-migration-backups')),
|
|
353
|
+
];
|
|
354
|
+
const credentialPurge = readCredentialPurgeState();
|
|
355
|
+
const inventory = buildProviderInventory({
|
|
356
|
+
catalog,
|
|
357
|
+
declarations,
|
|
358
|
+
policy: providerInventoryPolicy(config),
|
|
359
|
+
tombstones: lifecycle.tombstones,
|
|
360
|
+
additionalProviderIds: [
|
|
361
|
+
...Object.keys(lifecycle.tombstones ?? {}),
|
|
362
|
+
...recoveryTransactions.map((entry) => entry.provider_id).filter((id) => typeof id === 'string' && id.trim()),
|
|
363
|
+
],
|
|
364
|
+
activeJobs: hub.list(),
|
|
365
|
+
});
|
|
366
|
+
// Keep each authoritative source's credential reference with its own
|
|
367
|
+
// declaration. Reconstructing from the merged inventory would silently
|
|
368
|
+
// attach the first key to every layer when profile/settings differ.
|
|
369
|
+
const migrationDeclarations = buildProviderMigrationDeclarations(profileDeclarations, settingsDeclarations);
|
|
370
|
+
const routingReferences = [
|
|
371
|
+
...(config.flash_model_priority ?? []), ...(config.pro_model_priority ?? []),
|
|
372
|
+
...(config.worker?.model_policy?.priority ?? []), ...(config.worker?.model_policy?.escalation_priority ?? []),
|
|
373
|
+
...(config.review?.model_policy?.priority ?? []), ...(config.review?.model_policy?.escalation_priority ?? []),
|
|
374
|
+
];
|
|
375
|
+
const migration = buildProviderLayerMigrationPlan({
|
|
376
|
+
declarations: migrationDeclarations,
|
|
377
|
+
catalogProviders: catalog.providers,
|
|
378
|
+
harnessDefault: catalog.harness_default,
|
|
379
|
+
routingReferences,
|
|
380
|
+
tombstones: lifecycle.tombstones,
|
|
381
|
+
recoveryTransactions,
|
|
382
|
+
catalogEvidence,
|
|
383
|
+
declarationEvidence,
|
|
384
|
+
lifecycleEvidence,
|
|
385
|
+
defaultEvidence,
|
|
386
|
+
});
|
|
387
|
+
return {
|
|
388
|
+
...inventory,
|
|
389
|
+
migration,
|
|
390
|
+
catalog_evidence: catalogEvidence,
|
|
391
|
+
declaration_evidence: declarationEvidence,
|
|
392
|
+
default_evidence: defaultEvidence,
|
|
393
|
+
lifecycle_evidence: lifecycleEvidence.ok ? { ok: true } : { ok: false, code: lifecycleEvidence.code },
|
|
394
|
+
credential_history_refs: Object.values(lifecycle.transactions ?? {})
|
|
395
|
+
.flatMap((transaction) => Array.isArray(transaction?.credential_refs) ? transaction.credential_refs : []),
|
|
396
|
+
credential_purged_refs: Object.keys(credentialPurge.purged),
|
|
397
|
+
credential_purge_unverified: Object.entries(credentialPurge.unverified).map(([referenceId, record]) => ({
|
|
398
|
+
reference_id: referenceId,
|
|
399
|
+
transaction_id: record.transaction_id,
|
|
400
|
+
state: record.state,
|
|
401
|
+
})),
|
|
402
|
+
lifecycle_transactions: Object.entries(lifecycle.transactions ?? {}).slice(-64).map(([transactionId, transaction]) => ({
|
|
403
|
+
transaction_id: transactionId,
|
|
404
|
+
provider_id: transaction.provider_id,
|
|
405
|
+
state: transaction.state,
|
|
406
|
+
...(transaction.updated_at ? { updated_at: transaction.updated_at } : {}),
|
|
407
|
+
...(transaction.expected_revision ? { expected_revision: transaction.expected_revision } : {}),
|
|
408
|
+
})),
|
|
409
|
+
recovery_transactions: recoveryTransactions,
|
|
410
|
+
};
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
function readProviderSourceRevisions() {
|
|
414
|
+
const profileFile = join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml');
|
|
415
|
+
const settingsFile = join(CONFIG_DIR, 'harness', 'settings.yaml');
|
|
416
|
+
const read = (file, inspect) => {
|
|
417
|
+
try {
|
|
418
|
+
if (!existsSync(file)) return { ok: false, code: 'PROVIDER_SOURCE_MISSING', revision: null };
|
|
419
|
+
const parsed = inspect(readFileSync(file, 'utf8'));
|
|
420
|
+
return parsed.ok ? { ok: true, revision: parsed.revision } : { ok: false, code: parsed.code, revision: parsed.revision ?? null };
|
|
421
|
+
} catch { return { ok: false, code: 'PROVIDER_SOURCE_UNAVAILABLE', revision: null }; }
|
|
422
|
+
};
|
|
423
|
+
return { profile: read(profileFile, inspectProviderProfile), settings: read(settingsFile, inspectProviderSettings) };
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
export function hubCanonicalEvents(job = {}) {
|
|
415
427
|
if (!job.id) return [];
|
|
416
428
|
const role = job.role === 'reviewer' ? 'reviewer' : 'worker';
|
|
417
429
|
const atStart = job.startedAt ?? null;
|
|
418
430
|
const atEnd = job.endedAt ?? atStart;
|
|
419
|
-
const definitions = [
|
|
420
|
-
['job.created', atStart, { client_job_id: job.client_job_id ?? null }],
|
|
421
|
-
['job.started', atStart, {}],
|
|
422
|
-
['runtime.bound', atStart, {
|
|
423
|
-
execution_plane: job.execution_context?.execution_plane ?? null,
|
|
424
|
-
profile: job.execution_context?.profile ?? null,
|
|
425
|
-
listen_port: Number.isInteger(job.execution_context?.listen_port) ? job.execution_context.listen_port : null,
|
|
426
|
-
runtime_id: job.execution_context?.runtime_id ?? null,
|
|
427
|
-
}],
|
|
428
|
-
['model.selected', atStart, { provider: job.provider ?? null, model: job.model ?? null, source: job.selection_source ?? null }],
|
|
429
|
-
['model.admitted', atStart, {
|
|
430
|
-
provider: job.provider ?? null,
|
|
431
|
-
model: job.model ?? null,
|
|
432
|
-
health_state: job.health_state ?? 'unprobed',
|
|
433
|
-
reason_code: job.health_reason_code ?? null,
|
|
434
|
-
observed_at: Number.isFinite(job.health_observed_at) ? job.health_observed_at : null,
|
|
435
|
-
}],
|
|
436
|
-
[role === 'reviewer' ? 'review.started' : 'worker.started', atStart, { run_id: job.id }],
|
|
437
|
-
['agent.created', atStart, { run_id: job.id }],
|
|
438
|
-
];
|
|
431
|
+
const definitions = [
|
|
432
|
+
['job.created', atStart, { client_job_id: job.client_job_id ?? null }],
|
|
433
|
+
['job.started', atStart, {}],
|
|
434
|
+
['runtime.bound', atStart, {
|
|
435
|
+
execution_plane: job.execution_context?.execution_plane ?? null,
|
|
436
|
+
profile: job.execution_context?.profile ?? null,
|
|
437
|
+
listen_port: Number.isInteger(job.execution_context?.listen_port) ? job.execution_context.listen_port : null,
|
|
438
|
+
runtime_id: job.execution_context?.runtime_id ?? null,
|
|
439
|
+
}],
|
|
440
|
+
['model.selected', atStart, { provider: job.provider ?? null, model: job.model ?? null, source: job.selection_source ?? null }],
|
|
441
|
+
['model.admitted', atStart, {
|
|
442
|
+
provider: job.provider ?? null,
|
|
443
|
+
model: job.model ?? null,
|
|
444
|
+
health_state: job.health_state ?? 'unprobed',
|
|
445
|
+
reason_code: job.health_reason_code ?? null,
|
|
446
|
+
observed_at: Number.isFinite(job.health_observed_at) ? job.health_observed_at : null,
|
|
447
|
+
}],
|
|
448
|
+
[role === 'reviewer' ? 'review.started' : 'worker.started', atStart, { run_id: job.id }],
|
|
449
|
+
['agent.created', atStart, { run_id: job.id }],
|
|
450
|
+
];
|
|
439
451
|
if (job.status !== 'running') {
|
|
440
452
|
definitions.push([
|
|
441
453
|
role === 'reviewer' ? 'review.completed' : 'worker.completed',
|
|
@@ -455,7 +467,7 @@ export function hubCanonicalEvents(job = {}) {
|
|
|
455
467
|
|
|
456
468
|
// ---------- job registry ----------
|
|
457
469
|
|
|
458
|
-
export function applyHubWorkspaceEvidence({ outcome, workspaceDiff, allowNoChanges = false, isolation = 'shared', role = 'worker' } = {}) {
|
|
470
|
+
export function applyHubWorkspaceEvidence({ outcome, workspaceDiff, allowNoChanges = false, isolation = 'shared', role = 'worker' } = {}) {
|
|
459
471
|
const changes = workspaceDiff?.changes ?? {};
|
|
460
472
|
const hasChanges = ['modified', 'deleted', 'renamed', 'untracked']
|
|
461
473
|
.some((key) => Array.isArray(changes[key]) && changes[key].length > 0);
|
|
@@ -465,71 +477,71 @@ export function applyHubWorkspaceEvidence({ outcome, workspaceDiff, allowNoChang
|
|
|
465
477
|
hasChanges,
|
|
466
478
|
allowNoChanges: allowNoChanges === true && isolation === 'worktree',
|
|
467
479
|
requireNoChangeAuthorization: role === 'worker',
|
|
468
|
-
});
|
|
469
|
-
}
|
|
470
|
-
|
|
471
|
-
/**
|
|
472
|
-
* Build the explicit provider callability probe used by the 3210 lifecycle
|
|
473
|
-
* route. It performs one token-bounded Harness stream through the same llm
|
|
474
|
-
* runtime used by WorkerRegistry; no credential values are read or returned.
|
|
475
|
-
*/
|
|
476
|
-
export function createProviderProbe(ctx) {
|
|
477
|
-
const llm = ctx?.llm ?? ctx?.get?.('llm');
|
|
478
|
-
if (!llm || typeof llm.stream !== 'function') return null;
|
|
479
|
-
return async ({ provider, model, signal }) => {
|
|
480
|
-
try {
|
|
481
|
-
const stream = llm.stream({
|
|
482
|
-
provider,
|
|
483
|
-
model,
|
|
484
|
-
messages: [userMessage('dsh-crew provider probe')],
|
|
485
|
-
maxTokens: 1,
|
|
486
|
-
temperature: 0,
|
|
487
|
-
signal,
|
|
488
|
-
});
|
|
489
|
-
for await (const chunk of stream) {
|
|
490
|
-
if (chunk?.type !== 'finish') continue;
|
|
491
|
-
const reason = chunk.reason;
|
|
492
|
-
if (reason?.kind === 'error' || reason?.kind === 'aborted') {
|
|
493
|
-
return { ok: false, error: reason.failure ?? { code: `PROVIDER_PROBE_${String(reason.kind).toUpperCase()}` } };
|
|
494
|
-
}
|
|
495
|
-
return { ok: true };
|
|
496
|
-
}
|
|
497
|
-
return { ok: false, error: { code: 'PROVIDER_PROBE_NO_FINISH' } };
|
|
498
|
-
} catch (error) {
|
|
499
|
-
return { ok: false, error };
|
|
500
|
-
}
|
|
501
|
-
};
|
|
502
|
-
}
|
|
503
|
-
|
|
504
|
-
/** Pick the first configured model for a provider, falling back to its live catalog. */
|
|
505
|
-
export function selectProviderProbeModel({ providerId, record, config } = {}) {
|
|
506
|
-
const advertised = new Set(Array.isArray(record?.models) ? record.models.filter((model) => typeof model === 'string') : []);
|
|
507
|
-
if (advertised.size === 0) return null;
|
|
508
|
-
const priorities = [
|
|
509
|
-
...(Array.isArray(config?.worker?.model_policy?.priority) ? config.worker.model_policy.priority : []),
|
|
510
|
-
...(Array.isArray(config?.flash_model_priority) ? config.flash_model_priority : []),
|
|
511
|
-
...(Array.isArray(config?.review?.model_policy?.priority) ? config.review.model_policy.priority : []),
|
|
512
|
-
...(Array.isArray(config?.pro_model_priority) ? config.pro_model_priority : []),
|
|
513
|
-
];
|
|
514
|
-
for (const ref of priorities) {
|
|
515
|
-
if (ref?.provider === providerId && advertised.has(ref.model)) return ref.model;
|
|
516
|
-
}
|
|
517
|
-
return [...advertised][0];
|
|
518
|
-
}
|
|
519
|
-
|
|
520
|
-
// Exported for the unit tests (test/hub-windows.test.mjs); instantiation
|
|
480
|
+
});
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
/**
|
|
484
|
+
* Build the explicit provider callability probe used by the 3210 lifecycle
|
|
485
|
+
* route. It performs one token-bounded Harness stream through the same llm
|
|
486
|
+
* runtime used by WorkerRegistry; no credential values are read or returned.
|
|
487
|
+
*/
|
|
488
|
+
export function createProviderProbe(ctx) {
|
|
489
|
+
const llm = ctx?.llm ?? ctx?.get?.('llm');
|
|
490
|
+
if (!llm || typeof llm.stream !== 'function') return null;
|
|
491
|
+
return async ({ provider, model, signal }) => {
|
|
492
|
+
try {
|
|
493
|
+
const stream = llm.stream({
|
|
494
|
+
provider,
|
|
495
|
+
model,
|
|
496
|
+
messages: [userMessage('dsh-crew provider probe')],
|
|
497
|
+
maxTokens: 1,
|
|
498
|
+
temperature: 0,
|
|
499
|
+
signal,
|
|
500
|
+
});
|
|
501
|
+
for await (const chunk of stream) {
|
|
502
|
+
if (chunk?.type !== 'finish') continue;
|
|
503
|
+
const reason = chunk.reason;
|
|
504
|
+
if (reason?.kind === 'error' || reason?.kind === 'aborted') {
|
|
505
|
+
return { ok: false, error: reason.failure ?? { code: `PROVIDER_PROBE_${String(reason.kind).toUpperCase()}` } };
|
|
506
|
+
}
|
|
507
|
+
return { ok: true };
|
|
508
|
+
}
|
|
509
|
+
return { ok: false, error: { code: 'PROVIDER_PROBE_NO_FINISH' } };
|
|
510
|
+
} catch (error) {
|
|
511
|
+
return { ok: false, error };
|
|
512
|
+
}
|
|
513
|
+
};
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
/** Pick the first configured model for a provider, falling back to its live catalog. */
|
|
517
|
+
export function selectProviderProbeModel({ providerId, record, config } = {}) {
|
|
518
|
+
const advertised = new Set(Array.isArray(record?.models) ? record.models.filter((model) => typeof model === 'string') : []);
|
|
519
|
+
if (advertised.size === 0) return null;
|
|
520
|
+
const priorities = [
|
|
521
|
+
...(Array.isArray(config?.worker?.model_policy?.priority) ? config.worker.model_policy.priority : []),
|
|
522
|
+
...(Array.isArray(config?.flash_model_priority) ? config.flash_model_priority : []),
|
|
523
|
+
...(Array.isArray(config?.review?.model_policy?.priority) ? config.review.model_policy.priority : []),
|
|
524
|
+
...(Array.isArray(config?.pro_model_priority) ? config.pro_model_priority : []),
|
|
525
|
+
];
|
|
526
|
+
for (const ref of priorities) {
|
|
527
|
+
if (ref?.provider === providerId && advertised.has(ref.model)) return ref.model;
|
|
528
|
+
}
|
|
529
|
+
return [...advertised][0];
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
// Exported for the unit tests (test/hub-windows.test.mjs); instantiation
|
|
521
533
|
// needs only a duck-typed ctx, so spawn()'s path guard is testable without a
|
|
522
534
|
// live DSH host.
|
|
523
|
-
export class WorkerRegistry { constructor(ctx) {
|
|
524
|
-
this.ctx = ctx;
|
|
525
|
-
this.jobs = new Map();
|
|
526
|
-
this.providerMutations = new Set();
|
|
527
|
-
this.providerMutationEpochs = new Map();
|
|
528
|
-
this.providerLeases = new Map();
|
|
529
|
-
this.nextId = 1;
|
|
530
|
-
this.shard = createShardWriter('hub');
|
|
531
|
-
this.healthStore = createProviderHealthStore();
|
|
532
|
-
}
|
|
535
|
+
export class WorkerRegistry { constructor(ctx) {
|
|
536
|
+
this.ctx = ctx;
|
|
537
|
+
this.jobs = new Map();
|
|
538
|
+
this.providerMutations = new Set();
|
|
539
|
+
this.providerMutationEpochs = new Map();
|
|
540
|
+
this.providerLeases = new Map();
|
|
541
|
+
this.nextId = 1;
|
|
542
|
+
this.shard = createShardWriter('hub');
|
|
543
|
+
this.healthStore = createProviderHealthStore();
|
|
544
|
+
}
|
|
533
545
|
|
|
534
546
|
view(job, withResult = false) {
|
|
535
547
|
const v = {
|
|
@@ -549,13 +561,13 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
549
561
|
task_status: job.outcome?.task_status ?? null,
|
|
550
562
|
workspace_evidence_ok: job.outcome?.workspace_evidence_ok ?? null,
|
|
551
563
|
review_verdict: job.review?.verdict ?? null,
|
|
552
|
-
workspace_diff_available: !!job.workspaceDiff && job.workspaceDiff.kind === 'git',
|
|
553
|
-
workspace_retained: job.workspace_retained === true,
|
|
554
|
-
cleanup_warning: job.cleanup_warning ?? null,
|
|
555
|
-
profile_id: job.profile_id ?? null,
|
|
556
|
-
workspace_context: job.workspace_context ?? null,
|
|
557
|
-
execution_context: job.execution_context ?? null,
|
|
558
|
-
event_cursor: hubCanonicalEvents(job).at(-1)?.sequence ?? 0,
|
|
564
|
+
workspace_diff_available: !!job.workspaceDiff && job.workspaceDiff.kind === 'git',
|
|
565
|
+
workspace_retained: job.workspace_retained === true,
|
|
566
|
+
cleanup_warning: job.cleanup_warning ?? null,
|
|
567
|
+
profile_id: job.profile_id ?? null,
|
|
568
|
+
workspace_context: job.workspace_context ?? null,
|
|
569
|
+
execution_context: job.execution_context ?? null,
|
|
570
|
+
event_cursor: hubCanonicalEvents(job).at(-1)?.sequence ?? 0,
|
|
559
571
|
};
|
|
560
572
|
if (withResult) {
|
|
561
573
|
v.result = job.result; v.error = job.error; v.stopReason = job.stopReason;
|
|
@@ -571,51 +583,51 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
571
583
|
return v;
|
|
572
584
|
}
|
|
573
585
|
|
|
574
|
-
list() {
|
|
575
|
-
return [...this.jobs.values()].map((job) => this.view(job));
|
|
576
|
-
}
|
|
577
|
-
|
|
578
|
-
beginProviderMutation(providerId) {
|
|
579
|
-
if (typeof providerId !== 'string' || !providerId.trim() || this.providerMutations.has(providerId)) return false;
|
|
580
|
-
this.providerMutationEpochs.set(providerId, (this.providerMutationEpochs.get(providerId) ?? 0) + 1);
|
|
581
|
-
this.providerMutations.add(providerId);
|
|
582
|
-
return true;
|
|
583
|
-
}
|
|
584
|
-
|
|
585
|
-
endProviderMutation(providerId) {
|
|
586
|
-
this.providerMutations.delete(providerId);
|
|
587
|
-
}
|
|
588
|
-
|
|
589
|
-
getProviderMutationEpoch(providerId) {
|
|
590
|
-
return this.providerMutationEpochs.get(providerId) ?? 0;
|
|
591
|
-
}
|
|
592
|
-
|
|
593
|
-
assertProviderNotMutating(providerId, expectedEpoch = undefined) {
|
|
594
|
-
if (this.providerMutations.has(providerId)
|
|
595
|
-
|| (expectedEpoch !== undefined && this.getProviderMutationEpoch(providerId) !== expectedEpoch)) {
|
|
596
|
-
throw Object.assign(new Error('provider mutation is active'), { code: 'PROVIDER_DELETE_BUSY' });
|
|
597
|
-
}
|
|
598
|
-
}
|
|
599
|
-
|
|
600
|
-
acquireProviderLease(providerId) {
|
|
601
|
-
this.providerLeases.set(providerId, (this.providerLeases.get(providerId) ?? 0) + 1);
|
|
602
|
-
}
|
|
603
|
-
|
|
604
|
-
releaseProviderLease(providerId) {
|
|
605
|
-
const count = this.providerLeases.get(providerId) ?? 0;
|
|
606
|
-
if (count <= 1) this.providerLeases.delete(providerId);
|
|
607
|
-
else this.providerLeases.set(providerId, count - 1);
|
|
608
|
-
}
|
|
609
|
-
|
|
610
|
-
hasProviderLease(providerId) {
|
|
611
|
-
return (this.providerLeases.get(providerId) ?? 0) > 0;
|
|
612
|
-
}
|
|
613
|
-
|
|
614
|
-
// Profile/settings are shared stores. A migration or delete of provider A
|
|
615
|
-
// must fence a concurrent lifecycle write for provider B in this Hub.
|
|
616
|
-
beginProviderStoreMutation() { return this.beginProviderMutation('__dsh-provider-store__'); }
|
|
617
|
-
endProviderStoreMutation() { this.endProviderMutation('__dsh-provider-store__'); }
|
|
618
|
-
hasProviderStoreMutation() { return this.providerMutations.has('__dsh-provider-store__'); }
|
|
586
|
+
list() {
|
|
587
|
+
return [...this.jobs.values()].map((job) => this.view(job));
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
beginProviderMutation(providerId) {
|
|
591
|
+
if (typeof providerId !== 'string' || !providerId.trim() || this.providerMutations.has(providerId)) return false;
|
|
592
|
+
this.providerMutationEpochs.set(providerId, (this.providerMutationEpochs.get(providerId) ?? 0) + 1);
|
|
593
|
+
this.providerMutations.add(providerId);
|
|
594
|
+
return true;
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
endProviderMutation(providerId) {
|
|
598
|
+
this.providerMutations.delete(providerId);
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
getProviderMutationEpoch(providerId) {
|
|
602
|
+
return this.providerMutationEpochs.get(providerId) ?? 0;
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
assertProviderNotMutating(providerId, expectedEpoch = undefined) {
|
|
606
|
+
if (this.providerMutations.has(providerId)
|
|
607
|
+
|| (expectedEpoch !== undefined && this.getProviderMutationEpoch(providerId) !== expectedEpoch)) {
|
|
608
|
+
throw Object.assign(new Error('provider mutation is active'), { code: 'PROVIDER_DELETE_BUSY' });
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
acquireProviderLease(providerId) {
|
|
613
|
+
this.providerLeases.set(providerId, (this.providerLeases.get(providerId) ?? 0) + 1);
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
releaseProviderLease(providerId) {
|
|
617
|
+
const count = this.providerLeases.get(providerId) ?? 0;
|
|
618
|
+
if (count <= 1) this.providerLeases.delete(providerId);
|
|
619
|
+
else this.providerLeases.set(providerId, count - 1);
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
hasProviderLease(providerId) {
|
|
623
|
+
return (this.providerLeases.get(providerId) ?? 0) > 0;
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
// Profile/settings are shared stores. A migration or delete of provider A
|
|
627
|
+
// must fence a concurrent lifecycle write for provider B in this Hub.
|
|
628
|
+
beginProviderStoreMutation() { return this.beginProviderMutation('__dsh-provider-store__'); }
|
|
629
|
+
endProviderStoreMutation() { this.endProviderMutation('__dsh-provider-store__'); }
|
|
630
|
+
hasProviderStoreMutation() { return this.providerMutations.has('__dsh-provider-store__'); }
|
|
619
631
|
|
|
620
632
|
publish() {
|
|
621
633
|
this.shard.publish([...this.jobs.values()].map((j) => this.view(j)));
|
|
@@ -629,7 +641,7 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
629
641
|
* `role` (worker | reviewer) records who does the work; `tier` remains the
|
|
630
642
|
* legacy model-class slot. Reviewer-role jobs always use the pro slot.
|
|
631
643
|
*/
|
|
632
|
-
async spawn({ task, tier = 'flash', role, attempt = 0, effort = 'max', cwd, source = 'api', preset, delivery = 'coding', client_job_id, requested_isolation, workspace_branch, timeout_seconds, profile_id, workspace_context, allow_no_changes, ingress = 'direct-3210' }) {
|
|
644
|
+
async spawn({ task, tier = 'flash', role, attempt = 0, effort = 'max', cwd, source = 'api', preset, delivery = 'coding', client_job_id, requested_isolation, workspace_branch, timeout_seconds, profile_id, workspace_context, allow_no_changes, ingress = 'direct-3210' }) {
|
|
633
645
|
// role is only honored when the caller explicitly names it; a legacy
|
|
634
646
|
// tier-only spawn (role === undefined) keeps the exact v0.1 resolution.
|
|
635
647
|
const hasRole = role === 'worker' || role === 'reviewer';
|
|
@@ -648,9 +660,9 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
648
660
|
// Provider routing: follow-dsh reads the DSH Models selection live; the
|
|
649
661
|
// default deepseek-official keeps legacy setups unchanged. Re-resolved on
|
|
650
662
|
// every spawn so a Models change takes effect on the next worker.
|
|
651
|
-
const cfg = normalizeGlobalConfig(this.getConfig?.() ?? {});
|
|
652
|
-
const lifecycleState = readProviderLifecycleState();
|
|
653
|
-
const healthGate = cfg.health_gate ?? cfg.worker?.model_policy?.health_gate;
|
|
663
|
+
const cfg = normalizeGlobalConfig(this.getConfig?.() ?? {});
|
|
664
|
+
const lifecycleState = readProviderLifecycleState();
|
|
665
|
+
const healthGate = cfg.health_gate ?? cfg.worker?.model_policy?.health_gate;
|
|
654
666
|
const workerProviderMode = normalizeWorkerProviderMode(cfg.worker_provider_mode);
|
|
655
667
|
const getCurrentSelection = () => this.ctx.get('agentDefaultModel')?.currentSelection?.();
|
|
656
668
|
let selection;
|
|
@@ -691,30 +703,30 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
691
703
|
// tier-only spawns resolve through the tier resolver exactly as before.
|
|
692
704
|
if (hasRole) {
|
|
693
705
|
const policy = resolveModelPolicy(cfg, effRole, { attempt });
|
|
694
|
-
selection = resolveModel({
|
|
695
|
-
role: effRole,
|
|
696
|
-
attempt,
|
|
697
|
-
policy,
|
|
698
|
-
catalog,
|
|
699
|
-
harnessDefault: catalog.harness_default ?? getCurrentSelection(),
|
|
700
|
-
healthStore: this.healthStore,
|
|
701
|
-
healthGate: policy.health_gate ?? healthGate,
|
|
702
|
-
allowFallback: cfg.allow_fallback !== false,
|
|
703
|
-
tombstones: lifecycleState.tombstones,
|
|
704
|
-
});
|
|
706
|
+
selection = resolveModel({
|
|
707
|
+
role: effRole,
|
|
708
|
+
attempt,
|
|
709
|
+
policy,
|
|
710
|
+
catalog,
|
|
711
|
+
harnessDefault: catalog.harness_default ?? getCurrentSelection(),
|
|
712
|
+
healthStore: this.healthStore,
|
|
713
|
+
healthGate: policy.health_gate ?? healthGate,
|
|
714
|
+
allowFallback: cfg.allow_fallback !== false,
|
|
715
|
+
tombstones: lifecycleState.tombstones,
|
|
716
|
+
});
|
|
705
717
|
} else {
|
|
706
718
|
selection = resolveWorkerModel({
|
|
707
719
|
tier: effTier,
|
|
708
720
|
priority: cfg[`${effTier}_model_priority`],
|
|
709
721
|
priorityConfigured: cfg[`${effTier}_model_priority_configured`],
|
|
710
722
|
fallback: cfg[`${effTier}_model_fallback`],
|
|
711
|
-
catalog,
|
|
712
|
-
harnessDefault: catalog.harness_default ?? getCurrentSelection(),
|
|
713
|
-
healthStore: this.healthStore,
|
|
714
|
-
healthGate,
|
|
715
|
-
allowFallback: cfg.allow_fallback !== false,
|
|
716
|
-
tombstones: lifecycleState.tombstones,
|
|
717
|
-
traceContext: {
|
|
723
|
+
catalog,
|
|
724
|
+
harnessDefault: catalog.harness_default ?? getCurrentSelection(),
|
|
725
|
+
healthStore: this.healthStore,
|
|
726
|
+
healthGate,
|
|
727
|
+
allowFallback: cfg.allow_fallback !== false,
|
|
728
|
+
tombstones: lifecycleState.tombstones,
|
|
729
|
+
traceContext: {
|
|
718
730
|
role: effRole,
|
|
719
731
|
logicalAttempt: attempt,
|
|
720
732
|
modelClassHint: effTier,
|
|
@@ -722,20 +734,20 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
722
734
|
candidateSet: attempt > 0 ? 'escalation' : 'primary',
|
|
723
735
|
},
|
|
724
736
|
});
|
|
725
|
-
}
|
|
726
|
-
if (!selection.ok) throw Object.assign(new Error(selection.message), { policyCode: selection.code });
|
|
727
|
-
}
|
|
728
|
-
const pendingLayerMigration = readProviderLayerMigrationTransactions(join(CONFIG_DIR, 'provider-migration-backups'))
|
|
729
|
-
.find((entry) => entry.unresolved === true || entry.provider_id === selection.provider);
|
|
730
|
-
if (pendingLayerMigration) throw Object.assign(new Error('provider layer migration is pending verification'), { code: 'PROVIDER_MIGRATION_PENDING' });
|
|
731
|
-
// A process restart clears the in-memory mutation fence, so persisted
|
|
732
|
-
// provider-delete transactions must also gate admission. Unknown-owner
|
|
733
|
-
// recovery entries fence globally; known entries fence their provider.
|
|
734
|
-
const pendingProviderDelete = readProviderRecoveryTransactions()
|
|
735
|
-
.find((entry) => entry.unresolved === true || entry.provider_id === selection.provider);
|
|
736
|
-
if (pendingProviderDelete) throw Object.assign(new Error('provider delete recovery is pending'), { code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
737
|
-
const providerMutationEpoch = this.getProviderMutationEpoch(selection.provider);
|
|
738
|
-
this.assertProviderNotMutating(selection.provider, providerMutationEpoch);
|
|
737
|
+
}
|
|
738
|
+
if (!selection.ok) throw Object.assign(new Error(selection.message), { policyCode: selection.code });
|
|
739
|
+
}
|
|
740
|
+
const pendingLayerMigration = readProviderLayerMigrationTransactions(join(CONFIG_DIR, 'provider-migration-backups'))
|
|
741
|
+
.find((entry) => entry.unresolved === true || entry.provider_id === selection.provider);
|
|
742
|
+
if (pendingLayerMigration) throw Object.assign(new Error('provider layer migration is pending verification'), { code: 'PROVIDER_MIGRATION_PENDING' });
|
|
743
|
+
// A process restart clears the in-memory mutation fence, so persisted
|
|
744
|
+
// provider-delete transactions must also gate admission. Unknown-owner
|
|
745
|
+
// recovery entries fence globally; known entries fence their provider.
|
|
746
|
+
const pendingProviderDelete = readProviderRecoveryTransactions()
|
|
747
|
+
.find((entry) => entry.unresolved === true || entry.provider_id === selection.provider);
|
|
748
|
+
if (pendingProviderDelete) throw Object.assign(new Error('provider delete recovery is pending'), { code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
749
|
+
const providerMutationEpoch = this.getProviderMutationEpoch(selection.provider);
|
|
750
|
+
this.assertProviderNotMutating(selection.provider, providerMutationEpoch);
|
|
739
751
|
|
|
740
752
|
// The worker always gets the auditable Delivery Contract appended (unless
|
|
741
753
|
// it already carries one), so its final message follows ## Diff / ## Tests
|
|
@@ -767,19 +779,19 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
767
779
|
id, client_job_id: client_job_id ?? null, sessionId, role: jobRole, attempt, tier: effTier, provider: selection.provider, model: selection.model,
|
|
768
780
|
selection_source: selection.source, selection_trace: selection.selection_trace ?? null,
|
|
769
781
|
effort, reasoning_effort: selection.reasoningEffort,
|
|
770
|
-
task, source, cwd: executionCwd, requested_cwd: cwd,
|
|
771
|
-
isolation: isolatedWorkspace ? 'worktree' : 'shared',
|
|
772
|
-
execution_context: (() => {
|
|
773
|
-
const runtime = getHubRuntimeIdentity();
|
|
774
|
-
return {
|
|
775
|
-
execution_plane: runtime.execution_plane,
|
|
776
|
-
profile: runtime.profile,
|
|
777
|
-
listen_port: runtime.listen_port,
|
|
778
|
-
runtime_id: runtime.runtime_id,
|
|
779
|
-
ingress: ingress === 'official-3080' ? 'official-3080' : 'direct-3210',
|
|
780
|
-
};
|
|
781
|
-
})(),
|
|
782
|
-
allow_no_changes: allow_no_changes === true,
|
|
782
|
+
task, source, cwd: executionCwd, requested_cwd: cwd,
|
|
783
|
+
isolation: isolatedWorkspace ? 'worktree' : 'shared',
|
|
784
|
+
execution_context: (() => {
|
|
785
|
+
const runtime = getHubRuntimeIdentity();
|
|
786
|
+
return {
|
|
787
|
+
execution_plane: runtime.execution_plane,
|
|
788
|
+
profile: runtime.profile,
|
|
789
|
+
listen_port: runtime.listen_port,
|
|
790
|
+
runtime_id: runtime.runtime_id,
|
|
791
|
+
ingress: ingress === 'official-3080' ? 'official-3080' : 'direct-3210',
|
|
792
|
+
};
|
|
793
|
+
})(),
|
|
794
|
+
allow_no_changes: allow_no_changes === true,
|
|
783
795
|
workspace_branch: workspace_branch ?? null, isolatedWorkspace,
|
|
784
796
|
profile_id: profile_id ?? null, workspace_context: workspace_context ?? null,
|
|
785
797
|
prompt: workerPrompt, delivery: delivery === 'review' ? 'review' : 'coding',
|
|
@@ -794,48 +806,48 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
794
806
|
// Abort latch: cancel()/timeout set this the moment the job becomes
|
|
795
807
|
// terminal, even while agents.create() is still pending, so a handle
|
|
796
808
|
// acquired after that point is disposed instead of being driven.
|
|
797
|
-
abortRequested: false,
|
|
798
|
-
abortPromise: null,
|
|
799
|
-
resolveAbort: null,
|
|
800
|
-
providerLease: false,
|
|
801
|
-
providerLeaseReleaseBlocked: false,
|
|
802
|
-
lateHandleCreated: false,
|
|
803
|
-
lateCreateCleanupPromise: null,
|
|
809
|
+
abortRequested: false,
|
|
810
|
+
abortPromise: null,
|
|
811
|
+
resolveAbort: null,
|
|
812
|
+
providerLease: false,
|
|
813
|
+
providerLeaseReleaseBlocked: false,
|
|
814
|
+
lateHandleCreated: false,
|
|
815
|
+
lateCreateCleanupPromise: null,
|
|
804
816
|
};
|
|
805
|
-
job.abortPromise = new Promise((resolve) => { job.resolveAbort = resolve; });
|
|
817
|
+
job.abortPromise = new Promise((resolve) => { job.resolveAbort = resolve; });
|
|
806
818
|
const disposeJobHandle = async () => {
|
|
807
819
|
if (!job.handle) return;
|
|
808
|
-
if (!job.handle_dispose_promise) {
|
|
809
|
-
try {
|
|
810
|
-
job.handle_dispose_promise = Promise.resolve(job.handle.dispose());
|
|
811
|
-
} catch (error) {
|
|
812
|
-
job.handle_dispose_promise = Promise.reject(error);
|
|
813
|
-
}
|
|
814
|
-
}
|
|
815
|
-
try {
|
|
816
|
-
await job.handle_dispose_promise;
|
|
817
|
-
} catch (error) {
|
|
818
|
-
// A failed disposer is not proof that the provider is idle. Keep the
|
|
819
|
-
// lease fenced even when the job is already terminal; the operator
|
|
820
|
-
// must restart/reclaim the owned runtime before deletion can proceed.
|
|
821
|
-
job.providerLeaseReleaseBlocked = true;
|
|
822
|
-
throw error;
|
|
823
|
-
}
|
|
820
|
+
if (!job.handle_dispose_promise) {
|
|
821
|
+
try {
|
|
822
|
+
job.handle_dispose_promise = Promise.resolve(job.handle.dispose());
|
|
823
|
+
} catch (error) {
|
|
824
|
+
job.handle_dispose_promise = Promise.reject(error);
|
|
825
|
+
}
|
|
826
|
+
}
|
|
827
|
+
try {
|
|
828
|
+
await job.handle_dispose_promise;
|
|
829
|
+
} catch (error) {
|
|
830
|
+
// A failed disposer is not proof that the provider is idle. Keep the
|
|
831
|
+
// lease fenced even when the job is already terminal; the operator
|
|
832
|
+
// must restart/reclaim the owned runtime before deletion can proceed.
|
|
833
|
+
job.providerLeaseReleaseBlocked = true;
|
|
834
|
+
throw error;
|
|
835
|
+
}
|
|
824
836
|
};
|
|
825
837
|
job.disposeHandle = disposeJobHandle;
|
|
826
838
|
// Read-only pre-run snapshot (async, never blocks dispatch): the audit
|
|
827
839
|
// only needs the before-state by the time the worker finishes. Non-repos
|
|
828
840
|
// degrade to { kind:'no-git' } instead of failing the job.
|
|
829
|
-
job.baseline = await captureWorkspaceBaseline({ cwd: executionCwd }).catch(() => ({ kind: 'no-git', reason: NOT_A_GIT_REPOSITORY, error: 'workspace audit failed' }));
|
|
830
|
-
try {
|
|
831
|
-
this.assertProviderNotMutating(selection.provider, providerMutationEpoch);
|
|
832
|
-
this.acquireProviderLease(selection.provider);
|
|
833
|
-
job.providerLease = true;
|
|
834
|
-
} catch (error) {
|
|
835
|
-
if (isolatedWorkspace) await cleanupIsolatedWorkspace({ repoRoot: isolatedWorkspace.repoRoot, worktreePath: isolatedWorkspace.worktreePath }).catch(() => {});
|
|
836
|
-
throw error;
|
|
837
|
-
}
|
|
838
|
-
this.jobs.set(id, job);
|
|
841
|
+
job.baseline = await captureWorkspaceBaseline({ cwd: executionCwd }).catch(() => ({ kind: 'no-git', reason: NOT_A_GIT_REPOSITORY, error: 'workspace audit failed' }));
|
|
842
|
+
try {
|
|
843
|
+
this.assertProviderNotMutating(selection.provider, providerMutationEpoch);
|
|
844
|
+
this.acquireProviderLease(selection.provider);
|
|
845
|
+
job.providerLease = true;
|
|
846
|
+
} catch (error) {
|
|
847
|
+
if (isolatedWorkspace) await cleanupIsolatedWorkspace({ repoRoot: isolatedWorkspace.repoRoot, worktreePath: isolatedWorkspace.worktreePath }).catch(() => {});
|
|
848
|
+
throw error;
|
|
849
|
+
}
|
|
850
|
+
this.jobs.set(id, job);
|
|
839
851
|
|
|
840
852
|
const onEvent = (session, event) => {
|
|
841
853
|
switch (event.type) {
|
|
@@ -866,7 +878,7 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
866
878
|
};
|
|
867
879
|
|
|
868
880
|
const run = async () => {
|
|
869
|
-
const createPromise = this.ctx.agents.create({
|
|
881
|
+
const createPromise = this.ctx.agents.create({
|
|
870
882
|
sessionId,
|
|
871
883
|
meta: { cwd: executionCwd, ...(presetId === undefined ? {} : { agentPreset: presetId }) },
|
|
872
884
|
agentOptions: { provider: selection.provider, model: selection.model },
|
|
@@ -880,23 +892,23 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
880
892
|
});
|
|
881
893
|
},
|
|
882
894
|
});
|
|
883
|
-
const created = await Promise.race([
|
|
884
|
-
createPromise.then((handle) => ({ handle })),
|
|
885
|
-
job.abortPromise.then(() => ({ aborted: true })),
|
|
886
|
-
]);
|
|
887
|
-
if (created.aborted) {
|
|
888
|
-
job.lateCreateCleanupPromise = createPromise.then(async (handle) => {
|
|
889
|
-
if (!handle) return;
|
|
890
|
-
job.lateHandleCreated = true;
|
|
891
|
-
job.handle = handle;
|
|
892
|
-
if (job.status !== 'running' || job.abortRequested) await job.disposeHandle();
|
|
893
|
-
}).catch((error) => {
|
|
894
|
-
job.cleanup_warning = `late agent creation failed: ${error?.message ?? String(error)}`;
|
|
895
|
-
throw error;
|
|
896
|
-
});
|
|
897
|
-
return;
|
|
898
|
-
}
|
|
899
|
-
const handle = created.handle;
|
|
895
|
+
const created = await Promise.race([
|
|
896
|
+
createPromise.then((handle) => ({ handle })),
|
|
897
|
+
job.abortPromise.then(() => ({ aborted: true })),
|
|
898
|
+
]);
|
|
899
|
+
if (created.aborted) {
|
|
900
|
+
job.lateCreateCleanupPromise = createPromise.then(async (handle) => {
|
|
901
|
+
if (!handle) return;
|
|
902
|
+
job.lateHandleCreated = true;
|
|
903
|
+
job.handle = handle;
|
|
904
|
+
if (job.status !== 'running' || job.abortRequested) await job.disposeHandle();
|
|
905
|
+
}).catch((error) => {
|
|
906
|
+
job.cleanup_warning = `late agent creation failed: ${error?.message ?? String(error)}`;
|
|
907
|
+
throw error;
|
|
908
|
+
});
|
|
909
|
+
return;
|
|
910
|
+
}
|
|
911
|
+
const handle = created.handle;
|
|
900
912
|
job.handle = handle;
|
|
901
913
|
// The job may have become terminal (cancel/timeout) while create() was
|
|
902
914
|
// pending; the old disposer was a no-op in that window. Dispose the
|
|
@@ -923,7 +935,7 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
923
935
|
// Cancel/timeout may have become terminal while waiting for the initial
|
|
924
936
|
// idle boundary; do not deliver the prompt to a job that is no longer
|
|
925
937
|
// running. The shared finally block still disposes the handle exactly once.
|
|
926
|
-
if (job.status !== 'running' || job.abortRequested) return;
|
|
938
|
+
if (job.status !== 'running' || job.abortRequested) return;
|
|
927
939
|
handle.agent.followup(userMessage(job.prompt));
|
|
928
940
|
await handle.agent.whenIdle();
|
|
929
941
|
job.result = job.lastAssistantText ?? '';
|
|
@@ -938,11 +950,11 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
938
950
|
if (job.status !== 'running') return;
|
|
939
951
|
job.abortRequested = true;
|
|
940
952
|
job.status = 'failed';
|
|
941
|
-
job.phase = JOB_PHASES.FAILED;
|
|
953
|
+
job.phase = JOB_PHASES.FAILED;
|
|
942
954
|
job.error = `job timed out after ${timeout_seconds}s`;
|
|
943
955
|
job.error_code = 'JOB_TIMEOUT';
|
|
944
956
|
job.endedAt = new Date().toISOString();
|
|
945
|
-
job.resolveAbort?.();
|
|
957
|
+
job.resolveAbort?.();
|
|
946
958
|
this.publish();
|
|
947
959
|
for (const waiter of job.waiters.splice(0)) waiter();
|
|
948
960
|
try { await disposeJobHandle(); } catch (error) {
|
|
@@ -962,23 +974,23 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
962
974
|
job.endedAt = new Date().toISOString();
|
|
963
975
|
job.currentTool = null;
|
|
964
976
|
let handleCleanupWarning = job.cleanup_warning ?? null;
|
|
965
|
-
try { await disposeJobHandle(); } catch (error) {
|
|
966
|
-
handleCleanupWarning = `agent cleanup failed: ${error?.message ?? String(error)}`;
|
|
967
|
-
}
|
|
968
|
-
if (job.providerLease) {
|
|
969
|
-
const releaseLease = () => {
|
|
970
|
-
if (!job.providerLease) return;
|
|
971
|
-
this.releaseProviderLease(job.provider);
|
|
972
|
-
job.providerLease = false;
|
|
973
|
-
};
|
|
974
|
-
if (job.lateCreateCleanupPromise) {
|
|
975
|
-
job.lateCreateCleanupPromise.then(releaseLease).catch(() => {
|
|
976
|
-
if (!job.lateHandleCreated && !job.providerLeaseReleaseBlocked) releaseLease();
|
|
977
|
-
// A late-created handle that failed disposal remains fenced by
|
|
978
|
-
// the provider lease until an operator restarts/reclaims it.
|
|
979
|
-
});
|
|
980
|
-
} else if (!job.providerLeaseReleaseBlocked) releaseLease();
|
|
981
|
-
}
|
|
977
|
+
try { await disposeJobHandle(); } catch (error) {
|
|
978
|
+
handleCleanupWarning = `agent cleanup failed: ${error?.message ?? String(error)}`;
|
|
979
|
+
}
|
|
980
|
+
if (job.providerLease) {
|
|
981
|
+
const releaseLease = () => {
|
|
982
|
+
if (!job.providerLease) return;
|
|
983
|
+
this.releaseProviderLease(job.provider);
|
|
984
|
+
job.providerLease = false;
|
|
985
|
+
};
|
|
986
|
+
if (job.lateCreateCleanupPromise) {
|
|
987
|
+
job.lateCreateCleanupPromise.then(releaseLease).catch(() => {
|
|
988
|
+
if (!job.lateHandleCreated && !job.providerLeaseReleaseBlocked) releaseLease();
|
|
989
|
+
// A late-created handle that failed disposal remains fenced by
|
|
990
|
+
// the provider lease until an operator restarts/reclaims it.
|
|
991
|
+
});
|
|
992
|
+
} else if (!job.providerLeaseReleaseBlocked) releaseLease();
|
|
993
|
+
}
|
|
982
994
|
// Delivery completeness is separate from execution status: a job can
|
|
983
995
|
// be done yet fail to report Diff/Tests/Risks (or Review sections for
|
|
984
996
|
// an automatic review). Parse whatever final message the worker
|
|
@@ -1014,23 +1026,23 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
1014
1026
|
job.workspaceDiff = job.baseline.kind === 'git'
|
|
1015
1027
|
? await captureWorkspaceDiff({ cwd: executionCwd, baseline: job.baseline }).catch(() => ({ kind: 'no-git', reason: NOT_A_GIT_REPOSITORY, error: 'workspace diff failed' }))
|
|
1016
1028
|
: job.baseline;
|
|
1017
|
-
job.outcome = applyHubWorkspaceEvidence({
|
|
1018
|
-
outcome: job.outcome,
|
|
1019
|
-
workspaceDiff: job.workspaceDiff,
|
|
1020
|
-
allowNoChanges: job.allow_no_changes,
|
|
1021
|
-
isolation: job.isolation,
|
|
1022
|
-
role: job.role,
|
|
1023
|
-
});
|
|
1024
|
-
// Record only sanitized callability evidence. Cancelled jobs are
|
|
1025
|
-
// operator actions, not provider health signals.
|
|
1026
|
-
if (job.status !== 'cancelled') {
|
|
1027
|
-
this.healthStore.record(job.provider, job.model, {
|
|
1028
|
-
ok: job.status === 'done',
|
|
1029
|
-
observed_at: Date.parse(job.endedAt ?? '') || Date.now(),
|
|
1030
|
-
error: { code: job.error_code, message: job.error },
|
|
1031
|
-
});
|
|
1032
|
-
}
|
|
1033
|
-
if (job.role === 'reviewer' && job.review && job.workspaceDiff?.kind === 'git') {
|
|
1029
|
+
job.outcome = applyHubWorkspaceEvidence({
|
|
1030
|
+
outcome: job.outcome,
|
|
1031
|
+
workspaceDiff: job.workspaceDiff,
|
|
1032
|
+
allowNoChanges: job.allow_no_changes,
|
|
1033
|
+
isolation: job.isolation,
|
|
1034
|
+
role: job.role,
|
|
1035
|
+
});
|
|
1036
|
+
// Record only sanitized callability evidence. Cancelled jobs are
|
|
1037
|
+
// operator actions, not provider health signals.
|
|
1038
|
+
if (job.status !== 'cancelled') {
|
|
1039
|
+
this.healthStore.record(job.provider, job.model, {
|
|
1040
|
+
ok: job.status === 'done',
|
|
1041
|
+
observed_at: Date.parse(job.endedAt ?? '') || Date.now(),
|
|
1042
|
+
error: { code: job.error_code, message: job.error },
|
|
1043
|
+
});
|
|
1044
|
+
}
|
|
1045
|
+
if (job.role === 'reviewer' && job.review && job.workspaceDiff?.kind === 'git') {
|
|
1034
1046
|
const changes = job.workspaceDiff.changes ?? {};
|
|
1035
1047
|
const mutated = ['modified', 'deleted', 'renamed', 'untracked'].some((key) => Array.isArray(changes[key]) && changes[key].length > 0);
|
|
1036
1048
|
if (mutated) {
|
|
@@ -1071,8 +1083,8 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
1071
1083
|
job.status = 'cancelled';
|
|
1072
1084
|
job.phase = JOB_PHASES.CANCELLED;
|
|
1073
1085
|
job.error = 'cancelled by request';
|
|
1074
|
-
job.resolveAbort?.();
|
|
1075
|
-
for (const waiter of job.waiters.splice(0)) waiter();
|
|
1086
|
+
job.resolveAbort?.();
|
|
1087
|
+
for (const waiter of job.waiters.splice(0)) waiter();
|
|
1076
1088
|
try { await job.disposeHandle?.(); } catch (error) {
|
|
1077
1089
|
job.cleanup_warning = `agent cleanup failed: ${error?.message ?? String(error)}`;
|
|
1078
1090
|
}
|
|
@@ -1236,10 +1248,10 @@ export function resolveHubSpawnPayload(payload, getConfig = () => ({}), dependen
|
|
|
1236
1248
|
|
|
1237
1249
|
// ---------- plugin entry ----------
|
|
1238
1250
|
|
|
1239
|
-
export async function apply(ctx) {
|
|
1240
|
-
seedLangFromHost(ctx);
|
|
1241
|
-
const hub = new WorkerRegistry(ctx);
|
|
1242
|
-
const builtInProviderProbe = createProviderProbe(ctx);
|
|
1251
|
+
export async function apply(ctx) {
|
|
1252
|
+
seedLangFromHost(ctx);
|
|
1253
|
+
const hub = new WorkerRegistry(ctx);
|
|
1254
|
+
const builtInProviderProbe = createProviderProbe(ctx);
|
|
1243
1255
|
try {
|
|
1244
1256
|
const { readGlobalConfig } = await import('../install/install.mjs');
|
|
1245
1257
|
hub.getConfig = () => readGlobalConfig();
|
|
@@ -1249,25 +1261,25 @@ export async function apply(ctx) {
|
|
|
1249
1261
|
// enforced its own), so the check uses the global config only. The route
|
|
1250
1262
|
// calls resolveHubSpawnPayload (above) to stamp the effective tier onto
|
|
1251
1263
|
// the spawn payload.
|
|
1252
|
-
const disposers = [];
|
|
1253
|
-
const PROVIDER_DELETE_PLAN_TTL_MS = 10 * 60 * 1000;
|
|
1254
|
-
const providerDeletePlans = new Map();
|
|
1255
|
-
const providerMigrationPlans = new Map();
|
|
1256
|
-
const providerMigrationOwners = new Map();
|
|
1257
|
-
const providerStoreMutationOwners = new Map();
|
|
1258
|
-
const credentialPurgePlans = new Map();
|
|
1259
|
-
const rememberProviderDeletePlan = (plan) => {
|
|
1260
|
-
providerDeletePlans.set(plan.plan_id, { plan, created_at: Date.now() });
|
|
1261
|
-
while (providerDeletePlans.size > 32) providerDeletePlans.delete(providerDeletePlans.keys().next().value);
|
|
1262
|
-
};
|
|
1263
|
-
const rememberProviderMigrationPlan = (plan) => {
|
|
1264
|
-
providerMigrationPlans.set(plan.plan_id, { plan, created_at: Date.now() });
|
|
1265
|
-
while (providerMigrationPlans.size > 32) providerMigrationPlans.delete(providerMigrationPlans.keys().next().value);
|
|
1266
|
-
};
|
|
1267
|
-
const rememberCredentialPurgePlan = (plan) => {
|
|
1268
|
-
credentialPurgePlans.set(plan.plan_id, { plan, created_at: Date.now() });
|
|
1269
|
-
while (credentialPurgePlans.size > 32) credentialPurgePlans.delete(credentialPurgePlans.keys().next().value);
|
|
1270
|
-
};
|
|
1264
|
+
const disposers = [];
|
|
1265
|
+
const PROVIDER_DELETE_PLAN_TTL_MS = 10 * 60 * 1000;
|
|
1266
|
+
const providerDeletePlans = new Map();
|
|
1267
|
+
const providerMigrationPlans = new Map();
|
|
1268
|
+
const providerMigrationOwners = new Map();
|
|
1269
|
+
const providerStoreMutationOwners = new Map();
|
|
1270
|
+
const credentialPurgePlans = new Map();
|
|
1271
|
+
const rememberProviderDeletePlan = (plan) => {
|
|
1272
|
+
providerDeletePlans.set(plan.plan_id, { plan, created_at: Date.now() });
|
|
1273
|
+
while (providerDeletePlans.size > 32) providerDeletePlans.delete(providerDeletePlans.keys().next().value);
|
|
1274
|
+
};
|
|
1275
|
+
const rememberProviderMigrationPlan = (plan) => {
|
|
1276
|
+
providerMigrationPlans.set(plan.plan_id, { plan, created_at: Date.now() });
|
|
1277
|
+
while (providerMigrationPlans.size > 32) providerMigrationPlans.delete(providerMigrationPlans.keys().next().value);
|
|
1278
|
+
};
|
|
1279
|
+
const rememberCredentialPurgePlan = (plan) => {
|
|
1280
|
+
credentialPurgePlans.set(plan.plan_id, { plan, created_at: Date.now() });
|
|
1281
|
+
while (credentialPurgePlans.size > 32) credentialPurgePlans.delete(credentialPurgePlans.keys().next().value);
|
|
1282
|
+
};
|
|
1271
1283
|
|
|
1272
1284
|
// Multimodal bridge: register describe_image / generate_image for the DS
|
|
1273
1285
|
// model. Config is read per call so settings-page edits apply live; the
|
|
@@ -1356,8 +1368,8 @@ export async function apply(ctx) {
|
|
|
1356
1368
|
// as its raw default (pro-only + no tier used to spawn flash).
|
|
1357
1369
|
const resolved = resolveHubSpawnPayload(payload, () => hub.getConfig?.() ?? {});
|
|
1358
1370
|
if (!resolved.ok) return sendJson(res, 400, { ok: false, error: resolved.error, code: resolved.code });
|
|
1359
|
-
const ingress = req.headers?.['x-dsh-crew-ingress'] === 'official-3080' ? 'official-3080' : 'direct-3210';
|
|
1360
|
-
const job = await hub.spawn({ ...resolved.payload, ingress });
|
|
1371
|
+
const ingress = req.headers?.['x-dsh-crew-ingress'] === 'official-3080' ? 'official-3080' : 'direct-3210';
|
|
1372
|
+
const job = await hub.spawn({ ...resolved.payload, ingress });
|
|
1361
1373
|
return sendJson(res, 200, { ok: true, job: hub.view(job) });
|
|
1362
1374
|
}
|
|
1363
1375
|
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'cancel') {
|
|
@@ -1437,25 +1449,25 @@ export async function apply(ctx) {
|
|
|
1437
1449
|
const workspaceReadiness = requestedWorkspaceId && !requestedContext
|
|
1438
1450
|
? { status: 'UNAVAILABLE', reason_code: 'WORKSPACE_CONTEXT_NOT_FOUND' }
|
|
1439
1451
|
: await assessWorkspaceReadiness({ cwd: requestedContext?.repo_root ?? null });
|
|
1440
|
-
const liveJobs = typeof hub.list === 'function' ? hub.list() : [];
|
|
1441
|
-
const executionRows = buildHubExecutionRows(liveJobs);
|
|
1442
|
-
const readinessMatrix = { rows: [
|
|
1443
|
-
{ id: 'hub_compatibility', status: 'PASS', reason_code: 'LIVE_CHECK_PASSED' },
|
|
1444
|
-
catalogStatus,
|
|
1445
|
-
...executionRows,
|
|
1446
|
-
] };
|
|
1447
|
-
const readinessSnapshot = buildRuntimeReadinessSnapshot({
|
|
1448
|
-
runtime: getHubRuntimeIdentity(),
|
|
1449
|
-
readinessMatrix,
|
|
1450
|
-
selections: { worker: liveJobs.find((job) => job?.role === 'worker') ?? null, reviewer: liveJobs.find((job) => job?.role === 'reviewer') ?? null },
|
|
1451
|
-
health: hub.healthStore.list(),
|
|
1452
|
-
jobs: liveJobs,
|
|
1453
|
-
workspace: workspaceReadiness,
|
|
1454
|
-
});
|
|
1455
|
-
const contract = buildExtensionContract({
|
|
1456
|
-
config,
|
|
1457
|
-
readinessMatrix,
|
|
1458
|
-
readinessSnapshot,
|
|
1452
|
+
const liveJobs = typeof hub.list === 'function' ? hub.list() : [];
|
|
1453
|
+
const executionRows = buildHubExecutionRows(liveJobs);
|
|
1454
|
+
const readinessMatrix = { rows: [
|
|
1455
|
+
{ id: 'hub_compatibility', status: 'PASS', reason_code: 'LIVE_CHECK_PASSED' },
|
|
1456
|
+
catalogStatus,
|
|
1457
|
+
...executionRows,
|
|
1458
|
+
] };
|
|
1459
|
+
const readinessSnapshot = buildRuntimeReadinessSnapshot({
|
|
1460
|
+
runtime: getHubRuntimeIdentity(),
|
|
1461
|
+
readinessMatrix,
|
|
1462
|
+
selections: { worker: liveJobs.find((job) => job?.role === 'worker') ?? null, reviewer: liveJobs.find((job) => job?.role === 'reviewer') ?? null },
|
|
1463
|
+
health: hub.healthStore.list(),
|
|
1464
|
+
jobs: liveJobs,
|
|
1465
|
+
workspace: workspaceReadiness,
|
|
1466
|
+
});
|
|
1467
|
+
const contract = buildExtensionContract({
|
|
1468
|
+
config,
|
|
1469
|
+
readinessMatrix,
|
|
1470
|
+
readinessSnapshot,
|
|
1459
1471
|
workspace: workspaceReadiness,
|
|
1460
1472
|
profiles,
|
|
1461
1473
|
runtime: getHubRuntimeIdentity(),
|
|
@@ -1482,6 +1494,111 @@ export async function apply(ctx) {
|
|
|
1482
1494
|
},
|
|
1483
1495
|
}));
|
|
1484
1496
|
|
|
1497
|
+
// Quick-config: the narrow, capability-projected write surface used by
|
|
1498
|
+
// the official 3080 quick-controls panel. Only the user-facing toggles
|
|
1499
|
+
// and model-priority lists are writable here; the FULL config endpoint
|
|
1500
|
+
// remains the single authority for everything else. This is a projection
|
|
1501
|
+
// over writeGlobalConfig, never a second config store.
|
|
1502
|
+
const projectQuickConfig = (config) => Object.fromEntries(
|
|
1503
|
+
[...QUICK_CONFIG_KEYS].map((key) => [key, config[key]]),
|
|
1504
|
+
);
|
|
1505
|
+
disposers.push(webServer.register({
|
|
1506
|
+
kind: 'exact', path: `${ROUTE_BASE}/quick-config`,
|
|
1507
|
+
handler: async (req, res) => {
|
|
1508
|
+
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
1509
|
+
try {
|
|
1510
|
+
const { readGlobalConfig, writeGlobalConfig } = await import(`../install/install.mjs?t=${Date.now()}`);
|
|
1511
|
+
if (req.method === 'GET') {
|
|
1512
|
+
return sendJson(res, 200, { ok: true, config: projectQuickConfig(readGlobalConfig()) });
|
|
1513
|
+
}
|
|
1514
|
+
if (req.method === 'POST') {
|
|
1515
|
+
const body = await readBody(req);
|
|
1516
|
+
const patch = {};
|
|
1517
|
+
for (const [key, value] of Object.entries(body ?? {})) {
|
|
1518
|
+
if (!QUICK_CONFIG_KEYS.includes(key)) {
|
|
1519
|
+
return sendJson(res, 403, { ok: false, code: 'QUICK_CONFIG_KEY_FORBIDDEN', error: `key not writable from the quick surface: ${key}` });
|
|
1520
|
+
}
|
|
1521
|
+
patch[key] = value;
|
|
1522
|
+
}
|
|
1523
|
+
const next = writeGlobalConfig(patch);
|
|
1524
|
+
return sendJson(res, 200, { ok: true, config: projectQuickConfig(next) });
|
|
1525
|
+
}
|
|
1526
|
+
return sendJson(res, 405, { ok: false, error: 'GET or POST' }, { allow: 'GET, POST' });
|
|
1527
|
+
} catch (err) {
|
|
1528
|
+
return sendJson(res, 500, { ok: false, error: err?.message ?? String(err) });
|
|
1529
|
+
}
|
|
1530
|
+
},
|
|
1531
|
+
}));
|
|
1532
|
+
|
|
1533
|
+
disposers.push(webServer.register({
|
|
1534
|
+
kind: 'exact', path: `${ROUTE_BASE}/quick-status`,
|
|
1535
|
+
handler: async (req, res) => {
|
|
1536
|
+
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
1537
|
+
if (req.method !== 'GET') return sendJson(res, 405, { ok: false, error: 'GET only' }, { allow: 'GET' });
|
|
1538
|
+
try {
|
|
1539
|
+
const { readGlobalConfig } = await import(`../install/install.mjs?t=${Date.now()}`);
|
|
1540
|
+
const runtime = getHubRuntimeIdentity();
|
|
1541
|
+
return sendJson(res, 200, {
|
|
1542
|
+
ok: true,
|
|
1543
|
+
ready: true,
|
|
1544
|
+
runtime: { runtime_version: runtime.runtime_version, dsh_version: runtime.dsh_version ?? null, runtime_id: runtime.runtime_id },
|
|
1545
|
+
config: projectQuickConfig(readGlobalConfig()),
|
|
1546
|
+
});
|
|
1547
|
+
} catch (err) {
|
|
1548
|
+
return sendJson(res, 500, { ok: false, error: err?.message ?? String(err) });
|
|
1549
|
+
}
|
|
1550
|
+
},
|
|
1551
|
+
}));
|
|
1552
|
+
|
|
1553
|
+
disposers.push(webServer.register({
|
|
1554
|
+
kind: 'exact', path: `${ROUTE_BASE}/runtime/restart-request`,
|
|
1555
|
+
handler: async (req, res) => {
|
|
1556
|
+
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
1557
|
+
if (req.method !== 'POST') return sendJson(res, 405, { ok: false, error: 'POST only' }, { allow: 'POST' });
|
|
1558
|
+
try {
|
|
1559
|
+
const { createRestartRequest, readSupervisorHeartbeat, supervisorStateRoot } = await import(`../supervisor/restart-request.mjs?t=${Date.now()}`);
|
|
1560
|
+
const appRoot = CONFIG_DIR;
|
|
1561
|
+
// The restart-request helpers take the Crew app root and append
|
|
1562
|
+
// supervisor/ internally — do NOT pre-compose the path here.
|
|
1563
|
+
const heartbeat = readSupervisorHeartbeat(appRoot);
|
|
1564
|
+
if (!heartbeat) {
|
|
1565
|
+
return sendJson(res, 503, { ok: false, code: 'CREW_SUPERVISOR_UNAVAILABLE', error: 'no fresh supervisor heartbeat; restart cannot be executed' });
|
|
1566
|
+
}
|
|
1567
|
+
let reason = null;
|
|
1568
|
+
try {
|
|
1569
|
+
const body = await readBody(req);
|
|
1570
|
+
reason = typeof body?.reason === 'string' && body.reason.length > 0 ? body.reason.slice(0, 200) : null;
|
|
1571
|
+
} catch { /* body optional */ }
|
|
1572
|
+
const created = createRestartRequest({ appRoot, runtimeIdentity: getHubRuntimeIdentity(), reason });
|
|
1573
|
+
if (!created.ok) return sendJson(res, 400, { ok: false, code: created.code, error: created.error });
|
|
1574
|
+
return sendJson(res, 202, { ok: true, state: 'RESTART_REQUESTED', request_id: created.request.request_id, expires_at: created.request.expires_at });
|
|
1575
|
+
} catch (err) {
|
|
1576
|
+
return sendJson(res, 500, { ok: false, error: err?.message ?? String(err) });
|
|
1577
|
+
}
|
|
1578
|
+
},
|
|
1579
|
+
}));
|
|
1580
|
+
|
|
1581
|
+
disposers.push(webServer.register({
|
|
1582
|
+
kind: 'exact', path: `${ROUTE_BASE}/runtime/restart-status`,
|
|
1583
|
+
handler: async (req, res) => {
|
|
1584
|
+
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
1585
|
+
if (req.method !== 'GET') return sendJson(res, 405, { ok: false, error: 'GET only' }, { allow: 'GET' });
|
|
1586
|
+
try {
|
|
1587
|
+
const { readRestartResult, readRestartRequest, supervisorStateRoot } = await import(`../supervisor/restart-request.mjs?t=${Date.now()}`);
|
|
1588
|
+
const appRoot = CONFIG_DIR;
|
|
1589
|
+
const requestId = new URL(req.url, 'http://localhost').searchParams.get('id');
|
|
1590
|
+
if (!requestId || !/^[0-9a-f-]{36}$/i.test(requestId)) return sendJson(res, 400, { ok: false, error: 'request id required' });
|
|
1591
|
+
const result = readRestartResult(appRoot, requestId);
|
|
1592
|
+
if (result) return sendJson(res, 200, { ok: true, state: result.state, request_id: result.request_id, detail: result.detail ?? null });
|
|
1593
|
+
const pending = readRestartRequest(appRoot, requestId);
|
|
1594
|
+
if (pending) return sendJson(res, 200, { ok: true, state: 'RESTART_REQUESTED', request_id: pending.request_id });
|
|
1595
|
+
return sendJson(res, 404, { ok: false, error: 'unknown restart request' });
|
|
1596
|
+
} catch (err) {
|
|
1597
|
+
return sendJson(res, 500, { ok: false, error: err?.message ?? String(err) });
|
|
1598
|
+
}
|
|
1599
|
+
},
|
|
1600
|
+
}));
|
|
1601
|
+
|
|
1485
1602
|
disposers.push(webServer.register({
|
|
1486
1603
|
kind: 'exact', path: `${ROUTE_BASE}/presets`,
|
|
1487
1604
|
handler: async (req, res) => {
|
|
@@ -1501,15 +1618,15 @@ export async function apply(ctx) {
|
|
|
1501
1618
|
},
|
|
1502
1619
|
}));
|
|
1503
1620
|
|
|
1504
|
-
disposers.push(webServer.register({
|
|
1505
|
-
kind: 'exact', path: `${ROUTE_BASE}/provider`,
|
|
1506
|
-
handler: async (req, res) => {
|
|
1507
|
-
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
1508
|
-
try {
|
|
1509
|
-
const config = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
1510
|
-
const lifecycleState = readProviderLifecycleState();
|
|
1511
|
-
const healthGate = config.health_gate ?? config.worker?.model_policy?.health_gate;
|
|
1512
|
-
const mode = normalizeWorkerProviderMode(config.worker_provider_mode);
|
|
1621
|
+
disposers.push(webServer.register({
|
|
1622
|
+
kind: 'exact', path: `${ROUTE_BASE}/provider`,
|
|
1623
|
+
handler: async (req, res) => {
|
|
1624
|
+
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
1625
|
+
try {
|
|
1626
|
+
const config = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
1627
|
+
const lifecycleState = readProviderLifecycleState();
|
|
1628
|
+
const healthGate = config.health_gate ?? config.worker?.model_policy?.health_gate;
|
|
1629
|
+
const mode = normalizeWorkerProviderMode(config.worker_provider_mode);
|
|
1513
1630
|
let selections;
|
|
1514
1631
|
if (mode === 'deepseek-official') {
|
|
1515
1632
|
selections = {
|
|
@@ -1527,14 +1644,14 @@ export async function apply(ctx) {
|
|
|
1527
1644
|
tier,
|
|
1528
1645
|
priority: config[`${tier}_model_priority`],
|
|
1529
1646
|
priorityConfigured: config[`${tier}_model_priority_configured`],
|
|
1530
|
-
fallback: config[`${tier}_model_fallback`],
|
|
1531
|
-
catalog,
|
|
1532
|
-
harnessDefault: catalog.harness_default,
|
|
1533
|
-
healthStore: hub.healthStore,
|
|
1534
|
-
healthGate,
|
|
1535
|
-
allowFallback: config.allow_fallback !== false,
|
|
1536
|
-
tombstones: lifecycleState.tombstones,
|
|
1537
|
-
});
|
|
1647
|
+
fallback: config[`${tier}_model_fallback`],
|
|
1648
|
+
catalog,
|
|
1649
|
+
harnessDefault: catalog.harness_default,
|
|
1650
|
+
healthStore: hub.healthStore,
|
|
1651
|
+
healthGate,
|
|
1652
|
+
allowFallback: config.allow_fallback !== false,
|
|
1653
|
+
tombstones: lifecycleState.tombstones,
|
|
1654
|
+
});
|
|
1538
1655
|
selections[tier] = selected.ok
|
|
1539
1656
|
? { provider: selected.provider, model: selected.model, source: selected.source }
|
|
1540
1657
|
: { code: selected.code, error: selected.message };
|
|
@@ -1549,829 +1666,829 @@ export async function apply(ctx) {
|
|
|
1549
1666
|
} catch (err) {
|
|
1550
1667
|
return sendJson(res, 503, { ok: false, code: 'MODEL_CATALOG_UNAVAILABLE', error: 'Unable to resolve Harness worker models.' });
|
|
1551
1668
|
}
|
|
1552
|
-
},
|
|
1553
|
-
}));
|
|
1554
|
-
|
|
1555
|
-
// Secret-free Harness provider inventory. This is deliberately separate
|
|
1556
|
-
// from /models (catalog) and /provider (effective routing): it reports
|
|
1557
|
-
// declaration provenance, ownership, lifecycle state and policy refs.
|
|
1558
|
-
disposers.push(webServer.register({
|
|
1559
|
-
kind: 'exact', path: `${ROUTE_BASE}/providers`,
|
|
1560
|
-
handler: async (req, res) => {
|
|
1561
|
-
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
1562
|
-
if (req.method !== 'GET') return sendJson(res, 405, { ok: false, error: 'GET only' }, { allow: 'GET' });
|
|
1563
|
-
try {
|
|
1564
|
-
const config = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
1565
|
-
const inventory = await readProviderInventorySnapshot(hub, ctx, config);
|
|
1566
|
-
return sendJson(res, 200, {
|
|
1567
|
-
ok: true,
|
|
1568
|
-
...inventory,
|
|
1569
|
-
runtime: getHubRuntimeIdentity(),
|
|
1570
|
-
});
|
|
1571
|
-
} catch {
|
|
1572
|
-
return sendJson(res, 503, { ok: false, code: 'PROVIDER_INVENTORY_UNAVAILABLE', error: 'Unable to read Harness provider inventory.' });
|
|
1573
|
-
}
|
|
1574
|
-
},
|
|
1575
|
-
}));
|
|
1576
|
-
|
|
1577
|
-
// Credential references are a separate, secret-free inventory. Provider
|
|
1578
|
-
// deletion can report these references, but never implies their purge.
|
|
1579
|
-
disposers.push(webServer.register({
|
|
1580
|
-
kind: 'exact', path: `${ROUTE_BASE}/credential-references`,
|
|
1581
|
-
handler: async (req, res) => {
|
|
1582
|
-
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
1583
|
-
if (req.method !== 'GET') return sendJson(res, 405, { ok: false, error: 'GET only' }, { allow: 'GET' });
|
|
1584
|
-
try {
|
|
1585
|
-
const config = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
1586
|
-
const snapshot = await readProviderInventorySnapshot(hub, ctx, config);
|
|
1587
|
-
const credentials = buildCredentialReferenceInventory({ providers: snapshot.records, additional_refs: snapshot.credential_history_refs, purged_refs: snapshot.credential_purged_refs });
|
|
1588
|
-
return sendJson(res, 200, { ok: true, ...credentials, unverified_purges: snapshot.credential_purge_unverified, runtime: getHubRuntimeIdentity() });
|
|
1589
|
-
} catch {
|
|
1590
|
-
return sendJson(res, 503, { ok: false, code: 'CREDENTIAL_REFERENCE_INVENTORY_UNAVAILABLE' });
|
|
1591
|
-
}
|
|
1592
|
-
},
|
|
1593
|
-
}));
|
|
1594
|
-
|
|
1595
|
-
disposers.push(webServer.register({
|
|
1596
|
-
kind: 'prefix', path: `${ROUTE_BASE}/credential-references`,
|
|
1597
|
-
handler: async (req, res) => {
|
|
1598
|
-
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
1599
|
-
let url;
|
|
1600
|
-
try { url = new URL(req.url, 'http://localhost'); } catch { return sendJson(res, 400, { ok: false, code: 'CREDENTIAL_PURGE_PLAN_INVALID' }); }
|
|
1601
|
-
const parts = url.pathname.slice(`${ROUTE_BASE}/credential-references`.length).split('/').filter(Boolean);
|
|
1602
|
-
try {
|
|
1603
|
-
const referenceId = decodeURIComponent(parts[0] ?? '');
|
|
1604
|
-
const body = await readBody(req);
|
|
1605
|
-
const config = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
1606
|
-
const snapshot = await readProviderInventorySnapshot(hub, ctx, config);
|
|
1607
|
-
const credentials = buildCredentialReferenceInventory({ providers: snapshot.records, additional_refs: snapshot.credential_history_refs, purged_refs: snapshot.credential_purged_refs });
|
|
1608
|
-
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'purge-plan') {
|
|
1609
|
-
const planned = planCredentialPurge({ inventory: credentials, referenceId, expectedRevision: body?.expected_revision });
|
|
1610
|
-
if (!planned.ok) return sendJson(res, 409, { ok: false, code: planned.code });
|
|
1611
|
-
rememberCredentialPurgePlan(planned.plan);
|
|
1612
|
-
return sendJson(res, 200, { ok: true, plan: planned.plan });
|
|
1613
|
-
}
|
|
1614
|
-
if (req.method === 'DELETE' && parts.length === 1) {
|
|
1615
|
-
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'CREDENTIAL_PURGE_CONFIRM_REQUIRED' });
|
|
1616
|
-
const stored = credentialPurgePlans.get(body?.plan_id);
|
|
1617
|
-
if (!stored || Date.now() - stored.created_at > PROVIDER_DELETE_PLAN_TTL_MS || stored.plan.reference_id !== referenceId) {
|
|
1618
|
-
credentialPurgePlans.delete(body?.plan_id);
|
|
1619
|
-
return sendJson(res, 409, { ok: false, code: 'CREDENTIAL_PURGE_PLAN_EXPIRED' });
|
|
1620
|
-
}
|
|
1621
|
-
const current = planCredentialPurge({ inventory: credentials, referenceId, expectedRevision: stored.plan.expected_revision });
|
|
1622
|
-
if (!current.ok) return sendJson(res, 409, { ok: false, code: current.code });
|
|
1623
|
-
if (current.plan.expected_revision !== stored.plan.expected_revision) return sendJson(res, 409, { ok: false, code: 'CREDENTIAL_REFERENCE_CHANGED' });
|
|
1624
|
-
credentialPurgePlans.delete(body.plan_id);
|
|
1625
|
-
const adapter = createCredentialPurgeFileHooks({
|
|
1626
|
-
credentialsFile: join(CONFIG_DIR, 'harness', '.credentials.yaml'),
|
|
1627
|
-
crewHome: join(CONFIG_DIR, 'harness'),
|
|
1628
|
-
});
|
|
1629
|
-
const result = await executeCredentialPurge(current.plan, {
|
|
1630
|
-
recheck: async (id) => {
|
|
1631
|
-
const freshSnapshot = await readProviderInventorySnapshot(hub, ctx, config);
|
|
1632
|
-
const freshCredentials = buildCredentialReferenceInventory({ providers: freshSnapshot.records, additional_refs: freshSnapshot.credential_history_refs, purged_refs: freshSnapshot.credential_purged_refs });
|
|
1633
|
-
const freshPlan = planCredentialPurge({ inventory: freshCredentials, referenceId: id });
|
|
1634
|
-
if (!freshPlan.ok) return { ok: false };
|
|
1635
|
-
const freshRecord = freshCredentials.records.find((entry) => entry.reference_id === id);
|
|
1636
|
-
return {
|
|
1637
|
-
ok: true,
|
|
1638
|
-
revision: freshPlan.plan.expected_revision,
|
|
1639
|
-
orphan: freshRecord?.orphan === true,
|
|
1640
|
-
ownership: freshRecord?.ownership,
|
|
1641
|
-
purge_capability: freshRecord?.purge_capability,
|
|
1642
|
-
};
|
|
1643
|
-
},
|
|
1644
|
-
...(adapter.ok ? { purge: adapter.purge, verify: adapter.verify } : {}),
|
|
1645
|
-
});
|
|
1646
|
-
if (result.state === 'PURGED' || result.state === 'VERIFIED') {
|
|
1647
|
-
try {
|
|
1648
|
-
writeCredentialPurgeState(recordCredentialPurgeOutcome(readCredentialPurgeState(), current.plan, result));
|
|
1649
|
-
} catch {
|
|
1650
|
-
return sendJson(res, 503, { ok: false, code: 'CREDENTIAL_PURGE_AUDIT_FAILED', state: result.state, reference_id: referenceId });
|
|
1651
|
-
}
|
|
1652
|
-
}
|
|
1653
|
-
return sendJson(res, result.ok ? 200 : 503, result);
|
|
1654
|
-
}
|
|
1655
|
-
return sendJson(res, 404, { ok: false, code: 'CREDENTIAL_REFERENCE_ENDPOINT_NOT_FOUND' });
|
|
1656
|
-
} catch (error) {
|
|
1657
|
-
return sendJson(res, 400, { ok: false, code: boundedMachineCodeFromError(error) ?? 'CREDENTIAL_PURGE_PLAN_INVALID' });
|
|
1658
|
-
}
|
|
1659
|
-
},
|
|
1660
|
-
}));
|
|
1661
|
-
|
|
1662
|
-
// Deletion is two-phase: this endpoint only computes a revision-bound,
|
|
1663
|
-
// secret-free impact plan. A later mutation endpoint must present that
|
|
1664
|
-
// plan and perform restart/verification through an owned supervisor.
|
|
1665
|
-
disposers.push(webServer.register({
|
|
1666
|
-
kind: 'prefix', path: `${ROUTE_BASE}/providers`,
|
|
1667
|
-
handler: async (req, res) => {
|
|
1668
|
-
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
1669
|
-
let url;
|
|
1670
|
-
try { url = new URL(req.url, 'http://localhost'); } catch { return sendJson(res, 400, { ok: false, code: 'PROVIDER_PLAN_INVALID' }); }
|
|
1671
|
-
const parts = url.pathname.slice(`${ROUTE_BASE}/providers`.length).split('/').filter(Boolean);
|
|
1672
|
-
try {
|
|
1673
|
-
if (req.method === 'GET' && parts.length === 1 && parts[0] === 'migration-status') {
|
|
1674
|
-
const config = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
1675
|
-
const inventory = await readProviderInventorySnapshot(hub, ctx, config);
|
|
1676
|
-
return sendJson(res, 200, { ok: true, ...inventory.migration, runtime: getHubRuntimeIdentity() });
|
|
1677
|
-
}
|
|
1678
|
-
const providerId = decodeURIComponent(parts[0]);
|
|
1679
|
-
const body = await readBody(req);
|
|
1680
|
-
if (req.method === 'POST' && parts[0] === '_recovery' && parts[1] === 'lock' && parts.length === 2) {
|
|
1681
|
-
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_CONFIRM_REQUIRED' });
|
|
1682
|
-
const hooks = createProviderDeleteFileHooks({
|
|
1683
|
-
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1684
|
-
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1685
|
-
configFile: join(CONFIG_DIR, 'config.json'),
|
|
1686
|
-
lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
1687
|
-
backupDir: join(CONFIG_DIR, 'provider-backups'),
|
|
1688
|
-
sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
|
|
1689
|
-
});
|
|
1690
|
-
const recovered = await hooks.recoverLock({ force: body?.force === true });
|
|
1691
|
-
const migrationHooks = createProviderLayerMigrationFileHooks({
|
|
1692
|
-
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1693
|
-
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1694
|
-
backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
|
|
1695
|
-
});
|
|
1696
|
-
const migrationRecovered = await migrationHooks.recoverLock({ confirm: true });
|
|
1697
|
-
const ok = recovered.ok === true && migrationRecovered.ok === true;
|
|
1698
|
-
return sendJson(res, ok ? 200 : 409, { ok, delete_lock: recovered, migration_lock: migrationRecovered });
|
|
1699
|
-
}
|
|
1700
|
-
if (req.method === 'POST' && parts[0] === '_recovery' && parts[1] === 'quarantine' && parts.length === 2) {
|
|
1701
|
-
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_QUARANTINE_CONFIRM_REQUIRED' });
|
|
1702
|
-
const actionId = typeof body?.action_id === 'string' && RECOVERY_ACTION_PATTERN.test(body.action_id)
|
|
1703
|
-
? body.action_id : null;
|
|
1704
|
-
if (!actionId) return sendJson(res, 400, { ok: false, code: 'PROVIDER_DELETE_PLAN_INVALID' });
|
|
1705
|
-
const recovery = [
|
|
1706
|
-
...readProviderRecoveryTransactions(),
|
|
1707
|
-
...readProviderLayerMigrationTransactions(join(CONFIG_DIR, 'provider-migration-backups')),
|
|
1708
|
-
].find((entry) => entry.action_id === actionId);
|
|
1709
|
-
if (!recovery) return sendJson(res, 404, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_NOT_FOUND' });
|
|
1710
|
-
if (recovery.recoverable === true) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_RECOVERABLE' });
|
|
1711
|
-
if (typeof recovery.storage_id !== 'string') return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_QUARANTINE_FAILED' });
|
|
1712
|
-
if (recovery.source === 'provider-layer-migration') {
|
|
1713
|
-
try {
|
|
1714
|
-
const migrationHooks = createProviderLayerMigrationFileHooks({
|
|
1715
|
-
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1716
|
-
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1717
|
-
backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
|
|
1718
|
-
});
|
|
1719
|
-
const quarantined = await migrationHooks.quarantine(actionId);
|
|
1720
|
-
return sendJson(res, 200, { ...quarantined, transaction_id: recovery.transaction_id ?? null });
|
|
1721
|
-
} catch (error) {
|
|
1722
|
-
const code = boundedMachineCodeFromError(error) ?? 'PROVIDER_MIGRATION_RECOVERY_QUARANTINE_FAILED';
|
|
1723
|
-
return sendJson(res, 409, { ok: false, code });
|
|
1724
|
-
}
|
|
1725
|
-
}
|
|
1726
|
-
try {
|
|
1727
|
-
const hooks = createProviderDeleteFileHooks({
|
|
1728
|
-
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1729
|
-
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1730
|
-
configFile: join(CONFIG_DIR, 'config.json'),
|
|
1731
|
-
lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
1732
|
-
backupDir: join(CONFIG_DIR, 'provider-backups'),
|
|
1733
|
-
sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
|
|
1734
|
-
afterLockAcquired: () => {
|
|
1735
|
-
const current = readProviderRecoveryTransactions().find((entry) => entry.action_id === actionId);
|
|
1736
|
-
if (!current || current.recoverable === true || current.storage_id !== recovery.storage_id) {
|
|
1737
|
-
throw Object.assign(new Error('provider recovery entry changed before quarantine'), { code: 'PROVIDER_DELETE_STATE_CHANGED' });
|
|
1738
|
-
}
|
|
1739
|
-
},
|
|
1740
|
-
});
|
|
1741
|
-
const quarantined = await hooks.quarantine(recovery.storage_id);
|
|
1742
|
-
return sendJson(res, 200, { ...quarantined, action_id: actionId, transaction_id: recovery.transaction_id ?? null });
|
|
1743
|
-
} catch (error) {
|
|
1744
|
-
const code = boundedMachineCodeFromError(error) ?? 'PROVIDER_DELETE_RECOVERY_QUARANTINE_FAILED';
|
|
1745
|
-
return sendJson(res, 409, { ok: false, code });
|
|
1746
|
-
}
|
|
1747
|
-
}
|
|
1748
|
-
const config = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
1749
|
-
const inventory = await readProviderInventorySnapshot(hub, ctx, config);
|
|
1750
|
-
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'migrate-plan') {
|
|
1751
|
-
if (!hasCompleteProviderCatalogEvidence(inventory.catalog_evidence)) return sendJson(res, 503, { ok: false, code: 'MODEL_CATALOG_UNAVAILABLE' });
|
|
1752
|
-
if (!hasCompleteProviderDeclarationEvidence(inventory.declaration_evidence)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_SOURCE_UNRESOLVED' });
|
|
1753
|
-
if (!hasAvailableProviderLifecycleEvidence(inventory.lifecycle_evidence)) return sendJson(res, 503, { ok: false, code: 'PROVIDER_LIFECYCLE_UNAVAILABLE' });
|
|
1754
|
-
if (inventory.default_evidence?.ok !== true) return sendJson(res, 409, { ok: false, code: inventory.default_evidence?.code ?? 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' });
|
|
1755
|
-
if (hasPendingProviderRecoveryTransactions(inventory)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
1756
|
-
const entry = inventory.migration.providers.find((candidate) => candidate.provider_id === providerId);
|
|
1757
|
-
if (!entry) return sendJson(res, 404, { ok: false, code: 'PROVIDER_MIGRATION_NOT_REQUIRED' });
|
|
1758
|
-
if (entry.action === 'blocked' || entry.action === 'collision-review') return sendJson(res, 409, { ok: false, code: entry.blocked_reason ?? 'PROVIDER_MIGRATION_BLOCKED' });
|
|
1759
|
-
const revisions = readProviderSourceRevisions();
|
|
1760
|
-
if (!revisions.profile.ok) return sendJson(res, 409, { ok: false, code: revisions.profile.code ?? 'PROVIDER_PROFILE_CHANGED' });
|
|
1761
|
-
if ((entry.action === 'promote-existing-user' || entry.harness_default) && !revisions.settings.ok) return sendJson(res, 409, { ok: false, code: revisions.settings.code ?? 'PROVIDER_SETTINGS_CHANGED' });
|
|
1762
|
-
const profileFile = join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml');
|
|
1763
|
-
const parsedMaterialization = readProviderMaterialization(readFileSync(profileFile, 'utf8'), { providerId, file: 'harness/profiles/dsh-crew/cordis.patch.yml' });
|
|
1764
|
-
if (!parsedMaterialization.ok) return sendJson(res, 409, { ok: false, code: parsedMaterialization.code });
|
|
1765
|
-
const materialization = parsedMaterialization.provider;
|
|
1766
|
-
let userMaterializationBefore;
|
|
1767
|
-
if (entry.action === 'promote-existing-user') {
|
|
1768
|
-
const settingsFile = join(CONFIG_DIR, 'harness', 'settings.yaml');
|
|
1769
|
-
if (!existsSync(settingsFile)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_SETTINGS_CHANGED' });
|
|
1770
|
-
const parsedUserMaterialization = readProviderSettingsMaterialization(readFileSync(settingsFile, 'utf8'), { providerId, file: 'harness/settings.yaml' });
|
|
1771
|
-
if (!parsedUserMaterialization.ok) return sendJson(res, 409, { ok: false, code: parsedUserMaterialization.code });
|
|
1772
|
-
userMaterializationBefore = parsedUserMaterialization.provider;
|
|
1773
|
-
}
|
|
1774
|
-
const plan = {
|
|
1775
|
-
schema_version: 1,
|
|
1776
|
-
kind: 'provider-layer-migration',
|
|
1777
|
-
plan_id: randomUUID(),
|
|
1778
|
-
provider_id: providerId,
|
|
1779
|
-
action: entry.action,
|
|
1780
|
-
source: entry.source,
|
|
1781
|
-
routing_references: entry.referenced_by,
|
|
1782
|
-
materialization: { provider: materialization },
|
|
1783
|
-
...(userMaterializationBefore ? { user_materialization_before: userMaterializationBefore } : {}),
|
|
1784
|
-
expected_revisions: {
|
|
1785
|
-
profile: revisions.profile.revision,
|
|
1786
|
-
settings: revisions.settings.ok ? revisions.settings.revision : null,
|
|
1787
|
-
},
|
|
1788
|
-
harness_default_before: inventory.harness_default ?? null,
|
|
1789
|
-
runtime_id_before: getHubRuntimeIdentity().runtime_id,
|
|
1790
|
-
restart_required: true,
|
|
1791
|
-
};
|
|
1792
|
-
rememberProviderMigrationPlan(plan);
|
|
1793
|
-
return sendJson(res, 200, { ok: true, plan });
|
|
1794
|
-
}
|
|
1795
|
-
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'migrate') {
|
|
1796
|
-
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_MIGRATION_CONFIRM_REQUIRED' });
|
|
1797
|
-
const stored = providerMigrationPlans.get(body?.plan_id);
|
|
1798
|
-
if (!stored || Date.now() - stored.created_at > PROVIDER_DELETE_PLAN_TTL_MS || stored.plan.provider_id !== providerId) {
|
|
1799
|
-
providerMigrationPlans.delete(body?.plan_id);
|
|
1800
|
-
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_PLAN_EXPIRED' });
|
|
1801
|
-
}
|
|
1802
|
-
const entry = inventory.migration.providers.find((candidate) => candidate.provider_id === providerId);
|
|
1803
|
-
if (!entry || entry.action !== stored.plan.action || entry.action === 'blocked' || entry.action === 'collision-review') return sendJson(res, 409, { ok: false, code: entry?.blocked_reason ?? 'PROVIDER_MIGRATION_PLAN_CHANGED' });
|
|
1804
|
-
if (inventory.default_evidence?.ok !== true) return sendJson(res, 409, { ok: false, code: inventory.default_evidence?.code ?? 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' });
|
|
1805
|
-
const revisions = readProviderSourceRevisions();
|
|
1806
|
-
const expected = stored.plan.expected_revisions ?? {};
|
|
1807
|
-
if (revisions.profile.revision !== expected.profile || (revisions.settings.ok ? revisions.settings.revision : null) !== expected.settings) {
|
|
1808
|
-
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_SOURCE_CHANGED' });
|
|
1809
|
-
}
|
|
1810
|
-
const hooks = createProviderLayerMigrationFileHooks({
|
|
1811
|
-
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1812
|
-
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1813
|
-
backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
|
|
1814
|
-
});
|
|
1815
|
-
let result;
|
|
1816
|
-
if (hub.hasProviderLease(providerId)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_IN_USE' });
|
|
1817
|
-
if (!hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
1818
|
-
if (!hub.beginProviderMutation(providerId)) {
|
|
1819
|
-
hub.endProviderStoreMutation();
|
|
1820
|
-
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
1821
|
-
}
|
|
1822
|
-
let retainProviderMutation = false;
|
|
1823
|
-
let retainProviderStoreMutation = false;
|
|
1824
|
-
try {
|
|
1825
|
-
await hooks.acquireLock();
|
|
1826
|
-
// The plan was created from a pre-lock snapshot. Re-read every
|
|
1827
|
-
// authoritative source after taking the shared store lock so a
|
|
1828
|
-
// concurrent catalog/default/recovery change cannot be hidden
|
|
1829
|
-
// behind unchanged profile/settings revisions.
|
|
1830
|
-
const lockedConfig = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
1831
|
-
const lockedInventory = await readProviderInventorySnapshot(hub, ctx, lockedConfig);
|
|
1832
|
-
const lockedEntry = lockedInventory.migration.providers.find((candidate) => candidate.provider_id === providerId);
|
|
1833
|
-
const expectedRouting = Array.isArray(stored.plan.routing_references) ? stored.plan.routing_references : [];
|
|
1834
|
-
const sameRouting = JSON.stringify(lockedEntry?.referenced_by ?? []) === JSON.stringify(expectedRouting);
|
|
1835
|
-
const sameDefault = JSON.stringify(lockedInventory.harness_default ?? null) === JSON.stringify(stored.plan.harness_default_before ?? null);
|
|
1836
|
-
const profileFile = join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml');
|
|
1837
|
-
const lockedMaterialization = existsSync(profileFile)
|
|
1838
|
-
? readProviderMaterialization(readFileSync(profileFile, 'utf8'), { providerId, file: 'harness/profiles/dsh-crew/cordis.patch.yml' })
|
|
1839
|
-
: { ok: false };
|
|
1840
|
-
const sameMaterialization = lockedMaterialization.ok === true
|
|
1841
|
-
&& JSON.stringify(lockedMaterialization.provider) === JSON.stringify(stored.plan.materialization?.provider ?? null);
|
|
1842
|
-
const settingsFile = join(CONFIG_DIR, 'harness', 'settings.yaml');
|
|
1843
|
-
const lockedUserMaterialization = stored.plan.action === 'promote-existing-user' && existsSync(settingsFile)
|
|
1844
|
-
? readProviderSettingsMaterialization(readFileSync(settingsFile, 'utf8'), { providerId, file: 'harness/settings.yaml' })
|
|
1845
|
-
: { ok: stored.plan.action !== 'promote-existing-user' };
|
|
1846
|
-
const sameUserMaterialization = stored.plan.action !== 'promote-existing-user'
|
|
1847
|
-
|| (lockedUserMaterialization.ok === true && JSON.stringify(lockedUserMaterialization.provider) === JSON.stringify(stored.plan.user_materialization_before ?? null));
|
|
1848
|
-
const lockedRevisions = readProviderSourceRevisions();
|
|
1849
|
-
const expectedLocked = stored.plan.expected_revisions ?? {};
|
|
1850
|
-
const sameRevisions = lockedRevisions.profile.revision === expectedLocked.profile
|
|
1851
|
-
&& (lockedRevisions.settings.ok ? lockedRevisions.settings.revision : null) === expectedLocked.settings;
|
|
1852
|
-
if (hasPendingProviderRecoveryTransactions(lockedInventory)) {
|
|
1853
|
-
return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
1854
|
-
}
|
|
1855
|
-
if (!lockedEntry || lockedEntry.action !== stored.plan.action || lockedEntry.collision !== null
|
|
1856
|
-
|| !sameRouting || !sameDefault || !sameMaterialization || !sameUserMaterialization || !sameRevisions
|
|
1857
|
-
|| lockedInventory.default_evidence?.ok !== true
|
|
1858
|
-
|| !hasCompleteProviderCatalogEvidence(lockedInventory.catalog_evidence)
|
|
1859
|
-
|| !hasCompleteProviderDeclarationEvidence(lockedInventory.declaration_evidence)
|
|
1860
|
-
|| !hasAvailableProviderLifecycleEvidence(lockedInventory.lifecycle_evidence)) {
|
|
1861
|
-
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_PLAN_CHANGED' });
|
|
1862
|
-
}
|
|
1863
|
-
if (hub.hasProviderLease(providerId)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_IN_USE' });
|
|
1864
|
-
result = await executeProviderLayerMigration(stored.plan, hooks, { confirm: true, deferRestart: true });
|
|
1865
|
-
if (result.ok && result.state === 'RESTART_PENDING') await hooks.setRestartRuntimeBaseline(getHubRuntimeIdentity().runtime_id);
|
|
1866
|
-
providerMigrationPlans.delete(body.plan_id);
|
|
1867
|
-
retainProviderMutation = result.state === 'RESTART_PENDING';
|
|
1868
|
-
if (retainProviderMutation) {
|
|
1869
|
-
providerMigrationOwners.set(providerId, stored.plan.plan_id);
|
|
1870
|
-
providerStoreMutationOwners.set(stored.plan.plan_id, providerId);
|
|
1871
|
-
retainProviderStoreMutation = true;
|
|
1872
|
-
}
|
|
1873
|
-
return sendJson(res, result.ok ? 202 : 409, { ok: result.ok, restart_required: result.restart_required === true, result });
|
|
1874
|
-
} finally {
|
|
1875
|
-
await hooks.release();
|
|
1876
|
-
if (!retainProviderMutation) hub.endProviderMutation(providerId);
|
|
1877
|
-
if (!retainProviderStoreMutation) hub.endProviderStoreMutation();
|
|
1878
|
-
}
|
|
1879
|
-
}
|
|
1880
|
-
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'verify-migration') {
|
|
1881
|
-
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_MIGRATION_VERIFY_CONFIRM_REQUIRED' });
|
|
1882
|
-
const transactionId = body?.transaction_id;
|
|
1883
|
-
const hooks = createProviderLayerMigrationFileHooks({
|
|
1884
|
-
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1885
|
-
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1886
|
-
backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
|
|
1887
|
-
existingMigrationId: transactionId,
|
|
1888
|
-
});
|
|
1889
|
-
let acquiredStoreFence = false;
|
|
1890
|
-
try {
|
|
1891
|
-
await hooks.acquireLock();
|
|
1892
|
-
await hooks.backup({});
|
|
1893
|
-
const plan = hooks.backupPlan();
|
|
1894
|
-
if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_PLAN_PROVIDER_MISMATCH' });
|
|
1895
|
-
const owner = providerMigrationOwners.get(providerId);
|
|
1896
|
-
if (providerStoreMutationOwners.get(plan.plan_id) !== providerId || !hub.hasProviderStoreMutation()) {
|
|
1897
|
-
if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
1898
|
-
providerStoreMutationOwners.set(plan.plan_id, providerId);
|
|
1899
|
-
acquiredStoreFence = true;
|
|
1900
|
-
}
|
|
1901
|
-
if (owner !== plan.plan_id) {
|
|
1902
|
-
if (hub.hasProviderLease(providerId) || !hub.beginProviderMutation(providerId)) {
|
|
1903
|
-
if (acquiredStoreFence && providerStoreMutationOwners.get(plan.plan_id) === providerId) { providerStoreMutationOwners.delete(plan.plan_id); hub.endProviderStoreMutation(); }
|
|
1904
|
-
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
1905
|
-
}
|
|
1906
|
-
providerMigrationOwners.set(providerId, plan.plan_id);
|
|
1907
|
-
}
|
|
1908
|
-
const verification = await hooks.verify(plan);
|
|
1909
|
-
const live = await readProviderInventorySnapshot(hub, ctx, config);
|
|
1910
|
-
verification.catalogEvidence = live.catalog_evidence;
|
|
1911
|
-
verification.defaultEvidence = live.default_evidence;
|
|
1912
|
-
verification.defaultPreserved = live.default_evidence?.ok === true
|
|
1913
|
-
&& JSON.stringify(live.harness_default ?? null) === JSON.stringify(plan.harness_default_before ?? null);
|
|
1914
|
-
verification.liveNativeRemovable = hasCompleteProviderCatalogEvidence(live.catalog_evidence)
|
|
1915
|
-
&& live.records.some((record) => record.id === providerId && record.lifecycle?.catalogued === true && record.native_removable === true);
|
|
1916
|
-
verification.runtimeRestarted = hasProviderRuntimeRestartEvidence(plan.runtime_id_before_restart ?? plan.runtime_id_before, getHubRuntimeIdentity().runtime_id);
|
|
1917
|
-
if (verification.nativeRemovable !== true || verification.liveNativeRemovable !== true || verification.defaultPreserved !== true || verification.runtimeRestarted !== true) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_VERIFY_FAILED', verification });
|
|
1918
|
-
await hooks.finalizeVerified();
|
|
1919
|
-
if (providerMigrationOwners.get(providerId) === plan.plan_id) {
|
|
1920
|
-
providerMigrationOwners.delete(providerId);
|
|
1921
|
-
hub.endProviderMutation(providerId);
|
|
1922
|
-
}
|
|
1923
|
-
if (providerStoreMutationOwners.get(plan.plan_id) === providerId) {
|
|
1924
|
-
providerStoreMutationOwners.delete(plan.plan_id);
|
|
1925
|
-
hub.endProviderStoreMutation();
|
|
1926
|
-
}
|
|
1927
|
-
return sendJson(res, 200, { ok: true, state: 'VERIFIED', transaction_id: plan.plan_id, provider_id: providerId, verification });
|
|
1928
|
-
} finally {
|
|
1929
|
-
await hooks.release();
|
|
1930
|
-
// Keep the transaction-owned fence on verification failure so
|
|
1931
|
-
// an incomplete migration cannot race a new Worker; the caller
|
|
1932
|
-
// must explicitly retry verification or roll it back.
|
|
1933
|
-
}
|
|
1934
|
-
}
|
|
1935
|
-
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'rollback-migration') {
|
|
1936
|
-
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_MIGRATION_ROLLBACK_CONFIRM_REQUIRED' });
|
|
1937
|
-
const hooks = createProviderLayerMigrationFileHooks({
|
|
1938
|
-
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1939
|
-
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1940
|
-
backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
|
|
1941
|
-
existingMigrationId: body?.transaction_id,
|
|
1942
|
-
});
|
|
1943
|
-
let acquiredMigrationFence = false;
|
|
1944
|
-
let retainProviderMutation = false;
|
|
1945
|
-
let acquiredStoreFence = false;
|
|
1946
|
-
try {
|
|
1947
|
-
await hooks.acquireLock();
|
|
1948
|
-
await hooks.backup({});
|
|
1949
|
-
const plan = hooks.backupPlan();
|
|
1950
|
-
if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_PLAN_PROVIDER_MISMATCH' });
|
|
1951
|
-
const owner = providerMigrationOwners.get(providerId);
|
|
1952
|
-
if (providerStoreMutationOwners.get(plan.plan_id) !== providerId || !hub.hasProviderStoreMutation()) {
|
|
1953
|
-
if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
1954
|
-
providerStoreMutationOwners.set(plan.plan_id, providerId);
|
|
1955
|
-
acquiredStoreFence = true;
|
|
1956
|
-
}
|
|
1957
|
-
if (owner !== plan.plan_id) {
|
|
1958
|
-
if (hub.hasProviderLease(providerId) || !hub.beginProviderMutation(providerId)) {
|
|
1959
|
-
if (acquiredStoreFence && providerStoreMutationOwners.get(plan.plan_id) === providerId) { providerStoreMutationOwners.delete(plan.plan_id); hub.endProviderStoreMutation(); }
|
|
1960
|
-
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
1961
|
-
}
|
|
1962
|
-
providerMigrationOwners.set(providerId, plan.plan_id);
|
|
1963
|
-
acquiredMigrationFence = true;
|
|
1964
|
-
retainProviderMutation = true;
|
|
1965
|
-
}
|
|
1966
|
-
const rollback = await hooks.rollback();
|
|
1967
|
-
await hooks.setRollbackRuntimeBaseline(getHubRuntimeIdentity().runtime_id);
|
|
1968
|
-
providerMigrationOwners.set(providerId, plan.plan_id);
|
|
1969
|
-
retainProviderMutation = true;
|
|
1970
|
-
return sendJson(res, 202, { ok: true, restart_required: true, transaction_id: plan.plan_id, provider_id: providerId, state: 'ROLLBACK_RESTART_PENDING' });
|
|
1971
|
-
} finally {
|
|
1972
|
-
await hooks.release();
|
|
1973
|
-
if (acquiredMigrationFence && !retainProviderMutation && providerMigrationOwners.get(providerId) === body?.transaction_id) {
|
|
1974
|
-
providerMigrationOwners.delete(providerId);
|
|
1975
|
-
hub.endProviderMutation(providerId);
|
|
1976
|
-
}
|
|
1977
|
-
}
|
|
1978
|
-
}
|
|
1979
|
-
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'verify-rollback-migration') {
|
|
1980
|
-
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_MIGRATION_ROLLBACK_VERIFY_CONFIRM_REQUIRED' });
|
|
1981
|
-
const transactionId = body?.transaction_id;
|
|
1982
|
-
const hooks = createProviderLayerMigrationFileHooks({
|
|
1983
|
-
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1984
|
-
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1985
|
-
backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
|
|
1986
|
-
existingMigrationId: transactionId,
|
|
1987
|
-
});
|
|
1988
|
-
let acquiredStoreFence = false;
|
|
1989
|
-
try {
|
|
1990
|
-
await hooks.acquireLock();
|
|
1991
|
-
await hooks.backup({});
|
|
1992
|
-
const plan = hooks.backupPlan();
|
|
1993
|
-
if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_PLAN_PROVIDER_MISMATCH' });
|
|
1994
|
-
const owner = providerMigrationOwners.get(providerId);
|
|
1995
|
-
if (providerStoreMutationOwners.get(plan.plan_id) !== providerId || !hub.hasProviderStoreMutation()) {
|
|
1996
|
-
if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
1997
|
-
providerStoreMutationOwners.set(plan.plan_id, providerId);
|
|
1998
|
-
acquiredStoreFence = true;
|
|
1999
|
-
}
|
|
2000
|
-
if (owner !== plan.plan_id) {
|
|
2001
|
-
if (hub.hasProviderLease(providerId) || !hub.beginProviderMutation(providerId)) {
|
|
2002
|
-
if (acquiredStoreFence && providerStoreMutationOwners.get(plan.plan_id) === providerId) { providerStoreMutationOwners.delete(plan.plan_id); hub.endProviderStoreMutation(); }
|
|
2003
|
-
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
2004
|
-
}
|
|
2005
|
-
providerMigrationOwners.set(providerId, plan.plan_id);
|
|
2006
|
-
}
|
|
2007
|
-
const verification = await hooks.verifyRollback(plan);
|
|
2008
|
-
const live = await readProviderInventorySnapshot(hub, ctx, config);
|
|
2009
|
-
verification.catalogEvidence = live.catalog_evidence;
|
|
2010
|
-
verification.defaultEvidence = live.default_evidence;
|
|
2011
|
-
verification.defaultPreserved = live.default_evidence?.ok === true
|
|
2012
|
-
&& JSON.stringify(live.harness_default ?? null) === JSON.stringify(plan.harness_default_before ?? null);
|
|
2013
|
-
verification.liveRestored = hasCompleteProviderCatalogEvidence(live.catalog_evidence)
|
|
2014
|
-
&& live.records.some((record) => record.id === providerId && record.lifecycle?.catalogued === true && record.native_removable === false);
|
|
2015
|
-
verification.runtimeRestarted = hasProviderRuntimeRestartEvidence(plan.rollback_runtime_id_before, getHubRuntimeIdentity().runtime_id);
|
|
2016
|
-
if (verification.ok !== true || verification.liveRestored !== true || verification.defaultPreserved !== true || verification.runtimeRestarted !== true) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_ROLLBACK_VERIFY_FAILED', verification });
|
|
2017
|
-
await hooks.finalizeRollback();
|
|
2018
|
-
if (providerMigrationOwners.get(providerId) === plan.plan_id) {
|
|
2019
|
-
providerMigrationOwners.delete(providerId);
|
|
2020
|
-
hub.endProviderMutation(providerId);
|
|
2021
|
-
}
|
|
2022
|
-
if (providerStoreMutationOwners.get(plan.plan_id) === providerId) {
|
|
2023
|
-
providerStoreMutationOwners.delete(plan.plan_id);
|
|
2024
|
-
hub.endProviderStoreMutation();
|
|
2025
|
-
}
|
|
2026
|
-
return sendJson(res, 200, { ok: true, state: 'ROLLED_BACK', transaction_id: plan.plan_id, provider_id: providerId, verification });
|
|
2027
|
-
} finally { await hooks.release(); }
|
|
2028
|
-
}
|
|
2029
|
-
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'delete-plan') {
|
|
2030
|
-
if (providerId !== 'deepseek-official' && hasPendingProviderRecoveryTransactions(inventory)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
2031
|
-
if (providerId !== 'deepseek-official' && !hasCompleteProviderCatalogEvidence(inventory.catalog_evidence)) return sendJson(res, 503, { ok: false, code: 'MODEL_CATALOG_UNAVAILABLE' });
|
|
2032
|
-
if (providerId !== 'deepseek-official' && !hasCompleteProviderDeclarationEvidence(inventory.declaration_evidence)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_SOURCE_UNRESOLVED' });
|
|
2033
|
-
if (providerId !== 'deepseek-official' && !hasAvailableProviderLifecycleEvidence(inventory.lifecycle_evidence)) return sendJson(res, 503, { ok: false, code: 'PROVIDER_LIFECYCLE_UNAVAILABLE' });
|
|
2034
|
-
if (providerId !== 'deepseek-official' && inventory.default_evidence?.ok !== true) return sendJson(res, 409, { ok: false, code: inventory.default_evidence?.code ?? 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' });
|
|
2035
|
-
const revisions = readProviderSourceRevisions();
|
|
2036
|
-
if (!revisions.profile.ok && !revisions.settings.ok) return sendJson(res, 409, { ok: false, code: revisions.profile.code ?? revisions.settings.code });
|
|
2037
|
-
const expectedRevision = (revisions.profile.ok ? revisions.profile.revision : null) ?? revisions.settings.revision;
|
|
2038
|
-
const planned = planProviderDelete({
|
|
2039
|
-
providerId,
|
|
2040
|
-
inventory,
|
|
2041
|
-
activeJobs: hub.list(),
|
|
2042
|
-
replacementDefault: body?.replacement_default,
|
|
2043
|
-
expectedRevision,
|
|
2044
|
-
expectedRevisions: {
|
|
2045
|
-
profile: revisions.profile.ok ? revisions.profile.revision : null,
|
|
2046
|
-
settings: revisions.settings.ok ? revisions.settings.revision : null,
|
|
2047
|
-
},
|
|
2048
|
-
});
|
|
2049
|
-
if (!planned.ok) return sendJson(res, 409, { ok: false, code: planned.code });
|
|
2050
|
-
const plan = { ...planned.plan, runtime_id_before: getHubRuntimeIdentity().runtime_id };
|
|
2051
|
-
rememberProviderDeletePlan(plan);
|
|
2052
|
-
return sendJson(res, 200, { ok: true, profile_revision: expectedRevision, plan });
|
|
2053
|
-
}
|
|
2054
|
-
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'probe') {
|
|
2055
|
-
const record = inventory.records.find((entry) => entry.id === providerId);
|
|
2056
|
-
if (!record) return sendJson(res, 404, { ok: false, code: 'PROVIDER_NOT_FOUND' });
|
|
2057
|
-
if (record.desired_state === 'absent') return sendJson(res, 409, { ok: false, code: 'PROVIDER_TOMBSTONED' });
|
|
2058
|
-
const model = selectProviderProbeModel({ providerId, record, config });
|
|
2059
|
-
if (!model) return sendJson(res, 409, { ok: false, code: 'PROVIDER_PROBE_MODEL_UNAVAILABLE' });
|
|
2060
|
-
let explicitProviderProbe;
|
|
2061
|
-
try { explicitProviderProbe = ctx?.providerProbe; } catch { explicitProviderProbe = undefined; }
|
|
2062
|
-
const providerProbe = typeof explicitProviderProbe === 'function' ? explicitProviderProbe : builtInProviderProbe;
|
|
2063
|
-
if (typeof providerProbe !== 'function') return sendJson(res, 503, { ok: false, code: 'PROVIDER_PROBE_UNAVAILABLE' });
|
|
2064
|
-
let observed;
|
|
2065
|
-
let timer;
|
|
2066
|
-
try {
|
|
2067
|
-
const timeout = new Promise((resolve) => { timer = setTimeout(() => resolve({ error: { name: 'TimeoutError' } }), 15_000); });
|
|
2068
|
-
observed = await Promise.race([
|
|
2069
|
-
Promise.resolve().then(() => providerProbe({ provider: providerId, model, signal: AbortSignal.timeout(15_000) })),
|
|
2070
|
-
timeout,
|
|
2071
|
-
]);
|
|
2072
|
-
} catch (error) {
|
|
2073
|
-
// Keep probe transport failures as bounded health evidence so a
|
|
2074
|
-
// credential/quota/timeout result is visible to routing policy;
|
|
2075
|
-
// never copy the raw provider error into the response.
|
|
2076
|
-
observed = { error };
|
|
2077
|
-
} finally {
|
|
2078
|
-
clearTimeout(timer);
|
|
2079
|
-
}
|
|
2080
|
-
const health = hub.healthStore.record(providerId, model, observed?.ok === true ? { ok: true } : { error: observed?.error ?? observed });
|
|
2081
|
-
return sendJson(res, 200, { ok: true, provider_id: providerId, model, health });
|
|
2082
|
-
}
|
|
2083
|
-
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'verify-delete') {
|
|
2084
|
-
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_DELETE_VERIFY_CONFIRM_REQUIRED' });
|
|
2085
|
-
const { transaction_id: transactionId } = body ?? {};
|
|
2086
|
-
const hooks = createProviderDeleteFileHooks({
|
|
2087
|
-
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
2088
|
-
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
2089
|
-
configFile: join(CONFIG_DIR, 'config.json'),
|
|
2090
|
-
lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
2091
|
-
backupDir: join(CONFIG_DIR, 'provider-backups'),
|
|
2092
|
-
sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
|
|
2093
|
-
existingBackupId: transactionId,
|
|
2094
|
-
expectedProviderId: providerId,
|
|
2095
|
-
});
|
|
2096
|
-
const plan = hooks.backupPlan();
|
|
2097
|
-
if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_PLAN_PROVIDER_MISMATCH' });
|
|
2098
|
-
await hooks.acquireLock();
|
|
2099
|
-
try {
|
|
2100
|
-
if (providerStoreMutationOwners.get(transactionId) !== providerId || !hub.hasProviderStoreMutation()) {
|
|
2101
|
-
if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_BUSY' });
|
|
2102
|
-
providerStoreMutationOwners.set(transactionId, providerId);
|
|
2103
|
-
}
|
|
2104
|
-
const verification = await hooks.verify(plan);
|
|
2105
|
-
const live = await readProviderInventorySnapshot(hub, ctx, config);
|
|
2106
|
-
verification.catalog_evidence = live.catalog_evidence;
|
|
2107
|
-
verification.default_evidence = live.default_evidence;
|
|
2108
|
-
verification.catalogAbsent = hasCompleteProviderCatalogEvidence(live.catalog_evidence)
|
|
2109
|
-
&& !live.records.some((entry) => entry.id === providerId && entry.lifecycle?.catalogued === true);
|
|
2110
|
-
verification.harnessDefault = plan.was_harness_default !== true || (
|
|
2111
|
-
live.harness_default?.provider === plan.replacement_default
|
|
2112
|
-
&& live.harness_default?.model === plan.replacement_default_model
|
|
2113
|
-
);
|
|
2114
|
-
verification.runtimeRestarted = hasProviderRuntimeRestartEvidence(plan.delete_runtime_id_before_restart, getHubRuntimeIdentity().runtime_id);
|
|
2115
|
-
if (verification?.providerAbsent !== true || !hasCompleteProviderCatalogEvidence(verification?.catalog_evidence) || verification?.default_evidence?.ok !== true || verification?.catalogAbsent !== true || verification?.harnessDefault !== true || verification?.runtimeRestarted !== true || verification?.routingClear !== true || verification?.tombstonePresent !== true) {
|
|
2116
|
-
return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_VERIFY_FAILED', verification });
|
|
2117
|
-
}
|
|
2118
|
-
await hooks.recordTransaction({ transaction_id: transactionId, provider_id: providerId, state: 'VERIFIED' }, plan);
|
|
2119
|
-
hub.endProviderMutation(providerId);
|
|
2120
|
-
if (providerStoreMutationOwners.get(transactionId) === providerId) {
|
|
2121
|
-
providerStoreMutationOwners.delete(transactionId);
|
|
2122
|
-
hub.endProviderStoreMutation();
|
|
2123
|
-
}
|
|
2124
|
-
return sendJson(res, 200, { ok: true, transaction_id: transactionId, provider_id: providerId, state: 'VERIFIED', verification });
|
|
2125
|
-
} finally {
|
|
2126
|
-
await hooks.release();
|
|
2127
|
-
}
|
|
2128
|
-
}
|
|
2129
|
-
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'rollback') {
|
|
2130
|
-
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_ROLLBACK_CONFIRM_REQUIRED' });
|
|
2131
|
-
const { transaction_id: transactionId } = body ?? {};
|
|
2132
|
-
const hooks = createProviderDeleteFileHooks({
|
|
2133
|
-
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
2134
|
-
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
2135
|
-
configFile: join(CONFIG_DIR, 'config.json'),
|
|
2136
|
-
lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
2137
|
-
backupDir: join(CONFIG_DIR, 'provider-backups'),
|
|
2138
|
-
sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
|
|
2139
|
-
existingBackupId: transactionId,
|
|
2140
|
-
expectedProviderId: providerId,
|
|
2141
|
-
});
|
|
2142
|
-
await hooks.acquireLock();
|
|
2143
|
-
let acquiredStoreMutation = false;
|
|
2144
|
-
let retainStoreMutation = false;
|
|
2145
|
-
try {
|
|
2146
|
-
const plan = hooks.backupPlan();
|
|
2147
|
-
if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_PLAN_PROVIDER_MISMATCH' });
|
|
2148
|
-
if (providerStoreMutationOwners.get(transactionId) !== providerId || !hub.hasProviderStoreMutation()) {
|
|
2149
|
-
if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_BUSY' });
|
|
2150
|
-
providerStoreMutationOwners.set(transactionId, providerId);
|
|
2151
|
-
acquiredStoreMutation = true;
|
|
2152
|
-
}
|
|
2153
|
-
await hooks.rollback(plan);
|
|
2154
|
-
await hooks.setRuntimeBaseline(getHubRuntimeIdentity().runtime_id, 'rollback');
|
|
2155
|
-
const rollbackPlan = hooks.backupPlan() ?? plan;
|
|
2156
|
-
try {
|
|
2157
|
-
await hooks.recordTransaction({ transaction_id: transactionId, provider_id: providerId, state: 'ROLLBACK_PENDING' }, rollbackPlan);
|
|
2158
|
-
} catch (error) {
|
|
2159
|
-
return sendJson(res, 409, { ok: false, code: boundedMachineCodeFromError(error) ?? 'PROVIDER_LIFECYCLE_RECORD_FAILED' });
|
|
2160
|
-
}
|
|
2161
|
-
retainStoreMutation = true;
|
|
2162
|
-
return sendJson(res, 202, { ok: true, restart_required: true, transaction_id: transactionId, provider_id: providerId, state: 'ROLLBACK_PENDING' });
|
|
2163
|
-
} finally {
|
|
2164
|
-
await hooks.release();
|
|
2165
|
-
if (acquiredStoreMutation && !retainStoreMutation && providerStoreMutationOwners.get(transactionId) === providerId) {
|
|
2166
|
-
providerStoreMutationOwners.delete(transactionId);
|
|
2167
|
-
hub.endProviderStoreMutation();
|
|
2168
|
-
}
|
|
2169
|
-
}
|
|
2170
|
-
}
|
|
2171
|
-
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'verify-rollback') {
|
|
2172
|
-
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_ROLLBACK_VERIFY_CONFIRM_REQUIRED' });
|
|
2173
|
-
const { transaction_id: transactionId } = body ?? {};
|
|
2174
|
-
const hooks = createProviderDeleteFileHooks({
|
|
2175
|
-
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
2176
|
-
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
2177
|
-
configFile: join(CONFIG_DIR, 'config.json'),
|
|
2178
|
-
lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
2179
|
-
backupDir: join(CONFIG_DIR, 'provider-backups'),
|
|
2180
|
-
sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
|
|
2181
|
-
existingBackupId: transactionId,
|
|
2182
|
-
expectedProviderId: providerId,
|
|
2183
|
-
});
|
|
2184
|
-
const plan = hooks.backupPlan();
|
|
2185
|
-
if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_PLAN_PROVIDER_MISMATCH' });
|
|
2186
|
-
await hooks.acquireLock();
|
|
2187
|
-
try {
|
|
2188
|
-
if (providerStoreMutationOwners.get(transactionId) !== providerId || !hub.hasProviderStoreMutation()) {
|
|
2189
|
-
if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_BUSY' });
|
|
2190
|
-
providerStoreMutationOwners.set(transactionId, providerId);
|
|
2191
|
-
}
|
|
2192
|
-
const verified = await hooks.verifyRollback(plan);
|
|
2193
|
-
const live = await readProviderInventorySnapshot(hub, ctx, config);
|
|
2194
|
-
verified.catalog_evidence = live.catalog_evidence;
|
|
2195
|
-
verified.default_evidence = live.default_evidence;
|
|
2196
|
-
verified.catalogPresent = hasCompleteProviderCatalogEvidence(live.catalog_evidence)
|
|
2197
|
-
&& live.records.some((entry) => entry.id === providerId && entry.lifecycle?.catalogued === true);
|
|
2198
|
-
verified.harnessDefaultRestored = plan.was_harness_default !== true || (
|
|
2199
|
-
live.harness_default?.provider === plan.harness_default_before?.provider
|
|
2200
|
-
&& live.harness_default?.model === plan.harness_default_before?.model
|
|
2201
|
-
);
|
|
2202
|
-
verified.runtimeRestarted = hasProviderRuntimeRestartEvidence(plan.rollback_runtime_id_before_restart, getHubRuntimeIdentity().runtime_id);
|
|
2203
|
-
if (verified?.ok !== true || !hasCompleteProviderCatalogEvidence(verified?.catalog_evidence) || verified?.default_evidence?.ok !== true || verified?.catalogPresent !== true || verified?.harnessDefaultRestored !== true || verified?.runtimeRestarted !== true) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_ROLLBACK_VERIFY_FAILED', verification: verified });
|
|
2204
|
-
await hooks.recordTransaction({ transaction_id: transactionId, provider_id: providerId, state: 'ROLLED_BACK' }, plan);
|
|
2205
|
-
hub.endProviderMutation(providerId);
|
|
2206
|
-
if (providerStoreMutationOwners.get(transactionId) === providerId) {
|
|
2207
|
-
providerStoreMutationOwners.delete(transactionId);
|
|
2208
|
-
hub.endProviderStoreMutation();
|
|
2209
|
-
}
|
|
2210
|
-
return sendJson(res, 200, { ok: true, transaction_id: transactionId, provider_id: providerId, state: 'ROLLED_BACK' });
|
|
2211
|
-
} finally {
|
|
2212
|
-
await hooks.release();
|
|
2213
|
-
}
|
|
2214
|
-
}
|
|
2215
|
-
if (req.method === 'DELETE' && parts.length === 1) {
|
|
2216
|
-
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_DELETE_CONFIRM_REQUIRED' });
|
|
2217
|
-
if (body?.purge_orphan_credentials === true) {
|
|
2218
|
-
return sendJson(res, 400, { ok: false, code: 'CREDENTIAL_PURGE_REQUIRES_EXPLICIT_CONFIRMATION' });
|
|
2219
|
-
}
|
|
2220
|
-
const stored = providerDeletePlans.get(body?.plan_id);
|
|
2221
|
-
if (!stored || Date.now() - stored.created_at > PROVIDER_DELETE_PLAN_TTL_MS || stored.plan.provider_id !== providerId) {
|
|
2222
|
-
providerDeletePlans.delete(body?.plan_id);
|
|
2223
|
-
return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_PLAN_EXPIRED' });
|
|
2224
|
-
}
|
|
2225
|
-
const revisions = readProviderSourceRevisions();
|
|
2226
|
-
if (!revisions.profile.ok && !revisions.settings.ok) return sendJson(res, 409, { ok: false, code: revisions.profile.code ?? revisions.settings.code });
|
|
2227
|
-
if (providerId !== 'deepseek-official' && !hasCompleteProviderCatalogEvidence(inventory.catalog_evidence)) return sendJson(res, 503, { ok: false, code: 'MODEL_CATALOG_UNAVAILABLE' });
|
|
2228
|
-
if (providerId !== 'deepseek-official' && !hasCompleteProviderDeclarationEvidence(inventory.declaration_evidence)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_SOURCE_UNRESOLVED' });
|
|
2229
|
-
if (providerId !== 'deepseek-official' && !hasAvailableProviderLifecycleEvidence(inventory.lifecycle_evidence)) return sendJson(res, 503, { ok: false, code: 'PROVIDER_LIFECYCLE_UNAVAILABLE' });
|
|
2230
|
-
if (providerId !== 'deepseek-official' && inventory.default_evidence?.ok !== true) return sendJson(res, 409, { ok: false, code: inventory.default_evidence?.code ?? 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' });
|
|
2231
|
-
const storedExpected = stored.plan.expected_revisions;
|
|
2232
|
-
const revisionsMatch = storedExpected
|
|
2233
|
-
? (storedExpected.profile === (revisions.profile.ok ? revisions.profile.revision : null)
|
|
2234
|
-
&& storedExpected.settings === (revisions.settings.ok ? revisions.settings.revision : null))
|
|
2235
|
-
: (body?.expected_revision === stored.plan.expected_revision && (revisions.profile.revision ?? revisions.settings.revision) === stored.plan.expected_revision);
|
|
2236
|
-
if (!revisionsMatch || body?.expected_revision !== stored.plan.expected_revision) {
|
|
2237
|
-
return sendJson(res, 409, { ok: false, code: 'PROVIDER_PROFILE_CHANGED' });
|
|
2238
|
-
}
|
|
2239
|
-
if (providerId !== 'deepseek-official' && hasPendingProviderRecoveryTransactions(inventory)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
2240
|
-
const refreshed = planProviderDelete({
|
|
2241
|
-
providerId,
|
|
2242
|
-
inventory,
|
|
2243
|
-
activeJobs: hub.list(),
|
|
2244
|
-
replacementDefault: stored.plan.replacement_default,
|
|
2245
|
-
expectedRevision: stored.plan.expected_revision,
|
|
2246
|
-
expectedRevisions: stored.plan.expected_revisions,
|
|
2247
|
-
});
|
|
2248
|
-
if (!refreshed.ok) return sendJson(res, 409, { ok: false, code: refreshed.code });
|
|
2249
|
-
if (refreshed.plan.replacement_default_model !== stored.plan.replacement_default_model) {
|
|
2250
|
-
return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_PLAN_CHANGED' });
|
|
2251
|
-
}
|
|
2252
|
-
let executionPlan = {
|
|
2253
|
-
...refreshed.plan,
|
|
2254
|
-
plan_id: stored.plan.plan_id,
|
|
2255
|
-
expected_revision: stored.plan.expected_revision,
|
|
2256
|
-
expected_revisions: stored.plan.expected_revisions,
|
|
2257
|
-
runtime_id_before: stored.plan.runtime_id_before,
|
|
2258
|
-
};
|
|
2259
|
-
if (!hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_BUSY' });
|
|
2260
|
-
if (!hub.beginProviderMutation(providerId)) {
|
|
2261
|
-
hub.endProviderStoreMutation();
|
|
2262
|
-
return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_BUSY' });
|
|
2263
|
-
}
|
|
2264
|
-
let hooks = null;
|
|
2265
|
-
let lockAcquired = false;
|
|
2266
|
-
let retainProviderMutation = false;
|
|
2267
|
-
let retainProviderStoreMutation = false;
|
|
2268
|
-
try {
|
|
2269
|
-
hooks = createProviderDeleteFileHooks({
|
|
2270
|
-
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
2271
|
-
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
2272
|
-
configFile: join(CONFIG_DIR, 'config.json'),
|
|
2273
|
-
lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
2274
|
-
backupDir: join(CONFIG_DIR, 'provider-backups'),
|
|
2275
|
-
sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
|
|
2276
|
-
runtimeIdProvider: () => getHubRuntimeIdentity().runtime_id,
|
|
2277
|
-
afterLockAcquired: () => {
|
|
2278
|
-
const pendingRecovery = [
|
|
2279
|
-
...readProviderRecoveryTransactions(),
|
|
2280
|
-
...readProviderLayerMigrationTransactions(join(CONFIG_DIR, 'provider-migration-backups')),
|
|
2281
|
-
];
|
|
2282
|
-
if (pendingRecovery.length > 0) {
|
|
2283
|
-
throw Object.assign(new Error('provider recovery transaction is pending'), { code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
2284
|
-
}
|
|
2285
|
-
if (hub.hasProviderLease(providerId)) {
|
|
2286
|
-
throw Object.assign(new Error('provider is in use'), { code: 'PROVIDER_IN_USE' });
|
|
2287
|
-
}
|
|
2288
|
-
},
|
|
2289
|
-
});
|
|
2290
|
-
// Hold the real shared store lock while rebuilding the live
|
|
2291
|
-
// inventory and delete plan. The pre-lock plan is only a
|
|
2292
|
-
// proposal; catalog/default/ownership/recovery facts must be
|
|
2293
|
-
// authoritative at the mutation boundary.
|
|
2294
|
-
await hooks.acquireLock();
|
|
2295
|
-
lockAcquired = true;
|
|
2296
|
-
const lockedConfig = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
2297
|
-
const lockedInventory = await readProviderInventorySnapshot(hub, ctx, lockedConfig);
|
|
2298
|
-
const lockedRevisions = readProviderSourceRevisions();
|
|
2299
|
-
const lockedExpected = stored.plan.expected_revisions;
|
|
2300
|
-
const lockedRevisionsMatch = lockedExpected
|
|
2301
|
-
? lockedExpected.profile === (lockedRevisions.profile.ok ? lockedRevisions.profile.revision : null)
|
|
2302
|
-
&& lockedExpected.settings === (lockedRevisions.settings.ok ? lockedRevisions.settings.revision : null)
|
|
2303
|
-
: false;
|
|
2304
|
-
if (providerId !== 'deepseek-official' && !hasCompleteProviderCatalogEvidence(lockedInventory.catalog_evidence)) throw Object.assign(new Error('provider catalog unavailable after lock'), { code: 'MODEL_CATALOG_UNAVAILABLE' });
|
|
2305
|
-
if (providerId !== 'deepseek-official' && !hasCompleteProviderDeclarationEvidence(lockedInventory.declaration_evidence)) throw Object.assign(new Error('provider declarations unavailable after lock'), { code: 'PROVIDER_DELETE_SOURCE_UNRESOLVED' });
|
|
2306
|
-
if (providerId !== 'deepseek-official' && !hasAvailableProviderLifecycleEvidence(lockedInventory.lifecycle_evidence)) throw Object.assign(new Error('provider lifecycle unavailable after lock'), { code: 'PROVIDER_LIFECYCLE_UNAVAILABLE' });
|
|
2307
|
-
if (providerId !== 'deepseek-official' && lockedInventory.default_evidence?.ok !== true) throw Object.assign(new Error('provider default authority changed after lock'), { code: lockedInventory.default_evidence?.code ?? 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' });
|
|
2308
|
-
if (providerId !== 'deepseek-official' && hasPendingProviderRecoveryTransactions(lockedInventory)) throw Object.assign(new Error('provider recovery transaction is pending'), { code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
2309
|
-
if (!lockedRevisionsMatch) throw Object.assign(new Error('provider source changed after lock'), { code: 'PROVIDER_PROFILE_CHANGED' });
|
|
2310
|
-
if (hub.hasProviderLease(providerId)) throw Object.assign(new Error('provider is in use'), { code: 'PROVIDER_IN_USE' });
|
|
2311
|
-
const lockedPlan = planProviderDelete({
|
|
2312
|
-
providerId,
|
|
2313
|
-
inventory: lockedInventory,
|
|
2314
|
-
activeJobs: hub.list(),
|
|
2315
|
-
replacementDefault: stored.plan.replacement_default,
|
|
2316
|
-
expectedRevision: stored.plan.expected_revision,
|
|
2317
|
-
expectedRevisions: stored.plan.expected_revisions,
|
|
2318
|
-
});
|
|
2319
|
-
if (!lockedPlan.ok) throw Object.assign(new Error('provider delete plan changed after lock'), { code: lockedPlan.code });
|
|
2320
|
-
const sameLockedPlan = lockedPlan.plan.replacement_default_model === stored.plan.replacement_default_model
|
|
2321
|
-
&& lockedPlan.plan.was_harness_default === stored.plan.was_harness_default
|
|
2322
|
-
&& JSON.stringify(lockedPlan.plan.declaration_authorities) === JSON.stringify(stored.plan.declaration_authorities)
|
|
2323
|
-
&& JSON.stringify(lockedPlan.plan.harness_default_before ?? null) === JSON.stringify(stored.plan.harness_default_before ?? null)
|
|
2324
|
-
&& JSON.stringify(lockedPlan.plan.credential_refs) === JSON.stringify(stored.plan.credential_refs);
|
|
2325
|
-
if (!sameLockedPlan) throw Object.assign(new Error('provider delete plan semantics changed after lock'), { code: 'PROVIDER_DELETE_PLAN_CHANGED' });
|
|
2326
|
-
executionPlan = {
|
|
2327
|
-
...lockedPlan.plan,
|
|
2328
|
-
plan_id: stored.plan.plan_id,
|
|
2329
|
-
expected_revision: stored.plan.expected_revision,
|
|
2330
|
-
expected_revisions: stored.plan.expected_revisions,
|
|
2331
|
-
runtime_id_before: stored.plan.runtime_id_before,
|
|
2332
|
-
};
|
|
2333
|
-
providerDeletePlans.delete(body.plan_id);
|
|
2334
|
-
const { executeProviderDelete } = await import('../provider-lifecycle.mjs');
|
|
2335
|
-
const result = await executeProviderDelete(executionPlan, hooks, { deferRestart: true });
|
|
2336
|
-
retainProviderMutation = result.state === 'RESTART_PENDING';
|
|
2337
|
-
if (retainProviderMutation) {
|
|
2338
|
-
providerStoreMutationOwners.set(executionPlan.plan_id, providerId);
|
|
2339
|
-
retainProviderStoreMutation = true;
|
|
2340
|
-
}
|
|
2341
|
-
return sendJson(res, result.state === 'RESTART_PENDING' ? 202 : 409, { ok: result.state === 'RESTART_PENDING', restart_required: true, result });
|
|
2342
|
-
} finally {
|
|
2343
|
-
if (lockAcquired && hooks) await hooks.release().catch(() => {});
|
|
2344
|
-
if (!retainProviderMutation) hub.endProviderMutation(providerId);
|
|
2345
|
-
if (!retainProviderStoreMutation) hub.endProviderStoreMutation();
|
|
2346
|
-
}
|
|
2347
|
-
}
|
|
2348
|
-
return sendJson(res, 404, { ok: false, error: 'unknown providers endpoint' });
|
|
2349
|
-
} catch (err) {
|
|
2350
|
-
const isProbe = req.method === 'POST' && parts.length === 2 && parts[1] === 'probe';
|
|
2351
|
-
const code = boundedMachineCodeFromError(err) ?? (isProbe ? 'PROVIDER_PROBE_FAILED' : 'PROVIDER_PLAN_INVALID');
|
|
2352
|
-
const status = code === 'PROVIDER_DELETE_RESTART_SUPERVISOR_UNAVAILABLE' || code.startsWith('PROVIDER_PROBE') ? 503
|
|
2353
|
-
: code.startsWith('PROVIDER_') || code.startsWith('CREDENTIAL_') ? 409 : 400;
|
|
2354
|
-
return sendJson(res, status, { ok: false, code });
|
|
2355
|
-
}
|
|
2356
|
-
},
|
|
2357
|
-
}));
|
|
2358
|
-
|
|
2359
|
-
disposers.push(webServer.register({
|
|
2360
|
-
kind: 'exact', path: `${ROUTE_BASE}/provider-health`,
|
|
2361
|
-
handler: async (req, res) => {
|
|
2362
|
-
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
2363
|
-
if (req.method !== 'GET') return sendJson(res, 405, { ok: false, error: 'GET only' }, { allow: 'GET' });
|
|
2364
|
-
return sendJson(res, 200, {
|
|
2365
|
-
ok: true,
|
|
2366
|
-
schema_version: 1,
|
|
2367
|
-
health: hub.healthStore.list(),
|
|
2368
|
-
runtime: getHubRuntimeIdentity(),
|
|
2369
|
-
});
|
|
2370
|
-
},
|
|
2371
|
-
}));
|
|
2372
|
-
|
|
2373
|
-
disposers.push(webServer.register({
|
|
2374
|
-
kind: 'exact', path: `${ROUTE_BASE}/models`,
|
|
1669
|
+
},
|
|
1670
|
+
}));
|
|
1671
|
+
|
|
1672
|
+
// Secret-free Harness provider inventory. This is deliberately separate
|
|
1673
|
+
// from /models (catalog) and /provider (effective routing): it reports
|
|
1674
|
+
// declaration provenance, ownership, lifecycle state and policy refs.
|
|
1675
|
+
disposers.push(webServer.register({
|
|
1676
|
+
kind: 'exact', path: `${ROUTE_BASE}/providers`,
|
|
1677
|
+
handler: async (req, res) => {
|
|
1678
|
+
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
1679
|
+
if (req.method !== 'GET') return sendJson(res, 405, { ok: false, error: 'GET only' }, { allow: 'GET' });
|
|
1680
|
+
try {
|
|
1681
|
+
const config = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
1682
|
+
const inventory = await readProviderInventorySnapshot(hub, ctx, config);
|
|
1683
|
+
return sendJson(res, 200, {
|
|
1684
|
+
ok: true,
|
|
1685
|
+
...inventory,
|
|
1686
|
+
runtime: getHubRuntimeIdentity(),
|
|
1687
|
+
});
|
|
1688
|
+
} catch {
|
|
1689
|
+
return sendJson(res, 503, { ok: false, code: 'PROVIDER_INVENTORY_UNAVAILABLE', error: 'Unable to read Harness provider inventory.' });
|
|
1690
|
+
}
|
|
1691
|
+
},
|
|
1692
|
+
}));
|
|
1693
|
+
|
|
1694
|
+
// Credential references are a separate, secret-free inventory. Provider
|
|
1695
|
+
// deletion can report these references, but never implies their purge.
|
|
1696
|
+
disposers.push(webServer.register({
|
|
1697
|
+
kind: 'exact', path: `${ROUTE_BASE}/credential-references`,
|
|
1698
|
+
handler: async (req, res) => {
|
|
1699
|
+
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
1700
|
+
if (req.method !== 'GET') return sendJson(res, 405, { ok: false, error: 'GET only' }, { allow: 'GET' });
|
|
1701
|
+
try {
|
|
1702
|
+
const config = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
1703
|
+
const snapshot = await readProviderInventorySnapshot(hub, ctx, config);
|
|
1704
|
+
const credentials = buildCredentialReferenceInventory({ providers: snapshot.records, additional_refs: snapshot.credential_history_refs, purged_refs: snapshot.credential_purged_refs });
|
|
1705
|
+
return sendJson(res, 200, { ok: true, ...credentials, unverified_purges: snapshot.credential_purge_unverified, runtime: getHubRuntimeIdentity() });
|
|
1706
|
+
} catch {
|
|
1707
|
+
return sendJson(res, 503, { ok: false, code: 'CREDENTIAL_REFERENCE_INVENTORY_UNAVAILABLE' });
|
|
1708
|
+
}
|
|
1709
|
+
},
|
|
1710
|
+
}));
|
|
1711
|
+
|
|
1712
|
+
disposers.push(webServer.register({
|
|
1713
|
+
kind: 'prefix', path: `${ROUTE_BASE}/credential-references`,
|
|
1714
|
+
handler: async (req, res) => {
|
|
1715
|
+
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
1716
|
+
let url;
|
|
1717
|
+
try { url = new URL(req.url, 'http://localhost'); } catch { return sendJson(res, 400, { ok: false, code: 'CREDENTIAL_PURGE_PLAN_INVALID' }); }
|
|
1718
|
+
const parts = url.pathname.slice(`${ROUTE_BASE}/credential-references`.length).split('/').filter(Boolean);
|
|
1719
|
+
try {
|
|
1720
|
+
const referenceId = decodeURIComponent(parts[0] ?? '');
|
|
1721
|
+
const body = await readBody(req);
|
|
1722
|
+
const config = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
1723
|
+
const snapshot = await readProviderInventorySnapshot(hub, ctx, config);
|
|
1724
|
+
const credentials = buildCredentialReferenceInventory({ providers: snapshot.records, additional_refs: snapshot.credential_history_refs, purged_refs: snapshot.credential_purged_refs });
|
|
1725
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'purge-plan') {
|
|
1726
|
+
const planned = planCredentialPurge({ inventory: credentials, referenceId, expectedRevision: body?.expected_revision });
|
|
1727
|
+
if (!planned.ok) return sendJson(res, 409, { ok: false, code: planned.code });
|
|
1728
|
+
rememberCredentialPurgePlan(planned.plan);
|
|
1729
|
+
return sendJson(res, 200, { ok: true, plan: planned.plan });
|
|
1730
|
+
}
|
|
1731
|
+
if (req.method === 'DELETE' && parts.length === 1) {
|
|
1732
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'CREDENTIAL_PURGE_CONFIRM_REQUIRED' });
|
|
1733
|
+
const stored = credentialPurgePlans.get(body?.plan_id);
|
|
1734
|
+
if (!stored || Date.now() - stored.created_at > PROVIDER_DELETE_PLAN_TTL_MS || stored.plan.reference_id !== referenceId) {
|
|
1735
|
+
credentialPurgePlans.delete(body?.plan_id);
|
|
1736
|
+
return sendJson(res, 409, { ok: false, code: 'CREDENTIAL_PURGE_PLAN_EXPIRED' });
|
|
1737
|
+
}
|
|
1738
|
+
const current = planCredentialPurge({ inventory: credentials, referenceId, expectedRevision: stored.plan.expected_revision });
|
|
1739
|
+
if (!current.ok) return sendJson(res, 409, { ok: false, code: current.code });
|
|
1740
|
+
if (current.plan.expected_revision !== stored.plan.expected_revision) return sendJson(res, 409, { ok: false, code: 'CREDENTIAL_REFERENCE_CHANGED' });
|
|
1741
|
+
credentialPurgePlans.delete(body.plan_id);
|
|
1742
|
+
const adapter = createCredentialPurgeFileHooks({
|
|
1743
|
+
credentialsFile: join(CONFIG_DIR, 'harness', '.credentials.yaml'),
|
|
1744
|
+
crewHome: join(CONFIG_DIR, 'harness'),
|
|
1745
|
+
});
|
|
1746
|
+
const result = await executeCredentialPurge(current.plan, {
|
|
1747
|
+
recheck: async (id) => {
|
|
1748
|
+
const freshSnapshot = await readProviderInventorySnapshot(hub, ctx, config);
|
|
1749
|
+
const freshCredentials = buildCredentialReferenceInventory({ providers: freshSnapshot.records, additional_refs: freshSnapshot.credential_history_refs, purged_refs: freshSnapshot.credential_purged_refs });
|
|
1750
|
+
const freshPlan = planCredentialPurge({ inventory: freshCredentials, referenceId: id });
|
|
1751
|
+
if (!freshPlan.ok) return { ok: false };
|
|
1752
|
+
const freshRecord = freshCredentials.records.find((entry) => entry.reference_id === id);
|
|
1753
|
+
return {
|
|
1754
|
+
ok: true,
|
|
1755
|
+
revision: freshPlan.plan.expected_revision,
|
|
1756
|
+
orphan: freshRecord?.orphan === true,
|
|
1757
|
+
ownership: freshRecord?.ownership,
|
|
1758
|
+
purge_capability: freshRecord?.purge_capability,
|
|
1759
|
+
};
|
|
1760
|
+
},
|
|
1761
|
+
...(adapter.ok ? { purge: adapter.purge, verify: adapter.verify } : {}),
|
|
1762
|
+
});
|
|
1763
|
+
if (result.state === 'PURGED' || result.state === 'VERIFIED') {
|
|
1764
|
+
try {
|
|
1765
|
+
writeCredentialPurgeState(recordCredentialPurgeOutcome(readCredentialPurgeState(), current.plan, result));
|
|
1766
|
+
} catch {
|
|
1767
|
+
return sendJson(res, 503, { ok: false, code: 'CREDENTIAL_PURGE_AUDIT_FAILED', state: result.state, reference_id: referenceId });
|
|
1768
|
+
}
|
|
1769
|
+
}
|
|
1770
|
+
return sendJson(res, result.ok ? 200 : 503, result);
|
|
1771
|
+
}
|
|
1772
|
+
return sendJson(res, 404, { ok: false, code: 'CREDENTIAL_REFERENCE_ENDPOINT_NOT_FOUND' });
|
|
1773
|
+
} catch (error) {
|
|
1774
|
+
return sendJson(res, 400, { ok: false, code: boundedMachineCodeFromError(error) ?? 'CREDENTIAL_PURGE_PLAN_INVALID' });
|
|
1775
|
+
}
|
|
1776
|
+
},
|
|
1777
|
+
}));
|
|
1778
|
+
|
|
1779
|
+
// Deletion is two-phase: this endpoint only computes a revision-bound,
|
|
1780
|
+
// secret-free impact plan. A later mutation endpoint must present that
|
|
1781
|
+
// plan and perform restart/verification through an owned supervisor.
|
|
1782
|
+
disposers.push(webServer.register({
|
|
1783
|
+
kind: 'prefix', path: `${ROUTE_BASE}/providers`,
|
|
1784
|
+
handler: async (req, res) => {
|
|
1785
|
+
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
1786
|
+
let url;
|
|
1787
|
+
try { url = new URL(req.url, 'http://localhost'); } catch { return sendJson(res, 400, { ok: false, code: 'PROVIDER_PLAN_INVALID' }); }
|
|
1788
|
+
const parts = url.pathname.slice(`${ROUTE_BASE}/providers`.length).split('/').filter(Boolean);
|
|
1789
|
+
try {
|
|
1790
|
+
if (req.method === 'GET' && parts.length === 1 && parts[0] === 'migration-status') {
|
|
1791
|
+
const config = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
1792
|
+
const inventory = await readProviderInventorySnapshot(hub, ctx, config);
|
|
1793
|
+
return sendJson(res, 200, { ok: true, ...inventory.migration, runtime: getHubRuntimeIdentity() });
|
|
1794
|
+
}
|
|
1795
|
+
const providerId = decodeURIComponent(parts[0]);
|
|
1796
|
+
const body = await readBody(req);
|
|
1797
|
+
if (req.method === 'POST' && parts[0] === '_recovery' && parts[1] === 'lock' && parts.length === 2) {
|
|
1798
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_CONFIRM_REQUIRED' });
|
|
1799
|
+
const hooks = createProviderDeleteFileHooks({
|
|
1800
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1801
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1802
|
+
configFile: join(CONFIG_DIR, 'config.json'),
|
|
1803
|
+
lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
1804
|
+
backupDir: join(CONFIG_DIR, 'provider-backups'),
|
|
1805
|
+
sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
|
|
1806
|
+
});
|
|
1807
|
+
const recovered = await hooks.recoverLock({ force: body?.force === true });
|
|
1808
|
+
const migrationHooks = createProviderLayerMigrationFileHooks({
|
|
1809
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1810
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1811
|
+
backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
|
|
1812
|
+
});
|
|
1813
|
+
const migrationRecovered = await migrationHooks.recoverLock({ confirm: true });
|
|
1814
|
+
const ok = recovered.ok === true && migrationRecovered.ok === true;
|
|
1815
|
+
return sendJson(res, ok ? 200 : 409, { ok, delete_lock: recovered, migration_lock: migrationRecovered });
|
|
1816
|
+
}
|
|
1817
|
+
if (req.method === 'POST' && parts[0] === '_recovery' && parts[1] === 'quarantine' && parts.length === 2) {
|
|
1818
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_QUARANTINE_CONFIRM_REQUIRED' });
|
|
1819
|
+
const actionId = typeof body?.action_id === 'string' && RECOVERY_ACTION_PATTERN.test(body.action_id)
|
|
1820
|
+
? body.action_id : null;
|
|
1821
|
+
if (!actionId) return sendJson(res, 400, { ok: false, code: 'PROVIDER_DELETE_PLAN_INVALID' });
|
|
1822
|
+
const recovery = [
|
|
1823
|
+
...readProviderRecoveryTransactions(),
|
|
1824
|
+
...readProviderLayerMigrationTransactions(join(CONFIG_DIR, 'provider-migration-backups')),
|
|
1825
|
+
].find((entry) => entry.action_id === actionId);
|
|
1826
|
+
if (!recovery) return sendJson(res, 404, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_NOT_FOUND' });
|
|
1827
|
+
if (recovery.recoverable === true) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_RECOVERABLE' });
|
|
1828
|
+
if (typeof recovery.storage_id !== 'string') return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_QUARANTINE_FAILED' });
|
|
1829
|
+
if (recovery.source === 'provider-layer-migration') {
|
|
1830
|
+
try {
|
|
1831
|
+
const migrationHooks = createProviderLayerMigrationFileHooks({
|
|
1832
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1833
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1834
|
+
backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
|
|
1835
|
+
});
|
|
1836
|
+
const quarantined = await migrationHooks.quarantine(actionId);
|
|
1837
|
+
return sendJson(res, 200, { ...quarantined, transaction_id: recovery.transaction_id ?? null });
|
|
1838
|
+
} catch (error) {
|
|
1839
|
+
const code = boundedMachineCodeFromError(error) ?? 'PROVIDER_MIGRATION_RECOVERY_QUARANTINE_FAILED';
|
|
1840
|
+
return sendJson(res, 409, { ok: false, code });
|
|
1841
|
+
}
|
|
1842
|
+
}
|
|
1843
|
+
try {
|
|
1844
|
+
const hooks = createProviderDeleteFileHooks({
|
|
1845
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1846
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1847
|
+
configFile: join(CONFIG_DIR, 'config.json'),
|
|
1848
|
+
lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
1849
|
+
backupDir: join(CONFIG_DIR, 'provider-backups'),
|
|
1850
|
+
sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
|
|
1851
|
+
afterLockAcquired: () => {
|
|
1852
|
+
const current = readProviderRecoveryTransactions().find((entry) => entry.action_id === actionId);
|
|
1853
|
+
if (!current || current.recoverable === true || current.storage_id !== recovery.storage_id) {
|
|
1854
|
+
throw Object.assign(new Error('provider recovery entry changed before quarantine'), { code: 'PROVIDER_DELETE_STATE_CHANGED' });
|
|
1855
|
+
}
|
|
1856
|
+
},
|
|
1857
|
+
});
|
|
1858
|
+
const quarantined = await hooks.quarantine(recovery.storage_id);
|
|
1859
|
+
return sendJson(res, 200, { ...quarantined, action_id: actionId, transaction_id: recovery.transaction_id ?? null });
|
|
1860
|
+
} catch (error) {
|
|
1861
|
+
const code = boundedMachineCodeFromError(error) ?? 'PROVIDER_DELETE_RECOVERY_QUARANTINE_FAILED';
|
|
1862
|
+
return sendJson(res, 409, { ok: false, code });
|
|
1863
|
+
}
|
|
1864
|
+
}
|
|
1865
|
+
const config = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
1866
|
+
const inventory = await readProviderInventorySnapshot(hub, ctx, config);
|
|
1867
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'migrate-plan') {
|
|
1868
|
+
if (!hasCompleteProviderCatalogEvidence(inventory.catalog_evidence)) return sendJson(res, 503, { ok: false, code: 'MODEL_CATALOG_UNAVAILABLE' });
|
|
1869
|
+
if (!hasCompleteProviderDeclarationEvidence(inventory.declaration_evidence)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_SOURCE_UNRESOLVED' });
|
|
1870
|
+
if (!hasAvailableProviderLifecycleEvidence(inventory.lifecycle_evidence)) return sendJson(res, 503, { ok: false, code: 'PROVIDER_LIFECYCLE_UNAVAILABLE' });
|
|
1871
|
+
if (inventory.default_evidence?.ok !== true) return sendJson(res, 409, { ok: false, code: inventory.default_evidence?.code ?? 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' });
|
|
1872
|
+
if (hasPendingProviderRecoveryTransactions(inventory)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
1873
|
+
const entry = inventory.migration.providers.find((candidate) => candidate.provider_id === providerId);
|
|
1874
|
+
if (!entry) return sendJson(res, 404, { ok: false, code: 'PROVIDER_MIGRATION_NOT_REQUIRED' });
|
|
1875
|
+
if (entry.action === 'blocked' || entry.action === 'collision-review') return sendJson(res, 409, { ok: false, code: entry.blocked_reason ?? 'PROVIDER_MIGRATION_BLOCKED' });
|
|
1876
|
+
const revisions = readProviderSourceRevisions();
|
|
1877
|
+
if (!revisions.profile.ok) return sendJson(res, 409, { ok: false, code: revisions.profile.code ?? 'PROVIDER_PROFILE_CHANGED' });
|
|
1878
|
+
if ((entry.action === 'promote-existing-user' || entry.harness_default) && !revisions.settings.ok) return sendJson(res, 409, { ok: false, code: revisions.settings.code ?? 'PROVIDER_SETTINGS_CHANGED' });
|
|
1879
|
+
const profileFile = join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml');
|
|
1880
|
+
const parsedMaterialization = readProviderMaterialization(readFileSync(profileFile, 'utf8'), { providerId, file: 'harness/profiles/dsh-crew/cordis.patch.yml' });
|
|
1881
|
+
if (!parsedMaterialization.ok) return sendJson(res, 409, { ok: false, code: parsedMaterialization.code });
|
|
1882
|
+
const materialization = parsedMaterialization.provider;
|
|
1883
|
+
let userMaterializationBefore;
|
|
1884
|
+
if (entry.action === 'promote-existing-user') {
|
|
1885
|
+
const settingsFile = join(CONFIG_DIR, 'harness', 'settings.yaml');
|
|
1886
|
+
if (!existsSync(settingsFile)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_SETTINGS_CHANGED' });
|
|
1887
|
+
const parsedUserMaterialization = readProviderSettingsMaterialization(readFileSync(settingsFile, 'utf8'), { providerId, file: 'harness/settings.yaml' });
|
|
1888
|
+
if (!parsedUserMaterialization.ok) return sendJson(res, 409, { ok: false, code: parsedUserMaterialization.code });
|
|
1889
|
+
userMaterializationBefore = parsedUserMaterialization.provider;
|
|
1890
|
+
}
|
|
1891
|
+
const plan = {
|
|
1892
|
+
schema_version: 1,
|
|
1893
|
+
kind: 'provider-layer-migration',
|
|
1894
|
+
plan_id: randomUUID(),
|
|
1895
|
+
provider_id: providerId,
|
|
1896
|
+
action: entry.action,
|
|
1897
|
+
source: entry.source,
|
|
1898
|
+
routing_references: entry.referenced_by,
|
|
1899
|
+
materialization: { provider: materialization },
|
|
1900
|
+
...(userMaterializationBefore ? { user_materialization_before: userMaterializationBefore } : {}),
|
|
1901
|
+
expected_revisions: {
|
|
1902
|
+
profile: revisions.profile.revision,
|
|
1903
|
+
settings: revisions.settings.ok ? revisions.settings.revision : null,
|
|
1904
|
+
},
|
|
1905
|
+
harness_default_before: inventory.harness_default ?? null,
|
|
1906
|
+
runtime_id_before: getHubRuntimeIdentity().runtime_id,
|
|
1907
|
+
restart_required: true,
|
|
1908
|
+
};
|
|
1909
|
+
rememberProviderMigrationPlan(plan);
|
|
1910
|
+
return sendJson(res, 200, { ok: true, plan });
|
|
1911
|
+
}
|
|
1912
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'migrate') {
|
|
1913
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_MIGRATION_CONFIRM_REQUIRED' });
|
|
1914
|
+
const stored = providerMigrationPlans.get(body?.plan_id);
|
|
1915
|
+
if (!stored || Date.now() - stored.created_at > PROVIDER_DELETE_PLAN_TTL_MS || stored.plan.provider_id !== providerId) {
|
|
1916
|
+
providerMigrationPlans.delete(body?.plan_id);
|
|
1917
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_PLAN_EXPIRED' });
|
|
1918
|
+
}
|
|
1919
|
+
const entry = inventory.migration.providers.find((candidate) => candidate.provider_id === providerId);
|
|
1920
|
+
if (!entry || entry.action !== stored.plan.action || entry.action === 'blocked' || entry.action === 'collision-review') return sendJson(res, 409, { ok: false, code: entry?.blocked_reason ?? 'PROVIDER_MIGRATION_PLAN_CHANGED' });
|
|
1921
|
+
if (inventory.default_evidence?.ok !== true) return sendJson(res, 409, { ok: false, code: inventory.default_evidence?.code ?? 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' });
|
|
1922
|
+
const revisions = readProviderSourceRevisions();
|
|
1923
|
+
const expected = stored.plan.expected_revisions ?? {};
|
|
1924
|
+
if (revisions.profile.revision !== expected.profile || (revisions.settings.ok ? revisions.settings.revision : null) !== expected.settings) {
|
|
1925
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_SOURCE_CHANGED' });
|
|
1926
|
+
}
|
|
1927
|
+
const hooks = createProviderLayerMigrationFileHooks({
|
|
1928
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1929
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1930
|
+
backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
|
|
1931
|
+
});
|
|
1932
|
+
let result;
|
|
1933
|
+
if (hub.hasProviderLease(providerId)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_IN_USE' });
|
|
1934
|
+
if (!hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
1935
|
+
if (!hub.beginProviderMutation(providerId)) {
|
|
1936
|
+
hub.endProviderStoreMutation();
|
|
1937
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
1938
|
+
}
|
|
1939
|
+
let retainProviderMutation = false;
|
|
1940
|
+
let retainProviderStoreMutation = false;
|
|
1941
|
+
try {
|
|
1942
|
+
await hooks.acquireLock();
|
|
1943
|
+
// The plan was created from a pre-lock snapshot. Re-read every
|
|
1944
|
+
// authoritative source after taking the shared store lock so a
|
|
1945
|
+
// concurrent catalog/default/recovery change cannot be hidden
|
|
1946
|
+
// behind unchanged profile/settings revisions.
|
|
1947
|
+
const lockedConfig = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
1948
|
+
const lockedInventory = await readProviderInventorySnapshot(hub, ctx, lockedConfig);
|
|
1949
|
+
const lockedEntry = lockedInventory.migration.providers.find((candidate) => candidate.provider_id === providerId);
|
|
1950
|
+
const expectedRouting = Array.isArray(stored.plan.routing_references) ? stored.plan.routing_references : [];
|
|
1951
|
+
const sameRouting = JSON.stringify(lockedEntry?.referenced_by ?? []) === JSON.stringify(expectedRouting);
|
|
1952
|
+
const sameDefault = JSON.stringify(lockedInventory.harness_default ?? null) === JSON.stringify(stored.plan.harness_default_before ?? null);
|
|
1953
|
+
const profileFile = join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml');
|
|
1954
|
+
const lockedMaterialization = existsSync(profileFile)
|
|
1955
|
+
? readProviderMaterialization(readFileSync(profileFile, 'utf8'), { providerId, file: 'harness/profiles/dsh-crew/cordis.patch.yml' })
|
|
1956
|
+
: { ok: false };
|
|
1957
|
+
const sameMaterialization = lockedMaterialization.ok === true
|
|
1958
|
+
&& JSON.stringify(lockedMaterialization.provider) === JSON.stringify(stored.plan.materialization?.provider ?? null);
|
|
1959
|
+
const settingsFile = join(CONFIG_DIR, 'harness', 'settings.yaml');
|
|
1960
|
+
const lockedUserMaterialization = stored.plan.action === 'promote-existing-user' && existsSync(settingsFile)
|
|
1961
|
+
? readProviderSettingsMaterialization(readFileSync(settingsFile, 'utf8'), { providerId, file: 'harness/settings.yaml' })
|
|
1962
|
+
: { ok: stored.plan.action !== 'promote-existing-user' };
|
|
1963
|
+
const sameUserMaterialization = stored.plan.action !== 'promote-existing-user'
|
|
1964
|
+
|| (lockedUserMaterialization.ok === true && JSON.stringify(lockedUserMaterialization.provider) === JSON.stringify(stored.plan.user_materialization_before ?? null));
|
|
1965
|
+
const lockedRevisions = readProviderSourceRevisions();
|
|
1966
|
+
const expectedLocked = stored.plan.expected_revisions ?? {};
|
|
1967
|
+
const sameRevisions = lockedRevisions.profile.revision === expectedLocked.profile
|
|
1968
|
+
&& (lockedRevisions.settings.ok ? lockedRevisions.settings.revision : null) === expectedLocked.settings;
|
|
1969
|
+
if (hasPendingProviderRecoveryTransactions(lockedInventory)) {
|
|
1970
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
1971
|
+
}
|
|
1972
|
+
if (!lockedEntry || lockedEntry.action !== stored.plan.action || lockedEntry.collision !== null
|
|
1973
|
+
|| !sameRouting || !sameDefault || !sameMaterialization || !sameUserMaterialization || !sameRevisions
|
|
1974
|
+
|| lockedInventory.default_evidence?.ok !== true
|
|
1975
|
+
|| !hasCompleteProviderCatalogEvidence(lockedInventory.catalog_evidence)
|
|
1976
|
+
|| !hasCompleteProviderDeclarationEvidence(lockedInventory.declaration_evidence)
|
|
1977
|
+
|| !hasAvailableProviderLifecycleEvidence(lockedInventory.lifecycle_evidence)) {
|
|
1978
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_PLAN_CHANGED' });
|
|
1979
|
+
}
|
|
1980
|
+
if (hub.hasProviderLease(providerId)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_IN_USE' });
|
|
1981
|
+
result = await executeProviderLayerMigration(stored.plan, hooks, { confirm: true, deferRestart: true });
|
|
1982
|
+
if (result.ok && result.state === 'RESTART_PENDING') await hooks.setRestartRuntimeBaseline(getHubRuntimeIdentity().runtime_id);
|
|
1983
|
+
providerMigrationPlans.delete(body.plan_id);
|
|
1984
|
+
retainProviderMutation = result.state === 'RESTART_PENDING';
|
|
1985
|
+
if (retainProviderMutation) {
|
|
1986
|
+
providerMigrationOwners.set(providerId, stored.plan.plan_id);
|
|
1987
|
+
providerStoreMutationOwners.set(stored.plan.plan_id, providerId);
|
|
1988
|
+
retainProviderStoreMutation = true;
|
|
1989
|
+
}
|
|
1990
|
+
return sendJson(res, result.ok ? 202 : 409, { ok: result.ok, restart_required: result.restart_required === true, result });
|
|
1991
|
+
} finally {
|
|
1992
|
+
await hooks.release();
|
|
1993
|
+
if (!retainProviderMutation) hub.endProviderMutation(providerId);
|
|
1994
|
+
if (!retainProviderStoreMutation) hub.endProviderStoreMutation();
|
|
1995
|
+
}
|
|
1996
|
+
}
|
|
1997
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'verify-migration') {
|
|
1998
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_MIGRATION_VERIFY_CONFIRM_REQUIRED' });
|
|
1999
|
+
const transactionId = body?.transaction_id;
|
|
2000
|
+
const hooks = createProviderLayerMigrationFileHooks({
|
|
2001
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
2002
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
2003
|
+
backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
|
|
2004
|
+
existingMigrationId: transactionId,
|
|
2005
|
+
});
|
|
2006
|
+
let acquiredStoreFence = false;
|
|
2007
|
+
try {
|
|
2008
|
+
await hooks.acquireLock();
|
|
2009
|
+
await hooks.backup({});
|
|
2010
|
+
const plan = hooks.backupPlan();
|
|
2011
|
+
if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_PLAN_PROVIDER_MISMATCH' });
|
|
2012
|
+
const owner = providerMigrationOwners.get(providerId);
|
|
2013
|
+
if (providerStoreMutationOwners.get(plan.plan_id) !== providerId || !hub.hasProviderStoreMutation()) {
|
|
2014
|
+
if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
2015
|
+
providerStoreMutationOwners.set(plan.plan_id, providerId);
|
|
2016
|
+
acquiredStoreFence = true;
|
|
2017
|
+
}
|
|
2018
|
+
if (owner !== plan.plan_id) {
|
|
2019
|
+
if (hub.hasProviderLease(providerId) || !hub.beginProviderMutation(providerId)) {
|
|
2020
|
+
if (acquiredStoreFence && providerStoreMutationOwners.get(plan.plan_id) === providerId) { providerStoreMutationOwners.delete(plan.plan_id); hub.endProviderStoreMutation(); }
|
|
2021
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
2022
|
+
}
|
|
2023
|
+
providerMigrationOwners.set(providerId, plan.plan_id);
|
|
2024
|
+
}
|
|
2025
|
+
const verification = await hooks.verify(plan);
|
|
2026
|
+
const live = await readProviderInventorySnapshot(hub, ctx, config);
|
|
2027
|
+
verification.catalogEvidence = live.catalog_evidence;
|
|
2028
|
+
verification.defaultEvidence = live.default_evidence;
|
|
2029
|
+
verification.defaultPreserved = live.default_evidence?.ok === true
|
|
2030
|
+
&& JSON.stringify(live.harness_default ?? null) === JSON.stringify(plan.harness_default_before ?? null);
|
|
2031
|
+
verification.liveNativeRemovable = hasCompleteProviderCatalogEvidence(live.catalog_evidence)
|
|
2032
|
+
&& live.records.some((record) => record.id === providerId && record.lifecycle?.catalogued === true && record.native_removable === true);
|
|
2033
|
+
verification.runtimeRestarted = hasProviderRuntimeRestartEvidence(plan.runtime_id_before_restart ?? plan.runtime_id_before, getHubRuntimeIdentity().runtime_id);
|
|
2034
|
+
if (verification.nativeRemovable !== true || verification.liveNativeRemovable !== true || verification.defaultPreserved !== true || verification.runtimeRestarted !== true) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_VERIFY_FAILED', verification });
|
|
2035
|
+
await hooks.finalizeVerified();
|
|
2036
|
+
if (providerMigrationOwners.get(providerId) === plan.plan_id) {
|
|
2037
|
+
providerMigrationOwners.delete(providerId);
|
|
2038
|
+
hub.endProviderMutation(providerId);
|
|
2039
|
+
}
|
|
2040
|
+
if (providerStoreMutationOwners.get(plan.plan_id) === providerId) {
|
|
2041
|
+
providerStoreMutationOwners.delete(plan.plan_id);
|
|
2042
|
+
hub.endProviderStoreMutation();
|
|
2043
|
+
}
|
|
2044
|
+
return sendJson(res, 200, { ok: true, state: 'VERIFIED', transaction_id: plan.plan_id, provider_id: providerId, verification });
|
|
2045
|
+
} finally {
|
|
2046
|
+
await hooks.release();
|
|
2047
|
+
// Keep the transaction-owned fence on verification failure so
|
|
2048
|
+
// an incomplete migration cannot race a new Worker; the caller
|
|
2049
|
+
// must explicitly retry verification or roll it back.
|
|
2050
|
+
}
|
|
2051
|
+
}
|
|
2052
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'rollback-migration') {
|
|
2053
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_MIGRATION_ROLLBACK_CONFIRM_REQUIRED' });
|
|
2054
|
+
const hooks = createProviderLayerMigrationFileHooks({
|
|
2055
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
2056
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
2057
|
+
backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
|
|
2058
|
+
existingMigrationId: body?.transaction_id,
|
|
2059
|
+
});
|
|
2060
|
+
let acquiredMigrationFence = false;
|
|
2061
|
+
let retainProviderMutation = false;
|
|
2062
|
+
let acquiredStoreFence = false;
|
|
2063
|
+
try {
|
|
2064
|
+
await hooks.acquireLock();
|
|
2065
|
+
await hooks.backup({});
|
|
2066
|
+
const plan = hooks.backupPlan();
|
|
2067
|
+
if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_PLAN_PROVIDER_MISMATCH' });
|
|
2068
|
+
const owner = providerMigrationOwners.get(providerId);
|
|
2069
|
+
if (providerStoreMutationOwners.get(plan.plan_id) !== providerId || !hub.hasProviderStoreMutation()) {
|
|
2070
|
+
if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
2071
|
+
providerStoreMutationOwners.set(plan.plan_id, providerId);
|
|
2072
|
+
acquiredStoreFence = true;
|
|
2073
|
+
}
|
|
2074
|
+
if (owner !== plan.plan_id) {
|
|
2075
|
+
if (hub.hasProviderLease(providerId) || !hub.beginProviderMutation(providerId)) {
|
|
2076
|
+
if (acquiredStoreFence && providerStoreMutationOwners.get(plan.plan_id) === providerId) { providerStoreMutationOwners.delete(plan.plan_id); hub.endProviderStoreMutation(); }
|
|
2077
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
2078
|
+
}
|
|
2079
|
+
providerMigrationOwners.set(providerId, plan.plan_id);
|
|
2080
|
+
acquiredMigrationFence = true;
|
|
2081
|
+
retainProviderMutation = true;
|
|
2082
|
+
}
|
|
2083
|
+
const rollback = await hooks.rollback();
|
|
2084
|
+
await hooks.setRollbackRuntimeBaseline(getHubRuntimeIdentity().runtime_id);
|
|
2085
|
+
providerMigrationOwners.set(providerId, plan.plan_id);
|
|
2086
|
+
retainProviderMutation = true;
|
|
2087
|
+
return sendJson(res, 202, { ok: true, restart_required: true, transaction_id: plan.plan_id, provider_id: providerId, state: 'ROLLBACK_RESTART_PENDING' });
|
|
2088
|
+
} finally {
|
|
2089
|
+
await hooks.release();
|
|
2090
|
+
if (acquiredMigrationFence && !retainProviderMutation && providerMigrationOwners.get(providerId) === body?.transaction_id) {
|
|
2091
|
+
providerMigrationOwners.delete(providerId);
|
|
2092
|
+
hub.endProviderMutation(providerId);
|
|
2093
|
+
}
|
|
2094
|
+
}
|
|
2095
|
+
}
|
|
2096
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'verify-rollback-migration') {
|
|
2097
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_MIGRATION_ROLLBACK_VERIFY_CONFIRM_REQUIRED' });
|
|
2098
|
+
const transactionId = body?.transaction_id;
|
|
2099
|
+
const hooks = createProviderLayerMigrationFileHooks({
|
|
2100
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
2101
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
2102
|
+
backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
|
|
2103
|
+
existingMigrationId: transactionId,
|
|
2104
|
+
});
|
|
2105
|
+
let acquiredStoreFence = false;
|
|
2106
|
+
try {
|
|
2107
|
+
await hooks.acquireLock();
|
|
2108
|
+
await hooks.backup({});
|
|
2109
|
+
const plan = hooks.backupPlan();
|
|
2110
|
+
if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_PLAN_PROVIDER_MISMATCH' });
|
|
2111
|
+
const owner = providerMigrationOwners.get(providerId);
|
|
2112
|
+
if (providerStoreMutationOwners.get(plan.plan_id) !== providerId || !hub.hasProviderStoreMutation()) {
|
|
2113
|
+
if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
2114
|
+
providerStoreMutationOwners.set(plan.plan_id, providerId);
|
|
2115
|
+
acquiredStoreFence = true;
|
|
2116
|
+
}
|
|
2117
|
+
if (owner !== plan.plan_id) {
|
|
2118
|
+
if (hub.hasProviderLease(providerId) || !hub.beginProviderMutation(providerId)) {
|
|
2119
|
+
if (acquiredStoreFence && providerStoreMutationOwners.get(plan.plan_id) === providerId) { providerStoreMutationOwners.delete(plan.plan_id); hub.endProviderStoreMutation(); }
|
|
2120
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
2121
|
+
}
|
|
2122
|
+
providerMigrationOwners.set(providerId, plan.plan_id);
|
|
2123
|
+
}
|
|
2124
|
+
const verification = await hooks.verifyRollback(plan);
|
|
2125
|
+
const live = await readProviderInventorySnapshot(hub, ctx, config);
|
|
2126
|
+
verification.catalogEvidence = live.catalog_evidence;
|
|
2127
|
+
verification.defaultEvidence = live.default_evidence;
|
|
2128
|
+
verification.defaultPreserved = live.default_evidence?.ok === true
|
|
2129
|
+
&& JSON.stringify(live.harness_default ?? null) === JSON.stringify(plan.harness_default_before ?? null);
|
|
2130
|
+
verification.liveRestored = hasCompleteProviderCatalogEvidence(live.catalog_evidence)
|
|
2131
|
+
&& live.records.some((record) => record.id === providerId && record.lifecycle?.catalogued === true && record.native_removable === false);
|
|
2132
|
+
verification.runtimeRestarted = hasProviderRuntimeRestartEvidence(plan.rollback_runtime_id_before, getHubRuntimeIdentity().runtime_id);
|
|
2133
|
+
if (verification.ok !== true || verification.liveRestored !== true || verification.defaultPreserved !== true || verification.runtimeRestarted !== true) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_ROLLBACK_VERIFY_FAILED', verification });
|
|
2134
|
+
await hooks.finalizeRollback();
|
|
2135
|
+
if (providerMigrationOwners.get(providerId) === plan.plan_id) {
|
|
2136
|
+
providerMigrationOwners.delete(providerId);
|
|
2137
|
+
hub.endProviderMutation(providerId);
|
|
2138
|
+
}
|
|
2139
|
+
if (providerStoreMutationOwners.get(plan.plan_id) === providerId) {
|
|
2140
|
+
providerStoreMutationOwners.delete(plan.plan_id);
|
|
2141
|
+
hub.endProviderStoreMutation();
|
|
2142
|
+
}
|
|
2143
|
+
return sendJson(res, 200, { ok: true, state: 'ROLLED_BACK', transaction_id: plan.plan_id, provider_id: providerId, verification });
|
|
2144
|
+
} finally { await hooks.release(); }
|
|
2145
|
+
}
|
|
2146
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'delete-plan') {
|
|
2147
|
+
if (providerId !== 'deepseek-official' && hasPendingProviderRecoveryTransactions(inventory)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
2148
|
+
if (providerId !== 'deepseek-official' && !hasCompleteProviderCatalogEvidence(inventory.catalog_evidence)) return sendJson(res, 503, { ok: false, code: 'MODEL_CATALOG_UNAVAILABLE' });
|
|
2149
|
+
if (providerId !== 'deepseek-official' && !hasCompleteProviderDeclarationEvidence(inventory.declaration_evidence)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_SOURCE_UNRESOLVED' });
|
|
2150
|
+
if (providerId !== 'deepseek-official' && !hasAvailableProviderLifecycleEvidence(inventory.lifecycle_evidence)) return sendJson(res, 503, { ok: false, code: 'PROVIDER_LIFECYCLE_UNAVAILABLE' });
|
|
2151
|
+
if (providerId !== 'deepseek-official' && inventory.default_evidence?.ok !== true) return sendJson(res, 409, { ok: false, code: inventory.default_evidence?.code ?? 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' });
|
|
2152
|
+
const revisions = readProviderSourceRevisions();
|
|
2153
|
+
if (!revisions.profile.ok && !revisions.settings.ok) return sendJson(res, 409, { ok: false, code: revisions.profile.code ?? revisions.settings.code });
|
|
2154
|
+
const expectedRevision = (revisions.profile.ok ? revisions.profile.revision : null) ?? revisions.settings.revision;
|
|
2155
|
+
const planned = planProviderDelete({
|
|
2156
|
+
providerId,
|
|
2157
|
+
inventory,
|
|
2158
|
+
activeJobs: hub.list(),
|
|
2159
|
+
replacementDefault: body?.replacement_default,
|
|
2160
|
+
expectedRevision,
|
|
2161
|
+
expectedRevisions: {
|
|
2162
|
+
profile: revisions.profile.ok ? revisions.profile.revision : null,
|
|
2163
|
+
settings: revisions.settings.ok ? revisions.settings.revision : null,
|
|
2164
|
+
},
|
|
2165
|
+
});
|
|
2166
|
+
if (!planned.ok) return sendJson(res, 409, { ok: false, code: planned.code });
|
|
2167
|
+
const plan = { ...planned.plan, runtime_id_before: getHubRuntimeIdentity().runtime_id };
|
|
2168
|
+
rememberProviderDeletePlan(plan);
|
|
2169
|
+
return sendJson(res, 200, { ok: true, profile_revision: expectedRevision, plan });
|
|
2170
|
+
}
|
|
2171
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'probe') {
|
|
2172
|
+
const record = inventory.records.find((entry) => entry.id === providerId);
|
|
2173
|
+
if (!record) return sendJson(res, 404, { ok: false, code: 'PROVIDER_NOT_FOUND' });
|
|
2174
|
+
if (record.desired_state === 'absent') return sendJson(res, 409, { ok: false, code: 'PROVIDER_TOMBSTONED' });
|
|
2175
|
+
const model = selectProviderProbeModel({ providerId, record, config });
|
|
2176
|
+
if (!model) return sendJson(res, 409, { ok: false, code: 'PROVIDER_PROBE_MODEL_UNAVAILABLE' });
|
|
2177
|
+
let explicitProviderProbe;
|
|
2178
|
+
try { explicitProviderProbe = ctx?.providerProbe; } catch { explicitProviderProbe = undefined; }
|
|
2179
|
+
const providerProbe = typeof explicitProviderProbe === 'function' ? explicitProviderProbe : builtInProviderProbe;
|
|
2180
|
+
if (typeof providerProbe !== 'function') return sendJson(res, 503, { ok: false, code: 'PROVIDER_PROBE_UNAVAILABLE' });
|
|
2181
|
+
let observed;
|
|
2182
|
+
let timer;
|
|
2183
|
+
try {
|
|
2184
|
+
const timeout = new Promise((resolve) => { timer = setTimeout(() => resolve({ error: { name: 'TimeoutError' } }), 15_000); });
|
|
2185
|
+
observed = await Promise.race([
|
|
2186
|
+
Promise.resolve().then(() => providerProbe({ provider: providerId, model, signal: AbortSignal.timeout(15_000) })),
|
|
2187
|
+
timeout,
|
|
2188
|
+
]);
|
|
2189
|
+
} catch (error) {
|
|
2190
|
+
// Keep probe transport failures as bounded health evidence so a
|
|
2191
|
+
// credential/quota/timeout result is visible to routing policy;
|
|
2192
|
+
// never copy the raw provider error into the response.
|
|
2193
|
+
observed = { error };
|
|
2194
|
+
} finally {
|
|
2195
|
+
clearTimeout(timer);
|
|
2196
|
+
}
|
|
2197
|
+
const health = hub.healthStore.record(providerId, model, observed?.ok === true ? { ok: true } : { error: observed?.error ?? observed });
|
|
2198
|
+
return sendJson(res, 200, { ok: true, provider_id: providerId, model, health });
|
|
2199
|
+
}
|
|
2200
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'verify-delete') {
|
|
2201
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_DELETE_VERIFY_CONFIRM_REQUIRED' });
|
|
2202
|
+
const { transaction_id: transactionId } = body ?? {};
|
|
2203
|
+
const hooks = createProviderDeleteFileHooks({
|
|
2204
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
2205
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
2206
|
+
configFile: join(CONFIG_DIR, 'config.json'),
|
|
2207
|
+
lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
2208
|
+
backupDir: join(CONFIG_DIR, 'provider-backups'),
|
|
2209
|
+
sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
|
|
2210
|
+
existingBackupId: transactionId,
|
|
2211
|
+
expectedProviderId: providerId,
|
|
2212
|
+
});
|
|
2213
|
+
const plan = hooks.backupPlan();
|
|
2214
|
+
if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_PLAN_PROVIDER_MISMATCH' });
|
|
2215
|
+
await hooks.acquireLock();
|
|
2216
|
+
try {
|
|
2217
|
+
if (providerStoreMutationOwners.get(transactionId) !== providerId || !hub.hasProviderStoreMutation()) {
|
|
2218
|
+
if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_BUSY' });
|
|
2219
|
+
providerStoreMutationOwners.set(transactionId, providerId);
|
|
2220
|
+
}
|
|
2221
|
+
const verification = await hooks.verify(plan);
|
|
2222
|
+
const live = await readProviderInventorySnapshot(hub, ctx, config);
|
|
2223
|
+
verification.catalog_evidence = live.catalog_evidence;
|
|
2224
|
+
verification.default_evidence = live.default_evidence;
|
|
2225
|
+
verification.catalogAbsent = hasCompleteProviderCatalogEvidence(live.catalog_evidence)
|
|
2226
|
+
&& !live.records.some((entry) => entry.id === providerId && entry.lifecycle?.catalogued === true);
|
|
2227
|
+
verification.harnessDefault = plan.was_harness_default !== true || (
|
|
2228
|
+
live.harness_default?.provider === plan.replacement_default
|
|
2229
|
+
&& live.harness_default?.model === plan.replacement_default_model
|
|
2230
|
+
);
|
|
2231
|
+
verification.runtimeRestarted = hasProviderRuntimeRestartEvidence(plan.delete_runtime_id_before_restart, getHubRuntimeIdentity().runtime_id);
|
|
2232
|
+
if (verification?.providerAbsent !== true || !hasCompleteProviderCatalogEvidence(verification?.catalog_evidence) || verification?.default_evidence?.ok !== true || verification?.catalogAbsent !== true || verification?.harnessDefault !== true || verification?.runtimeRestarted !== true || verification?.routingClear !== true || verification?.tombstonePresent !== true) {
|
|
2233
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_VERIFY_FAILED', verification });
|
|
2234
|
+
}
|
|
2235
|
+
await hooks.recordTransaction({ transaction_id: transactionId, provider_id: providerId, state: 'VERIFIED' }, plan);
|
|
2236
|
+
hub.endProviderMutation(providerId);
|
|
2237
|
+
if (providerStoreMutationOwners.get(transactionId) === providerId) {
|
|
2238
|
+
providerStoreMutationOwners.delete(transactionId);
|
|
2239
|
+
hub.endProviderStoreMutation();
|
|
2240
|
+
}
|
|
2241
|
+
return sendJson(res, 200, { ok: true, transaction_id: transactionId, provider_id: providerId, state: 'VERIFIED', verification });
|
|
2242
|
+
} finally {
|
|
2243
|
+
await hooks.release();
|
|
2244
|
+
}
|
|
2245
|
+
}
|
|
2246
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'rollback') {
|
|
2247
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_ROLLBACK_CONFIRM_REQUIRED' });
|
|
2248
|
+
const { transaction_id: transactionId } = body ?? {};
|
|
2249
|
+
const hooks = createProviderDeleteFileHooks({
|
|
2250
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
2251
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
2252
|
+
configFile: join(CONFIG_DIR, 'config.json'),
|
|
2253
|
+
lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
2254
|
+
backupDir: join(CONFIG_DIR, 'provider-backups'),
|
|
2255
|
+
sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
|
|
2256
|
+
existingBackupId: transactionId,
|
|
2257
|
+
expectedProviderId: providerId,
|
|
2258
|
+
});
|
|
2259
|
+
await hooks.acquireLock();
|
|
2260
|
+
let acquiredStoreMutation = false;
|
|
2261
|
+
let retainStoreMutation = false;
|
|
2262
|
+
try {
|
|
2263
|
+
const plan = hooks.backupPlan();
|
|
2264
|
+
if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_PLAN_PROVIDER_MISMATCH' });
|
|
2265
|
+
if (providerStoreMutationOwners.get(transactionId) !== providerId || !hub.hasProviderStoreMutation()) {
|
|
2266
|
+
if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_BUSY' });
|
|
2267
|
+
providerStoreMutationOwners.set(transactionId, providerId);
|
|
2268
|
+
acquiredStoreMutation = true;
|
|
2269
|
+
}
|
|
2270
|
+
await hooks.rollback(plan);
|
|
2271
|
+
await hooks.setRuntimeBaseline(getHubRuntimeIdentity().runtime_id, 'rollback');
|
|
2272
|
+
const rollbackPlan = hooks.backupPlan() ?? plan;
|
|
2273
|
+
try {
|
|
2274
|
+
await hooks.recordTransaction({ transaction_id: transactionId, provider_id: providerId, state: 'ROLLBACK_PENDING' }, rollbackPlan);
|
|
2275
|
+
} catch (error) {
|
|
2276
|
+
return sendJson(res, 409, { ok: false, code: boundedMachineCodeFromError(error) ?? 'PROVIDER_LIFECYCLE_RECORD_FAILED' });
|
|
2277
|
+
}
|
|
2278
|
+
retainStoreMutation = true;
|
|
2279
|
+
return sendJson(res, 202, { ok: true, restart_required: true, transaction_id: transactionId, provider_id: providerId, state: 'ROLLBACK_PENDING' });
|
|
2280
|
+
} finally {
|
|
2281
|
+
await hooks.release();
|
|
2282
|
+
if (acquiredStoreMutation && !retainStoreMutation && providerStoreMutationOwners.get(transactionId) === providerId) {
|
|
2283
|
+
providerStoreMutationOwners.delete(transactionId);
|
|
2284
|
+
hub.endProviderStoreMutation();
|
|
2285
|
+
}
|
|
2286
|
+
}
|
|
2287
|
+
}
|
|
2288
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'verify-rollback') {
|
|
2289
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_ROLLBACK_VERIFY_CONFIRM_REQUIRED' });
|
|
2290
|
+
const { transaction_id: transactionId } = body ?? {};
|
|
2291
|
+
const hooks = createProviderDeleteFileHooks({
|
|
2292
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
2293
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
2294
|
+
configFile: join(CONFIG_DIR, 'config.json'),
|
|
2295
|
+
lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
2296
|
+
backupDir: join(CONFIG_DIR, 'provider-backups'),
|
|
2297
|
+
sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
|
|
2298
|
+
existingBackupId: transactionId,
|
|
2299
|
+
expectedProviderId: providerId,
|
|
2300
|
+
});
|
|
2301
|
+
const plan = hooks.backupPlan();
|
|
2302
|
+
if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_PLAN_PROVIDER_MISMATCH' });
|
|
2303
|
+
await hooks.acquireLock();
|
|
2304
|
+
try {
|
|
2305
|
+
if (providerStoreMutationOwners.get(transactionId) !== providerId || !hub.hasProviderStoreMutation()) {
|
|
2306
|
+
if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_BUSY' });
|
|
2307
|
+
providerStoreMutationOwners.set(transactionId, providerId);
|
|
2308
|
+
}
|
|
2309
|
+
const verified = await hooks.verifyRollback(plan);
|
|
2310
|
+
const live = await readProviderInventorySnapshot(hub, ctx, config);
|
|
2311
|
+
verified.catalog_evidence = live.catalog_evidence;
|
|
2312
|
+
verified.default_evidence = live.default_evidence;
|
|
2313
|
+
verified.catalogPresent = hasCompleteProviderCatalogEvidence(live.catalog_evidence)
|
|
2314
|
+
&& live.records.some((entry) => entry.id === providerId && entry.lifecycle?.catalogued === true);
|
|
2315
|
+
verified.harnessDefaultRestored = plan.was_harness_default !== true || (
|
|
2316
|
+
live.harness_default?.provider === plan.harness_default_before?.provider
|
|
2317
|
+
&& live.harness_default?.model === plan.harness_default_before?.model
|
|
2318
|
+
);
|
|
2319
|
+
verified.runtimeRestarted = hasProviderRuntimeRestartEvidence(plan.rollback_runtime_id_before_restart, getHubRuntimeIdentity().runtime_id);
|
|
2320
|
+
if (verified?.ok !== true || !hasCompleteProviderCatalogEvidence(verified?.catalog_evidence) || verified?.default_evidence?.ok !== true || verified?.catalogPresent !== true || verified?.harnessDefaultRestored !== true || verified?.runtimeRestarted !== true) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_ROLLBACK_VERIFY_FAILED', verification: verified });
|
|
2321
|
+
await hooks.recordTransaction({ transaction_id: transactionId, provider_id: providerId, state: 'ROLLED_BACK' }, plan);
|
|
2322
|
+
hub.endProviderMutation(providerId);
|
|
2323
|
+
if (providerStoreMutationOwners.get(transactionId) === providerId) {
|
|
2324
|
+
providerStoreMutationOwners.delete(transactionId);
|
|
2325
|
+
hub.endProviderStoreMutation();
|
|
2326
|
+
}
|
|
2327
|
+
return sendJson(res, 200, { ok: true, transaction_id: transactionId, provider_id: providerId, state: 'ROLLED_BACK' });
|
|
2328
|
+
} finally {
|
|
2329
|
+
await hooks.release();
|
|
2330
|
+
}
|
|
2331
|
+
}
|
|
2332
|
+
if (req.method === 'DELETE' && parts.length === 1) {
|
|
2333
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_DELETE_CONFIRM_REQUIRED' });
|
|
2334
|
+
if (body?.purge_orphan_credentials === true) {
|
|
2335
|
+
return sendJson(res, 400, { ok: false, code: 'CREDENTIAL_PURGE_REQUIRES_EXPLICIT_CONFIRMATION' });
|
|
2336
|
+
}
|
|
2337
|
+
const stored = providerDeletePlans.get(body?.plan_id);
|
|
2338
|
+
if (!stored || Date.now() - stored.created_at > PROVIDER_DELETE_PLAN_TTL_MS || stored.plan.provider_id !== providerId) {
|
|
2339
|
+
providerDeletePlans.delete(body?.plan_id);
|
|
2340
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_PLAN_EXPIRED' });
|
|
2341
|
+
}
|
|
2342
|
+
const revisions = readProviderSourceRevisions();
|
|
2343
|
+
if (!revisions.profile.ok && !revisions.settings.ok) return sendJson(res, 409, { ok: false, code: revisions.profile.code ?? revisions.settings.code });
|
|
2344
|
+
if (providerId !== 'deepseek-official' && !hasCompleteProviderCatalogEvidence(inventory.catalog_evidence)) return sendJson(res, 503, { ok: false, code: 'MODEL_CATALOG_UNAVAILABLE' });
|
|
2345
|
+
if (providerId !== 'deepseek-official' && !hasCompleteProviderDeclarationEvidence(inventory.declaration_evidence)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_SOURCE_UNRESOLVED' });
|
|
2346
|
+
if (providerId !== 'deepseek-official' && !hasAvailableProviderLifecycleEvidence(inventory.lifecycle_evidence)) return sendJson(res, 503, { ok: false, code: 'PROVIDER_LIFECYCLE_UNAVAILABLE' });
|
|
2347
|
+
if (providerId !== 'deepseek-official' && inventory.default_evidence?.ok !== true) return sendJson(res, 409, { ok: false, code: inventory.default_evidence?.code ?? 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' });
|
|
2348
|
+
const storedExpected = stored.plan.expected_revisions;
|
|
2349
|
+
const revisionsMatch = storedExpected
|
|
2350
|
+
? (storedExpected.profile === (revisions.profile.ok ? revisions.profile.revision : null)
|
|
2351
|
+
&& storedExpected.settings === (revisions.settings.ok ? revisions.settings.revision : null))
|
|
2352
|
+
: (body?.expected_revision === stored.plan.expected_revision && (revisions.profile.revision ?? revisions.settings.revision) === stored.plan.expected_revision);
|
|
2353
|
+
if (!revisionsMatch || body?.expected_revision !== stored.plan.expected_revision) {
|
|
2354
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_PROFILE_CHANGED' });
|
|
2355
|
+
}
|
|
2356
|
+
if (providerId !== 'deepseek-official' && hasPendingProviderRecoveryTransactions(inventory)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
2357
|
+
const refreshed = planProviderDelete({
|
|
2358
|
+
providerId,
|
|
2359
|
+
inventory,
|
|
2360
|
+
activeJobs: hub.list(),
|
|
2361
|
+
replacementDefault: stored.plan.replacement_default,
|
|
2362
|
+
expectedRevision: stored.plan.expected_revision,
|
|
2363
|
+
expectedRevisions: stored.plan.expected_revisions,
|
|
2364
|
+
});
|
|
2365
|
+
if (!refreshed.ok) return sendJson(res, 409, { ok: false, code: refreshed.code });
|
|
2366
|
+
if (refreshed.plan.replacement_default_model !== stored.plan.replacement_default_model) {
|
|
2367
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_PLAN_CHANGED' });
|
|
2368
|
+
}
|
|
2369
|
+
let executionPlan = {
|
|
2370
|
+
...refreshed.plan,
|
|
2371
|
+
plan_id: stored.plan.plan_id,
|
|
2372
|
+
expected_revision: stored.plan.expected_revision,
|
|
2373
|
+
expected_revisions: stored.plan.expected_revisions,
|
|
2374
|
+
runtime_id_before: stored.plan.runtime_id_before,
|
|
2375
|
+
};
|
|
2376
|
+
if (!hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_BUSY' });
|
|
2377
|
+
if (!hub.beginProviderMutation(providerId)) {
|
|
2378
|
+
hub.endProviderStoreMutation();
|
|
2379
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_BUSY' });
|
|
2380
|
+
}
|
|
2381
|
+
let hooks = null;
|
|
2382
|
+
let lockAcquired = false;
|
|
2383
|
+
let retainProviderMutation = false;
|
|
2384
|
+
let retainProviderStoreMutation = false;
|
|
2385
|
+
try {
|
|
2386
|
+
hooks = createProviderDeleteFileHooks({
|
|
2387
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
2388
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
2389
|
+
configFile: join(CONFIG_DIR, 'config.json'),
|
|
2390
|
+
lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
2391
|
+
backupDir: join(CONFIG_DIR, 'provider-backups'),
|
|
2392
|
+
sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
|
|
2393
|
+
runtimeIdProvider: () => getHubRuntimeIdentity().runtime_id,
|
|
2394
|
+
afterLockAcquired: () => {
|
|
2395
|
+
const pendingRecovery = [
|
|
2396
|
+
...readProviderRecoveryTransactions(),
|
|
2397
|
+
...readProviderLayerMigrationTransactions(join(CONFIG_DIR, 'provider-migration-backups')),
|
|
2398
|
+
];
|
|
2399
|
+
if (pendingRecovery.length > 0) {
|
|
2400
|
+
throw Object.assign(new Error('provider recovery transaction is pending'), { code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
2401
|
+
}
|
|
2402
|
+
if (hub.hasProviderLease(providerId)) {
|
|
2403
|
+
throw Object.assign(new Error('provider is in use'), { code: 'PROVIDER_IN_USE' });
|
|
2404
|
+
}
|
|
2405
|
+
},
|
|
2406
|
+
});
|
|
2407
|
+
// Hold the real shared store lock while rebuilding the live
|
|
2408
|
+
// inventory and delete plan. The pre-lock plan is only a
|
|
2409
|
+
// proposal; catalog/default/ownership/recovery facts must be
|
|
2410
|
+
// authoritative at the mutation boundary.
|
|
2411
|
+
await hooks.acquireLock();
|
|
2412
|
+
lockAcquired = true;
|
|
2413
|
+
const lockedConfig = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
2414
|
+
const lockedInventory = await readProviderInventorySnapshot(hub, ctx, lockedConfig);
|
|
2415
|
+
const lockedRevisions = readProviderSourceRevisions();
|
|
2416
|
+
const lockedExpected = stored.plan.expected_revisions;
|
|
2417
|
+
const lockedRevisionsMatch = lockedExpected
|
|
2418
|
+
? lockedExpected.profile === (lockedRevisions.profile.ok ? lockedRevisions.profile.revision : null)
|
|
2419
|
+
&& lockedExpected.settings === (lockedRevisions.settings.ok ? lockedRevisions.settings.revision : null)
|
|
2420
|
+
: false;
|
|
2421
|
+
if (providerId !== 'deepseek-official' && !hasCompleteProviderCatalogEvidence(lockedInventory.catalog_evidence)) throw Object.assign(new Error('provider catalog unavailable after lock'), { code: 'MODEL_CATALOG_UNAVAILABLE' });
|
|
2422
|
+
if (providerId !== 'deepseek-official' && !hasCompleteProviderDeclarationEvidence(lockedInventory.declaration_evidence)) throw Object.assign(new Error('provider declarations unavailable after lock'), { code: 'PROVIDER_DELETE_SOURCE_UNRESOLVED' });
|
|
2423
|
+
if (providerId !== 'deepseek-official' && !hasAvailableProviderLifecycleEvidence(lockedInventory.lifecycle_evidence)) throw Object.assign(new Error('provider lifecycle unavailable after lock'), { code: 'PROVIDER_LIFECYCLE_UNAVAILABLE' });
|
|
2424
|
+
if (providerId !== 'deepseek-official' && lockedInventory.default_evidence?.ok !== true) throw Object.assign(new Error('provider default authority changed after lock'), { code: lockedInventory.default_evidence?.code ?? 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' });
|
|
2425
|
+
if (providerId !== 'deepseek-official' && hasPendingProviderRecoveryTransactions(lockedInventory)) throw Object.assign(new Error('provider recovery transaction is pending'), { code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
2426
|
+
if (!lockedRevisionsMatch) throw Object.assign(new Error('provider source changed after lock'), { code: 'PROVIDER_PROFILE_CHANGED' });
|
|
2427
|
+
if (hub.hasProviderLease(providerId)) throw Object.assign(new Error('provider is in use'), { code: 'PROVIDER_IN_USE' });
|
|
2428
|
+
const lockedPlan = planProviderDelete({
|
|
2429
|
+
providerId,
|
|
2430
|
+
inventory: lockedInventory,
|
|
2431
|
+
activeJobs: hub.list(),
|
|
2432
|
+
replacementDefault: stored.plan.replacement_default,
|
|
2433
|
+
expectedRevision: stored.plan.expected_revision,
|
|
2434
|
+
expectedRevisions: stored.plan.expected_revisions,
|
|
2435
|
+
});
|
|
2436
|
+
if (!lockedPlan.ok) throw Object.assign(new Error('provider delete plan changed after lock'), { code: lockedPlan.code });
|
|
2437
|
+
const sameLockedPlan = lockedPlan.plan.replacement_default_model === stored.plan.replacement_default_model
|
|
2438
|
+
&& lockedPlan.plan.was_harness_default === stored.plan.was_harness_default
|
|
2439
|
+
&& JSON.stringify(lockedPlan.plan.declaration_authorities) === JSON.stringify(stored.plan.declaration_authorities)
|
|
2440
|
+
&& JSON.stringify(lockedPlan.plan.harness_default_before ?? null) === JSON.stringify(stored.plan.harness_default_before ?? null)
|
|
2441
|
+
&& JSON.stringify(lockedPlan.plan.credential_refs) === JSON.stringify(stored.plan.credential_refs);
|
|
2442
|
+
if (!sameLockedPlan) throw Object.assign(new Error('provider delete plan semantics changed after lock'), { code: 'PROVIDER_DELETE_PLAN_CHANGED' });
|
|
2443
|
+
executionPlan = {
|
|
2444
|
+
...lockedPlan.plan,
|
|
2445
|
+
plan_id: stored.plan.plan_id,
|
|
2446
|
+
expected_revision: stored.plan.expected_revision,
|
|
2447
|
+
expected_revisions: stored.plan.expected_revisions,
|
|
2448
|
+
runtime_id_before: stored.plan.runtime_id_before,
|
|
2449
|
+
};
|
|
2450
|
+
providerDeletePlans.delete(body.plan_id);
|
|
2451
|
+
const { executeProviderDelete } = await import('../provider-lifecycle.mjs');
|
|
2452
|
+
const result = await executeProviderDelete(executionPlan, hooks, { deferRestart: true });
|
|
2453
|
+
retainProviderMutation = result.state === 'RESTART_PENDING';
|
|
2454
|
+
if (retainProviderMutation) {
|
|
2455
|
+
providerStoreMutationOwners.set(executionPlan.plan_id, providerId);
|
|
2456
|
+
retainProviderStoreMutation = true;
|
|
2457
|
+
}
|
|
2458
|
+
return sendJson(res, result.state === 'RESTART_PENDING' ? 202 : 409, { ok: result.state === 'RESTART_PENDING', restart_required: true, result });
|
|
2459
|
+
} finally {
|
|
2460
|
+
if (lockAcquired && hooks) await hooks.release().catch(() => {});
|
|
2461
|
+
if (!retainProviderMutation) hub.endProviderMutation(providerId);
|
|
2462
|
+
if (!retainProviderStoreMutation) hub.endProviderStoreMutation();
|
|
2463
|
+
}
|
|
2464
|
+
}
|
|
2465
|
+
return sendJson(res, 404, { ok: false, error: 'unknown providers endpoint' });
|
|
2466
|
+
} catch (err) {
|
|
2467
|
+
const isProbe = req.method === 'POST' && parts.length === 2 && parts[1] === 'probe';
|
|
2468
|
+
const code = boundedMachineCodeFromError(err) ?? (isProbe ? 'PROVIDER_PROBE_FAILED' : 'PROVIDER_PLAN_INVALID');
|
|
2469
|
+
const status = code === 'PROVIDER_DELETE_RESTART_SUPERVISOR_UNAVAILABLE' || code.startsWith('PROVIDER_PROBE') ? 503
|
|
2470
|
+
: code.startsWith('PROVIDER_') || code.startsWith('CREDENTIAL_') ? 409 : 400;
|
|
2471
|
+
return sendJson(res, status, { ok: false, code });
|
|
2472
|
+
}
|
|
2473
|
+
},
|
|
2474
|
+
}));
|
|
2475
|
+
|
|
2476
|
+
disposers.push(webServer.register({
|
|
2477
|
+
kind: 'exact', path: `${ROUTE_BASE}/provider-health`,
|
|
2478
|
+
handler: async (req, res) => {
|
|
2479
|
+
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
2480
|
+
if (req.method !== 'GET') return sendJson(res, 405, { ok: false, error: 'GET only' }, { allow: 'GET' });
|
|
2481
|
+
return sendJson(res, 200, {
|
|
2482
|
+
ok: true,
|
|
2483
|
+
schema_version: 1,
|
|
2484
|
+
health: hub.healthStore.list(),
|
|
2485
|
+
runtime: getHubRuntimeIdentity(),
|
|
2486
|
+
});
|
|
2487
|
+
},
|
|
2488
|
+
}));
|
|
2489
|
+
|
|
2490
|
+
disposers.push(webServer.register({
|
|
2491
|
+
kind: 'exact', path: `${ROUTE_BASE}/models`,
|
|
2375
2492
|
handler: async (req, res) => {
|
|
2376
2493
|
if (!isLoopbackRequest(req)) return sendJson(res, 403, { ok: false, error: 'loopback only' });
|
|
2377
2494
|
if (req.method !== 'GET' && req.method !== 'POST') return sendJson(res, 405, { ok: false, error: 'GET or POST' }, { allow: 'GET, POST' });
|