@ran-sh/dsh-crew 1.0.1 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -20
- package/README.zh.md +25 -18
- package/lib/client.js +336 -16
- package/official-web-bridge/lib/client.js +336 -16
- package/package.json +166 -166
- package/src/client/index.tsx +853 -716
- package/src/credential-reference.mjs +38 -0
- package/src/hub/index.mjs +890 -65
- package/src/install/install-legacy.mjs +184 -185
- package/src/install/npx-lifecycle.mjs +456 -398
- package/src/model-catalog.mjs +8 -1
- package/src/policy-legacy.mjs +2 -2
- package/src/provider-delete-adapters.mjs +1934 -451
- package/src/provider-inventory.mjs +184 -132
- package/src/provider-layer-migration-adapters.mjs +759 -0
- package/src/provider-layer-migration.mjs +198 -0
- package/src/provider-lifecycle-state.mjs +103 -95
- package/src/provider-lifecycle.mjs +252 -214
- package/src/provider-profile-store.mjs +378 -171
- package/src/provider-settings-store.mjs +291 -0
- package/src/provider-store-lock.mjs +67 -0
- package/src/runtime-identity.mjs +73 -72
package/src/hub/index.mjs
CHANGED
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
// Web UI session list), exposes a loopback jobs API for the CC/Codex MCP shim,
|
|
4
4
|
// and serves the one-click installer endpoints for the settings page.
|
|
5
5
|
|
|
6
|
-
import { randomUUID } from 'node:crypto';
|
|
7
|
-
import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
|
|
8
|
-
import { join, isAbsolute } from 'node:path';
|
|
6
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
7
|
+
import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
|
|
8
|
+
import { dirname, join, isAbsolute, resolve as resolvePath } from 'node:path';
|
|
9
9
|
import { homedir } from 'node:os';
|
|
10
10
|
import { createShardWriter, readMergedStatus } from '../status-shard.mjs';
|
|
11
11
|
import {
|
|
@@ -34,16 +34,19 @@ import { buildHubExecutionRows } from '../config-readiness.mjs';
|
|
|
34
34
|
import { localRequestCore, originLoopback } from '../local-request-guard.mjs';
|
|
35
35
|
import { raceWaiters } from '../removable-waiter.mjs';
|
|
36
36
|
import { buildProviderInventory } from '../provider-inventory.mjs';
|
|
37
|
-
import {
|
|
37
|
+
import { buildProviderLayerMigrationPlan, executeProviderLayerMigration } from '../provider-layer-migration.mjs';
|
|
38
|
+
import { inspectProviderProfile, readProviderDeclarations, readProviderMaterialization } from '../provider-profile-store.mjs';
|
|
39
|
+
import { inspectProviderSettings, readHarnessDefault, readProviderSettingsDeclarations, readProviderSettingsMaterialization } from '../provider-settings-store.mjs';
|
|
38
40
|
import { normalizeProviderLifecycleState } from '../provider-lifecycle-state.mjs';
|
|
39
41
|
import { createProviderHealthStore } from '../provider-health.mjs';
|
|
40
42
|
import { planProviderDelete } from '../provider-lifecycle.mjs';
|
|
41
|
-
import { createProviderDeleteFileHooks } from '../provider-delete-adapters.mjs';
|
|
43
|
+
import { createProviderDeleteFileHooks, isRecoverableProviderDeleteBackup, readProviderDeleteManifestFile } from '../provider-delete-adapters.mjs';
|
|
42
44
|
import { buildCredentialReferenceInventory } from '../credential-reference-inventory.mjs';
|
|
43
45
|
import { executeCredentialPurge, planCredentialPurge } from '../credential-lifecycle.mjs';
|
|
44
46
|
import { createCredentialPurgeFileHooks } from '../credential-purge-adapters.mjs';
|
|
45
47
|
import { normalizeCredentialPurgeState, recordCredentialPurgeOutcome } from '../credential-purge-state.mjs';
|
|
46
48
|
import { buildRuntimeReadinessSnapshot } from '../runtime-readiness-snapshot.mjs';
|
|
49
|
+
import { createProviderLayerMigrationFileHooks, readProviderLayerMigrationTransactions } from '../provider-layer-migration-adapters.mjs';
|
|
47
50
|
|
|
48
51
|
// policy.mjs is pure (no @deepseek-ai imports, no ctx access), so importing it
|
|
49
52
|
// here is safe for the profile-realm discipline: it never pulls in package
|
|
@@ -106,19 +109,102 @@ const LEGACY_TIER_MODELS = { flash: 'deepseek-v4-flash', pro: 'deepseek-v4-pro'
|
|
|
106
109
|
const ROLES = { worker: true, reviewer: true };
|
|
107
110
|
const CONFIG_DIR = join(homedir(), '.config', 'dsh-crew');
|
|
108
111
|
const CREDENTIAL_PURGE_STATE_FILE = join(CONFIG_DIR, 'credential-purge-lifecycle.json');
|
|
112
|
+
const RECOVERY_ACTION_PATTERN = /^[a-f0-9]{32}$/u;
|
|
109
113
|
|
|
110
|
-
function
|
|
114
|
+
function recoveryActionId(name) {
|
|
115
|
+
return createHash('sha256').update(String(name)).digest('hex').slice(0, 32);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function isRecoveryControlEntry(name) {
|
|
119
|
+
return name === '.quarantine'
|
|
120
|
+
|| name === '.delete.lock'
|
|
121
|
+
|| name === '.delete.reclaim.lock'
|
|
122
|
+
|| name === '.delete.recovery.lock'
|
|
123
|
+
|| name === '.delete.recovery.lock.claim'
|
|
124
|
+
|| /^\.delete\.lock\.[0-9a-f-]+\.(?:active|staging|retired)$/iu.test(name)
|
|
125
|
+
|| /^\.delete\.recovery\.lock\.[0-9a-f-]+\.stale$/iu.test(name);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function readProviderLifecycleStateStatus() {
|
|
129
|
+
const file = join(CONFIG_DIR, 'provider-lifecycle.json');
|
|
130
|
+
if (!existsSync(file)) return { ok: true, state: normalizeProviderLifecycleState() };
|
|
111
131
|
try {
|
|
112
|
-
return normalizeProviderLifecycleState(JSON.parse(readFileSync(
|
|
132
|
+
return { ok: true, state: normalizeProviderLifecycleState(JSON.parse(readFileSync(file, 'utf8'))) };
|
|
113
133
|
} catch {
|
|
114
|
-
return normalizeProviderLifecycleState();
|
|
134
|
+
return { ok: false, code: 'PROVIDER_LIFECYCLE_UNAVAILABLE', state: normalizeProviderLifecycleState() };
|
|
115
135
|
}
|
|
116
136
|
}
|
|
117
137
|
|
|
138
|
+
function readProviderLifecycleState() {
|
|
139
|
+
return readProviderLifecycleStateStatus().state;
|
|
140
|
+
}
|
|
141
|
+
|
|
118
142
|
function readCredentialPurgeState() {
|
|
119
143
|
try { return normalizeCredentialPurgeState(JSON.parse(readFileSync(CREDENTIAL_PURGE_STATE_FILE, 'utf8'))); } catch { return normalizeCredentialPurgeState(); }
|
|
120
144
|
}
|
|
121
145
|
|
|
146
|
+
export function readProviderRecoveryTransactions(root = join(CONFIG_DIR, 'provider-backups')) {
|
|
147
|
+
if (!existsSync(root)) return [];
|
|
148
|
+
try {
|
|
149
|
+
const managedRoot = dirname(resolvePath(root));
|
|
150
|
+
const paths = {
|
|
151
|
+
profile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
152
|
+
settings: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
153
|
+
config: join(CONFIG_DIR, 'config.json'),
|
|
154
|
+
lifecycle: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
155
|
+
};
|
|
156
|
+
const transactions = readdirSync(root, { withFileTypes: true }).filter((entry) => !isRecoveryControlEntry(entry.name)).flatMap((entry) => {
|
|
157
|
+
const transactionRoot = join(root, entry.name);
|
|
158
|
+
const manifestFile = join(root, entry.name, 'manifest.json');
|
|
159
|
+
const actionId = recoveryActionId(entry.name);
|
|
160
|
+
if (!entry.isDirectory()) {
|
|
161
|
+
let updatedAt;
|
|
162
|
+
try { updatedAt = statSync(transactionRoot).mtime.toISOString(); } catch { updatedAt = new Date(0).toISOString(); }
|
|
163
|
+
return [{ storage_id: entry.name, action_id: actionId, transaction_id: null, provider_id: null, phase: 'RECOVERY_UNRESOLVED', updated_at: updatedAt, recoverable: false, unresolved: true }];
|
|
164
|
+
}
|
|
165
|
+
try {
|
|
166
|
+
const manifest = readProviderDeleteManifestFile(manifestFile, managedRoot);
|
|
167
|
+
const transactionId = typeof manifest?.plan?.plan_id === 'string' ? manifest.plan.plan_id : null;
|
|
168
|
+
const providerId = typeof manifest?.provider_id === 'string' ? manifest.provider_id : null;
|
|
169
|
+
const phase = typeof manifest?.phase_journal?.phase === 'string' && manifest.phase_journal.phase.trim()
|
|
170
|
+
? manifest.phase_journal.phase.trim() : 'RECOVERY_INCOMPLETE';
|
|
171
|
+
const updatedAt = typeof manifest?.phase_journal?.updated_at === 'string' && manifest.phase_journal.updated_at.trim()
|
|
172
|
+
? manifest.phase_journal.updated_at : statSync(manifestFile).mtime.toISOString();
|
|
173
|
+
const hasPendingMutation = manifest?.mutation_journal && typeof manifest.mutation_journal === 'object'
|
|
174
|
+
&& Object.keys(manifest.mutation_journal).length > 0;
|
|
175
|
+
const recoverable = transactionId != null && providerId != null && isRecoverableProviderDeleteBackup(manifest, {
|
|
176
|
+
root: transactionRoot,
|
|
177
|
+
managedRoot,
|
|
178
|
+
paths,
|
|
179
|
+
expectedProviderId: providerId,
|
|
180
|
+
}) === true;
|
|
181
|
+
if (['ROLLED_BACK', 'VERIFIED'].includes(phase) && !hasPendingMutation && recoverable) return [];
|
|
182
|
+
return [{ storage_id: entry.name, action_id: actionId, transaction_id: transactionId ?? entry.name, provider_id: providerId, phase, updated_at: updatedAt, recoverable, unresolved: recoverable !== true }];
|
|
183
|
+
} catch {
|
|
184
|
+
let updatedAt;
|
|
185
|
+
try { updatedAt = statSync(transactionRoot).mtime.toISOString(); } catch { updatedAt = new Date(0).toISOString(); }
|
|
186
|
+
return [{
|
|
187
|
+
storage_id: entry.name,
|
|
188
|
+
action_id: actionId,
|
|
189
|
+
transaction_id: null,
|
|
190
|
+
provider_id: null,
|
|
191
|
+
phase: 'RECOVERY_UNRESOLVED',
|
|
192
|
+
updated_at: updatedAt,
|
|
193
|
+
recoverable: false,
|
|
194
|
+
unresolved: true,
|
|
195
|
+
}];
|
|
196
|
+
}
|
|
197
|
+
});
|
|
198
|
+
// Recovery is a safety gate, not a bounded history view. Never discard an
|
|
199
|
+
// older unresolved transaction merely because newer entries exist.
|
|
200
|
+
return transactions.sort((a, b) => a.updated_at.localeCompare(b.updated_at));
|
|
201
|
+
} catch {
|
|
202
|
+
let updatedAt;
|
|
203
|
+
try { updatedAt = statSync(root).mtime.toISOString(); } catch { updatedAt = new Date(0).toISOString(); }
|
|
204
|
+
return [{ storage_id: null, action_id: null, transaction_id: null, provider_id: null, phase: 'RECOVERY_UNRESOLVED', updated_at: updatedAt, recoverable: false, unresolved: true }];
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
122
208
|
function writeCredentialPurgeState(state) {
|
|
123
209
|
mkdirSync(CONFIG_DIR, { recursive: true });
|
|
124
210
|
const temp = `${CREDENTIAL_PURGE_STATE_FILE}.${process.pid}.${Date.now()}.tmp`;
|
|
@@ -144,34 +230,155 @@ function providerInventoryPolicy(config) {
|
|
|
144
230
|
};
|
|
145
231
|
}
|
|
146
232
|
|
|
233
|
+
export function hasCompleteProviderCatalogEvidence(evidence) {
|
|
234
|
+
return evidence?.ok === true && evidence.partial !== true;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
export function hasCompleteProviderDeclarationEvidence(evidence) {
|
|
238
|
+
return evidence?.ok === true
|
|
239
|
+
&& Object.values(evidence.sources ?? {}).every((source) => source?.present !== true || !source.code);
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
export function hasAvailableProviderLifecycleEvidence(evidence) {
|
|
243
|
+
return evidence?.ok === true;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
export function hasPendingProviderRecoveryTransactions(inventory) {
|
|
247
|
+
return Array.isArray(inventory?.recovery_transactions) && inventory.recovery_transactions.length > 0;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
export function hasProviderRuntimeRestartEvidence(beforeRuntimeId, currentRuntimeId) {
|
|
251
|
+
return typeof beforeRuntimeId === 'string' && beforeRuntimeId.trim() !== ''
|
|
252
|
+
&& typeof currentRuntimeId === 'string' && currentRuntimeId.trim() !== ''
|
|
253
|
+
&& beforeRuntimeId !== currentRuntimeId;
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/** Preserve profile/settings provenance when preparing the migration planner. */
|
|
257
|
+
export function buildProviderMigrationDeclarations(profileDeclarations = [], settingsDeclarations = []) {
|
|
258
|
+
return [
|
|
259
|
+
...(Array.isArray(profileDeclarations) ? profileDeclarations : []),
|
|
260
|
+
...(Array.isArray(settingsDeclarations) ? settingsDeclarations : []),
|
|
261
|
+
];
|
|
262
|
+
}
|
|
263
|
+
|
|
147
264
|
async function readProviderInventorySnapshot(hub, ctx, config) {
|
|
148
265
|
let catalog = { providers: [], harness_default: null };
|
|
266
|
+
let catalogEvidence = { ok: false, code: 'MODEL_CATALOG_UNAVAILABLE' };
|
|
149
267
|
try {
|
|
150
268
|
catalog = await readHarnessModelCatalog({
|
|
151
269
|
llm: ctx.llm ?? ctx.get('llm'),
|
|
152
270
|
getCurrentSelection: () => ctx.get('agentDefaultModel')?.currentSelection?.(),
|
|
153
271
|
});
|
|
272
|
+
catalogEvidence = {
|
|
273
|
+
ok: catalog.partial !== true,
|
|
274
|
+
...(catalog.partial === true ? { code: 'PROVIDER_CATALOG_INCOMPLETE' } : {}),
|
|
275
|
+
provider_count: Number.isInteger(catalog.provider_count) ? catalog.provider_count : catalog.providers.length,
|
|
276
|
+
refreshed_at: typeof catalog.refreshed_at === 'string' ? catalog.refreshed_at : null,
|
|
277
|
+
partial: catalog.partial === true,
|
|
278
|
+
};
|
|
154
279
|
} catch {}
|
|
155
280
|
const profileFile = join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml');
|
|
156
281
|
let declarations = [];
|
|
282
|
+
let profileDeclarations = [];
|
|
283
|
+
let settingsDeclarations = [];
|
|
284
|
+
const declarationEvidence = { ok: true, sources: {} };
|
|
157
285
|
try {
|
|
286
|
+
declarationEvidence.sources.profile = { present: existsSync(profileFile) };
|
|
158
287
|
if (existsSync(profileFile)) {
|
|
159
288
|
const source = readFileSync(profileFile, 'utf8');
|
|
160
289
|
const parsed = readProviderDeclarations(source, { file: 'harness/profiles/dsh-crew/cordis.patch.yml' });
|
|
161
|
-
if (parsed.ok)
|
|
290
|
+
if (parsed.ok) {
|
|
291
|
+
profileDeclarations = parsed.declarations;
|
|
292
|
+
declarations = profileDeclarations;
|
|
293
|
+
}
|
|
294
|
+
else { declarationEvidence.ok = false; declarationEvidence.sources.profile.code = parsed.code; }
|
|
162
295
|
}
|
|
163
|
-
} catch {}
|
|
164
|
-
const
|
|
296
|
+
} catch { declarationEvidence.ok = false; declarationEvidence.sources.profile = { present: true, code: 'PROVIDER_SOURCE_UNAVAILABLE' }; }
|
|
297
|
+
const settingsFile = join(CONFIG_DIR, 'harness', 'settings.yaml');
|
|
298
|
+
let settingsDefault = null;
|
|
299
|
+
let defaultEvidence = { ok: true, source: 'none' };
|
|
300
|
+
try {
|
|
301
|
+
declarationEvidence.sources.settings = { present: existsSync(settingsFile) };
|
|
302
|
+
if (existsSync(settingsFile)) {
|
|
303
|
+
const source = readFileSync(settingsFile, 'utf8');
|
|
304
|
+
const parsed = readProviderSettingsDeclarations(source, { file: 'harness/settings.yaml' });
|
|
305
|
+
if (parsed.ok) {
|
|
306
|
+
settingsDeclarations = parsed.declarations;
|
|
307
|
+
declarations = [...declarations, ...settingsDeclarations];
|
|
308
|
+
}
|
|
309
|
+
else { declarationEvidence.ok = false; declarationEvidence.sources.settings.code = parsed.code; }
|
|
310
|
+
const parsedDefault = readHarnessDefault(source);
|
|
311
|
+
if (parsedDefault.ok) settingsDefault = parsedDefault;
|
|
312
|
+
else defaultEvidence = { ok: false, code: 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' };
|
|
313
|
+
}
|
|
314
|
+
} catch {
|
|
315
|
+
declarationEvidence.ok = false;
|
|
316
|
+
declarationEvidence.sources.settings = { present: true, code: 'PROVIDER_SOURCE_UNAVAILABLE' };
|
|
317
|
+
defaultEvidence = { ok: false, code: 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' };
|
|
318
|
+
}
|
|
319
|
+
const liveDefault = catalog.harness_default && typeof catalog.harness_default.provider === 'string' && typeof catalog.harness_default.model === 'string'
|
|
320
|
+
? { provider: catalog.harness_default.provider, model: catalog.harness_default.model } : null;
|
|
321
|
+
const persistedDefault = settingsDefault ? { provider: settingsDefault.provider, model: settingsDefault.model } : null;
|
|
322
|
+
if (liveDefault || persistedDefault) {
|
|
323
|
+
if (!liveDefault || !persistedDefault) defaultEvidence = { ok: false, code: 'PROVIDER_DEFAULT_AUTHORITY_MISMATCH' };
|
|
324
|
+
else if (liveDefault.provider !== persistedDefault.provider || liveDefault.model !== persistedDefault.model) defaultEvidence = { ok: false, code: 'PROVIDER_DEFAULT_AUTHORITY_MISMATCH' };
|
|
325
|
+
}
|
|
326
|
+
// The persisted agent-default-model is the mutation authority. Keep it in
|
|
327
|
+
// the inventory even when the live selection is stale; the evidence gate
|
|
328
|
+
// above prevents destructive operations until both sources agree.
|
|
329
|
+
if (persistedDefault) {
|
|
330
|
+
catalog = {
|
|
331
|
+
...catalog,
|
|
332
|
+
harness_default: persistedDefault,
|
|
333
|
+
harness_default_authority: { kind: 'harness-settings', locator: settingsDefault.locator },
|
|
334
|
+
};
|
|
335
|
+
}
|
|
336
|
+
const lifecycleEvidence = readProviderLifecycleStateStatus();
|
|
337
|
+
const lifecycle = lifecycleEvidence.state;
|
|
338
|
+
const recoveryTransactions = [
|
|
339
|
+
...readProviderRecoveryTransactions(),
|
|
340
|
+
...readProviderLayerMigrationTransactions(join(CONFIG_DIR, 'provider-migration-backups')),
|
|
341
|
+
];
|
|
165
342
|
const credentialPurge = readCredentialPurgeState();
|
|
166
343
|
const inventory = buildProviderInventory({
|
|
167
344
|
catalog,
|
|
168
345
|
declarations,
|
|
169
346
|
policy: providerInventoryPolicy(config),
|
|
170
347
|
tombstones: lifecycle.tombstones,
|
|
348
|
+
additionalProviderIds: [
|
|
349
|
+
...Object.keys(lifecycle.tombstones ?? {}),
|
|
350
|
+
...recoveryTransactions.map((entry) => entry.provider_id).filter((id) => typeof id === 'string' && id.trim()),
|
|
351
|
+
],
|
|
171
352
|
activeJobs: hub.list(),
|
|
172
353
|
});
|
|
354
|
+
// Keep each authoritative source's credential reference with its own
|
|
355
|
+
// declaration. Reconstructing from the merged inventory would silently
|
|
356
|
+
// attach the first key to every layer when profile/settings differ.
|
|
357
|
+
const migrationDeclarations = buildProviderMigrationDeclarations(profileDeclarations, settingsDeclarations);
|
|
358
|
+
const routingReferences = [
|
|
359
|
+
...(config.flash_model_priority ?? []), ...(config.pro_model_priority ?? []),
|
|
360
|
+
...(config.worker?.model_policy?.priority ?? []), ...(config.worker?.model_policy?.escalation_priority ?? []),
|
|
361
|
+
...(config.review?.model_policy?.priority ?? []), ...(config.review?.model_policy?.escalation_priority ?? []),
|
|
362
|
+
];
|
|
363
|
+
const migration = buildProviderLayerMigrationPlan({
|
|
364
|
+
declarations: migrationDeclarations,
|
|
365
|
+
catalogProviders: catalog.providers,
|
|
366
|
+
harnessDefault: catalog.harness_default,
|
|
367
|
+
routingReferences,
|
|
368
|
+
tombstones: lifecycle.tombstones,
|
|
369
|
+
recoveryTransactions,
|
|
370
|
+
catalogEvidence,
|
|
371
|
+
declarationEvidence,
|
|
372
|
+
lifecycleEvidence,
|
|
373
|
+
defaultEvidence,
|
|
374
|
+
});
|
|
173
375
|
return {
|
|
174
376
|
...inventory,
|
|
377
|
+
migration,
|
|
378
|
+
catalog_evidence: catalogEvidence,
|
|
379
|
+
declaration_evidence: declarationEvidence,
|
|
380
|
+
default_evidence: defaultEvidence,
|
|
381
|
+
lifecycle_evidence: lifecycleEvidence.ok ? { ok: true } : { ok: false, code: lifecycleEvidence.code },
|
|
175
382
|
credential_history_refs: Object.values(lifecycle.transactions ?? {})
|
|
176
383
|
.flatMap((transaction) => Array.isArray(transaction?.credential_refs) ? transaction.credential_refs : []),
|
|
177
384
|
credential_purged_refs: Object.keys(credentialPurge.purged),
|
|
@@ -184,23 +391,27 @@ async function readProviderInventorySnapshot(hub, ctx, config) {
|
|
|
184
391
|
transaction_id: transactionId,
|
|
185
392
|
provider_id: transaction.provider_id,
|
|
186
393
|
state: transaction.state,
|
|
394
|
+
...(transaction.updated_at ? { updated_at: transaction.updated_at } : {}),
|
|
187
395
|
...(transaction.expected_revision ? { expected_revision: transaction.expected_revision } : {}),
|
|
188
396
|
})),
|
|
397
|
+
recovery_transactions: recoveryTransactions,
|
|
189
398
|
};
|
|
190
399
|
}
|
|
191
400
|
|
|
192
|
-
function
|
|
401
|
+
function readProviderSourceRevisions() {
|
|
193
402
|
const profileFile = join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml');
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
403
|
+
const settingsFile = join(CONFIG_DIR, 'harness', 'settings.yaml');
|
|
404
|
+
const read = (file, inspect) => {
|
|
405
|
+
try {
|
|
406
|
+
if (!existsSync(file)) return { ok: false, code: 'PROVIDER_SOURCE_MISSING', revision: null };
|
|
407
|
+
const parsed = inspect(readFileSync(file, 'utf8'));
|
|
408
|
+
return parsed.ok ? { ok: true, revision: parsed.revision } : { ok: false, code: parsed.code, revision: parsed.revision ?? null };
|
|
409
|
+
} catch { return { ok: false, code: 'PROVIDER_SOURCE_UNAVAILABLE', revision: null }; }
|
|
410
|
+
};
|
|
411
|
+
return { profile: read(profileFile, inspectProviderProfile), settings: read(settingsFile, inspectProviderSettings) };
|
|
201
412
|
}
|
|
202
|
-
|
|
203
|
-
export function hubCanonicalEvents(job = {}) {
|
|
413
|
+
|
|
414
|
+
export function hubCanonicalEvents(job = {}) {
|
|
204
415
|
if (!job.id) return [];
|
|
205
416
|
const role = job.role === 'reviewer' ? 'reviewer' : 'worker';
|
|
206
417
|
const atStart = job.startedAt ?? null;
|
|
@@ -312,6 +523,9 @@ export function selectProviderProbeModel({ providerId, record, config } = {}) {
|
|
|
312
523
|
export class WorkerRegistry { constructor(ctx) {
|
|
313
524
|
this.ctx = ctx;
|
|
314
525
|
this.jobs = new Map();
|
|
526
|
+
this.providerMutations = new Set();
|
|
527
|
+
this.providerMutationEpochs = new Map();
|
|
528
|
+
this.providerLeases = new Map();
|
|
315
529
|
this.nextId = 1;
|
|
316
530
|
this.shard = createShardWriter('hub');
|
|
317
531
|
this.healthStore = createProviderHealthStore();
|
|
@@ -357,9 +571,51 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
357
571
|
return v;
|
|
358
572
|
}
|
|
359
573
|
|
|
360
|
-
list() {
|
|
361
|
-
return [...this.jobs.values()].map((job) => this.view(job));
|
|
362
|
-
}
|
|
574
|
+
list() {
|
|
575
|
+
return [...this.jobs.values()].map((job) => this.view(job));
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
beginProviderMutation(providerId) {
|
|
579
|
+
if (typeof providerId !== 'string' || !providerId.trim() || this.providerMutations.has(providerId)) return false;
|
|
580
|
+
this.providerMutationEpochs.set(providerId, (this.providerMutationEpochs.get(providerId) ?? 0) + 1);
|
|
581
|
+
this.providerMutations.add(providerId);
|
|
582
|
+
return true;
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
endProviderMutation(providerId) {
|
|
586
|
+
this.providerMutations.delete(providerId);
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
getProviderMutationEpoch(providerId) {
|
|
590
|
+
return this.providerMutationEpochs.get(providerId) ?? 0;
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
assertProviderNotMutating(providerId, expectedEpoch = undefined) {
|
|
594
|
+
if (this.providerMutations.has(providerId)
|
|
595
|
+
|| (expectedEpoch !== undefined && this.getProviderMutationEpoch(providerId) !== expectedEpoch)) {
|
|
596
|
+
throw Object.assign(new Error('provider mutation is active'), { code: 'PROVIDER_DELETE_BUSY' });
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
acquireProviderLease(providerId) {
|
|
601
|
+
this.providerLeases.set(providerId, (this.providerLeases.get(providerId) ?? 0) + 1);
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
releaseProviderLease(providerId) {
|
|
605
|
+
const count = this.providerLeases.get(providerId) ?? 0;
|
|
606
|
+
if (count <= 1) this.providerLeases.delete(providerId);
|
|
607
|
+
else this.providerLeases.set(providerId, count - 1);
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
hasProviderLease(providerId) {
|
|
611
|
+
return (this.providerLeases.get(providerId) ?? 0) > 0;
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
// Profile/settings are shared stores. A migration or delete of provider A
|
|
615
|
+
// must fence a concurrent lifecycle write for provider B in this Hub.
|
|
616
|
+
beginProviderStoreMutation() { return this.beginProviderMutation('__dsh-provider-store__'); }
|
|
617
|
+
endProviderStoreMutation() { this.endProviderMutation('__dsh-provider-store__'); }
|
|
618
|
+
hasProviderStoreMutation() { return this.providerMutations.has('__dsh-provider-store__'); }
|
|
363
619
|
|
|
364
620
|
publish() {
|
|
365
621
|
this.shard.publish([...this.jobs.values()].map((j) => this.view(j)));
|
|
@@ -466,9 +722,20 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
466
722
|
candidateSet: attempt > 0 ? 'escalation' : 'primary',
|
|
467
723
|
},
|
|
468
724
|
});
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
}
|
|
725
|
+
}
|
|
726
|
+
if (!selection.ok) throw Object.assign(new Error(selection.message), { policyCode: selection.code });
|
|
727
|
+
}
|
|
728
|
+
const pendingLayerMigration = readProviderLayerMigrationTransactions(join(CONFIG_DIR, 'provider-migration-backups'))
|
|
729
|
+
.find((entry) => entry.unresolved === true || entry.provider_id === selection.provider);
|
|
730
|
+
if (pendingLayerMigration) throw Object.assign(new Error('provider layer migration is pending verification'), { code: 'PROVIDER_MIGRATION_PENDING' });
|
|
731
|
+
// A process restart clears the in-memory mutation fence, so persisted
|
|
732
|
+
// provider-delete transactions must also gate admission. Unknown-owner
|
|
733
|
+
// recovery entries fence globally; known entries fence their provider.
|
|
734
|
+
const pendingProviderDelete = readProviderRecoveryTransactions()
|
|
735
|
+
.find((entry) => entry.unresolved === true || entry.provider_id === selection.provider);
|
|
736
|
+
if (pendingProviderDelete) throw Object.assign(new Error('provider delete recovery is pending'), { code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
737
|
+
const providerMutationEpoch = this.getProviderMutationEpoch(selection.provider);
|
|
738
|
+
this.assertProviderNotMutating(selection.provider, providerMutationEpoch);
|
|
472
739
|
|
|
473
740
|
// The worker always gets the auditable Delivery Contract appended (unless
|
|
474
741
|
// it already carries one), so its final message follows ## Diff / ## Tests
|
|
@@ -527,9 +794,13 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
527
794
|
// Abort latch: cancel()/timeout set this the moment the job becomes
|
|
528
795
|
// terminal, even while agents.create() is still pending, so a handle
|
|
529
796
|
// acquired after that point is disposed instead of being driven.
|
|
530
|
-
abortRequested: false,
|
|
797
|
+
abortRequested: false,
|
|
531
798
|
abortPromise: null,
|
|
532
799
|
resolveAbort: null,
|
|
800
|
+
providerLease: false,
|
|
801
|
+
providerLeaseReleaseBlocked: false,
|
|
802
|
+
lateHandleCreated: false,
|
|
803
|
+
lateCreateCleanupPromise: null,
|
|
533
804
|
};
|
|
534
805
|
job.abortPromise = new Promise((resolve) => { job.resolveAbort = resolve; });
|
|
535
806
|
const disposeJobHandle = async () => {
|
|
@@ -541,14 +812,30 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
541
812
|
job.handle_dispose_promise = Promise.reject(error);
|
|
542
813
|
}
|
|
543
814
|
}
|
|
544
|
-
|
|
815
|
+
try {
|
|
816
|
+
await job.handle_dispose_promise;
|
|
817
|
+
} catch (error) {
|
|
818
|
+
// A failed disposer is not proof that the provider is idle. Keep the
|
|
819
|
+
// lease fenced even when the job is already terminal; the operator
|
|
820
|
+
// must restart/reclaim the owned runtime before deletion can proceed.
|
|
821
|
+
job.providerLeaseReleaseBlocked = true;
|
|
822
|
+
throw error;
|
|
823
|
+
}
|
|
545
824
|
};
|
|
546
825
|
job.disposeHandle = disposeJobHandle;
|
|
547
826
|
// Read-only pre-run snapshot (async, never blocks dispatch): the audit
|
|
548
827
|
// only needs the before-state by the time the worker finishes. Non-repos
|
|
549
828
|
// degrade to { kind:'no-git' } instead of failing the job.
|
|
550
|
-
job.baseline = await captureWorkspaceBaseline({ cwd: executionCwd }).catch(() => ({ kind: 'no-git', reason: NOT_A_GIT_REPOSITORY, error: 'workspace audit failed' }));
|
|
551
|
-
|
|
829
|
+
job.baseline = await captureWorkspaceBaseline({ cwd: executionCwd }).catch(() => ({ kind: 'no-git', reason: NOT_A_GIT_REPOSITORY, error: 'workspace audit failed' }));
|
|
830
|
+
try {
|
|
831
|
+
this.assertProviderNotMutating(selection.provider, providerMutationEpoch);
|
|
832
|
+
this.acquireProviderLease(selection.provider);
|
|
833
|
+
job.providerLease = true;
|
|
834
|
+
} catch (error) {
|
|
835
|
+
if (isolatedWorkspace) await cleanupIsolatedWorkspace({ repoRoot: isolatedWorkspace.repoRoot, worktreePath: isolatedWorkspace.worktreePath }).catch(() => {});
|
|
836
|
+
throw error;
|
|
837
|
+
}
|
|
838
|
+
this.jobs.set(id, job);
|
|
552
839
|
|
|
553
840
|
const onEvent = (session, event) => {
|
|
554
841
|
switch (event.type) {
|
|
@@ -598,14 +885,15 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
598
885
|
job.abortPromise.then(() => ({ aborted: true })),
|
|
599
886
|
]);
|
|
600
887
|
if (created.aborted) {
|
|
601
|
-
createPromise.then(async (handle) => {
|
|
602
|
-
if (handle
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
}
|
|
608
|
-
|
|
888
|
+
job.lateCreateCleanupPromise = createPromise.then(async (handle) => {
|
|
889
|
+
if (!handle) return;
|
|
890
|
+
job.lateHandleCreated = true;
|
|
891
|
+
job.handle = handle;
|
|
892
|
+
if (job.status !== 'running' || job.abortRequested) await job.disposeHandle();
|
|
893
|
+
}).catch((error) => {
|
|
894
|
+
job.cleanup_warning = `late agent creation failed: ${error?.message ?? String(error)}`;
|
|
895
|
+
throw error;
|
|
896
|
+
});
|
|
609
897
|
return;
|
|
610
898
|
}
|
|
611
899
|
const handle = created.handle;
|
|
@@ -674,9 +962,23 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
674
962
|
job.endedAt = new Date().toISOString();
|
|
675
963
|
job.currentTool = null;
|
|
676
964
|
let handleCleanupWarning = job.cleanup_warning ?? null;
|
|
677
|
-
try { await disposeJobHandle(); } catch (error) {
|
|
678
|
-
handleCleanupWarning = `agent cleanup failed: ${error?.message ?? String(error)}`;
|
|
679
|
-
}
|
|
965
|
+
try { await disposeJobHandle(); } catch (error) {
|
|
966
|
+
handleCleanupWarning = `agent cleanup failed: ${error?.message ?? String(error)}`;
|
|
967
|
+
}
|
|
968
|
+
if (job.providerLease) {
|
|
969
|
+
const releaseLease = () => {
|
|
970
|
+
if (!job.providerLease) return;
|
|
971
|
+
this.releaseProviderLease(job.provider);
|
|
972
|
+
job.providerLease = false;
|
|
973
|
+
};
|
|
974
|
+
if (job.lateCreateCleanupPromise) {
|
|
975
|
+
job.lateCreateCleanupPromise.then(releaseLease).catch(() => {
|
|
976
|
+
if (!job.lateHandleCreated && !job.providerLeaseReleaseBlocked) releaseLease();
|
|
977
|
+
// A late-created handle that failed disposal remains fenced by
|
|
978
|
+
// the provider lease until an operator restarts/reclaims it.
|
|
979
|
+
});
|
|
980
|
+
} else if (!job.providerLeaseReleaseBlocked) releaseLease();
|
|
981
|
+
}
|
|
680
982
|
// Delivery completeness is separate from execution status: a job can
|
|
681
983
|
// be done yet fail to report Diff/Tests/Risks (or Review sections for
|
|
682
984
|
// an automatic review). Parse whatever final message the worker
|
|
@@ -950,11 +1252,18 @@ export async function apply(ctx) {
|
|
|
950
1252
|
const disposers = [];
|
|
951
1253
|
const PROVIDER_DELETE_PLAN_TTL_MS = 10 * 60 * 1000;
|
|
952
1254
|
const providerDeletePlans = new Map();
|
|
1255
|
+
const providerMigrationPlans = new Map();
|
|
1256
|
+
const providerMigrationOwners = new Map();
|
|
1257
|
+
const providerStoreMutationOwners = new Map();
|
|
953
1258
|
const credentialPurgePlans = new Map();
|
|
954
1259
|
const rememberProviderDeletePlan = (plan) => {
|
|
955
1260
|
providerDeletePlans.set(plan.plan_id, { plan, created_at: Date.now() });
|
|
956
1261
|
while (providerDeletePlans.size > 32) providerDeletePlans.delete(providerDeletePlans.keys().next().value);
|
|
957
1262
|
};
|
|
1263
|
+
const rememberProviderMigrationPlan = (plan) => {
|
|
1264
|
+
providerMigrationPlans.set(plan.plan_id, { plan, created_at: Date.now() });
|
|
1265
|
+
while (providerMigrationPlans.size > 32) providerMigrationPlans.delete(providerMigrationPlans.keys().next().value);
|
|
1266
|
+
};
|
|
958
1267
|
const rememberCredentialPurgePlan = (plan) => {
|
|
959
1268
|
credentialPurgePlans.set(plan.plan_id, { plan, created_at: Date.now() });
|
|
960
1269
|
while (credentialPurgePlans.size > 32) credentialPurgePlans.delete(credentialPurgePlans.keys().next().value);
|
|
@@ -1361,23 +1670,386 @@ export async function apply(ctx) {
|
|
|
1361
1670
|
try { url = new URL(req.url, 'http://localhost'); } catch { return sendJson(res, 400, { ok: false, code: 'PROVIDER_PLAN_INVALID' }); }
|
|
1362
1671
|
const parts = url.pathname.slice(`${ROUTE_BASE}/providers`.length).split('/').filter(Boolean);
|
|
1363
1672
|
try {
|
|
1673
|
+
if (req.method === 'GET' && parts.length === 1 && parts[0] === 'migration-status') {
|
|
1674
|
+
const config = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
1675
|
+
const inventory = await readProviderInventorySnapshot(hub, ctx, config);
|
|
1676
|
+
return sendJson(res, 200, { ok: true, ...inventory.migration, runtime: getHubRuntimeIdentity() });
|
|
1677
|
+
}
|
|
1364
1678
|
const providerId = decodeURIComponent(parts[0]);
|
|
1365
1679
|
const body = await readBody(req);
|
|
1680
|
+
if (req.method === 'POST' && parts[0] === '_recovery' && parts[1] === 'lock' && parts.length === 2) {
|
|
1681
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_CONFIRM_REQUIRED' });
|
|
1682
|
+
const hooks = createProviderDeleteFileHooks({
|
|
1683
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1684
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1685
|
+
configFile: join(CONFIG_DIR, 'config.json'),
|
|
1686
|
+
lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
1687
|
+
backupDir: join(CONFIG_DIR, 'provider-backups'),
|
|
1688
|
+
sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
|
|
1689
|
+
});
|
|
1690
|
+
const recovered = await hooks.recoverLock({ force: body?.force === true });
|
|
1691
|
+
const migrationHooks = createProviderLayerMigrationFileHooks({
|
|
1692
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1693
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1694
|
+
backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
|
|
1695
|
+
});
|
|
1696
|
+
const migrationRecovered = await migrationHooks.recoverLock({ confirm: true });
|
|
1697
|
+
const ok = recovered.ok === true && migrationRecovered.ok === true;
|
|
1698
|
+
return sendJson(res, ok ? 200 : 409, { ok, delete_lock: recovered, migration_lock: migrationRecovered });
|
|
1699
|
+
}
|
|
1700
|
+
if (req.method === 'POST' && parts[0] === '_recovery' && parts[1] === 'quarantine' && parts.length === 2) {
|
|
1701
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_QUARANTINE_CONFIRM_REQUIRED' });
|
|
1702
|
+
const actionId = typeof body?.action_id === 'string' && RECOVERY_ACTION_PATTERN.test(body.action_id)
|
|
1703
|
+
? body.action_id : null;
|
|
1704
|
+
if (!actionId) return sendJson(res, 400, { ok: false, code: 'PROVIDER_DELETE_PLAN_INVALID' });
|
|
1705
|
+
const recovery = [
|
|
1706
|
+
...readProviderRecoveryTransactions(),
|
|
1707
|
+
...readProviderLayerMigrationTransactions(join(CONFIG_DIR, 'provider-migration-backups')),
|
|
1708
|
+
].find((entry) => entry.action_id === actionId);
|
|
1709
|
+
if (!recovery) return sendJson(res, 404, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_NOT_FOUND' });
|
|
1710
|
+
if (recovery.recoverable === true) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_RECOVERABLE' });
|
|
1711
|
+
if (typeof recovery.storage_id !== 'string') return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_QUARANTINE_FAILED' });
|
|
1712
|
+
if (recovery.source === 'provider-layer-migration') {
|
|
1713
|
+
try {
|
|
1714
|
+
const migrationHooks = createProviderLayerMigrationFileHooks({
|
|
1715
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1716
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1717
|
+
backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
|
|
1718
|
+
});
|
|
1719
|
+
const quarantined = await migrationHooks.quarantine(actionId);
|
|
1720
|
+
return sendJson(res, 200, { ...quarantined, transaction_id: recovery.transaction_id ?? null });
|
|
1721
|
+
} catch (error) {
|
|
1722
|
+
const code = boundedMachineCodeFromError(error) ?? 'PROVIDER_MIGRATION_RECOVERY_QUARANTINE_FAILED';
|
|
1723
|
+
return sendJson(res, 409, { ok: false, code });
|
|
1724
|
+
}
|
|
1725
|
+
}
|
|
1726
|
+
try {
|
|
1727
|
+
const hooks = createProviderDeleteFileHooks({
|
|
1728
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1729
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1730
|
+
configFile: join(CONFIG_DIR, 'config.json'),
|
|
1731
|
+
lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
1732
|
+
backupDir: join(CONFIG_DIR, 'provider-backups'),
|
|
1733
|
+
sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
|
|
1734
|
+
afterLockAcquired: () => {
|
|
1735
|
+
const current = readProviderRecoveryTransactions().find((entry) => entry.action_id === actionId);
|
|
1736
|
+
if (!current || current.recoverable === true || current.storage_id !== recovery.storage_id) {
|
|
1737
|
+
throw Object.assign(new Error('provider recovery entry changed before quarantine'), { code: 'PROVIDER_DELETE_STATE_CHANGED' });
|
|
1738
|
+
}
|
|
1739
|
+
},
|
|
1740
|
+
});
|
|
1741
|
+
const quarantined = await hooks.quarantine(recovery.storage_id);
|
|
1742
|
+
return sendJson(res, 200, { ...quarantined, action_id: actionId, transaction_id: recovery.transaction_id ?? null });
|
|
1743
|
+
} catch (error) {
|
|
1744
|
+
const code = boundedMachineCodeFromError(error) ?? 'PROVIDER_DELETE_RECOVERY_QUARANTINE_FAILED';
|
|
1745
|
+
return sendJson(res, 409, { ok: false, code });
|
|
1746
|
+
}
|
|
1747
|
+
}
|
|
1366
1748
|
const config = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
1367
1749
|
const inventory = await readProviderInventorySnapshot(hub, ctx, config);
|
|
1750
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'migrate-plan') {
|
|
1751
|
+
if (!hasCompleteProviderCatalogEvidence(inventory.catalog_evidence)) return sendJson(res, 503, { ok: false, code: 'MODEL_CATALOG_UNAVAILABLE' });
|
|
1752
|
+
if (!hasCompleteProviderDeclarationEvidence(inventory.declaration_evidence)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_SOURCE_UNRESOLVED' });
|
|
1753
|
+
if (!hasAvailableProviderLifecycleEvidence(inventory.lifecycle_evidence)) return sendJson(res, 503, { ok: false, code: 'PROVIDER_LIFECYCLE_UNAVAILABLE' });
|
|
1754
|
+
if (inventory.default_evidence?.ok !== true) return sendJson(res, 409, { ok: false, code: inventory.default_evidence?.code ?? 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' });
|
|
1755
|
+
if (hasPendingProviderRecoveryTransactions(inventory)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
1756
|
+
const entry = inventory.migration.providers.find((candidate) => candidate.provider_id === providerId);
|
|
1757
|
+
if (!entry) return sendJson(res, 404, { ok: false, code: 'PROVIDER_MIGRATION_NOT_REQUIRED' });
|
|
1758
|
+
if (entry.action === 'blocked' || entry.action === 'collision-review') return sendJson(res, 409, { ok: false, code: entry.blocked_reason ?? 'PROVIDER_MIGRATION_BLOCKED' });
|
|
1759
|
+
const revisions = readProviderSourceRevisions();
|
|
1760
|
+
if (!revisions.profile.ok) return sendJson(res, 409, { ok: false, code: revisions.profile.code ?? 'PROVIDER_PROFILE_CHANGED' });
|
|
1761
|
+
if ((entry.action === 'promote-existing-user' || entry.harness_default) && !revisions.settings.ok) return sendJson(res, 409, { ok: false, code: revisions.settings.code ?? 'PROVIDER_SETTINGS_CHANGED' });
|
|
1762
|
+
const profileFile = join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml');
|
|
1763
|
+
const parsedMaterialization = readProviderMaterialization(readFileSync(profileFile, 'utf8'), { providerId, file: 'harness/profiles/dsh-crew/cordis.patch.yml' });
|
|
1764
|
+
if (!parsedMaterialization.ok) return sendJson(res, 409, { ok: false, code: parsedMaterialization.code });
|
|
1765
|
+
const materialization = parsedMaterialization.provider;
|
|
1766
|
+
let userMaterializationBefore;
|
|
1767
|
+
if (entry.action === 'promote-existing-user') {
|
|
1768
|
+
const settingsFile = join(CONFIG_DIR, 'harness', 'settings.yaml');
|
|
1769
|
+
if (!existsSync(settingsFile)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_SETTINGS_CHANGED' });
|
|
1770
|
+
const parsedUserMaterialization = readProviderSettingsMaterialization(readFileSync(settingsFile, 'utf8'), { providerId, file: 'harness/settings.yaml' });
|
|
1771
|
+
if (!parsedUserMaterialization.ok) return sendJson(res, 409, { ok: false, code: parsedUserMaterialization.code });
|
|
1772
|
+
userMaterializationBefore = parsedUserMaterialization.provider;
|
|
1773
|
+
}
|
|
1774
|
+
const plan = {
|
|
1775
|
+
schema_version: 1,
|
|
1776
|
+
kind: 'provider-layer-migration',
|
|
1777
|
+
plan_id: randomUUID(),
|
|
1778
|
+
provider_id: providerId,
|
|
1779
|
+
action: entry.action,
|
|
1780
|
+
source: entry.source,
|
|
1781
|
+
routing_references: entry.referenced_by,
|
|
1782
|
+
materialization: { provider: materialization },
|
|
1783
|
+
...(userMaterializationBefore ? { user_materialization_before: userMaterializationBefore } : {}),
|
|
1784
|
+
expected_revisions: {
|
|
1785
|
+
profile: revisions.profile.revision,
|
|
1786
|
+
settings: revisions.settings.ok ? revisions.settings.revision : null,
|
|
1787
|
+
},
|
|
1788
|
+
harness_default_before: inventory.harness_default ?? null,
|
|
1789
|
+
runtime_id_before: getHubRuntimeIdentity().runtime_id,
|
|
1790
|
+
restart_required: true,
|
|
1791
|
+
};
|
|
1792
|
+
rememberProviderMigrationPlan(plan);
|
|
1793
|
+
return sendJson(res, 200, { ok: true, plan });
|
|
1794
|
+
}
|
|
1795
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'migrate') {
|
|
1796
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_MIGRATION_CONFIRM_REQUIRED' });
|
|
1797
|
+
const stored = providerMigrationPlans.get(body?.plan_id);
|
|
1798
|
+
if (!stored || Date.now() - stored.created_at > PROVIDER_DELETE_PLAN_TTL_MS || stored.plan.provider_id !== providerId) {
|
|
1799
|
+
providerMigrationPlans.delete(body?.plan_id);
|
|
1800
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_PLAN_EXPIRED' });
|
|
1801
|
+
}
|
|
1802
|
+
const entry = inventory.migration.providers.find((candidate) => candidate.provider_id === providerId);
|
|
1803
|
+
if (!entry || entry.action !== stored.plan.action || entry.action === 'blocked' || entry.action === 'collision-review') return sendJson(res, 409, { ok: false, code: entry?.blocked_reason ?? 'PROVIDER_MIGRATION_PLAN_CHANGED' });
|
|
1804
|
+
if (inventory.default_evidence?.ok !== true) return sendJson(res, 409, { ok: false, code: inventory.default_evidence?.code ?? 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' });
|
|
1805
|
+
const revisions = readProviderSourceRevisions();
|
|
1806
|
+
const expected = stored.plan.expected_revisions ?? {};
|
|
1807
|
+
if (revisions.profile.revision !== expected.profile || (revisions.settings.ok ? revisions.settings.revision : null) !== expected.settings) {
|
|
1808
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_SOURCE_CHANGED' });
|
|
1809
|
+
}
|
|
1810
|
+
const hooks = createProviderLayerMigrationFileHooks({
|
|
1811
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1812
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1813
|
+
backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
|
|
1814
|
+
});
|
|
1815
|
+
let result;
|
|
1816
|
+
if (hub.hasProviderLease(providerId)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_IN_USE' });
|
|
1817
|
+
if (!hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
1818
|
+
if (!hub.beginProviderMutation(providerId)) {
|
|
1819
|
+
hub.endProviderStoreMutation();
|
|
1820
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
1821
|
+
}
|
|
1822
|
+
let retainProviderMutation = false;
|
|
1823
|
+
let retainProviderStoreMutation = false;
|
|
1824
|
+
try {
|
|
1825
|
+
await hooks.acquireLock();
|
|
1826
|
+
// The plan was created from a pre-lock snapshot. Re-read every
|
|
1827
|
+
// authoritative source after taking the shared store lock so a
|
|
1828
|
+
// concurrent catalog/default/recovery change cannot be hidden
|
|
1829
|
+
// behind unchanged profile/settings revisions.
|
|
1830
|
+
const lockedConfig = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
1831
|
+
const lockedInventory = await readProviderInventorySnapshot(hub, ctx, lockedConfig);
|
|
1832
|
+
const lockedEntry = lockedInventory.migration.providers.find((candidate) => candidate.provider_id === providerId);
|
|
1833
|
+
const expectedRouting = Array.isArray(stored.plan.routing_references) ? stored.plan.routing_references : [];
|
|
1834
|
+
const sameRouting = JSON.stringify(lockedEntry?.referenced_by ?? []) === JSON.stringify(expectedRouting);
|
|
1835
|
+
const sameDefault = JSON.stringify(lockedInventory.harness_default ?? null) === JSON.stringify(stored.plan.harness_default_before ?? null);
|
|
1836
|
+
const profileFile = join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml');
|
|
1837
|
+
const lockedMaterialization = existsSync(profileFile)
|
|
1838
|
+
? readProviderMaterialization(readFileSync(profileFile, 'utf8'), { providerId, file: 'harness/profiles/dsh-crew/cordis.patch.yml' })
|
|
1839
|
+
: { ok: false };
|
|
1840
|
+
const sameMaterialization = lockedMaterialization.ok === true
|
|
1841
|
+
&& JSON.stringify(lockedMaterialization.provider) === JSON.stringify(stored.plan.materialization?.provider ?? null);
|
|
1842
|
+
const settingsFile = join(CONFIG_DIR, 'harness', 'settings.yaml');
|
|
1843
|
+
const lockedUserMaterialization = stored.plan.action === 'promote-existing-user' && existsSync(settingsFile)
|
|
1844
|
+
? readProviderSettingsMaterialization(readFileSync(settingsFile, 'utf8'), { providerId, file: 'harness/settings.yaml' })
|
|
1845
|
+
: { ok: stored.plan.action !== 'promote-existing-user' };
|
|
1846
|
+
const sameUserMaterialization = stored.plan.action !== 'promote-existing-user'
|
|
1847
|
+
|| (lockedUserMaterialization.ok === true && JSON.stringify(lockedUserMaterialization.provider) === JSON.stringify(stored.plan.user_materialization_before ?? null));
|
|
1848
|
+
const lockedRevisions = readProviderSourceRevisions();
|
|
1849
|
+
const expectedLocked = stored.plan.expected_revisions ?? {};
|
|
1850
|
+
const sameRevisions = lockedRevisions.profile.revision === expectedLocked.profile
|
|
1851
|
+
&& (lockedRevisions.settings.ok ? lockedRevisions.settings.revision : null) === expectedLocked.settings;
|
|
1852
|
+
if (hasPendingProviderRecoveryTransactions(lockedInventory)) {
|
|
1853
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
1854
|
+
}
|
|
1855
|
+
if (!lockedEntry || lockedEntry.action !== stored.plan.action || lockedEntry.collision !== null
|
|
1856
|
+
|| !sameRouting || !sameDefault || !sameMaterialization || !sameUserMaterialization || !sameRevisions
|
|
1857
|
+
|| lockedInventory.default_evidence?.ok !== true
|
|
1858
|
+
|| !hasCompleteProviderCatalogEvidence(lockedInventory.catalog_evidence)
|
|
1859
|
+
|| !hasCompleteProviderDeclarationEvidence(lockedInventory.declaration_evidence)
|
|
1860
|
+
|| !hasAvailableProviderLifecycleEvidence(lockedInventory.lifecycle_evidence)) {
|
|
1861
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_PLAN_CHANGED' });
|
|
1862
|
+
}
|
|
1863
|
+
if (hub.hasProviderLease(providerId)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_IN_USE' });
|
|
1864
|
+
result = await executeProviderLayerMigration(stored.plan, hooks, { confirm: true, deferRestart: true });
|
|
1865
|
+
if (result.ok && result.state === 'RESTART_PENDING') await hooks.setRestartRuntimeBaseline(getHubRuntimeIdentity().runtime_id);
|
|
1866
|
+
providerMigrationPlans.delete(body.plan_id);
|
|
1867
|
+
retainProviderMutation = result.state === 'RESTART_PENDING';
|
|
1868
|
+
if (retainProviderMutation) {
|
|
1869
|
+
providerMigrationOwners.set(providerId, stored.plan.plan_id);
|
|
1870
|
+
providerStoreMutationOwners.set(stored.plan.plan_id, providerId);
|
|
1871
|
+
retainProviderStoreMutation = true;
|
|
1872
|
+
}
|
|
1873
|
+
return sendJson(res, result.ok ? 202 : 409, { ok: result.ok, restart_required: result.restart_required === true, result });
|
|
1874
|
+
} finally {
|
|
1875
|
+
await hooks.release();
|
|
1876
|
+
if (!retainProviderMutation) hub.endProviderMutation(providerId);
|
|
1877
|
+
if (!retainProviderStoreMutation) hub.endProviderStoreMutation();
|
|
1878
|
+
}
|
|
1879
|
+
}
|
|
1880
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'verify-migration') {
|
|
1881
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_MIGRATION_VERIFY_CONFIRM_REQUIRED' });
|
|
1882
|
+
const transactionId = body?.transaction_id;
|
|
1883
|
+
const hooks = createProviderLayerMigrationFileHooks({
|
|
1884
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1885
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1886
|
+
backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
|
|
1887
|
+
existingMigrationId: transactionId,
|
|
1888
|
+
});
|
|
1889
|
+
let acquiredStoreFence = false;
|
|
1890
|
+
try {
|
|
1891
|
+
await hooks.acquireLock();
|
|
1892
|
+
await hooks.backup({});
|
|
1893
|
+
const plan = hooks.backupPlan();
|
|
1894
|
+
if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_PLAN_PROVIDER_MISMATCH' });
|
|
1895
|
+
const owner = providerMigrationOwners.get(providerId);
|
|
1896
|
+
if (providerStoreMutationOwners.get(plan.plan_id) !== providerId || !hub.hasProviderStoreMutation()) {
|
|
1897
|
+
if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
1898
|
+
providerStoreMutationOwners.set(plan.plan_id, providerId);
|
|
1899
|
+
acquiredStoreFence = true;
|
|
1900
|
+
}
|
|
1901
|
+
if (owner !== plan.plan_id) {
|
|
1902
|
+
if (hub.hasProviderLease(providerId) || !hub.beginProviderMutation(providerId)) {
|
|
1903
|
+
if (acquiredStoreFence && providerStoreMutationOwners.get(plan.plan_id) === providerId) { providerStoreMutationOwners.delete(plan.plan_id); hub.endProviderStoreMutation(); }
|
|
1904
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
1905
|
+
}
|
|
1906
|
+
providerMigrationOwners.set(providerId, plan.plan_id);
|
|
1907
|
+
}
|
|
1908
|
+
const verification = await hooks.verify(plan);
|
|
1909
|
+
const live = await readProviderInventorySnapshot(hub, ctx, config);
|
|
1910
|
+
verification.catalogEvidence = live.catalog_evidence;
|
|
1911
|
+
verification.defaultEvidence = live.default_evidence;
|
|
1912
|
+
verification.defaultPreserved = live.default_evidence?.ok === true
|
|
1913
|
+
&& JSON.stringify(live.harness_default ?? null) === JSON.stringify(plan.harness_default_before ?? null);
|
|
1914
|
+
verification.liveNativeRemovable = hasCompleteProviderCatalogEvidence(live.catalog_evidence)
|
|
1915
|
+
&& live.records.some((record) => record.id === providerId && record.lifecycle?.catalogued === true && record.native_removable === true);
|
|
1916
|
+
verification.runtimeRestarted = hasProviderRuntimeRestartEvidence(plan.runtime_id_before_restart ?? plan.runtime_id_before, getHubRuntimeIdentity().runtime_id);
|
|
1917
|
+
if (verification.nativeRemovable !== true || verification.liveNativeRemovable !== true || verification.defaultPreserved !== true || verification.runtimeRestarted !== true) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_VERIFY_FAILED', verification });
|
|
1918
|
+
await hooks.finalizeVerified();
|
|
1919
|
+
if (providerMigrationOwners.get(providerId) === plan.plan_id) {
|
|
1920
|
+
providerMigrationOwners.delete(providerId);
|
|
1921
|
+
hub.endProviderMutation(providerId);
|
|
1922
|
+
}
|
|
1923
|
+
if (providerStoreMutationOwners.get(plan.plan_id) === providerId) {
|
|
1924
|
+
providerStoreMutationOwners.delete(plan.plan_id);
|
|
1925
|
+
hub.endProviderStoreMutation();
|
|
1926
|
+
}
|
|
1927
|
+
return sendJson(res, 200, { ok: true, state: 'VERIFIED', transaction_id: plan.plan_id, provider_id: providerId, verification });
|
|
1928
|
+
} finally {
|
|
1929
|
+
await hooks.release();
|
|
1930
|
+
// Keep the transaction-owned fence on verification failure so
|
|
1931
|
+
// an incomplete migration cannot race a new Worker; the caller
|
|
1932
|
+
// must explicitly retry verification or roll it back.
|
|
1933
|
+
}
|
|
1934
|
+
}
|
|
1935
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'rollback-migration') {
|
|
1936
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_MIGRATION_ROLLBACK_CONFIRM_REQUIRED' });
|
|
1937
|
+
const hooks = createProviderLayerMigrationFileHooks({
|
|
1938
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1939
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1940
|
+
backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
|
|
1941
|
+
existingMigrationId: body?.transaction_id,
|
|
1942
|
+
});
|
|
1943
|
+
let acquiredMigrationFence = false;
|
|
1944
|
+
let retainProviderMutation = false;
|
|
1945
|
+
let acquiredStoreFence = false;
|
|
1946
|
+
try {
|
|
1947
|
+
await hooks.acquireLock();
|
|
1948
|
+
await hooks.backup({});
|
|
1949
|
+
const plan = hooks.backupPlan();
|
|
1950
|
+
if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_PLAN_PROVIDER_MISMATCH' });
|
|
1951
|
+
const owner = providerMigrationOwners.get(providerId);
|
|
1952
|
+
if (providerStoreMutationOwners.get(plan.plan_id) !== providerId || !hub.hasProviderStoreMutation()) {
|
|
1953
|
+
if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
1954
|
+
providerStoreMutationOwners.set(plan.plan_id, providerId);
|
|
1955
|
+
acquiredStoreFence = true;
|
|
1956
|
+
}
|
|
1957
|
+
if (owner !== plan.plan_id) {
|
|
1958
|
+
if (hub.hasProviderLease(providerId) || !hub.beginProviderMutation(providerId)) {
|
|
1959
|
+
if (acquiredStoreFence && providerStoreMutationOwners.get(plan.plan_id) === providerId) { providerStoreMutationOwners.delete(plan.plan_id); hub.endProviderStoreMutation(); }
|
|
1960
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
1961
|
+
}
|
|
1962
|
+
providerMigrationOwners.set(providerId, plan.plan_id);
|
|
1963
|
+
acquiredMigrationFence = true;
|
|
1964
|
+
retainProviderMutation = true;
|
|
1965
|
+
}
|
|
1966
|
+
const rollback = await hooks.rollback();
|
|
1967
|
+
await hooks.setRollbackRuntimeBaseline(getHubRuntimeIdentity().runtime_id);
|
|
1968
|
+
providerMigrationOwners.set(providerId, plan.plan_id);
|
|
1969
|
+
retainProviderMutation = true;
|
|
1970
|
+
return sendJson(res, 202, { ok: true, restart_required: true, transaction_id: plan.plan_id, provider_id: providerId, state: 'ROLLBACK_RESTART_PENDING' });
|
|
1971
|
+
} finally {
|
|
1972
|
+
await hooks.release();
|
|
1973
|
+
if (acquiredMigrationFence && !retainProviderMutation && providerMigrationOwners.get(providerId) === body?.transaction_id) {
|
|
1974
|
+
providerMigrationOwners.delete(providerId);
|
|
1975
|
+
hub.endProviderMutation(providerId);
|
|
1976
|
+
}
|
|
1977
|
+
}
|
|
1978
|
+
}
|
|
1979
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'verify-rollback-migration') {
|
|
1980
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_MIGRATION_ROLLBACK_VERIFY_CONFIRM_REQUIRED' });
|
|
1981
|
+
const transactionId = body?.transaction_id;
|
|
1982
|
+
const hooks = createProviderLayerMigrationFileHooks({
|
|
1983
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1984
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1985
|
+
backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
|
|
1986
|
+
existingMigrationId: transactionId,
|
|
1987
|
+
});
|
|
1988
|
+
let acquiredStoreFence = false;
|
|
1989
|
+
try {
|
|
1990
|
+
await hooks.acquireLock();
|
|
1991
|
+
await hooks.backup({});
|
|
1992
|
+
const plan = hooks.backupPlan();
|
|
1993
|
+
if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_PLAN_PROVIDER_MISMATCH' });
|
|
1994
|
+
const owner = providerMigrationOwners.get(providerId);
|
|
1995
|
+
if (providerStoreMutationOwners.get(plan.plan_id) !== providerId || !hub.hasProviderStoreMutation()) {
|
|
1996
|
+
if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
1997
|
+
providerStoreMutationOwners.set(plan.plan_id, providerId);
|
|
1998
|
+
acquiredStoreFence = true;
|
|
1999
|
+
}
|
|
2000
|
+
if (owner !== plan.plan_id) {
|
|
2001
|
+
if (hub.hasProviderLease(providerId) || !hub.beginProviderMutation(providerId)) {
|
|
2002
|
+
if (acquiredStoreFence && providerStoreMutationOwners.get(plan.plan_id) === providerId) { providerStoreMutationOwners.delete(plan.plan_id); hub.endProviderStoreMutation(); }
|
|
2003
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
2004
|
+
}
|
|
2005
|
+
providerMigrationOwners.set(providerId, plan.plan_id);
|
|
2006
|
+
}
|
|
2007
|
+
const verification = await hooks.verifyRollback(plan);
|
|
2008
|
+
const live = await readProviderInventorySnapshot(hub, ctx, config);
|
|
2009
|
+
verification.catalogEvidence = live.catalog_evidence;
|
|
2010
|
+
verification.defaultEvidence = live.default_evidence;
|
|
2011
|
+
verification.defaultPreserved = live.default_evidence?.ok === true
|
|
2012
|
+
&& JSON.stringify(live.harness_default ?? null) === JSON.stringify(plan.harness_default_before ?? null);
|
|
2013
|
+
verification.liveRestored = hasCompleteProviderCatalogEvidence(live.catalog_evidence)
|
|
2014
|
+
&& live.records.some((record) => record.id === providerId && record.lifecycle?.catalogued === true && record.native_removable === false);
|
|
2015
|
+
verification.runtimeRestarted = hasProviderRuntimeRestartEvidence(plan.rollback_runtime_id_before, getHubRuntimeIdentity().runtime_id);
|
|
2016
|
+
if (verification.ok !== true || verification.liveRestored !== true || verification.defaultPreserved !== true || verification.runtimeRestarted !== true) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_ROLLBACK_VERIFY_FAILED', verification });
|
|
2017
|
+
await hooks.finalizeRollback();
|
|
2018
|
+
if (providerMigrationOwners.get(providerId) === plan.plan_id) {
|
|
2019
|
+
providerMigrationOwners.delete(providerId);
|
|
2020
|
+
hub.endProviderMutation(providerId);
|
|
2021
|
+
}
|
|
2022
|
+
if (providerStoreMutationOwners.get(plan.plan_id) === providerId) {
|
|
2023
|
+
providerStoreMutationOwners.delete(plan.plan_id);
|
|
2024
|
+
hub.endProviderStoreMutation();
|
|
2025
|
+
}
|
|
2026
|
+
return sendJson(res, 200, { ok: true, state: 'ROLLED_BACK', transaction_id: plan.plan_id, provider_id: providerId, verification });
|
|
2027
|
+
} finally { await hooks.release(); }
|
|
2028
|
+
}
|
|
1368
2029
|
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'delete-plan') {
|
|
1369
|
-
|
|
1370
|
-
if (!
|
|
2030
|
+
if (providerId !== 'deepseek-official' && hasPendingProviderRecoveryTransactions(inventory)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
2031
|
+
if (providerId !== 'deepseek-official' && !hasCompleteProviderCatalogEvidence(inventory.catalog_evidence)) return sendJson(res, 503, { ok: false, code: 'MODEL_CATALOG_UNAVAILABLE' });
|
|
2032
|
+
if (providerId !== 'deepseek-official' && !hasCompleteProviderDeclarationEvidence(inventory.declaration_evidence)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_SOURCE_UNRESOLVED' });
|
|
2033
|
+
if (providerId !== 'deepseek-official' && !hasAvailableProviderLifecycleEvidence(inventory.lifecycle_evidence)) return sendJson(res, 503, { ok: false, code: 'PROVIDER_LIFECYCLE_UNAVAILABLE' });
|
|
2034
|
+
if (providerId !== 'deepseek-official' && inventory.default_evidence?.ok !== true) return sendJson(res, 409, { ok: false, code: inventory.default_evidence?.code ?? 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' });
|
|
2035
|
+
const revisions = readProviderSourceRevisions();
|
|
2036
|
+
if (!revisions.profile.ok && !revisions.settings.ok) return sendJson(res, 409, { ok: false, code: revisions.profile.code ?? revisions.settings.code });
|
|
2037
|
+
const expectedRevision = (revisions.profile.ok ? revisions.profile.revision : null) ?? revisions.settings.revision;
|
|
1371
2038
|
const planned = planProviderDelete({
|
|
1372
2039
|
providerId,
|
|
1373
2040
|
inventory,
|
|
1374
2041
|
activeJobs: hub.list(),
|
|
1375
2042
|
replacementDefault: body?.replacement_default,
|
|
1376
|
-
expectedRevision
|
|
2043
|
+
expectedRevision,
|
|
2044
|
+
expectedRevisions: {
|
|
2045
|
+
profile: revisions.profile.ok ? revisions.profile.revision : null,
|
|
2046
|
+
settings: revisions.settings.ok ? revisions.settings.revision : null,
|
|
2047
|
+
},
|
|
1377
2048
|
});
|
|
1378
2049
|
if (!planned.ok) return sendJson(res, 409, { ok: false, code: planned.code });
|
|
1379
|
-
|
|
1380
|
-
|
|
2050
|
+
const plan = { ...planned.plan, runtime_id_before: getHubRuntimeIdentity().runtime_id };
|
|
2051
|
+
rememberProviderDeletePlan(plan);
|
|
2052
|
+
return sendJson(res, 200, { ok: true, profile_revision: expectedRevision, plan });
|
|
1381
2053
|
}
|
|
1382
2054
|
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'probe') {
|
|
1383
2055
|
const record = inventory.records.find((entry) => entry.id === providerId);
|
|
@@ -1413,9 +2085,11 @@ export async function apply(ctx) {
|
|
|
1413
2085
|
const { transaction_id: transactionId } = body ?? {};
|
|
1414
2086
|
const hooks = createProviderDeleteFileHooks({
|
|
1415
2087
|
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
2088
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1416
2089
|
configFile: join(CONFIG_DIR, 'config.json'),
|
|
1417
2090
|
lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
1418
2091
|
backupDir: join(CONFIG_DIR, 'provider-backups'),
|
|
2092
|
+
sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
|
|
1419
2093
|
existingBackupId: transactionId,
|
|
1420
2094
|
expectedProviderId: providerId,
|
|
1421
2095
|
});
|
|
@@ -1423,11 +2097,30 @@ export async function apply(ctx) {
|
|
|
1423
2097
|
if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_PLAN_PROVIDER_MISMATCH' });
|
|
1424
2098
|
await hooks.acquireLock();
|
|
1425
2099
|
try {
|
|
2100
|
+
if (providerStoreMutationOwners.get(transactionId) !== providerId || !hub.hasProviderStoreMutation()) {
|
|
2101
|
+
if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_BUSY' });
|
|
2102
|
+
providerStoreMutationOwners.set(transactionId, providerId);
|
|
2103
|
+
}
|
|
1426
2104
|
const verification = await hooks.verify(plan);
|
|
1427
|
-
|
|
2105
|
+
const live = await readProviderInventorySnapshot(hub, ctx, config);
|
|
2106
|
+
verification.catalog_evidence = live.catalog_evidence;
|
|
2107
|
+
verification.default_evidence = live.default_evidence;
|
|
2108
|
+
verification.catalogAbsent = hasCompleteProviderCatalogEvidence(live.catalog_evidence)
|
|
2109
|
+
&& !live.records.some((entry) => entry.id === providerId && entry.lifecycle?.catalogued === true);
|
|
2110
|
+
verification.harnessDefault = plan.was_harness_default !== true || (
|
|
2111
|
+
live.harness_default?.provider === plan.replacement_default
|
|
2112
|
+
&& live.harness_default?.model === plan.replacement_default_model
|
|
2113
|
+
);
|
|
2114
|
+
verification.runtimeRestarted = hasProviderRuntimeRestartEvidence(plan.delete_runtime_id_before_restart, getHubRuntimeIdentity().runtime_id);
|
|
2115
|
+
if (verification?.providerAbsent !== true || !hasCompleteProviderCatalogEvidence(verification?.catalog_evidence) || verification?.default_evidence?.ok !== true || verification?.catalogAbsent !== true || verification?.harnessDefault !== true || verification?.runtimeRestarted !== true || verification?.routingClear !== true || verification?.tombstonePresent !== true) {
|
|
1428
2116
|
return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_VERIFY_FAILED', verification });
|
|
1429
2117
|
}
|
|
1430
2118
|
await hooks.recordTransaction({ transaction_id: transactionId, provider_id: providerId, state: 'VERIFIED' }, plan);
|
|
2119
|
+
hub.endProviderMutation(providerId);
|
|
2120
|
+
if (providerStoreMutationOwners.get(transactionId) === providerId) {
|
|
2121
|
+
providerStoreMutationOwners.delete(transactionId);
|
|
2122
|
+
hub.endProviderStoreMutation();
|
|
2123
|
+
}
|
|
1431
2124
|
return sendJson(res, 200, { ok: true, transaction_id: transactionId, provider_id: providerId, state: 'VERIFIED', verification });
|
|
1432
2125
|
} finally {
|
|
1433
2126
|
await hooks.release();
|
|
@@ -1438,25 +2131,41 @@ export async function apply(ctx) {
|
|
|
1438
2131
|
const { transaction_id: transactionId } = body ?? {};
|
|
1439
2132
|
const hooks = createProviderDeleteFileHooks({
|
|
1440
2133
|
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
2134
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1441
2135
|
configFile: join(CONFIG_DIR, 'config.json'),
|
|
1442
2136
|
lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
1443
2137
|
backupDir: join(CONFIG_DIR, 'provider-backups'),
|
|
2138
|
+
sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
|
|
1444
2139
|
existingBackupId: transactionId,
|
|
1445
2140
|
expectedProviderId: providerId,
|
|
1446
2141
|
});
|
|
1447
2142
|
await hooks.acquireLock();
|
|
2143
|
+
let acquiredStoreMutation = false;
|
|
2144
|
+
let retainStoreMutation = false;
|
|
1448
2145
|
try {
|
|
1449
2146
|
const plan = hooks.backupPlan();
|
|
1450
2147
|
if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_PLAN_PROVIDER_MISMATCH' });
|
|
2148
|
+
if (providerStoreMutationOwners.get(transactionId) !== providerId || !hub.hasProviderStoreMutation()) {
|
|
2149
|
+
if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_BUSY' });
|
|
2150
|
+
providerStoreMutationOwners.set(transactionId, providerId);
|
|
2151
|
+
acquiredStoreMutation = true;
|
|
2152
|
+
}
|
|
1451
2153
|
await hooks.rollback(plan);
|
|
2154
|
+
await hooks.setRuntimeBaseline(getHubRuntimeIdentity().runtime_id, 'rollback');
|
|
2155
|
+
const rollbackPlan = hooks.backupPlan() ?? plan;
|
|
1452
2156
|
try {
|
|
1453
|
-
await hooks.recordTransaction({ transaction_id: transactionId, provider_id: providerId, state: 'ROLLBACK_PENDING' },
|
|
2157
|
+
await hooks.recordTransaction({ transaction_id: transactionId, provider_id: providerId, state: 'ROLLBACK_PENDING' }, rollbackPlan);
|
|
1454
2158
|
} catch (error) {
|
|
1455
2159
|
return sendJson(res, 409, { ok: false, code: boundedMachineCodeFromError(error) ?? 'PROVIDER_LIFECYCLE_RECORD_FAILED' });
|
|
1456
2160
|
}
|
|
2161
|
+
retainStoreMutation = true;
|
|
1457
2162
|
return sendJson(res, 202, { ok: true, restart_required: true, transaction_id: transactionId, provider_id: providerId, state: 'ROLLBACK_PENDING' });
|
|
1458
2163
|
} finally {
|
|
1459
2164
|
await hooks.release();
|
|
2165
|
+
if (acquiredStoreMutation && !retainStoreMutation && providerStoreMutationOwners.get(transactionId) === providerId) {
|
|
2166
|
+
providerStoreMutationOwners.delete(transactionId);
|
|
2167
|
+
hub.endProviderStoreMutation();
|
|
2168
|
+
}
|
|
1460
2169
|
}
|
|
1461
2170
|
}
|
|
1462
2171
|
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'verify-rollback') {
|
|
@@ -1464,9 +2173,11 @@ export async function apply(ctx) {
|
|
|
1464
2173
|
const { transaction_id: transactionId } = body ?? {};
|
|
1465
2174
|
const hooks = createProviderDeleteFileHooks({
|
|
1466
2175
|
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
2176
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1467
2177
|
configFile: join(CONFIG_DIR, 'config.json'),
|
|
1468
2178
|
lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
1469
2179
|
backupDir: join(CONFIG_DIR, 'provider-backups'),
|
|
2180
|
+
sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
|
|
1470
2181
|
existingBackupId: transactionId,
|
|
1471
2182
|
expectedProviderId: providerId,
|
|
1472
2183
|
});
|
|
@@ -1474,9 +2185,28 @@ export async function apply(ctx) {
|
|
|
1474
2185
|
if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_PLAN_PROVIDER_MISMATCH' });
|
|
1475
2186
|
await hooks.acquireLock();
|
|
1476
2187
|
try {
|
|
2188
|
+
if (providerStoreMutationOwners.get(transactionId) !== providerId || !hub.hasProviderStoreMutation()) {
|
|
2189
|
+
if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_BUSY' });
|
|
2190
|
+
providerStoreMutationOwners.set(transactionId, providerId);
|
|
2191
|
+
}
|
|
1477
2192
|
const verified = await hooks.verifyRollback(plan);
|
|
1478
|
-
|
|
2193
|
+
const live = await readProviderInventorySnapshot(hub, ctx, config);
|
|
2194
|
+
verified.catalog_evidence = live.catalog_evidence;
|
|
2195
|
+
verified.default_evidence = live.default_evidence;
|
|
2196
|
+
verified.catalogPresent = hasCompleteProviderCatalogEvidence(live.catalog_evidence)
|
|
2197
|
+
&& live.records.some((entry) => entry.id === providerId && entry.lifecycle?.catalogued === true);
|
|
2198
|
+
verified.harnessDefaultRestored = plan.was_harness_default !== true || (
|
|
2199
|
+
live.harness_default?.provider === plan.harness_default_before?.provider
|
|
2200
|
+
&& live.harness_default?.model === plan.harness_default_before?.model
|
|
2201
|
+
);
|
|
2202
|
+
verified.runtimeRestarted = hasProviderRuntimeRestartEvidence(plan.rollback_runtime_id_before_restart, getHubRuntimeIdentity().runtime_id);
|
|
2203
|
+
if (verified?.ok !== true || !hasCompleteProviderCatalogEvidence(verified?.catalog_evidence) || verified?.default_evidence?.ok !== true || verified?.catalogPresent !== true || verified?.harnessDefaultRestored !== true || verified?.runtimeRestarted !== true) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_ROLLBACK_VERIFY_FAILED', verification: verified });
|
|
1479
2204
|
await hooks.recordTransaction({ transaction_id: transactionId, provider_id: providerId, state: 'ROLLED_BACK' }, plan);
|
|
2205
|
+
hub.endProviderMutation(providerId);
|
|
2206
|
+
if (providerStoreMutationOwners.get(transactionId) === providerId) {
|
|
2207
|
+
providerStoreMutationOwners.delete(transactionId);
|
|
2208
|
+
hub.endProviderStoreMutation();
|
|
2209
|
+
}
|
|
1480
2210
|
return sendJson(res, 200, { ok: true, transaction_id: transactionId, provider_id: providerId, state: 'ROLLED_BACK' });
|
|
1481
2211
|
} finally {
|
|
1482
2212
|
await hooks.release();
|
|
@@ -1492,33 +2222,128 @@ export async function apply(ctx) {
|
|
|
1492
2222
|
providerDeletePlans.delete(body?.plan_id);
|
|
1493
2223
|
return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_PLAN_EXPIRED' });
|
|
1494
2224
|
}
|
|
1495
|
-
const
|
|
1496
|
-
if (!profile.ok) return sendJson(res, 409, { ok: false, code: profile.code });
|
|
1497
|
-
if (
|
|
2225
|
+
const revisions = readProviderSourceRevisions();
|
|
2226
|
+
if (!revisions.profile.ok && !revisions.settings.ok) return sendJson(res, 409, { ok: false, code: revisions.profile.code ?? revisions.settings.code });
|
|
2227
|
+
if (providerId !== 'deepseek-official' && !hasCompleteProviderCatalogEvidence(inventory.catalog_evidence)) return sendJson(res, 503, { ok: false, code: 'MODEL_CATALOG_UNAVAILABLE' });
|
|
2228
|
+
if (providerId !== 'deepseek-official' && !hasCompleteProviderDeclarationEvidence(inventory.declaration_evidence)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_SOURCE_UNRESOLVED' });
|
|
2229
|
+
if (providerId !== 'deepseek-official' && !hasAvailableProviderLifecycleEvidence(inventory.lifecycle_evidence)) return sendJson(res, 503, { ok: false, code: 'PROVIDER_LIFECYCLE_UNAVAILABLE' });
|
|
2230
|
+
if (providerId !== 'deepseek-official' && inventory.default_evidence?.ok !== true) return sendJson(res, 409, { ok: false, code: inventory.default_evidence?.code ?? 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' });
|
|
2231
|
+
const storedExpected = stored.plan.expected_revisions;
|
|
2232
|
+
const revisionsMatch = storedExpected
|
|
2233
|
+
? (storedExpected.profile === (revisions.profile.ok ? revisions.profile.revision : null)
|
|
2234
|
+
&& storedExpected.settings === (revisions.settings.ok ? revisions.settings.revision : null))
|
|
2235
|
+
: (body?.expected_revision === stored.plan.expected_revision && (revisions.profile.revision ?? revisions.settings.revision) === stored.plan.expected_revision);
|
|
2236
|
+
if (!revisionsMatch || body?.expected_revision !== stored.plan.expected_revision) {
|
|
1498
2237
|
return sendJson(res, 409, { ok: false, code: 'PROVIDER_PROFILE_CHANGED' });
|
|
1499
2238
|
}
|
|
2239
|
+
if (providerId !== 'deepseek-official' && hasPendingProviderRecoveryTransactions(inventory)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
1500
2240
|
const refreshed = planProviderDelete({
|
|
1501
2241
|
providerId,
|
|
1502
2242
|
inventory,
|
|
1503
2243
|
activeJobs: hub.list(),
|
|
1504
2244
|
replacementDefault: stored.plan.replacement_default,
|
|
1505
|
-
expectedRevision:
|
|
2245
|
+
expectedRevision: stored.plan.expected_revision,
|
|
2246
|
+
expectedRevisions: stored.plan.expected_revisions,
|
|
1506
2247
|
});
|
|
1507
2248
|
if (!refreshed.ok) return sendJson(res, 409, { ok: false, code: refreshed.code });
|
|
1508
2249
|
if (refreshed.plan.replacement_default_model !== stored.plan.replacement_default_model) {
|
|
1509
2250
|
return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_PLAN_CHANGED' });
|
|
1510
2251
|
}
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
}
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
2252
|
+
let executionPlan = {
|
|
2253
|
+
...refreshed.plan,
|
|
2254
|
+
plan_id: stored.plan.plan_id,
|
|
2255
|
+
expected_revision: stored.plan.expected_revision,
|
|
2256
|
+
expected_revisions: stored.plan.expected_revisions,
|
|
2257
|
+
runtime_id_before: stored.plan.runtime_id_before,
|
|
2258
|
+
};
|
|
2259
|
+
if (!hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_BUSY' });
|
|
2260
|
+
if (!hub.beginProviderMutation(providerId)) {
|
|
2261
|
+
hub.endProviderStoreMutation();
|
|
2262
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_BUSY' });
|
|
2263
|
+
}
|
|
2264
|
+
let hooks = null;
|
|
2265
|
+
let lockAcquired = false;
|
|
2266
|
+
let retainProviderMutation = false;
|
|
2267
|
+
let retainProviderStoreMutation = false;
|
|
2268
|
+
try {
|
|
2269
|
+
hooks = createProviderDeleteFileHooks({
|
|
2270
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
2271
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
2272
|
+
configFile: join(CONFIG_DIR, 'config.json'),
|
|
2273
|
+
lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
2274
|
+
backupDir: join(CONFIG_DIR, 'provider-backups'),
|
|
2275
|
+
sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
|
|
2276
|
+
runtimeIdProvider: () => getHubRuntimeIdentity().runtime_id,
|
|
2277
|
+
afterLockAcquired: () => {
|
|
2278
|
+
const pendingRecovery = [
|
|
2279
|
+
...readProviderRecoveryTransactions(),
|
|
2280
|
+
...readProviderLayerMigrationTransactions(join(CONFIG_DIR, 'provider-migration-backups')),
|
|
2281
|
+
];
|
|
2282
|
+
if (pendingRecovery.length > 0) {
|
|
2283
|
+
throw Object.assign(new Error('provider recovery transaction is pending'), { code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
2284
|
+
}
|
|
2285
|
+
if (hub.hasProviderLease(providerId)) {
|
|
2286
|
+
throw Object.assign(new Error('provider is in use'), { code: 'PROVIDER_IN_USE' });
|
|
2287
|
+
}
|
|
2288
|
+
},
|
|
2289
|
+
});
|
|
2290
|
+
// Hold the real shared store lock while rebuilding the live
|
|
2291
|
+
// inventory and delete plan. The pre-lock plan is only a
|
|
2292
|
+
// proposal; catalog/default/ownership/recovery facts must be
|
|
2293
|
+
// authoritative at the mutation boundary.
|
|
2294
|
+
await hooks.acquireLock();
|
|
2295
|
+
lockAcquired = true;
|
|
2296
|
+
const lockedConfig = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
2297
|
+
const lockedInventory = await readProviderInventorySnapshot(hub, ctx, lockedConfig);
|
|
2298
|
+
const lockedRevisions = readProviderSourceRevisions();
|
|
2299
|
+
const lockedExpected = stored.plan.expected_revisions;
|
|
2300
|
+
const lockedRevisionsMatch = lockedExpected
|
|
2301
|
+
? lockedExpected.profile === (lockedRevisions.profile.ok ? lockedRevisions.profile.revision : null)
|
|
2302
|
+
&& lockedExpected.settings === (lockedRevisions.settings.ok ? lockedRevisions.settings.revision : null)
|
|
2303
|
+
: false;
|
|
2304
|
+
if (providerId !== 'deepseek-official' && !hasCompleteProviderCatalogEvidence(lockedInventory.catalog_evidence)) throw Object.assign(new Error('provider catalog unavailable after lock'), { code: 'MODEL_CATALOG_UNAVAILABLE' });
|
|
2305
|
+
if (providerId !== 'deepseek-official' && !hasCompleteProviderDeclarationEvidence(lockedInventory.declaration_evidence)) throw Object.assign(new Error('provider declarations unavailable after lock'), { code: 'PROVIDER_DELETE_SOURCE_UNRESOLVED' });
|
|
2306
|
+
if (providerId !== 'deepseek-official' && !hasAvailableProviderLifecycleEvidence(lockedInventory.lifecycle_evidence)) throw Object.assign(new Error('provider lifecycle unavailable after lock'), { code: 'PROVIDER_LIFECYCLE_UNAVAILABLE' });
|
|
2307
|
+
if (providerId !== 'deepseek-official' && lockedInventory.default_evidence?.ok !== true) throw Object.assign(new Error('provider default authority changed after lock'), { code: lockedInventory.default_evidence?.code ?? 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' });
|
|
2308
|
+
if (providerId !== 'deepseek-official' && hasPendingProviderRecoveryTransactions(lockedInventory)) throw Object.assign(new Error('provider recovery transaction is pending'), { code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
2309
|
+
if (!lockedRevisionsMatch) throw Object.assign(new Error('provider source changed after lock'), { code: 'PROVIDER_PROFILE_CHANGED' });
|
|
2310
|
+
if (hub.hasProviderLease(providerId)) throw Object.assign(new Error('provider is in use'), { code: 'PROVIDER_IN_USE' });
|
|
2311
|
+
const lockedPlan = planProviderDelete({
|
|
2312
|
+
providerId,
|
|
2313
|
+
inventory: lockedInventory,
|
|
2314
|
+
activeJobs: hub.list(),
|
|
2315
|
+
replacementDefault: stored.plan.replacement_default,
|
|
2316
|
+
expectedRevision: stored.plan.expected_revision,
|
|
2317
|
+
expectedRevisions: stored.plan.expected_revisions,
|
|
2318
|
+
});
|
|
2319
|
+
if (!lockedPlan.ok) throw Object.assign(new Error('provider delete plan changed after lock'), { code: lockedPlan.code });
|
|
2320
|
+
const sameLockedPlan = lockedPlan.plan.replacement_default_model === stored.plan.replacement_default_model
|
|
2321
|
+
&& lockedPlan.plan.was_harness_default === stored.plan.was_harness_default
|
|
2322
|
+
&& JSON.stringify(lockedPlan.plan.declaration_authorities) === JSON.stringify(stored.plan.declaration_authorities)
|
|
2323
|
+
&& JSON.stringify(lockedPlan.plan.harness_default_before ?? null) === JSON.stringify(stored.plan.harness_default_before ?? null)
|
|
2324
|
+
&& JSON.stringify(lockedPlan.plan.credential_refs) === JSON.stringify(stored.plan.credential_refs);
|
|
2325
|
+
if (!sameLockedPlan) throw Object.assign(new Error('provider delete plan semantics changed after lock'), { code: 'PROVIDER_DELETE_PLAN_CHANGED' });
|
|
2326
|
+
executionPlan = {
|
|
2327
|
+
...lockedPlan.plan,
|
|
2328
|
+
plan_id: stored.plan.plan_id,
|
|
2329
|
+
expected_revision: stored.plan.expected_revision,
|
|
2330
|
+
expected_revisions: stored.plan.expected_revisions,
|
|
2331
|
+
runtime_id_before: stored.plan.runtime_id_before,
|
|
2332
|
+
};
|
|
2333
|
+
providerDeletePlans.delete(body.plan_id);
|
|
2334
|
+
const { executeProviderDelete } = await import('../provider-lifecycle.mjs');
|
|
2335
|
+
const result = await executeProviderDelete(executionPlan, hooks, { deferRestart: true });
|
|
2336
|
+
retainProviderMutation = result.state === 'RESTART_PENDING';
|
|
2337
|
+
if (retainProviderMutation) {
|
|
2338
|
+
providerStoreMutationOwners.set(executionPlan.plan_id, providerId);
|
|
2339
|
+
retainProviderStoreMutation = true;
|
|
2340
|
+
}
|
|
2341
|
+
return sendJson(res, result.state === 'RESTART_PENDING' ? 202 : 409, { ok: result.state === 'RESTART_PENDING', restart_required: true, result });
|
|
2342
|
+
} finally {
|
|
2343
|
+
if (lockAcquired && hooks) await hooks.release().catch(() => {});
|
|
2344
|
+
if (!retainProviderMutation) hub.endProviderMutation(providerId);
|
|
2345
|
+
if (!retainProviderStoreMutation) hub.endProviderStoreMutation();
|
|
2346
|
+
}
|
|
1522
2347
|
}
|
|
1523
2348
|
return sendJson(res, 404, { ok: false, error: 'unknown providers endpoint' });
|
|
1524
2349
|
} catch (err) {
|