@ran-sh/dsh-crew 1.0.1 → 1.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/hub/index.mjs CHANGED
@@ -3,9 +3,9 @@
3
3
  // Web UI session list), exposes a loopback jobs API for the CC/Codex MCP shim,
4
4
  // and serves the one-click installer endpoints for the settings page.
5
5
 
6
- import { randomUUID } from 'node:crypto';
7
- import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
8
- import { join, isAbsolute } from 'node:path';
6
+ import { createHash, randomUUID } from 'node:crypto';
7
+ import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
8
+ import { dirname, join, isAbsolute, resolve as resolvePath } from 'node:path';
9
9
  import { homedir } from 'node:os';
10
10
  import { createShardWriter, readMergedStatus } from '../status-shard.mjs';
11
11
  import {
@@ -34,16 +34,19 @@ import { buildHubExecutionRows } from '../config-readiness.mjs';
34
34
  import { localRequestCore, originLoopback } from '../local-request-guard.mjs';
35
35
  import { raceWaiters } from '../removable-waiter.mjs';
36
36
  import { buildProviderInventory } from '../provider-inventory.mjs';
37
- import { inspectProviderProfile, readProviderDeclarations } from '../provider-profile-store.mjs';
37
+ import { buildProviderLayerMigrationPlan, executeProviderLayerMigration } from '../provider-layer-migration.mjs';
38
+ import { inspectProviderProfile, readProviderDeclarations, readProviderMaterialization } from '../provider-profile-store.mjs';
39
+ import { inspectProviderSettings, readHarnessDefault, readProviderSettingsDeclarations, readProviderSettingsMaterialization } from '../provider-settings-store.mjs';
38
40
  import { normalizeProviderLifecycleState } from '../provider-lifecycle-state.mjs';
39
41
  import { createProviderHealthStore } from '../provider-health.mjs';
40
42
  import { planProviderDelete } from '../provider-lifecycle.mjs';
41
- import { createProviderDeleteFileHooks } from '../provider-delete-adapters.mjs';
43
+ import { createProviderDeleteFileHooks, isRecoverableProviderDeleteBackup, readProviderDeleteManifestFile } from '../provider-delete-adapters.mjs';
42
44
  import { buildCredentialReferenceInventory } from '../credential-reference-inventory.mjs';
43
45
  import { executeCredentialPurge, planCredentialPurge } from '../credential-lifecycle.mjs';
44
46
  import { createCredentialPurgeFileHooks } from '../credential-purge-adapters.mjs';
45
47
  import { normalizeCredentialPurgeState, recordCredentialPurgeOutcome } from '../credential-purge-state.mjs';
46
48
  import { buildRuntimeReadinessSnapshot } from '../runtime-readiness-snapshot.mjs';
49
+ import { createProviderLayerMigrationFileHooks, readProviderLayerMigrationTransactions } from '../provider-layer-migration-adapters.mjs';
47
50
 
48
51
  // policy.mjs is pure (no @deepseek-ai imports, no ctx access), so importing it
49
52
  // here is safe for the profile-realm discipline: it never pulls in package
@@ -106,19 +109,102 @@ const LEGACY_TIER_MODELS = { flash: 'deepseek-v4-flash', pro: 'deepseek-v4-pro'
106
109
  const ROLES = { worker: true, reviewer: true };
107
110
  const CONFIG_DIR = join(homedir(), '.config', 'dsh-crew');
108
111
  const CREDENTIAL_PURGE_STATE_FILE = join(CONFIG_DIR, 'credential-purge-lifecycle.json');
112
+ const RECOVERY_ACTION_PATTERN = /^[a-f0-9]{32}$/u;
109
113
 
110
- function readProviderLifecycleState() {
114
+ function recoveryActionId(name) {
115
+ return createHash('sha256').update(String(name)).digest('hex').slice(0, 32);
116
+ }
117
+
118
+ function isRecoveryControlEntry(name) {
119
+ return name === '.quarantine'
120
+ || name === '.delete.lock'
121
+ || name === '.delete.reclaim.lock'
122
+ || name === '.delete.recovery.lock'
123
+ || name === '.delete.recovery.lock.claim'
124
+ || /^\.delete\.lock\.[0-9a-f-]+\.(?:active|staging|retired)$/iu.test(name)
125
+ || /^\.delete\.recovery\.lock\.[0-9a-f-]+\.stale$/iu.test(name);
126
+ }
127
+
128
+ function readProviderLifecycleStateStatus() {
129
+ const file = join(CONFIG_DIR, 'provider-lifecycle.json');
130
+ if (!existsSync(file)) return { ok: true, state: normalizeProviderLifecycleState() };
111
131
  try {
112
- return normalizeProviderLifecycleState(JSON.parse(readFileSync(join(CONFIG_DIR, 'provider-lifecycle.json'), 'utf8')));
132
+ return { ok: true, state: normalizeProviderLifecycleState(JSON.parse(readFileSync(file, 'utf8'))) };
113
133
  } catch {
114
- return normalizeProviderLifecycleState();
134
+ return { ok: false, code: 'PROVIDER_LIFECYCLE_UNAVAILABLE', state: normalizeProviderLifecycleState() };
115
135
  }
116
136
  }
117
137
 
138
+ function readProviderLifecycleState() {
139
+ return readProviderLifecycleStateStatus().state;
140
+ }
141
+
118
142
  function readCredentialPurgeState() {
119
143
  try { return normalizeCredentialPurgeState(JSON.parse(readFileSync(CREDENTIAL_PURGE_STATE_FILE, 'utf8'))); } catch { return normalizeCredentialPurgeState(); }
120
144
  }
121
145
 
146
+ export function readProviderRecoveryTransactions(root = join(CONFIG_DIR, 'provider-backups')) {
147
+ if (!existsSync(root)) return [];
148
+ try {
149
+ const managedRoot = dirname(resolvePath(root));
150
+ const paths = {
151
+ profile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
152
+ settings: join(CONFIG_DIR, 'harness', 'settings.yaml'),
153
+ config: join(CONFIG_DIR, 'config.json'),
154
+ lifecycle: join(CONFIG_DIR, 'provider-lifecycle.json'),
155
+ };
156
+ const transactions = readdirSync(root, { withFileTypes: true }).filter((entry) => !isRecoveryControlEntry(entry.name)).flatMap((entry) => {
157
+ const transactionRoot = join(root, entry.name);
158
+ const manifestFile = join(root, entry.name, 'manifest.json');
159
+ const actionId = recoveryActionId(entry.name);
160
+ if (!entry.isDirectory()) {
161
+ let updatedAt;
162
+ try { updatedAt = statSync(transactionRoot).mtime.toISOString(); } catch { updatedAt = new Date(0).toISOString(); }
163
+ return [{ storage_id: entry.name, action_id: actionId, transaction_id: null, provider_id: null, phase: 'RECOVERY_UNRESOLVED', updated_at: updatedAt, recoverable: false, unresolved: true }];
164
+ }
165
+ try {
166
+ const manifest = readProviderDeleteManifestFile(manifestFile, managedRoot);
167
+ const transactionId = typeof manifest?.plan?.plan_id === 'string' ? manifest.plan.plan_id : null;
168
+ const providerId = typeof manifest?.provider_id === 'string' ? manifest.provider_id : null;
169
+ const phase = typeof manifest?.phase_journal?.phase === 'string' && manifest.phase_journal.phase.trim()
170
+ ? manifest.phase_journal.phase.trim() : 'RECOVERY_INCOMPLETE';
171
+ const updatedAt = typeof manifest?.phase_journal?.updated_at === 'string' && manifest.phase_journal.updated_at.trim()
172
+ ? manifest.phase_journal.updated_at : statSync(manifestFile).mtime.toISOString();
173
+ const hasPendingMutation = manifest?.mutation_journal && typeof manifest.mutation_journal === 'object'
174
+ && Object.keys(manifest.mutation_journal).length > 0;
175
+ const recoverable = transactionId != null && providerId != null && isRecoverableProviderDeleteBackup(manifest, {
176
+ root: transactionRoot,
177
+ managedRoot,
178
+ paths,
179
+ expectedProviderId: providerId,
180
+ }) === true;
181
+ if (['ROLLED_BACK', 'VERIFIED'].includes(phase) && !hasPendingMutation && recoverable) return [];
182
+ return [{ storage_id: entry.name, action_id: actionId, transaction_id: transactionId ?? entry.name, provider_id: providerId, phase, updated_at: updatedAt, recoverable, unresolved: recoverable !== true }];
183
+ } catch {
184
+ let updatedAt;
185
+ try { updatedAt = statSync(transactionRoot).mtime.toISOString(); } catch { updatedAt = new Date(0).toISOString(); }
186
+ return [{
187
+ storage_id: entry.name,
188
+ action_id: actionId,
189
+ transaction_id: null,
190
+ provider_id: null,
191
+ phase: 'RECOVERY_UNRESOLVED',
192
+ updated_at: updatedAt,
193
+ recoverable: false,
194
+ unresolved: true,
195
+ }];
196
+ }
197
+ });
198
+ // Recovery is a safety gate, not a bounded history view. Never discard an
199
+ // older unresolved transaction merely because newer entries exist.
200
+ return transactions.sort((a, b) => a.updated_at.localeCompare(b.updated_at));
201
+ } catch {
202
+ let updatedAt;
203
+ try { updatedAt = statSync(root).mtime.toISOString(); } catch { updatedAt = new Date(0).toISOString(); }
204
+ return [{ storage_id: null, action_id: null, transaction_id: null, provider_id: null, phase: 'RECOVERY_UNRESOLVED', updated_at: updatedAt, recoverable: false, unresolved: true }];
205
+ }
206
+ }
207
+
122
208
  function writeCredentialPurgeState(state) {
123
209
  mkdirSync(CONFIG_DIR, { recursive: true });
124
210
  const temp = `${CREDENTIAL_PURGE_STATE_FILE}.${process.pid}.${Date.now()}.tmp`;
@@ -144,34 +230,155 @@ function providerInventoryPolicy(config) {
144
230
  };
145
231
  }
146
232
 
233
+ export function hasCompleteProviderCatalogEvidence(evidence) {
234
+ return evidence?.ok === true && evidence.partial !== true;
235
+ }
236
+
237
+ export function hasCompleteProviderDeclarationEvidence(evidence) {
238
+ return evidence?.ok === true
239
+ && Object.values(evidence.sources ?? {}).every((source) => source?.present !== true || !source.code);
240
+ }
241
+
242
+ export function hasAvailableProviderLifecycleEvidence(evidence) {
243
+ return evidence?.ok === true;
244
+ }
245
+
246
+ export function hasPendingProviderRecoveryTransactions(inventory) {
247
+ return Array.isArray(inventory?.recovery_transactions) && inventory.recovery_transactions.length > 0;
248
+ }
249
+
250
+ export function hasProviderRuntimeRestartEvidence(beforeRuntimeId, currentRuntimeId) {
251
+ return typeof beforeRuntimeId === 'string' && beforeRuntimeId.trim() !== ''
252
+ && typeof currentRuntimeId === 'string' && currentRuntimeId.trim() !== ''
253
+ && beforeRuntimeId !== currentRuntimeId;
254
+ }
255
+
256
+ /** Preserve profile/settings provenance when preparing the migration planner. */
257
+ export function buildProviderMigrationDeclarations(profileDeclarations = [], settingsDeclarations = []) {
258
+ return [
259
+ ...(Array.isArray(profileDeclarations) ? profileDeclarations : []),
260
+ ...(Array.isArray(settingsDeclarations) ? settingsDeclarations : []),
261
+ ];
262
+ }
263
+
147
264
  async function readProviderInventorySnapshot(hub, ctx, config) {
148
265
  let catalog = { providers: [], harness_default: null };
266
+ let catalogEvidence = { ok: false, code: 'MODEL_CATALOG_UNAVAILABLE' };
149
267
  try {
150
268
  catalog = await readHarnessModelCatalog({
151
269
  llm: ctx.llm ?? ctx.get('llm'),
152
270
  getCurrentSelection: () => ctx.get('agentDefaultModel')?.currentSelection?.(),
153
271
  });
272
+ catalogEvidence = {
273
+ ok: catalog.partial !== true,
274
+ ...(catalog.partial === true ? { code: 'PROVIDER_CATALOG_INCOMPLETE' } : {}),
275
+ provider_count: Number.isInteger(catalog.provider_count) ? catalog.provider_count : catalog.providers.length,
276
+ refreshed_at: typeof catalog.refreshed_at === 'string' ? catalog.refreshed_at : null,
277
+ partial: catalog.partial === true,
278
+ };
154
279
  } catch {}
155
280
  const profileFile = join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml');
156
281
  let declarations = [];
282
+ let profileDeclarations = [];
283
+ let settingsDeclarations = [];
284
+ const declarationEvidence = { ok: true, sources: {} };
157
285
  try {
286
+ declarationEvidence.sources.profile = { present: existsSync(profileFile) };
158
287
  if (existsSync(profileFile)) {
159
288
  const source = readFileSync(profileFile, 'utf8');
160
289
  const parsed = readProviderDeclarations(source, { file: 'harness/profiles/dsh-crew/cordis.patch.yml' });
161
- if (parsed.ok) declarations = parsed.declarations;
290
+ if (parsed.ok) {
291
+ profileDeclarations = parsed.declarations;
292
+ declarations = profileDeclarations;
293
+ }
294
+ else { declarationEvidence.ok = false; declarationEvidence.sources.profile.code = parsed.code; }
162
295
  }
163
- } catch {}
164
- const lifecycle = readProviderLifecycleState();
296
+ } catch { declarationEvidence.ok = false; declarationEvidence.sources.profile = { present: true, code: 'PROVIDER_SOURCE_UNAVAILABLE' }; }
297
+ const settingsFile = join(CONFIG_DIR, 'harness', 'settings.yaml');
298
+ let settingsDefault = null;
299
+ let defaultEvidence = { ok: true, source: 'none' };
300
+ try {
301
+ declarationEvidence.sources.settings = { present: existsSync(settingsFile) };
302
+ if (existsSync(settingsFile)) {
303
+ const source = readFileSync(settingsFile, 'utf8');
304
+ const parsed = readProviderSettingsDeclarations(source, { file: 'harness/settings.yaml' });
305
+ if (parsed.ok) {
306
+ settingsDeclarations = parsed.declarations;
307
+ declarations = [...declarations, ...settingsDeclarations];
308
+ }
309
+ else { declarationEvidence.ok = false; declarationEvidence.sources.settings.code = parsed.code; }
310
+ const parsedDefault = readHarnessDefault(source);
311
+ if (parsedDefault.ok) settingsDefault = parsedDefault;
312
+ else defaultEvidence = { ok: false, code: 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' };
313
+ }
314
+ } catch {
315
+ declarationEvidence.ok = false;
316
+ declarationEvidence.sources.settings = { present: true, code: 'PROVIDER_SOURCE_UNAVAILABLE' };
317
+ defaultEvidence = { ok: false, code: 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' };
318
+ }
319
+ const liveDefault = catalog.harness_default && typeof catalog.harness_default.provider === 'string' && typeof catalog.harness_default.model === 'string'
320
+ ? { provider: catalog.harness_default.provider, model: catalog.harness_default.model } : null;
321
+ const persistedDefault = settingsDefault ? { provider: settingsDefault.provider, model: settingsDefault.model } : null;
322
+ if (liveDefault || persistedDefault) {
323
+ if (!liveDefault || !persistedDefault) defaultEvidence = { ok: false, code: 'PROVIDER_DEFAULT_AUTHORITY_MISMATCH' };
324
+ else if (liveDefault.provider !== persistedDefault.provider || liveDefault.model !== persistedDefault.model) defaultEvidence = { ok: false, code: 'PROVIDER_DEFAULT_AUTHORITY_MISMATCH' };
325
+ }
326
+ // The persisted agent-default-model is the mutation authority. Keep it in
327
+ // the inventory even when the live selection is stale; the evidence gate
328
+ // above prevents destructive operations until both sources agree.
329
+ if (persistedDefault) {
330
+ catalog = {
331
+ ...catalog,
332
+ harness_default: persistedDefault,
333
+ harness_default_authority: { kind: 'harness-settings', locator: settingsDefault.locator },
334
+ };
335
+ }
336
+ const lifecycleEvidence = readProviderLifecycleStateStatus();
337
+ const lifecycle = lifecycleEvidence.state;
338
+ const recoveryTransactions = [
339
+ ...readProviderRecoveryTransactions(),
340
+ ...readProviderLayerMigrationTransactions(join(CONFIG_DIR, 'provider-migration-backups')),
341
+ ];
165
342
  const credentialPurge = readCredentialPurgeState();
166
343
  const inventory = buildProviderInventory({
167
344
  catalog,
168
345
  declarations,
169
346
  policy: providerInventoryPolicy(config),
170
347
  tombstones: lifecycle.tombstones,
348
+ additionalProviderIds: [
349
+ ...Object.keys(lifecycle.tombstones ?? {}),
350
+ ...recoveryTransactions.map((entry) => entry.provider_id).filter((id) => typeof id === 'string' && id.trim()),
351
+ ],
171
352
  activeJobs: hub.list(),
172
353
  });
354
+ // Keep each authoritative source's credential reference with its own
355
+ // declaration. Reconstructing from the merged inventory would silently
356
+ // attach the first key to every layer when profile/settings differ.
357
+ const migrationDeclarations = buildProviderMigrationDeclarations(profileDeclarations, settingsDeclarations);
358
+ const routingReferences = [
359
+ ...(config.flash_model_priority ?? []), ...(config.pro_model_priority ?? []),
360
+ ...(config.worker?.model_policy?.priority ?? []), ...(config.worker?.model_policy?.escalation_priority ?? []),
361
+ ...(config.review?.model_policy?.priority ?? []), ...(config.review?.model_policy?.escalation_priority ?? []),
362
+ ];
363
+ const migration = buildProviderLayerMigrationPlan({
364
+ declarations: migrationDeclarations,
365
+ catalogProviders: catalog.providers,
366
+ harnessDefault: catalog.harness_default,
367
+ routingReferences,
368
+ tombstones: lifecycle.tombstones,
369
+ recoveryTransactions,
370
+ catalogEvidence,
371
+ declarationEvidence,
372
+ lifecycleEvidence,
373
+ defaultEvidence,
374
+ });
173
375
  return {
174
376
  ...inventory,
377
+ migration,
378
+ catalog_evidence: catalogEvidence,
379
+ declaration_evidence: declarationEvidence,
380
+ default_evidence: defaultEvidence,
381
+ lifecycle_evidence: lifecycleEvidence.ok ? { ok: true } : { ok: false, code: lifecycleEvidence.code },
175
382
  credential_history_refs: Object.values(lifecycle.transactions ?? {})
176
383
  .flatMap((transaction) => Array.isArray(transaction?.credential_refs) ? transaction.credential_refs : []),
177
384
  credential_purged_refs: Object.keys(credentialPurge.purged),
@@ -184,23 +391,27 @@ async function readProviderInventorySnapshot(hub, ctx, config) {
184
391
  transaction_id: transactionId,
185
392
  provider_id: transaction.provider_id,
186
393
  state: transaction.state,
394
+ ...(transaction.updated_at ? { updated_at: transaction.updated_at } : {}),
187
395
  ...(transaction.expected_revision ? { expected_revision: transaction.expected_revision } : {}),
188
396
  })),
397
+ recovery_transactions: recoveryTransactions,
189
398
  };
190
399
  }
191
400
 
192
- function readProviderProfileRevision() {
401
+ function readProviderSourceRevisions() {
193
402
  const profileFile = join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml');
194
- try {
195
- if (!existsSync(profileFile)) return { ok: false, code: 'PROVIDER_PROFILE_SCHEMA_UNSUPPORTED' };
196
- const parsed = inspectProviderProfile(readFileSync(profileFile, 'utf8'));
197
- return parsed.ok ? { ok: true, revision: parsed.revision } : { ok: false, code: parsed.code };
198
- } catch {
199
- return { ok: false, code: 'PROVIDER_PROFILE_SCHEMA_UNSUPPORTED' };
200
- }
403
+ const settingsFile = join(CONFIG_DIR, 'harness', 'settings.yaml');
404
+ const read = (file, inspect) => {
405
+ try {
406
+ if (!existsSync(file)) return { ok: false, code: 'PROVIDER_SOURCE_MISSING', revision: null };
407
+ const parsed = inspect(readFileSync(file, 'utf8'));
408
+ return parsed.ok ? { ok: true, revision: parsed.revision } : { ok: false, code: parsed.code, revision: parsed.revision ?? null };
409
+ } catch { return { ok: false, code: 'PROVIDER_SOURCE_UNAVAILABLE', revision: null }; }
410
+ };
411
+ return { profile: read(profileFile, inspectProviderProfile), settings: read(settingsFile, inspectProviderSettings) };
201
412
  }
202
-
203
- export function hubCanonicalEvents(job = {}) {
413
+
414
+ export function hubCanonicalEvents(job = {}) {
204
415
  if (!job.id) return [];
205
416
  const role = job.role === 'reviewer' ? 'reviewer' : 'worker';
206
417
  const atStart = job.startedAt ?? null;
@@ -312,6 +523,9 @@ export function selectProviderProbeModel({ providerId, record, config } = {}) {
312
523
  export class WorkerRegistry { constructor(ctx) {
313
524
  this.ctx = ctx;
314
525
  this.jobs = new Map();
526
+ this.providerMutations = new Set();
527
+ this.providerMutationEpochs = new Map();
528
+ this.providerLeases = new Map();
315
529
  this.nextId = 1;
316
530
  this.shard = createShardWriter('hub');
317
531
  this.healthStore = createProviderHealthStore();
@@ -357,9 +571,51 @@ export class WorkerRegistry { constructor(ctx) {
357
571
  return v;
358
572
  }
359
573
 
360
- list() {
361
- return [...this.jobs.values()].map((job) => this.view(job));
362
- }
574
+ list() {
575
+ return [...this.jobs.values()].map((job) => this.view(job));
576
+ }
577
+
578
+ beginProviderMutation(providerId) {
579
+ if (typeof providerId !== 'string' || !providerId.trim() || this.providerMutations.has(providerId)) return false;
580
+ this.providerMutationEpochs.set(providerId, (this.providerMutationEpochs.get(providerId) ?? 0) + 1);
581
+ this.providerMutations.add(providerId);
582
+ return true;
583
+ }
584
+
585
+ endProviderMutation(providerId) {
586
+ this.providerMutations.delete(providerId);
587
+ }
588
+
589
+ getProviderMutationEpoch(providerId) {
590
+ return this.providerMutationEpochs.get(providerId) ?? 0;
591
+ }
592
+
593
+ assertProviderNotMutating(providerId, expectedEpoch = undefined) {
594
+ if (this.providerMutations.has(providerId)
595
+ || (expectedEpoch !== undefined && this.getProviderMutationEpoch(providerId) !== expectedEpoch)) {
596
+ throw Object.assign(new Error('provider mutation is active'), { code: 'PROVIDER_DELETE_BUSY' });
597
+ }
598
+ }
599
+
600
+ acquireProviderLease(providerId) {
601
+ this.providerLeases.set(providerId, (this.providerLeases.get(providerId) ?? 0) + 1);
602
+ }
603
+
604
+ releaseProviderLease(providerId) {
605
+ const count = this.providerLeases.get(providerId) ?? 0;
606
+ if (count <= 1) this.providerLeases.delete(providerId);
607
+ else this.providerLeases.set(providerId, count - 1);
608
+ }
609
+
610
+ hasProviderLease(providerId) {
611
+ return (this.providerLeases.get(providerId) ?? 0) > 0;
612
+ }
613
+
614
+ // Profile/settings are shared stores. A migration or delete of provider A
615
+ // must fence a concurrent lifecycle write for provider B in this Hub.
616
+ beginProviderStoreMutation() { return this.beginProviderMutation('__dsh-provider-store__'); }
617
+ endProviderStoreMutation() { this.endProviderMutation('__dsh-provider-store__'); }
618
+ hasProviderStoreMutation() { return this.providerMutations.has('__dsh-provider-store__'); }
363
619
 
364
620
  publish() {
365
621
  this.shard.publish([...this.jobs.values()].map((j) => this.view(j)));
@@ -466,9 +722,20 @@ export class WorkerRegistry { constructor(ctx) {
466
722
  candidateSet: attempt > 0 ? 'escalation' : 'primary',
467
723
  },
468
724
  });
469
- }
470
- if (!selection.ok) throw Object.assign(new Error(selection.message), { policyCode: selection.code });
471
- }
725
+ }
726
+ if (!selection.ok) throw Object.assign(new Error(selection.message), { policyCode: selection.code });
727
+ }
728
+ const pendingLayerMigration = readProviderLayerMigrationTransactions(join(CONFIG_DIR, 'provider-migration-backups'))
729
+ .find((entry) => entry.unresolved === true || entry.provider_id === selection.provider);
730
+ if (pendingLayerMigration) throw Object.assign(new Error('provider layer migration is pending verification'), { code: 'PROVIDER_MIGRATION_PENDING' });
731
+ // A process restart clears the in-memory mutation fence, so persisted
732
+ // provider-delete transactions must also gate admission. Unknown-owner
733
+ // recovery entries fence globally; known entries fence their provider.
734
+ const pendingProviderDelete = readProviderRecoveryTransactions()
735
+ .find((entry) => entry.unresolved === true || entry.provider_id === selection.provider);
736
+ if (pendingProviderDelete) throw Object.assign(new Error('provider delete recovery is pending'), { code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
737
+ const providerMutationEpoch = this.getProviderMutationEpoch(selection.provider);
738
+ this.assertProviderNotMutating(selection.provider, providerMutationEpoch);
472
739
 
473
740
  // The worker always gets the auditable Delivery Contract appended (unless
474
741
  // it already carries one), so its final message follows ## Diff / ## Tests
@@ -527,9 +794,13 @@ export class WorkerRegistry { constructor(ctx) {
527
794
  // Abort latch: cancel()/timeout set this the moment the job becomes
528
795
  // terminal, even while agents.create() is still pending, so a handle
529
796
  // acquired after that point is disposed instead of being driven.
530
- abortRequested: false,
797
+ abortRequested: false,
531
798
  abortPromise: null,
532
799
  resolveAbort: null,
800
+ providerLease: false,
801
+ providerLeaseReleaseBlocked: false,
802
+ lateHandleCreated: false,
803
+ lateCreateCleanupPromise: null,
533
804
  };
534
805
  job.abortPromise = new Promise((resolve) => { job.resolveAbort = resolve; });
535
806
  const disposeJobHandle = async () => {
@@ -541,14 +812,30 @@ export class WorkerRegistry { constructor(ctx) {
541
812
  job.handle_dispose_promise = Promise.reject(error);
542
813
  }
543
814
  }
544
- await job.handle_dispose_promise;
815
+ try {
816
+ await job.handle_dispose_promise;
817
+ } catch (error) {
818
+ // A failed disposer is not proof that the provider is idle. Keep the
819
+ // lease fenced even when the job is already terminal; the operator
820
+ // must restart/reclaim the owned runtime before deletion can proceed.
821
+ job.providerLeaseReleaseBlocked = true;
822
+ throw error;
823
+ }
545
824
  };
546
825
  job.disposeHandle = disposeJobHandle;
547
826
  // Read-only pre-run snapshot (async, never blocks dispatch): the audit
548
827
  // only needs the before-state by the time the worker finishes. Non-repos
549
828
  // degrade to { kind:'no-git' } instead of failing the job.
550
- job.baseline = await captureWorkspaceBaseline({ cwd: executionCwd }).catch(() => ({ kind: 'no-git', reason: NOT_A_GIT_REPOSITORY, error: 'workspace audit failed' }));
551
- this.jobs.set(id, job);
829
+ job.baseline = await captureWorkspaceBaseline({ cwd: executionCwd }).catch(() => ({ kind: 'no-git', reason: NOT_A_GIT_REPOSITORY, error: 'workspace audit failed' }));
830
+ try {
831
+ this.assertProviderNotMutating(selection.provider, providerMutationEpoch);
832
+ this.acquireProviderLease(selection.provider);
833
+ job.providerLease = true;
834
+ } catch (error) {
835
+ if (isolatedWorkspace) await cleanupIsolatedWorkspace({ repoRoot: isolatedWorkspace.repoRoot, worktreePath: isolatedWorkspace.worktreePath }).catch(() => {});
836
+ throw error;
837
+ }
838
+ this.jobs.set(id, job);
552
839
 
553
840
  const onEvent = (session, event) => {
554
841
  switch (event.type) {
@@ -598,14 +885,15 @@ export class WorkerRegistry { constructor(ctx) {
598
885
  job.abortPromise.then(() => ({ aborted: true })),
599
886
  ]);
600
887
  if (created.aborted) {
601
- createPromise.then(async (handle) => {
602
- if (handle && (job.status !== 'running' || job.abortRequested)) {
603
- job.handle = handle;
604
- try { await job.disposeHandle(); } catch (error) {
605
- job.cleanup_warning = `agent cleanup failed: ${error?.message ?? String(error)}`;
606
- }
607
- }
608
- }).catch(() => {});
888
+ job.lateCreateCleanupPromise = createPromise.then(async (handle) => {
889
+ if (!handle) return;
890
+ job.lateHandleCreated = true;
891
+ job.handle = handle;
892
+ if (job.status !== 'running' || job.abortRequested) await job.disposeHandle();
893
+ }).catch((error) => {
894
+ job.cleanup_warning = `late agent creation failed: ${error?.message ?? String(error)}`;
895
+ throw error;
896
+ });
609
897
  return;
610
898
  }
611
899
  const handle = created.handle;
@@ -674,9 +962,23 @@ export class WorkerRegistry { constructor(ctx) {
674
962
  job.endedAt = new Date().toISOString();
675
963
  job.currentTool = null;
676
964
  let handleCleanupWarning = job.cleanup_warning ?? null;
677
- try { await disposeJobHandle(); } catch (error) {
678
- handleCleanupWarning = `agent cleanup failed: ${error?.message ?? String(error)}`;
679
- }
965
+ try { await disposeJobHandle(); } catch (error) {
966
+ handleCleanupWarning = `agent cleanup failed: ${error?.message ?? String(error)}`;
967
+ }
968
+ if (job.providerLease) {
969
+ const releaseLease = () => {
970
+ if (!job.providerLease) return;
971
+ this.releaseProviderLease(job.provider);
972
+ job.providerLease = false;
973
+ };
974
+ if (job.lateCreateCleanupPromise) {
975
+ job.lateCreateCleanupPromise.then(releaseLease).catch(() => {
976
+ if (!job.lateHandleCreated && !job.providerLeaseReleaseBlocked) releaseLease();
977
+ // A late-created handle that failed disposal remains fenced by
978
+ // the provider lease until an operator restarts/reclaims it.
979
+ });
980
+ } else if (!job.providerLeaseReleaseBlocked) releaseLease();
981
+ }
680
982
  // Delivery completeness is separate from execution status: a job can
681
983
  // be done yet fail to report Diff/Tests/Risks (or Review sections for
682
984
  // an automatic review). Parse whatever final message the worker
@@ -950,11 +1252,18 @@ export async function apply(ctx) {
950
1252
  const disposers = [];
951
1253
  const PROVIDER_DELETE_PLAN_TTL_MS = 10 * 60 * 1000;
952
1254
  const providerDeletePlans = new Map();
1255
+ const providerMigrationPlans = new Map();
1256
+ const providerMigrationOwners = new Map();
1257
+ const providerStoreMutationOwners = new Map();
953
1258
  const credentialPurgePlans = new Map();
954
1259
  const rememberProviderDeletePlan = (plan) => {
955
1260
  providerDeletePlans.set(plan.plan_id, { plan, created_at: Date.now() });
956
1261
  while (providerDeletePlans.size > 32) providerDeletePlans.delete(providerDeletePlans.keys().next().value);
957
1262
  };
1263
+ const rememberProviderMigrationPlan = (plan) => {
1264
+ providerMigrationPlans.set(plan.plan_id, { plan, created_at: Date.now() });
1265
+ while (providerMigrationPlans.size > 32) providerMigrationPlans.delete(providerMigrationPlans.keys().next().value);
1266
+ };
958
1267
  const rememberCredentialPurgePlan = (plan) => {
959
1268
  credentialPurgePlans.set(plan.plan_id, { plan, created_at: Date.now() });
960
1269
  while (credentialPurgePlans.size > 32) credentialPurgePlans.delete(credentialPurgePlans.keys().next().value);
@@ -1361,23 +1670,386 @@ export async function apply(ctx) {
1361
1670
  try { url = new URL(req.url, 'http://localhost'); } catch { return sendJson(res, 400, { ok: false, code: 'PROVIDER_PLAN_INVALID' }); }
1362
1671
  const parts = url.pathname.slice(`${ROUTE_BASE}/providers`.length).split('/').filter(Boolean);
1363
1672
  try {
1673
+ if (req.method === 'GET' && parts.length === 1 && parts[0] === 'migration-status') {
1674
+ const config = normalizeGlobalConfig(hub.getConfig?.() ?? {});
1675
+ const inventory = await readProviderInventorySnapshot(hub, ctx, config);
1676
+ return sendJson(res, 200, { ok: true, ...inventory.migration, runtime: getHubRuntimeIdentity() });
1677
+ }
1364
1678
  const providerId = decodeURIComponent(parts[0]);
1365
1679
  const body = await readBody(req);
1680
+ if (req.method === 'POST' && parts[0] === '_recovery' && parts[1] === 'lock' && parts.length === 2) {
1681
+ if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_CONFIRM_REQUIRED' });
1682
+ const hooks = createProviderDeleteFileHooks({
1683
+ profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
1684
+ settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
1685
+ configFile: join(CONFIG_DIR, 'config.json'),
1686
+ lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
1687
+ backupDir: join(CONFIG_DIR, 'provider-backups'),
1688
+ sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
1689
+ });
1690
+ const recovered = await hooks.recoverLock({ force: body?.force === true });
1691
+ const migrationHooks = createProviderLayerMigrationFileHooks({
1692
+ profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
1693
+ settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
1694
+ backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
1695
+ });
1696
+ const migrationRecovered = await migrationHooks.recoverLock({ confirm: true });
1697
+ const ok = recovered.ok === true && migrationRecovered.ok === true;
1698
+ return sendJson(res, ok ? 200 : 409, { ok, delete_lock: recovered, migration_lock: migrationRecovered });
1699
+ }
1700
+ if (req.method === 'POST' && parts[0] === '_recovery' && parts[1] === 'quarantine' && parts.length === 2) {
1701
+ if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_QUARANTINE_CONFIRM_REQUIRED' });
1702
+ const actionId = typeof body?.action_id === 'string' && RECOVERY_ACTION_PATTERN.test(body.action_id)
1703
+ ? body.action_id : null;
1704
+ if (!actionId) return sendJson(res, 400, { ok: false, code: 'PROVIDER_DELETE_PLAN_INVALID' });
1705
+ const recovery = [
1706
+ ...readProviderRecoveryTransactions(),
1707
+ ...readProviderLayerMigrationTransactions(join(CONFIG_DIR, 'provider-migration-backups')),
1708
+ ].find((entry) => entry.action_id === actionId);
1709
+ if (!recovery) return sendJson(res, 404, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_NOT_FOUND' });
1710
+ if (recovery.recoverable === true) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_RECOVERABLE' });
1711
+ if (typeof recovery.storage_id !== 'string') return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_QUARANTINE_FAILED' });
1712
+ if (recovery.source === 'provider-layer-migration') {
1713
+ try {
1714
+ const migrationHooks = createProviderLayerMigrationFileHooks({
1715
+ profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
1716
+ settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
1717
+ backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
1718
+ });
1719
+ const quarantined = await migrationHooks.quarantine(actionId);
1720
+ return sendJson(res, 200, { ...quarantined, transaction_id: recovery.transaction_id ?? null });
1721
+ } catch (error) {
1722
+ const code = boundedMachineCodeFromError(error) ?? 'PROVIDER_MIGRATION_RECOVERY_QUARANTINE_FAILED';
1723
+ return sendJson(res, 409, { ok: false, code });
1724
+ }
1725
+ }
1726
+ try {
1727
+ const hooks = createProviderDeleteFileHooks({
1728
+ profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
1729
+ settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
1730
+ configFile: join(CONFIG_DIR, 'config.json'),
1731
+ lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
1732
+ backupDir: join(CONFIG_DIR, 'provider-backups'),
1733
+ sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
1734
+ afterLockAcquired: () => {
1735
+ const current = readProviderRecoveryTransactions().find((entry) => entry.action_id === actionId);
1736
+ if (!current || current.recoverable === true || current.storage_id !== recovery.storage_id) {
1737
+ throw Object.assign(new Error('provider recovery entry changed before quarantine'), { code: 'PROVIDER_DELETE_STATE_CHANGED' });
1738
+ }
1739
+ },
1740
+ });
1741
+ const quarantined = await hooks.quarantine(recovery.storage_id);
1742
+ return sendJson(res, 200, { ...quarantined, action_id: actionId, transaction_id: recovery.transaction_id ?? null });
1743
+ } catch (error) {
1744
+ const code = boundedMachineCodeFromError(error) ?? 'PROVIDER_DELETE_RECOVERY_QUARANTINE_FAILED';
1745
+ return sendJson(res, 409, { ok: false, code });
1746
+ }
1747
+ }
1366
1748
  const config = normalizeGlobalConfig(hub.getConfig?.() ?? {});
1367
1749
  const inventory = await readProviderInventorySnapshot(hub, ctx, config);
1750
+ if (req.method === 'POST' && parts.length === 2 && parts[1] === 'migrate-plan') {
1751
+ if (!hasCompleteProviderCatalogEvidence(inventory.catalog_evidence)) return sendJson(res, 503, { ok: false, code: 'MODEL_CATALOG_UNAVAILABLE' });
1752
+ if (!hasCompleteProviderDeclarationEvidence(inventory.declaration_evidence)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_SOURCE_UNRESOLVED' });
1753
+ if (!hasAvailableProviderLifecycleEvidence(inventory.lifecycle_evidence)) return sendJson(res, 503, { ok: false, code: 'PROVIDER_LIFECYCLE_UNAVAILABLE' });
1754
+ if (inventory.default_evidence?.ok !== true) return sendJson(res, 409, { ok: false, code: inventory.default_evidence?.code ?? 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' });
1755
+ if (hasPendingProviderRecoveryTransactions(inventory)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
1756
+ const entry = inventory.migration.providers.find((candidate) => candidate.provider_id === providerId);
1757
+ if (!entry) return sendJson(res, 404, { ok: false, code: 'PROVIDER_MIGRATION_NOT_REQUIRED' });
1758
+ if (entry.action === 'blocked' || entry.action === 'collision-review') return sendJson(res, 409, { ok: false, code: entry.blocked_reason ?? 'PROVIDER_MIGRATION_BLOCKED' });
1759
+ const revisions = readProviderSourceRevisions();
1760
+ if (!revisions.profile.ok) return sendJson(res, 409, { ok: false, code: revisions.profile.code ?? 'PROVIDER_PROFILE_CHANGED' });
1761
+ if ((entry.action === 'promote-existing-user' || entry.harness_default) && !revisions.settings.ok) return sendJson(res, 409, { ok: false, code: revisions.settings.code ?? 'PROVIDER_SETTINGS_CHANGED' });
1762
+ const profileFile = join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml');
1763
+ const parsedMaterialization = readProviderMaterialization(readFileSync(profileFile, 'utf8'), { providerId, file: 'harness/profiles/dsh-crew/cordis.patch.yml' });
1764
+ if (!parsedMaterialization.ok) return sendJson(res, 409, { ok: false, code: parsedMaterialization.code });
1765
+ const materialization = parsedMaterialization.provider;
1766
+ let userMaterializationBefore;
1767
+ if (entry.action === 'promote-existing-user') {
1768
+ const settingsFile = join(CONFIG_DIR, 'harness', 'settings.yaml');
1769
+ if (!existsSync(settingsFile)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_SETTINGS_CHANGED' });
1770
+ const parsedUserMaterialization = readProviderSettingsMaterialization(readFileSync(settingsFile, 'utf8'), { providerId, file: 'harness/settings.yaml' });
1771
+ if (!parsedUserMaterialization.ok) return sendJson(res, 409, { ok: false, code: parsedUserMaterialization.code });
1772
+ userMaterializationBefore = parsedUserMaterialization.provider;
1773
+ }
1774
+ const plan = {
1775
+ schema_version: 1,
1776
+ kind: 'provider-layer-migration',
1777
+ plan_id: randomUUID(),
1778
+ provider_id: providerId,
1779
+ action: entry.action,
1780
+ source: entry.source,
1781
+ routing_references: entry.referenced_by,
1782
+ materialization: { provider: materialization },
1783
+ ...(userMaterializationBefore ? { user_materialization_before: userMaterializationBefore } : {}),
1784
+ expected_revisions: {
1785
+ profile: revisions.profile.revision,
1786
+ settings: revisions.settings.ok ? revisions.settings.revision : null,
1787
+ },
1788
+ harness_default_before: inventory.harness_default ?? null,
1789
+ runtime_id_before: getHubRuntimeIdentity().runtime_id,
1790
+ restart_required: true,
1791
+ };
1792
+ rememberProviderMigrationPlan(plan);
1793
+ return sendJson(res, 200, { ok: true, plan });
1794
+ }
1795
+ if (req.method === 'POST' && parts.length === 2 && parts[1] === 'migrate') {
1796
+ if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_MIGRATION_CONFIRM_REQUIRED' });
1797
+ const stored = providerMigrationPlans.get(body?.plan_id);
1798
+ if (!stored || Date.now() - stored.created_at > PROVIDER_DELETE_PLAN_TTL_MS || stored.plan.provider_id !== providerId) {
1799
+ providerMigrationPlans.delete(body?.plan_id);
1800
+ return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_PLAN_EXPIRED' });
1801
+ }
1802
+ const entry = inventory.migration.providers.find((candidate) => candidate.provider_id === providerId);
1803
+ if (!entry || entry.action !== stored.plan.action || entry.action === 'blocked' || entry.action === 'collision-review') return sendJson(res, 409, { ok: false, code: entry?.blocked_reason ?? 'PROVIDER_MIGRATION_PLAN_CHANGED' });
1804
+ if (inventory.default_evidence?.ok !== true) return sendJson(res, 409, { ok: false, code: inventory.default_evidence?.code ?? 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' });
1805
+ const revisions = readProviderSourceRevisions();
1806
+ const expected = stored.plan.expected_revisions ?? {};
1807
+ if (revisions.profile.revision !== expected.profile || (revisions.settings.ok ? revisions.settings.revision : null) !== expected.settings) {
1808
+ return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_SOURCE_CHANGED' });
1809
+ }
1810
+ const hooks = createProviderLayerMigrationFileHooks({
1811
+ profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
1812
+ settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
1813
+ backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
1814
+ });
1815
+ let result;
1816
+ if (hub.hasProviderLease(providerId)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_IN_USE' });
1817
+ if (!hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
1818
+ if (!hub.beginProviderMutation(providerId)) {
1819
+ hub.endProviderStoreMutation();
1820
+ return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
1821
+ }
1822
+ let retainProviderMutation = false;
1823
+ let retainProviderStoreMutation = false;
1824
+ try {
1825
+ await hooks.acquireLock();
1826
+ // The plan was created from a pre-lock snapshot. Re-read every
1827
+ // authoritative source after taking the shared store lock so a
1828
+ // concurrent catalog/default/recovery change cannot be hidden
1829
+ // behind unchanged profile/settings revisions.
1830
+ const lockedConfig = normalizeGlobalConfig(hub.getConfig?.() ?? {});
1831
+ const lockedInventory = await readProviderInventorySnapshot(hub, ctx, lockedConfig);
1832
+ const lockedEntry = lockedInventory.migration.providers.find((candidate) => candidate.provider_id === providerId);
1833
+ const expectedRouting = Array.isArray(stored.plan.routing_references) ? stored.plan.routing_references : [];
1834
+ const sameRouting = JSON.stringify(lockedEntry?.referenced_by ?? []) === JSON.stringify(expectedRouting);
1835
+ const sameDefault = JSON.stringify(lockedInventory.harness_default ?? null) === JSON.stringify(stored.plan.harness_default_before ?? null);
1836
+ const profileFile = join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml');
1837
+ const lockedMaterialization = existsSync(profileFile)
1838
+ ? readProviderMaterialization(readFileSync(profileFile, 'utf8'), { providerId, file: 'harness/profiles/dsh-crew/cordis.patch.yml' })
1839
+ : { ok: false };
1840
+ const sameMaterialization = lockedMaterialization.ok === true
1841
+ && JSON.stringify(lockedMaterialization.provider) === JSON.stringify(stored.plan.materialization?.provider ?? null);
1842
+ const settingsFile = join(CONFIG_DIR, 'harness', 'settings.yaml');
1843
+ const lockedUserMaterialization = stored.plan.action === 'promote-existing-user' && existsSync(settingsFile)
1844
+ ? readProviderSettingsMaterialization(readFileSync(settingsFile, 'utf8'), { providerId, file: 'harness/settings.yaml' })
1845
+ : { ok: stored.plan.action !== 'promote-existing-user' };
1846
+ const sameUserMaterialization = stored.plan.action !== 'promote-existing-user'
1847
+ || (lockedUserMaterialization.ok === true && JSON.stringify(lockedUserMaterialization.provider) === JSON.stringify(stored.plan.user_materialization_before ?? null));
1848
+ const lockedRevisions = readProviderSourceRevisions();
1849
+ const expectedLocked = stored.plan.expected_revisions ?? {};
1850
+ const sameRevisions = lockedRevisions.profile.revision === expectedLocked.profile
1851
+ && (lockedRevisions.settings.ok ? lockedRevisions.settings.revision : null) === expectedLocked.settings;
1852
+ if (hasPendingProviderRecoveryTransactions(lockedInventory)) {
1853
+ return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
1854
+ }
1855
+ if (!lockedEntry || lockedEntry.action !== stored.plan.action || lockedEntry.collision !== null
1856
+ || !sameRouting || !sameDefault || !sameMaterialization || !sameUserMaterialization || !sameRevisions
1857
+ || lockedInventory.default_evidence?.ok !== true
1858
+ || !hasCompleteProviderCatalogEvidence(lockedInventory.catalog_evidence)
1859
+ || !hasCompleteProviderDeclarationEvidence(lockedInventory.declaration_evidence)
1860
+ || !hasAvailableProviderLifecycleEvidence(lockedInventory.lifecycle_evidence)) {
1861
+ return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_PLAN_CHANGED' });
1862
+ }
1863
+ if (hub.hasProviderLease(providerId)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_IN_USE' });
1864
+ result = await executeProviderLayerMigration(stored.plan, hooks, { confirm: true, deferRestart: true });
1865
+ if (result.ok && result.state === 'RESTART_PENDING') await hooks.setRestartRuntimeBaseline(getHubRuntimeIdentity().runtime_id);
1866
+ providerMigrationPlans.delete(body.plan_id);
1867
+ retainProviderMutation = result.state === 'RESTART_PENDING';
1868
+ if (retainProviderMutation) {
1869
+ providerMigrationOwners.set(providerId, stored.plan.plan_id);
1870
+ providerStoreMutationOwners.set(stored.plan.plan_id, providerId);
1871
+ retainProviderStoreMutation = true;
1872
+ }
1873
+ return sendJson(res, result.ok ? 202 : 409, { ok: result.ok, restart_required: result.restart_required === true, result });
1874
+ } finally {
1875
+ await hooks.release();
1876
+ if (!retainProviderMutation) hub.endProviderMutation(providerId);
1877
+ if (!retainProviderStoreMutation) hub.endProviderStoreMutation();
1878
+ }
1879
+ }
1880
+ if (req.method === 'POST' && parts.length === 2 && parts[1] === 'verify-migration') {
1881
+ if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_MIGRATION_VERIFY_CONFIRM_REQUIRED' });
1882
+ const transactionId = body?.transaction_id;
1883
+ const hooks = createProviderLayerMigrationFileHooks({
1884
+ profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
1885
+ settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
1886
+ backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
1887
+ existingMigrationId: transactionId,
1888
+ });
1889
+ let acquiredStoreFence = false;
1890
+ try {
1891
+ await hooks.acquireLock();
1892
+ await hooks.backup({});
1893
+ const plan = hooks.backupPlan();
1894
+ if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_PLAN_PROVIDER_MISMATCH' });
1895
+ const owner = providerMigrationOwners.get(providerId);
1896
+ if (providerStoreMutationOwners.get(plan.plan_id) !== providerId || !hub.hasProviderStoreMutation()) {
1897
+ if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
1898
+ providerStoreMutationOwners.set(plan.plan_id, providerId);
1899
+ acquiredStoreFence = true;
1900
+ }
1901
+ if (owner !== plan.plan_id) {
1902
+ if (hub.hasProviderLease(providerId) || !hub.beginProviderMutation(providerId)) {
1903
+ if (acquiredStoreFence && providerStoreMutationOwners.get(plan.plan_id) === providerId) { providerStoreMutationOwners.delete(plan.plan_id); hub.endProviderStoreMutation(); }
1904
+ return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
1905
+ }
1906
+ providerMigrationOwners.set(providerId, plan.plan_id);
1907
+ }
1908
+ const verification = await hooks.verify(plan);
1909
+ const live = await readProviderInventorySnapshot(hub, ctx, config);
1910
+ verification.catalogEvidence = live.catalog_evidence;
1911
+ verification.defaultEvidence = live.default_evidence;
1912
+ verification.defaultPreserved = live.default_evidence?.ok === true
1913
+ && JSON.stringify(live.harness_default ?? null) === JSON.stringify(plan.harness_default_before ?? null);
1914
+ verification.liveNativeRemovable = hasCompleteProviderCatalogEvidence(live.catalog_evidence)
1915
+ && live.records.some((record) => record.id === providerId && record.lifecycle?.catalogued === true && record.native_removable === true);
1916
+ verification.runtimeRestarted = hasProviderRuntimeRestartEvidence(plan.runtime_id_before_restart ?? plan.runtime_id_before, getHubRuntimeIdentity().runtime_id);
1917
+ if (verification.nativeRemovable !== true || verification.liveNativeRemovable !== true || verification.defaultPreserved !== true || verification.runtimeRestarted !== true) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_VERIFY_FAILED', verification });
1918
+ await hooks.finalizeVerified();
1919
+ if (providerMigrationOwners.get(providerId) === plan.plan_id) {
1920
+ providerMigrationOwners.delete(providerId);
1921
+ hub.endProviderMutation(providerId);
1922
+ }
1923
+ if (providerStoreMutationOwners.get(plan.plan_id) === providerId) {
1924
+ providerStoreMutationOwners.delete(plan.plan_id);
1925
+ hub.endProviderStoreMutation();
1926
+ }
1927
+ return sendJson(res, 200, { ok: true, state: 'VERIFIED', transaction_id: plan.plan_id, provider_id: providerId, verification });
1928
+ } finally {
1929
+ await hooks.release();
1930
+ // Keep the transaction-owned fence on verification failure so
1931
+ // an incomplete migration cannot race a new Worker; the caller
1932
+ // must explicitly retry verification or roll it back.
1933
+ }
1934
+ }
1935
+ if (req.method === 'POST' && parts.length === 2 && parts[1] === 'rollback-migration') {
1936
+ if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_MIGRATION_ROLLBACK_CONFIRM_REQUIRED' });
1937
+ const hooks = createProviderLayerMigrationFileHooks({
1938
+ profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
1939
+ settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
1940
+ backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
1941
+ existingMigrationId: body?.transaction_id,
1942
+ });
1943
+ let acquiredMigrationFence = false;
1944
+ let retainProviderMutation = false;
1945
+ let acquiredStoreFence = false;
1946
+ try {
1947
+ await hooks.acquireLock();
1948
+ await hooks.backup({});
1949
+ const plan = hooks.backupPlan();
1950
+ if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_PLAN_PROVIDER_MISMATCH' });
1951
+ const owner = providerMigrationOwners.get(providerId);
1952
+ if (providerStoreMutationOwners.get(plan.plan_id) !== providerId || !hub.hasProviderStoreMutation()) {
1953
+ if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
1954
+ providerStoreMutationOwners.set(plan.plan_id, providerId);
1955
+ acquiredStoreFence = true;
1956
+ }
1957
+ if (owner !== plan.plan_id) {
1958
+ if (hub.hasProviderLease(providerId) || !hub.beginProviderMutation(providerId)) {
1959
+ if (acquiredStoreFence && providerStoreMutationOwners.get(plan.plan_id) === providerId) { providerStoreMutationOwners.delete(plan.plan_id); hub.endProviderStoreMutation(); }
1960
+ return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
1961
+ }
1962
+ providerMigrationOwners.set(providerId, plan.plan_id);
1963
+ acquiredMigrationFence = true;
1964
+ retainProviderMutation = true;
1965
+ }
1966
+ const rollback = await hooks.rollback();
1967
+ await hooks.setRollbackRuntimeBaseline(getHubRuntimeIdentity().runtime_id);
1968
+ providerMigrationOwners.set(providerId, plan.plan_id);
1969
+ retainProviderMutation = true;
1970
+ return sendJson(res, 202, { ok: true, restart_required: true, transaction_id: plan.plan_id, provider_id: providerId, state: 'ROLLBACK_RESTART_PENDING' });
1971
+ } finally {
1972
+ await hooks.release();
1973
+ if (acquiredMigrationFence && !retainProviderMutation && providerMigrationOwners.get(providerId) === body?.transaction_id) {
1974
+ providerMigrationOwners.delete(providerId);
1975
+ hub.endProviderMutation(providerId);
1976
+ }
1977
+ }
1978
+ }
1979
+ if (req.method === 'POST' && parts.length === 2 && parts[1] === 'verify-rollback-migration') {
1980
+ if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_MIGRATION_ROLLBACK_VERIFY_CONFIRM_REQUIRED' });
1981
+ const transactionId = body?.transaction_id;
1982
+ const hooks = createProviderLayerMigrationFileHooks({
1983
+ profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
1984
+ settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
1985
+ backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
1986
+ existingMigrationId: transactionId,
1987
+ });
1988
+ let acquiredStoreFence = false;
1989
+ try {
1990
+ await hooks.acquireLock();
1991
+ await hooks.backup({});
1992
+ const plan = hooks.backupPlan();
1993
+ if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_PLAN_PROVIDER_MISMATCH' });
1994
+ const owner = providerMigrationOwners.get(providerId);
1995
+ if (providerStoreMutationOwners.get(plan.plan_id) !== providerId || !hub.hasProviderStoreMutation()) {
1996
+ if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
1997
+ providerStoreMutationOwners.set(plan.plan_id, providerId);
1998
+ acquiredStoreFence = true;
1999
+ }
2000
+ if (owner !== plan.plan_id) {
2001
+ if (hub.hasProviderLease(providerId) || !hub.beginProviderMutation(providerId)) {
2002
+ if (acquiredStoreFence && providerStoreMutationOwners.get(plan.plan_id) === providerId) { providerStoreMutationOwners.delete(plan.plan_id); hub.endProviderStoreMutation(); }
2003
+ return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
2004
+ }
2005
+ providerMigrationOwners.set(providerId, plan.plan_id);
2006
+ }
2007
+ const verification = await hooks.verifyRollback(plan);
2008
+ const live = await readProviderInventorySnapshot(hub, ctx, config);
2009
+ verification.catalogEvidence = live.catalog_evidence;
2010
+ verification.defaultEvidence = live.default_evidence;
2011
+ verification.defaultPreserved = live.default_evidence?.ok === true
2012
+ && JSON.stringify(live.harness_default ?? null) === JSON.stringify(plan.harness_default_before ?? null);
2013
+ verification.liveRestored = hasCompleteProviderCatalogEvidence(live.catalog_evidence)
2014
+ && live.records.some((record) => record.id === providerId && record.lifecycle?.catalogued === true && record.native_removable === false);
2015
+ verification.runtimeRestarted = hasProviderRuntimeRestartEvidence(plan.rollback_runtime_id_before, getHubRuntimeIdentity().runtime_id);
2016
+ if (verification.ok !== true || verification.liveRestored !== true || verification.defaultPreserved !== true || verification.runtimeRestarted !== true) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_ROLLBACK_VERIFY_FAILED', verification });
2017
+ await hooks.finalizeRollback();
2018
+ if (providerMigrationOwners.get(providerId) === plan.plan_id) {
2019
+ providerMigrationOwners.delete(providerId);
2020
+ hub.endProviderMutation(providerId);
2021
+ }
2022
+ if (providerStoreMutationOwners.get(plan.plan_id) === providerId) {
2023
+ providerStoreMutationOwners.delete(plan.plan_id);
2024
+ hub.endProviderStoreMutation();
2025
+ }
2026
+ return sendJson(res, 200, { ok: true, state: 'ROLLED_BACK', transaction_id: plan.plan_id, provider_id: providerId, verification });
2027
+ } finally { await hooks.release(); }
2028
+ }
1368
2029
  if (req.method === 'POST' && parts.length === 2 && parts[1] === 'delete-plan') {
1369
- const profile = readProviderProfileRevision();
1370
- if (!profile.ok) return sendJson(res, 409, { ok: false, code: profile.code });
2030
+ if (providerId !== 'deepseek-official' && hasPendingProviderRecoveryTransactions(inventory)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
2031
+ if (providerId !== 'deepseek-official' && !hasCompleteProviderCatalogEvidence(inventory.catalog_evidence)) return sendJson(res, 503, { ok: false, code: 'MODEL_CATALOG_UNAVAILABLE' });
2032
+ if (providerId !== 'deepseek-official' && !hasCompleteProviderDeclarationEvidence(inventory.declaration_evidence)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_SOURCE_UNRESOLVED' });
2033
+ if (providerId !== 'deepseek-official' && !hasAvailableProviderLifecycleEvidence(inventory.lifecycle_evidence)) return sendJson(res, 503, { ok: false, code: 'PROVIDER_LIFECYCLE_UNAVAILABLE' });
2034
+ if (providerId !== 'deepseek-official' && inventory.default_evidence?.ok !== true) return sendJson(res, 409, { ok: false, code: inventory.default_evidence?.code ?? 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' });
2035
+ const revisions = readProviderSourceRevisions();
2036
+ if (!revisions.profile.ok && !revisions.settings.ok) return sendJson(res, 409, { ok: false, code: revisions.profile.code ?? revisions.settings.code });
2037
+ const expectedRevision = (revisions.profile.ok ? revisions.profile.revision : null) ?? revisions.settings.revision;
1371
2038
  const planned = planProviderDelete({
1372
2039
  providerId,
1373
2040
  inventory,
1374
2041
  activeJobs: hub.list(),
1375
2042
  replacementDefault: body?.replacement_default,
1376
- expectedRevision: profile.revision,
2043
+ expectedRevision,
2044
+ expectedRevisions: {
2045
+ profile: revisions.profile.ok ? revisions.profile.revision : null,
2046
+ settings: revisions.settings.ok ? revisions.settings.revision : null,
2047
+ },
1377
2048
  });
1378
2049
  if (!planned.ok) return sendJson(res, 409, { ok: false, code: planned.code });
1379
- rememberProviderDeletePlan(planned.plan);
1380
- return sendJson(res, 200, { ok: true, profile_revision: profile.revision, plan: planned.plan });
2050
+ const plan = { ...planned.plan, runtime_id_before: getHubRuntimeIdentity().runtime_id };
2051
+ rememberProviderDeletePlan(plan);
2052
+ return sendJson(res, 200, { ok: true, profile_revision: expectedRevision, plan });
1381
2053
  }
1382
2054
  if (req.method === 'POST' && parts.length === 2 && parts[1] === 'probe') {
1383
2055
  const record = inventory.records.find((entry) => entry.id === providerId);
@@ -1413,9 +2085,11 @@ export async function apply(ctx) {
1413
2085
  const { transaction_id: transactionId } = body ?? {};
1414
2086
  const hooks = createProviderDeleteFileHooks({
1415
2087
  profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
2088
+ settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
1416
2089
  configFile: join(CONFIG_DIR, 'config.json'),
1417
2090
  lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
1418
2091
  backupDir: join(CONFIG_DIR, 'provider-backups'),
2092
+ sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
1419
2093
  existingBackupId: transactionId,
1420
2094
  expectedProviderId: providerId,
1421
2095
  });
@@ -1423,11 +2097,30 @@ export async function apply(ctx) {
1423
2097
  if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_PLAN_PROVIDER_MISMATCH' });
1424
2098
  await hooks.acquireLock();
1425
2099
  try {
2100
+ if (providerStoreMutationOwners.get(transactionId) !== providerId || !hub.hasProviderStoreMutation()) {
2101
+ if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_BUSY' });
2102
+ providerStoreMutationOwners.set(transactionId, providerId);
2103
+ }
1426
2104
  const verification = await hooks.verify(plan);
1427
- if (verification?.providerAbsent !== true || verification?.routingClear !== true || verification?.tombstonePresent !== true) {
2105
+ const live = await readProviderInventorySnapshot(hub, ctx, config);
2106
+ verification.catalog_evidence = live.catalog_evidence;
2107
+ verification.default_evidence = live.default_evidence;
2108
+ verification.catalogAbsent = hasCompleteProviderCatalogEvidence(live.catalog_evidence)
2109
+ && !live.records.some((entry) => entry.id === providerId && entry.lifecycle?.catalogued === true);
2110
+ verification.harnessDefault = plan.was_harness_default !== true || (
2111
+ live.harness_default?.provider === plan.replacement_default
2112
+ && live.harness_default?.model === plan.replacement_default_model
2113
+ );
2114
+ verification.runtimeRestarted = hasProviderRuntimeRestartEvidence(plan.delete_runtime_id_before_restart, getHubRuntimeIdentity().runtime_id);
2115
+ if (verification?.providerAbsent !== true || !hasCompleteProviderCatalogEvidence(verification?.catalog_evidence) || verification?.default_evidence?.ok !== true || verification?.catalogAbsent !== true || verification?.harnessDefault !== true || verification?.runtimeRestarted !== true || verification?.routingClear !== true || verification?.tombstonePresent !== true) {
1428
2116
  return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_VERIFY_FAILED', verification });
1429
2117
  }
1430
2118
  await hooks.recordTransaction({ transaction_id: transactionId, provider_id: providerId, state: 'VERIFIED' }, plan);
2119
+ hub.endProviderMutation(providerId);
2120
+ if (providerStoreMutationOwners.get(transactionId) === providerId) {
2121
+ providerStoreMutationOwners.delete(transactionId);
2122
+ hub.endProviderStoreMutation();
2123
+ }
1431
2124
  return sendJson(res, 200, { ok: true, transaction_id: transactionId, provider_id: providerId, state: 'VERIFIED', verification });
1432
2125
  } finally {
1433
2126
  await hooks.release();
@@ -1438,25 +2131,41 @@ export async function apply(ctx) {
1438
2131
  const { transaction_id: transactionId } = body ?? {};
1439
2132
  const hooks = createProviderDeleteFileHooks({
1440
2133
  profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
2134
+ settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
1441
2135
  configFile: join(CONFIG_DIR, 'config.json'),
1442
2136
  lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
1443
2137
  backupDir: join(CONFIG_DIR, 'provider-backups'),
2138
+ sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
1444
2139
  existingBackupId: transactionId,
1445
2140
  expectedProviderId: providerId,
1446
2141
  });
1447
2142
  await hooks.acquireLock();
2143
+ let acquiredStoreMutation = false;
2144
+ let retainStoreMutation = false;
1448
2145
  try {
1449
2146
  const plan = hooks.backupPlan();
1450
2147
  if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_PLAN_PROVIDER_MISMATCH' });
2148
+ if (providerStoreMutationOwners.get(transactionId) !== providerId || !hub.hasProviderStoreMutation()) {
2149
+ if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_BUSY' });
2150
+ providerStoreMutationOwners.set(transactionId, providerId);
2151
+ acquiredStoreMutation = true;
2152
+ }
1451
2153
  await hooks.rollback(plan);
2154
+ await hooks.setRuntimeBaseline(getHubRuntimeIdentity().runtime_id, 'rollback');
2155
+ const rollbackPlan = hooks.backupPlan() ?? plan;
1452
2156
  try {
1453
- await hooks.recordTransaction({ transaction_id: transactionId, provider_id: providerId, state: 'ROLLBACK_PENDING' }, plan);
2157
+ await hooks.recordTransaction({ transaction_id: transactionId, provider_id: providerId, state: 'ROLLBACK_PENDING' }, rollbackPlan);
1454
2158
  } catch (error) {
1455
2159
  return sendJson(res, 409, { ok: false, code: boundedMachineCodeFromError(error) ?? 'PROVIDER_LIFECYCLE_RECORD_FAILED' });
1456
2160
  }
2161
+ retainStoreMutation = true;
1457
2162
  return sendJson(res, 202, { ok: true, restart_required: true, transaction_id: transactionId, provider_id: providerId, state: 'ROLLBACK_PENDING' });
1458
2163
  } finally {
1459
2164
  await hooks.release();
2165
+ if (acquiredStoreMutation && !retainStoreMutation && providerStoreMutationOwners.get(transactionId) === providerId) {
2166
+ providerStoreMutationOwners.delete(transactionId);
2167
+ hub.endProviderStoreMutation();
2168
+ }
1460
2169
  }
1461
2170
  }
1462
2171
  if (req.method === 'POST' && parts.length === 2 && parts[1] === 'verify-rollback') {
@@ -1464,9 +2173,11 @@ export async function apply(ctx) {
1464
2173
  const { transaction_id: transactionId } = body ?? {};
1465
2174
  const hooks = createProviderDeleteFileHooks({
1466
2175
  profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
2176
+ settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
1467
2177
  configFile: join(CONFIG_DIR, 'config.json'),
1468
2178
  lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
1469
2179
  backupDir: join(CONFIG_DIR, 'provider-backups'),
2180
+ sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
1470
2181
  existingBackupId: transactionId,
1471
2182
  expectedProviderId: providerId,
1472
2183
  });
@@ -1474,9 +2185,28 @@ export async function apply(ctx) {
1474
2185
  if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_PLAN_PROVIDER_MISMATCH' });
1475
2186
  await hooks.acquireLock();
1476
2187
  try {
2188
+ if (providerStoreMutationOwners.get(transactionId) !== providerId || !hub.hasProviderStoreMutation()) {
2189
+ if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_BUSY' });
2190
+ providerStoreMutationOwners.set(transactionId, providerId);
2191
+ }
1477
2192
  const verified = await hooks.verifyRollback(plan);
1478
- if (verified?.ok !== true) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_ROLLBACK_VERIFY_FAILED' });
2193
+ const live = await readProviderInventorySnapshot(hub, ctx, config);
2194
+ verified.catalog_evidence = live.catalog_evidence;
2195
+ verified.default_evidence = live.default_evidence;
2196
+ verified.catalogPresent = hasCompleteProviderCatalogEvidence(live.catalog_evidence)
2197
+ && live.records.some((entry) => entry.id === providerId && entry.lifecycle?.catalogued === true);
2198
+ verified.harnessDefaultRestored = plan.was_harness_default !== true || (
2199
+ live.harness_default?.provider === plan.harness_default_before?.provider
2200
+ && live.harness_default?.model === plan.harness_default_before?.model
2201
+ );
2202
+ verified.runtimeRestarted = hasProviderRuntimeRestartEvidence(plan.rollback_runtime_id_before_restart, getHubRuntimeIdentity().runtime_id);
2203
+ if (verified?.ok !== true || !hasCompleteProviderCatalogEvidence(verified?.catalog_evidence) || verified?.default_evidence?.ok !== true || verified?.catalogPresent !== true || verified?.harnessDefaultRestored !== true || verified?.runtimeRestarted !== true) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_ROLLBACK_VERIFY_FAILED', verification: verified });
1479
2204
  await hooks.recordTransaction({ transaction_id: transactionId, provider_id: providerId, state: 'ROLLED_BACK' }, plan);
2205
+ hub.endProviderMutation(providerId);
2206
+ if (providerStoreMutationOwners.get(transactionId) === providerId) {
2207
+ providerStoreMutationOwners.delete(transactionId);
2208
+ hub.endProviderStoreMutation();
2209
+ }
1480
2210
  return sendJson(res, 200, { ok: true, transaction_id: transactionId, provider_id: providerId, state: 'ROLLED_BACK' });
1481
2211
  } finally {
1482
2212
  await hooks.release();
@@ -1492,33 +2222,128 @@ export async function apply(ctx) {
1492
2222
  providerDeletePlans.delete(body?.plan_id);
1493
2223
  return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_PLAN_EXPIRED' });
1494
2224
  }
1495
- const profile = readProviderProfileRevision();
1496
- if (!profile.ok) return sendJson(res, 409, { ok: false, code: profile.code });
1497
- if (body?.expected_revision !== stored.plan.expected_revision || profile.revision !== stored.plan.expected_revision) {
2225
+ const revisions = readProviderSourceRevisions();
2226
+ if (!revisions.profile.ok && !revisions.settings.ok) return sendJson(res, 409, { ok: false, code: revisions.profile.code ?? revisions.settings.code });
2227
+ if (providerId !== 'deepseek-official' && !hasCompleteProviderCatalogEvidence(inventory.catalog_evidence)) return sendJson(res, 503, { ok: false, code: 'MODEL_CATALOG_UNAVAILABLE' });
2228
+ if (providerId !== 'deepseek-official' && !hasCompleteProviderDeclarationEvidence(inventory.declaration_evidence)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_SOURCE_UNRESOLVED' });
2229
+ if (providerId !== 'deepseek-official' && !hasAvailableProviderLifecycleEvidence(inventory.lifecycle_evidence)) return sendJson(res, 503, { ok: false, code: 'PROVIDER_LIFECYCLE_UNAVAILABLE' });
2230
+ if (providerId !== 'deepseek-official' && inventory.default_evidence?.ok !== true) return sendJson(res, 409, { ok: false, code: inventory.default_evidence?.code ?? 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' });
2231
+ const storedExpected = stored.plan.expected_revisions;
2232
+ const revisionsMatch = storedExpected
2233
+ ? (storedExpected.profile === (revisions.profile.ok ? revisions.profile.revision : null)
2234
+ && storedExpected.settings === (revisions.settings.ok ? revisions.settings.revision : null))
2235
+ : (body?.expected_revision === stored.plan.expected_revision && (revisions.profile.revision ?? revisions.settings.revision) === stored.plan.expected_revision);
2236
+ if (!revisionsMatch || body?.expected_revision !== stored.plan.expected_revision) {
1498
2237
  return sendJson(res, 409, { ok: false, code: 'PROVIDER_PROFILE_CHANGED' });
1499
2238
  }
2239
+ if (providerId !== 'deepseek-official' && hasPendingProviderRecoveryTransactions(inventory)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
1500
2240
  const refreshed = planProviderDelete({
1501
2241
  providerId,
1502
2242
  inventory,
1503
2243
  activeJobs: hub.list(),
1504
2244
  replacementDefault: stored.plan.replacement_default,
1505
- expectedRevision: profile.revision,
2245
+ expectedRevision: stored.plan.expected_revision,
2246
+ expectedRevisions: stored.plan.expected_revisions,
1506
2247
  });
1507
2248
  if (!refreshed.ok) return sendJson(res, 409, { ok: false, code: refreshed.code });
1508
2249
  if (refreshed.plan.replacement_default_model !== stored.plan.replacement_default_model) {
1509
2250
  return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_PLAN_CHANGED' });
1510
2251
  }
1511
- const executionPlan = { ...refreshed.plan, plan_id: stored.plan.plan_id, expected_revision: stored.plan.expected_revision };
1512
- const hooks = createProviderDeleteFileHooks({
1513
- profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
1514
- configFile: join(CONFIG_DIR, 'config.json'),
1515
- lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
1516
- backupDir: join(CONFIG_DIR, 'provider-backups'),
1517
- });
1518
- providerDeletePlans.delete(body.plan_id);
1519
- const { executeProviderDelete } = await import('../provider-lifecycle.mjs');
1520
- const result = await executeProviderDelete(executionPlan, hooks, { deferRestart: true });
1521
- return sendJson(res, result.state === 'RESTART_PENDING' ? 202 : 409, { ok: result.state === 'RESTART_PENDING', restart_required: true, result });
2252
+ let executionPlan = {
2253
+ ...refreshed.plan,
2254
+ plan_id: stored.plan.plan_id,
2255
+ expected_revision: stored.plan.expected_revision,
2256
+ expected_revisions: stored.plan.expected_revisions,
2257
+ runtime_id_before: stored.plan.runtime_id_before,
2258
+ };
2259
+ if (!hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_BUSY' });
2260
+ if (!hub.beginProviderMutation(providerId)) {
2261
+ hub.endProviderStoreMutation();
2262
+ return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_BUSY' });
2263
+ }
2264
+ let hooks = null;
2265
+ let lockAcquired = false;
2266
+ let retainProviderMutation = false;
2267
+ let retainProviderStoreMutation = false;
2268
+ try {
2269
+ hooks = createProviderDeleteFileHooks({
2270
+ profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
2271
+ settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
2272
+ configFile: join(CONFIG_DIR, 'config.json'),
2273
+ lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
2274
+ backupDir: join(CONFIG_DIR, 'provider-backups'),
2275
+ sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
2276
+ runtimeIdProvider: () => getHubRuntimeIdentity().runtime_id,
2277
+ afterLockAcquired: () => {
2278
+ const pendingRecovery = [
2279
+ ...readProviderRecoveryTransactions(),
2280
+ ...readProviderLayerMigrationTransactions(join(CONFIG_DIR, 'provider-migration-backups')),
2281
+ ];
2282
+ if (pendingRecovery.length > 0) {
2283
+ throw Object.assign(new Error('provider recovery transaction is pending'), { code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
2284
+ }
2285
+ if (hub.hasProviderLease(providerId)) {
2286
+ throw Object.assign(new Error('provider is in use'), { code: 'PROVIDER_IN_USE' });
2287
+ }
2288
+ },
2289
+ });
2290
+ // Hold the real shared store lock while rebuilding the live
2291
+ // inventory and delete plan. The pre-lock plan is only a
2292
+ // proposal; catalog/default/ownership/recovery facts must be
2293
+ // authoritative at the mutation boundary.
2294
+ await hooks.acquireLock();
2295
+ lockAcquired = true;
2296
+ const lockedConfig = normalizeGlobalConfig(hub.getConfig?.() ?? {});
2297
+ const lockedInventory = await readProviderInventorySnapshot(hub, ctx, lockedConfig);
2298
+ const lockedRevisions = readProviderSourceRevisions();
2299
+ const lockedExpected = stored.plan.expected_revisions;
2300
+ const lockedRevisionsMatch = lockedExpected
2301
+ ? lockedExpected.profile === (lockedRevisions.profile.ok ? lockedRevisions.profile.revision : null)
2302
+ && lockedExpected.settings === (lockedRevisions.settings.ok ? lockedRevisions.settings.revision : null)
2303
+ : false;
2304
+ if (providerId !== 'deepseek-official' && !hasCompleteProviderCatalogEvidence(lockedInventory.catalog_evidence)) throw Object.assign(new Error('provider catalog unavailable after lock'), { code: 'MODEL_CATALOG_UNAVAILABLE' });
2305
+ if (providerId !== 'deepseek-official' && !hasCompleteProviderDeclarationEvidence(lockedInventory.declaration_evidence)) throw Object.assign(new Error('provider declarations unavailable after lock'), { code: 'PROVIDER_DELETE_SOURCE_UNRESOLVED' });
2306
+ if (providerId !== 'deepseek-official' && !hasAvailableProviderLifecycleEvidence(lockedInventory.lifecycle_evidence)) throw Object.assign(new Error('provider lifecycle unavailable after lock'), { code: 'PROVIDER_LIFECYCLE_UNAVAILABLE' });
2307
+ if (providerId !== 'deepseek-official' && lockedInventory.default_evidence?.ok !== true) throw Object.assign(new Error('provider default authority changed after lock'), { code: lockedInventory.default_evidence?.code ?? 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' });
2308
+ if (providerId !== 'deepseek-official' && hasPendingProviderRecoveryTransactions(lockedInventory)) throw Object.assign(new Error('provider recovery transaction is pending'), { code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
2309
+ if (!lockedRevisionsMatch) throw Object.assign(new Error('provider source changed after lock'), { code: 'PROVIDER_PROFILE_CHANGED' });
2310
+ if (hub.hasProviderLease(providerId)) throw Object.assign(new Error('provider is in use'), { code: 'PROVIDER_IN_USE' });
2311
+ const lockedPlan = planProviderDelete({
2312
+ providerId,
2313
+ inventory: lockedInventory,
2314
+ activeJobs: hub.list(),
2315
+ replacementDefault: stored.plan.replacement_default,
2316
+ expectedRevision: stored.plan.expected_revision,
2317
+ expectedRevisions: stored.plan.expected_revisions,
2318
+ });
2319
+ if (!lockedPlan.ok) throw Object.assign(new Error('provider delete plan changed after lock'), { code: lockedPlan.code });
2320
+ const sameLockedPlan = lockedPlan.plan.replacement_default_model === stored.plan.replacement_default_model
2321
+ && lockedPlan.plan.was_harness_default === stored.plan.was_harness_default
2322
+ && JSON.stringify(lockedPlan.plan.declaration_authorities) === JSON.stringify(stored.plan.declaration_authorities)
2323
+ && JSON.stringify(lockedPlan.plan.harness_default_before ?? null) === JSON.stringify(stored.plan.harness_default_before ?? null)
2324
+ && JSON.stringify(lockedPlan.plan.credential_refs) === JSON.stringify(stored.plan.credential_refs);
2325
+ if (!sameLockedPlan) throw Object.assign(new Error('provider delete plan semantics changed after lock'), { code: 'PROVIDER_DELETE_PLAN_CHANGED' });
2326
+ executionPlan = {
2327
+ ...lockedPlan.plan,
2328
+ plan_id: stored.plan.plan_id,
2329
+ expected_revision: stored.plan.expected_revision,
2330
+ expected_revisions: stored.plan.expected_revisions,
2331
+ runtime_id_before: stored.plan.runtime_id_before,
2332
+ };
2333
+ providerDeletePlans.delete(body.plan_id);
2334
+ const { executeProviderDelete } = await import('../provider-lifecycle.mjs');
2335
+ const result = await executeProviderDelete(executionPlan, hooks, { deferRestart: true });
2336
+ retainProviderMutation = result.state === 'RESTART_PENDING';
2337
+ if (retainProviderMutation) {
2338
+ providerStoreMutationOwners.set(executionPlan.plan_id, providerId);
2339
+ retainProviderStoreMutation = true;
2340
+ }
2341
+ return sendJson(res, result.state === 'RESTART_PENDING' ? 202 : 409, { ok: result.state === 'RESTART_PENDING', restart_required: true, result });
2342
+ } finally {
2343
+ if (lockAcquired && hooks) await hooks.release().catch(() => {});
2344
+ if (!retainProviderMutation) hub.endProviderMutation(providerId);
2345
+ if (!retainProviderStoreMutation) hub.endProviderStoreMutation();
2346
+ }
1522
2347
  }
1523
2348
  return sendJson(res, 404, { ok: false, error: 'unknown providers endpoint' });
1524
2349
  } catch (err) {