@ran-sh/dsh-crew 1.0.2 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/README.zh.md +4 -0
- package/lib/client.js +183 -6
- package/official-web-bridge/lib/client.js +183 -6
- package/package.json +1 -1
- package/src/client/index.tsx +92 -4
- package/src/credential-reference.mjs +38 -0
- package/src/hub/index.mjs +483 -14
- package/src/install/npx-lifecycle.mjs +456 -398
- package/src/model-catalog.mjs +8 -1
- package/src/provider-delete-adapters.mjs +165 -17
- package/src/provider-inventory.mjs +17 -3
- package/src/provider-layer-migration-adapters.mjs +759 -0
- package/src/provider-layer-migration.mjs +198 -0
- package/src/provider-lifecycle-state.mjs +4 -1
- package/src/provider-profile-store.mjs +193 -4
- package/src/provider-settings-store.mjs +82 -6
- package/src/provider-store-lock.mjs +67 -0
- package/src/runtime-identity.mjs +2 -1
package/src/hub/index.mjs
CHANGED
|
@@ -34,8 +34,9 @@ import { buildHubExecutionRows } from '../config-readiness.mjs';
|
|
|
34
34
|
import { localRequestCore, originLoopback } from '../local-request-guard.mjs';
|
|
35
35
|
import { raceWaiters } from '../removable-waiter.mjs';
|
|
36
36
|
import { buildProviderInventory } from '../provider-inventory.mjs';
|
|
37
|
-
import {
|
|
38
|
-
import {
|
|
37
|
+
import { buildProviderLayerMigrationPlan, executeProviderLayerMigration } from '../provider-layer-migration.mjs';
|
|
38
|
+
import { inspectProviderProfile, readProviderDeclarations, readProviderMaterialization } from '../provider-profile-store.mjs';
|
|
39
|
+
import { inspectProviderSettings, readHarnessDefault, readProviderSettingsDeclarations, readProviderSettingsMaterialization } from '../provider-settings-store.mjs';
|
|
39
40
|
import { normalizeProviderLifecycleState } from '../provider-lifecycle-state.mjs';
|
|
40
41
|
import { createProviderHealthStore } from '../provider-health.mjs';
|
|
41
42
|
import { planProviderDelete } from '../provider-lifecycle.mjs';
|
|
@@ -45,6 +46,7 @@ import { executeCredentialPurge, planCredentialPurge } from '../credential-lifec
|
|
|
45
46
|
import { createCredentialPurgeFileHooks } from '../credential-purge-adapters.mjs';
|
|
46
47
|
import { normalizeCredentialPurgeState, recordCredentialPurgeOutcome } from '../credential-purge-state.mjs';
|
|
47
48
|
import { buildRuntimeReadinessSnapshot } from '../runtime-readiness-snapshot.mjs';
|
|
49
|
+
import { createProviderLayerMigrationFileHooks, readProviderLayerMigrationTransactions } from '../provider-layer-migration-adapters.mjs';
|
|
48
50
|
|
|
49
51
|
// policy.mjs is pure (no @deepseek-ai imports, no ctx access), so importing it
|
|
50
52
|
// here is safe for the profile-realm discipline: it never pulls in package
|
|
@@ -193,7 +195,9 @@ export function readProviderRecoveryTransactions(root = join(CONFIG_DIR, 'provid
|
|
|
193
195
|
}];
|
|
194
196
|
}
|
|
195
197
|
});
|
|
196
|
-
|
|
198
|
+
// Recovery is a safety gate, not a bounded history view. Never discard an
|
|
199
|
+
// older unresolved transaction merely because newer entries exist.
|
|
200
|
+
return transactions.sort((a, b) => a.updated_at.localeCompare(b.updated_at));
|
|
197
201
|
} catch {
|
|
198
202
|
let updatedAt;
|
|
199
203
|
try { updatedAt = statSync(root).mtime.toISOString(); } catch { updatedAt = new Date(0).toISOString(); }
|
|
@@ -249,6 +253,14 @@ export function hasProviderRuntimeRestartEvidence(beforeRuntimeId, currentRuntim
|
|
|
249
253
|
&& beforeRuntimeId !== currentRuntimeId;
|
|
250
254
|
}
|
|
251
255
|
|
|
256
|
+
/** Preserve profile/settings provenance when preparing the migration planner. */
|
|
257
|
+
export function buildProviderMigrationDeclarations(profileDeclarations = [], settingsDeclarations = []) {
|
|
258
|
+
return [
|
|
259
|
+
...(Array.isArray(profileDeclarations) ? profileDeclarations : []),
|
|
260
|
+
...(Array.isArray(settingsDeclarations) ? settingsDeclarations : []),
|
|
261
|
+
];
|
|
262
|
+
}
|
|
263
|
+
|
|
252
264
|
async function readProviderInventorySnapshot(hub, ctx, config) {
|
|
253
265
|
let catalog = { providers: [], harness_default: null };
|
|
254
266
|
let catalogEvidence = { ok: false, code: 'MODEL_CATALOG_UNAVAILABLE' };
|
|
@@ -267,13 +279,18 @@ async function readProviderInventorySnapshot(hub, ctx, config) {
|
|
|
267
279
|
} catch {}
|
|
268
280
|
const profileFile = join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml');
|
|
269
281
|
let declarations = [];
|
|
282
|
+
let profileDeclarations = [];
|
|
283
|
+
let settingsDeclarations = [];
|
|
270
284
|
const declarationEvidence = { ok: true, sources: {} };
|
|
271
285
|
try {
|
|
272
286
|
declarationEvidence.sources.profile = { present: existsSync(profileFile) };
|
|
273
287
|
if (existsSync(profileFile)) {
|
|
274
288
|
const source = readFileSync(profileFile, 'utf8');
|
|
275
289
|
const parsed = readProviderDeclarations(source, { file: 'harness/profiles/dsh-crew/cordis.patch.yml' });
|
|
276
|
-
if (parsed.ok)
|
|
290
|
+
if (parsed.ok) {
|
|
291
|
+
profileDeclarations = parsed.declarations;
|
|
292
|
+
declarations = profileDeclarations;
|
|
293
|
+
}
|
|
277
294
|
else { declarationEvidence.ok = false; declarationEvidence.sources.profile.code = parsed.code; }
|
|
278
295
|
}
|
|
279
296
|
} catch { declarationEvidence.ok = false; declarationEvidence.sources.profile = { present: true, code: 'PROVIDER_SOURCE_UNAVAILABLE' }; }
|
|
@@ -285,7 +302,10 @@ async function readProviderInventorySnapshot(hub, ctx, config) {
|
|
|
285
302
|
if (existsSync(settingsFile)) {
|
|
286
303
|
const source = readFileSync(settingsFile, 'utf8');
|
|
287
304
|
const parsed = readProviderSettingsDeclarations(source, { file: 'harness/settings.yaml' });
|
|
288
|
-
if (parsed.ok)
|
|
305
|
+
if (parsed.ok) {
|
|
306
|
+
settingsDeclarations = parsed.declarations;
|
|
307
|
+
declarations = [...declarations, ...settingsDeclarations];
|
|
308
|
+
}
|
|
289
309
|
else { declarationEvidence.ok = false; declarationEvidence.sources.settings.code = parsed.code; }
|
|
290
310
|
const parsedDefault = readHarnessDefault(source);
|
|
291
311
|
if (parsedDefault.ok) settingsDefault = parsedDefault;
|
|
@@ -315,7 +335,10 @@ async function readProviderInventorySnapshot(hub, ctx, config) {
|
|
|
315
335
|
}
|
|
316
336
|
const lifecycleEvidence = readProviderLifecycleStateStatus();
|
|
317
337
|
const lifecycle = lifecycleEvidence.state;
|
|
318
|
-
const recoveryTransactions =
|
|
338
|
+
const recoveryTransactions = [
|
|
339
|
+
...readProviderRecoveryTransactions(),
|
|
340
|
+
...readProviderLayerMigrationTransactions(join(CONFIG_DIR, 'provider-migration-backups')),
|
|
341
|
+
];
|
|
319
342
|
const credentialPurge = readCredentialPurgeState();
|
|
320
343
|
const inventory = buildProviderInventory({
|
|
321
344
|
catalog,
|
|
@@ -328,8 +351,30 @@ async function readProviderInventorySnapshot(hub, ctx, config) {
|
|
|
328
351
|
],
|
|
329
352
|
activeJobs: hub.list(),
|
|
330
353
|
});
|
|
354
|
+
// Keep each authoritative source's credential reference with its own
|
|
355
|
+
// declaration. Reconstructing from the merged inventory would silently
|
|
356
|
+
// attach the first key to every layer when profile/settings differ.
|
|
357
|
+
const migrationDeclarations = buildProviderMigrationDeclarations(profileDeclarations, settingsDeclarations);
|
|
358
|
+
const routingReferences = [
|
|
359
|
+
...(config.flash_model_priority ?? []), ...(config.pro_model_priority ?? []),
|
|
360
|
+
...(config.worker?.model_policy?.priority ?? []), ...(config.worker?.model_policy?.escalation_priority ?? []),
|
|
361
|
+
...(config.review?.model_policy?.priority ?? []), ...(config.review?.model_policy?.escalation_priority ?? []),
|
|
362
|
+
];
|
|
363
|
+
const migration = buildProviderLayerMigrationPlan({
|
|
364
|
+
declarations: migrationDeclarations,
|
|
365
|
+
catalogProviders: catalog.providers,
|
|
366
|
+
harnessDefault: catalog.harness_default,
|
|
367
|
+
routingReferences,
|
|
368
|
+
tombstones: lifecycle.tombstones,
|
|
369
|
+
recoveryTransactions,
|
|
370
|
+
catalogEvidence,
|
|
371
|
+
declarationEvidence,
|
|
372
|
+
lifecycleEvidence,
|
|
373
|
+
defaultEvidence,
|
|
374
|
+
});
|
|
331
375
|
return {
|
|
332
376
|
...inventory,
|
|
377
|
+
migration,
|
|
333
378
|
catalog_evidence: catalogEvidence,
|
|
334
379
|
declaration_evidence: declarationEvidence,
|
|
335
380
|
default_evidence: defaultEvidence,
|
|
@@ -565,6 +610,12 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
565
610
|
hasProviderLease(providerId) {
|
|
566
611
|
return (this.providerLeases.get(providerId) ?? 0) > 0;
|
|
567
612
|
}
|
|
613
|
+
|
|
614
|
+
// Profile/settings are shared stores. A migration or delete of provider A
|
|
615
|
+
// must fence a concurrent lifecycle write for provider B in this Hub.
|
|
616
|
+
beginProviderStoreMutation() { return this.beginProviderMutation('__dsh-provider-store__'); }
|
|
617
|
+
endProviderStoreMutation() { this.endProviderMutation('__dsh-provider-store__'); }
|
|
618
|
+
hasProviderStoreMutation() { return this.providerMutations.has('__dsh-provider-store__'); }
|
|
568
619
|
|
|
569
620
|
publish() {
|
|
570
621
|
this.shard.publish([...this.jobs.values()].map((j) => this.view(j)));
|
|
@@ -671,9 +722,18 @@ export class WorkerRegistry { constructor(ctx) {
|
|
|
671
722
|
candidateSet: attempt > 0 ? 'escalation' : 'primary',
|
|
672
723
|
},
|
|
673
724
|
});
|
|
674
|
-
}
|
|
675
|
-
if (!selection.ok) throw Object.assign(new Error(selection.message), { policyCode: selection.code });
|
|
676
725
|
}
|
|
726
|
+
if (!selection.ok) throw Object.assign(new Error(selection.message), { policyCode: selection.code });
|
|
727
|
+
}
|
|
728
|
+
const pendingLayerMigration = readProviderLayerMigrationTransactions(join(CONFIG_DIR, 'provider-migration-backups'))
|
|
729
|
+
.find((entry) => entry.unresolved === true || entry.provider_id === selection.provider);
|
|
730
|
+
if (pendingLayerMigration) throw Object.assign(new Error('provider layer migration is pending verification'), { code: 'PROVIDER_MIGRATION_PENDING' });
|
|
731
|
+
// A process restart clears the in-memory mutation fence, so persisted
|
|
732
|
+
// provider-delete transactions must also gate admission. Unknown-owner
|
|
733
|
+
// recovery entries fence globally; known entries fence their provider.
|
|
734
|
+
const pendingProviderDelete = readProviderRecoveryTransactions()
|
|
735
|
+
.find((entry) => entry.unresolved === true || entry.provider_id === selection.provider);
|
|
736
|
+
if (pendingProviderDelete) throw Object.assign(new Error('provider delete recovery is pending'), { code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
677
737
|
const providerMutationEpoch = this.getProviderMutationEpoch(selection.provider);
|
|
678
738
|
this.assertProviderNotMutating(selection.provider, providerMutationEpoch);
|
|
679
739
|
|
|
@@ -1192,11 +1252,18 @@ export async function apply(ctx) {
|
|
|
1192
1252
|
const disposers = [];
|
|
1193
1253
|
const PROVIDER_DELETE_PLAN_TTL_MS = 10 * 60 * 1000;
|
|
1194
1254
|
const providerDeletePlans = new Map();
|
|
1255
|
+
const providerMigrationPlans = new Map();
|
|
1256
|
+
const providerMigrationOwners = new Map();
|
|
1257
|
+
const providerStoreMutationOwners = new Map();
|
|
1195
1258
|
const credentialPurgePlans = new Map();
|
|
1196
1259
|
const rememberProviderDeletePlan = (plan) => {
|
|
1197
1260
|
providerDeletePlans.set(plan.plan_id, { plan, created_at: Date.now() });
|
|
1198
1261
|
while (providerDeletePlans.size > 32) providerDeletePlans.delete(providerDeletePlans.keys().next().value);
|
|
1199
1262
|
};
|
|
1263
|
+
const rememberProviderMigrationPlan = (plan) => {
|
|
1264
|
+
providerMigrationPlans.set(plan.plan_id, { plan, created_at: Date.now() });
|
|
1265
|
+
while (providerMigrationPlans.size > 32) providerMigrationPlans.delete(providerMigrationPlans.keys().next().value);
|
|
1266
|
+
};
|
|
1200
1267
|
const rememberCredentialPurgePlan = (plan) => {
|
|
1201
1268
|
credentialPurgePlans.set(plan.plan_id, { plan, created_at: Date.now() });
|
|
1202
1269
|
while (credentialPurgePlans.size > 32) credentialPurgePlans.delete(credentialPurgePlans.keys().next().value);
|
|
@@ -1603,6 +1670,11 @@ export async function apply(ctx) {
|
|
|
1603
1670
|
try { url = new URL(req.url, 'http://localhost'); } catch { return sendJson(res, 400, { ok: false, code: 'PROVIDER_PLAN_INVALID' }); }
|
|
1604
1671
|
const parts = url.pathname.slice(`${ROUTE_BASE}/providers`.length).split('/').filter(Boolean);
|
|
1605
1672
|
try {
|
|
1673
|
+
if (req.method === 'GET' && parts.length === 1 && parts[0] === 'migration-status') {
|
|
1674
|
+
const config = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
1675
|
+
const inventory = await readProviderInventorySnapshot(hub, ctx, config);
|
|
1676
|
+
return sendJson(res, 200, { ok: true, ...inventory.migration, runtime: getHubRuntimeIdentity() });
|
|
1677
|
+
}
|
|
1606
1678
|
const providerId = decodeURIComponent(parts[0]);
|
|
1607
1679
|
const body = await readBody(req);
|
|
1608
1680
|
if (req.method === 'POST' && parts[0] === '_recovery' && parts[1] === 'lock' && parts.length === 2) {
|
|
@@ -1613,19 +1685,44 @@ export async function apply(ctx) {
|
|
|
1613
1685
|
configFile: join(CONFIG_DIR, 'config.json'),
|
|
1614
1686
|
lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
1615
1687
|
backupDir: join(CONFIG_DIR, 'provider-backups'),
|
|
1688
|
+
sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
|
|
1616
1689
|
});
|
|
1617
1690
|
const recovered = await hooks.recoverLock({ force: body?.force === true });
|
|
1618
|
-
|
|
1691
|
+
const migrationHooks = createProviderLayerMigrationFileHooks({
|
|
1692
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1693
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1694
|
+
backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
|
|
1695
|
+
});
|
|
1696
|
+
const migrationRecovered = await migrationHooks.recoverLock({ confirm: true });
|
|
1697
|
+
const ok = recovered.ok === true && migrationRecovered.ok === true;
|
|
1698
|
+
return sendJson(res, ok ? 200 : 409, { ok, delete_lock: recovered, migration_lock: migrationRecovered });
|
|
1619
1699
|
}
|
|
1620
1700
|
if (req.method === 'POST' && parts[0] === '_recovery' && parts[1] === 'quarantine' && parts.length === 2) {
|
|
1621
1701
|
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_QUARANTINE_CONFIRM_REQUIRED' });
|
|
1622
1702
|
const actionId = typeof body?.action_id === 'string' && RECOVERY_ACTION_PATTERN.test(body.action_id)
|
|
1623
1703
|
? body.action_id : null;
|
|
1624
1704
|
if (!actionId) return sendJson(res, 400, { ok: false, code: 'PROVIDER_DELETE_PLAN_INVALID' });
|
|
1625
|
-
const recovery =
|
|
1705
|
+
const recovery = [
|
|
1706
|
+
...readProviderRecoveryTransactions(),
|
|
1707
|
+
...readProviderLayerMigrationTransactions(join(CONFIG_DIR, 'provider-migration-backups')),
|
|
1708
|
+
].find((entry) => entry.action_id === actionId);
|
|
1626
1709
|
if (!recovery) return sendJson(res, 404, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_NOT_FOUND' });
|
|
1627
1710
|
if (recovery.recoverable === true) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_RECOVERABLE' });
|
|
1628
1711
|
if (typeof recovery.storage_id !== 'string') return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_QUARANTINE_FAILED' });
|
|
1712
|
+
if (recovery.source === 'provider-layer-migration') {
|
|
1713
|
+
try {
|
|
1714
|
+
const migrationHooks = createProviderLayerMigrationFileHooks({
|
|
1715
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1716
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1717
|
+
backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
|
|
1718
|
+
});
|
|
1719
|
+
const quarantined = await migrationHooks.quarantine(actionId);
|
|
1720
|
+
return sendJson(res, 200, { ...quarantined, transaction_id: recovery.transaction_id ?? null });
|
|
1721
|
+
} catch (error) {
|
|
1722
|
+
const code = boundedMachineCodeFromError(error) ?? 'PROVIDER_MIGRATION_RECOVERY_QUARANTINE_FAILED';
|
|
1723
|
+
return sendJson(res, 409, { ok: false, code });
|
|
1724
|
+
}
|
|
1725
|
+
}
|
|
1629
1726
|
try {
|
|
1630
1727
|
const hooks = createProviderDeleteFileHooks({
|
|
1631
1728
|
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
@@ -1633,6 +1730,7 @@ export async function apply(ctx) {
|
|
|
1633
1730
|
configFile: join(CONFIG_DIR, 'config.json'),
|
|
1634
1731
|
lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
1635
1732
|
backupDir: join(CONFIG_DIR, 'provider-backups'),
|
|
1733
|
+
sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
|
|
1636
1734
|
afterLockAcquired: () => {
|
|
1637
1735
|
const current = readProviderRecoveryTransactions().find((entry) => entry.action_id === actionId);
|
|
1638
1736
|
if (!current || current.recoverable === true || current.storage_id !== recovery.storage_id) {
|
|
@@ -1649,6 +1747,285 @@ export async function apply(ctx) {
|
|
|
1649
1747
|
}
|
|
1650
1748
|
const config = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
1651
1749
|
const inventory = await readProviderInventorySnapshot(hub, ctx, config);
|
|
1750
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'migrate-plan') {
|
|
1751
|
+
if (!hasCompleteProviderCatalogEvidence(inventory.catalog_evidence)) return sendJson(res, 503, { ok: false, code: 'MODEL_CATALOG_UNAVAILABLE' });
|
|
1752
|
+
if (!hasCompleteProviderDeclarationEvidence(inventory.declaration_evidence)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_SOURCE_UNRESOLVED' });
|
|
1753
|
+
if (!hasAvailableProviderLifecycleEvidence(inventory.lifecycle_evidence)) return sendJson(res, 503, { ok: false, code: 'PROVIDER_LIFECYCLE_UNAVAILABLE' });
|
|
1754
|
+
if (inventory.default_evidence?.ok !== true) return sendJson(res, 409, { ok: false, code: inventory.default_evidence?.code ?? 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' });
|
|
1755
|
+
if (hasPendingProviderRecoveryTransactions(inventory)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
1756
|
+
const entry = inventory.migration.providers.find((candidate) => candidate.provider_id === providerId);
|
|
1757
|
+
if (!entry) return sendJson(res, 404, { ok: false, code: 'PROVIDER_MIGRATION_NOT_REQUIRED' });
|
|
1758
|
+
if (entry.action === 'blocked' || entry.action === 'collision-review') return sendJson(res, 409, { ok: false, code: entry.blocked_reason ?? 'PROVIDER_MIGRATION_BLOCKED' });
|
|
1759
|
+
const revisions = readProviderSourceRevisions();
|
|
1760
|
+
if (!revisions.profile.ok) return sendJson(res, 409, { ok: false, code: revisions.profile.code ?? 'PROVIDER_PROFILE_CHANGED' });
|
|
1761
|
+
if ((entry.action === 'promote-existing-user' || entry.harness_default) && !revisions.settings.ok) return sendJson(res, 409, { ok: false, code: revisions.settings.code ?? 'PROVIDER_SETTINGS_CHANGED' });
|
|
1762
|
+
const profileFile = join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml');
|
|
1763
|
+
const parsedMaterialization = readProviderMaterialization(readFileSync(profileFile, 'utf8'), { providerId, file: 'harness/profiles/dsh-crew/cordis.patch.yml' });
|
|
1764
|
+
if (!parsedMaterialization.ok) return sendJson(res, 409, { ok: false, code: parsedMaterialization.code });
|
|
1765
|
+
const materialization = parsedMaterialization.provider;
|
|
1766
|
+
let userMaterializationBefore;
|
|
1767
|
+
if (entry.action === 'promote-existing-user') {
|
|
1768
|
+
const settingsFile = join(CONFIG_DIR, 'harness', 'settings.yaml');
|
|
1769
|
+
if (!existsSync(settingsFile)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_SETTINGS_CHANGED' });
|
|
1770
|
+
const parsedUserMaterialization = readProviderSettingsMaterialization(readFileSync(settingsFile, 'utf8'), { providerId, file: 'harness/settings.yaml' });
|
|
1771
|
+
if (!parsedUserMaterialization.ok) return sendJson(res, 409, { ok: false, code: parsedUserMaterialization.code });
|
|
1772
|
+
userMaterializationBefore = parsedUserMaterialization.provider;
|
|
1773
|
+
}
|
|
1774
|
+
const plan = {
|
|
1775
|
+
schema_version: 1,
|
|
1776
|
+
kind: 'provider-layer-migration',
|
|
1777
|
+
plan_id: randomUUID(),
|
|
1778
|
+
provider_id: providerId,
|
|
1779
|
+
action: entry.action,
|
|
1780
|
+
source: entry.source,
|
|
1781
|
+
routing_references: entry.referenced_by,
|
|
1782
|
+
materialization: { provider: materialization },
|
|
1783
|
+
...(userMaterializationBefore ? { user_materialization_before: userMaterializationBefore } : {}),
|
|
1784
|
+
expected_revisions: {
|
|
1785
|
+
profile: revisions.profile.revision,
|
|
1786
|
+
settings: revisions.settings.ok ? revisions.settings.revision : null,
|
|
1787
|
+
},
|
|
1788
|
+
harness_default_before: inventory.harness_default ?? null,
|
|
1789
|
+
runtime_id_before: getHubRuntimeIdentity().runtime_id,
|
|
1790
|
+
restart_required: true,
|
|
1791
|
+
};
|
|
1792
|
+
rememberProviderMigrationPlan(plan);
|
|
1793
|
+
return sendJson(res, 200, { ok: true, plan });
|
|
1794
|
+
}
|
|
1795
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'migrate') {
|
|
1796
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_MIGRATION_CONFIRM_REQUIRED' });
|
|
1797
|
+
const stored = providerMigrationPlans.get(body?.plan_id);
|
|
1798
|
+
if (!stored || Date.now() - stored.created_at > PROVIDER_DELETE_PLAN_TTL_MS || stored.plan.provider_id !== providerId) {
|
|
1799
|
+
providerMigrationPlans.delete(body?.plan_id);
|
|
1800
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_PLAN_EXPIRED' });
|
|
1801
|
+
}
|
|
1802
|
+
const entry = inventory.migration.providers.find((candidate) => candidate.provider_id === providerId);
|
|
1803
|
+
if (!entry || entry.action !== stored.plan.action || entry.action === 'blocked' || entry.action === 'collision-review') return sendJson(res, 409, { ok: false, code: entry?.blocked_reason ?? 'PROVIDER_MIGRATION_PLAN_CHANGED' });
|
|
1804
|
+
if (inventory.default_evidence?.ok !== true) return sendJson(res, 409, { ok: false, code: inventory.default_evidence?.code ?? 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' });
|
|
1805
|
+
const revisions = readProviderSourceRevisions();
|
|
1806
|
+
const expected = stored.plan.expected_revisions ?? {};
|
|
1807
|
+
if (revisions.profile.revision !== expected.profile || (revisions.settings.ok ? revisions.settings.revision : null) !== expected.settings) {
|
|
1808
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_SOURCE_CHANGED' });
|
|
1809
|
+
}
|
|
1810
|
+
const hooks = createProviderLayerMigrationFileHooks({
|
|
1811
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1812
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1813
|
+
backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
|
|
1814
|
+
});
|
|
1815
|
+
let result;
|
|
1816
|
+
if (hub.hasProviderLease(providerId)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_IN_USE' });
|
|
1817
|
+
if (!hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
1818
|
+
if (!hub.beginProviderMutation(providerId)) {
|
|
1819
|
+
hub.endProviderStoreMutation();
|
|
1820
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
1821
|
+
}
|
|
1822
|
+
let retainProviderMutation = false;
|
|
1823
|
+
let retainProviderStoreMutation = false;
|
|
1824
|
+
try {
|
|
1825
|
+
await hooks.acquireLock();
|
|
1826
|
+
// The plan was created from a pre-lock snapshot. Re-read every
|
|
1827
|
+
// authoritative source after taking the shared store lock so a
|
|
1828
|
+
// concurrent catalog/default/recovery change cannot be hidden
|
|
1829
|
+
// behind unchanged profile/settings revisions.
|
|
1830
|
+
const lockedConfig = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
1831
|
+
const lockedInventory = await readProviderInventorySnapshot(hub, ctx, lockedConfig);
|
|
1832
|
+
const lockedEntry = lockedInventory.migration.providers.find((candidate) => candidate.provider_id === providerId);
|
|
1833
|
+
const expectedRouting = Array.isArray(stored.plan.routing_references) ? stored.plan.routing_references : [];
|
|
1834
|
+
const sameRouting = JSON.stringify(lockedEntry?.referenced_by ?? []) === JSON.stringify(expectedRouting);
|
|
1835
|
+
const sameDefault = JSON.stringify(lockedInventory.harness_default ?? null) === JSON.stringify(stored.plan.harness_default_before ?? null);
|
|
1836
|
+
const profileFile = join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml');
|
|
1837
|
+
const lockedMaterialization = existsSync(profileFile)
|
|
1838
|
+
? readProviderMaterialization(readFileSync(profileFile, 'utf8'), { providerId, file: 'harness/profiles/dsh-crew/cordis.patch.yml' })
|
|
1839
|
+
: { ok: false };
|
|
1840
|
+
const sameMaterialization = lockedMaterialization.ok === true
|
|
1841
|
+
&& JSON.stringify(lockedMaterialization.provider) === JSON.stringify(stored.plan.materialization?.provider ?? null);
|
|
1842
|
+
const settingsFile = join(CONFIG_DIR, 'harness', 'settings.yaml');
|
|
1843
|
+
const lockedUserMaterialization = stored.plan.action === 'promote-existing-user' && existsSync(settingsFile)
|
|
1844
|
+
? readProviderSettingsMaterialization(readFileSync(settingsFile, 'utf8'), { providerId, file: 'harness/settings.yaml' })
|
|
1845
|
+
: { ok: stored.plan.action !== 'promote-existing-user' };
|
|
1846
|
+
const sameUserMaterialization = stored.plan.action !== 'promote-existing-user'
|
|
1847
|
+
|| (lockedUserMaterialization.ok === true && JSON.stringify(lockedUserMaterialization.provider) === JSON.stringify(stored.plan.user_materialization_before ?? null));
|
|
1848
|
+
const lockedRevisions = readProviderSourceRevisions();
|
|
1849
|
+
const expectedLocked = stored.plan.expected_revisions ?? {};
|
|
1850
|
+
const sameRevisions = lockedRevisions.profile.revision === expectedLocked.profile
|
|
1851
|
+
&& (lockedRevisions.settings.ok ? lockedRevisions.settings.revision : null) === expectedLocked.settings;
|
|
1852
|
+
if (hasPendingProviderRecoveryTransactions(lockedInventory)) {
|
|
1853
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
1854
|
+
}
|
|
1855
|
+
if (!lockedEntry || lockedEntry.action !== stored.plan.action || lockedEntry.collision !== null
|
|
1856
|
+
|| !sameRouting || !sameDefault || !sameMaterialization || !sameUserMaterialization || !sameRevisions
|
|
1857
|
+
|| lockedInventory.default_evidence?.ok !== true
|
|
1858
|
+
|| !hasCompleteProviderCatalogEvidence(lockedInventory.catalog_evidence)
|
|
1859
|
+
|| !hasCompleteProviderDeclarationEvidence(lockedInventory.declaration_evidence)
|
|
1860
|
+
|| !hasAvailableProviderLifecycleEvidence(lockedInventory.lifecycle_evidence)) {
|
|
1861
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_PLAN_CHANGED' });
|
|
1862
|
+
}
|
|
1863
|
+
if (hub.hasProviderLease(providerId)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_IN_USE' });
|
|
1864
|
+
result = await executeProviderLayerMigration(stored.plan, hooks, { confirm: true, deferRestart: true });
|
|
1865
|
+
if (result.ok && result.state === 'RESTART_PENDING') await hooks.setRestartRuntimeBaseline(getHubRuntimeIdentity().runtime_id);
|
|
1866
|
+
providerMigrationPlans.delete(body.plan_id);
|
|
1867
|
+
retainProviderMutation = result.state === 'RESTART_PENDING';
|
|
1868
|
+
if (retainProviderMutation) {
|
|
1869
|
+
providerMigrationOwners.set(providerId, stored.plan.plan_id);
|
|
1870
|
+
providerStoreMutationOwners.set(stored.plan.plan_id, providerId);
|
|
1871
|
+
retainProviderStoreMutation = true;
|
|
1872
|
+
}
|
|
1873
|
+
return sendJson(res, result.ok ? 202 : 409, { ok: result.ok, restart_required: result.restart_required === true, result });
|
|
1874
|
+
} finally {
|
|
1875
|
+
await hooks.release();
|
|
1876
|
+
if (!retainProviderMutation) hub.endProviderMutation(providerId);
|
|
1877
|
+
if (!retainProviderStoreMutation) hub.endProviderStoreMutation();
|
|
1878
|
+
}
|
|
1879
|
+
}
|
|
1880
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'verify-migration') {
|
|
1881
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_MIGRATION_VERIFY_CONFIRM_REQUIRED' });
|
|
1882
|
+
const transactionId = body?.transaction_id;
|
|
1883
|
+
const hooks = createProviderLayerMigrationFileHooks({
|
|
1884
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1885
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1886
|
+
backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
|
|
1887
|
+
existingMigrationId: transactionId,
|
|
1888
|
+
});
|
|
1889
|
+
let acquiredStoreFence = false;
|
|
1890
|
+
try {
|
|
1891
|
+
await hooks.acquireLock();
|
|
1892
|
+
await hooks.backup({});
|
|
1893
|
+
const plan = hooks.backupPlan();
|
|
1894
|
+
if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_PLAN_PROVIDER_MISMATCH' });
|
|
1895
|
+
const owner = providerMigrationOwners.get(providerId);
|
|
1896
|
+
if (providerStoreMutationOwners.get(plan.plan_id) !== providerId || !hub.hasProviderStoreMutation()) {
|
|
1897
|
+
if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
1898
|
+
providerStoreMutationOwners.set(plan.plan_id, providerId);
|
|
1899
|
+
acquiredStoreFence = true;
|
|
1900
|
+
}
|
|
1901
|
+
if (owner !== plan.plan_id) {
|
|
1902
|
+
if (hub.hasProviderLease(providerId) || !hub.beginProviderMutation(providerId)) {
|
|
1903
|
+
if (acquiredStoreFence && providerStoreMutationOwners.get(plan.plan_id) === providerId) { providerStoreMutationOwners.delete(plan.plan_id); hub.endProviderStoreMutation(); }
|
|
1904
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
1905
|
+
}
|
|
1906
|
+
providerMigrationOwners.set(providerId, plan.plan_id);
|
|
1907
|
+
}
|
|
1908
|
+
const verification = await hooks.verify(plan);
|
|
1909
|
+
const live = await readProviderInventorySnapshot(hub, ctx, config);
|
|
1910
|
+
verification.catalogEvidence = live.catalog_evidence;
|
|
1911
|
+
verification.defaultEvidence = live.default_evidence;
|
|
1912
|
+
verification.defaultPreserved = live.default_evidence?.ok === true
|
|
1913
|
+
&& JSON.stringify(live.harness_default ?? null) === JSON.stringify(plan.harness_default_before ?? null);
|
|
1914
|
+
verification.liveNativeRemovable = hasCompleteProviderCatalogEvidence(live.catalog_evidence)
|
|
1915
|
+
&& live.records.some((record) => record.id === providerId && record.lifecycle?.catalogued === true && record.native_removable === true);
|
|
1916
|
+
verification.runtimeRestarted = hasProviderRuntimeRestartEvidence(plan.runtime_id_before_restart ?? plan.runtime_id_before, getHubRuntimeIdentity().runtime_id);
|
|
1917
|
+
if (verification.nativeRemovable !== true || verification.liveNativeRemovable !== true || verification.defaultPreserved !== true || verification.runtimeRestarted !== true) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_VERIFY_FAILED', verification });
|
|
1918
|
+
await hooks.finalizeVerified();
|
|
1919
|
+
if (providerMigrationOwners.get(providerId) === plan.plan_id) {
|
|
1920
|
+
providerMigrationOwners.delete(providerId);
|
|
1921
|
+
hub.endProviderMutation(providerId);
|
|
1922
|
+
}
|
|
1923
|
+
if (providerStoreMutationOwners.get(plan.plan_id) === providerId) {
|
|
1924
|
+
providerStoreMutationOwners.delete(plan.plan_id);
|
|
1925
|
+
hub.endProviderStoreMutation();
|
|
1926
|
+
}
|
|
1927
|
+
return sendJson(res, 200, { ok: true, state: 'VERIFIED', transaction_id: plan.plan_id, provider_id: providerId, verification });
|
|
1928
|
+
} finally {
|
|
1929
|
+
await hooks.release();
|
|
1930
|
+
// Keep the transaction-owned fence on verification failure so
|
|
1931
|
+
// an incomplete migration cannot race a new Worker; the caller
|
|
1932
|
+
// must explicitly retry verification or roll it back.
|
|
1933
|
+
}
|
|
1934
|
+
}
|
|
1935
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'rollback-migration') {
|
|
1936
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_MIGRATION_ROLLBACK_CONFIRM_REQUIRED' });
|
|
1937
|
+
const hooks = createProviderLayerMigrationFileHooks({
|
|
1938
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1939
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1940
|
+
backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
|
|
1941
|
+
existingMigrationId: body?.transaction_id,
|
|
1942
|
+
});
|
|
1943
|
+
let acquiredMigrationFence = false;
|
|
1944
|
+
let retainProviderMutation = false;
|
|
1945
|
+
let acquiredStoreFence = false;
|
|
1946
|
+
try {
|
|
1947
|
+
await hooks.acquireLock();
|
|
1948
|
+
await hooks.backup({});
|
|
1949
|
+
const plan = hooks.backupPlan();
|
|
1950
|
+
if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_PLAN_PROVIDER_MISMATCH' });
|
|
1951
|
+
const owner = providerMigrationOwners.get(providerId);
|
|
1952
|
+
if (providerStoreMutationOwners.get(plan.plan_id) !== providerId || !hub.hasProviderStoreMutation()) {
|
|
1953
|
+
if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
1954
|
+
providerStoreMutationOwners.set(plan.plan_id, providerId);
|
|
1955
|
+
acquiredStoreFence = true;
|
|
1956
|
+
}
|
|
1957
|
+
if (owner !== plan.plan_id) {
|
|
1958
|
+
if (hub.hasProviderLease(providerId) || !hub.beginProviderMutation(providerId)) {
|
|
1959
|
+
if (acquiredStoreFence && providerStoreMutationOwners.get(plan.plan_id) === providerId) { providerStoreMutationOwners.delete(plan.plan_id); hub.endProviderStoreMutation(); }
|
|
1960
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
1961
|
+
}
|
|
1962
|
+
providerMigrationOwners.set(providerId, plan.plan_id);
|
|
1963
|
+
acquiredMigrationFence = true;
|
|
1964
|
+
retainProviderMutation = true;
|
|
1965
|
+
}
|
|
1966
|
+
const rollback = await hooks.rollback();
|
|
1967
|
+
await hooks.setRollbackRuntimeBaseline(getHubRuntimeIdentity().runtime_id);
|
|
1968
|
+
providerMigrationOwners.set(providerId, plan.plan_id);
|
|
1969
|
+
retainProviderMutation = true;
|
|
1970
|
+
return sendJson(res, 202, { ok: true, restart_required: true, transaction_id: plan.plan_id, provider_id: providerId, state: 'ROLLBACK_RESTART_PENDING' });
|
|
1971
|
+
} finally {
|
|
1972
|
+
await hooks.release();
|
|
1973
|
+
if (acquiredMigrationFence && !retainProviderMutation && providerMigrationOwners.get(providerId) === body?.transaction_id) {
|
|
1974
|
+
providerMigrationOwners.delete(providerId);
|
|
1975
|
+
hub.endProviderMutation(providerId);
|
|
1976
|
+
}
|
|
1977
|
+
}
|
|
1978
|
+
}
|
|
1979
|
+
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'verify-rollback-migration') {
|
|
1980
|
+
if (body?.confirm !== true) return sendJson(res, 400, { ok: false, code: 'PROVIDER_MIGRATION_ROLLBACK_VERIFY_CONFIRM_REQUIRED' });
|
|
1981
|
+
const transactionId = body?.transaction_id;
|
|
1982
|
+
const hooks = createProviderLayerMigrationFileHooks({
|
|
1983
|
+
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1984
|
+
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1985
|
+
backupDir: join(CONFIG_DIR, 'provider-migration-backups'),
|
|
1986
|
+
existingMigrationId: transactionId,
|
|
1987
|
+
});
|
|
1988
|
+
let acquiredStoreFence = false;
|
|
1989
|
+
try {
|
|
1990
|
+
await hooks.acquireLock();
|
|
1991
|
+
await hooks.backup({});
|
|
1992
|
+
const plan = hooks.backupPlan();
|
|
1993
|
+
if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_PLAN_PROVIDER_MISMATCH' });
|
|
1994
|
+
const owner = providerMigrationOwners.get(providerId);
|
|
1995
|
+
if (providerStoreMutationOwners.get(plan.plan_id) !== providerId || !hub.hasProviderStoreMutation()) {
|
|
1996
|
+
if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
1997
|
+
providerStoreMutationOwners.set(plan.plan_id, providerId);
|
|
1998
|
+
acquiredStoreFence = true;
|
|
1999
|
+
}
|
|
2000
|
+
if (owner !== plan.plan_id) {
|
|
2001
|
+
if (hub.hasProviderLease(providerId) || !hub.beginProviderMutation(providerId)) {
|
|
2002
|
+
if (acquiredStoreFence && providerStoreMutationOwners.get(plan.plan_id) === providerId) { providerStoreMutationOwners.delete(plan.plan_id); hub.endProviderStoreMutation(); }
|
|
2003
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_BUSY' });
|
|
2004
|
+
}
|
|
2005
|
+
providerMigrationOwners.set(providerId, plan.plan_id);
|
|
2006
|
+
}
|
|
2007
|
+
const verification = await hooks.verifyRollback(plan);
|
|
2008
|
+
const live = await readProviderInventorySnapshot(hub, ctx, config);
|
|
2009
|
+
verification.catalogEvidence = live.catalog_evidence;
|
|
2010
|
+
verification.defaultEvidence = live.default_evidence;
|
|
2011
|
+
verification.defaultPreserved = live.default_evidence?.ok === true
|
|
2012
|
+
&& JSON.stringify(live.harness_default ?? null) === JSON.stringify(plan.harness_default_before ?? null);
|
|
2013
|
+
verification.liveRestored = hasCompleteProviderCatalogEvidence(live.catalog_evidence)
|
|
2014
|
+
&& live.records.some((record) => record.id === providerId && record.lifecycle?.catalogued === true && record.native_removable === false);
|
|
2015
|
+
verification.runtimeRestarted = hasProviderRuntimeRestartEvidence(plan.rollback_runtime_id_before, getHubRuntimeIdentity().runtime_id);
|
|
2016
|
+
if (verification.ok !== true || verification.liveRestored !== true || verification.defaultPreserved !== true || verification.runtimeRestarted !== true) return sendJson(res, 409, { ok: false, code: 'PROVIDER_MIGRATION_ROLLBACK_VERIFY_FAILED', verification });
|
|
2017
|
+
await hooks.finalizeRollback();
|
|
2018
|
+
if (providerMigrationOwners.get(providerId) === plan.plan_id) {
|
|
2019
|
+
providerMigrationOwners.delete(providerId);
|
|
2020
|
+
hub.endProviderMutation(providerId);
|
|
2021
|
+
}
|
|
2022
|
+
if (providerStoreMutationOwners.get(plan.plan_id) === providerId) {
|
|
2023
|
+
providerStoreMutationOwners.delete(plan.plan_id);
|
|
2024
|
+
hub.endProviderStoreMutation();
|
|
2025
|
+
}
|
|
2026
|
+
return sendJson(res, 200, { ok: true, state: 'ROLLED_BACK', transaction_id: plan.plan_id, provider_id: providerId, verification });
|
|
2027
|
+
} finally { await hooks.release(); }
|
|
2028
|
+
}
|
|
1652
2029
|
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'delete-plan') {
|
|
1653
2030
|
if (providerId !== 'deepseek-official' && hasPendingProviderRecoveryTransactions(inventory)) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
1654
2031
|
if (providerId !== 'deepseek-official' && !hasCompleteProviderCatalogEvidence(inventory.catalog_evidence)) return sendJson(res, 503, { ok: false, code: 'MODEL_CATALOG_UNAVAILABLE' });
|
|
@@ -1712,6 +2089,7 @@ export async function apply(ctx) {
|
|
|
1712
2089
|
configFile: join(CONFIG_DIR, 'config.json'),
|
|
1713
2090
|
lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
1714
2091
|
backupDir: join(CONFIG_DIR, 'provider-backups'),
|
|
2092
|
+
sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
|
|
1715
2093
|
existingBackupId: transactionId,
|
|
1716
2094
|
expectedProviderId: providerId,
|
|
1717
2095
|
});
|
|
@@ -1719,6 +2097,10 @@ export async function apply(ctx) {
|
|
|
1719
2097
|
if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_PLAN_PROVIDER_MISMATCH' });
|
|
1720
2098
|
await hooks.acquireLock();
|
|
1721
2099
|
try {
|
|
2100
|
+
if (providerStoreMutationOwners.get(transactionId) !== providerId || !hub.hasProviderStoreMutation()) {
|
|
2101
|
+
if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_BUSY' });
|
|
2102
|
+
providerStoreMutationOwners.set(transactionId, providerId);
|
|
2103
|
+
}
|
|
1722
2104
|
const verification = await hooks.verify(plan);
|
|
1723
2105
|
const live = await readProviderInventorySnapshot(hub, ctx, config);
|
|
1724
2106
|
verification.catalog_evidence = live.catalog_evidence;
|
|
@@ -1735,6 +2117,10 @@ export async function apply(ctx) {
|
|
|
1735
2117
|
}
|
|
1736
2118
|
await hooks.recordTransaction({ transaction_id: transactionId, provider_id: providerId, state: 'VERIFIED' }, plan);
|
|
1737
2119
|
hub.endProviderMutation(providerId);
|
|
2120
|
+
if (providerStoreMutationOwners.get(transactionId) === providerId) {
|
|
2121
|
+
providerStoreMutationOwners.delete(transactionId);
|
|
2122
|
+
hub.endProviderStoreMutation();
|
|
2123
|
+
}
|
|
1738
2124
|
return sendJson(res, 200, { ok: true, transaction_id: transactionId, provider_id: providerId, state: 'VERIFIED', verification });
|
|
1739
2125
|
} finally {
|
|
1740
2126
|
await hooks.release();
|
|
@@ -1749,13 +2135,21 @@ export async function apply(ctx) {
|
|
|
1749
2135
|
configFile: join(CONFIG_DIR, 'config.json'),
|
|
1750
2136
|
lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
1751
2137
|
backupDir: join(CONFIG_DIR, 'provider-backups'),
|
|
2138
|
+
sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
|
|
1752
2139
|
existingBackupId: transactionId,
|
|
1753
2140
|
expectedProviderId: providerId,
|
|
1754
2141
|
});
|
|
1755
2142
|
await hooks.acquireLock();
|
|
2143
|
+
let acquiredStoreMutation = false;
|
|
2144
|
+
let retainStoreMutation = false;
|
|
1756
2145
|
try {
|
|
1757
2146
|
const plan = hooks.backupPlan();
|
|
1758
2147
|
if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_PLAN_PROVIDER_MISMATCH' });
|
|
2148
|
+
if (providerStoreMutationOwners.get(transactionId) !== providerId || !hub.hasProviderStoreMutation()) {
|
|
2149
|
+
if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_BUSY' });
|
|
2150
|
+
providerStoreMutationOwners.set(transactionId, providerId);
|
|
2151
|
+
acquiredStoreMutation = true;
|
|
2152
|
+
}
|
|
1759
2153
|
await hooks.rollback(plan);
|
|
1760
2154
|
await hooks.setRuntimeBaseline(getHubRuntimeIdentity().runtime_id, 'rollback');
|
|
1761
2155
|
const rollbackPlan = hooks.backupPlan() ?? plan;
|
|
@@ -1764,9 +2158,14 @@ export async function apply(ctx) {
|
|
|
1764
2158
|
} catch (error) {
|
|
1765
2159
|
return sendJson(res, 409, { ok: false, code: boundedMachineCodeFromError(error) ?? 'PROVIDER_LIFECYCLE_RECORD_FAILED' });
|
|
1766
2160
|
}
|
|
2161
|
+
retainStoreMutation = true;
|
|
1767
2162
|
return sendJson(res, 202, { ok: true, restart_required: true, transaction_id: transactionId, provider_id: providerId, state: 'ROLLBACK_PENDING' });
|
|
1768
2163
|
} finally {
|
|
1769
2164
|
await hooks.release();
|
|
2165
|
+
if (acquiredStoreMutation && !retainStoreMutation && providerStoreMutationOwners.get(transactionId) === providerId) {
|
|
2166
|
+
providerStoreMutationOwners.delete(transactionId);
|
|
2167
|
+
hub.endProviderStoreMutation();
|
|
2168
|
+
}
|
|
1770
2169
|
}
|
|
1771
2170
|
}
|
|
1772
2171
|
if (req.method === 'POST' && parts.length === 2 && parts[1] === 'verify-rollback') {
|
|
@@ -1778,6 +2177,7 @@ export async function apply(ctx) {
|
|
|
1778
2177
|
configFile: join(CONFIG_DIR, 'config.json'),
|
|
1779
2178
|
lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
1780
2179
|
backupDir: join(CONFIG_DIR, 'provider-backups'),
|
|
2180
|
+
sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
|
|
1781
2181
|
existingBackupId: transactionId,
|
|
1782
2182
|
expectedProviderId: providerId,
|
|
1783
2183
|
});
|
|
@@ -1785,6 +2185,10 @@ export async function apply(ctx) {
|
|
|
1785
2185
|
if (!plan || plan.provider_id !== providerId) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_PLAN_PROVIDER_MISMATCH' });
|
|
1786
2186
|
await hooks.acquireLock();
|
|
1787
2187
|
try {
|
|
2188
|
+
if (providerStoreMutationOwners.get(transactionId) !== providerId || !hub.hasProviderStoreMutation()) {
|
|
2189
|
+
if (hub.hasProviderStoreMutation() || !hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_BUSY' });
|
|
2190
|
+
providerStoreMutationOwners.set(transactionId, providerId);
|
|
2191
|
+
}
|
|
1788
2192
|
const verified = await hooks.verifyRollback(plan);
|
|
1789
2193
|
const live = await readProviderInventorySnapshot(hub, ctx, config);
|
|
1790
2194
|
verified.catalog_evidence = live.catalog_evidence;
|
|
@@ -1799,6 +2203,10 @@ export async function apply(ctx) {
|
|
|
1799
2203
|
if (verified?.ok !== true || !hasCompleteProviderCatalogEvidence(verified?.catalog_evidence) || verified?.default_evidence?.ok !== true || verified?.catalogPresent !== true || verified?.harnessDefaultRestored !== true || verified?.runtimeRestarted !== true) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_ROLLBACK_VERIFY_FAILED', verification: verified });
|
|
1800
2204
|
await hooks.recordTransaction({ transaction_id: transactionId, provider_id: providerId, state: 'ROLLED_BACK' }, plan);
|
|
1801
2205
|
hub.endProviderMutation(providerId);
|
|
2206
|
+
if (providerStoreMutationOwners.get(transactionId) === providerId) {
|
|
2207
|
+
providerStoreMutationOwners.delete(transactionId);
|
|
2208
|
+
hub.endProviderStoreMutation();
|
|
2209
|
+
}
|
|
1802
2210
|
return sendJson(res, 200, { ok: true, transaction_id: transactionId, provider_id: providerId, state: 'ROLLED_BACK' });
|
|
1803
2211
|
} finally {
|
|
1804
2212
|
await hooks.release();
|
|
@@ -1841,25 +2249,37 @@ export async function apply(ctx) {
|
|
|
1841
2249
|
if (refreshed.plan.replacement_default_model !== stored.plan.replacement_default_model) {
|
|
1842
2250
|
return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_PLAN_CHANGED' });
|
|
1843
2251
|
}
|
|
1844
|
-
|
|
2252
|
+
let executionPlan = {
|
|
1845
2253
|
...refreshed.plan,
|
|
1846
2254
|
plan_id: stored.plan.plan_id,
|
|
1847
2255
|
expected_revision: stored.plan.expected_revision,
|
|
1848
2256
|
expected_revisions: stored.plan.expected_revisions,
|
|
1849
2257
|
runtime_id_before: stored.plan.runtime_id_before,
|
|
1850
2258
|
};
|
|
1851
|
-
if (!hub.
|
|
2259
|
+
if (!hub.beginProviderStoreMutation()) return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_BUSY' });
|
|
2260
|
+
if (!hub.beginProviderMutation(providerId)) {
|
|
2261
|
+
hub.endProviderStoreMutation();
|
|
2262
|
+
return sendJson(res, 409, { ok: false, code: 'PROVIDER_DELETE_BUSY' });
|
|
2263
|
+
}
|
|
2264
|
+
let hooks = null;
|
|
2265
|
+
let lockAcquired = false;
|
|
1852
2266
|
let retainProviderMutation = false;
|
|
2267
|
+
let retainProviderStoreMutation = false;
|
|
1853
2268
|
try {
|
|
1854
|
-
|
|
2269
|
+
hooks = createProviderDeleteFileHooks({
|
|
1855
2270
|
profileFile: join(CONFIG_DIR, 'harness', 'profiles', 'dsh-crew', 'cordis.patch.yml'),
|
|
1856
2271
|
settingsFile: join(CONFIG_DIR, 'harness', 'settings.yaml'),
|
|
1857
2272
|
configFile: join(CONFIG_DIR, 'config.json'),
|
|
1858
2273
|
lifecycleFile: join(CONFIG_DIR, 'provider-lifecycle.json'),
|
|
1859
2274
|
backupDir: join(CONFIG_DIR, 'provider-backups'),
|
|
2275
|
+
sharedLockFile: join(CONFIG_DIR, 'provider-store.lock'),
|
|
1860
2276
|
runtimeIdProvider: () => getHubRuntimeIdentity().runtime_id,
|
|
1861
2277
|
afterLockAcquired: () => {
|
|
1862
|
-
|
|
2278
|
+
const pendingRecovery = [
|
|
2279
|
+
...readProviderRecoveryTransactions(),
|
|
2280
|
+
...readProviderLayerMigrationTransactions(join(CONFIG_DIR, 'provider-migration-backups')),
|
|
2281
|
+
];
|
|
2282
|
+
if (pendingRecovery.length > 0) {
|
|
1863
2283
|
throw Object.assign(new Error('provider recovery transaction is pending'), { code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
1864
2284
|
}
|
|
1865
2285
|
if (hub.hasProviderLease(providerId)) {
|
|
@@ -1867,13 +2287,62 @@ export async function apply(ctx) {
|
|
|
1867
2287
|
}
|
|
1868
2288
|
},
|
|
1869
2289
|
});
|
|
2290
|
+
// Hold the real shared store lock while rebuilding the live
|
|
2291
|
+
// inventory and delete plan. The pre-lock plan is only a
|
|
2292
|
+
// proposal; catalog/default/ownership/recovery facts must be
|
|
2293
|
+
// authoritative at the mutation boundary.
|
|
2294
|
+
await hooks.acquireLock();
|
|
2295
|
+
lockAcquired = true;
|
|
2296
|
+
const lockedConfig = normalizeGlobalConfig(hub.getConfig?.() ?? {});
|
|
2297
|
+
const lockedInventory = await readProviderInventorySnapshot(hub, ctx, lockedConfig);
|
|
2298
|
+
const lockedRevisions = readProviderSourceRevisions();
|
|
2299
|
+
const lockedExpected = stored.plan.expected_revisions;
|
|
2300
|
+
const lockedRevisionsMatch = lockedExpected
|
|
2301
|
+
? lockedExpected.profile === (lockedRevisions.profile.ok ? lockedRevisions.profile.revision : null)
|
|
2302
|
+
&& lockedExpected.settings === (lockedRevisions.settings.ok ? lockedRevisions.settings.revision : null)
|
|
2303
|
+
: false;
|
|
2304
|
+
if (providerId !== 'deepseek-official' && !hasCompleteProviderCatalogEvidence(lockedInventory.catalog_evidence)) throw Object.assign(new Error('provider catalog unavailable after lock'), { code: 'MODEL_CATALOG_UNAVAILABLE' });
|
|
2305
|
+
if (providerId !== 'deepseek-official' && !hasCompleteProviderDeclarationEvidence(lockedInventory.declaration_evidence)) throw Object.assign(new Error('provider declarations unavailable after lock'), { code: 'PROVIDER_DELETE_SOURCE_UNRESOLVED' });
|
|
2306
|
+
if (providerId !== 'deepseek-official' && !hasAvailableProviderLifecycleEvidence(lockedInventory.lifecycle_evidence)) throw Object.assign(new Error('provider lifecycle unavailable after lock'), { code: 'PROVIDER_LIFECYCLE_UNAVAILABLE' });
|
|
2307
|
+
if (providerId !== 'deepseek-official' && lockedInventory.default_evidence?.ok !== true) throw Object.assign(new Error('provider default authority changed after lock'), { code: lockedInventory.default_evidence?.code ?? 'PROVIDER_DEFAULT_AUTHORITY_UNAVAILABLE' });
|
|
2308
|
+
if (providerId !== 'deepseek-official' && hasPendingProviderRecoveryTransactions(lockedInventory)) throw Object.assign(new Error('provider recovery transaction is pending'), { code: 'PROVIDER_DELETE_RECOVERY_PENDING' });
|
|
2309
|
+
if (!lockedRevisionsMatch) throw Object.assign(new Error('provider source changed after lock'), { code: 'PROVIDER_PROFILE_CHANGED' });
|
|
2310
|
+
if (hub.hasProviderLease(providerId)) throw Object.assign(new Error('provider is in use'), { code: 'PROVIDER_IN_USE' });
|
|
2311
|
+
const lockedPlan = planProviderDelete({
|
|
2312
|
+
providerId,
|
|
2313
|
+
inventory: lockedInventory,
|
|
2314
|
+
activeJobs: hub.list(),
|
|
2315
|
+
replacementDefault: stored.plan.replacement_default,
|
|
2316
|
+
expectedRevision: stored.plan.expected_revision,
|
|
2317
|
+
expectedRevisions: stored.plan.expected_revisions,
|
|
2318
|
+
});
|
|
2319
|
+
if (!lockedPlan.ok) throw Object.assign(new Error('provider delete plan changed after lock'), { code: lockedPlan.code });
|
|
2320
|
+
const sameLockedPlan = lockedPlan.plan.replacement_default_model === stored.plan.replacement_default_model
|
|
2321
|
+
&& lockedPlan.plan.was_harness_default === stored.plan.was_harness_default
|
|
2322
|
+
&& JSON.stringify(lockedPlan.plan.declaration_authorities) === JSON.stringify(stored.plan.declaration_authorities)
|
|
2323
|
+
&& JSON.stringify(lockedPlan.plan.harness_default_before ?? null) === JSON.stringify(stored.plan.harness_default_before ?? null)
|
|
2324
|
+
&& JSON.stringify(lockedPlan.plan.credential_refs) === JSON.stringify(stored.plan.credential_refs);
|
|
2325
|
+
if (!sameLockedPlan) throw Object.assign(new Error('provider delete plan semantics changed after lock'), { code: 'PROVIDER_DELETE_PLAN_CHANGED' });
|
|
2326
|
+
executionPlan = {
|
|
2327
|
+
...lockedPlan.plan,
|
|
2328
|
+
plan_id: stored.plan.plan_id,
|
|
2329
|
+
expected_revision: stored.plan.expected_revision,
|
|
2330
|
+
expected_revisions: stored.plan.expected_revisions,
|
|
2331
|
+
runtime_id_before: stored.plan.runtime_id_before,
|
|
2332
|
+
};
|
|
1870
2333
|
providerDeletePlans.delete(body.plan_id);
|
|
1871
2334
|
const { executeProviderDelete } = await import('../provider-lifecycle.mjs');
|
|
1872
2335
|
const result = await executeProviderDelete(executionPlan, hooks, { deferRestart: true });
|
|
1873
2336
|
retainProviderMutation = result.state === 'RESTART_PENDING';
|
|
2337
|
+
if (retainProviderMutation) {
|
|
2338
|
+
providerStoreMutationOwners.set(executionPlan.plan_id, providerId);
|
|
2339
|
+
retainProviderStoreMutation = true;
|
|
2340
|
+
}
|
|
1874
2341
|
return sendJson(res, result.state === 'RESTART_PENDING' ? 202 : 409, { ok: result.state === 'RESTART_PENDING', restart_required: true, result });
|
|
1875
2342
|
} finally {
|
|
2343
|
+
if (lockAcquired && hooks) await hooks.release().catch(() => {});
|
|
1876
2344
|
if (!retainProviderMutation) hub.endProviderMutation(providerId);
|
|
2345
|
+
if (!retainProviderStoreMutation) hub.endProviderStoreMutation();
|
|
1877
2346
|
}
|
|
1878
2347
|
}
|
|
1879
2348
|
return sendJson(res, 404, { ok: false, error: 'unknown providers endpoint' });
|