@stage-labs/metro 0.1.0-beta.289 → 0.1.0-beta.290
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/runtime/marketplace/plugin/.claude-plugin/plugin.json +1 -1
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/gates.ts +6 -4
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/migration-api.ts +55 -0
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/migration-control.ts +269 -0
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/migration-guard.ts +90 -0
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/migration-operations.ts +196 -0
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/migration-preview.ts +46 -11
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/migration-source.ts +23 -0
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/migration-state.ts +122 -0
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/oauth-routes.ts +68 -45
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/oauth.ts +11 -4
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/org-runtime-discovery.ts +4 -1
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/org-runtime.ts +95 -41
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/org-sync-control.ts +37 -3
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/org-sync-watch.ts +15 -1
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/plugin-sync.ts +5 -3
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/publication.ts +20 -5
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/relay-session.ts +1 -0
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/relay.ts +26 -11
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/store.ts +71 -50
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/tools.ts +2 -1
- package/runtime/node_modules/@metro-labs/daemon/src/routes/local-mode.ts +5 -2
- package/runtime/node_modules/@metro-labs/daemon/src/routes/session-apis.ts +3 -0
- package/runtime/node_modules/@metro-labs/http/package.json +2 -1
- package/runtime/node_modules/@metro-labs/http/src/connector-delivery.ts +6 -41
- package/runtime/node_modules/@metro-labs/http/src/connector-migration.ts +176 -0
- package/runtime/node_modules/@metro-labs/http/src/connector-policy.ts +3 -0
- package/runtime/node_modules/@metro-labs/http/src/transfer-crypto.ts +44 -0
- package/runtime/runtime.json +1 -1
- package/runtime/sdk-runner/runner.json +1 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "metro",
|
|
3
|
-
"version": "0.1.0-beta.
|
|
3
|
+
"version": "0.1.0-beta.290",
|
|
4
4
|
"description": "Metro for Claude Code, installed by the metro daemon on this machine: every connector the agent holds as its own MCP server relayed through the daemon, the orchestrator-only guard on the main thread, and the standing rules loaded at each session start. No vendor credential ever sits in a config file.",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Bonustrack",
|
|
@@ -20,23 +20,25 @@ export interface ConnectorGate {
|
|
|
20
20
|
}
|
|
21
21
|
|
|
22
22
|
let gates: ConnectorGate[] = [];
|
|
23
|
+
let managed = new Set<string>();
|
|
23
24
|
|
|
24
25
|
const normalized = (name: string): string => name.replace(/[^a-zA-Z0-9_-]/g, '_');
|
|
25
26
|
|
|
26
|
-
function approvalChanged(gate: ConnectorGate, tool: string, next: GateRow | undefined, keys:
|
|
27
|
+
function approvalChanged(gate: ConnectorGate, tool: string, next: GateRow | undefined, keys: ReadonlyMap<string, string>): boolean {
|
|
27
28
|
if (next === undefined || gate.server !== keys.get(gate.id) || JSON.stringify([gate.name, gate.policy]) !== JSON.stringify([next.name, next.config.policy ?? {}])) return true;
|
|
28
29
|
const before = decide(gate.policy, { name: tool, group: toolGroupOf(gate, tool) });
|
|
29
30
|
const after = decide(next.config.policy, { name: tool, group: next.config.toolGroups?.[tool] === 'read' ? 'read' : 'write' });
|
|
30
31
|
return before !== after;
|
|
31
32
|
}
|
|
32
33
|
|
|
33
|
-
export function registerConnectors(rows: readonly GateRow[]): void {
|
|
34
|
-
const
|
|
34
|
+
export function registerConnectors(rows: readonly GateRow[], managedIds: readonly string[] = [], keys: ReadonlyMap<string, string> = serverKeysOf([...rows])): void {
|
|
35
|
+
const nextManaged = new Set(managedIds);
|
|
35
36
|
revokeToolPrompts((tool) => {
|
|
36
37
|
const found = connectorToolOf(tool);
|
|
37
|
-
if (found === undefined || !isOrgRelayId(found.gate.id)) return false;
|
|
38
|
+
if (found === undefined || (!isOrgRelayId(found.gate.id) && !managed.has(found.gate.id) && !nextManaged.has(found.gate.id))) return false;
|
|
38
39
|
return approvalChanged(found.gate, found.tool, rows.find((row) => row.id === found.gate.id), keys);
|
|
39
40
|
});
|
|
41
|
+
managed = nextManaged;
|
|
40
42
|
gates = rows.map((row) => ({
|
|
41
43
|
id: row.id,
|
|
42
44
|
name: row.name,
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import type { IncomingMessage, ServerResponse } from 'node:http';
|
|
2
|
+
import { isRecord } from '@metro-labs/core/is-record';
|
|
3
|
+
import { parseId } from '@metro-labs/core/ids';
|
|
4
|
+
import { ApiError } from '@metro-labs/http/api-error';
|
|
5
|
+
import { readJsonBody, sessionRoute } from '@metro-labs/http/api-http';
|
|
6
|
+
import type { ConnectorMigrations, MigrationAction } from './migration-control.js';
|
|
7
|
+
|
|
8
|
+
const PATH = '/api/connectors/migrations';
|
|
9
|
+
const actions: readonly MigrationAction[] = ['prepare', 'reconcile', 'cutover', 'rollback', 'resolve'];
|
|
10
|
+
export type MigrationApiDeps = Pick<ConnectorMigrations, 'status' | 'run'>;
|
|
11
|
+
|
|
12
|
+
interface MigrationInput {
|
|
13
|
+
sourceId: string;
|
|
14
|
+
action: MigrationAction;
|
|
15
|
+
scope: string;
|
|
16
|
+
operationId: string | null;
|
|
17
|
+
uncertainty: string | null;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function operationOf(value: unknown): string | null {
|
|
21
|
+
if (value === null) return null;
|
|
22
|
+
const id = parseId(value);
|
|
23
|
+
if (id === null) throw new ApiError('A migration needs its current operation ID, or null for an unprepared source.', 400);
|
|
24
|
+
return id;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function receiptOf(body: Record<string, unknown>, action: MigrationAction): string | null {
|
|
28
|
+
if (action !== 'resolve') {
|
|
29
|
+
if (Object.hasOwn(body, 'uncertainty')) throw new ApiError('Only request settlement accepts an uncertainty receipt.', 400);
|
|
30
|
+
return null;
|
|
31
|
+
}
|
|
32
|
+
if (typeof body.uncertainty !== 'string' || !/^[a-f0-9]{64}$/.test(body.uncertainty)) throw new ApiError('Confirming settled vendor requests needs the current uncertainty receipt.', 400);
|
|
33
|
+
return body.uncertainty;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function inputOf(body: unknown): MigrationInput {
|
|
37
|
+
if (!isRecord(body) || Object.keys(body).some((key) => !['scope', 'sourceId', 'action', 'operationId', 'uncertainty'].includes(key)) || typeof body.scope !== 'string' || !/^[a-f0-9]{64}$/.test(body.scope)) {
|
|
38
|
+
throw new ApiError('A migration needs its connector, action and current enrollment scope.', 400);
|
|
39
|
+
}
|
|
40
|
+
const sourceId = parseId(body.sourceId);
|
|
41
|
+
const action = actions.find((value) => value === body.action);
|
|
42
|
+
if (sourceId === null || action === undefined) throw new ApiError('A migration needs a valid connector and action.', 400);
|
|
43
|
+
return { sourceId, action, scope: body.scope, operationId: operationOf(body.operationId), uncertainty: receiptOf(body, action) };
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function handleMigrationRequest(req: IncomingMessage, res: ServerResponse, deps: MigrationApiDeps): boolean {
|
|
47
|
+
return sessionRoute(req, res, { methods: { [PATH]: ['GET', 'POST'] }, admin: true, label: 'connector-migration-api' }, async (session) => {
|
|
48
|
+
const body = req.method === 'POST' ? await readJsonBody(req) : null;
|
|
49
|
+
const current = deps.status();
|
|
50
|
+
if (current.owner === null || current.owner !== session.subject) throw new ApiError('This agent belongs to another organization.', 403);
|
|
51
|
+
if (req.method !== 'POST') return current;
|
|
52
|
+
const { sourceId, action, scope, operationId, uncertainty } = inputOf(body);
|
|
53
|
+
return deps.run(sourceId, action, scope, operationId, uncertainty);
|
|
54
|
+
});
|
|
55
|
+
}
|
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
import { newId } from '@metro-labs/core/ids';
|
|
2
|
+
import { log } from '@metro-labs/core/log';
|
|
3
|
+
import type { ConnectorSyncStatus } from '@metro-labs/core/connector-sync';
|
|
4
|
+
import { ApiError } from '@metro-labs/http/api-error';
|
|
5
|
+
import type { DeliverySnapshot } from '@metro-labs/http/connector-delivery';
|
|
6
|
+
import { migrationResponseOf, sealConnectorMigration, type MigrationResponse, type MigrationView as RemoteMigration } from '@metro-labs/http/connector-migration';
|
|
7
|
+
import { agentsDir } from '../agents/files.js';
|
|
8
|
+
import { readAgentKey, type AgentKey } from './agent-key.js';
|
|
9
|
+
import { agentCall, METRO_API, type MetroApi } from './metro-api.js';
|
|
10
|
+
import { migrationSource } from './migration-preview.js';
|
|
11
|
+
import { migrationPayload } from './migration-source.js';
|
|
12
|
+
import { localMigrationIdle } from './migration-guard.js';
|
|
13
|
+
import { migrationUncertaintyReceipt, resolveMigrationUncertainty } from './migration-operations.js';
|
|
14
|
+
import { localRollbackMigrationSource } from './store.js';
|
|
15
|
+
import { freshOrgConnectors, orgConnectorMigrationIdle, orgConnectorSyncStatus, refreshOrgConnectorRuntime } from './org-sync-watch.js';
|
|
16
|
+
import { migrationRecord, migrationRecords, saveMigration, type MigrationRecord } from './migration-state.js';
|
|
17
|
+
|
|
18
|
+
export type MigrationAction = 'prepare' | 'reconcile' | 'cutover' | 'rollback' | 'resolve';
|
|
19
|
+
interface Context { key: AgentKey; owner: string; agentId: string; scope: string }
|
|
20
|
+
export interface MigrationView extends Pick<MigrationRecord, 'id' | 'sourceId' | 'name' | 'destinationId' | 'phase' | 'auth' | 'reason'> {
|
|
21
|
+
uncertainty: string | null;
|
|
22
|
+
}
|
|
23
|
+
export interface MigrationStatus {
|
|
24
|
+
scope: string;
|
|
25
|
+
owner: string | null;
|
|
26
|
+
agentId: string | null;
|
|
27
|
+
enabled: boolean;
|
|
28
|
+
enrolled: boolean;
|
|
29
|
+
migrations: MigrationView[];
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
interface MigrationDeps {
|
|
33
|
+
dir?: string;
|
|
34
|
+
api?: MetroApi;
|
|
35
|
+
status?: () => ConnectorSyncStatus;
|
|
36
|
+
key?: () => AgentKey | null;
|
|
37
|
+
fresh?: () => Promise<DeliverySnapshot>;
|
|
38
|
+
changed?: () => void;
|
|
39
|
+
localIdle?: (id: string) => boolean;
|
|
40
|
+
orgIdle?: (id: string) => boolean;
|
|
41
|
+
restore?: (id: string, oauth: boolean) => void;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const conflict = (message: string): ApiError => new ApiError(message, 409);
|
|
45
|
+
const uncertain = (): ApiError => new ApiError('The migration request was not confirmed. Refresh and reconcile this connector before trying another action; an interrupted request may already have reached Metro.', 503);
|
|
46
|
+
const viewOf = ({ id, sourceId, name, destinationId, phase, auth, reason }: MigrationRecord, uncertainty: string | null): MigrationView => ({ id, sourceId, name, destinationId, phase, auth, reason, uncertainty });
|
|
47
|
+
const reasons: Record<RemoteMigration['readiness'], string | null> = { pending: 'not_verified', ready: null, signin: 'sign_in', changed: 'destination_changed', unavailable: 'unavailable' };
|
|
48
|
+
const samePolicy = (a: MigrationRecord['policy'], b: MigrationRecord['policy']): boolean =>
|
|
49
|
+
(a.read ?? 'allow') === (b.read ?? 'allow') && (a.write ?? 'allow') === (b.write ?? 'allow') && JSON.stringify(Object.entries(a.tools ?? {}).sort()) === JSON.stringify(Object.entries(b.tools ?? {}).sort());
|
|
50
|
+
|
|
51
|
+
function received(record: MigrationRecord, snapshot: DeliverySnapshot): boolean {
|
|
52
|
+
const delivered = snapshot.connectors.find((row) => row.id === record.destinationId);
|
|
53
|
+
return delivered?.state === 'ready' && delivered.name === record.name && samePolicy(delivered.policy, record.policy);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export class ConnectorMigrations {
|
|
57
|
+
private readonly dir: string;
|
|
58
|
+
private readonly api: MetroApi;
|
|
59
|
+
private running = false;
|
|
60
|
+
|
|
61
|
+
constructor(private readonly deps: MigrationDeps = {}) {
|
|
62
|
+
this.dir = deps.dir ?? agentsDir();
|
|
63
|
+
this.api = deps.api ?? METRO_API;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
private syncStatus(): ConnectorSyncStatus { return (this.deps.status ?? orgConnectorSyncStatus)(); }
|
|
67
|
+
private key(): AgentKey | null { return this.deps.key === undefined ? readAgentKey(this.dir) : this.deps.key(); }
|
|
68
|
+
private changed(): void { (this.deps.changed ?? refreshOrgConnectorRuntime)(); }
|
|
69
|
+
private localIdle(id: string): boolean { return this.deps.localIdle?.(id) ?? localMigrationIdle(id, this.dir); }
|
|
70
|
+
private orgIdle(id: string): boolean { return (this.deps.orgIdle ?? orgConnectorMigrationIdle)(id); }
|
|
71
|
+
|
|
72
|
+
status(): MigrationStatus {
|
|
73
|
+
const { scope, owner, agentId, enabled, enrolled } = this.syncStatus();
|
|
74
|
+
const keyId = this.key()?.keyId;
|
|
75
|
+
const migrations = migrationRecords(this.dir).filter((row) => row.owner === owner).map((row) => {
|
|
76
|
+
const uncertainty = migrationUncertaintyReceipt(row.sourceId, this.dir);
|
|
77
|
+
const reason = uncertainty === null ? row.reason : 'requests_uncertain';
|
|
78
|
+
return viewOf({ ...row, reason: row.agentId !== agentId || row.keyId !== keyId ? 'identity_changed' : reason }, uncertainty);
|
|
79
|
+
});
|
|
80
|
+
return { scope, owner, agentId, enabled, enrolled, migrations };
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
private context(scope: string, requireSync: boolean): Context {
|
|
84
|
+
const status = this.syncStatus();
|
|
85
|
+
const key = this.key();
|
|
86
|
+
if (scope !== status.scope) throw conflict('The organization, agent or enrollment changed. Refresh before continuing the migration.');
|
|
87
|
+
if (!status.enrolled || key?.enrollment?.organization !== status.owner || key.enrollment.server !== status.agentId) throw conflict('Enroll this agent with the current organization before migrating connectors.');
|
|
88
|
+
if (requireSync && !status.enabled) throw conflict('Enable organization connector sync on the Server page before preparing or switching a connector.');
|
|
89
|
+
return { key, owner: key.enrollment.organization, agentId: key.enrollment.server, scope };
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
private check(context: Context, record?: MigrationRecord): void {
|
|
93
|
+
const current = this.context(context.scope, false);
|
|
94
|
+
if (current.key.keyId !== context.key.keyId || current.owner !== context.owner || current.agentId !== context.agentId) throw conflict('The migration identity changed. Refresh before continuing.');
|
|
95
|
+
if (record !== undefined && (record.owner !== current.owner || record.agentId !== current.agentId || record.keyId !== current.key.keyId)) throw conflict('This migration belongs to a different enrollment. Restore its original enrollment before reconciling it.');
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
private save(context: Context, record: MigrationRecord): MigrationRecord {
|
|
99
|
+
this.check(context, record);
|
|
100
|
+
saveMigration(record, this.dir);
|
|
101
|
+
this.changed();
|
|
102
|
+
return record;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
private source(record: MigrationRecord): ReturnType<typeof migrationPayload> {
|
|
106
|
+
const payload = migrationPayload(migrationSource(record.sourceId, this.dir));
|
|
107
|
+
if (payload.sourceStamp !== record.sourceStamp) throw conflict('The retained local connector changed. Restore its original settings before continuing; Metro will not overwrite it.');
|
|
108
|
+
return payload;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
private async call(context: Context, record: MigrationRecord, action: 'begin' | 'prepare' | 'status' | 'activate' | 'rollback', sealed?: ReturnType<typeof sealConnectorMigration>): Promise<MigrationResponse> {
|
|
112
|
+
this.check(context, record);
|
|
113
|
+
let response: MigrationResponse;
|
|
114
|
+
try {
|
|
115
|
+
response = migrationResponseOf(await agentCall(this.api, context.key, '/api/agents/connector-migrations', {
|
|
116
|
+
action, operationId: record.id, sourceId: record.sourceId, ...(sealed === undefined ? {} : { sealed }),
|
|
117
|
+
}, { timeoutMs: 60_000 }));
|
|
118
|
+
} catch {
|
|
119
|
+
throw uncertain();
|
|
120
|
+
}
|
|
121
|
+
this.check(context, record);
|
|
122
|
+
const remote = response.migration;
|
|
123
|
+
if (remote.operationId !== record.id || remote.sourceId !== record.sourceId || (record.destinationId !== null && remote.destinationId !== record.destinationId)) throw uncertain();
|
|
124
|
+
if (remote.sourceStamp !== null && remote.sourceStamp !== record.sourceStamp) throw conflict('The prepared destination no longer matches this local connector. Roll back this preparation before trying again.');
|
|
125
|
+
return response;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
private accept(context: Context, record: MigrationRecord, remote: RemoteMigration): MigrationRecord {
|
|
129
|
+
if (remote.phase !== 'preparing' && remote.phase !== 'rolled_back' && (remote.name !== record.name || remote.auth !== record.auth || remote.policy === null || !samePolicy(record.policy, remote.policy))) {
|
|
130
|
+
return this.save(context, { ...record, destinationId: remote.destinationId, reason: 'destination_changed' });
|
|
131
|
+
}
|
|
132
|
+
return this.save(context, { ...record, destinationId: remote.destinationId, reason: reasons[remote.readiness] });
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
private create(context: Context, id: string): MigrationRecord {
|
|
136
|
+
const payload = migrationPayload(migrationSource(id, this.dir));
|
|
137
|
+
return this.save(context, {
|
|
138
|
+
id: newId(), owner: context.owner, agentId: context.agentId, keyId: context.key.keyId,
|
|
139
|
+
sourceId: id, name: payload.name, destinationId: null, phase: 'preparing', auth: payload.auth.kind,
|
|
140
|
+
sourceStamp: payload.sourceStamp, policy: payload.policy, reason: null,
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
private async prepare(context: Context, record: MigrationRecord): Promise<void> {
|
|
145
|
+
const payload = this.source(record);
|
|
146
|
+
const begun = await this.call(context, record, 'begin');
|
|
147
|
+
record = this.accept(context, record, begun.migration);
|
|
148
|
+
if (begun.migration.phase === 'prepared') {
|
|
149
|
+
this.save(context, { ...record, phase: 'prepared' });
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
152
|
+
if (begun.migration.phase !== 'preparing' || begun.receiving === undefined) throw uncertain();
|
|
153
|
+
const receiving = begun.receiving;
|
|
154
|
+
const expected = { owner: context.owner, agent: context.agentId, keyId: context.key.keyId, operationId: record.id, sourceId: record.sourceId, destinationId: record.destinationId };
|
|
155
|
+
for (const [key, value] of Object.entries(expected)) if (receiving.context[key as keyof typeof receiving.context] !== value) throw uncertain();
|
|
156
|
+
const sealed = sealConnectorMigration(receiving.publicKey, receiving.context, payload);
|
|
157
|
+
const { migration } = await this.call(context, record, 'prepare', sealed);
|
|
158
|
+
record = this.accept(context, record, migration);
|
|
159
|
+
if (migration.phase !== 'prepared') throw uncertain();
|
|
160
|
+
this.save(context, { ...record, phase: 'prepared' });
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
private waiting(context: Context, record: MigrationRecord, includeOrganization = false): boolean {
|
|
164
|
+
const uncertain = migrationUncertaintyReceipt(record.sourceId, this.dir) !== null;
|
|
165
|
+
if (!uncertain && this.localIdle(record.sourceId) && (!includeOrganization || this.orgIdle(record.sourceId))) return false;
|
|
166
|
+
this.save(context, { ...record, reason: uncertain ? 'requests_uncertain' : 'waiting_for_local' });
|
|
167
|
+
return true;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
private async cutover(context: Context, record: MigrationRecord): Promise<void> {
|
|
171
|
+
this.context(context.scope, true);
|
|
172
|
+
this.source(record);
|
|
173
|
+
if (record.phase === 'prepared') {
|
|
174
|
+
const { migration } = await this.call(context, record, 'status');
|
|
175
|
+
record = this.accept(context, record, migration);
|
|
176
|
+
if (migration.phase !== 'prepared' || record.reason !== null) return;
|
|
177
|
+
record = this.save(context, { ...record, phase: 'cutover' });
|
|
178
|
+
}
|
|
179
|
+
if (this.waiting(context, record)) return;
|
|
180
|
+
const { migration } = await this.call(context, record, 'activate');
|
|
181
|
+
record = this.accept(context, record, migration);
|
|
182
|
+
if (migration.phase !== 'active' || record.reason !== null) return;
|
|
183
|
+
const snapshot = await (this.deps.fresh ?? freshOrgConnectors)().catch(() => {
|
|
184
|
+
throw new ApiError('Fresh organization delivery could not be verified. The source stays stopped; restore sync and reconcile this migration.', 503);
|
|
185
|
+
});
|
|
186
|
+
this.check(context, record);
|
|
187
|
+
this.context(context.scope, true);
|
|
188
|
+
if (!received(record, snapshot)) {
|
|
189
|
+
this.save(context, { ...record, reason: 'sync_unavailable' });
|
|
190
|
+
return;
|
|
191
|
+
}
|
|
192
|
+
this.save(context, { ...record, phase: 'active', reason: null });
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
private async rollback(context: Context, record: MigrationRecord): Promise<void> {
|
|
196
|
+
this.source(record);
|
|
197
|
+
record = this.save(context, { ...record, phase: 'rollback' });
|
|
198
|
+
if (this.waiting(context, record, true)) return;
|
|
199
|
+
const { migration } = await this.call(context, record, 'rollback');
|
|
200
|
+
if (migration.phase !== 'rolled_back') {
|
|
201
|
+
this.accept(context, record, migration);
|
|
202
|
+
return;
|
|
203
|
+
}
|
|
204
|
+
this.check(context, record);
|
|
205
|
+
if (this.deps.restore !== undefined) this.deps.restore(record.sourceId, record.auth === 'oauth');
|
|
206
|
+
else localRollbackMigrationSource(record.sourceId, record.auth === 'oauth', this.dir);
|
|
207
|
+
this.save(context, { ...record, phase: 'rolled_back', reason: record.auth === 'oauth' ? 'sign_in' : null });
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
private async reconcile(context: Context, record: MigrationRecord): Promise<void> {
|
|
211
|
+
if (record.phase === 'preparing') return this.prepare(context, record);
|
|
212
|
+
if (record.phase === 'cutover') return this.cutover(context, record);
|
|
213
|
+
if (record.phase === 'rollback') return this.rollback(context, record);
|
|
214
|
+
if (record.phase === 'rolled_back') return;
|
|
215
|
+
const { migration } = await this.call(context, record, 'status');
|
|
216
|
+
this.accept(context, record, migration);
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
private obtain(context: Context, sourceId: string, action: MigrationAction): MigrationRecord | null {
|
|
220
|
+
const record = migrationRecord(sourceId, this.dir);
|
|
221
|
+
if (record !== null) this.check(context, record);
|
|
222
|
+
if (action === 'prepare' && (record === null || record.phase === 'rolled_back')) return this.create(context, sourceId);
|
|
223
|
+
if (record === null && action !== 'reconcile') throw conflict('Prepare this connector before moving it.');
|
|
224
|
+
return record;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
private perform(context: Context, record: MigrationRecord, action: MigrationAction): Promise<void> {
|
|
228
|
+
if (action === 'prepare' && record.phase === 'preparing') return this.prepare(context, record);
|
|
229
|
+
if (action === 'cutover' && (record.phase === 'prepared' || record.phase === 'cutover')) return this.cutover(context, record);
|
|
230
|
+
if (action === 'rollback' && record.phase !== 'rolled_back') return this.rollback(context, record);
|
|
231
|
+
return this.reconcile(context, record);
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
private resolve(context: Context, record: MigrationRecord, receipt: string | null): void {
|
|
235
|
+
this.check(context, record);
|
|
236
|
+
if (receipt === null || receipt !== migrationUncertaintyReceipt(record.sourceId, this.dir)) throw conflict('The outstanding request receipt changed. Refresh before confirming that vendor requests have settled.');
|
|
237
|
+
if (!resolveMigrationUncertainty(record.sourceId, receipt, this.dir)) throw conflict('Requests are still running or their receipt changed. Wait for vendor completion, then refresh before confirming settlement.');
|
|
238
|
+
this.save(context, { ...record, reason: record.phase === 'prepared' ? 'not_verified' : null });
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
private validateIntent(sourceId: string, action: MigrationAction, operationId: string | null, receipt: string | null): void {
|
|
242
|
+
if ((migrationRecord(sourceId, this.dir)?.id ?? null) !== operationId) throw conflict('This connector migration changed. Refresh before acting on its current operation.');
|
|
243
|
+
if (action === 'resolve' && (receipt === null || receipt !== migrationUncertaintyReceipt(sourceId, this.dir))) throw conflict('The outstanding request receipt changed. Refresh before confirming that vendor requests have settled.');
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
async run(sourceId: string, action: MigrationAction, scope: string, operationId: string | null, uncertainty: string | null = null): Promise<MigrationStatus> {
|
|
247
|
+
if (this.running) throw conflict('Another migration request is still running on this agent. Refresh after it finishes.');
|
|
248
|
+
const context = this.context(scope, action === 'prepare' || action === 'cutover');
|
|
249
|
+
this.validateIntent(sourceId, action, operationId, uncertainty);
|
|
250
|
+
this.running = true;
|
|
251
|
+
try {
|
|
252
|
+
const record = this.obtain(context, sourceId, action);
|
|
253
|
+
if (record !== null) {
|
|
254
|
+
if (action === 'resolve') this.resolve(context, record, uncertainty);
|
|
255
|
+
else await this.perform(context, record, action);
|
|
256
|
+
}
|
|
257
|
+
this.check(context);
|
|
258
|
+
return this.status();
|
|
259
|
+
} catch (error) {
|
|
260
|
+
const record = migrationRecord(sourceId, this.dir);
|
|
261
|
+
if (record !== null) {
|
|
262
|
+
try { this.save(context, { ...record, reason: 'unavailable' }); } catch {
|
|
263
|
+
log.warn('connector migration: the interrupted operation needs reconciliation in its original enrollment');
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
throw error;
|
|
267
|
+
} finally { this.running = false; }
|
|
268
|
+
}
|
|
269
|
+
}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { ApiError } from '@metro-labs/http/api-error';
|
|
2
|
+
import type { FetchLike } from '@metro-labs/core/stations/oauth';
|
|
3
|
+
import { agentsDir } from '../agents/files.js';
|
|
4
|
+
import type { AgentKey } from './agent-key.js';
|
|
5
|
+
import { migrationRecord, migrationRecords } from './migration-state.js';
|
|
6
|
+
import { assertMigrationOutcomeKnown, beginMigrationOperation, migrationOperationsIdle, migrationRequestEffectful, type MigrationOperation } from './migration-operations.js';
|
|
7
|
+
|
|
8
|
+
const locked = (phase: string): boolean => phase !== 'rolled_back';
|
|
9
|
+
const retired = (phase: string): boolean => phase === 'cutover' || phase === 'active' || phase === 'rollback';
|
|
10
|
+
const refused = (): ApiError => new ApiError('This local connector is locked while it moves to the organization.', 409);
|
|
11
|
+
|
|
12
|
+
export function migrationSource(id: string, dir = agentsDir()): ReturnType<typeof migrationRecords>[number] | null {
|
|
13
|
+
const record = migrationRecord(id, dir);
|
|
14
|
+
return record !== null && locked(record.phase) ? record : null;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function localMigrationEnabled(id: string, dir = agentsDir()): boolean {
|
|
18
|
+
const record = migrationSource(id, dir);
|
|
19
|
+
return record === null || !retired(record.phase);
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function assertLocalMigrationUse(id: string, dir = agentsDir()): void {
|
|
23
|
+
const record = migrationSource(id, dir);
|
|
24
|
+
if (record === null) return;
|
|
25
|
+
if (retired(record.phase)) throw refused();
|
|
26
|
+
assertMigrationOutcomeKnown(id, dir);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function assertLocalMigrationEdit(id: string, name: string | undefined, dir = agentsDir()): void {
|
|
30
|
+
if (migrationRecords(dir).some((record) => locked(record.phase) && (record.sourceId === id || record.name === name))) throw refused();
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function migrationExportableRows<T extends { id: string; name: string }>(rows: readonly T[], dir = agentsDir()): T[] {
|
|
34
|
+
const records = migrationRecords(dir).filter((record) => locked(record.phase));
|
|
35
|
+
return rows.filter((row) => !records.some((record) => record.sourceId === row.id || record.name === row.name));
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export function localMigrationIdle(id: string, dir = agentsDir()): boolean {
|
|
39
|
+
return migrationOperationsIdle(id, dir, migrationSource(id, dir) !== null);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export interface LocalMigrationOperation extends MigrationOperation {
|
|
43
|
+
check: () => void;
|
|
44
|
+
fetch: FetchLike;
|
|
45
|
+
close: () => void;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function beginLocalMigrationOperation(id: string, dir = agentsDir(), edit = false, name?: string): LocalMigrationOperation {
|
|
49
|
+
let closed = false;
|
|
50
|
+
const check = (): void => {
|
|
51
|
+
if (closed) throw refused();
|
|
52
|
+
if (edit) assertLocalMigrationEdit(id, name, dir);
|
|
53
|
+
else assertLocalMigrationUse(id, dir);
|
|
54
|
+
};
|
|
55
|
+
check();
|
|
56
|
+
const tracked = beginMigrationOperation(id, dir, migrationSource(id, dir) !== null);
|
|
57
|
+
const dispatch: MigrationOperation['dispatch'] = (send, options) => {
|
|
58
|
+
check();
|
|
59
|
+
return tracked.dispatch(send, { ...options, protectedSource: migrationSource(id, dir) !== null });
|
|
60
|
+
};
|
|
61
|
+
return {
|
|
62
|
+
check,
|
|
63
|
+
dispatch,
|
|
64
|
+
fetch: (input, init) => dispatch(() => fetch(input, init), { effectful: migrationRequestEffectful(init?.method, init?.body) }),
|
|
65
|
+
close: () => { closed = true; tracked.close(); },
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export async function withLocalMigrationOperation<T>(id: string, dir: string, run: (operation: LocalMigrationOperation) => Promise<T>, edit = false, name?: string): Promise<T> {
|
|
70
|
+
const operation = beginLocalMigrationOperation(id, dir, edit, name);
|
|
71
|
+
try { return await run(operation); }
|
|
72
|
+
finally { operation.close(); }
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export interface MigrationRuntimeMapping {
|
|
76
|
+
aliases: { sourceId: string; destinationId: string; name: string }[];
|
|
77
|
+
reserved: string[];
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export function migrationRuntimeMapping(owner: string | null, key: AgentKey | null, dir = agentsDir()): MigrationRuntimeMapping {
|
|
81
|
+
const records = migrationRecords(dir).filter((record) => locked(record.phase));
|
|
82
|
+
const aliases = records.flatMap((record) => {
|
|
83
|
+
if (record.phase !== 'active' || record.destinationId === null || owner === null || key === null) return [];
|
|
84
|
+
if (record.owner !== owner || record.keyId !== key.keyId || record.owner !== key.enrollment?.organization || record.agentId !== key.enrollment.server) return [];
|
|
85
|
+
return [{ sourceId: record.sourceId, destinationId: record.destinationId, name: record.name }];
|
|
86
|
+
});
|
|
87
|
+
const reserved = records.flatMap((record) => record.destinationId === null ? [] : [record.destinationId]);
|
|
88
|
+
if (new Set(aliases.map((alias) => alias.sourceId)).size !== aliases.length || new Set(reserved).size !== reserved.length) throw refused();
|
|
89
|
+
return { aliases, reserved };
|
|
90
|
+
}
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
import { createHash, randomBytes } from 'node:crypto';
|
|
2
|
+
import { closeSync, existsSync, fsyncSync, openSync, readFileSync, writeFileSync } from 'node:fs';
|
|
3
|
+
import { join, resolve } from 'node:path';
|
|
4
|
+
import { isRecord } from '@metro-labs/core/is-record';
|
|
5
|
+
import { parseId } from '@metro-labs/core/ids';
|
|
6
|
+
import { ensureSecureDir, writeSecure } from '@metro-labs/core/secure-fs';
|
|
7
|
+
import { ApiError } from '@metro-labs/http/api-error';
|
|
8
|
+
import { agentsDir } from '../agents/files.js';
|
|
9
|
+
|
|
10
|
+
interface RequestOperation {
|
|
11
|
+
id: string;
|
|
12
|
+
source: string;
|
|
13
|
+
dir: string;
|
|
14
|
+
lease: string;
|
|
15
|
+
durable: boolean;
|
|
16
|
+
abandoned: boolean;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export interface MigrationOperation {
|
|
20
|
+
dispatch: (send: () => Promise<Response>, options?: { protectedSource?: boolean; effectful?: boolean }) => Promise<Response>;
|
|
21
|
+
close: () => void;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const leases = new Map<string, Set<string>>();
|
|
25
|
+
const requests = new Map<string, RequestOperation>();
|
|
26
|
+
const held = new Map<string, RequestOperation>();
|
|
27
|
+
const keyOf = (source: string, dir: string): string => JSON.stringify([resolve(dir), source]);
|
|
28
|
+
const unavailable = (): ApiError => new ApiError('Protected connector request records cannot be read. Restore their files before using this connector.', 503);
|
|
29
|
+
const uncertain = (): ApiError => new ApiError('A previous connector request has an unknown outcome. Inspect the vendor and acknowledge it before continuing. Do not retry the tool.', 409);
|
|
30
|
+
const CONTROL = new Set(['initialize', 'ping', 'tools/list', 'notifications/initialized', 'notifications/cancelled']);
|
|
31
|
+
|
|
32
|
+
export function migrationRequestEffectful(method: string | undefined, body: unknown): boolean {
|
|
33
|
+
const verb = (method ?? 'GET').toUpperCase();
|
|
34
|
+
if (['GET', 'HEAD', 'DELETE'].includes(verb)) return false;
|
|
35
|
+
if (verb !== 'POST') return true;
|
|
36
|
+
const text = typeof body === 'string' ? body : body instanceof Uint8Array ? new TextDecoder().decode(body) : null;
|
|
37
|
+
if (text === null) return true;
|
|
38
|
+
try {
|
|
39
|
+
const value: unknown = JSON.parse(text);
|
|
40
|
+
const batch: unknown[] = Array.isArray(value) ? value : [value];
|
|
41
|
+
return batch.length === 0 || batch.some((item) => !isRecord(item) || item.jsonrpc !== '2.0' || typeof item.method !== 'string' || !CONTROL.has(item.method));
|
|
42
|
+
} catch { return true; }
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function paths(source: string, dir: string): { data: string; marker: string } {
|
|
46
|
+
if (parseId(source) === null) throw unavailable();
|
|
47
|
+
const data = join(dir, `connector-migration-operations-${source}.json`);
|
|
48
|
+
return { data, marker: `${data}.initialized` };
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function flush(path: string): void {
|
|
52
|
+
const fd = openSync(path, 'r');
|
|
53
|
+
try { fsyncSync(fd); } finally { closeSync(fd); }
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function pendingOf(text: string): string[] {
|
|
57
|
+
const value: unknown = JSON.parse(text);
|
|
58
|
+
if (!isRecord(value) || value.version !== 1 || !Array.isArray(value.pending) || value.pending.length > 10_000) throw unavailable();
|
|
59
|
+
const ids: unknown[] = value.pending;
|
|
60
|
+
if (!ids.every((id): id is string => typeof id === 'string' && /^[a-f0-9]{32}$/.test(id))) throw unavailable();
|
|
61
|
+
if (new Set(ids).size !== ids.length) throw unavailable();
|
|
62
|
+
return ids;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function readPending(source: string, dir: string, initialize = false): string[] {
|
|
66
|
+
const { data, marker } = paths(source, dir);
|
|
67
|
+
try {
|
|
68
|
+
if (initialize && !existsSync(data) && !existsSync(marker)) return [];
|
|
69
|
+
if (readFileSync(marker, 'utf8') !== '1\n') throw unavailable();
|
|
70
|
+
return pendingOf(readFileSync(data, 'utf8'));
|
|
71
|
+
} catch { throw unavailable(); }
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function writePending(source: string, dir: string, pending: string[]): void {
|
|
75
|
+
const { data, marker } = paths(source, dir);
|
|
76
|
+
ensureSecureDir(dir);
|
|
77
|
+
if (!existsSync(marker)) {
|
|
78
|
+
writeFileSync(marker, '1\n', { flag: 'wx', mode: 0o600 });
|
|
79
|
+
flush(marker);
|
|
80
|
+
flush(dir);
|
|
81
|
+
}
|
|
82
|
+
writeSecure(data, `${JSON.stringify({ version: 1, pending })}\n`);
|
|
83
|
+
flush(data);
|
|
84
|
+
flush(dir);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export function captureMigrationOperations(source: string, dir: string, initialize: boolean): void {
|
|
88
|
+
const pending = new Set(readPending(source, dir, initialize));
|
|
89
|
+
const active = [...held.values()].filter((request) => keyOf(request.source, request.dir) === keyOf(source, dir));
|
|
90
|
+
for (const request of active) pending.add(request.id);
|
|
91
|
+
writePending(source, dir, [...pending]);
|
|
92
|
+
for (const request of active) request.durable = true;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export function migrationUncertaintyReceipt(source: string, dir = agentsDir()): string | null {
|
|
96
|
+
const unknown = readPending(source, dir).filter((id) => !requests.has(id)).sort();
|
|
97
|
+
return unknown.length === 0 ? null : createHash('sha256').update(JSON.stringify([1, source, unknown])).digest('hex');
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export function assertMigrationOutcomeKnown(source: string, dir = agentsDir()): void {
|
|
101
|
+
if (migrationUncertaintyReceipt(source, dir) !== null) throw uncertain();
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
export function migrationOperationsIdle(source: string, dir: string, protectedSource: boolean): boolean {
|
|
105
|
+
if (leases.has(keyOf(source, dir))) return false;
|
|
106
|
+
return !protectedSource || readPending(source, dir).length === 0;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export function resolveMigrationUncertainty(source: string, expectedReceipt: string, dir = agentsDir()): boolean {
|
|
110
|
+
if (leases.has(keyOf(source, dir))) return false;
|
|
111
|
+
const receipt = migrationUncertaintyReceipt(source, dir);
|
|
112
|
+
if (receipt === null || receipt !== expectedReceipt) return false;
|
|
113
|
+
writePending(source, dir, []);
|
|
114
|
+
return true;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function finish(request: RequestOperation, completed: boolean): void {
|
|
118
|
+
if (request.abandoned) return;
|
|
119
|
+
requests.delete(request.id);
|
|
120
|
+
if (completed) held.delete(request.id);
|
|
121
|
+
if (completed && request.durable) writePending(request.source, request.dir, readPending(request.source, request.dir).filter((id) => id !== request.id));
|
|
122
|
+
request.abandoned = true;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function observed(response: Response, request: RequestOperation): Response {
|
|
126
|
+
if (response.status === 401 || response.status === 403) {
|
|
127
|
+
finish(request, true);
|
|
128
|
+
return response;
|
|
129
|
+
}
|
|
130
|
+
if (response.body === null) {
|
|
131
|
+
finish(request, response.ok);
|
|
132
|
+
return response;
|
|
133
|
+
}
|
|
134
|
+
const reader = response.body.getReader();
|
|
135
|
+
const body = new ReadableStream<Uint8Array>({
|
|
136
|
+
async pull(controller) {
|
|
137
|
+
try {
|
|
138
|
+
const result = await reader.read();
|
|
139
|
+
if (result.done) {
|
|
140
|
+
finish(request, response.ok);
|
|
141
|
+
controller.close();
|
|
142
|
+
} else controller.enqueue(result.value);
|
|
143
|
+
} catch (error) {
|
|
144
|
+
finish(request, false);
|
|
145
|
+
controller.error(error);
|
|
146
|
+
}
|
|
147
|
+
},
|
|
148
|
+
async cancel(reason: unknown) {
|
|
149
|
+
finish(request, false);
|
|
150
|
+
await reader.cancel(reason);
|
|
151
|
+
},
|
|
152
|
+
});
|
|
153
|
+
return new Response(body, { status: response.status, statusText: response.statusText, headers: response.headers });
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
async function dispatch(source: string, dir: string, lease: string, protectedSource: boolean, effectful: boolean, send: () => Promise<Response>): Promise<Response> {
|
|
157
|
+
if (protectedSource) assertMigrationOutcomeKnown(source, dir);
|
|
158
|
+
if (!effectful) return send();
|
|
159
|
+
const request: RequestOperation = { id: randomBytes(16).toString('hex'), source, dir, lease, durable: false, abandoned: false };
|
|
160
|
+
if (protectedSource) {
|
|
161
|
+
const pending = readPending(source, dir);
|
|
162
|
+
if (pending.length >= 10_000) throw unavailable();
|
|
163
|
+
writePending(source, dir, [...pending, request.id]);
|
|
164
|
+
request.durable = true;
|
|
165
|
+
}
|
|
166
|
+
requests.set(request.id, request);
|
|
167
|
+
held.set(request.id, request);
|
|
168
|
+
try { return observed(await send(), request); }
|
|
169
|
+
catch (error) { finish(request, false); throw error; }
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
export function beginMigrationOperation(source: string, dir: string, protectedSource: boolean): MigrationOperation {
|
|
173
|
+
if (protectedSource) assertMigrationOutcomeKnown(source, dir);
|
|
174
|
+
const key = keyOf(source, dir);
|
|
175
|
+
const lease = randomBytes(16).toString('hex');
|
|
176
|
+
const active = leases.get(key) ?? new Set<string>();
|
|
177
|
+
active.add(lease);
|
|
178
|
+
leases.set(key, active);
|
|
179
|
+
let closed = false;
|
|
180
|
+
return {
|
|
181
|
+
dispatch: (send, options = {}) => {
|
|
182
|
+
if (closed) return Promise.reject(uncertain());
|
|
183
|
+
return dispatch(source, dir, lease, options.protectedSource ?? protectedSource, options.effectful ?? true, send);
|
|
184
|
+
},
|
|
185
|
+
close: () => {
|
|
186
|
+
if (closed) return;
|
|
187
|
+
closed = true;
|
|
188
|
+
for (const request of held.values()) if (request.lease === lease) {
|
|
189
|
+
finish(request, false);
|
|
190
|
+
held.delete(request.id);
|
|
191
|
+
}
|
|
192
|
+
active.delete(lease);
|
|
193
|
+
if (active.size === 0) leases.delete(key);
|
|
194
|
+
},
|
|
195
|
+
};
|
|
196
|
+
}
|