@stage-labs/metro 0.1.0-beta.291 → 0.1.0-beta.293
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/runtime/marketplace/plugin/.claude-plugin/plugin.json +1 -1
- package/runtime/marketplace/plugin/bin/policy.mjs +4 -1
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/config.ts +2 -0
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/gates.ts +27 -23
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/migration-api.ts +1 -1
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/migration-control.ts +107 -30
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/migration-guard.ts +17 -13
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/migration-operations.ts +36 -6
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/migration-preview.ts +29 -13
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/migration-source.ts +18 -2
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/migration-state.ts +20 -3
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/oauth-routes.ts +2 -2
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/oauth.ts +2 -1
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/plugin-sync.ts +39 -12
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/publication.ts +22 -11
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/relay-passive.ts +96 -0
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/relay.ts +19 -10
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/server-keys.ts +46 -0
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/store.ts +21 -24
- package/runtime/node_modules/@metro-labs/daemon/src/connectors/transfer-store.ts +1 -4
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/permission-prompt.ts +6 -2
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/policy-gate.ts +1 -2
- package/runtime/node_modules/@metro-labs/http/package.json +2 -1
- package/runtime/node_modules/@metro-labs/http/src/connector-grant.ts +59 -0
- package/runtime/node_modules/@metro-labs/http/src/connector-mcp.ts +2 -0
- package/runtime/node_modules/@metro-labs/http/src/connector-migration.ts +23 -16
- package/runtime/runtime.json +1 -1
- package/runtime/sdk-runner/runner.json +1 -1
|
@@ -5,6 +5,7 @@ import { isRecord } from '@metro-labs/core/is-record';
|
|
|
5
5
|
import { ApiError } from '@metro-labs/http/api-error';
|
|
6
6
|
import { bodyField } from '@metro-labs/http/api-http';
|
|
7
7
|
import { connectorAuth, connectorClient, connectorName } from '@metro-labs/http/connector-input';
|
|
8
|
+
import { portableGrant } from '@metro-labs/http/connector-grant';
|
|
8
9
|
import { connectorPolicyOf } from '@metro-labs/http/connector-delivery';
|
|
9
10
|
import { readConfig } from './config.js';
|
|
10
11
|
import type { LocalConnectorRow } from './store.js';
|
|
@@ -12,9 +13,10 @@ import { agentsDir } from '../agents/files.js';
|
|
|
12
13
|
import { normalizePolicy, type ToolPolicy } from '../policy/policy.js';
|
|
13
14
|
import { connectorSyncScope, syncOptedIn, syncSetting } from './org-sync-setting.js';
|
|
14
15
|
import { readAgentKey } from './agent-key.js';
|
|
16
|
+
import { MigrationBlocked } from './migration-source.js';
|
|
15
17
|
import { migrationOrigin } from './migration-url.js';
|
|
16
18
|
|
|
17
|
-
export type MigrationIssue = 'invalid_config' | 'unsupported_url' | 'custom_policy' | 'oauth_sign_in' | 'header_reentry' | 'unused_oauth_client';
|
|
19
|
+
export type MigrationIssue = 'invalid_config' | 'unsupported_url' | 'custom_policy' | 'oauth_sign_in' | 'header_reentry' | 'unused_oauth_client' | 'oauth_unsupported';
|
|
18
20
|
|
|
19
21
|
export interface MigrationConnector {
|
|
20
22
|
id: string;
|
|
@@ -29,6 +31,7 @@ export interface MigrationConnector {
|
|
|
29
31
|
export interface ConnectorMigrationPreview {
|
|
30
32
|
migrationSupported: true;
|
|
31
33
|
oauthResourceSupported: true;
|
|
34
|
+
moveAllSupported: true;
|
|
32
35
|
enabled: boolean;
|
|
33
36
|
enrolled: boolean;
|
|
34
37
|
agentId: string | null;
|
|
@@ -54,7 +57,7 @@ function validClient(raw: unknown): boolean {
|
|
|
54
57
|
}
|
|
55
58
|
|
|
56
59
|
function oauthAuth(raw: Record<string, unknown>): 'oauth' {
|
|
57
|
-
if (!onlyKeys(raw, ['kind', 'accessToken', 'refreshToken', 'clientId', 'clientSecret', 'tokenEndpoint', 'issuer', 'scope', 'expiresAt'])) invalid();
|
|
60
|
+
if (!onlyKeys(raw, ['kind', 'accessToken', 'refreshToken', 'clientId', 'clientSecret', 'tokenEndpoint', 'issuer', 'scope', 'expiresAt', 'resource', 'tokenEndpointAuthMethod'])) invalid();
|
|
58
61
|
return 'oauth';
|
|
59
62
|
}
|
|
60
63
|
|
|
@@ -145,8 +148,17 @@ function rowIdentity(row: Record<string, unknown>, index: number): { id: string;
|
|
|
145
148
|
return { id: id ?? `invalid-${String(index + 1)}`, name: name ?? `Connector ${String(index + 1)}`, valid: id !== null && name !== null };
|
|
146
149
|
}
|
|
147
150
|
|
|
151
|
+
function grantIssues(row: Record<string, unknown>): MigrationIssue[] {
|
|
152
|
+
const grant = bodyField(row.config, 'auth');
|
|
153
|
+
if (!isRecord(grant) || grant.kind !== 'oauth') return ['oauth_sign_in'];
|
|
154
|
+
try {
|
|
155
|
+
const portable = portableGrant(grant, String(row.url));
|
|
156
|
+
return portable.expiresAt !== undefined && portable.expiresAt <= Date.now() && portable.refreshToken === undefined ? ['oauth_sign_in'] : [];
|
|
157
|
+
} catch { return ['oauth_unsupported']; }
|
|
158
|
+
}
|
|
159
|
+
|
|
148
160
|
function authIssues(row: Record<string, unknown>, auth: MigrationConnector['auth']): MigrationIssue[] {
|
|
149
|
-
if (auth === 'oauth') return
|
|
161
|
+
if (auth === 'oauth') return grantIssues(row);
|
|
150
162
|
const client = bodyField(row.config, 'client');
|
|
151
163
|
return (auth === 'none' || auth === 'header') && client !== undefined && client !== null ? ['unused_oauth_client'] : [];
|
|
152
164
|
}
|
|
@@ -189,21 +201,25 @@ function previewRows(dir: string): MigrationConnector[] {
|
|
|
189
201
|
const rows = readRows(dir).map(previewRow);
|
|
190
202
|
const seen = new Map<string, MigrationConnector>();
|
|
191
203
|
for (const row of rows) {
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
for (const duplicate of [prior, row]) if (!duplicate.issues.includes('invalid_config')) duplicate.issues.push('invalid_config');
|
|
196
|
-
}
|
|
197
|
-
seen.set(key, row);
|
|
204
|
+
const prior = seen.get(row.id);
|
|
205
|
+
if (prior !== undefined) {
|
|
206
|
+
for (const duplicate of [prior, row]) if (!duplicate.issues.includes('invalid_config')) duplicate.issues.push('invalid_config');
|
|
198
207
|
}
|
|
208
|
+
seen.set(row.id, row);
|
|
199
209
|
}
|
|
200
210
|
return rows;
|
|
201
211
|
}
|
|
202
212
|
|
|
203
|
-
|
|
213
|
+
function assertTransferable(source: MigrationConnector): void {
|
|
214
|
+
if (source.issues.includes('oauth_sign_in')) throw new MigrationBlocked('sign_in', 'No usable local OAuth grant is saved. Sign in locally before moving it.');
|
|
215
|
+
if (source.issues.includes('oauth_unsupported')) throw new MigrationBlocked('unsupported_grant', 'This OAuth grant has unsupported client authentication or provider binding. Keep it local or sign in separately.');
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
export function migrationSource(id: string, dir = agentsDir(), transfer = false): LocalConnectorRow {
|
|
204
219
|
const source = previewRows(dir).find((row) => row.id === id);
|
|
205
220
|
if (source === undefined) throw new ApiError('The local connector is missing.', 404);
|
|
206
|
-
if (source.issues.some((issue) => issue !== 'oauth_sign_in')) throw new ApiError('This connector cannot be migrated without changing its settings or permissions. Resolve the preview blockers first.', 409);
|
|
221
|
+
if (source.issues.some((issue) => issue !== 'oauth_sign_in' && issue !== 'oauth_unsupported')) throw new ApiError('This connector cannot be migrated without changing its settings or permissions. Resolve the preview blockers first.', 409);
|
|
222
|
+
if (transfer) assertTransferable(source);
|
|
207
223
|
const raw = readRows(dir).find((row) => isRecord(row) && row.id === id);
|
|
208
224
|
if (!isRecord(raw) || typeof raw.url !== 'string') throw unavailable();
|
|
209
225
|
return { id, name: source.name, url: raw.url, config: { ...readConfig(raw.config), oauth: source.auth === 'oauth', policy: source.policy } };
|
|
@@ -212,8 +228,8 @@ export function migrationSource(id: string, dir = agentsDir()): LocalConnectorRo
|
|
|
212
228
|
export function connectorMigrationPreview(owner: string | null, dir = agentsDir()): ConnectorMigrationPreview {
|
|
213
229
|
const enabled = syncOptedIn(dir) && syncSetting(dir, connectorSyncScope(owner, readAgentKey(dir))).enabled;
|
|
214
230
|
const liveEnabled = process.env.METRO_CONNECTORS_LIVE_ENABLED === 'true';
|
|
215
|
-
if (!enabled) return { migrationSupported: true, oauthResourceSupported: true, enabled, enrolled: false, agentId: null, owner, liveEnabled, connectors: [] };
|
|
231
|
+
if (!enabled) return { migrationSupported: true, oauthResourceSupported: true, moveAllSupported: true, enabled, enrolled: false, agentId: null, owner, liveEnabled, connectors: [] };
|
|
216
232
|
const enrollment = owner === null ? null : readAgentKey(dir)?.enrollment;
|
|
217
233
|
const agentId = enrollment?.organization === owner ? enrollment?.server ?? null : null;
|
|
218
|
-
return { migrationSupported: true, oauthResourceSupported: true, enabled, enrolled: agentId !== null, agentId, owner, liveEnabled, connectors: previewRows(dir) };
|
|
234
|
+
return { migrationSupported: true, oauthResourceSupported: true, moveAllSupported: true, enabled, enrolled: agentId !== null, agentId, owner, liveEnabled, connectors: previewRows(dir) };
|
|
219
235
|
}
|
|
@@ -2,12 +2,17 @@ import { createHash } from 'node:crypto';
|
|
|
2
2
|
import type { MigrationPayload } from '@metro-labs/http/connector-migration';
|
|
3
3
|
import { connectorPolicyOf } from '@metro-labs/http/connector-delivery';
|
|
4
4
|
import { ApiError } from '@metro-labs/http/api-error';
|
|
5
|
+
import { portableGrant } from '@metro-labs/http/connector-grant';
|
|
5
6
|
import { boundOAuthResource } from '@metro-labs/http/connector-public-url';
|
|
6
7
|
import type { LocalConnectorRow } from './store.js';
|
|
7
8
|
|
|
8
|
-
export
|
|
9
|
+
export class MigrationBlocked extends ApiError {
|
|
10
|
+
constructor(readonly reason: string, message: string) { super(message, 409); }
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export function migrationPayload(row: LocalConnectorRow, oauthResource?: string, transfer = false): MigrationPayload {
|
|
9
14
|
const oauth = row.config.oauth || row.config.auth.kind === 'oauth';
|
|
10
|
-
const auth =
|
|
15
|
+
const auth = sourceAuth(row, oauth, transfer);
|
|
11
16
|
if (!oauth && oauthResource !== undefined) throw new ApiError('Only an OAuth connector accepts a public OAuth resource.', 400);
|
|
12
17
|
return stamped({
|
|
13
18
|
name: row.name, url: row.url, auth, client: row.config.client, policy: connectorPolicyOf(row.config.policy ?? {}),
|
|
@@ -15,6 +20,11 @@ export function migrationPayload(row: LocalConnectorRow, oauthResource?: string)
|
|
|
15
20
|
});
|
|
16
21
|
}
|
|
17
22
|
|
|
23
|
+
function sourceAuth(row: LocalConnectorRow, oauth: boolean, transfer: boolean): MigrationPayload['auth'] {
|
|
24
|
+
if (!oauth) return row.config.auth;
|
|
25
|
+
return transfer ? { kind: 'oauth', grant: transferableGrant(row) } : { kind: 'oauth' };
|
|
26
|
+
}
|
|
27
|
+
|
|
18
28
|
function stamped(value: Omit<MigrationPayload, 'sourceStamp'>): MigrationPayload {
|
|
19
29
|
const policy = value.policy;
|
|
20
30
|
const canonical = {
|
|
@@ -24,3 +34,9 @@ function stamped(value: Omit<MigrationPayload, 'sourceStamp'>): MigrationPayload
|
|
|
24
34
|
const sourceStamp = createHash('sha256').update(JSON.stringify(canonical)).digest('base64url');
|
|
25
35
|
return { ...value, sourceStamp };
|
|
26
36
|
}
|
|
37
|
+
|
|
38
|
+
function transferableGrant(row: LocalConnectorRow): ReturnType<typeof portableGrant> {
|
|
39
|
+
if (row.config.auth.kind !== 'oauth') throw new MigrationBlocked('sign_in', 'This connector has no saved OAuth grant. Sign in locally before moving it.');
|
|
40
|
+
try { return portableGrant(row.config.auth, row.url); }
|
|
41
|
+
catch { throw new MigrationBlocked('unsupported_grant', 'This OAuth grant has unsupported client authentication or provider binding. Keep it local or sign in separately.'); }
|
|
42
|
+
}
|
|
@@ -10,7 +10,7 @@ import { publicOAuthResource } from '@metro-labs/http/connector-public-url';
|
|
|
10
10
|
import { agentsDir } from '../agents/files.js';
|
|
11
11
|
import { captureMigrationOperations } from './migration-operations.js';
|
|
12
12
|
|
|
13
|
-
export type MigrationPhase = 'preparing' | 'prepared' | 'cutover' | 'active' | 'rollback' | 'rolled_back';
|
|
13
|
+
export type MigrationPhase = 'recovering' | 'freezing' | 'preparing' | 'prepared' | 'cutover' | 'active' | 'rollback' | 'rolled_back';
|
|
14
14
|
export type MigrationAuth = 'none' | 'header' | 'oauth';
|
|
15
15
|
|
|
16
16
|
export interface MigrationRecord {
|
|
@@ -24,14 +24,18 @@ export interface MigrationRecord {
|
|
|
24
24
|
phase: MigrationPhase;
|
|
25
25
|
auth: MigrationAuth;
|
|
26
26
|
oauthResource?: string;
|
|
27
|
+
move?: boolean;
|
|
28
|
+
grantImported?: boolean;
|
|
29
|
+
captured?: boolean;
|
|
30
|
+
recoveryReceipt?: string;
|
|
27
31
|
sourceStamp: string;
|
|
28
32
|
policy: ConnectorPolicy;
|
|
29
33
|
reason: string | null;
|
|
30
34
|
}
|
|
31
35
|
|
|
32
|
-
const phases: readonly MigrationPhase[] = ['preparing', 'prepared', 'cutover', 'active', 'rollback', 'rolled_back'];
|
|
36
|
+
const phases: readonly MigrationPhase[] = ['recovering', 'freezing', 'preparing', 'prepared', 'cutover', 'active', 'rollback', 'rolled_back'];
|
|
33
37
|
const auths: readonly MigrationAuth[] = ['none', 'header', 'oauth'];
|
|
34
|
-
const fields = ['id', 'owner', 'agentId', 'keyId', 'sourceId', 'name', 'destinationId', 'phase', 'auth', 'oauthResource', 'sourceStamp', 'policy', 'reason'];
|
|
38
|
+
const fields = ['id', 'owner', 'agentId', 'keyId', 'sourceId', 'name', 'destinationId', 'phase', 'auth', 'oauthResource', 'sourceStamp', 'policy', 'reason', 'move', 'grantImported', 'captured', 'recoveryReceipt'];
|
|
35
39
|
const unavailable = (): ApiError => new ApiError('Protected connector migration records cannot be read. Restore their file or permissions before using connectors.', 503);
|
|
36
40
|
const DIGEST = /^[A-Za-z0-9_-]{43}$/;
|
|
37
41
|
const pathOf = (dir: string): string => join(dir, 'connector-migrations.json');
|
|
@@ -48,12 +52,25 @@ function field(value: unknown, pattern: RegExp): string {
|
|
|
48
52
|
return value;
|
|
49
53
|
}
|
|
50
54
|
|
|
55
|
+
function movementOf(value: Record<string, unknown>): Pick<MigrationRecord, 'move' | 'grantImported' | 'captured' | 'recoveryReceipt'> {
|
|
56
|
+
const result: Pick<MigrationRecord, 'move' | 'grantImported' | 'captured' | 'recoveryReceipt'> = {};
|
|
57
|
+
for (const key of ['move', 'grantImported', 'captured'] as const) {
|
|
58
|
+
const flag = value[key];
|
|
59
|
+
if (flag === undefined) continue;
|
|
60
|
+
if (typeof flag !== 'boolean') throw unavailable();
|
|
61
|
+
result[key] = flag;
|
|
62
|
+
}
|
|
63
|
+
if (value.recoveryReceipt !== undefined) result.recoveryReceipt = field(value.recoveryReceipt, /^[a-f0-9]{64}$/);
|
|
64
|
+
return result;
|
|
65
|
+
}
|
|
66
|
+
|
|
51
67
|
function recordOf(value: unknown): MigrationRecord {
|
|
52
68
|
if (!isRecord(value) || Object.keys(value).some((key) => !fields.includes(key))) throw unavailable();
|
|
53
69
|
const phase = phases.find((item) => item === value.phase);
|
|
54
70
|
const auth = auths.find((item) => item === value.auth);
|
|
55
71
|
if (phase === undefined || auth === undefined || (value.oauthResource !== undefined && auth !== 'oauth')) throw unavailable();
|
|
56
72
|
return {
|
|
73
|
+
...movementOf(value),
|
|
57
74
|
...(value.oauthResource === undefined ? {} : { oauthResource: publicOAuthResource(value.oauthResource) }),
|
|
58
75
|
id: requiredId(value.id), sourceId: requiredId(value.sourceId), agentId: requiredId(value.agentId),
|
|
59
76
|
owner: field(value.owner, /^org_[A-Za-z0-9]{1,128}$/), keyId: field(value.keyId, DIGEST), sourceStamp: field(value.sourceStamp, DIGEST),
|
|
@@ -51,7 +51,7 @@ export async function startOAuth(
|
|
|
51
51
|
returnTo,
|
|
52
52
|
}, operation.fetch);
|
|
53
53
|
const row = await deps.createPendingConnector({
|
|
54
|
-
name
|
|
54
|
+
name,
|
|
55
55
|
url: bodyField(body, 'url'),
|
|
56
56
|
clientId,
|
|
57
57
|
clientSecret,
|
|
@@ -69,7 +69,7 @@ export async function startOAuth(
|
|
|
69
69
|
'connector-api: stored unconnected, sending the user to sign in',
|
|
70
70
|
);
|
|
71
71
|
sendJson(req, res, 201, { ...payload(row), authorizeUrl: authorize });
|
|
72
|
-
}, true
|
|
72
|
+
}, true);
|
|
73
73
|
}
|
|
74
74
|
|
|
75
75
|
export async function handleConnect(
|
|
@@ -102,6 +102,7 @@ function authOf(pendingAuth: PendingAuth, tokens: OAuthTokens): OAuthAuth {
|
|
|
102
102
|
kind: 'oauth',
|
|
103
103
|
issuer: pendingAuth.server.issuer,
|
|
104
104
|
tokenEndpoint: pendingAuth.server.tokenEndpoint,
|
|
105
|
+
resource: pendingAuth.resource,
|
|
105
106
|
clientId: pendingAuth.client.clientId,
|
|
106
107
|
...(pendingAuth.client.clientSecret === undefined
|
|
107
108
|
? {}
|
|
@@ -156,7 +157,7 @@ export async function refreshOAuth(
|
|
|
156
157
|
server,
|
|
157
158
|
client,
|
|
158
159
|
auth.refreshToken,
|
|
159
|
-
resource,
|
|
160
|
+
auth.resource ?? resource,
|
|
160
161
|
auth.scope,
|
|
161
162
|
send,
|
|
162
163
|
);
|
|
@@ -33,21 +33,48 @@ export function serverKey(name: string): string {
|
|
|
33
33
|
return slug === '' ? 'connector' : slug;
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
-
|
|
36
|
+
function legacyOwners(rows: PluginRow[], published: PluginRow[]): Map<string, string> {
|
|
37
|
+
const visible = new Set(published.map((row) => row.id));
|
|
37
38
|
const taken = new Map<string, number>();
|
|
38
|
-
const
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
const
|
|
42
|
-
taken.
|
|
43
|
-
|
|
39
|
+
const owners = new Map<string, string>();
|
|
40
|
+
const reserved = new Set<string>();
|
|
41
|
+
for (const row of rows.filter((entry) => !isOrgRelayId(entry.id))) {
|
|
42
|
+
const slug = serverKey(row.name);
|
|
43
|
+
const seen = taken.get(slug) ?? 0;
|
|
44
|
+
taken.set(slug, seen + 1);
|
|
45
|
+
const key = seen === 0 ? slug : `${slug}-${row.id.toLowerCase().replace(/[^a-z0-9]/g, '').slice(0, 4)}`;
|
|
46
|
+
reserved.add(key);
|
|
47
|
+
if (visible.has(row.id)) owners.set(key, row.id);
|
|
48
|
+
}
|
|
49
|
+
for (const row of rows.filter((entry) => isOrgRelayId(entry.id) && visible.has(entry.id)).sort((a, b) => a.id.localeCompare(b.id))) {
|
|
50
|
+
const key = availableKey(orgKey(row), reserved);
|
|
51
|
+
reserved.add(key);
|
|
52
|
+
owners.set(key, row.id);
|
|
44
53
|
}
|
|
54
|
+
return new Map([...owners].map(([key, id]) => [id, key]));
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const orgKey = (row: PluginRow): string => `org-${serverKey(row.name).slice(0, 35)}-${Buffer.from(row.id.slice(4)).toString('hex')}`;
|
|
58
|
+
|
|
59
|
+
function availableKey(wanted: string, reserved: ReadonlySet<string>): string {
|
|
60
|
+
let key = wanted;
|
|
61
|
+
let suffix = 1;
|
|
62
|
+
while (reserved.has(key)) key = `${wanted}-${String(suffix++)}`;
|
|
63
|
+
return key;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export function serverKeysOf(rows: PluginRow[], saved?: ReadonlyMap<string, string>, published: PluginRow[] = rows): Map<string, string> {
|
|
67
|
+
const keys = new Map(saved ?? legacyOwners(rows, published));
|
|
45
68
|
const reserved = new Set(keys.values());
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
69
|
+
const local = rows.filter((row) => !isOrgRelayId(row.id));
|
|
70
|
+
const organization = rows.filter((row) => isOrgRelayId(row.id)).sort((a, b) => a.id.localeCompare(b.id));
|
|
71
|
+
for (const row of [...local, ...organization]) {
|
|
72
|
+
if (keys.has(row.id)) continue;
|
|
73
|
+
const slug = serverKey(row.name);
|
|
74
|
+
const wanted = isOrgRelayId(row.id)
|
|
75
|
+
? orgKey(row)
|
|
76
|
+
: reserved.has(slug) ? `${slug}-${row.id.toLowerCase().replace(/[^a-z0-9]/g, '').slice(0, 4)}` : slug;
|
|
77
|
+
const key = availableKey(wanted, reserved);
|
|
51
78
|
reserved.add(key);
|
|
52
79
|
keys.set(row.id, key);
|
|
53
80
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { registerConnectors, type GateRow } from './gates.js';
|
|
2
|
-
import {
|
|
2
|
+
import { syncPluginServers } from './plugin-sync.js';
|
|
3
|
+
import { persistentServerKeys } from './server-keys.js';
|
|
3
4
|
import { isOrgRelayId } from './relay-id.js';
|
|
4
5
|
import { migrationRecords } from './migration-state.js';
|
|
5
6
|
|
|
@@ -17,14 +18,7 @@ function metadata(rows: readonly GateRow[]): GateRow[] {
|
|
|
17
18
|
}));
|
|
18
19
|
}
|
|
19
20
|
|
|
20
|
-
|
|
21
|
-
let records: ReturnType<typeof migrationRecords>;
|
|
22
|
-
try { records = migrationRecords().filter((record) => record.phase !== 'rolled_back'); }
|
|
23
|
-
catch (error) {
|
|
24
|
-
registerConnectors([], local.map((row) => row.id));
|
|
25
|
-
syncPluginServers([]);
|
|
26
|
-
throw error;
|
|
27
|
-
}
|
|
21
|
+
function publishedRows(records: ReturnType<typeof migrationRecords>): GateRow[] {
|
|
28
22
|
const retired = new Set(records.filter((record) => !['preparing', 'prepared'].includes(record.phase)).map((record) => record.sourceId));
|
|
29
23
|
const aliases = new Map(organization.filter((row) => records.some((record) => record.phase === 'active' && record.sourceId === row.id && record.name === row.name)).map((row) => [row.id, row]));
|
|
30
24
|
const rows = local.flatMap((row) => {
|
|
@@ -32,8 +26,25 @@ export function publishConnectors(): void {
|
|
|
32
26
|
return alias !== undefined ? [alias] : retired.has(row.id) ? [] : [row];
|
|
33
27
|
});
|
|
34
28
|
rows.push(...organization.filter((row) => isOrgRelayId(row.id) && !records.some((record) => row.id === `org-${record.destinationId ?? ''}`)));
|
|
35
|
-
|
|
36
|
-
|
|
29
|
+
return rows;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export function publishConnectors(): void {
|
|
33
|
+
let rows: GateRow[];
|
|
34
|
+
let keys: Map<string, string>;
|
|
35
|
+
try {
|
|
36
|
+
const records = migrationRecords().filter((record) => record.phase !== 'rolled_back');
|
|
37
|
+
rows = publishedRows(records);
|
|
38
|
+
const active = new Set(records.filter((record) => record.phase === 'active').map((record) => record.sourceId));
|
|
39
|
+
const reserved = [...rows, ...local.filter((row) => active.has(row.id))];
|
|
40
|
+
keys = persistentServerKeys([...local, ...organization.filter((row) => isOrgRelayId(row.id))], undefined, reserved);
|
|
41
|
+
}
|
|
42
|
+
catch (error) {
|
|
43
|
+
registerConnectors([]);
|
|
44
|
+
syncPluginServers([]);
|
|
45
|
+
throw error;
|
|
46
|
+
}
|
|
47
|
+
registerConnectors(rows, keys);
|
|
37
48
|
syncPluginServers(rows, { keys });
|
|
38
49
|
}
|
|
39
50
|
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import type { IncomingMessage, ServerResponse } from 'node:http';
|
|
2
|
+
import { JSONRPCNotificationSchema, ServerNotificationSchema } from '@modelcontextprotocol/sdk/types.js';
|
|
3
|
+
import { isRecord } from '@metro-labs/core/is-record';
|
|
4
|
+
import type { MigrationOperation } from './migration-operations.js';
|
|
5
|
+
|
|
6
|
+
export const isRelaySse = (contentType: string | null | undefined): boolean => /^text\/event-stream\s*(?:;|$)/i.test(contentType ?? '');
|
|
7
|
+
const MAX_FRAME = 64 * 1024;
|
|
8
|
+
const NOTIFICATIONS = new Set(['notifications/tools/list_changed', 'notifications/resources/list_changed', 'notifications/prompts/list_changed', 'notifications/resources/updated', 'notifications/message']);
|
|
9
|
+
|
|
10
|
+
export class PassiveSseListener {
|
|
11
|
+
forwarding = false;
|
|
12
|
+
private pending: Buffer[] = [];
|
|
13
|
+
private length = 0;
|
|
14
|
+
private data: string[] = [];
|
|
15
|
+
private fields = false;
|
|
16
|
+
private size = 0;
|
|
17
|
+
private unsafe = false;
|
|
18
|
+
private skipLf = false;
|
|
19
|
+
private first = true;
|
|
20
|
+
private readonly decoder = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true });
|
|
21
|
+
|
|
22
|
+
idle(): boolean { return !this.unsafe && this.length === 0 && !this.fields && !this.forwarding; }
|
|
23
|
+
keepaliveAllowed(): boolean { return this.unsafe || this.idle(); }
|
|
24
|
+
|
|
25
|
+
observe(chunk: Uint8Array): void {
|
|
26
|
+
if (this.unsafe) return;
|
|
27
|
+
this.scan(chunk);
|
|
28
|
+
if (this.unsafe) { this.pending = []; this.length = 0; this.data = []; }
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
private append(bytes: Uint8Array): void {
|
|
32
|
+
this.length += bytes.length;
|
|
33
|
+
if (this.length + this.size > MAX_FRAME) this.unsafe = true;
|
|
34
|
+
else if (bytes.length > 0) this.pending.push(Buffer.from(bytes));
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
private scan(bytes: Uint8Array): void {
|
|
38
|
+
let start = 0;
|
|
39
|
+
for (let at = 0; at < bytes.length; at += 1) {
|
|
40
|
+
if (at - start + this.length + this.size > MAX_FRAME) { this.unsafe = true; return; }
|
|
41
|
+
const byte = bytes[at];
|
|
42
|
+
if (this.skipLf && byte === 10) { this.skipLf = false; start = at + 1; continue; }
|
|
43
|
+
this.skipLf = false;
|
|
44
|
+
if (byte !== 10 && byte !== 13) continue;
|
|
45
|
+
this.append(bytes.subarray(start, at));
|
|
46
|
+
try { this.line(this.decoder.decode(Buffer.concat(this.pending, this.length)), this.length); }
|
|
47
|
+
catch { this.unsafe = true; }
|
|
48
|
+
this.pending = []; this.length = 0;
|
|
49
|
+
this.skipLf = byte === 13;
|
|
50
|
+
start = at + 1;
|
|
51
|
+
if (this.unsafe) return;
|
|
52
|
+
}
|
|
53
|
+
this.append(bytes.subarray(start));
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
private line(raw: string, length: number): void {
|
|
57
|
+
const line = this.first && raw.charCodeAt(0) === 0xfeff ? raw.slice(1) : raw;
|
|
58
|
+
this.first = false;
|
|
59
|
+
if (line === '') {
|
|
60
|
+
if (this.data.length > 0 && !this.notification()) this.unsafe = true;
|
|
61
|
+
this.data = []; this.fields = false; this.size = 0;
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
if (line.startsWith(':')) return;
|
|
65
|
+
this.size += length;
|
|
66
|
+
if (this.size > MAX_FRAME) { this.unsafe = true; return; }
|
|
67
|
+
this.fields = true;
|
|
68
|
+
const colon = line.indexOf(':');
|
|
69
|
+
const field = colon < 0 ? line : line.slice(0, colon);
|
|
70
|
+
const value = colon < 0 ? '' : line.slice(colon + 1).replace(/^ /, '');
|
|
71
|
+
this.field(field, value);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
private field(field: string, value: string): void {
|
|
75
|
+
if (field === 'data') this.data.push(value);
|
|
76
|
+
else if (field === 'event') { if (value !== '' && value !== 'message') this.unsafe = true; }
|
|
77
|
+
else if (field === 'retry') { if (!/^\d+$/.test(value)) this.unsafe = true; }
|
|
78
|
+
else this.unsafe = true;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
private notification(): boolean {
|
|
82
|
+
const text = this.data.join('\n');
|
|
83
|
+
if (text === '') return true;
|
|
84
|
+
const value: unknown = JSON.parse(text);
|
|
85
|
+
return isRecord(value) && typeof value.method === 'string' && NOTIFICATIONS.has(value.method)
|
|
86
|
+
&& JSONRPCNotificationSchema.safeParse(value).success && ServerNotificationSchema.safeParse(value).success;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export function passiveRelayListener(req: IncomingMessage, res: ServerResponse, upstream: Response, operation: MigrationOperation, stop: () => void): PassiveSseListener | undefined {
|
|
91
|
+
if (req.method !== 'GET' || req.headers['last-event-id'] !== undefined || req.headers['transfer-encoding'] !== undefined || Number(req.headers['content-length'] ?? 0) !== 0) return undefined;
|
|
92
|
+
if (upstream.status !== 200 || upstream.body === null || !isRelaySse(upstream.headers.get('content-type'))) return undefined;
|
|
93
|
+
const listener = new PassiveSseListener();
|
|
94
|
+
operation.passiveListener({ idle: () => listener.idle() && res.writableLength === 0 && !res.destroyed, stop });
|
|
95
|
+
return listener;
|
|
96
|
+
}
|
|
@@ -3,6 +3,7 @@ import { whyUnreachable } from '@metro-labs/http/connector-reach';
|
|
|
3
3
|
import { noteHealth } from './health.js';
|
|
4
4
|
import type { RelayTarget } from './relay-target.js';
|
|
5
5
|
import type { RelaySession } from './relay-session.js';
|
|
6
|
+
import { isRelaySse, passiveRelayListener, type PassiveSseListener } from './relay-passive.js';
|
|
6
7
|
import { isOrgRelayId } from './relay-id.js';
|
|
7
8
|
import { exchangeOrg } from './org-relay.js';
|
|
8
9
|
import { assertLocalMigrationUse, migrationSource, withLocalMigrationOperation } from './migration-guard.js';
|
|
@@ -131,12 +132,12 @@ interface Keepalive {
|
|
|
131
132
|
stop: () => void;
|
|
132
133
|
}
|
|
133
134
|
|
|
134
|
-
function startKeepalive(res: ServerResponse): Keepalive {
|
|
135
|
+
function startKeepalive(res: ServerResponse, listener?: PassiveSseListener): Keepalive {
|
|
135
136
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
136
137
|
const arm = (): void => {
|
|
137
138
|
timer = setTimeout(() => {
|
|
138
139
|
try {
|
|
139
|
-
res.write(': keepalive\n\n');
|
|
140
|
+
if (listener?.keepaliveAllowed() !== false) res.write(': keepalive\n\n');
|
|
140
141
|
} catch {
|
|
141
142
|
log.debug('relay: keepalive write failed');
|
|
142
143
|
}
|
|
@@ -172,13 +173,17 @@ async function pumpBody(
|
|
|
172
173
|
res: ServerResponse,
|
|
173
174
|
body: ReadableStream<Uint8Array>,
|
|
174
175
|
touch: () => void,
|
|
176
|
+
listener?: PassiveSseListener,
|
|
175
177
|
): Promise<void> {
|
|
176
178
|
const reader = body.getReader();
|
|
177
179
|
for (;;) {
|
|
178
180
|
const { done, value } = await reader.read();
|
|
179
181
|
if (done) return;
|
|
182
|
+
listener?.observe(value);
|
|
180
183
|
touch();
|
|
184
|
+
if (listener !== undefined) listener.forwarding = true;
|
|
181
185
|
if (!res.write(value)) await drainedOrClosed(res);
|
|
186
|
+
if (listener !== undefined) listener.forwarding = false;
|
|
182
187
|
}
|
|
183
188
|
}
|
|
184
189
|
|
|
@@ -191,22 +196,23 @@ function passedHeaders(upstream: Response): Record<string, string> {
|
|
|
191
196
|
return headers;
|
|
192
197
|
}
|
|
193
198
|
|
|
194
|
-
async function pipe(res: ServerResponse, upstream: Response, answers: Record<string, unknown>[]): Promise<void> {
|
|
199
|
+
async function pipe(res: ServerResponse, upstream: Response, answers: Record<string, unknown>[], listener?: PassiveSseListener): Promise<void> {
|
|
195
200
|
const headers = passedHeaders(upstream);
|
|
196
201
|
if (answers.length > 0 && (upstream.status === 200 || upstream.status === 202)) {
|
|
197
202
|
await mergeBlocked(res, upstream, headers, answers);
|
|
198
203
|
return;
|
|
199
204
|
}
|
|
200
|
-
const sse = (headers['content-type']
|
|
205
|
+
const sse = isRelaySse(headers['content-type']);
|
|
201
206
|
if (sse) headers['x-accel-buffering'] = 'no';
|
|
202
207
|
res.writeHead(upstream.status, headers);
|
|
208
|
+
if (sse) res.flushHeaders();
|
|
203
209
|
if (upstream.body === null) {
|
|
204
210
|
res.end();
|
|
205
211
|
return;
|
|
206
212
|
}
|
|
207
|
-
const keepalive = sse ? startKeepalive(res) : null;
|
|
213
|
+
const keepalive = sse ? startKeepalive(res, listener) : null;
|
|
208
214
|
try {
|
|
209
|
-
await pumpBody(res, upstream.body, () => keepalive?.touch());
|
|
215
|
+
await pumpBody(res, upstream.body, () => keepalive?.touch(), listener);
|
|
210
216
|
} finally {
|
|
211
217
|
keepalive?.stop();
|
|
212
218
|
}
|
|
@@ -262,7 +268,7 @@ async function relayExchange(
|
|
|
262
268
|
res.socket?.removeListener('close', bail);
|
|
263
269
|
});
|
|
264
270
|
try {
|
|
265
|
-
await relayOnce(req, res, connectorId, deps, control
|
|
271
|
+
await relayOnce(req, res, connectorId, deps, control);
|
|
266
272
|
} catch (err) {
|
|
267
273
|
if (!control.signal.aborted) throw err;
|
|
268
274
|
log.debug({ connector: connectorId }, 'relay: the client left before the answer ended');
|
|
@@ -276,6 +282,7 @@ async function deliver(
|
|
|
276
282
|
deps: RelayApiDeps,
|
|
277
283
|
out: Exchanged,
|
|
278
284
|
answers: Record<string, unknown>[],
|
|
285
|
+
listener?: PassiveSseListener,
|
|
279
286
|
): Promise<void> {
|
|
280
287
|
if (out.kind === 'missing') {
|
|
281
288
|
answer(res, 404, { error: 'no such connector' });
|
|
@@ -295,7 +302,7 @@ async function deliver(
|
|
|
295
302
|
}
|
|
296
303
|
const { status } = out.upstream;
|
|
297
304
|
noteHealth(connectorId, status < 400, status < 400 ? null : `the connector answered ${String(status)}`);
|
|
298
|
-
await pipe(res, out.upstream, answers);
|
|
305
|
+
await pipe(res, out.upstream, answers, listener);
|
|
299
306
|
}
|
|
300
307
|
|
|
301
308
|
async function relayOrg(req: IncomingMessage, res: ServerResponse, id: string, deps: RelayApiDeps, read: Uint8Array<ArrayBuffer> | null, signal: AbortSignal): Promise<void> {
|
|
@@ -316,8 +323,9 @@ async function relayOnce(
|
|
|
316
323
|
res: ServerResponse,
|
|
317
324
|
connectorId: string,
|
|
318
325
|
deps: RelayApiDeps,
|
|
319
|
-
|
|
326
|
+
control: AbortController,
|
|
320
327
|
): Promise<void> {
|
|
328
|
+
const { signal } = control;
|
|
321
329
|
const read = req.method === 'POST' ? await readCapped(req) : null;
|
|
322
330
|
const dir = deps.migrationDir ?? agentsDir();
|
|
323
331
|
if (isOrgRelayId(connectorId) || migrationSource(connectorId, dir)?.phase === 'active') return relayOrg(req, res, connectorId, deps, read, signal);
|
|
@@ -328,7 +336,8 @@ async function relayOnce(
|
|
|
328
336
|
return;
|
|
329
337
|
}
|
|
330
338
|
const out = await exchange(req, connectorId, deps, screened === null ? read : screened.forward, signal, operation.dispatch);
|
|
331
|
-
|
|
339
|
+
const listener = out.kind === 'response' ? passiveRelayListener(req, res, out.upstream, operation, () => { control.abort(); }) : undefined;
|
|
340
|
+
await deliver(res, connectorId, deps, out, screened?.answers ?? [], listener);
|
|
332
341
|
});
|
|
333
342
|
}
|
|
334
343
|
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { isRecord } from '@metro-labs/core/is-record';
|
|
4
|
+
import { writeSecure } from '@metro-labs/core/secure-fs';
|
|
5
|
+
import { agentsDir } from '../agents/files.js';
|
|
6
|
+
import { serverKeysOf, type PluginRow } from './plugin-sync.js';
|
|
7
|
+
|
|
8
|
+
const invalid = (): Error => new Error('Invalid connector server keys. Restore the saved namespace mapping and initialization marker.');
|
|
9
|
+
|
|
10
|
+
function readKeys(path: string, marker: string): Map<string, string> | undefined {
|
|
11
|
+
try {
|
|
12
|
+
const marked = existsSync(marker);
|
|
13
|
+
if (marked && readFileSync(marker, 'utf8') !== '1\n') throw invalid();
|
|
14
|
+
if (!existsSync(path)) {
|
|
15
|
+
if (marked) throw invalid();
|
|
16
|
+
return undefined;
|
|
17
|
+
}
|
|
18
|
+
if (!marked) throw invalid();
|
|
19
|
+
return parseKeys(readFileSync(path, 'utf8'));
|
|
20
|
+
} catch { throw invalid(); }
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function parseKeys(text: string): Map<string, string> {
|
|
24
|
+
const raw: unknown = JSON.parse(text);
|
|
25
|
+
if (!isRecord(raw) || raw.version !== 1 || !isRecord(raw.keys)) throw invalid();
|
|
26
|
+
const keys = new Map<string, string>();
|
|
27
|
+
const reserved = new Set<string>();
|
|
28
|
+
for (const [id, key] of Object.entries(raw.keys)) {
|
|
29
|
+
if (typeof key !== 'string' || !/^[a-z0-9-]+$/.test(key) || reserved.has(key)) throw invalid();
|
|
30
|
+
reserved.add(key);
|
|
31
|
+
keys.set(id, key);
|
|
32
|
+
}
|
|
33
|
+
return keys;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function persistentServerKeys(rows: PluginRow[], dir = agentsDir(), published: PluginRow[] = rows): Map<string, string> {
|
|
37
|
+
const path = join(dir, 'connector-server-keys.json');
|
|
38
|
+
const marker = join(dir, 'connector-server-keys.initialized');
|
|
39
|
+
const saved = readKeys(path, marker);
|
|
40
|
+
const keys = serverKeysOf(rows, saved, published);
|
|
41
|
+
if (keys.size > 0 && keys.size !== saved?.size) {
|
|
42
|
+
if (saved === undefined) writeSecure(marker, '1\n');
|
|
43
|
+
writeSecure(path, `${JSON.stringify({ version: 1, keys: Object.fromEntries(keys) }, null, 2)}\n`);
|
|
44
|
+
}
|
|
45
|
+
return keys;
|
|
46
|
+
}
|