@stage-labs/metro 0.1.0-beta.7 → 0.1.0-beta.71

package/runtime/node_modules/@metro-labs/mcp/src/db/local-connectors.ts

@@ -0,0 +1,277 @@
+import { join } from 'node:path';
+import { readJson, writeSecure } from '../daemon/secure-fs.js';
+import { errMsg, log } from '../daemon/log.js';
+import {
+  connectorUrlText,
+  ConnectorVerifyError,
+  parseConnectorUrl,
+  verifyRemoteMcp,
+  type ConnectorAuth,
+  type OAuthAuth,
+} from '../daemon/connector-verify.js';
+import { oauthExpired, refreshOAuth } from '../daemon/connector-oauth.js';
+import { advertisesOAuth } from '../daemon/oauth-discovery.js';
+import {
+  ConnectorError,
+  connectorAuth,
+  connectorName,
+  readConfig,
+  stamp,
+  type ConnectorConfig,
+} from './connector-config.js';
+import {
+  bearerHeaders,
+  fixedTarget,
+  unrefreshedTarget,
+  type RelayTarget,
+} from './connector-relay.js';
+import {
+  connectorFromRow,
+  UNVERIFIED,
+  type Connector,
+  type ConnectorCheck,
+  type ConnectorInput,
+  type DeletedConnector,
+} from './connectors.js';
+import { agentsDir } from './file-source.js';
+import { assertLocalOwner, LOCAL_PROJECT_ID } from './file-admin.js';
+import { newId } from './ids.js';
+import type { LoadedConnector } from './materialize.js';
+
+export interface LocalConnectorRow {
+  id: string;
+  name: string;
+  url: string;
+  transport: 'http';
+  config: ConnectorConfig;
+}
+
+const FILE = 'connectors.json';
+const filePath = (dir: string): string => join(dir, FILE);
+
+const isRow = (v: unknown): v is { id: string; name: string; url: string; config?: unknown } =>
+  typeof v === 'object' &&
+  v !== null &&
+  typeof (v as { id?: unknown }).id === 'string' &&
+  typeof (v as { name?: unknown }).name === 'string' &&
+  typeof (v as { url?: unknown }).url === 'string';
+
+const asList = (value: unknown): unknown[] =>
+  Array.isArray(value) ? value.map((item: unknown) => item) : [];
+
+export function readLocalConnectors(dir = agentsDir()): LocalConnectorRow[] {
+  const raw = readJson<{ connectors?: unknown }>(filePath(dir), {}, {
+    warn: 'connectors.json: malformed, ignoring',
+  });
+  return asList(raw.connectors).filter(isRow).map((r) => ({
+    id: r.id,
+    name: r.name,
+    url: r.url,
+    transport: 'http' as const,
+    config: readConfig(r.config),
+  }));
+}
+
+function writeRows(dir: string, rows: LocalConnectorRow[]): void {
+  writeSecure(filePath(dir), `${JSON.stringify({ version: 1, connectors: rows }, null, 2)}\n`);
+}
+
+const missing = (): ConnectorError => new ConnectorError('no such connector', 404);
+
+function ownedRows(subject: string, project: string, dir: string): LocalConnectorRow[] {
+  assertLocalOwner(subject, dir);
+  if (project !== LOCAL_PROJECT_ID) throw new ConnectorError('no such project', 404);
+  return readLocalConnectors(dir);
+}
+
+function rowOrThrow(subject: string, id: string, dir: string): LocalConnectorRow {
+  assertLocalOwner(subject, dir);
+  const row = readLocalConnectors(dir).find((r) => r.id === id);
+  if (row === undefined) throw missing();
+  return row;
+}
+
+function replace(dir: string, row: LocalConnectorRow): Connector {
+  writeRows(dir, readLocalConnectors(dir).map((r) => (r.id === row.id ? row : r)));
+  return connectorFromRow(row);
+}
+
+async function oauthCapable(url: URL, auth: ConnectorAuth): Promise<boolean> {
+  if (auth.kind === 'oauth') return true;
+  if (auth.kind === 'header') return false;
+  return advertisesOAuth(url);
+}
+
+export function localImportConnectors(rows: LoadedConnector[], dir = agentsDir()): number {
+  const current = readLocalConnectors(dir);
+  for (const row of rows) {
+    const other = current.find((r) => r.id !== row.id && r.name === row.name);
+    if (other !== undefined)
+      throw new ConnectorError(`a connector named '${row.name}' already exists on this daemon; rename it first`, 409);
+  }
+  const imported: LocalConnectorRow[] = rows.map((row) => ({
+    id: row.id,
+    name: row.name,
+    url: row.url,
+    transport: 'http',
+    config: readConfig(row.config),
+  }));
+  const ids = new Set(imported.map((r) => r.id));
+  writeRows(dir, [...current.filter((r) => !ids.has(r.id)), ...imported]);
+  return imported.length;
+}
+
+export async function localListConnectors(
+  subject: string,
+  project: string,
+  dir = agentsDir(),
+): Promise<Connector[]> {
+  return Promise.resolve(ownedRows(subject, project, dir).map(connectorFromRow));
+}
+
+export async function localGetConnector(subject: string, id: string, dir = agentsDir()): Promise<Connector> {
+  return Promise.resolve(connectorFromRow(rowOrThrow(subject, id, dir)));
+}
+
+function assertNameFree(name: string, exceptId: string | null, dir: string): void {
+  if (readLocalConnectors(dir).some((r) => r.id !== exceptId && r.name === name))
+    throw new ConnectorError(`a connector named '${name}' already exists on this daemon`, 409);
+}
+
+function insert(dir: string, name: string, url: URL, config: ConnectorConfig): Connector {
+  assertNameFree(name, null, dir);
+  const row: LocalConnectorRow = { id: newId(), name, url: connectorUrlText(url), transport: 'http', config };
+  writeRows(dir, [...readLocalConnectors(dir), row]);
+  return connectorFromRow(row);
+}
+
+export async function localCreateConnector(
+  subject: string,
+  project: string,
+  input: ConnectorInput,
+  dir = agentsDir(),
+): Promise<Connector> {
+  ownedRows(subject, project, dir);
+  const name = connectorName(input.name);
+  const url = parseConnectorUrl(input.url);
+  const auth = connectorAuth(input.header, input.value);
+  const verified = stamp(await verifyRemoteMcp(url, auth));
+  return insert(dir, name, url, {
+    auth,
+    createdAt: new Date().toISOString(),
+    verified,
+    oauth: await oauthCapable(url, auth),
+  });
+}
+
+export async function localCreatePendingConnector(
+  subject: string,
+  project: string,
+  input: { name: unknown; url: unknown },
+  dir = agentsDir(),
+): Promise<Connector> {
+  ownedRows(subject, project, dir);
+  const name = connectorName(input.name);
+  const url = parseConnectorUrl(input.url);
+  return Promise.resolve(
+    insert(dir, name, url, { auth: { kind: 'none' }, createdAt: new Date().toISOString(), verified: UNVERIFIED, oauth: true }),
+  );
+}
+
+export async function localReconnectConnector(
+  subject: string,
+  id: string,
+  auth: OAuthAuth,
+  dir = agentsDir(),
+): Promise<Connector> {
+  const row = rowOrThrow(subject, id, dir);
+  const verified = stamp(await verifyRemoteMcp(parseConnectorUrl(row.url), auth));
+  return replace(dir, { ...row, config: { ...row.config, auth, verified, oauth: true } });
+}
+
+export async function localDisconnectConnector(subject: string, id: string, dir = agentsDir()): Promise<Connector> {
+  const row = rowOrThrow(subject, id, dir);
+  if (row.config.auth.kind !== 'oauth') throw new ConnectorError('that connector is not signed in', 400);
+  return Promise.resolve(replace(dir, { ...row, config: { ...row.config, auth: { kind: 'none' }, oauth: true } }));
+}
+
+async function freshAuth(auth: ConnectorAuth, resource: string): Promise<ConnectorAuth> {
+  if (auth.kind !== 'oauth' || !oauthExpired(auth)) return auth;
+  return refreshOAuth(auth, resource);
+}
+
+export async function localVerifyConnector(subject: string, id: string, dir = agentsDir()): Promise<ConnectorCheck> {
+  const row = rowOrThrow(subject, id, dir);
+  try {
+    const url = parseConnectorUrl(row.url);
+    const auth = await freshAuth(row.config.auth, url.toString());
+    const verified = stamp(await verifyRemoteMcp(url, auth));
+    const oauth = row.config.oauth || (await oauthCapable(url, auth));
+    replace(dir, { ...row, config: { ...row.config, auth, verified, oauth } });
+    return { id: row.id, name: row.name, ok: true, verified };
+  } catch (err) {
+    if (!(err instanceof ConnectorVerifyError)) throw err;
+    return { id: row.id, name: row.name, ok: false, reason: err.message };
+  }
+}
+
+export async function localRenameConnector(subject: string, id: string, raw: string, dir = agentsDir()): Promise<Connector> {
+  const name = connectorName(raw);
+  const row = rowOrThrow(subject, id, dir);
+  assertNameFree(name, id, dir);
+  return Promise.resolve(replace(dir, { ...row, name }));
+}
+
+export async function localDeleteConnector(subject: string, id: string, dir = agentsDir()): Promise<DeletedConnector> {
+  const row = rowOrThrow(subject, id, dir);
+  writeRows(dir, readLocalConnectors(dir).filter((r) => r.id !== id));
+  return Promise.resolve({ id: row.id, name: row.name });
+}
+
+const byIds = (ids: string[], dir: string): LocalConnectorRow[] =>
+  readLocalConnectors(dir).filter((r) => ids.includes(r.id));
+
+export async function localConnectorNamesByIds(ids: string[], dir = agentsDir()): Promise<{ id: string; name: string }[]> {
+  return Promise.resolve(byIds(ids, dir).map((r) => ({ id: r.id, name: r.name })));
+}
+
+const inflight = new Map<string, Promise<OAuthAuth>>();
+
+function refreshOnce(row: LocalConnectorRow, auth: OAuthAuth, dir: string): Promise<OAuthAuth> {
+  const running = inflight.get(row.id);
+  if (running !== undefined) return running;
+  const job = refreshOAuth(auth, parseConnectorUrl(row.url).toString())
+    .then((fresh) => {
+      replace(dir, { ...row, config: { ...row.config, auth: fresh } });
+      return fresh;
+    })
+    .finally(() => {
+      inflight.delete(row.id);
+    });
+  inflight.set(row.id, job);
+  return job;
+}
+
+async function oauthTarget(row: LocalConnectorRow, auth: OAuthAuth, force: boolean, dir: string): Promise<RelayTarget> {
+  if (!force && !oauthExpired(auth)) return { kind: 'ok', url: row.url, headers: bearerHeaders(auth.accessToken) };
+  try {
+    const fresh = await refreshOnce(row, auth, dir);
+    return { kind: 'ok', url: row.url, headers: bearerHeaders(fresh.accessToken) };
+  } catch (err) {
+    log.warn({ id: row.id, err: errMsg(err) }, 'local relay: token refresh failed');
+    return unrefreshedTarget(row.url, auth, force);
+  }
+}
+
+export async function localRelayTarget(
+  connectorId: string,
+  force: boolean,
+  dir = agentsDir(),
+): Promise<RelayTarget> {
+  const row = readLocalConnectors(dir).find((r) => r.id === connectorId);
+  if (row === undefined) return { kind: 'missing' };
+  parseConnectorUrl(row.url);
+  const auth = row.config.auth;
+  if (auth.kind === 'oauth') return oauthTarget(row, auth, force, dir);
+  return fixedTarget(row.url, auth, force);
+}

package/runtime/node_modules/@metro-labs/mcp/src/db/materialize.ts

@@ -0,0 +1,253 @@
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { log } from '../daemon/log.js';
+import { writeSecure } from '../daemon/secure-fs.js';
+import { isLocalMode, trainsDir } from '../daemon/paths.js';
+import {
+  setAgentMap,
+  setAllowlistMap,
+  type AgentMap,
+  type AgentNameMap,
+  type AllowlistMap,
+} from './agent-map.js';
+import { setKeyMap } from './key-map.js';
+import { STATIONS, type StationName } from './stations.js';
+
+interface StationTarget {
+  file: string;
+  fileEnv: string;
+  trainImport: string | null;
+}
+
+export interface LoadedAccount {
+  station: StationName;
+  id: string;
+  allowlist: string[] | null;
+  config: Record<string, unknown>;
+}
+
+export interface LoadedConnector {
+  id: string;
+  name: string;
+  url: string;
+  transport: 'http';
+  config: Record<string, unknown>;
+}
+
+export interface LoadedAgent {
+  id: string;
+  name: string;
+  accounts: LoadedAccount[];
+  key: string | null;
+  connectors?: LoadedConnector[];
+}
+
+const METRO_DIR = join(homedir(), '.metro');
+
+const STATION_TARGETS: Record<StationName, StationTarget> = {
+  xmtp: {
+    file: 'xmtp-accounts.json',
+    fileEnv: 'XMTP_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/xmtp/train',
+  },
+  'telegram-bot': {
+    file: 'telegram-bot-accounts.json',
+    fileEnv: 'TELEGRAM_BOT_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/telegram-bot/train',
+  },
+  'telegram': {
+    file: 'telegram-accounts.json',
+    fileEnv: 'TELEGRAM_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/telegram/train',
+  },
+  'discord-bot': {
+    file: 'discord-bot-accounts.json',
+    fileEnv: 'DISCORD_BOT_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/discord-bot/train',
+  },
+  whatsapp: {
+    file: 'whatsapp-accounts.json',
+    fileEnv: 'WHATSAPP_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/whatsapp/train',
+  },
+  webhook: {
+    file: 'webhook-accounts.json',
+    fileEnv: 'WEBHOOK_ACCOUNTS_FILE',
+    trainImport: null,
+  },
+};
+
+function accountFilePath(station: StationName): string {
+  const target = STATION_TARGETS[station];
+  return process.env[target.fileEnv] ?? join(METRO_DIR, target.file);
+}
+
+export type StationSource = () => Promise<LoadedAgent[]>;
+
+export const MOVABLE_STATIONS = new Set<StationName>(
+  STATIONS.filter((s) => s !== 'webhook'),
+);
+
+function trainStubPath(station: StationName): string {
+  return join(trainsDir(), `${station}.ts`);
+}
+
+function currentText(path: string): string | null {
+  try {
+    return readFileSync(path, 'utf8');
+  } catch {
+    return null;
+  }
+}
+
+export function writeIfChanged(path: string, content: string): boolean {
+  if (currentText(path) === content) return false;
+  writeFileSync(path, content);
+  return true;
+}
+
+export const stationRunsHere = (station: StationName): boolean =>
+  isLocalMode() || !MOVABLE_STATIONS.has(station);
+
+interface WrittenStations {
+  active: Map<StationName, number>;
+  changed: StationName[];
+}
+
+function writeStations(list: LoadedAgent[]): WrittenStations {
+  mkdirSync(METRO_DIR, { recursive: true });
+  mkdirSync(trainsDir(), { recursive: true });
+
+  const byStation = new Map<StationName, LoadedAccount[]>();
+  const map: AgentMap = {};
+  const names: AgentNameMap = {};
+  const allow: AllowlistMap = {};
+  for (const agent of list) {
+    names[agent.id] = agent.name;
+    for (const a of agent.accounts) {
+      if (!(a.station in STATION_TARGETS)) {
+        log.error(
+          { station: a.station, id: a.id },
+          'unknown station — this metro is older or newer than the row; skipping it',
+        );
+        continue;
+      }
+      map[`${a.station}/${a.id}`] = agent.id;
+      if (a.allowlist) allow[`${a.station}/${a.id}`] = a.allowlist;
+      if (stationRunsHere(a.station)) {
+        const cur = byStation.get(a.station);
+        if (cur) cur.push(a);
+        else byStation.set(a.station, [a]);
+      }
+    }
+  }
+  setAgentMap(map, names);
+  setAllowlistMap(allow);
+
+  const active = new Map<StationName, number>();
+  const changed: StationName[] = [];
+  for (const [station, accts] of byStation) {
+    const records = accts.map((a) => ({ id: a.id, ...a.config }));
+    const path = accountFilePath(station);
+    const content = JSON.stringify(records, null, 2);
+    if (currentText(path) !== content) {
+      writeSecure(path, content);
+      changed.push(station);
+    }
+    const { trainImport } = STATION_TARGETS[station];
+    if (trainImport !== null)
+      writeIfChanged(trainStubPath(station), `import '${trainImport}';\n`);
+    active.set(station, accts.length);
+  }
+  return { active, changed };
+}
+
+const BLANKED = '[]\n';
+
+function pruneStations(present: Map<StationName, number>): StationName[] {
+  const removed: StationName[] = [];
+  for (const station of Object.keys(STATION_TARGETS) as StationName[]) {
+    if (present.has(station)) continue;
+    const hasTrain = STATION_TARGETS[station].trainImport !== null;
+    const stub = trainStubPath(station);
+    const stubGone = !hasTrain || !existsSync(stub);
+    const file = currentText(accountFilePath(station));
+    const fileInert = file === null || file === BLANKED;
+    if (stubGone && fileInert) continue;
+    if (!fileInert) writeSecure(accountFilePath(station), BLANKED);
+    if (hasTrain) rmSync(stub, { force: true });
+    removed.push(station);
+  }
+  return removed;
+}
+
+const stationLabels = (m: Map<StationName, number>): string[] =>
+  [...m].map(([station, n]) => `${station}(${n})`);
+
+function applyKeyMap(list: LoadedAgent[]): void {
+  lastKey = list.find((a) => a.key !== null)?.key ?? null;
+  setKeyMap(
+    list.flatMap((agent) =>
+      agent.key === null ? [] : [{ key: agent.key, agentId: agent.id }],
+    ),
+  );
+}
+
+let lastKey: string | null = null;
+
+export function localAgentKey(): string | null {
+  return lastKey;
+}
+
+async function loadAndWrite(
+  source: StationSource,
+): Promise<WrittenStations & { agents: number }> {
+  const list = await source();
+  applyKeyMap(list);
+  return { ...writeStations(list), agents: list.length };
+}
+
+export interface MaterializeOptions {
+  allowEmpty?: boolean;
+}
+
+export async function materializeFrom(
+  source: StationSource,
+  opts: MaterializeOptions = {},
+): Promise<void> {
+  const { active, agents: found } = await loadAndWrite(source);
+
+  if (found === 0 && opts.allowEmpty !== true)
+    throw new Error('no agents found — nothing to materialize');
+  const removed = pruneStations(active);
+  log.info(
+    { stations: stationLabels(active), agents: found, removed },
+    'materialized station accounts',
+  );
+}
+
+export interface ReloadedStations {
+  active: StationName[];
+  removed: StationName[];
+  changed: StationName[];
+}
+
+export async function reloadFrom(
+  source: StationSource,
+): Promise<ReloadedStations> {
+  const { active, changed } = await loadAndWrite(source);
+  const removed = pruneStations(active);
+  if (removed.length > 0 || changed.length > 0)
+    log.info(
+      { stations: stationLabels(active), removed, changed },
+      'reloaded station accounts',
+    );
+  return { active: [...active.keys()], removed, changed };
+}

package/runtime/node_modules/@metro-labs/mcp/src/db/schema.ts

@@ -0,0 +1,33 @@
+import { index, jsonb, pgTable, text, uniqueIndex } from 'drizzle-orm/pg-core';
+
+import { STATIONS, type ConnectorTransport, type StationName } from './stations.js';
+
+export { STATIONS, type ConnectorTransport, type StationName };
+
+export type ProjectRole = 'admin' | 'member';
+
+
+export const vaults = pgTable(
+  'vaults',
+  {
+    id: text('id').primaryKey(),
+    owner: text('owner').notNull(),
+    name: text('name').notNull(),
+    stations: jsonb('stations').notNull(),
+    envelope: jsonb('envelope').notNull(),
+    syncedAt: text('synced_at').notNull(),
+  },
+  (t) => [index('vaults_owner_idx').on(t.owner)],
+);
+
+export const servers = pgTable(
+  'servers',
+  {
+    id: text('id').primaryKey(),
+    owner: text('owner').notNull(),
+    host: text('host').notNull(),
+    name: text('name'),
+    addedAt: text('added_at').notNull(),
+  },
+  (t) => [uniqueIndex('servers_owner_host_idx').on(t.owner, t.host), index('servers_owner_idx').on(t.owner)],
+);

package/runtime/node_modules/@metro-labs/mcp/src/db/servers.ts

@@ -0,0 +1,83 @@
+import { and, asc, eq } from 'drizzle-orm';
+import { ApiError } from '../daemon/api-error.js';
+import { isRecord } from '../daemon/is-record.js';
+import { getDb } from './client.js';
+import { newId, parseId } from './ids.js';
+import { servers } from './schema.js';
+import { normalizeAddress } from './address.js';
+import { parseServerHost, parseServerName, type ServerEntry } from '../daemon/server-types.js';
+
+export class ServerListError extends ApiError {}
+
+const missing = (): ServerListError => new ServerListError('no such server', 404);
+
+function ownerOf(subject: string): string {
+  const address = normalizeAddress(subject);
+  if (address === null) throw new ServerListError('the server list needs a signed identity', 403);
+  return address;
+}
+
+function idOf(raw: string): string {
+  const id = parseId(raw);
+  if (id === null) throw missing();
+  return id;
+}
+
+const entryOf = (row: { id: string; host: string; name: string | null; addedAt: string }): ServerEntry => ({
+  id: row.id,
+  host: row.host,
+  name: row.name,
+  addedAt: row.addedAt,
+});
+
+const columns = { id: servers.id, host: servers.host, name: servers.name, addedAt: servers.addedAt };
+
+export async function listServersForOwner(subject: string): Promise<ServerEntry[]> {
+  const owner = ownerOf(subject);
+  const rows = await getDb().select(columns).from(servers).where(eq(servers.owner, owner)).orderBy(asc(servers.addedAt));
+  return rows.map(entryOf);
+}
+
+export async function addServerForOwner(subject: string, body: unknown): Promise<ServerEntry> {
+  const owner = ownerOf(subject);
+  const host = parseServerHost(isRecord(body) && typeof body.host === 'string' ? body.host : '');
+  if (host === null) throw new ServerListError('host is not a server address', 400);
+  const name = parseServerName(isRecord(body) ? body.name : undefined);
+  const db = getDb();
+  const held = await db.select(columns).from(servers).where(and(eq(servers.owner, owner), eq(servers.host, host)));
+  const row = held[0];
+  if (row !== undefined) {
+    if (name === null) return entryOf(row);
+    await db.update(servers).set({ name }).where(eq(servers.id, row.id));
+    return entryOf({ ...row, name });
+  }
+  const next = { id: newId(), owner, host, name, addedAt: new Date().toISOString() };
+  await db.insert(servers).values(next);
+  return entryOf(next);
+}
+
+export async function renameServerForOwner(subject: string, rawId: string, body: unknown): Promise<ServerEntry> {
+  const owner = ownerOf(subject);
+  const id = idOf(rawId);
+  const name = parseServerName(isRecord(body) ? body.name : undefined);
+  const rows = await getDb()
+    .update(servers)
+    .set({ name })
+    .where(and(eq(servers.id, id), eq(servers.owner, owner)))
+    .returning(columns);
+  const row = rows[0];
+  if (row === undefined) throw missing();
+  return entryOf(row);
+}
+
+export async function deleteServerForOwner(subject: string, rawId: string): Promise<{ id: string; host: string }> {
+  const owner = ownerOf(subject);
+  const id = idOf(rawId);
+  const gone = await getDb()
+    .delete(servers)
+    .where(and(eq(servers.id, id), eq(servers.owner, owner)))
+    .returning({ id: servers.id, host: servers.host });
+  const row = gone[0];
+  if (row === undefined) throw missing();
+  return row;
+}

package/runtime/node_modules/@metro-labs/mcp/src/db/stations.ts

@@ -0,0 +1,12 @@
+export const STATIONS = [
+  'xmtp',
+  'telegram-bot',
+  'telegram',
+  'discord-bot',
+  'whatsapp',
+  'webhook',
+] as const;
+
+export type StationName = (typeof STATIONS)[number];
+
+export type ConnectorTransport = 'http' | 'sse';