@stage-labs/metro 0.1.0-beta.2 → 0.1.0-beta.21

package/runtime/node_modules/@metro-labs/mcp/src/daemon/boot.ts

@@ -0,0 +1,322 @@
+import type { ProjectApiDeps } from './project-api.js';
+import { join } from 'node:path';
+import { type Server } from 'node:http';
+import { selfLine, userSelf } from './events.js';
+import { setTrainCallBackend } from './train-call.js';
+import { errMsg, log, logFatalSync } from './log.js';
+import { acquireLock, loadMetroEnv, STATE_DIR } from './paths.js';
+import { installCrashGuard, markDaemonReady } from './crash-guard.js';
+import {
+  loadTunnelConfig,
+  Tunnel,
+  warnOnLegacyWebhooks,
+  webhookPort,
+} from './tunnel.js';
+import { TrainSupervisor, TRAINS_DIR } from './supervisor.js';
+import {
+  makeEmit,
+  startWebhookServer,
+  trainEventToMetroEvent,
+} from './http.js';
+import { type RunApiDeps } from './run-api.js';
+import {
+  claimRuntime,
+  fenceRuntime,
+  runtimeLabels,
+  touchRuntime,
+} from '../db/runtimes.js';
+import {
+  mintRuntimeCodeForEmail,
+  releaseRuntimeForEmail,
+} from '../db/runtime-admin.js';
+import { loadAgentForRuntime, localAgentKey } from '../db/materialize.js';
+import {
+  agentLiveness,
+  closeAgentSession,
+  createMetroMcp,
+} from '../mcp/index.js';
+import { metroCall } from '../mcp/ctx.js';
+import { gatherAccountsForAgents } from '../mcp/accounts.js';
+import {
+  accountStationCapabilities,
+  stationByName,
+} from '../stations/registry.js';
+import { prepareAccount } from '../stations/attach.js';
+import {
+  materializeFrom,
+  materializeFromDb,
+  reloadAccountsFromDb,
+  reloadFrom,
+} from '../db/materialize.js';
+import {
+  httpSource,
+  runtimeConfigFromEnv,
+  startRuntimePoller,
+} from './runtime-source.js';
+import {  createAgentForEmail,  deleteAgentForEmail,  listAgentsForEmail,  ownedAgentOrThrow,  resetAgentKeyForEmail,    type ResetAgentKey,} from '../db/agent-admin.js';
+import {
+  attachAccountToAgent,
+  detachAccountFromAgent,
+} from '../db/account-attach.js';
+import {
+  createConnectorForEmail,
+  createPendingConnectorForEmail,
+  deleteConnectorForEmail,
+  disconnectConnectorForEmail,
+  getConnectorForEmail,
+  listConnectorsForEmail,
+  listFreshConnectorsByIds,
+  renameConnectorForEmail,
+  reconnectConnectorForEmail,
+  verifyConnectorForEmail,
+} from '../db/connectors.js';
+import {
+  addMemberForEmail,
+  createProjectForEmail,
+  deleteProjectForEmail,
+  listMembersForEmail,
+  listProjectsForEmail,
+  removeMemberForEmail,
+  renameProjectForEmail,
+  setMemberRoleForEmail,
+} from '../db/projects.js';
+import {
+  addToCollectionForEmail,
+  createCollectionForEmail,
+  deleteCollectionForEmail,
+  getCollectionForEmail,
+  listCollectionsForEmail,
+  removeFromCollectionForEmail,
+  renameCollectionForEmail,
+} from '../db/connector-collections.js';
+import type { StationName } from '../db/schema.js';
+import { AttachSessions } from './attach-session.js';
+import { startUploadReaper } from './upload-store.js';
+import type { AgentApiDeps } from './agent-api.js';
+import type { ConnectorApiDeps } from './connector-api.js';
+
+installCrashGuard();
+loadMetroEnv();
+acquireLock(join(STATE_DIR, '.tail-lock'));
+
+const self = userSelf();
+log.info({ self, line: selfLine() }, 'user identity');
+
+process.stdout.on('error', (err) => {
+  if ((err as NodeJS.ErrnoException).code !== 'EPIPE')
+    log.warn({ err: errMsg(err) }, 'stdout error');
+});
+
+const supervisor = new TrainSupervisor();
+const emit = makeEmit();
+
+supervisor.onTrainEvent((env, train) => {
+  const entry = trainEventToMetroEvent(env, train);
+  if (entry) emit(entry);
+});
+
+let webhookServer: Server | null = null;
+const tunnelCfg = loadTunnelConfig();
+const tunnel = tunnelCfg ? new Tunnel(tunnelCfg, webhookPort()) : null;
+
+setTrainCallBackend((train, action, args) =>
+  supervisor.call(train, action, args),
+);
+
+const runtime = runtimeConfigFromEnv();
+const localSource = runtime === null ? null : httpSource(runtime);
+
+function announceLocalEndpoint(): void {
+  const key = localAgentKey();
+  if (key === null) {
+    log.warn('this agent has no key — reset it in the web UI to connect an agent');
+    return;
+  }
+  const url = `http://127.0.0.1:${String(webhookPort())}/mcp?token=${key}`;
+  process.stderr.write(
+    `\nConnect an agent on this machine:\n\n  claude mcp add --transport http metro "${url}"\n\n`,
+  );
+}
+
+async function applyStations(removed: StationName[]): Promise<void> {
+  for (const station of removed) await supervisor.stopTrain(station);
+}
+
+async function syncLocal(): Promise<void> {
+  if (localSource === null) return;
+  const before = new Set(supervisor.running());
+  const { active, removed } = await reloadFrom(localSource);
+  await applyStations(removed);
+  for (const station of active)
+    if (before.has(station)) supervisor.requestReload(station);
+}
+
+async function stopLocalStations(): Promise<void> {
+  for (const station of supervisor.running()) await supervisor.stopTrain(station);
+}
+
+async function syncStations(station: StationName): Promise<void> {
+  const { removed } = await reloadAccountsFromDb();
+  if (stationByName(station)?.hasTrain === false) return;
+  if (removed.includes(station)) await supervisor.stopTrain(station);
+  else supervisor.requestReload(station);
+}
+
+const attachSessions = new AttachSessions({
+  authorize: async (owner) => {
+    await ownedAgentOrThrow(owner.email, owner.agentId);
+  },
+  complete: async (owner, station, config) => {
+    const ref = await attachAccountToAgent(
+      owner.email,
+      owner.agentId,
+      station,
+      config,
+    );
+    const activated = await syncStations(station).then(
+      () => true,
+      (err: unknown) => {
+        log.warn(
+          { station, err: errMsg(err) },
+          'attach-session: station reload failed, the change lands at the next boot',
+        );
+        return false;
+      },
+    );
+    return { accountId: ref.accountId, activated };
+  },
+});
+
+async function resetAgentKey(
+  email: string,
+  id: string,
+): Promise<ResetAgentKey> {
+  const reset = await resetAgentKeyForEmail(email, id);
+  const closed = await closeAgentSession(id);
+  log.info(
+    { agent: reset.name, id: reset.id, sessionClosed: closed },
+    'agent-api: key rotated, live session dropped',
+  );
+  return reset;
+}
+
+const agentApi: AgentApiDeps = {
+  attachSessions,
+  listAgents: listAgentsForEmail,
+  createAgent: createAgentForEmail,
+  deleteAgent: deleteAgentForEmail,
+  resetKey: resetAgentKey,
+  gatherAccounts: gatherAccountsForAgents,
+  capabilities: accountStationCapabilities,
+  liveness: agentLiveness,
+  mintRuntimeCode: mintRuntimeCodeForEmail,
+  releaseRuntime: releaseRuntimeForEmail,
+  runtimes: runtimeLabels,
+  prepareAccount,
+  attachAccount: attachAccountToAgent,
+  detachAccount: detachAccountFromAgent,
+  syncStations,
+};
+
+const runApi: RunApiDeps = {
+  claimRuntime,
+  fenceRuntime,
+  touchRuntime,
+  loadAgent: loadAgentForRuntime,
+};
+
+const projectApi: ProjectApiDeps = {
+  listProjects: listProjectsForEmail,
+  createProject: createProjectForEmail,
+  renameProject: renameProjectForEmail,
+  deleteProject: deleteProjectForEmail,
+  listMembers: listMembersForEmail,
+  addMember: addMemberForEmail,
+  setMemberRole: setMemberRoleForEmail,
+  removeMember: removeMemberForEmail,
+};
+
+const connectorApi: ConnectorApiDeps = {
+  listConnectors: listConnectorsForEmail,
+  freshConnectorsByIds: listFreshConnectorsByIds,
+  listCollections: listCollectionsForEmail,
+  getCollection: getCollectionForEmail,
+  createCollection: createCollectionForEmail,
+  renameCollection: renameCollectionForEmail,
+  deleteCollection: deleteCollectionForEmail,
+  addToCollection: addToCollectionForEmail,
+  removeFromCollection: removeFromCollectionForEmail,
+  renameConnector: renameConnectorForEmail,
+  createConnector: createConnectorForEmail,
+  createPendingConnector: createPendingConnectorForEmail,
+  reconnectConnector: reconnectConnectorForEmail,
+  getConnector: getConnectorForEmail,
+  verifyConnector: verifyConnectorForEmail,
+  disconnectConnector: disconnectConnectorForEmail,
+  deleteConnector: deleteConnectorForEmail,
+};
+
+async function main(): Promise<void> {
+  if (localSource === null) await materializeFromDb();
+  else await materializeFrom(localSource);
+  warnOnLegacyWebhooks();
+  supervisor.start();
+  const metroMcp = await createMetroMcp();
+  webhookServer = await startWebhookServer(
+    emit,
+    { agentApi, connectorApi, projectApi, runApi },
+    metroMcp.httpHandler,
+    metroCall,
+  );
+  metroMcp.startInbound();
+  startUploadReaper();
+  if (localSource !== null) {
+    startRuntimePoller({ sync: syncLocal, stopAll: stopLocalStations });
+    announceLocalEndpoint();
+  }
+  tunnel?.start();
+  log.info(
+    { tunnel: !!tunnel, trainsDir: TRAINS_DIR, mcp: '/' },
+    'dispatcher ready',
+  );
+  markDaemonReady();
+}
+
+const SHUTDOWN_TIMEOUT_MS = 3_000;
+let shuttingDown = false;
+async function shutdown(): Promise<void> {
+  if (shuttingDown) return;
+  shuttingDown = true;
+  log.info('dispatcher shutting down');
+  tunnel?.stop();
+  if (webhookServer) {
+    const server = webhookServer;
+    await Promise.race([
+      new Promise<void>((r) => {
+        server.close(() => {
+          r();
+        });
+      }),
+      new Promise<void>((r) => {
+        setTimeout(r, SHUTDOWN_TIMEOUT_MS).unref();
+      }),
+    ]);
+  }
+  await attachSessions.stop();
+  await supervisor.stop();
+  process.exit(0);
+}
+const onShutdown = (): void => {
+  shutdown().catch((err: unknown) => {
+    log.error({ err: errMsg(err) }, 'dispatcher: shutdown failed');
+    process.exit(1);
+  });
+};
+if (process.env.METRO_STDIN_SHUTDOWN === '1')
+  process.stdin.on('end', onShutdown).on('close', onShutdown);
+for (const sig of ['SIGINT', 'SIGTERM'] as const) process.on(sig, onShutdown);
+
+await main().catch((err: unknown) => {
+  logFatalSync({ err: errMsg(err) }, 'dispatcher failed to start');
+  process.exit(1);
+});

package/runtime/node_modules/@metro-labs/mcp/src/daemon/cli-pair-api.ts

@@ -0,0 +1,111 @@
+import type { IncomingMessage, ServerResponse } from 'node:http';
+import { errMsg, log } from './log.js';
+import {
+  bodyField,
+  cliIdentity,
+  cors,
+  readJsonBody,
+  sendJson,
+} from './api-http.js';
+import { CLI_CODE_RE, takeCliCode } from './cli-pair.js';
+import { mcpServersJson } from './connector-json.js';
+import { sessionTtlFromEnv } from './google-oauth.js';
+import { signCliToken } from './session.js';
+import type { ConnectorApiDeps } from './connector-api.js';
+
+const CLAIM_PATH = '/api/cli/claim';
+const MCP_PATH = '/api/cli/mcp';
+const SESSION_PATH = '/api/cli/session';
+
+const ALLOWED: Record<string, string> = {
+  [CLAIM_PATH]: 'POST',
+  [MCP_PATH]: 'GET',
+  [SESSION_PATH]: 'GET',
+};
+
+function secretOrNull(): string | null {
+  const secret = process.env.METRO_SESSION_SECRET?.trim() ?? '';
+  return secret === '' ? null : secret;
+}
+
+async function handleClaim(
+  req: IncomingMessage,
+  res: ServerResponse,
+  deps: ConnectorApiDeps,
+): Promise<void> {
+  const secret = secretOrNull();
+  if (secret === null) {
+    sendJson(req, res, 503, { error: 'sign-in is not configured' });
+    return;
+  }
+  const code = bodyField(await readJsonBody(req), 'code');
+  const taken =
+    typeof code === 'string' && CLI_CODE_RE.test(code)
+      ? takeCliCode(code)
+      : undefined;
+  if (taken === undefined) {
+    sendJson(req, res, 400, {
+      error: 'that code has expired or was already used',
+    });
+    return;
+  }
+  const collection = await deps.getCollection(taken.email, taken.collectionId);
+  const token = signCliToken(
+    { email: taken.email, collectionId: collection.id },
+    secret,
+    { ttlSec: sessionTtlFromEnv() },
+  );
+  log.info({ email: taken.email, collection: collection.id }, 'cli: code claimed');
+  sendJson(req, res, 200, { token, email: taken.email, collection: collection.name });
+}
+
+async function handleRead(
+  req: IncomingMessage,
+  res: ServerResponse,
+  deps: ConnectorApiDeps,
+  path: string,
+): Promise<void> {
+  const who = cliIdentity(req);
+  if (who === null) {
+    sendJson(req, res, 401, { error: 'unauthorized' });
+    return;
+  }
+  const collection = await deps.getCollection(who.email, who.collectionId);
+  if (path === SESSION_PATH) {
+    sendJson(req, res, 200, { email: who.email, collection: collection.name });
+    return;
+  }
+  const rows = await deps.freshConnectorsByIds(collection.connectorIds);
+  sendJson(req, res, 200, { json: mcpServersJson(rows), collection: collection.name });
+}
+
+export function handleCliPairRequest(
+  req: IncomingMessage,
+  res: ServerResponse,
+  deps: ConnectorApiDeps,
+): boolean {
+  const path = (req.url ?? '').split('?')[0] ?? '';
+  const method = ALLOWED[path];
+  if (method === undefined) {
+    if (!path.startsWith('/api/cli/') && path !== '/api/cli') return false;
+    sendJson(req, res, 404, { error: 'no such route' });
+    return true;
+  }
+  if (req.method === 'OPTIONS') {
+    res.writeHead(204, cors(req)).end();
+    return true;
+  }
+  if (req.method !== method) {
+    sendJson(req, res, 405, { error: 'method not allowed' });
+    return true;
+  }
+  const work =
+    path === CLAIM_PATH
+      ? handleClaim(req, res, deps)
+      : handleRead(req, res, deps, path);
+  work.catch((err: unknown) => {
+    log.warn({ err: errMsg(err) }, 'cli: unhandled error');
+    if (!res.headersSent) sendJson(req, res, 500, { error: 'cli api failed' });
+  });
+  return true;
+}

package/runtime/node_modules/@metro-labs/mcp/src/daemon/cli-pair.ts

@@ -0,0 +1,20 @@
+import { makeCodeStore } from './pair-codes.js';
+
+export interface CliCode {
+  email: string;
+  collectionId: string;
+}
+
+const store = makeCodeStore<CliCode>('mc');
+
+export const CLI_CODE_RE = store.pattern;
+
+export const mintCliCode = (
+  entry: CliCode,
+  now?: number,
+): { code: string; expiresAt: number } => store.mint(entry, now);
+
+export const takeCliCode = (code: string, now?: number): CliCode | undefined =>
+  store.take(code, now);
+
+export const cliCodeCount = (): number => store.count();

package/runtime/node_modules/@metro-labs/mcp/src/daemon/collection-api.ts

@@ -0,0 +1,202 @@
+import type { IncomingMessage, ServerResponse } from 'node:http';
+import { errMsg, log } from './log.js';
+import {
+  apiFailure,
+  apiSession,
+  projectParam,
+  bodyField,
+  cors,
+  readJsonBody,
+  sendJson,
+  type ApiSession,
+} from './api-http.js';
+import { parseId } from '../db/ids.js';
+import { mintCliCode } from './cli-pair.js';
+import type { ConnectorApiDeps } from './connector-api.js';
+
+const PREFIX = '/api/collections';
+
+type Routable =
+  | { kind: 'index' }
+  | { kind: 'collection'; id: string }
+  | { kind: 'rename'; id: string }
+  | { kind: 'code'; id: string }
+  | { kind: 'items'; id: string }
+  | { kind: 'item'; id: string; connectorId: string };
+
+type Target = Routable | { kind: 'unknown' } | null;
+
+const ALLOWED: Record<Routable['kind'], string[]> = {
+  index: ['GET', 'POST'],
+  collection: ['GET', 'DELETE'],
+  rename: ['POST'],
+  code: ['POST'],
+  items: ['POST'],
+  item: ['DELETE'],
+};
+
+function itemTarget(id: string, rest: string[]): Target {
+  if (rest.length === 1) return { kind: 'items', id };
+  if (rest.length > 2) return { kind: 'unknown' };
+  const connectorId = parseId(rest[1] ?? '');
+  return connectorId === null
+    ? { kind: 'unknown' }
+    : { kind: 'item', id, connectorId };
+}
+
+function subTarget(id: string, rest: string[]): Target {
+  if (rest.length === 0) return { kind: 'collection', id };
+  const head = rest[0];
+  if (head === 'items') return itemTarget(id, rest);
+  if (rest.length > 1) return { kind: 'unknown' };
+  if (head === 'rename') return { kind: 'rename', id };
+  if (head === 'code') return { kind: 'code', id };
+  return { kind: 'unknown' };
+}
+
+function target(path: string): Target {
+  if (path === PREFIX || path === `${PREFIX}/`) return { kind: 'index' };
+  if (!path.startsWith(`${PREFIX}/`)) return null;
+  const segments = path.slice(PREFIX.length + 1).split('/').filter(Boolean);
+  const head = segments[0];
+  if (head === undefined) return { kind: 'index' };
+  const id = parseId(head);
+  return id === null ? { kind: 'unknown' } : subTarget(id, segments.slice(1));
+}
+
+async function nameFrom(req: IncomingMessage): Promise<string | null> {
+  const name = bodyField(await readJsonBody(req), 'name');
+  return typeof name === 'string' ? name : null;
+}
+
+async function handleIndex(
+  req: IncomingMessage,
+  res: ServerResponse,
+  deps: ConnectorApiDeps,
+  session: ApiSession,
+): Promise<void> {
+  const project = projectParam(req);
+  if (project === null) {
+    sendJson(req, res, 400, { error: 'a project is required' });
+    return;
+  }
+  if (req.method === 'GET') {
+    sendJson(req, res, 200, {
+      collections: await deps.listCollections(session.email, project),
+    });
+    return;
+  }
+  const name = await nameFrom(req);
+  if (name === null) {
+    sendJson(req, res, 400, { error: 'name is required' });
+    return;
+  }
+  const list = await deps.createCollection(session.email, project, name);
+  log.info({ id: list.id, name: list.name }, 'collection-api: created collection');
+  sendJson(req, res, 201, list);
+}
+
+async function handleRename(
+  req: IncomingMessage,
+  res: ServerResponse,
+  deps: ConnectorApiDeps,
+  session: ApiSession,
+  id: string,
+): Promise<void> {
+  const name = await nameFrom(req);
+  if (name === null) {
+    sendJson(req, res, 400, { error: 'name is required' });
+    return;
+  }
+  sendJson(req, res, 200, await deps.renameCollection(session.email, id, name));
+}
+
+async function handleItems(
+  req: IncomingMessage,
+  res: ServerResponse,
+  deps: ConnectorApiDeps,
+  session: ApiSession,
+  id: string,
+): Promise<void> {
+  const connectorId = bodyField(await readJsonBody(req), 'connectorId');
+  if (typeof connectorId !== 'string' || parseId(connectorId) === null) {
+    sendJson(req, res, 400, { error: 'connectorId is required' });
+    return;
+  }
+  sendJson(req, res, 200, await deps.addToCollection(session.email, id, connectorId));
+}
+
+async function handleCode(
+  req: IncomingMessage,
+  res: ServerResponse,
+  deps: ConnectorApiDeps,
+  session: ApiSession,
+  id: string,
+): Promise<void> {
+  const collection = await deps.getCollection(session.email, id);
+  const minted = mintCliCode({ email: session.email, collectionId: collection.id });
+  log.info({ id: collection.id, email: session.email }, 'collection-api: minted cli code');
+  sendJson(req, res, 200, { ...minted, collection: collection.name });
+}
+
+async function route(
+  req: IncomingMessage,
+  res: ServerResponse,
+  deps: ConnectorApiDeps,
+  session: ApiSession,
+  tgt: Routable,
+): Promise<void> {
+  try {
+    if (tgt.kind === 'index')
+      await handleIndex(req, res, deps, session);
+    else if (tgt.kind === 'rename')
+      await handleRename(req, res, deps, session, tgt.id);
+    else if (tgt.kind === 'code')
+      await handleCode(req, res, deps, session, tgt.id);
+    else if (tgt.kind === 'items')
+      await handleItems(req, res, deps, session, tgt.id);
+    else if (tgt.kind === 'item')
+      sendJson(
+        req,
+        res,
+        200,
+        await deps.removeFromCollection(session.email, tgt.id, tgt.connectorId),
+      );
+    else if (req.method === 'DELETE')
+      sendJson(req, res, 200, await deps.deleteCollection(session.email, tgt.id));
+    else sendJson(req, res, 200, await deps.getCollection(session.email, tgt.id));
+  } catch (err) {
+    apiFailure(req, res, err, 'collection-api');
+  }
+}
+
+export function handleCollectionApiRequest(
+  req: IncomingMessage,
+  res: ServerResponse,
+  deps: ConnectorApiDeps,
+): boolean {
+  const tgt = target((req.url ?? '').split('?')[0] ?? '');
+  if (tgt === null) return false;
+  if (req.method === 'OPTIONS') {
+    res.writeHead(204, cors(req)).end();
+    return true;
+  }
+  if (tgt.kind === 'unknown') {
+    sendJson(req, res, 404, { error: 'no such collection' });
+    return true;
+  }
+  if (!ALLOWED[tgt.kind].includes(req.method ?? '')) {
+    sendJson(req, res, 405, { error: 'method not allowed' });
+    return true;
+  }
+  const session = apiSession(req);
+  if (!session) {
+    sendJson(req, res, 401, { error: 'unauthorized' });
+    return true;
+  }
+  route(req, res, deps, session, tgt).catch((err: unknown) => {
+    log.warn({ err: errMsg(err) }, 'list-api: unhandled error');
+    if (!res.headersSent) sendJson(req, res, 500, { error: 'list api failed' });
+  });
+  return true;
+}