@stage-labs/metro 0.1.0-beta.21 → 0.1.0-beta.210

package/runtime/node_modules/@metro-labs/{mcp/src/daemon → daemon/src/stations}/attach-session.ts

@@ -1,17 +1,18 @@
 import { randomBytes } from 'node:crypto';
-import { ApiError } from './api-error.js';
-import { errMsg, log } from './log.js';
-import type { StationName } from '../db/schema.js';
-import {
-  startInteractiveAttach,
-  type AttachDriver,
-  type AttachOutcome,
-  type AttachPrompt,
-  type AttachStep,
-  type DriverHooks,
-  type InteractiveStation,
-  type StartedAttach,
-} from '../stations/attach-interactive.js';
+import { ApiError } from '@metro-labs/http/api-error';
+import { errMsg, log } from '@metro-labs/core/log';
+import type { StationName } from '@metro-labs/core/station-names';
+import { ensureStationDeps } from './runtime-deps.js';
+import { startInteractiveAttach, type InteractiveStation } from './attach-interactive.js';
+import type {
+  AttachDriver,
+  AttachOutcome,
+  AttachPrompt,
+  AttachStep,
+  DriverHooks,
+  StartedAttach,
+  StepInput,
+} from './attach-driver.js';
 
 export type StartAttach = (
   station: InteractiveStation,
@@ -28,7 +29,6 @@ const MAX_PER_AGENT = 2;
 const MAX_TOTAL = 40;
 
 export interface AttachOwner {
-  email: string;
   agentId: string;
 }
 
@@ -40,6 +40,9 @@ export interface AttachView {
   prompt: string;
   qr: string | null;
   pairingCode: string | null;
+  userCode: string | null;
+  verificationUri: string | null;
+  authorizeUrl: string | null;
   accountId: string | null;
   identity: Record<string, string>;
   activated: boolean;
@@ -47,13 +50,13 @@ export interface AttachView {
   expiresAt: number;
 }
 
-export type CompleteAttach = (
+type CompleteAttach = (
   owner: AttachOwner,
   station: StationName,
   config: Record<string, unknown>,
 ) => Promise<{ accountId: string; activated: boolean }>;
 
-export type AuthorizeAttach = (owner: AttachOwner) => Promise<void>;
+type AuthorizeAttach = (owner: AttachOwner) => Promise<void>;
 
 export interface AttachSessionDeps {
   authorize: AuthorizeAttach;
@@ -82,6 +85,9 @@ function blankView(attachId: string, station: string): AttachView {
     prompt: '',
     qr: null,
     pairingCode: null,
+    userCode: null,
+    verificationUri: null,
+    authorizeUrl: null,
     accountId: null,
     identity: {},
     activated: false,
@@ -108,11 +114,7 @@ export class AttachSessions {
 
   private own(owner: AttachOwner, attachId: string): Session {
     const session = this.sessions.get(attachId);
-    if (
-      session?.owner.email !== owner.email ||
-      session.owner.agentId !== owner.agentId
-    )
-      throw missing();
+    if (session?.owner.agentId !== owner.agentId) throw missing();
     return session;
   }
 
@@ -136,6 +138,9 @@ export class AttachSessions {
     Object.assign(session.view, patch, {
       qr: null,
       pairingCode: null,
+      userCode: null,
+      verificationUri: null,
+      authorizeUrl: null,
       step: null,
       expiresAt: Date.now() + SETTLED_TTL_MS,
     });
@@ -183,6 +188,9 @@ export class AttachSessions {
         session.view.prompt = p.prompt;
         session.view.qr = p.qr ?? null;
         session.view.pairingCode = p.pairingCode ?? null;
+        session.view.userCode = p.userCode ?? null;
+        session.view.verificationUri = p.verificationUri ?? null;
+        session.view.authorizeUrl = p.authorizeUrl ?? null;
       },
       done: (o: AttachOutcome): void => {
         this.finish(attachId, station, o).catch((err: unknown) => {
@@ -216,12 +224,14 @@ export class AttachSessions {
     };
     this.sessions.set(attachId, session);
     try {
+      ensureStationDeps(station);
       const started = await (this.deps.start ?? startInteractiveAttach)(
         station,
         input,
         this.hooksFor(attachId, station),
       );
       session.driver = started.driver;
+      if (started.expiresAt !== undefined) session.view.expiresAt = started.expiresAt;
       this.hooksFor(attachId, station).prompt(started.prompt);
     } catch (err) {
       this.sessions.delete(attachId);
@@ -241,7 +251,7 @@ export class AttachSessions {
   async submit(
     owner: AttachOwner,
     attachId: string,
-    input: { code?: unknown; password?: unknown },
+    input: StepInput,
   ): Promise<AttachView> {
     const session = this.own(owner, attachId);
     const driver = session.driver;

package/runtime/node_modules/@metro-labs/{mcp → daemon}/src/stations/attach-xmtp.ts

@@ -1,7 +1,8 @@
 import { randomBytes } from 'node:crypto';
+import { stringOf } from '@metro-labs/http/api-http';
 import { rmSync } from 'node:fs';
 import { fileURLToPath } from 'node:url';
-import { errMsg, log } from '../daemon/log.js';
+import { errMsg, log } from '@metro-labs/core/log';
 
 export class XmtpAttachError extends Error {}
 
@@ -43,8 +44,6 @@ interface VerifyReply {
   error?: unknown;
 }
 
-const str = (value: unknown): string => (typeof value === 'string' ? value : '');
-
 function readReply(raw: string): VerifyReply {
   const line = raw.trim().split('\n').at(-1) ?? '';
   try {
@@ -89,7 +88,7 @@ export async function verifyXmtpKeyOutOfProcess(
   dbPath: string,
 ): Promise<XmtpIdentity> {
   const raw = await runVerifier(
-    JSON.stringify({ privateKey, dbPath }),
+    JSON.stringify({ privateKey, dbPath, smart: true }),
   ).catch((err: unknown) => {
     log.warn({ err: errMsg(err) }, 'attach-xmtp: could not run the inbox check');
     throw new XmtpAttachError(UNREACHABLE);
@@ -97,13 +96,13 @@ export async function verifyXmtpKeyOutOfProcess(
   const reply = readReply(raw);
   if (reply.ok !== true)
     throw new XmtpAttachError(
-      withoutKey(str(reply.error) || UNREACHABLE, privateKey),
+      withoutKey(stringOf(reply.error) || UNREACHABLE, privateKey),
     );
   const identity = {
-    inboxId: str(reply.inboxId),
-    address: str(reply.address),
-    installationId: str(reply.installationId),
-    dbPath: str(reply.dbPath),
+    inboxId: stringOf(reply.inboxId),
+    address: stringOf(reply.address),
+    installationId: stringOf(reply.installationId),
+    dbPath: stringOf(reply.dbPath),
   };
   if (identity.inboxId === '' || identity.address === '')
     throw new XmtpAttachError(UNREACHABLE);

package/runtime/node_modules/@metro-labs/{mcp → daemon}/src/stations/attach.ts

@@ -2,13 +2,20 @@ import { randomBytes } from 'node:crypto';
 import {
   DiscordTokenError,
   verifyDiscordBotToken,
-} from '@metro-labs/discord/verify';
+} from '@metro-labs/discord-bot/verify';
 import {
   TelegramTokenError,
   verifyTelegramBotToken,
-} from '@metro-labs/telegram/verify';
-import { ApiError } from '../daemon/api-error.js';
-import { publicBaseOrDefault } from '../daemon/attach-serve.js';
+} from '@metro-labs/telegram-bot/verify';
+import {
+  parsePrivateKey,
+  ThreemaGatewayError,
+  verifyThreemaGateway,
+} from '@metro-labs/threema/verify';
+import { ApiError } from '@metro-labs/http/api-error';
+import { bodyField } from '@metro-labs/http/api-http';
+import { ensureStationDeps } from './runtime-deps.js';
+import { publicBaseOrDefault } from '../files/attach-serve.js';
 import {
   discardXmtpDb,
   newXmtpDbPath,
@@ -18,8 +25,9 @@ import {
 } from './attach-xmtp.js';
 
 export const ATTACHABLE_STATIONS = [
-  'discord',
-  'telegram',
+  'discord-bot',
+  'telegram-bot',
+  'threema',
   'xmtp',
   'webhook',
 ] as const;
@@ -31,6 +39,9 @@ export class StationAttachError extends ApiError {}
 export interface AttachInput {
   station: AttachStation;
   token?: unknown;
+  gatewayId?: unknown;
+  secret?: unknown;
+  privateKey?: unknown;
 }
 
 export interface OneTimeSecret {
@@ -52,6 +63,16 @@ const SECP256K1_ORDER = BigInt(
   '0xfffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141',
 );
 
+export function attachInputOf(station: AttachStation, body: unknown): AttachInput {
+  return {
+    station,
+    token: bodyField(body, 'token'),
+    gatewayId: bodyField(body, 'gatewayId'),
+    secret: bodyField(body, 'secret'),
+    privateKey: bodyField(body, 'privateKey'),
+  };
+}
+
 export function isAttachStation(raw: unknown): raw is AttachStation {
   return (
     typeof raw === 'string' &&
@@ -137,7 +158,7 @@ async function prepareXmtp(verify: VerifyXmtpKey): Promise<PreparedAccount> {
     ),
   );
   return {
-    config: { privateKey, dbPath },
+    config: { privateKey, dbPath, smart: true },
     identity: { inboxId: identity.inboxId, address: identity.address },
     secret: {
       label: 'xmtp private key',
@@ -162,6 +183,49 @@ export function newWebhookId(): string {
   return String(WEBHOOK_ID_FLOOR + (raw % (9n * WEBHOOK_ID_FLOOR)));
 }
 
+export const threemaCallbackUrl = (callbackId: string, token: string): string =>
+  `${publicBaseOrDefault().replace(/\/+$/, '')}/api/threema/${callbackId}/${token}`;
+
+function requireText(raw: unknown, label: string): string {
+  const text = typeof raw === 'string' ? raw.trim() : '';
+  if (text === '')
+    throw new StationAttachError(`the Threema ${label} is required`, 400);
+  return text;
+}
+
+async function prepareThreema(input: AttachInput): Promise<PreparedAccount> {
+  const gatewayId = requireText(input.gatewayId, 'Gateway ID');
+  const secret = requireText(input.secret, 'API secret');
+  const privateKey = requireText(input.privateKey, 'private key');
+  try {
+    const identity = await verifyThreemaGateway({ gatewayId, secret, privateKey });
+    const callbackId = newWebhookId();
+    const callbackToken = randomBytes(48).toString('base64url');
+    return {
+      config: {
+        gatewayId: identity.gatewayId,
+        secret,
+        privateKey: parsePrivateKey(privateKey) ?? privateKey,
+        callbackId,
+        callbackToken,
+        createdAt: new Date().toISOString(),
+      },
+      identity: {
+        gatewayId: identity.gatewayId,
+        credits: String(identity.credits),
+        callback: threemaCallbackUrl(callbackId, callbackToken),
+      },
+    };
+  } catch (err) {
+    if (err instanceof StationAttachError) throw err;
+    return rejected(
+      err,
+      err instanceof ThreemaGatewayError,
+      'Threema rejected those Gateway credentials',
+    );
+  }
+}
+
 function prepareWebhook(): PreparedAccount {
   const secret = randomBytes(48).toString('base64url');
   const webhookId = newWebhookId();
@@ -175,8 +239,10 @@ export async function prepareAccount(
   input: AttachInput,
   verify: VerifyXmtpKey = verifyXmtpKeyOutOfProcess,
 ): Promise<PreparedAccount> {
-  if (input.station === 'discord') return prepareDiscord(input.token);
-  if (input.station === 'telegram') return prepareTelegram(input.token);
+  ensureStationDeps(input.station);
+  if (input.station === 'discord-bot') return prepareDiscord(input.token);
+  if (input.station === 'telegram-bot') return prepareTelegram(input.token);
   if (input.station === 'webhook') return prepareWebhook();
+  if (input.station === 'threema') return prepareThreema(input);
   return prepareXmtp(verify);
 }

package/runtime/node_modules/@metro-labs/daemon/src/stations/materialize.ts

@@ -0,0 +1,265 @@
+import { setPolicies, type PolicyTarget, type ToolPolicy } from '../policy/policy.js';
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { log } from '@metro-labs/core/log';
+import { writeSecure } from '@metro-labs/core/secure-fs';
+import { trainsDir } from '../boot/paths.js';
+import {
+  setAgentMap,
+  setAllowlistMap,
+  setApproversMap,
+  setDisabledAccounts,
+  type AgentMap,
+  type AgentNameMap,
+  type AllowlistMap,
+} from '../agents/map.js';
+import { setKeyMap } from '../agents/keys.js';
+import { STATIONS, type StationName } from '@metro-labs/core/station-names';
+
+interface StationTarget {
+  file: string;
+  fileEnv: string;
+  trainImport: string | null;
+}
+
+export interface LoadedAccount {
+  station: StationName;
+  id: string;
+  allowlist: string[] | null;
+  approvers?: string[];
+  enabled?: boolean;
+  policy?: ToolPolicy;
+  config: Record<string, unknown>;
+}
+
+export interface LoadedConnector {
+  id: string;
+  name: string;
+  url: string;
+  config: Record<string, unknown>;
+}
+
+export interface LoadedAgent {
+  id: string;
+  name: string;
+  accounts: LoadedAccount[];
+  key: string | null;
+}
+
+const METRO_DIR = join(homedir(), '.metro');
+
+const STATION_TARGETS: Record<StationName, StationTarget> = {
+  xmtp: {
+    file: 'xmtp-accounts.json',
+    fileEnv: 'XMTP_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/xmtp/train',
+  },
+  'telegram-bot': {
+    file: 'telegram-bot-accounts.json',
+    fileEnv: 'TELEGRAM_BOT_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/telegram-bot/train',
+  },
+  'telegram': {
+    file: 'telegram-accounts.json',
+    fileEnv: 'TELEGRAM_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/telegram/train',
+  },
+  'discord-bot': {
+    file: 'discord-bot-accounts.json',
+    fileEnv: 'DISCORD_BOT_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/discord-bot/train',
+  },
+  whatsapp: {
+    file: 'whatsapp-accounts.json',
+    fileEnv: 'WHATSAPP_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/whatsapp/train',
+  },
+  threema: {
+    file: 'threema-accounts.json',
+    fileEnv: 'THREEMA_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/threema/train',
+  },
+  outlook: {
+    file: 'outlook-accounts.json',
+    fileEnv: 'OUTLOOK_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/outlook/train',
+  },
+  webhook: {
+    file: 'webhook-accounts.json',
+    fileEnv: 'WEBHOOK_ACCOUNTS_FILE',
+    trainImport: null,
+  },
+};
+
+export function accountFilePath(station: StationName): string {
+  const target = STATION_TARGETS[station];
+  return process.env[target.fileEnv] ?? join(METRO_DIR, target.file);
+}
+
+export type StationSource = () => Promise<LoadedAgent[]>;
+
+export const MOVABLE_STATIONS = new Set<StationName>(
+  STATIONS.filter((s) => s !== 'webhook'),
+);
+
+function trainStubPath(station: StationName): string {
+  return join(trainsDir(), `${station}.ts`);
+}
+
+function currentText(path: string): string | null {
+  try {
+    return readFileSync(path, 'utf8');
+  } catch {
+    return null;
+  }
+}
+
+export function writeIfChanged(path: string, content: string): boolean {
+  if (currentText(path) === content) return false;
+  writeFileSync(path, content);
+  return true;
+}
+
+interface WrittenStations {
+  active: Map<StationName, number>;
+  changed: StationName[];
+}
+
+const channelPolicies = (list: LoadedAgent[]): [PolicyTarget, ToolPolicy][] =>
+  list.flatMap((agent) =>
+    agent.accounts.flatMap((a): [PolicyTarget, ToolPolicy][] =>
+      a.policy === undefined ? [] : [[{ kind: 'channel', station: a.station, account: a.id }, a.policy]],
+    ),
+  );
+
+const approversOf = (list: LoadedAgent[]): Record<string, string[]> =>
+  Object.fromEntries(
+    list.flatMap((agent) => agent.accounts.flatMap((a) => (a.approvers === undefined ? [] : [[`${a.station}/${a.id}`, a.approvers]]))),
+  );
+
+function writeStations(list: LoadedAgent[]): WrittenStations {
+  mkdirSync(METRO_DIR, { recursive: true });
+  mkdirSync(trainsDir(), { recursive: true });
+
+  const byStation = new Map<StationName, LoadedAccount[]>();
+  const map: AgentMap = {};
+  const names: AgentNameMap = {};
+  const allow: AllowlistMap = {};
+  const disabled = new Set<string>();
+  for (const agent of list) {
+    names[agent.id] = agent.name;
+    for (const a of agent.accounts) {
+      if (!(a.station in STATION_TARGETS)) {
+        log.error(
+          { station: a.station, id: a.id },
+          'unknown station — this metro is older or newer than the row; skipping it',
+        );
+        continue;
+      }
+      map[`${a.station}/${a.id}`] = agent.id;
+      if (a.allowlist) allow[`${a.station}/${a.id}`] = a.allowlist;
+      if (a.enabled === false) {
+        disabled.add(`${a.station}/${a.id}`);
+        continue;
+      }
+      const cur = byStation.get(a.station);
+      if (cur) cur.push(a);
+      else byStation.set(a.station, [a]);
+    }
+  }
+  setAgentMap(map, names);
+  setAllowlistMap(allow);
+  setApproversMap(approversOf(list));
+  setDisabledAccounts(disabled);
+  setPolicies('channel', channelPolicies(list));
+
+  const active = new Map<StationName, number>();
+  const changed: StationName[] = [];
+  for (const [station, accts] of byStation) {
+    const records = accts.map((a) => ({ id: a.id, ...a.config }));
+    const path = accountFilePath(station);
+    const content = JSON.stringify(records, null, 2);
+    if (currentText(path) !== content) {
+      writeSecure(path, content);
+      changed.push(station);
+    }
+    const { trainImport } = STATION_TARGETS[station];
+    if (trainImport !== null)
+      writeIfChanged(trainStubPath(station), `import '${trainImport}';\n`);
+    active.set(station, accts.length);
+  }
+  return { active, changed };
+}
+
+const BLANKED = '[]\n';
+
+function pruneStations(present: Map<StationName, number>): StationName[] {
+  const removed: StationName[] = [];
+  for (const station of Object.keys(STATION_TARGETS) as StationName[]) {
+    if (present.has(station)) continue;
+    const hasTrain = STATION_TARGETS[station].trainImport !== null;
+    const stub = trainStubPath(station);
+    const stubGone = !hasTrain || !existsSync(stub);
+    const file = currentText(accountFilePath(station));
+    const fileInert = file === null || file === BLANKED;
+    if (stubGone && fileInert) continue;
+    if (!fileInert) writeSecure(accountFilePath(station), BLANKED);
+    if (hasTrain) rmSync(stub, { force: true });
+    removed.push(station);
+  }
+  return removed;
+}
+
+const stationLabels = (m: Map<StationName, number>): string[] =>
+  [...m].map(([station, n]) => `${station}(${n})`);
+
+function applyKeyMap(list: LoadedAgent[]): void {
+  setKeyMap(
+    list.flatMap((agent) =>
+      agent.key === null ? [] : [{ key: agent.key, agentId: agent.id }],
+    ),
+  );
+}
+
+async function loadAndWrite(
+  source: StationSource,
+): Promise<WrittenStations & { agents: number }> {
+  const list = await source();
+  applyKeyMap(list);
+  return { ...writeStations(list), agents: list.length };
+}
+
+export async function materializeFrom(source: StationSource): Promise<void> {
+  const { active, agents: found } = await loadAndWrite(source);
+  const removed = pruneStations(active);
+  log.info(
+    { stations: stationLabels(active), agents: found, removed },
+    'materialized station accounts',
+  );
+}
+
+export interface ReloadedStations {
+  active: StationName[];
+  removed: StationName[];
+  changed: StationName[];
+}
+
+export async function reloadFrom(
+  source: StationSource,
+): Promise<ReloadedStations> {
+  const { active, changed } = await loadAndWrite(source);
+  const removed = pruneStations(active);
+  if (removed.length > 0 || changed.length > 0)
+    log.info(
+      { stations: stationLabels(active), removed, changed },
+      'reloaded station accounts',
+    );
+  return { active: [...active.keys()], removed, changed };
+}

package/runtime/node_modules/@metro-labs/{mcp → daemon}/src/stations/registry.ts

@@ -1,18 +1,23 @@
-import type { Station } from './types.js';
-import { Line } from './lines.js';
+import type { Station } from '@metro-labs/core/stations/types';
+import { Line } from '@metro-labs/core/lines';
+import { errMsg, log } from '@metro-labs/core/log';
 import { xmtpStation } from '@metro-labs/xmtp';
+import { telegramBotStation } from '@metro-labs/telegram-bot';
 import { telegramStation } from '@metro-labs/telegram';
-import { telegramUserStation } from '@metro-labs/telegram-user';
-import { discordStation } from '@metro-labs/discord';
+import { discordBotStation } from '@metro-labs/discord-bot';
 import { whatsappStation } from '@metro-labs/whatsapp';
+import { threemaStation } from '@metro-labs/threema';
+import { outlookStation } from '@metro-labs/outlook';
 import { webhookStation } from '@metro-labs/webhook';
 
 export const STATIONS: readonly Station[] = [
   xmtpStation,
+  telegramBotStation,
   telegramStation,
-  telegramUserStation,
-  discordStation,
+  discordBotStation,
   whatsappStation,
+  threemaStation,
+  outlookStation,
   webhookStation,
 ];
 
@@ -33,3 +38,14 @@ export const accountStationCapabilities = (): Record<string, string[]> => {
     if (s.hasAccounts) out[s.name] = [...s.messageVerbs].sort();
   return out;
 };
+
+export function forgetOrphans(known: readonly { station: string; id: string }[]): void {
+  for (const s of STATIONS) {
+    if (s.forgetExcept === undefined) continue;
+    try {
+      s.forgetExcept(known.filter((a) => a.station === s.name).map((a) => a.id));
+    } catch (err) {
+      log.warn({ station: s.name, err: errMsg(err) }, 'stations: could not remove the files of detached accounts');
+    }
+  }
+}