@stage-labs/metro 0.1.0-beta.2 → 0.1.0-beta.200

package/runtime/node_modules/@metro-labs/daemon/src/stations/attach.ts

@@ -0,0 +1,248 @@
+import { randomBytes } from 'node:crypto';
+import {
+  DiscordTokenError,
+  verifyDiscordBotToken,
+} from '@metro-labs/discord-bot/verify';
+import {
+  TelegramTokenError,
+  verifyTelegramBotToken,
+} from '@metro-labs/telegram-bot/verify';
+import {
+  parsePrivateKey,
+  ThreemaGatewayError,
+  verifyThreemaGateway,
+} from '@metro-labs/threema/verify';
+import { ApiError } from '@metro-labs/http/api-error';
+import { bodyField } from '@metro-labs/http/api-http';
+import { ensureStationDeps } from './runtime-deps.js';
+import { publicBaseOrDefault } from '../files/attach-serve.js';
+import {
+  discardXmtpDb,
+  newXmtpDbPath,
+  verifyXmtpKeyOutOfProcess,
+  XmtpAttachError,
+  type VerifyXmtpKey,
+} from './attach-xmtp.js';
+
+export const ATTACHABLE_STATIONS = [
+  'discord-bot',
+  'telegram-bot',
+  'threema',
+  'xmtp',
+  'webhook',
+] as const;
+
+export type AttachStation = (typeof ATTACHABLE_STATIONS)[number];
+
+export class StationAttachError extends ApiError {}
+
+export interface AttachInput {
+  station: AttachStation;
+  token?: unknown;
+  gatewayId?: unknown;
+  secret?: unknown;
+  privateKey?: unknown;
+}
+
+export interface OneTimeSecret {
+  label: string;
+  value: string;
+  note: string;
+}
+
+export interface PreparedAccount {
+  config: Record<string, unknown>;
+  identity: Record<string, string>;
+  secret?: OneTimeSecret;
+  discard?: () => void;
+}
+
+const TOKEN_RE = /^[A-Za-z0-9._:-]{8,256}$/;
+
+const SECP256K1_ORDER = BigInt(
+  '0xfffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141',
+);
+
+export function attachInputOf(station: AttachStation, body: unknown): AttachInput {
+  return {
+    station,
+    token: bodyField(body, 'token'),
+    gatewayId: bodyField(body, 'gatewayId'),
+    secret: bodyField(body, 'secret'),
+    privateKey: bodyField(body, 'privateKey'),
+  };
+}
+
+export function isAttachStation(raw: unknown): raw is AttachStation {
+  return (
+    typeof raw === 'string' &&
+    (ATTACHABLE_STATIONS as readonly string[]).includes(raw)
+  );
+}
+
+function requireToken(raw: unknown, label: string): string {
+  const token = typeof raw === 'string' ? raw.trim() : '';
+  if (token === '')
+    throw new StationAttachError(`a ${label} bot token is required`, 400);
+  if (!TOKEN_RE.test(token))
+    throw new StationAttachError(
+      `that does not look like a ${label} bot token`,
+      400,
+    );
+  return token;
+}
+
+function rejected(err: unknown, own: boolean, fallback: string): never {
+  throw new StationAttachError(
+    own && err instanceof Error ? err.message : fallback,
+    400,
+  );
+}
+
+async function prepareDiscord(raw: unknown): Promise<PreparedAccount> {
+  const token = requireToken(raw, 'Discord');
+  try {
+    const bot = await verifyDiscordBotToken(token);
+    if (!bot.messageContent)
+      throw new StationAttachError(
+        'this Discord application does not have the Message Content intent enabled. Turn it on under Bot / Privileged Gateway Intents in the developer portal, then attach it again',
+        400,
+      );
+    return {
+      config: { token },
+      identity: { userId: bot.userId, username: bot.username },
+    };
+  } catch (err) {
+    if (err instanceof StationAttachError) throw err;
+    return rejected(
+      err,
+      err instanceof DiscordTokenError,
+      'Discord rejected that bot token',
+    );
+  }
+}
+
+async function prepareTelegram(raw: unknown): Promise<PreparedAccount> {
+  const token = requireToken(raw, 'Telegram');
+  try {
+    const bot = await verifyTelegramBotToken(token);
+    return {
+      config: { token },
+      identity: { botId: String(bot.botId), username: bot.username },
+    };
+  } catch (err) {
+    return rejected(
+      err,
+      err instanceof TelegramTokenError,
+      'Telegram rejected that bot token',
+    );
+  }
+}
+
+export function newXmtpPrivateKey(): string {
+  for (;;) {
+    const hex = randomBytes(32).toString('hex');
+    const value = BigInt(`0x${hex}`);
+    if (value > 0n && value < SECP256K1_ORDER) return `0x${hex}`;
+  }
+}
+
+async function prepareXmtp(verify: VerifyXmtpKey): Promise<PreparedAccount> {
+  const privateKey = newXmtpPrivateKey();
+  const dbPath = newXmtpDbPath();
+  const identity = await verify(privateKey, dbPath).catch((err: unknown) =>
+    rejected(
+      err,
+      err instanceof XmtpAttachError,
+      'Metro could not open an XMTP inbox with the key it generated, so nothing was attached',
+    ),
+  );
+  return {
+    config: { privateKey, dbPath, smart: true },
+    identity: { inboxId: identity.inboxId, address: identity.address },
+    secret: {
+      label: 'xmtp private key',
+      value: privateKey,
+      note: 'Metro generated this key, opened inbox ' +
+        `${identity.inboxId} with it, and stores it with the account. ` +
+        'It is shown once, here, and is never returned by any API again.',
+    },
+    discard: () => {
+      discardXmtpDb(identity.dbPath);
+    },
+  };
+}
+
+export const hookUrl = (webhookId: string, secret: string): string =>
+  `${publicBaseOrDefault().replace(/\/+$/, '')}/api/webhooks/${webhookId}/${secret}`;
+
+const WEBHOOK_ID_FLOOR = 10n ** 18n;
+
+export function newWebhookId(): string {
+  const raw = BigInt(`0x${randomBytes(8).toString('hex')}`);
+  return String(WEBHOOK_ID_FLOOR + (raw % (9n * WEBHOOK_ID_FLOOR)));
+}
+
+export const threemaCallbackUrl = (callbackId: string, token: string): string =>
+  `${publicBaseOrDefault().replace(/\/+$/, '')}/api/threema/${callbackId}/${token}`;
+
+function requireText(raw: unknown, label: string): string {
+  const text = typeof raw === 'string' ? raw.trim() : '';
+  if (text === '')
+    throw new StationAttachError(`the Threema ${label} is required`, 400);
+  return text;
+}
+
+async function prepareThreema(input: AttachInput): Promise<PreparedAccount> {
+  const gatewayId = requireText(input.gatewayId, 'Gateway ID');
+  const secret = requireText(input.secret, 'API secret');
+  const privateKey = requireText(input.privateKey, 'private key');
+  try {
+    const identity = await verifyThreemaGateway({ gatewayId, secret, privateKey });
+    const callbackId = newWebhookId();
+    const callbackToken = randomBytes(48).toString('base64url');
+    return {
+      config: {
+        gatewayId: identity.gatewayId,
+        secret,
+        privateKey: parsePrivateKey(privateKey) ?? privateKey,
+        callbackId,
+        callbackToken,
+        createdAt: new Date().toISOString(),
+      },
+      identity: {
+        gatewayId: identity.gatewayId,
+        credits: String(identity.credits),
+        callback: threemaCallbackUrl(callbackId, callbackToken),
+      },
+    };
+  } catch (err) {
+    if (err instanceof StationAttachError) throw err;
+    return rejected(
+      err,
+      err instanceof ThreemaGatewayError,
+      'Threema rejected those Gateway credentials',
+    );
+  }
+}
+
+function prepareWebhook(): PreparedAccount {
+  const secret = randomBytes(48).toString('base64url');
+  const webhookId = newWebhookId();
+  return {
+    config: { secret, webhookId, createdAt: new Date().toISOString() },
+    identity: { endpoint: hookUrl(webhookId, secret) },
+  };
+}
+
+export async function prepareAccount(
+  input: AttachInput,
+  verify: VerifyXmtpKey = verifyXmtpKeyOutOfProcess,
+): Promise<PreparedAccount> {
+  ensureStationDeps(input.station);
+  if (input.station === 'discord-bot') return prepareDiscord(input.token);
+  if (input.station === 'telegram-bot') return prepareTelegram(input.token);
+  if (input.station === 'webhook') return prepareWebhook();
+  if (input.station === 'threema') return prepareThreema(input);
+  return prepareXmtp(verify);
+}

package/runtime/node_modules/@metro-labs/daemon/src/stations/materialize.ts

@@ -0,0 +1,265 @@
+import { setPolicies, type PolicyTarget, type ToolPolicy } from '../policy/policy.js';
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { log } from '@metro-labs/core/log';
+import { writeSecure } from '@metro-labs/core/secure-fs';
+import { trainsDir } from '../boot/paths.js';
+import {
+  setAgentMap,
+  setAllowlistMap,
+  setApproversMap,
+  setDisabledAccounts,
+  type AgentMap,
+  type AgentNameMap,
+  type AllowlistMap,
+} from '../agents/map.js';
+import { setKeyMap } from '../agents/keys.js';
+import { STATIONS, type StationName } from '@metro-labs/core/station-names';
+
+interface StationTarget {
+  file: string;
+  fileEnv: string;
+  trainImport: string | null;
+}
+
+export interface LoadedAccount {
+  station: StationName;
+  id: string;
+  allowlist: string[] | null;
+  approvers?: string[];
+  enabled?: boolean;
+  policy?: ToolPolicy;
+  config: Record<string, unknown>;
+}
+
+export interface LoadedConnector {
+  id: string;
+  name: string;
+  url: string;
+  config: Record<string, unknown>;
+}
+
+export interface LoadedAgent {
+  id: string;
+  name: string;
+  accounts: LoadedAccount[];
+  key: string | null;
+}
+
+const METRO_DIR = join(homedir(), '.metro');
+
+const STATION_TARGETS: Record<StationName, StationTarget> = {
+  xmtp: {
+    file: 'xmtp-accounts.json',
+    fileEnv: 'XMTP_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/xmtp/train',
+  },
+  'telegram-bot': {
+    file: 'telegram-bot-accounts.json',
+    fileEnv: 'TELEGRAM_BOT_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/telegram-bot/train',
+  },
+  'telegram': {
+    file: 'telegram-accounts.json',
+    fileEnv: 'TELEGRAM_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/telegram/train',
+  },
+  'discord-bot': {
+    file: 'discord-bot-accounts.json',
+    fileEnv: 'DISCORD_BOT_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/discord-bot/train',
+  },
+  whatsapp: {
+    file: 'whatsapp-accounts.json',
+    fileEnv: 'WHATSAPP_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/whatsapp/train',
+  },
+  threema: {
+    file: 'threema-accounts.json',
+    fileEnv: 'THREEMA_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/threema/train',
+  },
+  outlook: {
+    file: 'outlook-accounts.json',
+    fileEnv: 'OUTLOOK_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/outlook/train',
+  },
+  webhook: {
+    file: 'webhook-accounts.json',
+    fileEnv: 'WEBHOOK_ACCOUNTS_FILE',
+    trainImport: null,
+  },
+};
+
+export function accountFilePath(station: StationName): string {
+  const target = STATION_TARGETS[station];
+  return process.env[target.fileEnv] ?? join(METRO_DIR, target.file);
+}
+
+export type StationSource = () => Promise<LoadedAgent[]>;
+
+export const MOVABLE_STATIONS = new Set<StationName>(
+  STATIONS.filter((s) => s !== 'webhook'),
+);
+
+function trainStubPath(station: StationName): string {
+  return join(trainsDir(), `${station}.ts`);
+}
+
+function currentText(path: string): string | null {
+  try {
+    return readFileSync(path, 'utf8');
+  } catch {
+    return null;
+  }
+}
+
+export function writeIfChanged(path: string, content: string): boolean {
+  if (currentText(path) === content) return false;
+  writeFileSync(path, content);
+  return true;
+}
+
+interface WrittenStations {
+  active: Map<StationName, number>;
+  changed: StationName[];
+}
+
+const channelPolicies = (list: LoadedAgent[]): [PolicyTarget, ToolPolicy][] =>
+  list.flatMap((agent) =>
+    agent.accounts.flatMap((a): [PolicyTarget, ToolPolicy][] =>
+      a.policy === undefined ? [] : [[{ kind: 'channel', station: a.station, account: a.id }, a.policy]],
+    ),
+  );
+
+const approversOf = (list: LoadedAgent[]): Record<string, string[]> =>
+  Object.fromEntries(
+    list.flatMap((agent) => agent.accounts.flatMap((a) => (a.approvers === undefined ? [] : [[`${a.station}/${a.id}`, a.approvers]]))),
+  );
+
+function writeStations(list: LoadedAgent[]): WrittenStations {
+  mkdirSync(METRO_DIR, { recursive: true });
+  mkdirSync(trainsDir(), { recursive: true });
+
+  const byStation = new Map<StationName, LoadedAccount[]>();
+  const map: AgentMap = {};
+  const names: AgentNameMap = {};
+  const allow: AllowlistMap = {};
+  const disabled = new Set<string>();
+  for (const agent of list) {
+    names[agent.id] = agent.name;
+    for (const a of agent.accounts) {
+      if (!(a.station in STATION_TARGETS)) {
+        log.error(
+          { station: a.station, id: a.id },
+          'unknown station — this metro is older or newer than the row; skipping it',
+        );
+        continue;
+      }
+      map[`${a.station}/${a.id}`] = agent.id;
+      if (a.allowlist) allow[`${a.station}/${a.id}`] = a.allowlist;
+      if (a.enabled === false) {
+        disabled.add(`${a.station}/${a.id}`);
+        continue;
+      }
+      const cur = byStation.get(a.station);
+      if (cur) cur.push(a);
+      else byStation.set(a.station, [a]);
+    }
+  }
+  setAgentMap(map, names);
+  setAllowlistMap(allow);
+  setApproversMap(approversOf(list));
+  setDisabledAccounts(disabled);
+  setPolicies('channel', channelPolicies(list));
+
+  const active = new Map<StationName, number>();
+  const changed: StationName[] = [];
+  for (const [station, accts] of byStation) {
+    const records = accts.map((a) => ({ id: a.id, ...a.config }));
+    const path = accountFilePath(station);
+    const content = JSON.stringify(records, null, 2);
+    if (currentText(path) !== content) {
+      writeSecure(path, content);
+      changed.push(station);
+    }
+    const { trainImport } = STATION_TARGETS[station];
+    if (trainImport !== null)
+      writeIfChanged(trainStubPath(station), `import '${trainImport}';\n`);
+    active.set(station, accts.length);
+  }
+  return { active, changed };
+}
+
+const BLANKED = '[]\n';
+
+function pruneStations(present: Map<StationName, number>): StationName[] {
+  const removed: StationName[] = [];
+  for (const station of Object.keys(STATION_TARGETS) as StationName[]) {
+    if (present.has(station)) continue;
+    const hasTrain = STATION_TARGETS[station].trainImport !== null;
+    const stub = trainStubPath(station);
+    const stubGone = !hasTrain || !existsSync(stub);
+    const file = currentText(accountFilePath(station));
+    const fileInert = file === null || file === BLANKED;
+    if (stubGone && fileInert) continue;
+    if (!fileInert) writeSecure(accountFilePath(station), BLANKED);
+    if (hasTrain) rmSync(stub, { force: true });
+    removed.push(station);
+  }
+  return removed;
+}
+
+const stationLabels = (m: Map<StationName, number>): string[] =>
+  [...m].map(([station, n]) => `${station}(${n})`);
+
+function applyKeyMap(list: LoadedAgent[]): void {
+  setKeyMap(
+    list.flatMap((agent) =>
+      agent.key === null ? [] : [{ key: agent.key, agentId: agent.id }],
+    ),
+  );
+}
+
+async function loadAndWrite(
+  source: StationSource,
+): Promise<WrittenStations & { agents: number }> {
+  const list = await source();
+  applyKeyMap(list);
+  return { ...writeStations(list), agents: list.length };
+}
+
+export async function materializeFrom(source: StationSource): Promise<void> {
+  const { active, agents: found } = await loadAndWrite(source);
+  const removed = pruneStations(active);
+  log.info(
+    { stations: stationLabels(active), agents: found, removed },
+    'materialized station accounts',
+  );
+}
+
+export interface ReloadedStations {
+  active: StationName[];
+  removed: StationName[];
+  changed: StationName[];
+}
+
+export async function reloadFrom(
+  source: StationSource,
+): Promise<ReloadedStations> {
+  const { active, changed } = await loadAndWrite(source);
+  const removed = pruneStations(active);
+  if (removed.length > 0 || changed.length > 0)
+    log.info(
+      { stations: stationLabels(active), removed, changed },
+      'reloaded station accounts',
+    );
+  return { active: [...active.keys()], removed, changed };
+}

package/runtime/node_modules/@metro-labs/daemon/src/stations/registry.ts

@@ -0,0 +1,51 @@
+import type { Station } from '@metro-labs/core/stations/types';
+import { Line } from '@metro-labs/core/lines';
+import { errMsg, log } from '@metro-labs/core/log';
+import { xmtpStation } from '@metro-labs/xmtp';
+import { telegramBotStation } from '@metro-labs/telegram-bot';
+import { telegramStation } from '@metro-labs/telegram';
+import { discordBotStation } from '@metro-labs/discord-bot';
+import { whatsappStation } from '@metro-labs/whatsapp';
+import { threemaStation } from '@metro-labs/threema';
+import { outlookStation } from '@metro-labs/outlook';
+import { webhookStation } from '@metro-labs/webhook';
+
+export const STATIONS: readonly Station[] = [
+  xmtpStation,
+  telegramBotStation,
+  telegramStation,
+  discordBotStation,
+  whatsappStation,
+  threemaStation,
+  outlookStation,
+  webhookStation,
+];
+
+export const stationByName = (name: string): Station | undefined =>
+  STATIONS.find((s) => s.name === name);
+
+export const stationForLine = (line: string): Station | undefined => {
+  const s = Line.station(line);
+  return s ? stationByName(s) : undefined;
+};
+
+export const accountStationNames = (): string[] =>
+  STATIONS.filter((s) => s.hasAccounts).map((s) => s.name);
+
+export const accountStationCapabilities = (): Record<string, string[]> => {
+  const out: Record<string, string[]> = {};
+  for (const s of STATIONS)
+    if (s.hasAccounts) out[s.name] = [...s.messageVerbs].sort();
+  return out;
+};
+
+export function forgetOrphans(known: readonly { station: string; id: string }[]): void {
+  for (const s of STATIONS) {
+    if (s.forgetExcept === undefined) continue;
+    try {
+      s.forgetExcept(known.filter((a) => a.station === s.name).map((a) => a.id));
+    } catch (err) {
+      log.warn({ station: s.name, err: errMsg(err) }, 'stations: could not remove the files of detached accounts');
+    }
+  }
+}

package/runtime/node_modules/@metro-labs/daemon/src/stations/runtime-deps.ts

@@ -0,0 +1,82 @@
+import { spawnSync } from 'node:child_process';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { log } from '@metro-labs/core/log';
+import { isRecord } from '@metro-labs/core/is-record';
+import type { StationName } from '@metro-labs/core/station-names';
+import { knownAccounts } from '../agents/map.js';
+
+const INSTALL_TIMEOUT_MS = 15 * 60_000;
+
+export interface RuntimeManifest {
+  core: Record<string, string>;
+  stations: Record<string, Record<string, string>>;
+}
+
+export interface RuntimeStore {
+  dir: string;
+  manifest: RuntimeManifest;
+}
+
+const ranges = (raw: unknown): Record<string, string> =>
+  isRecord(raw)
+    ? Object.fromEntries(Object.entries(raw).filter((e): e is [string, string] => typeof e[1] === 'string'))
+    : {};
+
+export function readManifest(path: string): RuntimeManifest {
+  const raw: unknown = JSON.parse(readFileSync(path, 'utf8'));
+  if (!isRecord(raw)) throw new Error(`${path} is not a runtime manifest`);
+  const stations = isRecord(raw.stations) ? raw.stations : {};
+  return {
+    core: ranges(raw.core),
+    stations: Object.fromEntries(Object.entries(stations).map(([name, deps]) => [name, ranges(deps)])),
+  };
+}
+
+export function runtimeStore(): RuntimeStore | null {
+  const dir = process.env.METRO_RUNTIME_STORE?.trim() ?? '';
+  const manifest = process.env.METRO_RUNTIME_MANIFEST?.trim() ?? '';
+  if (dir === '' || manifest === '') return null;
+  return { dir, manifest: readManifest(manifest) };
+}
+
+export function dependenciesFor(manifest: RuntimeManifest, stations: Iterable<string>): Record<string, string> {
+  const out: Record<string, string> = { ...manifest.core };
+  for (const station of new Set(stations)) Object.assign(out, manifest.stations[station] ?? {});
+  return Object.fromEntries(Object.entries(out).sort(([a], [b]) => a.localeCompare(b)));
+}
+
+const packageText = (deps: Record<string, string>): string =>
+  `${JSON.stringify({ name: 'metro-runtime', private: true, dependencies: deps }, null, 2)}\n`;
+
+function current(dir: string): string | null {
+  try {
+    return readFileSync(join(dir, 'package.json'), 'utf8');
+  } catch {
+    return null;
+  }
+}
+
+export function installRuntime(store: RuntimeStore, stations: Iterable<string>): boolean {
+  const wanted = packageText(dependenciesFor(store.manifest, stations));
+  const installed = existsSync(join(store.dir, 'node_modules', '.metro-installed'));
+  if (installed && current(store.dir) === wanted) return false;
+  mkdirSync(store.dir, { recursive: true });
+  writeFileSync(join(store.dir, 'package.json'), wanted);
+  log.info({ dir: store.dir }, 'runtime: installing the channel SDKs this machine needs');
+  const run = spawnSync('bun', ['install', '--no-summary', '--no-progress'], {
+    cwd: store.dir,
+    stdio: ['ignore', 'inherit', 'inherit'],
+    timeout: INSTALL_TIMEOUT_MS,
+  });
+  if (run.error !== undefined || run.status !== 0)
+    throw new Error(`bun install failed in ${store.dir}: ${run.error?.message ?? `exit ${String(run.status)}`}`);
+  writeFileSync(join(store.dir, 'node_modules', '.metro-installed'), `${new Date().toISOString()}\n`);
+  return true;
+}
+
+export function ensureStationDeps(station: StationName): void {
+  const store = runtimeStore();
+  if (store === null) return;
+  installRuntime(store, [...knownAccounts().map((a) => a.station), station]);
+}