@stage-labs/metro 0.1.0-beta.13 → 0.1.0-beta.131

package/runtime/node_modules/@metro-labs/daemon/src/stations/attach-xmtp.ts

@@ -0,0 +1,111 @@
+import { randomBytes } from 'node:crypto';
+import { rmSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { errMsg, log } from '@metro-labs/core/log';
+
+export class XmtpAttachError extends Error {}
+
+export interface XmtpIdentity {
+  inboxId: string;
+  address: string;
+  installationId: string;
+  dbPath: string;
+}
+
+export type VerifyXmtpKey = (
+  privateKey: string,
+  dbPath: string,
+) => Promise<XmtpIdentity>;
+
+const VERIFY_TIMEOUT_MS = 90_000;
+const UNREACHABLE =
+  'Metro could not open an XMTP inbox with the key it generated, so nothing was attached';
+
+export function newXmtpDbPath(): string {
+  return `~/.metro/xmtp-production-${randomBytes(8).toString('hex')}.db3`;
+}
+
+export function discardXmtpDb(dbPath: string): void {
+  for (const suffix of ['', '-wal', '-shm'])
+    rmSync(`${dbPath}${suffix}`, { force: true });
+}
+
+export function withoutKey(message: string, privateKey: string): string {
+  return message.split(privateKey).join('[redacted]');
+}
+
+interface VerifyReply {
+  ok?: unknown;
+  inboxId?: unknown;
+  address?: unknown;
+  installationId?: unknown;
+  dbPath?: unknown;
+  error?: unknown;
+}
+
+const str = (value: unknown): string => (typeof value === 'string' ? value : '');
+
+function readReply(raw: string): VerifyReply {
+  const line = raw.trim().split('\n').at(-1) ?? '';
+  try {
+    const parsed: unknown = JSON.parse(line);
+    return typeof parsed === 'object' && parsed !== null ? parsed : {};
+  } catch {
+    return {};
+  }
+}
+
+async function runVerifier(payload: string): Promise<string> {
+  const script = fileURLToPath(import.meta.resolve('@metro-labs/xmtp/verify'));
+  const proc = Bun.spawn(['bun', 'run', script], {
+    stdin: 'pipe',
+    stdout: 'pipe',
+    stderr: 'pipe',
+  });
+  await proc.stdin.write(payload);
+  await proc.stdin.end();
+  const timer = setTimeout(() => {
+    proc.kill();
+  }, VERIFY_TIMEOUT_MS);
+  try {
+    const [out, err, code] = await Promise.all([
+      new Response(proc.stdout).text(),
+      new Response(proc.stderr).text(),
+      proc.exited,
+    ]);
+    if (code !== 0)
+      log.warn(
+        { code, stderr: err.trim().slice(-500) },
+        'attach-xmtp: the inbox check exited badly',
+      );
+    return out;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+export async function verifyXmtpKeyOutOfProcess(
+  privateKey: string,
+  dbPath: string,
+): Promise<XmtpIdentity> {
+  const raw = await runVerifier(
+    JSON.stringify({ privateKey, dbPath }),
+  ).catch((err: unknown) => {
+    log.warn({ err: errMsg(err) }, 'attach-xmtp: could not run the inbox check');
+    throw new XmtpAttachError(UNREACHABLE);
+  });
+  const reply = readReply(raw);
+  if (reply.ok !== true)
+    throw new XmtpAttachError(
+      withoutKey(str(reply.error) || UNREACHABLE, privateKey),
+    );
+  const identity = {
+    inboxId: str(reply.inboxId),
+    address: str(reply.address),
+    installationId: str(reply.installationId),
+    dbPath: str(reply.dbPath),
+  };
+  if (identity.inboxId === '' || identity.address === '')
+    throw new XmtpAttachError(UNREACHABLE);
+  return identity;
+}

package/runtime/node_modules/@metro-labs/daemon/src/stations/attach.ts

@@ -0,0 +1,248 @@
+import { randomBytes } from 'node:crypto';
+import {
+  DiscordTokenError,
+  verifyDiscordBotToken,
+} from '@metro-labs/discord-bot/verify';
+import {
+  TelegramTokenError,
+  verifyTelegramBotToken,
+} from '@metro-labs/telegram-bot/verify';
+import {
+  parsePrivateKey,
+  ThreemaGatewayError,
+  verifyThreemaGateway,
+} from '@metro-labs/threema/verify';
+import { ApiError } from '@metro-labs/http/api-error';
+import { bodyField } from '@metro-labs/http/api-http';
+import { ensureStationDeps } from './runtime-deps.js';
+import { publicBaseOrDefault } from '../files/attach-serve.js';
+import {
+  discardXmtpDb,
+  newXmtpDbPath,
+  verifyXmtpKeyOutOfProcess,
+  XmtpAttachError,
+  type VerifyXmtpKey,
+} from './attach-xmtp.js';
+
+export const ATTACHABLE_STATIONS = [
+  'discord-bot',
+  'telegram-bot',
+  'threema',
+  'xmtp',
+  'webhook',
+] as const;
+
+export type AttachStation = (typeof ATTACHABLE_STATIONS)[number];
+
+export class StationAttachError extends ApiError {}
+
+export interface AttachInput {
+  station: AttachStation;
+  token?: unknown;
+  gatewayId?: unknown;
+  secret?: unknown;
+  privateKey?: unknown;
+}
+
+export interface OneTimeSecret {
+  label: string;
+  value: string;
+  note: string;
+}
+
+export interface PreparedAccount {
+  config: Record<string, unknown>;
+  identity: Record<string, string>;
+  secret?: OneTimeSecret;
+  discard?: () => void;
+}
+
+const TOKEN_RE = /^[A-Za-z0-9._:-]{8,256}$/;
+
+const SECP256K1_ORDER = BigInt(
+  '0xfffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141',
+);
+
+export function attachInputOf(station: AttachStation, body: unknown): AttachInput {
+  return {
+    station,
+    token: bodyField(body, 'token'),
+    gatewayId: bodyField(body, 'gatewayId'),
+    secret: bodyField(body, 'secret'),
+    privateKey: bodyField(body, 'privateKey'),
+  };
+}
+
+export function isAttachStation(raw: unknown): raw is AttachStation {
+  return (
+    typeof raw === 'string' &&
+    (ATTACHABLE_STATIONS as readonly string[]).includes(raw)
+  );
+}
+
+function requireToken(raw: unknown, label: string): string {
+  const token = typeof raw === 'string' ? raw.trim() : '';
+  if (token === '')
+    throw new StationAttachError(`a ${label} bot token is required`, 400);
+  if (!TOKEN_RE.test(token))
+    throw new StationAttachError(
+      `that does not look like a ${label} bot token`,
+      400,
+    );
+  return token;
+}
+
+function rejected(err: unknown, own: boolean, fallback: string): never {
+  throw new StationAttachError(
+    own && err instanceof Error ? err.message : fallback,
+    400,
+  );
+}
+
+async function prepareDiscord(raw: unknown): Promise<PreparedAccount> {
+  const token = requireToken(raw, 'Discord');
+  try {
+    const bot = await verifyDiscordBotToken(token);
+    if (!bot.messageContent)
+      throw new StationAttachError(
+        'this Discord application does not have the Message Content intent enabled. Turn it on under Bot / Privileged Gateway Intents in the developer portal, then attach it again',
+        400,
+      );
+    return {
+      config: { token },
+      identity: { userId: bot.userId, username: bot.username },
+    };
+  } catch (err) {
+    if (err instanceof StationAttachError) throw err;
+    return rejected(
+      err,
+      err instanceof DiscordTokenError,
+      'Discord rejected that bot token',
+    );
+  }
+}
+
+async function prepareTelegram(raw: unknown): Promise<PreparedAccount> {
+  const token = requireToken(raw, 'Telegram');
+  try {
+    const bot = await verifyTelegramBotToken(token);
+    return {
+      config: { token },
+      identity: { botId: String(bot.botId), username: bot.username },
+    };
+  } catch (err) {
+    return rejected(
+      err,
+      err instanceof TelegramTokenError,
+      'Telegram rejected that bot token',
+    );
+  }
+}
+
+export function newXmtpPrivateKey(): string {
+  for (;;) {
+    const hex = randomBytes(32).toString('hex');
+    const value = BigInt(`0x${hex}`);
+    if (value > 0n && value < SECP256K1_ORDER) return `0x${hex}`;
+  }
+}
+
+async function prepareXmtp(verify: VerifyXmtpKey): Promise<PreparedAccount> {
+  const privateKey = newXmtpPrivateKey();
+  const dbPath = newXmtpDbPath();
+  const identity = await verify(privateKey, dbPath).catch((err: unknown) =>
+    rejected(
+      err,
+      err instanceof XmtpAttachError,
+      'Metro could not open an XMTP inbox with the key it generated, so nothing was attached',
+    ),
+  );
+  return {
+    config: { privateKey, dbPath },
+    identity: { inboxId: identity.inboxId, address: identity.address },
+    secret: {
+      label: 'xmtp private key',
+      value: privateKey,
+      note: 'Metro generated this key, opened inbox ' +
+        `${identity.inboxId} with it, and stores it with the account. ` +
+        'It is shown once, here, and is never returned by any API again.',
+    },
+    discard: () => {
+      discardXmtpDb(identity.dbPath);
+    },
+  };
+}
+
+export const hookUrl = (webhookId: string, secret: string): string =>
+  `${publicBaseOrDefault().replace(/\/+$/, '')}/api/webhooks/${webhookId}/${secret}`;
+
+const WEBHOOK_ID_FLOOR = 10n ** 18n;
+
+export function newWebhookId(): string {
+  const raw = BigInt(`0x${randomBytes(8).toString('hex')}`);
+  return String(WEBHOOK_ID_FLOOR + (raw % (9n * WEBHOOK_ID_FLOOR)));
+}
+
+export const threemaCallbackUrl = (callbackId: string, token: string): string =>
+  `${publicBaseOrDefault().replace(/\/+$/, '')}/api/threema/${callbackId}/${token}`;
+
+function requireText(raw: unknown, label: string): string {
+  const text = typeof raw === 'string' ? raw.trim() : '';
+  if (text === '')
+    throw new StationAttachError(`the Threema ${label} is required`, 400);
+  return text;
+}
+
+async function prepareThreema(input: AttachInput): Promise<PreparedAccount> {
+  const gatewayId = requireText(input.gatewayId, 'Gateway ID');
+  const secret = requireText(input.secret, 'API secret');
+  const privateKey = requireText(input.privateKey, 'private key');
+  try {
+    const identity = await verifyThreemaGateway({ gatewayId, secret, privateKey });
+    const callbackId = newWebhookId();
+    const callbackToken = randomBytes(48).toString('base64url');
+    return {
+      config: {
+        gatewayId: identity.gatewayId,
+        secret,
+        privateKey: parsePrivateKey(privateKey) ?? privateKey,
+        callbackId,
+        callbackToken,
+        createdAt: new Date().toISOString(),
+      },
+      identity: {
+        gatewayId: identity.gatewayId,
+        credits: String(identity.credits),
+        callback: threemaCallbackUrl(callbackId, callbackToken),
+      },
+    };
+  } catch (err) {
+    if (err instanceof StationAttachError) throw err;
+    return rejected(
+      err,
+      err instanceof ThreemaGatewayError,
+      'Threema rejected those Gateway credentials',
+    );
+  }
+}
+
+function prepareWebhook(): PreparedAccount {
+  const secret = randomBytes(48).toString('base64url');
+  const webhookId = newWebhookId();
+  return {
+    config: { secret, webhookId, createdAt: new Date().toISOString() },
+    identity: { endpoint: hookUrl(webhookId, secret) },
+  };
+}
+
+export async function prepareAccount(
+  input: AttachInput,
+  verify: VerifyXmtpKey = verifyXmtpKeyOutOfProcess,
+): Promise<PreparedAccount> {
+  ensureStationDeps(input.station);
+  if (input.station === 'discord-bot') return prepareDiscord(input.token);
+  if (input.station === 'telegram-bot') return prepareTelegram(input.token);
+  if (input.station === 'webhook') return prepareWebhook();
+  if (input.station === 'threema') return prepareThreema(input);
+  return prepareXmtp(verify);
+}

package/runtime/node_modules/@metro-labs/daemon/src/stations/materialize.ts

@@ -0,0 +1,261 @@
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { log } from '@metro-labs/core/log';
+import { writeSecure } from '@metro-labs/core/secure-fs';
+import { trainsDir } from '../boot/paths.js';
+import {
+  setAgentMap,
+  setAllowlistMap,
+  setDisabledAccounts,
+  type AgentMap,
+  type AgentNameMap,
+  type AllowlistMap,
+} from '../agents/map.js';
+import { setKeyMap } from '../agents/keys.js';
+import { STATIONS, type StationName } from '@metro-labs/core/station-names';
+
+interface StationTarget {
+  file: string;
+  fileEnv: string;
+  trainImport: string | null;
+}
+
+export interface LoadedAccount {
+  station: StationName;
+  id: string;
+  allowlist: string[] | null;
+  enabled?: boolean;
+  config: Record<string, unknown>;
+}
+
+export interface LoadedConnector {
+  id: string;
+  name: string;
+  url: string;
+  transport: 'http';
+  config: Record<string, unknown>;
+}
+
+export interface LoadedAgent {
+  id: string;
+  name: string;
+  accounts: LoadedAccount[];
+  key: string | null;
+  connectors?: LoadedConnector[];
+}
+
+const METRO_DIR = join(homedir(), '.metro');
+
+const STATION_TARGETS: Record<StationName, StationTarget> = {
+  xmtp: {
+    file: 'xmtp-accounts.json',
+    fileEnv: 'XMTP_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/xmtp/train',
+  },
+  'telegram-bot': {
+    file: 'telegram-bot-accounts.json',
+    fileEnv: 'TELEGRAM_BOT_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/telegram-bot/train',
+  },
+  'telegram': {
+    file: 'telegram-accounts.json',
+    fileEnv: 'TELEGRAM_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/telegram/train',
+  },
+  'discord-bot': {
+    file: 'discord-bot-accounts.json',
+    fileEnv: 'DISCORD_BOT_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/discord-bot/train',
+  },
+  whatsapp: {
+    file: 'whatsapp-accounts.json',
+    fileEnv: 'WHATSAPP_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/whatsapp/train',
+  },
+  threema: {
+    file: 'threema-accounts.json',
+    fileEnv: 'THREEMA_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/threema/train',
+  },
+  webhook: {
+    file: 'webhook-accounts.json',
+    fileEnv: 'WEBHOOK_ACCOUNTS_FILE',
+    trainImport: null,
+  },
+};
+
+export function accountFilePath(station: StationName): string {
+  const target = STATION_TARGETS[station];
+  return process.env[target.fileEnv] ?? join(METRO_DIR, target.file);
+}
+
+export type StationSource = () => Promise<LoadedAgent[]>;
+
+export const MOVABLE_STATIONS = new Set<StationName>(
+  STATIONS.filter((s) => s !== 'webhook'),
+);
+
+function trainStubPath(station: StationName): string {
+  return join(trainsDir(), `${station}.ts`);
+}
+
+function currentText(path: string): string | null {
+  try {
+    return readFileSync(path, 'utf8');
+  } catch {
+    return null;
+  }
+}
+
+export function writeIfChanged(path: string, content: string): boolean {
+  if (currentText(path) === content) return false;
+  writeFileSync(path, content);
+  return true;
+}
+
+interface WrittenStations {
+  active: Map<StationName, number>;
+  changed: StationName[];
+}
+
+function writeStations(list: LoadedAgent[]): WrittenStations {
+  mkdirSync(METRO_DIR, { recursive: true });
+  mkdirSync(trainsDir(), { recursive: true });
+
+  const byStation = new Map<StationName, LoadedAccount[]>();
+  const map: AgentMap = {};
+  const names: AgentNameMap = {};
+  const allow: AllowlistMap = {};
+  const disabled = new Set<string>();
+  for (const agent of list) {
+    names[agent.id] = agent.name;
+    for (const a of agent.accounts) {
+      if (!(a.station in STATION_TARGETS)) {
+        log.error(
+          { station: a.station, id: a.id },
+          'unknown station — this metro is older or newer than the row; skipping it',
+        );
+        continue;
+      }
+      map[`${a.station}/${a.id}`] = agent.id;
+      if (a.allowlist) allow[`${a.station}/${a.id}`] = a.allowlist;
+      if (a.enabled === false) {
+        disabled.add(`${a.station}/${a.id}`);
+        continue;
+      }
+      const cur = byStation.get(a.station);
+      if (cur) cur.push(a);
+      else byStation.set(a.station, [a]);
+    }
+  }
+  setAgentMap(map, names);
+  setAllowlistMap(allow);
+  setDisabledAccounts(disabled);
+
+  const active = new Map<StationName, number>();
+  const changed: StationName[] = [];
+  for (const [station, accts] of byStation) {
+    const records = accts.map((a) => ({ id: a.id, ...a.config }));
+    const path = accountFilePath(station);
+    const content = JSON.stringify(records, null, 2);
+    if (currentText(path) !== content) {
+      writeSecure(path, content);
+      changed.push(station);
+    }
+    const { trainImport } = STATION_TARGETS[station];
+    if (trainImport !== null)
+      writeIfChanged(trainStubPath(station), `import '${trainImport}';\n`);
+    active.set(station, accts.length);
+  }
+  return { active, changed };
+}
+
+const BLANKED = '[]\n';
+
+function pruneStations(present: Map<StationName, number>): StationName[] {
+  const removed: StationName[] = [];
+  for (const station of Object.keys(STATION_TARGETS) as StationName[]) {
+    if (present.has(station)) continue;
+    const hasTrain = STATION_TARGETS[station].trainImport !== null;
+    const stub = trainStubPath(station);
+    const stubGone = !hasTrain || !existsSync(stub);
+    const file = currentText(accountFilePath(station));
+    const fileInert = file === null || file === BLANKED;
+    if (stubGone && fileInert) continue;
+    if (!fileInert) writeSecure(accountFilePath(station), BLANKED);
+    if (hasTrain) rmSync(stub, { force: true });
+    removed.push(station);
+  }
+  return removed;
+}
+
+const stationLabels = (m: Map<StationName, number>): string[] =>
+  [...m].map(([station, n]) => `${station}(${n})`);
+
+function applyKeyMap(list: LoadedAgent[]): void {
+  lastKey = list.find((a) => a.key !== null)?.key ?? null;
+  setKeyMap(
+    list.flatMap((agent) =>
+      agent.key === null ? [] : [{ key: agent.key, agentId: agent.id }],
+    ),
+  );
+}
+
+let lastKey: string | null = null;
+
+export function localAgentKey(): string | null {
+  return lastKey;
+}
+
+async function loadAndWrite(
+  source: StationSource,
+): Promise<WrittenStations & { agents: number }> {
+  const list = await source();
+  applyKeyMap(list);
+  return { ...writeStations(list), agents: list.length };
+}
+
+export interface MaterializeOptions {
+  allowEmpty?: boolean;
+}
+
+export async function materializeFrom(
+  source: StationSource,
+  opts: MaterializeOptions = {},
+): Promise<void> {
+  const { active, agents: found } = await loadAndWrite(source);
+
+  if (found === 0 && opts.allowEmpty !== true)
+    throw new Error('no agents found — nothing to materialize');
+  const removed = pruneStations(active);
+  log.info(
+    { stations: stationLabels(active), agents: found, removed },
+    'materialized station accounts',
+  );
+}
+
+export interface ReloadedStations {
+  active: StationName[];
+  removed: StationName[];
+  changed: StationName[];
+}
+
+export async function reloadFrom(
+  source: StationSource,
+): Promise<ReloadedStations> {
+  const { active, changed } = await loadAndWrite(source);
+  const removed = pruneStations(active);
+  if (removed.length > 0 || changed.length > 0)
+    log.info(
+      { stations: stationLabels(active), removed, changed },
+      'reloaded station accounts',
+    );
+  return { active: [...active.keys()], removed, changed };
+}

package/runtime/node_modules/@metro-labs/daemon/src/stations/registry.ts

@@ -0,0 +1,37 @@
+import type { Station } from '@metro-labs/core/stations/types';
+import { Line } from '@metro-labs/core/lines';
+import { xmtpStation } from '@metro-labs/xmtp';
+import { telegramBotStation } from '@metro-labs/telegram-bot';
+import { telegramStation } from '@metro-labs/telegram';
+import { discordBotStation } from '@metro-labs/discord-bot';
+import { whatsappStation } from '@metro-labs/whatsapp';
+import { threemaStation } from '@metro-labs/threema';
+import { webhookStation } from '@metro-labs/webhook';
+
+export const STATIONS: readonly Station[] = [
+  xmtpStation,
+  telegramBotStation,
+  telegramStation,
+  discordBotStation,
+  whatsappStation,
+  threemaStation,
+  webhookStation,
+];
+
+export const stationByName = (name: string): Station | undefined =>
+  STATIONS.find((s) => s.name === name);
+
+export const stationForLine = (line: string): Station | undefined => {
+  const s = Line.station(line);
+  return s ? stationByName(s) : undefined;
+};
+
+export const accountStationNames = (): string[] =>
+  STATIONS.filter((s) => s.hasAccounts).map((s) => s.name);
+
+export const accountStationCapabilities = (): Record<string, string[]> => {
+  const out: Record<string, string[]> = {};
+  for (const s of STATIONS)
+    if (s.hasAccounts) out[s.name] = [...s.messageVerbs].sort();
+  return out;
+};