@stage-labs/metro 0.1.0-beta.8 → 0.1.0-beta.80

package/runtime/node_modules/@metro-labs/daemon/src/stations/attach.ts

@@ -0,0 +1,184 @@
+import { randomBytes } from 'node:crypto';
+import {
+  DiscordTokenError,
+  verifyDiscordBotToken,
+} from '@metro-labs/discord-bot/verify';
+import {
+  TelegramTokenError,
+  verifyTelegramBotToken,
+} from '@metro-labs/telegram-bot/verify';
+import { ApiError } from '@metro-labs/http/api-error';
+import { ensureStationDeps } from './runtime-deps.js';
+import { publicBaseOrDefault } from '../files/attach-serve.js';
+import {
+  discardXmtpDb,
+  newXmtpDbPath,
+  verifyXmtpKeyOutOfProcess,
+  XmtpAttachError,
+  type VerifyXmtpKey,
+} from './attach-xmtp.js';
+
+export const ATTACHABLE_STATIONS = [
+  'discord-bot',
+  'telegram-bot',
+  'xmtp',
+  'webhook',
+] as const;
+
+export type AttachStation = (typeof ATTACHABLE_STATIONS)[number];
+
+export class StationAttachError extends ApiError {}
+
+export interface AttachInput {
+  station: AttachStation;
+  token?: unknown;
+}
+
+export interface OneTimeSecret {
+  label: string;
+  value: string;
+  note: string;
+}
+
+export interface PreparedAccount {
+  config: Record<string, unknown>;
+  identity: Record<string, string>;
+  secret?: OneTimeSecret;
+  discard?: () => void;
+}
+
+const TOKEN_RE = /^[A-Za-z0-9._:-]{8,256}$/;
+
+const SECP256K1_ORDER = BigInt(
+  '0xfffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141',
+);
+
+export function isAttachStation(raw: unknown): raw is AttachStation {
+  return (
+    typeof raw === 'string' &&
+    (ATTACHABLE_STATIONS as readonly string[]).includes(raw)
+  );
+}
+
+function requireToken(raw: unknown, label: string): string {
+  const token = typeof raw === 'string' ? raw.trim() : '';
+  if (token === '')
+    throw new StationAttachError(`a ${label} bot token is required`, 400);
+  if (!TOKEN_RE.test(token))
+    throw new StationAttachError(
+      `that does not look like a ${label} bot token`,
+      400,
+    );
+  return token;
+}
+
+function rejected(err: unknown, own: boolean, fallback: string): never {
+  throw new StationAttachError(
+    own && err instanceof Error ? err.message : fallback,
+    400,
+  );
+}
+
+async function prepareDiscord(raw: unknown): Promise<PreparedAccount> {
+  const token = requireToken(raw, 'Discord');
+  try {
+    const bot = await verifyDiscordBotToken(token);
+    if (!bot.messageContent)
+      throw new StationAttachError(
+        'this Discord application does not have the Message Content intent enabled. Turn it on under Bot / Privileged Gateway Intents in the developer portal, then attach it again',
+        400,
+      );
+    return {
+      config: { token },
+      identity: { userId: bot.userId, username: bot.username },
+    };
+  } catch (err) {
+    if (err instanceof StationAttachError) throw err;
+    return rejected(
+      err,
+      err instanceof DiscordTokenError,
+      'Discord rejected that bot token',
+    );
+  }
+}
+
+async function prepareTelegram(raw: unknown): Promise<PreparedAccount> {
+  const token = requireToken(raw, 'Telegram');
+  try {
+    const bot = await verifyTelegramBotToken(token);
+    return {
+      config: { token },
+      identity: { botId: String(bot.botId), username: bot.username },
+    };
+  } catch (err) {
+    return rejected(
+      err,
+      err instanceof TelegramTokenError,
+      'Telegram rejected that bot token',
+    );
+  }
+}
+
+export function newXmtpPrivateKey(): string {
+  for (;;) {
+    const hex = randomBytes(32).toString('hex');
+    const value = BigInt(`0x${hex}`);
+    if (value > 0n && value < SECP256K1_ORDER) return `0x${hex}`;
+  }
+}
+
+async function prepareXmtp(verify: VerifyXmtpKey): Promise<PreparedAccount> {
+  const privateKey = newXmtpPrivateKey();
+  const dbPath = newXmtpDbPath();
+  const identity = await verify(privateKey, dbPath).catch((err: unknown) =>
+    rejected(
+      err,
+      err instanceof XmtpAttachError,
+      'Metro could not open an XMTP inbox with the key it generated, so nothing was attached',
+    ),
+  );
+  return {
+    config: { privateKey, dbPath },
+    identity: { inboxId: identity.inboxId, address: identity.address },
+    secret: {
+      label: 'xmtp private key',
+      value: privateKey,
+      note: 'Metro generated this key, opened inbox ' +
+        `${identity.inboxId} with it, and stores it with the account. ` +
+        'It is shown once, here, and is never returned by any API again.',
+    },
+    discard: () => {
+      discardXmtpDb(identity.dbPath);
+    },
+  };
+}
+
+export const hookUrl = (webhookId: string, secret: string): string =>
+  `${publicBaseOrDefault().replace(/\/+$/, '')}/api/webhooks/${webhookId}/${secret}`;
+
+const WEBHOOK_ID_FLOOR = 10n ** 18n;
+
+export function newWebhookId(): string {
+  const raw = BigInt(`0x${randomBytes(8).toString('hex')}`);
+  return String(WEBHOOK_ID_FLOOR + (raw % (9n * WEBHOOK_ID_FLOOR)));
+}
+
+function prepareWebhook(): PreparedAccount {
+  const secret = randomBytes(48).toString('base64url');
+  const webhookId = newWebhookId();
+  return {
+    config: { secret, webhookId, createdAt: new Date().toISOString() },
+    identity: { endpoint: hookUrl(webhookId, secret) },
+  };
+}
+
+export async function prepareAccount(
+  input: AttachInput,
+  verify: VerifyXmtpKey = verifyXmtpKeyOutOfProcess,
+): Promise<PreparedAccount> {
+  ensureStationDeps(input.station);
+  if (input.station === 'discord-bot') return prepareDiscord(input.token);
+  if (input.station === 'telegram-bot') return prepareTelegram(input.token);
+  if (input.station === 'webhook') return prepareWebhook();
+  return prepareXmtp(verify);
+}

package/runtime/node_modules/@metro-labs/daemon/src/stations/materialize.ts

@@ -0,0 +1,248 @@
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { log } from '@metro-labs/core/log';
+import { writeSecure } from '@metro-labs/core/secure-fs';
+import { trainsDir } from '../boot/paths.js';
+import {
+  setAgentMap,
+  setAllowlistMap,
+  type AgentMap,
+  type AgentNameMap,
+  type AllowlistMap,
+} from '../agents/map.js';
+import { setKeyMap } from '../agents/keys.js';
+import { STATIONS, type StationName } from '@metro-labs/core/station-names';
+
+interface StationTarget {
+  file: string;
+  fileEnv: string;
+  trainImport: string | null;
+}
+
+export interface LoadedAccount {
+  station: StationName;
+  id: string;
+  allowlist: string[] | null;
+  config: Record<string, unknown>;
+}
+
+export interface LoadedConnector {
+  id: string;
+  name: string;
+  url: string;
+  transport: 'http';
+  config: Record<string, unknown>;
+}
+
+export interface LoadedAgent {
+  id: string;
+  name: string;
+  accounts: LoadedAccount[];
+  key: string | null;
+  connectors?: LoadedConnector[];
+}
+
+const METRO_DIR = join(homedir(), '.metro');
+
+const STATION_TARGETS: Record<StationName, StationTarget> = {
+  xmtp: {
+    file: 'xmtp-accounts.json',
+    fileEnv: 'XMTP_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/xmtp/train',
+  },
+  'telegram-bot': {
+    file: 'telegram-bot-accounts.json',
+    fileEnv: 'TELEGRAM_BOT_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/telegram-bot/train',
+  },
+  'telegram': {
+    file: 'telegram-accounts.json',
+    fileEnv: 'TELEGRAM_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/telegram/train',
+  },
+  'discord-bot': {
+    file: 'discord-bot-accounts.json',
+    fileEnv: 'DISCORD_BOT_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/discord-bot/train',
+  },
+  whatsapp: {
+    file: 'whatsapp-accounts.json',
+    fileEnv: 'WHATSAPP_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/whatsapp/train',
+  },
+  webhook: {
+    file: 'webhook-accounts.json',
+    fileEnv: 'WEBHOOK_ACCOUNTS_FILE',
+    trainImport: null,
+  },
+};
+
+function accountFilePath(station: StationName): string {
+  const target = STATION_TARGETS[station];
+  return process.env[target.fileEnv] ?? join(METRO_DIR, target.file);
+}
+
+export type StationSource = () => Promise<LoadedAgent[]>;
+
+export const MOVABLE_STATIONS = new Set<StationName>(
+  STATIONS.filter((s) => s !== 'webhook'),
+);
+
+function trainStubPath(station: StationName): string {
+  return join(trainsDir(), `${station}.ts`);
+}
+
+function currentText(path: string): string | null {
+  try {
+    return readFileSync(path, 'utf8');
+  } catch {
+    return null;
+  }
+}
+
+export function writeIfChanged(path: string, content: string): boolean {
+  if (currentText(path) === content) return false;
+  writeFileSync(path, content);
+  return true;
+}
+
+interface WrittenStations {
+  active: Map<StationName, number>;
+  changed: StationName[];
+}
+
+function writeStations(list: LoadedAgent[]): WrittenStations {
+  mkdirSync(METRO_DIR, { recursive: true });
+  mkdirSync(trainsDir(), { recursive: true });
+
+  const byStation = new Map<StationName, LoadedAccount[]>();
+  const map: AgentMap = {};
+  const names: AgentNameMap = {};
+  const allow: AllowlistMap = {};
+  for (const agent of list) {
+    names[agent.id] = agent.name;
+    for (const a of agent.accounts) {
+      if (!(a.station in STATION_TARGETS)) {
+        log.error(
+          { station: a.station, id: a.id },
+          'unknown station — this metro is older or newer than the row; skipping it',
+        );
+        continue;
+      }
+      map[`${a.station}/${a.id}`] = agent.id;
+      if (a.allowlist) allow[`${a.station}/${a.id}`] = a.allowlist;
+      const cur = byStation.get(a.station);
+      if (cur) cur.push(a);
+      else byStation.set(a.station, [a]);
+    }
+  }
+  setAgentMap(map, names);
+  setAllowlistMap(allow);
+
+  const active = new Map<StationName, number>();
+  const changed: StationName[] = [];
+  for (const [station, accts] of byStation) {
+    const records = accts.map((a) => ({ id: a.id, ...a.config }));
+    const path = accountFilePath(station);
+    const content = JSON.stringify(records, null, 2);
+    if (currentText(path) !== content) {
+      writeSecure(path, content);
+      changed.push(station);
+    }
+    const { trainImport } = STATION_TARGETS[station];
+    if (trainImport !== null)
+      writeIfChanged(trainStubPath(station), `import '${trainImport}';\n`);
+    active.set(station, accts.length);
+  }
+  return { active, changed };
+}
+
+const BLANKED = '[]\n';
+
+function pruneStations(present: Map<StationName, number>): StationName[] {
+  const removed: StationName[] = [];
+  for (const station of Object.keys(STATION_TARGETS) as StationName[]) {
+    if (present.has(station)) continue;
+    const hasTrain = STATION_TARGETS[station].trainImport !== null;
+    const stub = trainStubPath(station);
+    const stubGone = !hasTrain || !existsSync(stub);
+    const file = currentText(accountFilePath(station));
+    const fileInert = file === null || file === BLANKED;
+    if (stubGone && fileInert) continue;
+    if (!fileInert) writeSecure(accountFilePath(station), BLANKED);
+    if (hasTrain) rmSync(stub, { force: true });
+    removed.push(station);
+  }
+  return removed;
+}
+
+const stationLabels = (m: Map<StationName, number>): string[] =>
+  [...m].map(([station, n]) => `${station}(${n})`);
+
+function applyKeyMap(list: LoadedAgent[]): void {
+  lastKey = list.find((a) => a.key !== null)?.key ?? null;
+  setKeyMap(
+    list.flatMap((agent) =>
+      agent.key === null ? [] : [{ key: agent.key, agentId: agent.id }],
+    ),
+  );
+}
+
+let lastKey: string | null = null;
+
+export function localAgentKey(): string | null {
+  return lastKey;
+}
+
+async function loadAndWrite(
+  source: StationSource,
+): Promise<WrittenStations & { agents: number }> {
+  const list = await source();
+  applyKeyMap(list);
+  return { ...writeStations(list), agents: list.length };
+}
+
+export interface MaterializeOptions {
+  allowEmpty?: boolean;
+}
+
+export async function materializeFrom(
+  source: StationSource,
+  opts: MaterializeOptions = {},
+): Promise<void> {
+  const { active, agents: found } = await loadAndWrite(source);
+
+  if (found === 0 && opts.allowEmpty !== true)
+    throw new Error('no agents found — nothing to materialize');
+  const removed = pruneStations(active);
+  log.info(
+    { stations: stationLabels(active), agents: found, removed },
+    'materialized station accounts',
+  );
+}
+
+export interface ReloadedStations {
+  active: StationName[];
+  removed: StationName[];
+  changed: StationName[];
+}
+
+export async function reloadFrom(
+  source: StationSource,
+): Promise<ReloadedStations> {
+  const { active, changed } = await loadAndWrite(source);
+  const removed = pruneStations(active);
+  if (removed.length > 0 || changed.length > 0)
+    log.info(
+      { stations: stationLabels(active), removed, changed },
+      'reloaded station accounts',
+    );
+  return { active: [...active.keys()], removed, changed };
+}

package/runtime/node_modules/@metro-labs/daemon/src/stations/registry.ts

@@ -0,0 +1,35 @@
+import type { Station } from '@metro-labs/core/stations/types';
+import { Line } from '@metro-labs/core/lines';
+import { xmtpStation } from '@metro-labs/xmtp';
+import { telegramBotStation } from '@metro-labs/telegram-bot';
+import { telegramStation } from '@metro-labs/telegram';
+import { discordBotStation } from '@metro-labs/discord-bot';
+import { whatsappStation } from '@metro-labs/whatsapp';
+import { webhookStation } from '@metro-labs/webhook';
+
+export const STATIONS: readonly Station[] = [
+  xmtpStation,
+  telegramBotStation,
+  telegramStation,
+  discordBotStation,
+  whatsappStation,
+  webhookStation,
+];
+
+export const stationByName = (name: string): Station | undefined =>
+  STATIONS.find((s) => s.name === name);
+
+export const stationForLine = (line: string): Station | undefined => {
+  const s = Line.station(line);
+  return s ? stationByName(s) : undefined;
+};
+
+export const accountStationNames = (): string[] =>
+  STATIONS.filter((s) => s.hasAccounts).map((s) => s.name);
+
+export const accountStationCapabilities = (): Record<string, string[]> => {
+  const out: Record<string, string[]> = {};
+  for (const s of STATIONS)
+    if (s.hasAccounts) out[s.name] = [...s.messageVerbs].sort();
+  return out;
+};

package/runtime/node_modules/@metro-labs/daemon/src/stations/runtime-deps.ts

@@ -0,0 +1,82 @@
+import { spawnSync } from 'node:child_process';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { log } from '@metro-labs/core/log';
+import { isRecord } from '@metro-labs/core/is-record';
+import type { StationName } from '@metro-labs/core/station-names';
+import { knownAccounts } from '../agents/map.js';
+
+const INSTALL_TIMEOUT_MS = 15 * 60_000;
+
+export interface RuntimeManifest {
+  core: Record<string, string>;
+  stations: Record<string, Record<string, string>>;
+}
+
+export interface RuntimeStore {
+  dir: string;
+  manifest: RuntimeManifest;
+}
+
+const ranges = (raw: unknown): Record<string, string> =>
+  isRecord(raw)
+    ? Object.fromEntries(Object.entries(raw).filter((e): e is [string, string] => typeof e[1] === 'string'))
+    : {};
+
+export function readManifest(path: string): RuntimeManifest {
+  const raw: unknown = JSON.parse(readFileSync(path, 'utf8'));
+  if (!isRecord(raw)) throw new Error(`${path} is not a runtime manifest`);
+  const stations = isRecord(raw.stations) ? raw.stations : {};
+  return {
+    core: ranges(raw.core),
+    stations: Object.fromEntries(Object.entries(stations).map(([name, deps]) => [name, ranges(deps)])),
+  };
+}
+
+export function runtimeStore(): RuntimeStore | null {
+  const dir = process.env.METRO_RUNTIME_STORE?.trim() ?? '';
+  const manifest = process.env.METRO_RUNTIME_MANIFEST?.trim() ?? '';
+  if (dir === '' || manifest === '') return null;
+  return { dir, manifest: readManifest(manifest) };
+}
+
+export function dependenciesFor(manifest: RuntimeManifest, stations: Iterable<string>): Record<string, string> {
+  const out: Record<string, string> = { ...manifest.core };
+  for (const station of new Set(stations)) Object.assign(out, manifest.stations[station] ?? {});
+  return Object.fromEntries(Object.entries(out).sort(([a], [b]) => a.localeCompare(b)));
+}
+
+const packageText = (deps: Record<string, string>): string =>
+  `${JSON.stringify({ name: 'metro-runtime', private: true, dependencies: deps }, null, 2)}\n`;
+
+function current(dir: string): string | null {
+  try {
+    return readFileSync(join(dir, 'package.json'), 'utf8');
+  } catch {
+    return null;
+  }
+}
+
+export function installRuntime(store: RuntimeStore, stations: Iterable<string>): boolean {
+  const wanted = packageText(dependenciesFor(store.manifest, stations));
+  const installed = existsSync(join(store.dir, 'node_modules', '.metro-installed'));
+  if (installed && current(store.dir) === wanted) return false;
+  mkdirSync(store.dir, { recursive: true });
+  writeFileSync(join(store.dir, 'package.json'), wanted);
+  log.info({ dir: store.dir }, 'runtime: installing the channel SDKs this machine needs');
+  const run = spawnSync('bun', ['install', '--no-summary', '--no-progress'], {
+    cwd: store.dir,
+    stdio: ['ignore', 'inherit', 'inherit'],
+    timeout: INSTALL_TIMEOUT_MS,
+  });
+  if (run.error !== undefined || run.status !== 0)
+    throw new Error(`bun install failed in ${store.dir}: ${run.error?.message ?? `exit ${String(run.status)}`}`);
+  writeFileSync(join(store.dir, 'node_modules', '.metro-installed'), `${new Date().toISOString()}\n`);
+  return true;
+}
+
+export function ensureStationDeps(station: StationName): void {
+  const store = runtimeStore();
+  if (store === null) return;
+  installRuntime(store, [...knownAccounts().map((a) => a.station), station]);
+}

package/runtime/node_modules/@metro-labs/daemon/src/stations/supervisor-io.ts

@@ -0,0 +1,97 @@
+import { watch, type FSWatcher } from 'node:fs';
+import { join, parse as parsePath } from 'node:path';
+import { errMsg, log } from '@metro-labs/core/log';
+import type { Pending } from '@metro-labs/core/trains/protocol';
+
+export const RESTART_BACKOFFS_MS = [1_000, 5_000, 30_000] as const;
+export const MAX_CONSECUTIVE_FAILS = 5;
+export const HOT_RELOAD_DEBOUNCE_MS = 300;
+export const TRAIN_EXT = /\.(ts|js|mjs)$/;
+
+export interface TrainState {
+  name: string;
+  path: string;
+  proc: ReturnType<typeof Bun.spawn> | null;
+  pending: Map<string, Pending>;
+  buf: string;
+  errBuf: string;
+  failCount: number;
+  restartTimer: ReturnType<typeof setTimeout> | null;
+  startedAt: string | null;
+  stopped: boolean;
+}
+
+export function startWatcher(
+  dir: string,
+  timers: Map<string, ReturnType<typeof setTimeout>>,
+  onChange: (name: string, path: string) => void,
+): FSWatcher | null {
+  try {
+    const w = watch(dir, (_event, filename) => {
+      if (!filename) return;
+      const base = filename;
+      if (!TRAIN_EXT.test(base) || base.startsWith('_') || base.startsWith('.'))
+        return;
+      const name = parsePath(base).name;
+      const existing = timers.get(name);
+      if (existing) clearTimeout(existing);
+      timers.set(
+        name,
+        setTimeout(() => {
+          timers.delete(name);
+          onChange(name, join(dir, base));
+        }, HOT_RELOAD_DEBOUNCE_MS),
+      );
+    });
+    w.on('error', (err) => {
+      log.warn({ err: errMsg(err) }, 'train hot-reload: watcher error');
+    });
+    log.info({ dir }, 'train hot-reload: watching');
+    return w;
+  } catch (err) {
+    log.warn(
+      { err: errMsg(err) },
+      'train hot-reload: watch unavailable (on-demand restart only)',
+    );
+    return null;
+  }
+}
+
+export function killGracefully(
+  proc: ReturnType<typeof Bun.spawn> | null,
+): Promise<unknown> | null {
+  if (proc?.exitCode !== null) return null;
+  try {
+    proc.kill('SIGTERM');
+  } catch {
+  }
+  const grace = setTimeout(() => {
+    try {
+      proc.kill('SIGKILL');
+    } catch {
+    }
+  }, 2_000);
+  return proc.exited.finally(() => {
+    clearTimeout(grace);
+  });
+}
+
+export async function pumpStream(
+  stream: unknown,
+  name: string,
+  kind: 'stdout' | 'stderr',
+  onChunk: (s: string) => void,
+): Promise<void> {
+  if (!stream || typeof stream === 'number') return;
+  const reader = (stream as ReadableStream<Uint8Array>).getReader();
+  const dec = new TextDecoder();
+  try {
+    while (true) {
+      const { value, done } = await reader.read();
+      if (done) break;
+      onChunk(dec.decode(value, { stream: true }));
+    }
+  } catch (err) {
+    log.debug({ name, err: errMsg(err) }, `train: ${kind} pump ended`);
+  }
+}