@stage-labs/metro 0.1.0-beta.3 → 0.1.0-beta.300

package/runtime/node_modules/@metro-labs/daemon/src/stations/materialize.ts

@@ -0,0 +1,267 @@
+import { setPolicies, type PolicyTarget, type ToolPolicy } from '../policy/policy.js';
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { log } from '@metro-labs/core/log';
+import { writeSecure } from '@metro-labs/core/secure-fs';
+import { trainsDir } from '../boot/paths.js';
+import {
+  setAgentMap,
+  setAllowlistMap,
+  setApproversMap,
+  setDisabledAccounts,
+  type AgentMap,
+  type AgentNameMap,
+  type AllowlistMap,
+} from '../agents/map.js';
+import { setKeyMap } from '../agents/keys.js';
+import { STATIONS, type StationName } from '@metro-labs/core/station-names';
+
+interface StationTarget {
+  file: string;
+  fileEnv: string;
+  trainImport: string | null;
+}
+
+export interface LoadedAccount {
+  station: StationName;
+  id: string;
+  allowlist: string[] | null;
+  approvers?: string[];
+  enabled?: boolean;
+  policy?: ToolPolicy;
+  config: Record<string, unknown>;
+}
+
+export interface LoadedConnector {
+  id: string;
+  name: string;
+  url: string;
+  config: Record<string, unknown>;
+}
+
+export interface LoadedAgent {
+  id: string;
+  name: string;
+  accounts: LoadedAccount[];
+  key: string | null;
+}
+
+const METRO_DIR = join(homedir(), '.metro');
+
+const STATION_TARGETS: Record<StationName, StationTarget> = {
+  xmtp: {
+    file: 'xmtp-accounts.json',
+    fileEnv: 'XMTP_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/xmtp/train',
+  },
+  'telegram-bot': {
+    file: 'telegram-bot-accounts.json',
+    fileEnv: 'TELEGRAM_BOT_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/telegram-bot/train',
+  },
+  'telegram': {
+    file: 'telegram-accounts.json',
+    fileEnv: 'TELEGRAM_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/telegram/train',
+  },
+  'discord-bot': {
+    file: 'discord-bot-accounts.json',
+    fileEnv: 'DISCORD_BOT_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/discord-bot/train',
+  },
+  whatsapp: {
+    file: 'whatsapp-accounts.json',
+    fileEnv: 'WHATSAPP_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/whatsapp/train',
+  },
+  threema: {
+    file: 'threema-accounts.json',
+    fileEnv: 'THREEMA_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/threema/train',
+  },
+  outlook: {
+    file: 'outlook-accounts.json',
+    fileEnv: 'OUTLOOK_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/outlook/train',
+  },
+  gmail: {
+    file: 'gmail-accounts.json',
+    fileEnv: 'GMAIL_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/gmail/train',
+  },
+  webhook: {
+    file: 'webhook-accounts.json',
+    fileEnv: 'WEBHOOK_ACCOUNTS_FILE',
+    trainImport: null,
+  },
+};
+
+export function accountFilePath(station: StationName): string {
+  const target = STATION_TARGETS[station];
+  return process.env[target.fileEnv] ?? join(METRO_DIR, target.file);
+}
+
+export type StationSource = () => Promise<LoadedAgent[]>;
+
+export const MOVABLE_STATIONS = new Set<StationName>(
+  STATIONS.filter((s) => s !== 'webhook'),
+);
+
+function trainStubPath(station: StationName): string {
+  return join(trainsDir(), `${station}.ts`);
+}
+
+function currentText(path: string): string | null {
+  try {
+    return readFileSync(path, 'utf8');
+  } catch {
+    return null;
+  }
+}
+
+export function writeIfChanged(path: string, content: string): boolean {
+  if (currentText(path) === content) return false;
+  writeFileSync(path, content);
+  return true;
+}
+
+interface WrittenStations {
+  active: Map<StationName, number>;
+  changed: StationName[];
+}
+
+const channelPolicies = (list: LoadedAgent[]): [PolicyTarget, ToolPolicy][] =>
+  list.flatMap((agent) =>
+    agent.accounts.flatMap((a): [PolicyTarget, ToolPolicy][] =>
+      a.policy === undefined ? [] : [[{ kind: 'channel', station: a.station, account: a.id }, a.policy]],
+    ),
+  );
+
+const approversOf = (list: LoadedAgent[]): Record<string, string[]> =>
+  Object.fromEntries(
+    list.flatMap((agent) => agent.accounts.flatMap((a) => (a.approvers === undefined ? [] : [[`${a.station}/${a.id}`, a.approvers]]))),
+  );
+
+function writeStations(list: LoadedAgent[]): WrittenStations {
+  mkdirSync(METRO_DIR, { recursive: true });
+  mkdirSync(trainsDir(), { recursive: true });
+
+  const byStation = new Map<StationName, LoadedAccount[]>();
+  const map: AgentMap = {};
+  const names: AgentNameMap = {};
+  const allow: AllowlistMap = {};
+  const disabled = new Set<string>();
+  for (const agent of list) {
+    names[agent.id] = agent.name;
+    for (const a of agent.accounts) {
+      if (!(a.station in STATION_TARGETS)) {
+        log.error(
+          { station: a.station, id: a.id },
+          'unknown station — this metro is older or newer than the row; skipping it',
+        );
+        continue;
+      }
+      map[`${a.station}/${a.id}`] = agent.id;
+      if (a.allowlist) allow[`${a.station}/${a.id}`] = a.allowlist;
+      if (a.enabled === false) disabled.add(`${a.station}/${a.id}`);
+      const cur = byStation.get(a.station);
+      if (cur) cur.push(a);
+      else byStation.set(a.station, [a]);
+    }
+  }
+  setAgentMap(map, names);
+  setAllowlistMap(allow);
+  setApproversMap(approversOf(list));
+  setDisabledAccounts(disabled);
+  setPolicies('channel', channelPolicies(list));
+
+  const active = new Map<StationName, number>();
+  const changed: StationName[] = [];
+  for (const [station, accts] of byStation) {
+    const records = accts.map((a) => ({ id: a.id, ...a.config }));
+    const path = accountFilePath(station);
+    const content = JSON.stringify(records, null, 2);
+    if (currentText(path) !== content) {
+      writeSecure(path, content);
+      changed.push(station);
+    }
+    const { trainImport } = STATION_TARGETS[station];
+    if (trainImport !== null)
+      writeIfChanged(trainStubPath(station), `import '${trainImport}';\n`);
+    active.set(station, accts.length);
+  }
+  return { active, changed };
+}
+
+const BLANKED = '[]\n';
+
+function pruneStations(present: Map<StationName, number>): StationName[] {
+  const removed: StationName[] = [];
+  for (const station of Object.keys(STATION_TARGETS) as StationName[]) {
+    if (present.has(station)) continue;
+    const hasTrain = STATION_TARGETS[station].trainImport !== null;
+    const stub = trainStubPath(station);
+    const stubGone = !hasTrain || !existsSync(stub);
+    const file = currentText(accountFilePath(station));
+    const fileInert = file === null || file === BLANKED;
+    if (stubGone && fileInert) continue;
+    if (!fileInert) writeSecure(accountFilePath(station), BLANKED);
+    if (hasTrain) rmSync(stub, { force: true });
+    removed.push(station);
+  }
+  return removed;
+}
+
+const stationLabels = (m: Map<StationName, number>): string[] =>
+  [...m].map(([station, n]) => `${station}(${n})`);
+
+function applyKeyMap(list: LoadedAgent[]): void {
+  setKeyMap(
+    list.flatMap((agent) =>
+      agent.key === null ? [] : [{ key: agent.key, agentId: agent.id }],
+    ),
+  );
+}
+
+async function loadAndWrite(
+  source: StationSource,
+): Promise<WrittenStations & { agents: number }> {
+  const list = await source();
+  applyKeyMap(list);
+  return { ...writeStations(list), agents: list.length };
+}
+
+export async function materializeFrom(source: StationSource): Promise<void> {
+  const { active, agents: found } = await loadAndWrite(source);
+  const removed = pruneStations(active);
+  log.info(
+    { stations: stationLabels(active), agents: found, removed },
+    'materialized station accounts',
+  );
+}
+
+export interface ReloadedStations {
+  active: StationName[];
+  removed: StationName[];
+  changed: StationName[];
+}
+
+export async function reloadFrom(
+  source: StationSource,
+): Promise<ReloadedStations> {
+  const { active, changed } = await loadAndWrite(source);
+  const removed = pruneStations(active);
+  if (removed.length > 0 || changed.length > 0)
+    log.info(
+      { stations: stationLabels(active), removed, changed },
+      'reloaded station accounts',
+    );
+  return { active: [...active.keys()], removed, changed };
+}

package/runtime/node_modules/@metro-labs/daemon/src/stations/registry.ts

@@ -0,0 +1,53 @@
+import type { Station } from '@metro-labs/core/stations/types';
+import { Line } from '@metro-labs/core/lines';
+import { errMsg, log } from '@metro-labs/core/log';
+import { xmtpStation } from '@metro-labs/xmtp';
+import { telegramBotStation } from '@metro-labs/telegram-bot';
+import { telegramStation } from '@metro-labs/telegram';
+import { discordBotStation } from '@metro-labs/discord-bot';
+import { whatsappStation } from '@metro-labs/whatsapp';
+import { threemaStation } from '@metro-labs/threema';
+import { outlookStation } from '@metro-labs/outlook';
+import { gmailStation } from '@metro-labs/gmail';
+import { webhookStation } from '@metro-labs/webhook';
+
+export const STATIONS: readonly Station[] = [
+  xmtpStation,
+  telegramBotStation,
+  telegramStation,
+  discordBotStation,
+  whatsappStation,
+  threemaStation,
+  outlookStation,
+  gmailStation,
+  webhookStation,
+];
+
+export const stationByName = (name: string): Station | undefined =>
+  STATIONS.find((s) => s.name === name);
+
+export const stationForLine = (line: string): Station | undefined => {
+  const s = Line.station(line);
+  return s ? stationByName(s) : undefined;
+};
+
+export const accountStationNames = (): string[] =>
+  STATIONS.filter((s) => s.hasAccounts).map((s) => s.name);
+
+export const accountStationCapabilities = (): Record<string, string[]> => {
+  const out: Record<string, string[]> = {};
+  for (const s of STATIONS)
+    if (s.hasAccounts) out[s.name] = [...s.messageVerbs].sort();
+  return out;
+};
+
+export function forgetOrphans(known: readonly { station: string; id: string }[]): void {
+  for (const s of STATIONS) {
+    if (s.forgetExcept === undefined) continue;
+    try {
+      s.forgetExcept(known.filter((a) => a.station === s.name).map((a) => a.id));
+    } catch (err) {
+      log.warn({ station: s.name, err: errMsg(err) }, 'stations: could not remove the files of detached accounts');
+    }
+  }
+}

package/runtime/node_modules/@metro-labs/daemon/src/stations/runtime-deps.ts

@@ -0,0 +1,135 @@
+import { spawnSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { copyFileSync, existsSync, lstatSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { log } from '@metro-labs/core/log';
+import { isRecord } from '@metro-labs/core/is-record';
+import type { StationName } from '@metro-labs/core/station-names';
+import { knownAccounts } from '../agents/map.js';
+
+const INSTALL_TIMEOUT_MS = 15 * 60_000;
+
+export interface RuntimeManifest {
+  core: Record<string, string>;
+  stations: Record<string, Record<string, string>>;
+  patchedDependencies?: Record<string, string>;
+}
+
+export interface RuntimeStore {
+  dir: string;
+  sources: string;
+  manifest: RuntimeManifest;
+}
+
+const ranges = (raw: unknown): Record<string, string> =>
+  isRecord(raw)
+    ? Object.fromEntries(Object.entries(raw).filter((e): e is [string, string] => typeof e[1] === 'string'))
+    : {};
+
+function patchesOf(raw: unknown): Record<string, string> {
+  if (!isRecord(raw)) throw new Error('Invalid runtime patch map');
+  return Object.fromEntries(Object.entries(raw).map(([name, path]) => {
+    if (typeof path !== 'string' || !/^patches\/[a-f0-9]{64}\.patch$/.test(path))
+      throw new Error(`Invalid runtime patch path for ${name}`);
+    return [name, path];
+  }));
+}
+
+export function readManifest(path: string): RuntimeManifest {
+  const raw: unknown = JSON.parse(readFileSync(path, 'utf8'));
+  if (!isRecord(raw)) throw new Error(`${path} is not a runtime manifest`);
+  const stations = isRecord(raw.stations) ? raw.stations : {};
+  return {
+    core: ranges(raw.core),
+    stations: Object.fromEntries(Object.entries(stations).map(([name, deps]) => [name, ranges(deps)])),
+    ...(raw.patchedDependencies === undefined ? {} : { patchedDependencies: patchesOf(raw.patchedDependencies) }),
+  };
+}
+
+export function runtimeStore(): RuntimeStore | null {
+  const dir = process.env.METRO_RUNTIME_STORE?.trim() ?? '';
+  const manifest = process.env.METRO_RUNTIME_MANIFEST?.trim() ?? '';
+  if (dir === '' || manifest === '') return null;
+  return { dir, sources: dirname(manifest), manifest: readManifest(manifest) };
+}
+
+export function dependenciesFor(manifest: RuntimeManifest, stations: Iterable<string>): Record<string, string> {
+  const out: Record<string, string> = { ...manifest.core };
+  for (const station of new Set(stations)) Object.assign(out, manifest.stations[station] ?? {});
+  return Object.fromEntries(Object.entries(out).sort(([a], [b]) => a.localeCompare(b)));
+}
+
+function packageText(deps: Record<string, string>, patches: Record<string, string>): string {
+  const patchedDependencies = Object.fromEntries(
+    Object.entries(deps).flatMap(([name, version]) => {
+      const key = `${name}@${version}`;
+      const path = patches[key];
+      return path === undefined ? [] : [[key, path]];
+    }),
+  );
+  return `${JSON.stringify({
+    name: 'metro-runtime',
+    private: true,
+    dependencies: deps,
+    ...(Object.keys(patchedDependencies).length === 0 ? {} : { patchedDependencies }),
+  }, null, 2)}\n`;
+}
+
+function current(dir: string): string | null {
+  try {
+    return readFileSync(join(dir, 'package.json'), 'utf8');
+  } catch {
+    return null;
+  }
+}
+
+function copyPatches(store: RuntimeStore): void {
+  const paths = Object.values(patchesOf(store.manifest.patchedDependencies ?? {}));
+  if (paths.length === 0) return;
+  const destination = join(store.dir, 'patches');
+  mkdirSync(destination, { recursive: true });
+  if (!lstatSync(join(store.sources, 'patches')).isDirectory() || !lstatSync(destination).isDirectory())
+    throw new Error('Invalid runtime patch directory');
+  for (const path of new Set(paths)) {
+    const source = join(store.sources, path);
+    const target = join(store.dir, path);
+    if (!lstatSync(source).isFile() || lstatSync(target, { throwIfNoEntry: false })?.isFile() === false)
+      throw new Error(`Invalid runtime patch file: ${path}`);
+    const hash = createHash('sha256').update(readFileSync(source)).digest('hex');
+    if (path !== `patches/${hash}.patch`) throw new Error(`Invalid runtime patch contents: ${path}`);
+    copyFileSync(source, target);
+  }
+}
+
+export function installRuntime(store: RuntimeStore, stations: Iterable<string>): boolean {
+  const wanted = packageText(dependenciesFor(store.manifest, stations), store.manifest.patchedDependencies ?? {});
+  const marker = join(store.dir, 'node_modules', '.metro-installed');
+  try {
+    copyPatches(store);
+  } catch (err) {
+    rmSync(marker, { force: true });
+    throw err;
+  }
+  if (existsSync(marker) && current(store.dir) === wanted) return false;
+  rmSync(marker, { force: true });
+  mkdirSync(store.dir, { recursive: true });
+  writeFileSync(join(store.dir, 'package.json'), wanted);
+  log.info({ dir: store.dir }, 'runtime: installing the channel SDKs this machine needs');
+  const run = spawnSync('bun', ['install', '--no-summary', '--no-progress'], {
+    cwd: store.dir,
+    stdio: ['ignore', 'inherit', 'inherit'],
+    timeout: INSTALL_TIMEOUT_MS,
+  });
+  if (run.error !== undefined || run.status !== 0)
+    throw new Error(`bun install failed in ${store.dir}: ${run.error?.message ?? `exit ${String(run.status)}`}`);
+  writeFileSync(marker, `${new Date().toISOString()}\n`);
+  return true;
+}
+
+export function ensureStationDeps(station?: StationName): void {
+  const store = runtimeStore();
+  if (store === null) return;
+  const stations = knownAccounts().map((a) => a.station);
+  if (station !== undefined) stations.push(station);
+  installRuntime(store, stations);
+}

package/runtime/node_modules/@metro-labs/daemon/src/stations/supervisor-io.ts

@@ -0,0 +1,97 @@
+import { watch, type FSWatcher } from 'node:fs';
+import { join, parse as parsePath } from 'node:path';
+import { errMsg, log } from '@metro-labs/core/log';
+import type { Pending } from '@metro-labs/core/trains/protocol';
+
+export const RESTART_BACKOFFS_MS = [1_000, 5_000, 30_000] as const;
+export const MAX_CONSECUTIVE_FAILS = 5;
+export const HOT_RELOAD_DEBOUNCE_MS = 300;
+const TRAIN_EXT = /\.(ts|js|mjs)$/;
+
+export interface TrainState {
+  name: string;
+  path: string;
+  proc: ReturnType<typeof Bun.spawn> | null;
+  pending: Map<string, Pending>;
+  buf: string;
+  errBuf: string;
+  failCount: number;
+  restartTimer: ReturnType<typeof setTimeout> | null;
+  startedAt: string | null;
+  stopped: boolean;
+}
+
+export function startWatcher(
+  dir: string,
+  timers: Map<string, ReturnType<typeof setTimeout>>,
+  onChange: (name: string, path: string) => void,
+): FSWatcher | null {
+  try {
+    const w = watch(dir, (_event, filename) => {
+      if (!filename) return;
+      const base = filename;
+      if (!TRAIN_EXT.test(base) || base.startsWith('_') || base.startsWith('.'))
+        return;
+      const name = parsePath(base).name;
+      const existing = timers.get(name);
+      if (existing) clearTimeout(existing);
+      timers.set(
+        name,
+        setTimeout(() => {
+          timers.delete(name);
+          onChange(name, join(dir, base));
+        }, HOT_RELOAD_DEBOUNCE_MS),
+      );
+    });
+    w.on('error', (err) => {
+      log.warn({ err: errMsg(err) }, 'train hot-reload: watcher error');
+    });
+    log.info({ dir }, 'train hot-reload: watching');
+    return w;
+  } catch (err) {
+    log.warn(
+      { err: errMsg(err) },
+      'train hot-reload: watch unavailable (on-demand restart only)',
+    );
+    return null;
+  }
+}
+
+export function killGracefully(
+  proc: ReturnType<typeof Bun.spawn> | null,
+): Promise<unknown> | null {
+  if (proc?.exitCode !== null) return null;
+  try {
+    proc.kill('SIGTERM');
+  } catch {
+  }
+  const grace = setTimeout(() => {
+    try {
+      proc.kill('SIGKILL');
+    } catch {
+    }
+  }, 2_000);
+  return proc.exited.finally(() => {
+    clearTimeout(grace);
+  });
+}
+
+export async function pumpStream(
+  stream: unknown,
+  name: string,
+  kind: 'stdout' | 'stderr',
+  onChunk: (s: string) => void,
+): Promise<void> {
+  if (!stream || typeof stream === 'number') return;
+  const reader = (stream as ReadableStream<Uint8Array>).getReader();
+  const dec = new TextDecoder();
+  try {
+    while (true) {
+      const { value, done } = await reader.read();
+      if (done) break;
+      onChunk(dec.decode(value, { stream: true }));
+    }
+  } catch (err) {
+    log.debug({ name, err: errMsg(err) }, `train: ${kind} pump ended`);
+  }
+}