@stage-labs/metro 0.1.0-beta.26 → 0.1.0-beta.261

package/runtime/node_modules/@metro-labs/{mcp/src/db → daemon/src/stations}/materialize.ts

@@ -1,3 +1,4 @@
+import { setPolicies, type PolicyTarget, type ToolPolicy } from '../policy/policy.js';
 import {
   existsSync,
   mkdirSync,
@@ -6,22 +7,21 @@ import {
   writeFileSync,
 } from 'node:fs';
 import { homedir } from 'node:os';
-import { ID_RE } from './ids.js';
 import { join } from 'node:path';
-import { eq } from 'drizzle-orm';
-import { log } from '../daemon/log.js';
-import { writeSecure } from '../daemon/secure-fs.js';
-import { closeDb, databaseUrl, getDb } from './client.js';
+import { log } from '@metro-labs/core/log';
+import { writeSecure } from '@metro-labs/core/secure-fs';
+import { trainsDir } from '../boot/paths.js';
 import {
   setAgentMap,
   setAllowlistMap,
+  setApproversMap,
+  setDisabledAccounts,
   type AgentMap,
   type AgentNameMap,
   type AllowlistMap,
-} from './agent-map.js';
-import { setKeyMap } from './key-map.js';
-import { carryTokenStores, PREVIOUS_ACCOUNT_ID } from './token-carry.js';
-import { STATIONS, stations, agents, type StationName } from './schema.js';
+} from '../agents/map.js';
+import { setKeyMap } from '../agents/keys.js';
+import { STATIONS, type StationName } from '@metro-labs/core/station-names';
 
 interface StationTarget {
   file: string;
@@ -33,6 +33,16 @@ export interface LoadedAccount {
   station: StationName;
   id: string;
   allowlist: string[] | null;
+  approvers?: string[];
+  enabled?: boolean;
+  policy?: ToolPolicy;
+  config: Record<string, unknown>;
+}
+
+export interface LoadedConnector {
+  id: string;
+  name: string;
+  url: string;
   config: Record<string, unknown>;
 }
 
@@ -44,7 +54,6 @@ export interface LoadedAgent {
 }
 
 const METRO_DIR = join(homedir(), '.metro');
-const TRAINS_DIR = process.env.METRO_TRAINS_DIR ?? join(METRO_DIR, 'trains');
 
 const STATION_TARGETS: Record<StationName, StationTarget> = {
   xmtp: {
@@ -72,6 +81,21 @@ const STATION_TARGETS: Record<StationName, StationTarget> = {
     fileEnv: 'WHATSAPP_ACCOUNTS_FILE',
     trainImport: '@metro-labs/whatsapp/train',
   },
+  threema: {
+    file: 'threema-accounts.json',
+    fileEnv: 'THREEMA_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/threema/train',
+  },
+  outlook: {
+    file: 'outlook-accounts.json',
+    fileEnv: 'OUTLOOK_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/outlook/train',
+  },
+  gmail: {
+    file: 'gmail-accounts.json',
+    fileEnv: 'GMAIL_ACCOUNTS_FILE',
+    trainImport: '@metro-labs/gmail/train',
+  },
   webhook: {
     file: 'webhook-accounts.json',
     fileEnv: 'WEBHOOK_ACCOUNTS_FILE',
@@ -79,137 +103,19 @@ const STATION_TARGETS: Record<StationName, StationTarget> = {
   },
 };
 
-function accountFilePath(station: StationName): string {
+export function accountFilePath(station: StationName): string {
   const target = STATION_TARGETS[station];
   return process.env[target.fileEnv] ?? join(METRO_DIR, target.file);
 }
 
-function agentFilter(): string | undefined {
-  const v = process.env.METRO_AGENT?.trim();
-  if (!v) return undefined;
-  if (!ID_RE.test(v))
-    throw new Error(`METRO_AGENT must be an 11-character agent id, got '${v}'`);
-  return v;
-}
-
 export type StationSource = () => Promise<LoadedAgent[]>;
 
 export const MOVABLE_STATIONS = new Set<StationName>(
   STATIONS.filter((s) => s !== 'webhook'),
 );
 
-export function unmovableStations(list: LoadedAccount[]): StationName[] {
-  return [
-    ...new Set(
-      list
-        .map((a) => a.station)
-        .filter((station) => !MOVABLE_STATIONS.has(station)),
-    ),
-  ];
-}
-
-export interface StationRecord {
-  id: string;
-  agentId: string;
-  station: StationName;
-}
-
-export async function listAllStations(): Promise<StationRecord[]> {
-  if (!databaseUrl()) return [];
-  const rows = await getDb()
-    .select({
-      id: stations.id,
-      agentId: stations.agentId,
-      station: stations.station,
-    })
-    .from(stations);
-  return rows;
-}
-
-export async function loadAllStationsFor(
-  agentId: string,
-): Promise<LoadedAccount[]> {
-  const rows = await getDb()
-    .select()
-    .from(stations)
-    .where(eq(stations.agentId, agentId));
-  return rows.map((r) => ({
-    station: r.station,
-    id: r.id,
-    allowlist: r.allowlist,
-    config: r.config as Record<string, unknown>,
-  }));
-}
-
-export async function loadAgentForRuntime(
-  agentId: string,
-): Promise<LoadedAgent> {
-  const db = getDb();
-  const rows = await db.select().from(agents).where(eq(agents.id, agentId));
-  const a = rows[0];
-  if (a === undefined) throw new Error(`no such agent '${agentId}'`);
-  const acctRows = await db
-    .select()
-    .from(stations)
-    .where(eq(stations.agentId, a.id));
-  return {
-    id: a.id,
-    name: a.name,
-    key: a.key,
-    accounts: acctRows
-      .filter((r) => MOVABLE_STATIONS.has(r.station))
-      .map((r) => ({
-        station: r.station,
-        id: r.id,
-        allowlist: r.allowlist,
-        config: r.config as Record<string, unknown>,
-      })),
-  };
-}
-
-export const pgSource: StationSource = () => loadAgents();
-
-async function loadAgents(): Promise<LoadedAgent[]> {
-  const db = getDb();
-  const only = agentFilter();
-  const agentRows =
-    only !== undefined
-      ? await db.select().from(agents).where(eq(agents.id, only))
-      : await db.select().from(agents);
-  if (only !== undefined && agentRows.length === 0)
-    throw new Error(`METRO_AGENT=${only} does not match any agent`);
-
-  const out: LoadedAgent[] = [];
-  for (const a of agentRows) {
-    const acctRows = await db
-      .select()
-      .from(stations)
-      .where(eq(stations.agentId, a.id));
-    out.push({
-      id: a.id,
-      name: a.name,
-      accounts: acctRows.map((r) => ({
-        station: r.station,
-        id: r.id,
-        allowlist: r.allowlist,
-        config: r.config as Record<string, unknown>,
-      })),
-      key: a.runtimeId === null ? a.key : null,
-    });
-  }
-  return out;
-}
-
-function trainConfig(
-  config: Record<string, unknown>,
-): Record<string, unknown> {
-  return Object.fromEntries(
-    Object.entries(config).filter(([key]) => key !== PREVIOUS_ACCOUNT_ID),
-  );
-}
-
 function trainStubPath(station: StationName): string {
-  return join(TRAINS_DIR, `${station}.ts`);
+  return join(trainsDir(), `${station}.ts`);
 }
 
 function currentText(path: string): string | null {
@@ -226,25 +132,32 @@ export function writeIfChanged(path: string, content: string): boolean {
   return true;
 }
 
-export const isLocalRuntime = (): boolean =>
-  (process.env.METRO_RUN_TOKEN?.trim() ?? '') !== '';
-
-export const stationRunsHere = (station: StationName): boolean =>
-  isLocalRuntime() || !MOVABLE_STATIONS.has(station);
-
 interface WrittenStations {
   active: Map<StationName, number>;
   changed: StationName[];
 }
 
+const channelPolicies = (list: LoadedAgent[]): [PolicyTarget, ToolPolicy][] =>
+  list.flatMap((agent) =>
+    agent.accounts.flatMap((a): [PolicyTarget, ToolPolicy][] =>
+      a.policy === undefined ? [] : [[{ kind: 'channel', station: a.station, account: a.id }, a.policy]],
+    ),
+  );
+
+const approversOf = (list: LoadedAgent[]): Record<string, string[]> =>
+  Object.fromEntries(
+    list.flatMap((agent) => agent.accounts.flatMap((a) => (a.approvers === undefined ? [] : [[`${a.station}/${a.id}`, a.approvers]]))),
+  );
+
 function writeStations(list: LoadedAgent[]): WrittenStations {
   mkdirSync(METRO_DIR, { recursive: true });
-  mkdirSync(TRAINS_DIR, { recursive: true });
+  mkdirSync(trainsDir(), { recursive: true });
 
   const byStation = new Map<StationName, LoadedAccount[]>();
   const map: AgentMap = {};
   const names: AgentNameMap = {};
   const allow: AllowlistMap = {};
+  const disabled = new Set<string>();
   for (const agent of list) {
     names[agent.id] = agent.name;
     for (const a of agent.accounts) {
@@ -257,21 +170,22 @@ function writeStations(list: LoadedAgent[]): WrittenStations {
       }
       map[`${a.station}/${a.id}`] = agent.id;
       if (a.allowlist) allow[`${a.station}/${a.id}`] = a.allowlist;
-      if (stationRunsHere(a.station)) {
-        const cur = byStation.get(a.station);
-        if (cur) cur.push(a);
-        else byStation.set(a.station, [a]);
-      }
+      if (a.enabled === false) disabled.add(`${a.station}/${a.id}`);
+      const cur = byStation.get(a.station);
+      if (cur) cur.push(a);
+      else byStation.set(a.station, [a]);
     }
   }
   setAgentMap(map, names);
   setAllowlistMap(allow);
-  carryTokenStores(list.flatMap((agent) => agent.accounts));
+  setApproversMap(approversOf(list));
+  setDisabledAccounts(disabled);
+  setPolicies('channel', channelPolicies(list));
 
   const active = new Map<StationName, number>();
   const changed: StationName[] = [];
   for (const [station, accts] of byStation) {
-    const records = accts.map((a) => ({ id: a.id, ...trainConfig(a.config) }));
+    const records = accts.map((a) => ({ id: a.id, ...a.config }));
     const path = accountFilePath(station);
     const content = JSON.stringify(records, null, 2);
     if (currentText(path) !== content) {
@@ -309,7 +223,6 @@ const stationLabels = (m: Map<StationName, number>): string[] =>
   [...m].map(([station, n]) => `${station}(${n})`);
 
 function applyKeyMap(list: LoadedAgent[]): void {
-  lastKey = list.find((a) => a.key !== null)?.key ?? null;
   setKeyMap(
     list.flatMap((agent) =>
       agent.key === null ? [] : [{ key: agent.key, agentId: agent.id }],
@@ -317,12 +230,6 @@ function applyKeyMap(list: LoadedAgent[]): void {
   );
 }
 
-let lastKey: string | null = null;
-
-export function localAgentKey(): string | null {
-  return lastKey;
-}
-
 async function loadAndWrite(
   source: StationSource,
 ): Promise<WrittenStations & { agents: number }> {
@@ -333,24 +240,13 @@ async function loadAndWrite(
 
 export async function materializeFrom(source: StationSource): Promise<void> {
   const { active, agents: found } = await loadAndWrite(source);
-
-  if (found === 0) throw new Error('no agents found — nothing to materialize');
+  const removed = pruneStations(active);
   log.info(
-    { stations: stationLabels(active), agents: found },
+    { stations: stationLabels(active), agents: found, removed },
     'materialized station accounts',
   );
 }
 
-export async function materializeFromDb(): Promise<void> {
-  if (!databaseUrl())
-    throw new Error('DATABASE_URL is not set — accounts load from Postgres');
-  try {
-    await materializeFrom(pgSource);
-  } finally {
-    await closeDb();
-  }
-}
-
 export interface ReloadedStations {
   active: StationName[];
   removed: StationName[];
@@ -369,14 +265,3 @@ export async function reloadFrom(
     );
   return { active: [...active.keys()], removed, changed };
 }
-
-export async function reloadAccountsFromDb(): Promise<ReloadedStations> {
-  if (!databaseUrl())
-    throw new Error('DATABASE_URL is not set — accounts load from Postgres');
-  return reloadFrom(pgSource);
-}
-
-if (import.meta.main) {
-  await materializeFromDb();
-  process.exit(0);
-}

package/runtime/node_modules/@metro-labs/{mcp → daemon}/src/stations/registry.ts

@@ -1,10 +1,14 @@
-import type { Station } from './types.js';
-import { Line } from './lines.js';
+import type { Station } from '@metro-labs/core/stations/types';
+import { Line } from '@metro-labs/core/lines';
+import { errMsg, log } from '@metro-labs/core/log';
 import { xmtpStation } from '@metro-labs/xmtp';
 import { telegramBotStation } from '@metro-labs/telegram-bot';
 import { telegramStation } from '@metro-labs/telegram';
 import { discordBotStation } from '@metro-labs/discord-bot';
 import { whatsappStation } from '@metro-labs/whatsapp';
+import { threemaStation } from '@metro-labs/threema';
+import { outlookStation } from '@metro-labs/outlook';
+import { gmailStation } from '@metro-labs/gmail';
 import { webhookStation } from '@metro-labs/webhook';
 
 export const STATIONS: readonly Station[] = [
@@ -13,6 +17,9 @@ export const STATIONS: readonly Station[] = [
   telegramStation,
   discordBotStation,
   whatsappStation,
+  threemaStation,
+  outlookStation,
+  gmailStation,
   webhookStation,
 ];
 
@@ -33,3 +40,14 @@ export const accountStationCapabilities = (): Record<string, string[]> => {
     if (s.hasAccounts) out[s.name] = [...s.messageVerbs].sort();
   return out;
 };
+
+export function forgetOrphans(known: readonly { station: string; id: string }[]): void {
+  for (const s of STATIONS) {
+    if (s.forgetExcept === undefined) continue;
+    try {
+      s.forgetExcept(known.filter((a) => a.station === s.name).map((a) => a.id));
+    } catch (err) {
+      log.warn({ station: s.name, err: errMsg(err) }, 'stations: could not remove the files of detached accounts');
+    }
+  }
+}

package/runtime/node_modules/@metro-labs/daemon/src/stations/runtime-deps.ts

@@ -0,0 +1,82 @@
+import { spawnSync } from 'node:child_process';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { log } from '@metro-labs/core/log';
+import { isRecord } from '@metro-labs/core/is-record';
+import type { StationName } from '@metro-labs/core/station-names';
+import { knownAccounts } from '../agents/map.js';
+
+const INSTALL_TIMEOUT_MS = 15 * 60_000;
+
+export interface RuntimeManifest {
+  core: Record<string, string>;
+  stations: Record<string, Record<string, string>>;
+}
+
+export interface RuntimeStore {
+  dir: string;
+  manifest: RuntimeManifest;
+}
+
+const ranges = (raw: unknown): Record<string, string> =>
+  isRecord(raw)
+    ? Object.fromEntries(Object.entries(raw).filter((e): e is [string, string] => typeof e[1] === 'string'))
+    : {};
+
+export function readManifest(path: string): RuntimeManifest {
+  const raw: unknown = JSON.parse(readFileSync(path, 'utf8'));
+  if (!isRecord(raw)) throw new Error(`${path} is not a runtime manifest`);
+  const stations = isRecord(raw.stations) ? raw.stations : {};
+  return {
+    core: ranges(raw.core),
+    stations: Object.fromEntries(Object.entries(stations).map(([name, deps]) => [name, ranges(deps)])),
+  };
+}
+
+export function runtimeStore(): RuntimeStore | null {
+  const dir = process.env.METRO_RUNTIME_STORE?.trim() ?? '';
+  const manifest = process.env.METRO_RUNTIME_MANIFEST?.trim() ?? '';
+  if (dir === '' || manifest === '') return null;
+  return { dir, manifest: readManifest(manifest) };
+}
+
+export function dependenciesFor(manifest: RuntimeManifest, stations: Iterable<string>): Record<string, string> {
+  const out: Record<string, string> = { ...manifest.core };
+  for (const station of new Set(stations)) Object.assign(out, manifest.stations[station] ?? {});
+  return Object.fromEntries(Object.entries(out).sort(([a], [b]) => a.localeCompare(b)));
+}
+
+const packageText = (deps: Record<string, string>): string =>
+  `${JSON.stringify({ name: 'metro-runtime', private: true, dependencies: deps }, null, 2)}\n`;
+
+function current(dir: string): string | null {
+  try {
+    return readFileSync(join(dir, 'package.json'), 'utf8');
+  } catch {
+    return null;
+  }
+}
+
+export function installRuntime(store: RuntimeStore, stations: Iterable<string>): boolean {
+  const wanted = packageText(dependenciesFor(store.manifest, stations));
+  const installed = existsSync(join(store.dir, 'node_modules', '.metro-installed'));
+  if (installed && current(store.dir) === wanted) return false;
+  mkdirSync(store.dir, { recursive: true });
+  writeFileSync(join(store.dir, 'package.json'), wanted);
+  log.info({ dir: store.dir }, 'runtime: installing the channel SDKs this machine needs');
+  const run = spawnSync('bun', ['install', '--no-summary', '--no-progress'], {
+    cwd: store.dir,
+    stdio: ['ignore', 'inherit', 'inherit'],
+    timeout: INSTALL_TIMEOUT_MS,
+  });
+  if (run.error !== undefined || run.status !== 0)
+    throw new Error(`bun install failed in ${store.dir}: ${run.error?.message ?? `exit ${String(run.status)}`}`);
+  writeFileSync(join(store.dir, 'node_modules', '.metro-installed'), `${new Date().toISOString()}\n`);
+  return true;
+}
+
+export function ensureStationDeps(station: StationName): void {
+  const store = runtimeStore();
+  if (store === null) return;
+  installRuntime(store, [...knownAccounts().map((a) => a.station), station]);
+}

package/runtime/node_modules/@metro-labs/{mcp/src/daemon → daemon/src/stations}/supervisor-io.ts

@@ -1,12 +1,12 @@
 import { watch, type FSWatcher } from 'node:fs';
 import { join, parse as parsePath } from 'node:path';
-import { errMsg, log } from './log.js';
-import type { Pending } from './protocol.js';
+import { errMsg, log } from '@metro-labs/core/log';
+import type { Pending } from '@metro-labs/core/trains/protocol';
 
 export const RESTART_BACKOFFS_MS = [1_000, 5_000, 30_000] as const;
 export const MAX_CONSECUTIVE_FAILS = 5;
 export const HOT_RELOAD_DEBOUNCE_MS = 300;
-export const TRAIN_EXT = /\.(ts|js|mjs)$/;
+const TRAIN_EXT = /\.(ts|js|mjs)$/;
 
 export interface TrainState {
   name: string;

package/runtime/node_modules/@metro-labs/{mcp/src/daemon → daemon/src/stations}/supervisor.ts

@@ -1,8 +1,7 @@
 import { mkdirSync, statSync, type FSWatcher } from 'node:fs';
-import { homedir } from 'node:os';
 import { join } from 'node:path';
-import { errMsg, log } from './log.js';
-import { daemonSelf } from './events.js';
+import { errMsg, log } from '@metro-labs/core/log';
+import { userSelf } from '@metro-labs/core/events';
 import {
   drainLines,
   failAllPending,
@@ -12,7 +11,7 @@ import {
   sendCall,
   type TrainCallResponse,
   type TrainEvent,
-} from './protocol.js';
+} from '@metro-labs/core/trains/protocol';
 import {
   HOT_RELOAD_DEBOUNCE_MS,
   MAX_CONSECUTIVE_FAILS,
@@ -22,9 +21,7 @@ import {
   startWatcher,
   type TrainState,
 } from './supervisor-io.js';
-
-export const TRAINS_DIR =
-  process.env.METRO_TRAINS_DIR ?? join(homedir(), '.metro', 'trains');
+import { trainsDir } from '../boot/paths.js';
 
 const DYLD_FALLBACK_ENV: Record<string, string> =
   process.platform === 'darwin'
@@ -45,7 +42,7 @@ export class TrainSupervisor {
   private watcher: FSWatcher | null = null;
   private reloadTimers = new Map<string, ReturnType<typeof setTimeout>>();
 
-  constructor(private dir: string = TRAINS_DIR) {}
+  constructor(private dir: string = trainsDir()) {}
 
   running(): string[] {
     return [...this.trains.keys()];
@@ -212,7 +209,7 @@ export class TrainSupervisor {
           ...process.env,
           ...DYLD_FALLBACK_ENV,
           METRO_TRAIN_NAME: state.name,
-          METRO_SELF_URI: daemonSelf(),
+          METRO_SELF_URI: userSelf(),
         },
       });
       state.proc = proc;

package/runtime/node_modules/@metro-labs/daemon/src/stations/threema-callbacks.ts

@@ -0,0 +1,32 @@
+import { readJson } from '@metro-labs/core/secure-fs';
+import { accountFilePath } from './materialize.js';
+
+export interface ThreemaCallback {
+  id: string;
+  callbackId: string;
+  callbackToken: string;
+}
+
+const str = (v: unknown): string | undefined =>
+  typeof v === 'string' && v !== '' ? v : undefined;
+
+function toCallback(row: unknown): ThreemaCallback | null {
+  if (typeof row !== 'object' || row === null) return null;
+  const rec = row as Record<string, unknown>;
+  const id = str(rec.id);
+  const callbackId = str(rec.callbackId);
+  const callbackToken = str(rec.callbackToken);
+  if (id === undefined || callbackId === undefined || callbackToken === undefined) return null;
+  return { id, callbackId, callbackToken };
+}
+
+export function listThreemaCallbacks(): ThreemaCallback[] {
+  const raw = readJson<unknown[]>(accountFilePath('threema'), [], {
+    warn: 'threema-accounts.json: malformed, ignoring',
+  });
+  if (!Array.isArray(raw)) return [];
+  return raw.map(toCallback).filter((c): c is ThreemaCallback => c !== null);
+}
+
+export const findThreemaCallback = (callbackId: string): ThreemaCallback | undefined =>
+  listThreemaCallbacks().find((c) => c.callbackId === callbackId);

package/runtime/node_modules/@metro-labs/{mcp/src/daemon → daemon/src/stations}/train-call.ts

@@ -1,4 +1,4 @@
-import type { TrainCallResponse } from './protocol.js';
+import type { TrainCallResponse } from '@metro-labs/core/trains/protocol';
 
 export type TrainCallBackend = (
   train: string,

package/runtime/node_modules/@metro-labs/daemon/src/stations/webhook-endpoints.ts

@@ -0,0 +1,51 @@
+import type { Endpoint } from '@metro-labs/core/endpoints';
+import { timingSafeEqual } from 'node:crypto';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { readJson } from '@metro-labs/core/secure-fs';
+
+interface AccountRecord {
+  id?: unknown;
+  webhookId?: unknown;
+  label?: unknown;
+  secret?: unknown;
+  createdAt?: unknown;
+}
+
+const accountsFile = (): string =>
+  process.env.WEBHOOK_ACCOUNTS_FILE ??
+  join(homedir(), '.metro', 'webhook-accounts.json');
+
+const str = (v: unknown): string | undefined =>
+  typeof v === 'string' && v !== '' ? v : undefined;
+
+function toEndpoint(raw: AccountRecord): Endpoint | null {
+  const id = str(raw.id);
+  if (id === undefined) return null;
+  return {
+    id,
+    webhookId: str(raw.webhookId),
+    label: str(raw.label) ?? id,
+    secret: str(raw.secret),
+    createdAt: str(raw.createdAt) ?? '',
+  };
+}
+
+export function listEndpoints(): Endpoint[] {
+  const raw = readJson<AccountRecord[]>(accountsFile(), [], {
+    warn: 'webhook-accounts.json: malformed, ignoring',
+  });
+  if (!Array.isArray(raw)) return [];
+  return raw.map(toEndpoint).filter((e): e is Endpoint => e !== null);
+}
+
+export const findEndpointByWebhookId = (
+  webhookId: string,
+): Endpoint | undefined =>
+  listEndpoints().find((e) => e.webhookId === webhookId);
+
+export function tokenMatches(secret: string, given: string): boolean {
+  const want = Buffer.from(secret);
+  const got = Buffer.from(given);
+  return want.length === got.length && timingSafeEqual(want, got);
+}