@stage-labs/metro 0.1.0-beta.2 → 0.1.0-beta.21

package/runtime/node_modules/@metro-labs/mcp/src/stations/attach-resolve.ts

@@ -0,0 +1,200 @@
+import {
+  assertAttachmentSize,
+  assertContentLength,
+  guessMime,
+  saveBufferToCache,
+} from './attachments.js';
+import {
+  assertInlineTotal,
+  decodeInline,
+  removeInlineTemp,
+  splitInlineData,
+  writeInlineTemp,
+} from './attach-inline.js';
+import { realpathSync } from 'node:fs';
+import { readUpload, UPLOAD_TTL_MS } from '../daemon/upload-store.js';
+import type { CanonicalAttachment } from './types.js';
+
+export interface ResolvedAttachment {
+  path: string;
+  mime: string;
+  name: string;
+  bytes: number;
+  temp?: string;
+}
+
+export interface ResolveOptions {
+  allowed?: Set<string>;
+}
+
+interface InlineBudget {
+  used: number;
+}
+
+const FETCH_TIMEOUT_MS = 60_000;
+
+const basenameOf = (src: string): string =>
+  src.split('?')[0]?.split('#')[0]?.split('/').filter(Boolean).pop() ?? '';
+
+const isHttpUrl = (src: string): boolean => /^https?:\/\//i.test(src);
+
+const mintId = (): string => Math.random().toString(36).slice(2, 12);
+
+const HIDDEN_SEGMENT = /(^|\/)\.[^/]/;
+
+function assertReadablePath(path: string): void {
+  let real: string;
+  try {
+    real = realpathSync(path);
+  } catch {
+    return;
+  }
+  if (HIDDEN_SEGMENT.test(real))
+    throw new Error(
+      `attachment path '${path}' resolves into a hidden directory ` +
+        `('${real}') and will not be read. \`path\` is read on the daemon ` +
+        'machine; put the file somewhere non-hidden or pass `upload` instead.',
+    );
+}
+
+async function fromPath(
+  a: CanonicalAttachment,
+  path: string,
+): Promise<ResolvedAttachment> {
+  assertReadablePath(path);
+  const file = Bun.file(path);
+  if (!(await file.exists()))
+    throw new Error(
+      `attachment path '${path}' does not exist on the metro host. ` +
+        '`path` is read on the daemon machine, not on the caller\'s machine; ' +
+        'pass `url` instead if the file is not already on the daemon.',
+    );
+  assertAttachmentSize(file.size);
+  return {
+    path,
+    mime: a.mime ?? guessMime(path),
+    name: a.name ?? basenameOf(path) ?? 'attachment',
+    bytes: file.size,
+  };
+}
+
+async function fromUrl(
+  a: CanonicalAttachment,
+  url: string,
+): Promise<ResolvedAttachment> {
+  if (!isHttpUrl(url))
+    throw new Error(
+      `attachment url '${url}' is not an http(s) url; use \`path\` for a local file`,
+    );
+  const res = await fetch(url, {
+    signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+  });
+  if (!res.ok)
+    throw new Error(`attachment fetch failed ${res.status} for '${url}'`);
+  assertContentLength(res.headers.get('content-length'));
+  const data = new Uint8Array(await res.arrayBuffer());
+  const mime =
+    a.mime ?? res.headers.get('content-type')?.split(';')[0] ?? guessMime(url);
+  const name = a.name ?? basenameOf(url) ?? 'attachment';
+  const saved = await saveBufferToCache(data, `out${mintId()}`, 0, {
+    mime,
+    name,
+  });
+  return { path: saved.path, mime, name, bytes: data.length };
+}
+
+async function fromData(
+  a: CanonicalAttachment,
+  data: string,
+  budget: InlineBudget,
+): Promise<ResolvedAttachment> {
+  const name = a.name ?? 'attachment';
+  const source = splitInlineData(data);
+  const bytes = decodeInline(source.data, `'${name}'`);
+  budget.used += bytes.length;
+  assertInlineTotal(budget.used);
+  const { dir, path } = await writeInlineTemp(bytes, a.name);
+  return {
+    path,
+    mime: a.mime ?? source.mime ?? guessMime(name),
+    name,
+    bytes: bytes.length,
+    temp: dir,
+  };
+}
+
+const uploadMissing = (id: string): Error =>
+  new Error(
+    `upload '${id}' is not a live upload of yours. Either it was never created, or its ` +
+      'bytes were never pushed to `POST /api/uploads`, or it belongs to another agent, or ' +
+      `it expired (an upload lives ${Math.round(UPLOAD_TTL_MS / 60_000)} minutes). ` +
+      'Create a new one with `create_upload` and push the file again.',
+  );
+
+function fromUpload(
+  a: CanonicalAttachment,
+  id: string,
+  allowed: Set<string> | undefined,
+): ResolvedAttachment {
+  const rec = allowed === undefined ? undefined : readUpload(id);
+  if (rec === undefined || !(allowed?.has(rec.agentId) ?? false))
+    throw uploadMissing(id);
+  const name = a.name ?? rec.name;
+  return { path: rec.path, mime: a.mime ?? rec.mime, name, bytes: rec.bytes };
+}
+
+export const SOURCE_KEYS = ['path', 'url', 'data', 'upload'] as const;
+
+const sourcesOf = (a: CanonicalAttachment): string[] =>
+  SOURCE_KEYS.filter((k) => {
+    const v = a[k];
+    return typeof v === 'string' && v.length > 0;
+  });
+
+export async function resolveAttachment(
+  a: CanonicalAttachment,
+  budget: InlineBudget = { used: 0 },
+  opts: ResolveOptions = {},
+): Promise<ResolvedAttachment> {
+  const sources = sourcesOf(a);
+  if (sources.length === 0)
+    throw new Error(
+      'attachment requires exactly one of `upload`, `data`, `url` or `path`',
+    );
+  if (sources.length > 1)
+    throw new Error(
+      `attachment names ${sources.length} sources (${sources
+        .map((s) => `\`${s}\``)
+        .join(', ')}); pass exactly one of \`upload\`, \`data\`, \`url\` or \`path\``,
+    );
+  if (a.upload) return fromUpload(a, a.upload, opts.allowed);
+  if (a.data) return fromData(a, a.data, budget);
+  if (a.path) return fromPath(a, a.path);
+  return fromUrl(a, a.url ?? '');
+}
+
+export async function cleanupAttachments(
+  atts: ResolvedAttachment[],
+): Promise<void> {
+  for (const a of atts) if (a.temp) await removeInlineTemp(a.temp);
+}
+
+export async function resolveAttachments(
+  atts: CanonicalAttachment[],
+  opts: ResolveOptions = {},
+): Promise<ResolvedAttachment[]> {
+  const out: ResolvedAttachment[] = [];
+  const budget: InlineBudget = { used: 0 };
+  for (let i = 0; i < atts.length; i++) {
+    const a = atts[i];
+    if (!a) continue;
+    try {
+      out.push(await resolveAttachment(a, budget, opts));
+    } catch (e) {
+      await cleanupAttachments(out);
+      const why = e instanceof Error ? e.message : String(e);
+      throw new Error(`attachment ${i + 1} of ${atts.length}: ${why}`);
+    }
+  }
+  return out;
+}

package/runtime/node_modules/@metro-labs/mcp/src/stations/attach-xmtp.ts

@@ -0,0 +1,111 @@
+import { randomBytes } from 'node:crypto';
+import { rmSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { errMsg, log } from '../daemon/log.js';
+
+export class XmtpAttachError extends Error {}
+
+export interface XmtpIdentity {
+  inboxId: string;
+  address: string;
+  installationId: string;
+  dbPath: string;
+}
+
+export type VerifyXmtpKey = (
+  privateKey: string,
+  dbPath: string,
+) => Promise<XmtpIdentity>;
+
+const VERIFY_TIMEOUT_MS = 90_000;
+const UNREACHABLE =
+  'Metro could not open an XMTP inbox with the key it generated, so nothing was attached';
+
+export function newXmtpDbPath(): string {
+  return `~/.metro/xmtp-production-${randomBytes(8).toString('hex')}.db3`;
+}
+
+export function discardXmtpDb(dbPath: string): void {
+  for (const suffix of ['', '-wal', '-shm'])
+    rmSync(`${dbPath}${suffix}`, { force: true });
+}
+
+export function withoutKey(message: string, privateKey: string): string {
+  return message.split(privateKey).join('[redacted]');
+}
+
+interface VerifyReply {
+  ok?: unknown;
+  inboxId?: unknown;
+  address?: unknown;
+  installationId?: unknown;
+  dbPath?: unknown;
+  error?: unknown;
+}
+
+const str = (value: unknown): string => (typeof value === 'string' ? value : '');
+
+function readReply(raw: string): VerifyReply {
+  const line = raw.trim().split('\n').at(-1) ?? '';
+  try {
+    const parsed: unknown = JSON.parse(line);
+    return typeof parsed === 'object' && parsed !== null ? parsed : {};
+  } catch {
+    return {};
+  }
+}
+
+async function runVerifier(payload: string): Promise<string> {
+  const script = fileURLToPath(import.meta.resolve('@metro-labs/xmtp/verify'));
+  const proc = Bun.spawn(['bun', 'run', script], {
+    stdin: 'pipe',
+    stdout: 'pipe',
+    stderr: 'pipe',
+  });
+  await proc.stdin.write(payload);
+  await proc.stdin.end();
+  const timer = setTimeout(() => {
+    proc.kill();
+  }, VERIFY_TIMEOUT_MS);
+  try {
+    const [out, err, code] = await Promise.all([
+      new Response(proc.stdout).text(),
+      new Response(proc.stderr).text(),
+      proc.exited,
+    ]);
+    if (code !== 0)
+      log.warn(
+        { code, stderr: err.trim().slice(-500) },
+        'attach-xmtp: the inbox check exited badly',
+      );
+    return out;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+export async function verifyXmtpKeyOutOfProcess(
+  privateKey: string,
+  dbPath: string,
+): Promise<XmtpIdentity> {
+  const raw = await runVerifier(
+    JSON.stringify({ privateKey, dbPath }),
+  ).catch((err: unknown) => {
+    log.warn({ err: errMsg(err) }, 'attach-xmtp: could not run the inbox check');
+    throw new XmtpAttachError(UNREACHABLE);
+  });
+  const reply = readReply(raw);
+  if (reply.ok !== true)
+    throw new XmtpAttachError(
+      withoutKey(str(reply.error) || UNREACHABLE, privateKey),
+    );
+  const identity = {
+    inboxId: str(reply.inboxId),
+    address: str(reply.address),
+    installationId: str(reply.installationId),
+    dbPath: str(reply.dbPath),
+  };
+  if (identity.inboxId === '' || identity.address === '')
+    throw new XmtpAttachError(UNREACHABLE);
+  return identity;
+}

package/runtime/node_modules/@metro-labs/mcp/src/stations/attach.ts

@@ -0,0 +1,182 @@
+import { randomBytes } from 'node:crypto';
+import {
+  DiscordTokenError,
+  verifyDiscordBotToken,
+} from '@metro-labs/discord/verify';
+import {
+  TelegramTokenError,
+  verifyTelegramBotToken,
+} from '@metro-labs/telegram/verify';
+import { ApiError } from '../daemon/api-error.js';
+import { publicBaseOrDefault } from '../daemon/attach-serve.js';
+import {
+  discardXmtpDb,
+  newXmtpDbPath,
+  verifyXmtpKeyOutOfProcess,
+  XmtpAttachError,
+  type VerifyXmtpKey,
+} from './attach-xmtp.js';
+
+export const ATTACHABLE_STATIONS = [
+  'discord',
+  'telegram',
+  'xmtp',
+  'webhook',
+] as const;
+
+export type AttachStation = (typeof ATTACHABLE_STATIONS)[number];
+
+export class StationAttachError extends ApiError {}
+
+export interface AttachInput {
+  station: AttachStation;
+  token?: unknown;
+}
+
+export interface OneTimeSecret {
+  label: string;
+  value: string;
+  note: string;
+}
+
+export interface PreparedAccount {
+  config: Record<string, unknown>;
+  identity: Record<string, string>;
+  secret?: OneTimeSecret;
+  discard?: () => void;
+}
+
+const TOKEN_RE = /^[A-Za-z0-9._:-]{8,256}$/;
+
+const SECP256K1_ORDER = BigInt(
+  '0xfffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141',
+);
+
+export function isAttachStation(raw: unknown): raw is AttachStation {
+  return (
+    typeof raw === 'string' &&
+    (ATTACHABLE_STATIONS as readonly string[]).includes(raw)
+  );
+}
+
+function requireToken(raw: unknown, label: string): string {
+  const token = typeof raw === 'string' ? raw.trim() : '';
+  if (token === '')
+    throw new StationAttachError(`a ${label} bot token is required`, 400);
+  if (!TOKEN_RE.test(token))
+    throw new StationAttachError(
+      `that does not look like a ${label} bot token`,
+      400,
+    );
+  return token;
+}
+
+function rejected(err: unknown, own: boolean, fallback: string): never {
+  throw new StationAttachError(
+    own && err instanceof Error ? err.message : fallback,
+    400,
+  );
+}
+
+async function prepareDiscord(raw: unknown): Promise<PreparedAccount> {
+  const token = requireToken(raw, 'Discord');
+  try {
+    const bot = await verifyDiscordBotToken(token);
+    if (!bot.messageContent)
+      throw new StationAttachError(
+        'this Discord application does not have the Message Content intent enabled. Turn it on under Bot / Privileged Gateway Intents in the developer portal, then attach it again',
+        400,
+      );
+    return {
+      config: { token },
+      identity: { userId: bot.userId, username: bot.username },
+    };
+  } catch (err) {
+    if (err instanceof StationAttachError) throw err;
+    return rejected(
+      err,
+      err instanceof DiscordTokenError,
+      'Discord rejected that bot token',
+    );
+  }
+}
+
+async function prepareTelegram(raw: unknown): Promise<PreparedAccount> {
+  const token = requireToken(raw, 'Telegram');
+  try {
+    const bot = await verifyTelegramBotToken(token);
+    return {
+      config: { token },
+      identity: { botId: String(bot.botId), username: bot.username },
+    };
+  } catch (err) {
+    return rejected(
+      err,
+      err instanceof TelegramTokenError,
+      'Telegram rejected that bot token',
+    );
+  }
+}
+
+export function newXmtpPrivateKey(): string {
+  for (;;) {
+    const hex = randomBytes(32).toString('hex');
+    const value = BigInt(`0x${hex}`);
+    if (value > 0n && value < SECP256K1_ORDER) return `0x${hex}`;
+  }
+}
+
+async function prepareXmtp(verify: VerifyXmtpKey): Promise<PreparedAccount> {
+  const privateKey = newXmtpPrivateKey();
+  const dbPath = newXmtpDbPath();
+  const identity = await verify(privateKey, dbPath).catch((err: unknown) =>
+    rejected(
+      err,
+      err instanceof XmtpAttachError,
+      'Metro could not open an XMTP inbox with the key it generated, so nothing was attached',
+    ),
+  );
+  return {
+    config: { privateKey, dbPath },
+    identity: { inboxId: identity.inboxId, address: identity.address },
+    secret: {
+      label: 'xmtp private key',
+      value: privateKey,
+      note: 'Metro generated this key, opened inbox ' +
+        `${identity.inboxId} with it, and stores it with the account. ` +
+        'It is shown once, here, and is never returned by any API again.',
+    },
+    discard: () => {
+      discardXmtpDb(identity.dbPath);
+    },
+  };
+}
+
+export const hookUrl = (webhookId: string, secret: string): string =>
+  `${publicBaseOrDefault().replace(/\/+$/, '')}/api/webhooks/${webhookId}/${secret}`;
+
+const WEBHOOK_ID_FLOOR = 10n ** 18n;
+
+export function newWebhookId(): string {
+  const raw = BigInt(`0x${randomBytes(8).toString('hex')}`);
+  return String(WEBHOOK_ID_FLOOR + (raw % (9n * WEBHOOK_ID_FLOOR)));
+}
+
+function prepareWebhook(): PreparedAccount {
+  const secret = randomBytes(48).toString('base64url');
+  const webhookId = newWebhookId();
+  return {
+    config: { secret, webhookId, createdAt: new Date().toISOString() },
+    identity: { endpoint: hookUrl(webhookId, secret) },
+  };
+}
+
+export async function prepareAccount(
+  input: AttachInput,
+  verify: VerifyXmtpKey = verifyXmtpKeyOutOfProcess,
+): Promise<PreparedAccount> {
+  if (input.station === 'discord') return prepareDiscord(input.token);
+  if (input.station === 'telegram') return prepareTelegram(input.token);
+  if (input.station === 'webhook') return prepareWebhook();
+  return prepareXmtp(verify);
+}

package/runtime/node_modules/@metro-labs/mcp/src/stations/attachments.ts

@@ -0,0 +1,196 @@
+export const attachDir = (): string =>
+  process.env.METRO_XMTP_ATTACH_DIR ??
+  `${process.env.HOME}/.cache/metro/messenger-uploads`;
+
+const CACHE_NAME_RE = /^msg_[A-Za-z0-9]{1,16}_\d+\.[A-Za-z0-9]{1,5}$/;
+
+export const resolveCachedAttachment = (name: string): string | null =>
+  CACHE_NAME_RE.test(name) ? `${attachDir()}/${name}` : null;
+
+const MIME_EXT: Record<string, string> = {
+  'image/jpeg': 'jpg',
+  'image/jpg': 'jpg',
+  'image/png': 'png',
+  'image/gif': 'gif',
+  'image/webp': 'webp',
+  'image/heic': 'heic',
+  'audio/mp4': 'm4a',
+  'audio/mpeg': 'mp3',
+  'audio/ogg': 'ogg',
+  'audio/wav': 'wav',
+  'video/mp4': 'mp4',
+  'video/quicktime': 'mov',
+  'video/webm': 'webm',
+  'application/pdf': 'pdf',
+};
+
+const extFromName = (filename: string | undefined): string | undefined => {
+  const ext = filename
+    ?.split('?')[0]
+    ?.split('#')[0]
+    ?.split('.')
+    .pop()
+    ?.toLowerCase();
+  return ext !== undefined && ext.length >= 1 && ext.length <= 5
+    ? ext
+    : undefined;
+};
+
+export const extFromMime = (mime: string | undefined): string | undefined => {
+  if (mime === undefined) return undefined;
+  const mapped = MIME_EXT[mime];
+  if (mapped) return mapped;
+  if (mime.startsWith('image/')) return mime.slice(6).replace('jpeg', 'jpg');
+  return undefined;
+};
+
+const extFor = (
+  filename: string | undefined,
+  mime: string | undefined,
+): string => extFromName(filename) ?? extFromMime(mime) ?? 'bin';
+
+const cacheFileName = (
+  messageId: string,
+  index: number,
+  filename: string | undefined,
+  mime: string | undefined,
+): string => {
+  const safeId =
+    messageId.replace(/[^a-zA-Z0-9]/g, '').slice(0, 16) || 'unknown';
+  return `msg_${safeId}_${index}.${extFor(filename, mime)}`;
+};
+
+export interface SavedAttachment {
+  path: string;
+  mime?: string;
+  name?: string;
+  bytes: number;
+}
+
+export const MAX_ATTACHMENT_BYTES = Number(
+  process.env.METRO_ATTACH_MAX_BYTES ?? 100 * 1024 * 1024,
+);
+
+export function assertAttachmentSize(bytes: number): void {
+  if (bytes > MAX_ATTACHMENT_BYTES)
+    throw new Error(
+      `attachment size ${bytes} exceeds limit of ${MAX_ATTACHMENT_BYTES} bytes`,
+    );
+}
+
+export function assertContentLength(header: string | null | undefined): void {
+  if (header == null) return;
+  const n = Number(header);
+  if (Number.isFinite(n)) assertAttachmentSize(n);
+}
+
+const cachePathFor = async (
+  messageId: string,
+  index: number,
+  meta: { mime?: string; name?: string },
+): Promise<string> => {
+  const { mkdir } = await import('node:fs/promises');
+  const nodePath = await import('node:path');
+  const dir = attachDir();
+  await mkdir(dir, { recursive: true });
+  return nodePath.join(
+    dir,
+    cacheFileName(messageId, index, meta.name, meta.mime),
+  );
+};
+
+export const saveBufferToCache = async (
+  data: Uint8Array,
+  messageId: string,
+  index: number,
+  meta: { mime?: string; name?: string },
+): Promise<SavedAttachment> => {
+  assertAttachmentSize(data.length);
+  const { writeFile } = await import('node:fs/promises');
+  const path = await cachePathFor(messageId, index, meta);
+  await writeFile(path, data);
+  return { path, mime: meta.mime, name: meta.name, bytes: data.length };
+};
+
+export const saveStreamToCache = async (
+  chunks: AsyncIterable<Uint8Array>,
+  messageId: string,
+  index: number,
+  meta: { mime?: string; name?: string },
+): Promise<SavedAttachment> => {
+  const { open, rename, rm } = await import('node:fs/promises');
+  const path = await cachePathFor(messageId, index, meta);
+  const part = `${path}.part`;
+  const handle = await open(part, 'w');
+  let bytes = 0;
+  try {
+    for await (const chunk of chunks) {
+      bytes += chunk.length;
+      assertAttachmentSize(bytes);
+      await handle.write(chunk);
+    }
+  } catch (err) {
+    await handle.close();
+    await rm(part, { force: true });
+    throw err;
+  }
+  await handle.close();
+  await rename(part, path);
+  return { path, mime: meta.mime, name: meta.name, bytes };
+};
+
+export const guessMime = (path: string): string => {
+  const ext = (path.split('.').pop() ?? '').toLowerCase();
+  const map: Record<string, string> = {
+    png: 'image/png',
+    jpg: 'image/jpeg',
+    jpeg: 'image/jpeg',
+    gif: 'image/gif',
+    webp: 'image/webp',
+    heic: 'image/heic',
+    mp4: 'video/mp4',
+    mov: 'video/quicktime',
+    webm: 'video/webm',
+    m4v: 'video/x-m4v',
+    m4a: 'audio/mp4',
+    mp3: 'audio/mpeg',
+    ogg: 'audio/ogg',
+    wav: 'audio/wav',
+    pdf: 'application/pdf',
+  };
+  return map[ext] ?? 'application/octet-stream';
+};
+
+export const isImageMime = (mime: string): boolean =>
+  mime.toLowerCase().startsWith('image/');
+export const isImageExt = (path: string): boolean =>
+  /\.(png|jpe?g|gif|webp|heic|bmp|svg)$/i.test(path);
+
+export const appendFile = async (
+  form: FormData,
+  field: string,
+  path: string,
+  name: string,
+): Promise<void> => {
+  const data = await Bun.file(path).arrayBuffer();
+  form.append(field, new Blob([data]), name);
+};
+
+export const kindOf = (mime: string, path: string): string => {
+  if (isImageMime(mime) || isImageExt(path)) return 'image';
+  if (mime.startsWith('video/')) return 'video';
+  if (mime.startsWith('audio/')) return 'audio';
+  return 'file';
+};
+
+export const toCanonical = (a: {
+  path: string;
+  mime: string;
+  name: string;
+}): Record<string, unknown> => ({
+  kind: kindOf(a.mime, a.path),
+  path: a.path,
+  url: a.path,
+  name: a.name,
+  mime: a.mime,
+});