@stage-labs/metro 0.1.0-beta.2 → 0.1.0-beta.200

package/runtime/node_modules/@metro-labs/daemon/src/files/attach-owner.ts

@@ -0,0 +1,32 @@
+import { readFileSync } from 'node:fs';
+import { ID_RE } from '@metro-labs/core/ids';
+import { resolveCachedAttachment } from '@metro-labs/core/stations/attachments';
+import { writeSecure } from '@metro-labs/core/secure-fs';
+
+const OWNER_SUFFIX = '.owner';
+
+export function recordOwner(path: string, agentId: string): void {
+  writeSecure(`${path}${OWNER_SUFFIX}`, `${agentId}\n`);
+}
+
+export function ownerOf(path: string): string | undefined {
+  let raw: string;
+  try {
+    raw = readFileSync(`${path}${OWNER_SUFFIX}`, 'utf8');
+  } catch {
+    return undefined;
+  }
+  const agentId = raw.trim();
+  return ID_RE.test(agentId) ? agentId : undefined;
+}
+
+export function recordAttachmentOwner(name: string, agentId: string): void {
+  const path = resolveCachedAttachment(name);
+  if (path === null) return;
+  recordOwner(path, agentId);
+}
+
+export function attachmentOwner(name: string): string | undefined {
+  const path = resolveCachedAttachment(name);
+  return path === null ? undefined : ownerOf(path);
+}

package/runtime/node_modules/@metro-labs/daemon/src/files/attach-reaper.ts

@@ -0,0 +1,174 @@
+import { readdirSync, rmSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import {
+  attachDir,
+  resolveCachedAttachment,
+} from '@metro-labs/core/stations/attachments';
+import { errMsg, log } from '@metro-labs/core/log';
+
+const DAY_MS = 24 * 60 * 60 * 1000;
+const MB = 1024 * 1024;
+const DEFAULT_TTL_DAYS = 7;
+const DEFAULT_MAX_MB = 2048;
+const PART_TTL_MS = 60 * 60 * 1000;
+const SWEEP_MS = 10 * 60 * 1000;
+const SIDECARS = ['.owner', '.grant'] as const;
+
+export interface AttachSweepOptions {
+  dir?: string;
+  now?: number;
+  ttlMs?: number;
+  maxBytes?: number;
+}
+
+export interface AttachSweepResult {
+  expired: number;
+  capped: number;
+  orphans: number;
+  parts: number;
+  freedBytes: number;
+}
+
+interface Entry {
+  name: string;
+  mtime: number;
+  size: number;
+}
+
+const positive = (raw: string | undefined, fallback: number): number => {
+  const value = Number(raw);
+  return Number.isFinite(value) && value > 0 ? value : fallback;
+};
+
+export const attachTtlMs = (): number =>
+  positive(process.env.METRO_ATTACH_TTL_DAYS, DEFAULT_TTL_DAYS) * DAY_MS;
+
+export const attachMaxBytes = (): number =>
+  positive(process.env.METRO_ATTACH_MAX_MB, DEFAULT_MAX_MB) * MB;
+
+const cacheShaped = (name: string): boolean =>
+  resolveCachedAttachment(name) !== null;
+
+function stripSuffix(name: string, suffixes: readonly string[]): string | undefined {
+  const suffix = suffixes.find((s) => name.endsWith(s));
+  if (suffix === undefined) return undefined;
+  const base = name.slice(0, -suffix.length);
+  return cacheShaped(base) ? base : undefined;
+}
+
+function listEntries(dir: string): Entry[] {
+  let names: string[];
+  try {
+    names = readdirSync(dir);
+  } catch {
+    return [];
+  }
+  const entries: Entry[] = [];
+  for (const name of names) {
+    try {
+      const stat = statSync(join(dir, name));
+      if (stat.isFile()) entries.push({ name, mtime: stat.mtimeMs, size: stat.size });
+    } catch {
+      continue;
+    }
+  }
+  return entries;
+}
+
+function removeEntry(dir: string, name: string): void {
+  rmSync(join(dir, name), { force: true });
+}
+
+function removeWithSidecars(dir: string, name: string): void {
+  removeEntry(dir, name);
+  for (const suffix of SIDECARS) removeEntry(dir, `${name}${suffix}`);
+}
+
+interface Sweep {
+  dir: string;
+  now: number;
+  entries: Entry[];
+  result: AttachSweepResult;
+}
+
+function sweepParts(sweep: Sweep): Set<string> {
+  const fresh = new Set<string>();
+  for (const entry of sweep.entries) {
+    const base = stripSuffix(entry.name, ['.part']);
+    if (base === undefined) continue;
+    if (sweep.now - entry.mtime <= PART_TTL_MS) {
+      fresh.add(base);
+      continue;
+    }
+    removeEntry(sweep.dir, entry.name);
+    sweep.result.parts += 1;
+    sweep.result.freedBytes += entry.size;
+  }
+  return fresh;
+}
+
+function sweepExpired(sweep: Sweep, ttlMs: number): Entry[] {
+  const kept: Entry[] = [];
+  for (const entry of sweep.entries) {
+    if (!cacheShaped(entry.name)) continue;
+    if (sweep.now - entry.mtime <= ttlMs) {
+      kept.push(entry);
+      continue;
+    }
+    removeWithSidecars(sweep.dir, entry.name);
+    sweep.result.expired += 1;
+    sweep.result.freedBytes += entry.size;
+  }
+  return kept;
+}
+
+function sweepOrphans(sweep: Sweep, busy: Set<string>): void {
+  const present = new Set(sweep.entries.map((e) => e.name));
+  for (const entry of sweep.entries) {
+    const base = stripSuffix(entry.name, SIDECARS);
+    if (base === undefined || present.has(base) || busy.has(base)) continue;
+    removeEntry(sweep.dir, entry.name);
+    sweep.result.orphans += 1;
+  }
+}
+
+function capSize(sweep: Sweep, kept: Entry[], maxBytes: number): void {
+  let total = kept.reduce((sum, e) => sum + e.size, 0);
+  for (const entry of [...kept].sort((a, b) => a.mtime - b.mtime)) {
+    if (total <= maxBytes) return;
+    removeWithSidecars(sweep.dir, entry.name);
+    total -= entry.size;
+    sweep.result.capped += 1;
+    sweep.result.freedBytes += entry.size;
+  }
+}
+
+export function sweepAttachments(options: AttachSweepOptions = {}): AttachSweepResult {
+  const dir = options.dir ?? attachDir();
+  const sweep: Sweep = {
+    dir,
+    now: options.now ?? Date.now(),
+    entries: listEntries(dir),
+    result: { expired: 0, capped: 0, orphans: 0, parts: 0, freedBytes: 0 },
+  };
+  const busy = sweepParts(sweep);
+  const kept = sweepExpired(sweep, options.ttlMs ?? attachTtlMs());
+  sweepOrphans(sweep, busy);
+  capSize(sweep, kept, options.maxBytes ?? attachMaxBytes());
+  return sweep.result;
+}
+
+export function startAttachReaper(): void {
+  const run = (): void => {
+    try {
+      const swept = sweepAttachments();
+      if (swept.expired + swept.capped + swept.orphans + swept.parts > 0) {
+        log.info(swept, 'attachments: reaped the cache');
+      }
+    } catch (err) {
+      log.warn({ err: errMsg(err) }, 'attachments: sweep failed');
+    }
+  };
+  run();
+  setInterval(run, SWEEP_MS).unref?.();
+}

package/runtime/node_modules/@metro-labs/daemon/src/files/attach-serve.ts

@@ -0,0 +1,123 @@
+import { createReadStream } from 'node:fs';
+import { stat } from 'node:fs/promises';
+import type { IncomingMessage, ServerResponse } from 'node:http';
+import {
+  guessMime,
+  resolveCachedAttachment,
+} from '@metro-labs/core/stations/attachments';
+import {
+  allowedAgents,
+  authenticate,
+  extractToken,
+} from '../mcp/request-identity.js';
+import { attachmentOwner, recordAttachmentOwner } from './attach-owner.js';
+import { grantAllows, issueAttachmentGrant } from './attach-grant.js';
+import { errMsg, log } from '@metro-labs/core/log';
+import { currentTunnelUrl } from '../net/tunnel.js';
+
+export const webhookPort = (): number => Number(process.env.METRO_WEBHOOK_PORT) || 8420;
+
+function authorized(req: IncomingMessage, name: string): boolean {
+  const owner = attachmentOwner(name);
+  if (owner === undefined) return false;
+  if (grantAllows(name, owner, extractToken(req) ?? '')) return true;
+  return allowedAgents(authenticate(req) ?? undefined).has(
+    owner,
+  );
+}
+
+export function publicBaseUrl(): string | null {
+  const env = process.env.METRO_PUBLIC_URL?.trim();
+  if (env) return env.replace(/\/+$/, '');
+  return currentTunnelUrl();
+}
+
+export const loopbackBase = (): string => `http://127.0.0.1:${String(webhookPort())}`;
+
+export const publicBaseOrDefault = (): string => publicBaseUrl() ?? loopbackBase();
+
+export function attachmentUrl(
+  pathOrName: string,
+  agentId: string,
+): string | null {
+  const base = publicBaseUrl() ?? loopbackBase();
+  if (!base) return null;
+  const name = pathOrName.split('/').pop();
+  if (!name || !resolveCachedAttachment(name)) return null;
+  recordAttachmentOwner(name, agentId);
+  const token = issueAttachmentGrant(name, agentId);
+  if (token === undefined) return null;
+  return `${base}/attach/${encodeURIComponent(name)}?token=${encodeURIComponent(token)}`;
+}
+
+export function attachmentEventUrl(
+  payload: Record<string, unknown>,
+  agentId: string,
+): string | null {
+  if (payload.contentType !== 'attachmentSaved') return null;
+  if (typeof payload.url === 'string' && payload.url.length > 0) return null;
+  const p = payload.attachmentPath ?? payload.localPath;
+  return typeof p === 'string' ? attachmentUrl(p, agentId) : null;
+}
+
+async function serveFile(
+  res: ServerResponse,
+  method: string,
+  name: string,
+): Promise<void> {
+  const path = resolveCachedAttachment(name);
+  if (!path) {
+    res.writeHead(404).end();
+    return;
+  }
+  let size: number;
+  try {
+    size = (await stat(path)).size;
+  } catch {
+    res.writeHead(404).end();
+    return;
+  }
+  res.writeHead(200, {
+    'content-type': guessMime(path),
+    'content-length': String(size),
+    'cache-control': 'private, max-age=86400',
+  });
+  if (method === 'HEAD') {
+    res.end();
+    return;
+  }
+  const stream = createReadStream(path);
+  stream.on('error', (err) => {
+    log.warn({ err: errMsg(err) }, 'attach: read stream error');
+    res.end();
+  });
+  stream.pipe(res);
+}
+
+export function handleAttachRequest(
+  req: IncomingMessage,
+  res: ServerResponse,
+): boolean {
+  const [rawPath] = (req.url ?? '').split('?', 2);
+  const m = /^\/attach\/([^/]+)$/.exec(rawPath ?? '');
+  if (!m) return false;
+  const method = req.method ?? 'GET';
+  if (method !== 'GET' && method !== 'HEAD') {
+    res.writeHead(405).end();
+    return true;
+  }
+  const name = decodeURIComponent(m[1] ?? '');
+  if (resolveCachedAttachment(name) === null) {
+    res.writeHead(404).end();
+    return true;
+  }
+  if (!authorized(req, name)) {
+    res.writeHead(401).end();
+    return true;
+  }
+  serveFile(res, method, name).catch((err: unknown) => {
+    log.warn({ err: errMsg(err) }, 'attach: serve error');
+    if (!res.headersSent) res.writeHead(500).end();
+  });
+  return true;
+}

package/runtime/node_modules/@metro-labs/daemon/src/files/upload-api.ts

@@ -0,0 +1,278 @@
+import { open, rename, rm, type FileHandle } from 'node:fs/promises';
+import type { IncomingMessage, ServerResponse } from 'node:http';
+import { guessMime } from '@metro-labs/core/stations/attachments';
+import { safeFileName } from '../stations/attach-inline.js';
+import {
+  allowedAgents,
+  authenticate,
+  extractToken,
+} from '../mcp/request-identity.js';
+import { ApiError } from '@metro-labs/http/api-error';
+import { apiFailure, cors, sendJson } from '@metro-labs/http/api-http';
+import { log } from '@metro-labs/core/log';
+import { writeSecure } from '@metro-labs/core/secure-fs';
+import {
+  createUploadSlot,
+  liveUploadBytes,
+  MAX_LIVE_UPLOAD_BYTES,
+  MAX_UPLOAD_BYTES,
+  readUpload,
+  removeUpload,
+  sweepUploads,
+  uploadPath,
+  uploadSlot,
+  uploadTicketAllows,
+  UPLOAD_TTL_MS,
+  type UploadRecord,
+} from './upload-store.js';
+
+const PREFIX = '/api/uploads';
+const ITEM_METHODS = ['PUT', 'POST', 'DELETE'];
+const DRAIN_MAX = 2 * MAX_UPLOAD_BYTES;
+const VAGUE_MIMES = new Set([
+  'application/x-www-form-urlencoded',
+  'application/octet-stream',
+  '',
+]);
+
+const mib = (n: number): string => `${Math.round(n / (1024 * 1024))} MiB`;
+
+const uploadLimit = (): string =>
+  `${mib(MAX_UPLOAD_BYTES)} (${MAX_UPLOAD_BYTES} bytes)`;
+
+const tooLarge = (): string =>
+  `upload exceeds the ${uploadLimit()} per-file limit; ` +
+  'send a link in `text` for anything larger';
+
+const noRoom = (): string =>
+  `metro is already holding close to ${mib(MAX_LIVE_UPLOAD_BYTES)} of pending uploads, ` +
+  `which is the daemon-wide cap; each one expires ${Math.round(UPLOAD_TTL_MS / 60_000)} ` +
+  'minutes after it is created, or DELETE one you no longer need';
+
+type Target =
+  | { kind: 'collection' }
+  | { kind: 'item'; id: string }
+  | { kind: 'unknown' }
+  | null;
+
+export function target(path: string): Target {
+  if (path === PREFIX || path === `${PREFIX}/`) return { kind: 'collection' };
+  if (!path.startsWith(`${PREFIX}/`)) return null;
+  const rest = path.slice(PREFIX.length + 1).split('/').filter(Boolean);
+  const id = rest.length === 1 ? rest[0] : undefined;
+  if (id === undefined || uploadPath(id) === null) return { kind: 'unknown' };
+  return { kind: 'item', id };
+}
+
+const query = (req: IncomingMessage, key: string): string | undefined =>
+  new URL(req.url ?? '/', 'http://localhost').searchParams.get(key) ?? undefined;
+
+const identityScope = (req: IncomingMessage): Set<string> =>
+  allowedAgents(authenticate(req) ?? undefined);
+
+function mimeFor(req: IncomingMessage, name: string): string {
+  const explicit = query(req, 'mime');
+  if (explicit !== undefined && explicit !== '') return explicit;
+  const header = (req.headers['content-type'] ?? '').split(';')[0]?.trim() ?? '';
+  return VAGUE_MIMES.has(header.toLowerCase()) ? guessMime(name) : header;
+}
+
+function roomForUpload(): number {
+  sweepUploads();
+  const room = MAX_LIVE_UPLOAD_BYTES - liveUploadBytes();
+  if (room <= 0) throw new ApiError(noRoom(), 507);
+  return room;
+}
+
+const overflowAt = (total: number, room: number): ApiError | undefined => {
+  if (total > MAX_UPLOAD_BYTES) return new ApiError(tooLarge(), 413);
+  return total > room ? new ApiError(noRoom(), 507) : undefined;
+};
+
+function declaredLength(req: IncomingMessage): number | undefined {
+  const n = Number(req.headers['content-length']);
+  return Number.isFinite(n) && n >= 0 ? n : undefined;
+}
+
+function assertRoomForDeclared(req: IncomingMessage, room: number): void {
+  const declared = declaredLength(req);
+  if (declared === undefined) return;
+  if (declared > MAX_UPLOAD_BYTES) throw new ApiError(tooLarge(), 413);
+  if (declared > room) throw new ApiError(noRoom(), 507);
+}
+
+async function drainInto(
+  req: IncomingMessage,
+  handle: FileHandle,
+  room: number,
+): Promise<number> {
+  let over: ApiError | undefined;
+  let total = 0;
+  try {
+    for await (const chunk of req) {
+      const buf = chunk as Buffer;
+      total += buf.length;
+      over ??= overflowAt(total, room);
+      if (over === undefined) await handle.write(buf);
+      else if (total > DRAIN_MAX) break;
+    }
+  } finally {
+    await handle.close();
+  }
+  if (over !== undefined) throw over;
+  if (total === 0) throw new ApiError('upload body is empty', 400);
+  return total;
+}
+
+async function streamToSlot(req: IncomingMessage, id: string): Promise<number> {
+  const part = uploadPath(id, '.part');
+  const final = uploadPath(id);
+  if (part === null || final === null) throw new ApiError('bad upload id', 400);
+  const room = roomForUpload();
+  assertRoomForDeclared(req, room);
+  const handle = await open(part, 'wx', 0o600).catch(() => {
+    throw new ApiError('this upload is already being written', 409);
+  });
+  const total = await drainInto(req, handle, room).catch(
+    async (err: unknown) => {
+      await rm(part, { force: true });
+      throw err;
+    },
+  );
+  await rename(part, final);
+  return total;
+}
+
+const payload = (rec: UploadRecord): Record<string, unknown> => ({
+  id: rec.id,
+  name: rec.name,
+  mime: rec.mime,
+  bytes: rec.bytes,
+  expires_at: new Date(rec.expiresAt).toISOString(),
+  attachment: { upload: rec.id },
+});
+
+async function fill(
+  req: IncomingMessage,
+  res: ServerResponse,
+  id: string,
+  status: number,
+): Promise<void> {
+  const bytes = await streamToSlot(req, id);
+  const rec = readUpload(id);
+  if (rec === undefined) throw new ApiError('upload expired while storing', 410);
+  log.info({ upload: id, agent: rec.agentId, bytes }, 'uploads: stored');
+  sendJson(req, res, status, payload(rec));
+}
+
+async function handleCreate(
+  req: IncomingMessage,
+  res: ServerResponse,
+): Promise<void> {
+  const [owner] = identityScope(req);
+  if (owner === undefined) throw new ApiError('unauthorized', 401);
+  const name = safeFileName(query(req, 'name'));
+  const id = createUploadSlot(owner, {
+    name,
+    mime: mimeFor(req, name),
+  });
+  try {
+    await fill(req, res, id, 201);
+  } catch (err) {
+    removeUpload(id);
+    throw err;
+  }
+}
+
+function authorizeItem(req: IncomingMessage, id: string): string {
+  const slot = uploadSlot(id);
+  if (slot === undefined) throw new ApiError('no such upload', 404);
+  if (uploadTicketAllows(id, slot.agentId, extractToken(req) ?? ''))
+    return slot.agentId;
+  if (identityScope(req).has(slot.agentId)) return slot.agentId;
+  throw new ApiError('no such upload', 404);
+}
+
+function relabelSlot(id: string, name: string, mime: string | undefined): void {
+  const slot = uploadSlot(id);
+  const path = uploadPath(id, '.meta');
+  if (slot === undefined || path === null) return;
+  writeSecure(
+    path,
+    JSON.stringify({
+      name,
+      mime: mime !== undefined && mime !== '' ? mime : guessMime(name),
+      createdAt: slot.createdAt,
+    }),
+  );
+}
+
+async function handleFill(
+  req: IncomingMessage,
+  res: ServerResponse,
+  id: string,
+): Promise<void> {
+  authorizeItem(req, id);
+  if (readUpload(id) !== undefined)
+    throw new ApiError('this upload already holds bytes', 409);
+  const name = query(req, 'name');
+  if (name !== undefined)
+    relabelSlot(id, safeFileName(name), query(req, 'mime'));
+  await fill(req, res, id, 200);
+}
+
+function handleDelete(
+  req: IncomingMessage,
+  res: ServerResponse,
+  id: string,
+): void {
+  authorizeItem(req, id);
+  removeUpload(id);
+  log.info({ upload: id }, 'uploads: deleted');
+  sendJson(req, res, 200, { id, deleted: true });
+}
+
+async function dispatch(
+  req: IncomingMessage,
+  res: ServerResponse,
+  tgt: { kind: 'collection' } | { kind: 'item'; id: string },
+): Promise<void> {
+  if (tgt.kind === 'collection') return handleCreate(req, res);
+  if (req.method === 'DELETE') handleDelete(req, res, tgt.id);
+  else await handleFill(req, res, tgt.id);
+}
+
+function closeIfBodyUnread(req: IncomingMessage, res: ServerResponse): void {
+  if (req.readableEnded) return;
+  const drop = (): void => {
+    req.socket?.destroy();
+  };
+  if (res.writableFinished) drop();
+  else res.once('finish', drop);
+}
+
+export function handleUploadRequest(
+  req: IncomingMessage,
+  res: ServerResponse,
+): boolean {
+  const tgt = target((req.url ?? '').split('?')[0] ?? '');
+  if (tgt === null) return false;
+  if (req.method === 'OPTIONS') {
+    res.writeHead(204, cors(req)).end();
+    return true;
+  }
+  if (tgt.kind === 'unknown') {
+    sendJson(req, res, 404, { error: 'no such upload' });
+    return true;
+  }
+  const methods = tgt.kind === 'collection' ? ['POST'] : ITEM_METHODS;
+  if (!methods.includes(req.method ?? '')) {
+    sendJson(req, res, 405, { error: 'method not allowed' });
+    return true;
+  }
+  dispatch(req, res, tgt).catch((err: unknown) => {
+    apiFailure(req, res, err, 'upload-api');
+    closeIfBodyUnread(req, res);
+  });
+  return true;
+}