@stage-labs/metro 0.1.0-beta.31 → 0.1.0-beta.310

package/runtime/node_modules/@metro-labs/daemon/src/vault/proxy.ts

@@ -0,0 +1,147 @@
+import { spawn, type ChildProcess } from 'node:child_process';
+import { stringOf } from '@metro-labs/http/api-http';
+import { randomBytes } from 'node:crypto';
+import { basename } from 'node:path';
+import { createInterface } from 'node:readline';
+import { isRecord } from '@metro-labs/core/is-record';
+import { errMsg, log } from '@metro-labs/core/log';
+import { MANAGEMENT_KEY_ENV } from './config.js';
+import { ports } from './paths.js';
+
+export interface ProxyRequest {
+  at: string;
+  method: string;
+  host: string;
+  path: string;
+  status: number | null;
+  action: string;
+  swapped: string[];
+}
+
+const RECENT_MAX = 200;
+const BACKOFF_MAX_MS = 30_000;
+
+const recent: ProxyRequest[] = [];
+const state: { child: ChildProcess | null; wanted: { bin: string; config: string } | null; key: string; backoff: number; lastError: string | null; startedAt: number } = {
+  child: null,
+  wanted: null,
+  key: randomBytes(24).toString('hex'),
+  backoff: 1000,
+  lastError: null,
+  startedAt: 0,
+};
+
+function swappedIds(transforms: unknown): string[] {
+  if (!Array.isArray(transforms)) return [];
+  return transforms.flatMap((t) => {
+    const swapped = isRecord(t) && isRecord(t.annotations) && Array.isArray(t.annotations.swapped) ? t.annotations.swapped : [];
+    return swapped.flatMap((s) => (isRecord(s) && typeof s.secret === 'string' ? [basename(s.secret)] : []));
+  });
+}
+
+export function requestOf(line: string): ProxyRequest | null {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(line);
+  } catch {
+    return null;
+  }
+  if (!isRecord(parsed) || parsed.msg !== 'request' || !isRecord(parsed.audit)) return null;
+  const audit = parsed.audit;
+  if (audit.method === 'CONNECT') return null;
+  return {
+    at: stringOf(parsed.time),
+    method: stringOf(audit.method),
+    host: stringOf(audit.host),
+    path: stringOf(audit.path),
+    status: typeof audit.status_code === 'number' ? audit.status_code : null,
+    action: stringOf(audit.action),
+    swapped: swappedIds(parsed.request_transforms),
+  };
+}
+
+function noteLine(line: string): void {
+  const request = requestOf(line);
+  if (request !== null) {
+    recent.push(request);
+    if (recent.length > RECENT_MAX) recent.shift();
+    return;
+  }
+  if (/"level":"(WARN|ERROR)"/.test(line)) log.warn({ line: line.slice(0, 500) }, 'vault: iron-proxy');
+}
+
+function launch(): void {
+  const wanted = state.wanted;
+  if (wanted === null || state.child !== null) return;
+  const child = spawn(wanted.bin, ['-config', wanted.config], { stdio: ['ignore', 'pipe', 'pipe'], env: { PATH: process.env.PATH ?? '/usr/bin:/bin', [MANAGEMENT_KEY_ENV]: state.key } });
+  state.child = child;
+  state.startedAt = Date.now();
+  for (const stream of [child.stdout, child.stderr]) if (stream !== null) createInterface({ input: stream }).on('line', noteLine);
+  child.on('error', (err) => {
+    state.lastError = errMsg(err);
+  });
+  child.on('exit', (code) => {
+    state.child = null;
+    if (state.wanted === null) return;
+    state.lastError = `iron-proxy stopped (exit ${String(code)})`;
+    log.warn({ code, retryInMs: state.backoff }, 'vault: iron-proxy stopped, starting it again');
+    const wait = state.backoff;
+    state.backoff = Date.now() - state.startedAt > 60_000 ? 1000 : Math.min(state.backoff * 2, BACKOFF_MAX_MS);
+    setTimeout(launch, wait).unref();
+  });
+}
+
+async function reload(): Promise<boolean> {
+  try {
+    const res = await fetch(`http://127.0.0.1:${String(ports().management)}/v1/reload`, {
+      method: 'POST',
+      headers: { authorization: `Bearer ${state.key}` },
+      signal: AbortSignal.timeout(10_000),
+    });
+    return res.ok;
+  } catch {
+    return false;
+  }
+}
+
+async function listening(): Promise<boolean> {
+  try {
+    await fetch(`http://127.0.0.1:${String(ports().management)}/v1/reload`, { method: 'GET', signal: AbortSignal.timeout(2_000) });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function waitListening(ms = 10_000): Promise<boolean> {
+  const until = Date.now() + ms;
+  while (Date.now() < until) {
+    if (await listening()) return true;
+    await new Promise((r) => setTimeout(r, 200));
+  }
+  return false;
+}
+
+export async function runProxy(bin: string, config: string): Promise<void> {
+  const fresh = state.child === null;
+  state.wanted = { bin, config };
+  if (fresh) launch();
+  else if (await reload()) {
+    state.lastError = null;
+    return;
+  } else {
+    state.child?.kill('SIGTERM');
+    await new Promise((r) => setTimeout(r, state.backoff + 500));
+  }
+  if (!(await waitListening())) throw new Error(state.lastError ?? 'iron-proxy did not start');
+  state.lastError = null;
+}
+
+export function stopProxy(): void {
+  state.wanted = null;
+  state.child?.kill('SIGTERM');
+}
+
+export const proxyRunning = (): boolean => state.child !== null;
+export const proxyError = (): string | null => state.lastError;
+export const recentRequests = (): ProxyRequest[] => [...recent].reverse();

package/runtime/node_modules/@metro-labs/daemon/src/vault/store.ts

@@ -0,0 +1,129 @@
+import { existsSync, rmSync } from 'node:fs';
+import { stringOf } from '@metro-labs/http/api-http';
+import { ApiError } from '@metro-labs/http/api-error';
+import { isRecord } from '@metro-labs/core/is-record';
+import { newId } from '@metro-labs/core/ids';
+import { ensureSecureDir, readJson, writeSecure } from '@metro-labs/core/secure-fs';
+import { stateFile, valueFile, valuesDir, vaultDir } from './paths.js';
+
+export interface VaultSecret {
+  id: string;
+  name: string;
+  env: string;
+  hosts: string[];
+  createdAt: string;
+  updatedAt: string;
+}
+
+export interface VaultState {
+  version: 1;
+  enabled: boolean;
+  secrets: VaultSecret[];
+}
+
+export interface SecretInput {
+  name?: unknown;
+  env?: unknown;
+  hosts?: unknown;
+  value?: unknown;
+}
+
+const ENV_RE = /^[A-Z][A-Z0-9_]{2,63}$/;
+const HOST_RE = /^(\*\.)?[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+$/;
+const RESERVED_ENV = new Set(['HOME', 'PATH', 'USER', 'SHELL', 'LANG', 'TERM', 'LOGNAME', 'HTTP_PROXY', 'HTTPS_PROXY', 'NO_PROXY', 'ALL_PROXY']);
+const MAX_SECRETS = 100;
+const MAX_HOSTS = 20;
+const MAX_VALUE = 16 * 1024;
+
+function secretOf(raw: unknown): VaultSecret | null {
+  if (!isRecord(raw) || typeof raw.id !== 'string' || !ENV_RE.test(stringOf(raw.env))) return null;
+  const hosts = Array.isArray(raw.hosts) ? raw.hosts.filter((h): h is string => typeof h === 'string' && HOST_RE.test(h)) : [];
+  return { id: raw.id, name: stringOf(raw.name) || stringOf(raw.env), env: stringOf(raw.env), hosts, createdAt: stringOf(raw.createdAt), updatedAt: stringOf(raw.updatedAt) };
+}
+
+export function readVault(dir = vaultDir()): VaultState {
+  const raw = readJson<unknown>(stateFile(dir), null);
+  if (!isRecord(raw)) return { version: 1, enabled: false, secrets: [] };
+  const secrets = Array.isArray(raw.secrets) ? raw.secrets.map(secretOf).filter((s): s is VaultSecret => s !== null) : [];
+  return { version: 1, enabled: raw.enabled === true, secrets };
+}
+
+function writeVault(state: VaultState, dir = vaultDir()): void {
+  ensureSecureDir(dir);
+  writeSecure(stateFile(dir), `${JSON.stringify(state, null, 2)}\n`);
+}
+
+function normalizeName(raw: unknown): string {
+  const name = stringOf(raw).trim();
+  if (name === '' || name.length > 80) throw new ApiError('a name of 1 to 80 characters is required', 400);
+  return name;
+}
+
+function normalizeEnv(raw: unknown): string {
+  const env = stringOf(raw).trim();
+  if (!ENV_RE.test(env) || RESERVED_ENV.has(env))
+    throw new ApiError('the variable name must be capital letters, digits and _, 3 to 64 long, like OPENAI_API_KEY', 400);
+  return env;
+}
+
+function normalizeHosts(raw: unknown): string[] {
+  if (!Array.isArray(raw)) throw new ApiError('hosts must be a list of websites, like api.openai.com', 400);
+  const hosts = [...new Set(raw.map((h) => stringOf(h).trim().toLowerCase().replace(/^https?:\/\//, '').replace(/\/.*$/, '')).filter((h) => h !== ''))];
+  if (hosts.length === 0 || hosts.length > MAX_HOSTS) throw new ApiError(`a secret needs 1 to ${String(MAX_HOSTS)} websites`, 400);
+  const bad = hosts.find((h) => !HOST_RE.test(h));
+  if (bad !== undefined) throw new ApiError(`'${bad}' is not a website name; write it as api.example.com or *.example.com`, 400);
+  return hosts;
+}
+
+function normalizeValue(raw: unknown): string {
+  const value = stringOf(raw).trim();
+  if (value === '' || value.length > MAX_VALUE) throw new ApiError('a value of 1 to 16384 characters is required', 400);
+  return value;
+}
+
+function saveValue(id: string, value: string, dir: string): void {
+  ensureSecureDir(valuesDir(dir));
+  writeSecure(valueFile(id, dir), value);
+}
+
+export function addSecret(input: SecretInput, dir = vaultDir(), now = new Date()): VaultSecret {
+  const state = readVault(dir);
+  if (state.secrets.length >= MAX_SECRETS) throw new ApiError(`at most ${String(MAX_SECRETS)} secrets`, 400);
+  const env = normalizeEnv(input.env);
+  if (state.secrets.some((s) => s.env === env)) throw new ApiError(`${env} is already a secret`, 409);
+  const at = now.toISOString();
+  const secret: VaultSecret = { id: newId(), name: normalizeName(input.name ?? env), env, hosts: normalizeHosts(input.hosts), createdAt: at, updatedAt: at };
+  saveValue(secret.id, normalizeValue(input.value), dir);
+  writeVault({ ...state, secrets: [...state.secrets, secret] }, dir);
+  return secret;
+}
+
+export function updateSecret(id: string, input: SecretInput, dir = vaultDir(), now = new Date()): VaultSecret {
+  const state = readVault(dir);
+  const found = state.secrets.find((s) => s.id === id);
+  if (found === undefined) throw new ApiError('no such secret', 404);
+  const next: VaultSecret = {
+    ...found,
+    ...(input.name === undefined ? {} : { name: normalizeName(input.name) }),
+    ...(input.hosts === undefined ? {} : { hosts: normalizeHosts(input.hosts) }),
+    updatedAt: now.toISOString(),
+  };
+  if (input.value !== undefined) saveValue(id, normalizeValue(input.value), dir);
+  writeVault({ ...state, secrets: state.secrets.map((s) => (s.id === id ? next : s)) }, dir);
+  return next;
+}
+
+export function removeSecret(id: string, dir = vaultDir()): void {
+  const state = readVault(dir);
+  if (!state.secrets.some((s) => s.id === id)) throw new ApiError('no such secret', 404);
+  writeVault({ ...state, secrets: state.secrets.filter((s) => s.id !== id) }, dir);
+  rmSync(valueFile(id, dir), { force: true });
+}
+
+export function setVaultEnabled(enabled: boolean, dir = vaultDir()): VaultState {
+  const next = { ...readVault(dir), enabled };
+  writeVault(next, dir);
+  return next;
+}
+
+export const hasValue = (id: string, dir = vaultDir()): boolean => existsSync(valueFile(id, dir));

package/runtime/node_modules/@metro-labs/daemon/src/vault/trust.ts

@@ -0,0 +1,44 @@
+import { spawnSync } from 'node:child_process';
+import { existsSync, readFileSync } from 'node:fs';
+import { errMsg, log } from '@metro-labs/core/log';
+import { asUser, type AgentUser } from '../agent-user/user.js';
+import { TRUSTED_CA } from './paths.js';
+import { mustHelper, rootHelper } from '../metro-user/privilege.js';
+
+const has = (bin: string): boolean => spawnSync('sh', ['-c', `command -v ${bin}`], { stdio: 'ignore' }).status === 0;
+
+export function trustSystem(caFile: string): void {
+  const same = existsSync(TRUSTED_CA) && readFileSync(TRUSTED_CA, 'utf8') === readFileSync(caFile, 'utf8');
+  if (!same) mustHelper(['trust-ca']);
+}
+
+export function untrustSystem(): void {
+  if (existsSync(TRUSTED_CA)) mustHelper(['untrust-ca']);
+}
+
+const NSS_SCRIPT = [
+  'set -e',
+  'db="$HOME/.pki/nssdb"',
+  'mkdir -p "$db"',
+  '[ -f "$db/cert9.db" ] || certutil -d "sql:$db" -N --empty-password',
+  'certutil -d "sql:$db" -D -n "Metro vault" >/dev/null 2>&1 || true',
+  'certutil -d "sql:$db" -A -t "C,," -n "Metro vault" -i "$1"',
+].join('\n');
+
+function ensureCertutil(): boolean {
+  if (has('certutil')) return true;
+  return rootHelper(['install-nss-tools']).status === 0 && has('certutil');
+}
+
+export function trustBrowsers(user: AgentUser): string | null {
+  try {
+    if (!ensureCertutil()) return 'certutil is missing, so browsers the agent starts will not trust the vault (apt-get install libnss3-tools)';
+    const [file, argv] = asUser(user, 'sh', ['-c', NSS_SCRIPT, 'metro', TRUSTED_CA]);
+    const run = spawnSync(file, argv, { encoding: 'utf8', timeout: 30_000 });
+    if (run.status !== 0) return `browsers will not trust the vault: ${run.stderr.trim()}`;
+    return null;
+  } catch (err) {
+    log.warn({ err: errMsg(err) }, 'vault: could not add the certificate for browsers');
+    return errMsg(err);
+  }
+}

package/runtime/node_modules/@metro-labs/daemon/src/voice/api.ts

@@ -0,0 +1,14 @@
+import type { IncomingMessage, ServerResponse } from 'node:http';
+import { readJsonBody, sessionRoute } from '@metro-labs/http/api-http';
+import { patchVoice, publicVoice, readVoice, writeVoice } from './store.js';
+
+const PATH = '/api/voice';
+
+async function answer(req: IncomingMessage): Promise<unknown> {
+  if (req.method === 'PUT') writeVoice(patchVoice(readVoice(), await readJsonBody(req)));
+  return publicVoice(readVoice());
+}
+
+export function handleVoiceRequest(req: IncomingMessage, res: ServerResponse): boolean {
+  return sessionRoute(req, res, { methods: { [PATH]: ['GET', 'PUT'] }, admin: ['PUT'], label: 'voice-api' }, () => answer(req));
+}

package/runtime/node_modules/@metro-labs/daemon/src/voice/audio.ts

@@ -0,0 +1,193 @@
+import OpusScript from 'opusscript';
+import { errMsg, log } from '@metro-labs/core/log';
+
+const RATE = 48_000;
+const FRAME = 960;
+const FRAME_MS = 20;
+const MAX_LATE_FRAMES = 5;
+const MAX_QUEUED_FRAMES = 3_000;
+
+interface Frame {
+  pcm: Int16Array;
+  chars: number;
+}
+
+export function downsample3(pcm: Int16Array): Int16Array {
+  const out = new Int16Array(Math.floor(pcm.length / 3));
+  for (let i = 0; i < out.length; i += 1)
+    out[i] = Math.round(((pcm[i * 3] ?? 0) + (pcm[i * 3 + 1] ?? 0) + (pcm[i * 3 + 2] ?? 0)) / 3);
+  return out;
+}
+
+export function upsample2(pcm: Int16Array, carry: number): Int16Array {
+  const out = new Int16Array(pcm.length * 2);
+  let prev = carry;
+  for (let i = 0; i < pcm.length; i += 1) {
+    const cur = pcm[i] ?? 0;
+    out[i * 2] = Math.round((prev + cur) / 2);
+    out[i * 2 + 1] = cur;
+    prev = cur;
+  }
+  return out;
+}
+
+export const pcmOf = (bytes: Buffer): Int16Array => {
+  const copy = Buffer.from(bytes);
+  return new Int16Array(copy.buffer, copy.byteOffset, Math.floor(copy.length / 2));
+};
+
+export const bytesOf = (pcm: Int16Array): Buffer => Buffer.from(pcm.buffer, pcm.byteOffset, pcm.byteLength);
+
+export class OpusIn {
+  private readonly decoder = new OpusScript(RATE, 1, OpusScript.Application.VOIP);
+
+  decode(packet: Buffer): Int16Array | null {
+    try {
+      return pcmOf(this.decoder.decode(packet));
+    } catch (err) {
+      log.debug({ err: errMsg(err) }, 'voice: an audio packet could not be decoded');
+      return null;
+    }
+  }
+
+  close(): void {
+    this.decoder.delete();
+  }
+}
+
+export interface Playback {
+  send(opus: Buffer, timestamp: number, marker: boolean): void | Promise<void>;
+  played(chars: number): void;
+}
+
+export class Player {
+  private readonly encoder = new OpusScript(RATE, 1, OpusScript.Application.VOIP);
+  private readonly queue: Frame[] = [];
+  private pending = new Int16Array(0);
+  private pendingChars = 0;
+  private timer: ReturnType<typeof setTimeout> | null = null;
+  private started = 0;
+  private sent = 0;
+  private speaking = false;
+  private onIdle: (() => void) | null = null;
+  private onFirst: (() => void) | null = null;
+  private onFailed: ((reason: string) => void) | null = null;
+  private generation = 0;
+  private inFlight = 0;
+  private closed = false;
+
+  constructor(private readonly out: Playback) {}
+
+  get busy(): boolean {
+    return this.queue.length > 0 || this.pending.length > 0 || this.inFlight > 0;
+  }
+
+  whenFirstPlayed(fn: () => void): void {
+    this.onFirst = fn;
+  }
+
+  whenIdle(fn: () => void): void {
+    this.onIdle = fn;
+  }
+
+  whenFailed(fn: (reason: string) => void): void {
+    this.onFailed = fn;
+  }
+
+  push(pcm: Int16Array, chars: number): void {
+    if (this.closed) return;
+    if (this.queue.length + this.inFlight + Math.ceil((this.pending.length + pcm.length) / FRAME) > MAX_QUEUED_FRAMES) {
+      this.failed('audio queue is full');
+      return;
+    }
+    const joined = new Int16Array(this.pending.length + pcm.length);
+    joined.set(this.pending);
+    joined.set(pcm, this.pending.length);
+    let at = 0;
+    for (; at + FRAME <= joined.length; at += FRAME) this.queue.push({ pcm: joined.slice(at, at + FRAME), chars: 0 });
+    this.pending = joined.slice(at);
+    this.pendingChars += chars;
+    const last = this.queue.at(-1);
+    if (last !== undefined && this.pending.length === 0) {
+      last.chars += this.pendingChars;
+      this.pendingChars = 0;
+    }
+  }
+
+  finish(): void {
+    if (this.closed || this.pending.length === 0) return;
+    const pcm = new Int16Array(FRAME);
+    pcm.set(this.pending);
+    this.queue.push({ pcm, chars: this.pendingChars });
+    this.pending = new Int16Array(0);
+    this.pendingChars = 0;
+  }
+
+  clear(): void {
+    this.generation += 1;
+    this.inFlight = 0;
+    this.speaking = false;
+    this.queue.length = 0;
+    this.pending = new Int16Array(0);
+    this.pendingChars = 0;
+  }
+
+  start(): void {
+    if (this.closed || this.timer !== null) return;
+    this.started = performance.now();
+    this.sent = 0;
+    this.tick();
+  }
+
+  stop(): void {
+    if (this.closed) return;
+    this.closed = true;
+    if (this.timer !== null) clearTimeout(this.timer);
+    this.timer = null;
+    this.clear();
+    this.encoder.delete();
+  }
+
+  private tick(): void {
+    if (this.closed) return;
+    const due = Math.floor((performance.now() - this.started) / FRAME_MS) + 1;
+    if (due - this.sent > MAX_LATE_FRAMES) this.sent = due - 1;
+    while (!this.closed && this.sent < due) this.sendOne();
+    if (this.closed) return;
+    const next = this.started + this.sent * FRAME_MS;
+    this.timer = setTimeout(() => {
+      this.tick();
+    }, Math.max(0, next - performance.now()));
+  }
+
+  private sendOne(): void {
+    const frame = this.queue.shift();
+    const marker = frame !== undefined && !this.speaking;
+    this.speaking = frame !== undefined;
+    const pcm = frame?.pcm ?? new Int16Array(FRAME);
+    const timestamp = (this.sent * FRAME) >>> 0;
+    this.sent += 1;
+    const generation = this.generation;
+    if (frame !== undefined) this.inFlight += 1;
+    try {
+      const sent = this.out.send(this.encoder.encode(bytesOf(pcm), FRAME), timestamp, marker);
+      Promise.resolve(sent).then(() => {
+        if (this.closed || generation !== this.generation || frame === undefined) return;
+        this.inFlight -= 1;
+        if (marker) this.onFirst?.();
+        if (frame.chars > 0) this.out.played(frame.chars);
+        if (!this.busy) this.onIdle?.();
+      }).catch((err: unknown) => {
+        if (!this.closed && generation === this.generation) this.failed(errMsg(err));
+      });
+    } catch (err) {
+      if (!this.closed && generation === this.generation) this.failed(errMsg(err));
+    }
+  }
+
+  private failed(reason: string): void {
+    this.clear();
+    log.warn({ reason }, 'voice: audio playback failed');
+    this.onFailed?.(reason);
+  }
+}