@tuongaz/seeflow 0.1.101 → 0.1.103

package/src/share/sse-rate-limit.ts

@@ -0,0 +1,205 @@
+/**
+ * Token-bucket rate limiter + per-(flowId, nodeId) coalescer for SSE bridge
+ * frames. Sits between the SSE tap's event source and the relay broadcast so
+ * a noisy `node:status` storm cannot saturate the relay or peer browsers.
+ *
+ * Semantics:
+ * - Sustained rate `tokensPerSecond` (default 60) with bucket capacity
+ *   `burst` (default 120). Each non-terminal frame consumes one token.
+ * - Non-terminal frames that arrive with no tokens are queued. When a
+ *   `node:status` frame for the same `(flowId, nodeId)` is already pending,
+ *   the latest payload replaces the queued one (last-wins coalescing) and
+ *   `droppedFrames` is incremented for the displaced payload.
+ * - Terminal events (`node:done`, `node:error`) are NEVER coalesced or
+ *   dropped — they bypass the bucket and emit immediately.
+ * - When the queue exceeds `maxQueueDepth` (default = burst), the oldest
+ *   pending non-terminal frame is dropped and `droppedFrames` is incremented.
+ */
+
+import type { SsePayload } from './sse-frame.ts';
+
+const TERMINAL_TYPES: ReadonlySet<SsePayload['t']> = new Set(['node:done', 'node:error']);
+
+export interface RateLimitOptions {
+  /** Tokens added per second. Default 60. */
+  tokensPerSecond?: number;
+  /** Bucket capacity (burst size). Default 120. */
+  burst?: number;
+  /**
+   * Max pending non-terminal frames in the queue. Default = `burst`.
+   * Overflow drops the oldest pending frame.
+   */
+  maxQueueDepth?: number;
+  /** Outbound emit callback (post-rate-limit). */
+  onEmit: (frame: SsePayload) => void;
+  /** Monotonic time source in milliseconds. Defaults to `performance.now()`. */
+  now?: () => number;
+  /**
+   * Schedule `fn` to run after `ms` milliseconds, returning a cancel function.
+   * Defaults to `setTimeout`. Injected in tests for deterministic draining.
+   */
+  schedule?: (ms: number, fn: () => void) => () => void;
+}
+
+export interface RateLimitMetrics {
+  droppedFrames: number;
+  queueDepth: number;
+}
+
+export interface RateLimiter {
+  submit(frame: SsePayload): void;
+  metrics(): RateLimitMetrics;
+  /** Drain all queued frames immediately, ignoring tokens. */
+  flush(): void;
+  dispose(): void;
+}
+
+function defaultNow(): number {
+  return typeof performance !== 'undefined' ? performance.now() : Date.now();
+}
+
+function defaultSchedule(ms: number, fn: () => void): () => void {
+  const handle = setTimeout(fn, ms);
+  return () => clearTimeout(handle);
+}
+
+function coalesceKey(frame: SsePayload): string | null {
+  if (frame.t !== 'node:status') return null;
+  const data = frame.data;
+  const nodeId =
+    data && typeof data === 'object' ? (data as { nodeId?: unknown }).nodeId : undefined;
+  if (typeof nodeId !== 'string' || nodeId.length === 0) return null;
+  return `${frame.flowId}\x00${nodeId}`;
+}
+
+export function createRateLimiter(opts: RateLimitOptions): RateLimiter {
+  const tokensPerSecond = Math.max(0.0001, opts.tokensPerSecond ?? 60);
+  const burst = Math.max(1, opts.burst ?? 120);
+  const maxQueueDepth = Math.max(1, opts.maxQueueDepth ?? burst);
+  const now = opts.now ?? defaultNow;
+  const schedule = opts.schedule ?? defaultSchedule;
+
+  let tokens = burst;
+  let lastRefill = now();
+  let droppedFrames = 0;
+  let drainCancel: (() => void) | null = null;
+  let disposed = false;
+
+  const queue: SsePayload[] = [];
+  /** coalesceKey -> queue index */
+  const queueIndex = new Map<string, number>();
+
+  function safeEmit(frame: SsePayload): void {
+    try {
+      opts.onEmit(frame);
+    } catch (err) {
+      console.error('[sse-rate-limit] onEmit listener threw:', err);
+    }
+  }
+
+  function refill(): void {
+    const t = now();
+    const dt = (t - lastRefill) / 1000;
+    if (dt <= 0) return;
+    tokens = Math.min(burst, tokens + dt * tokensPerSecond);
+    lastRefill = t;
+  }
+
+  function rebuildIndex(): void {
+    queueIndex.clear();
+    for (let i = 0; i < queue.length; i++) {
+      const frame = queue[i];
+      if (!frame) continue;
+      const k = coalesceKey(frame);
+      if (k !== null) queueIndex.set(k, i);
+    }
+  }
+
+  function clearDrainTimer(): void {
+    if (drainCancel) {
+      drainCancel();
+      drainCancel = null;
+    }
+  }
+
+  function scheduleDrain(): void {
+    if (disposed || queue.length === 0 || drainCancel) return;
+    refill();
+    const needed = Math.max(0, 1 - tokens);
+    const waitMs = Math.ceil((needed / tokensPerSecond) * 1000);
+    drainCancel = schedule(Math.max(1, waitMs), () => {
+      drainCancel = null;
+      drainQueue();
+    });
+  }
+
+  function drainQueue(): void {
+    refill();
+    while (queue.length > 0 && tokens >= 1) {
+      const frame = queue.shift();
+      if (!frame) break;
+      tokens -= 1;
+      safeEmit(frame);
+    }
+    rebuildIndex();
+    scheduleDrain();
+  }
+
+  function enqueueNonTerminal(frame: SsePayload): void {
+    const key = coalesceKey(frame);
+    if (key !== null) {
+      const idx = queueIndex.get(key);
+      if (idx !== undefined && idx < queue.length && queue[idx] !== undefined) {
+        // Last-wins: replace the queued payload; older payload is "dropped".
+        queue[idx] = frame;
+        droppedFrames += 1;
+        return;
+      }
+    }
+    if (queue.length >= maxQueueDepth) {
+      queue.shift();
+      droppedFrames += 1;
+      rebuildIndex();
+    }
+    const newIdx = queue.length;
+    queue.push(frame);
+    if (key !== null) queueIndex.set(key, newIdx);
+  }
+
+  return {
+    submit(frame) {
+      if (disposed) return;
+
+      if (TERMINAL_TYPES.has(frame.t)) {
+        safeEmit(frame);
+        return;
+      }
+
+      refill();
+      if (queue.length === 0 && tokens >= 1) {
+        tokens -= 1;
+        safeEmit(frame);
+        return;
+      }
+
+      enqueueNonTerminal(frame);
+      scheduleDrain();
+    },
+    metrics() {
+      return { droppedFrames, queueDepth: queue.length };
+    },
+    flush() {
+      for (const frame of queue) safeEmit(frame);
+      queue.length = 0;
+      queueIndex.clear();
+      clearDrainTimer();
+    },
+    dispose() {
+      if (disposed) return;
+      disposed = true;
+      clearDrainTimer();
+      queue.length = 0;
+      queueIndex.clear();
+    },
+  };
+}

package/src/share/sse-tap.ts

@@ -0,0 +1,183 @@
+/**
+ * SSE tap: subscribes to the local EventBus for every flowId in the shared
+ * project, mirrors each bridged event into a per-flow ring buffer, and feeds
+ * the relay bridge via `onEvent`. A process-wide monotonic `seq` counter is
+ * stamped on every payload so peers can detect drops and out-of-order frames.
+ *
+ * Used by `apps/studio/src/share.ts` (US-067) to forward live runtime events
+ * to connected peers, and exposes a `snapshot()` of last-seen status per node
+ * so newly joined peers can be primed without replaying the entire buffer
+ * (US-069 / US-070).
+ */
+
+import type { EventBus, StudioEvent } from '../events.ts';
+import { type SsePayload, isSseEventType } from './sse-frame.ts';
+import {
+  type RateLimitMetrics,
+  type RateLimitOptions,
+  type RateLimiter,
+  createRateLimiter,
+} from './sse-rate-limit.ts';
+
+export interface SseTapOptions {
+  /** Called on every refresh to compute the desired subscription set. */
+  flowIds: () => string[];
+  /** Invoked synchronously with each bridged event's payload (post-rate-limit). */
+  onEvent: (frame: SsePayload) => void;
+  /** Per-flow ring-buffer cap; defaults to 50. */
+  bufferSize?: number;
+  /**
+   * Token-bucket rate limit + coalescing applied between the EventBus and
+   * `onEvent`. Pass `false` to disable (frames forwarded synchronously).
+   * Defaults to PRD-spec values: 60 frames/sec sustained, burst 120.
+   */
+  rateLimit?: false | Omit<RateLimitOptions, 'onEmit'>;
+}
+
+export interface SseTap {
+  start(): void;
+  stop(): void;
+  /** Re-syncs the subscription set against `opts.flowIds()`. Idempotent. */
+  refreshFlows(): void;
+  /**
+   * Latest per-node status payload, grouped by flow. Only carries node-level
+   * event types (`node:running`/`node:done`/`node:error`/`node:status`).
+   */
+  snapshot(): Record<string, Record<string, SsePayload>>;
+  /** Rate-limit metrics; zeros when rate limiting is disabled. */
+  metrics(): RateLimitMetrics;
+}
+
+const DEFAULT_BUFFER_SIZE = 50;
+
+const NODE_STATUS_TYPES: ReadonlySet<SsePayload['t']> = new Set([
+  'node:running',
+  'node:done',
+  'node:error',
+  'node:status',
+]);
+
+function extractNodeId(payload: unknown): string | null {
+  if (!payload || typeof payload !== 'object') return null;
+  const candidate = (payload as { nodeId?: unknown }).nodeId;
+  return typeof candidate === 'string' && candidate.length > 0 ? candidate : null;
+}
+
+export function createSseTap(events: EventBus, opts: SseTapOptions): SseTap {
+  const bufferSize = Math.max(1, opts.bufferSize ?? DEFAULT_BUFFER_SIZE);
+  let seqCounter = 0;
+  let started = false;
+
+  const unsubs = new Map<string, () => void>();
+  const buffers = new Map<string, SsePayload[]>();
+  const latestByNode = new Map<string, Map<string, SsePayload>>();
+
+  const limiter: RateLimiter | null =
+    opts.rateLimit === false
+      ? null
+      : createRateLimiter({ ...(opts.rateLimit ?? {}), onEmit: opts.onEvent });
+
+  function pushToBuffer(flowId: string, payload: SsePayload): void {
+    let buf = buffers.get(flowId);
+    if (!buf) {
+      buf = [];
+      buffers.set(flowId, buf);
+    }
+    buf.push(payload);
+    if (buf.length > bufferSize) buf.shift();
+  }
+
+  function recordLatestStatus(flowId: string, payload: SsePayload): void {
+    if (!NODE_STATUS_TYPES.has(payload.t)) return;
+    const nodeId = extractNodeId(payload.data);
+    if (!nodeId) return;
+    let nodeMap = latestByNode.get(flowId);
+    if (!nodeMap) {
+      nodeMap = new Map();
+      latestByNode.set(flowId, nodeMap);
+    }
+    nodeMap.set(nodeId, payload);
+  }
+
+  function makeSubscriber(flowId: string) {
+    return (event: StudioEvent): void => {
+      if (!isSseEventType(event.type)) return;
+      const payload: SsePayload = {
+        t: event.type,
+        flowId: event.flowId,
+        ts: event.ts,
+        data: event.payload,
+        seq: seqCounter++,
+      };
+      pushToBuffer(flowId, payload);
+      recordLatestStatus(flowId, payload);
+      if (limiter) {
+        limiter.submit(payload);
+      } else {
+        try {
+          opts.onEvent(payload);
+        } catch (err) {
+          console.error('[sse-tap] onEvent listener threw:', err);
+        }
+      }
+    };
+  }
+
+  function subscribeFlow(flowId: string): void {
+    if (unsubs.has(flowId)) return;
+    const off = events.subscribe(flowId, makeSubscriber(flowId));
+    unsubs.set(flowId, off);
+  }
+
+  function unsubscribeFlow(flowId: string): void {
+    const off = unsubs.get(flowId);
+    if (!off) return;
+    off();
+    unsubs.delete(flowId);
+    buffers.delete(flowId);
+    latestByNode.delete(flowId);
+  }
+
+  function syncSubscriptions(): void {
+    const desired = new Set(opts.flowIds());
+    for (const existing of [...unsubs.keys()]) {
+      if (!desired.has(existing)) unsubscribeFlow(existing);
+    }
+    for (const id of desired) subscribeFlow(id);
+  }
+
+  return {
+    start(): void {
+      if (started) return;
+      started = true;
+      syncSubscriptions();
+    },
+    stop(): void {
+      if (!started) return;
+      started = false;
+      for (const off of unsubs.values()) off();
+      unsubs.clear();
+      buffers.clear();
+      latestByNode.clear();
+      limiter?.dispose();
+    },
+    refreshFlows(): void {
+      if (!started) return;
+      syncSubscriptions();
+    },
+    snapshot(): Record<string, Record<string, SsePayload>> {
+      const out: Record<string, Record<string, SsePayload>> = {};
+      for (const [flowId, nodeMap] of latestByNode) {
+        const nodes: Record<string, SsePayload> = {};
+        for (const [nodeId, payload] of nodeMap) {
+          nodes[nodeId] = payload;
+        }
+        out[flowId] = nodes;
+      }
+      return out;
+    },
+    metrics(): RateLimitMetrics {
+      return limiter ? limiter.metrics() : { droppedFrames: 0, queueDepth: 0 };
+    },
+  };
+}

package/src/share-audit.ts

@@ -0,0 +1,267 @@
+/**
+ * Per-session JSONL audit log for inbound share frames.
+ *
+ * One file per session at `${opts.dir}/${opts.sessionId}.jsonl`. Each accepted
+ * or rejected envelope writes one JSON.stringify(entry) + '\n' line via
+ * `fs.appendFileSync` (or an injected appender for tests). The directory is
+ * created with mkdirSync recursive on first write.
+ */
+
+import { appendFileSync, mkdirSync } from 'node:fs';
+import { appendFile, mkdir, readFile } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import type { Envelope } from './share-envelope.ts';
+
+export type AuditVerdict = 'accept' | 'reject';
+
+/**
+ * Per-frame audit entry written by the WS message dispatcher. Coexists with
+ * `RpcAuditEntry`, `FileUploadAuditEntry`, and `AuditEntry` (the US-078 shape)
+ * on the same JSONL file; readers should treat each line as the union.
+ */
+export interface FrameAuditEntry {
+  ts: number;
+  peerId: string;
+  displayName: string;
+  type: Envelope['type'];
+  verdict: AuditVerdict;
+  reason?: string;
+}
+
+export interface AuditLog {
+  append(entry: FrameAuditEntry): void;
+  close(): Promise<void>;
+}
+
+export interface AuditLogOpts {
+  dir: string;
+  sessionId: string;
+  // Injected for tests so we don't need a tmp dir. Default uses fs.appendFileSync.
+  appendFn?: (filePath: string, line: string) => void;
+  // Injected for tests so we don't need to mkdir on disk. Default uses fs.mkdirSync.
+  mkdirFn?: (dir: string) => void;
+}
+
+const defaultAppend = (filePath: string, line: string): void => {
+  appendFileSync(filePath, line);
+};
+
+const defaultMkdir = (dir: string): void => {
+  mkdirSync(dir, { recursive: true });
+};
+
+/**
+ * RPC-specific audit entry shape. Used by US-038's `handleRpcFrame` after a
+ * dispatch attempt — written via `appendShareAudit` to the same per-session
+ * JSONL file `createAuditLog` writes to, so the two entry shapes coexist as a
+ * union on disk (callers reading the file should treat each line as
+ * `AuditEntry | RpcAuditEntry`).
+ */
+export interface RpcAuditEntry {
+  ts: number;
+  peerId: string;
+  op: string;
+  flowId: string;
+  ok: boolean;
+  reason?: string;
+  // Mirrors the `attributedTo` field on the outgoing `node-patched` broadcast
+  // so the audit trail records the same originator label peers will see. For
+  // peer-originated rpcs this is `{ peerId, displayName }` from the share
+  // controller's peer map; for host-local edits it is
+  // `{ peerId: 'host', displayName: hostDisplayName }`.
+  attributedTo?: { peerId: string; displayName: string };
+}
+
+export interface AppendShareAuditOpts {
+  // Override the default `~/.seeflow/share-history` root. Tests inject a
+  // tmpdir so they never touch the user's real audit dir.
+  dir?: string;
+  appendFn?: (filePath: string, line: string) => void;
+  mkdirFn?: (dir: string) => void;
+}
+
+const isSafeSessionId = (sessionId: string): boolean => {
+  if (sessionId.length === 0) return false;
+  if (sessionId === '.' || sessionId === '..') return false;
+  if (sessionId.includes('/') || sessionId.includes('\\')) return false;
+  if (sessionId.includes('\0')) return false;
+  return true;
+};
+
+/**
+ * File-upload audit entry. Mirrors the design doc shape:
+ * `{ peerId, op:'file-upload', nodeId, filename, size, sha256, ts, accept }`.
+ * Coexists with `RpcAuditEntry` + `AuditEntry` on the same JSONL file; readers
+ * should treat each line as the union.
+ */
+export interface FileUploadAuditEntry {
+  ts: number;
+  peerId: string;
+  op: 'file-upload';
+  nodeId: string;
+  filename: string;
+  size: number;
+  sha256: string;
+  accept: boolean;
+  reason?: string;
+}
+
+/**
+ * Append one JSON line to `<dir>/<sessionId>.jsonl`. The directory is created
+ * recursively on first write. Rejects sessionIds that contain path separators,
+ * NUL bytes, or `..` traversal attempts so a tampered peer can't drop frames
+ * outside the audit root.
+ */
+export function appendShareAudit(
+  sessionId: string,
+  entry: RpcAuditEntry | FileUploadAuditEntry,
+  opts: AppendShareAuditOpts = {},
+): void {
+  if (!isSafeSessionId(sessionId)) {
+    throw new Error(`invalid sessionId: ${JSON.stringify(sessionId)}`);
+  }
+  const dir = opts.dir ?? join(homedir(), '.seeflow', 'share-history');
+  const appendFn = opts.appendFn ?? defaultAppend;
+  const mkdirFn = opts.mkdirFn ?? defaultMkdir;
+  mkdirFn(dir);
+  const filePath = join(dir, `${sessionId}.jsonl`);
+  appendFn(filePath, `${JSON.stringify(entry)}\n`);
+}
+
+export function createAuditLog(opts: AuditLogOpts): AuditLog {
+  const appendFn = opts.appendFn ?? defaultAppend;
+  const mkdirFn = opts.mkdirFn ?? defaultMkdir;
+  const filePath = join(opts.dir, `${opts.sessionId}.jsonl`);
+  let dirReady = false;
+
+  return {
+    append(entry) {
+      if (!dirReady) {
+        mkdirFn(opts.dir);
+        dirReady = true;
+      }
+      appendFn(filePath, `${JSON.stringify(entry)}\n`);
+    },
+    async close() {
+      // No buffered writes — appendFileSync is synchronous. Hook here for any
+      // future flush logic.
+    },
+  };
+}
+
+/**
+ * Phase-8 audit shape covering RPCs, kicks, rotations, the kill-switch, plus
+ * host start/stop and peer join/leave. Coexists on disk with `FrameAuditEntry`,
+ * `RpcAuditEntry`, and `FileUploadAuditEntry` — readers should tolerate the
+ * union per-line.
+ */
+export type AuditKind =
+  | 'rpc-accept'
+  | 'rpc-reject'
+  | 'kick'
+  | 'rotate'
+  | 'kill-switch'
+  | 'host-start'
+  | 'host-stop'
+  | 'peer-join'
+  | 'peer-leave';
+
+export interface AuditEntry {
+  ts: number;
+  peerId: string | null;
+  displayName: string | null;
+  kind: AuditKind;
+  op?: string;
+  reason?: string;
+  details?: Record<string, unknown>;
+}
+
+export interface AuditLogger {
+  append(entry: Omit<AuditEntry, 'ts'>): Promise<void>;
+  list(opts?: {
+    limit?: number;
+    cursor?: number;
+  }): Promise<{ entries: AuditEntry[]; nextCursor: number | null }>;
+  close(): Promise<void>;
+}
+
+const auditLockChains = new Map<string, Promise<void>>();
+
+const defaultRoot = (): string => join(homedir(), '.seeflow', 'share-history');
+
+/**
+ * Build a per-session audit logger backed by `${root}/${sessionId}.jsonl`.
+ * `append` serializes inside-process per file so 10+ concurrent callers can't
+ * interleave partial lines; the kernel-level O_APPEND on `fs.appendFile` keeps
+ * cross-process writes safe up to PIPE_BUF. `list` paginates by byte offset so
+ * callers can resume from `nextCursor` without re-parsing what they've seen.
+ */
+export function createAuditLogger(sessionId: string, root?: string): AuditLogger {
+  if (!isSafeSessionId(sessionId)) {
+    throw new Error(`invalid sessionId: ${JSON.stringify(sessionId)}`);
+  }
+  const dir = root ?? defaultRoot();
+  const filePath = join(dir, `${sessionId}.jsonl`);
+  let dirReady = false;
+
+  const ensureDir = async (): Promise<void> => {
+    if (dirReady) return;
+    await mkdir(dir, { recursive: true });
+    dirReady = true;
+  };
+
+  const append = async (entry: Omit<AuditEntry, 'ts'>): Promise<void> => {
+    await ensureDir();
+    const full: AuditEntry = { ...entry, ts: Date.now() };
+    const line = `${JSON.stringify(full)}\n`;
+    const prev = auditLockChains.get(filePath) ?? Promise.resolve();
+    const task = prev.then(() => appendFile(filePath, line));
+    auditLockChains.set(
+      filePath,
+      task.then(
+        () => undefined,
+        () => undefined,
+      ),
+    );
+    await task;
+  };
+
+  const list = async (
+    opts: { limit?: number; cursor?: number } = {},
+  ): Promise<{ entries: AuditEntry[]; nextCursor: number | null }> => {
+    const limit = opts.limit ?? 200;
+    const cursor = opts.cursor ?? 0;
+    let buf: Buffer;
+    try {
+      buf = await readFile(filePath);
+    } catch (err) {
+      const code = (err as NodeJS.ErrnoException).code;
+      if (code === 'ENOENT') return { entries: [], nextCursor: null };
+      throw err;
+    }
+    const entries: AuditEntry[] = [];
+    let offset = cursor < 0 ? 0 : cursor;
+    while (offset < buf.length && entries.length < limit) {
+      const nl = buf.indexOf(0x0a, offset);
+      if (nl === -1) break;
+      const line = buf.subarray(offset, nl).toString('utf8');
+      offset = nl + 1;
+      if (line.length === 0) continue;
+      try {
+        entries.push(JSON.parse(line) as AuditEntry);
+      } catch {
+        // Skip corrupted line; advance past it so list() stays monotonic.
+      }
+    }
+    const nextCursor = offset >= buf.length ? null : offset;
+    return { entries, nextCursor };
+  };
+
+  const close = async (): Promise<void> => {
+    const chain = auditLockChains.get(filePath);
+    if (chain) await chain;
+  };
+
+  return { append, list, close };
+}