@tuongaz/seeflow 0.1.102 → 0.1.104

package/src/share-files-manifest.ts

@@ -0,0 +1,232 @@
+/**
+ * Host-side files-manifest builder. Walks the currently registered project's
+ * per-node folders once on session start and emits one `files-manifest` frame
+ * per joining peer so the peer's canvas can prime cache keys and render
+ * placeholder sizing BEFORE any `file-request` fires.
+ *
+ * Manifest is metadata only — no file contents on the wire. Each entry carries
+ * the file's size + a sha256 prefix (16 hex chars) so the peer can detect a
+ * cache miss without round-tripping the body. The full 64-hex sha is preserved
+ * inside file-request / file-redirect payloads (US-060) where integrity matters;
+ * here we use a prefix to keep the manifest small.
+ *
+ * Walk depth is capped at 2 (files directly under `<flowDir>/nodes/<nodeId>/`
+ * plus one level of subdirectory) so a peer that drops a folder of nested
+ * assets doesn't explode the manifest. Total entries are capped at 5000 —
+ * if exceeded the manifest is truncated with a warn; join is never blocked.
+ *
+ * Incremental updates: after the initial build, `recordFileWritten` / `recordNodeRemoved`
+ * keep the in-memory map in sync with `file-upload` node-patched broadcasts +
+ * `removeNodeDir` calls, so a peer that joins mid-session sees the same
+ * post-upload state every other peer has converged on.
+ */
+
+import { createHash } from 'node:crypto';
+import type { Dirent } from 'node:fs';
+import * as fsPromises from 'node:fs/promises';
+import { dirname, join, sep as pathSep } from 'node:path';
+import type { Registry } from './registry.ts';
+import type { FilesManifestEntry, FilesManifestPayload } from './share-envelope.ts';
+
+export const MAX_MANIFEST_ENTRIES = 5000;
+const ETAG_PREFIX_LEN = 16;
+
+interface InternalEntry {
+  nodeId: string;
+  relPath: string;
+  size: number;
+  etag: string;
+}
+
+export interface FilesManifestDeps {
+  registry: Registry;
+  // Test seams — defaults to node:fs/promises.
+  readdir?: (p: string) => Promise<Dirent[]>;
+  readFile?: (p: string) => Promise<Buffer>;
+  stat?: (p: string) => Promise<{ size: number }>;
+}
+
+export interface FilesManifestBuilder {
+  /** Walk every registered flow's nodes/ subtree and populate the in-memory map.
+   *  Safe to call repeatedly — replaces the existing state. */
+  init(): Promise<void>;
+  /** Snapshot of the current manifest, capped at MAX_MANIFEST_ENTRIES. */
+  build(): FilesManifestPayload;
+  /** Record (or replace) a single file entry. Called by share.ts when a
+   *  `file-upload` `node-patched` broadcast fires. */
+  recordFileWritten(input: {
+    absPath: string;
+    nodeId: string;
+    relPath: string;
+    size: number;
+    etag: string;
+  }): void;
+  /** Compute size + etag from raw bytes then `recordFileWritten`. Convenience
+   *  for callers that already hold the post-write buffer. */
+  recordFileWrittenFromBytes(input: {
+    absPath: string;
+    nodeId: string;
+    relPath: string;
+    bytes: Uint8Array;
+  }): void;
+  /** Drop every entry whose `absPath` lives under the given node directory. */
+  recordNodeRemoved(nodeDirAbsPath: string): void;
+  /** Test/debug visibility into the underlying map. */
+  size(): number;
+}
+
+const NODE_ID_RE = /^node-[A-Za-z0-9]{10}$/;
+
+const defaultReaddir = (p: string): Promise<Dirent[]> =>
+  fsPromises.readdir(p, { withFileTypes: true });
+const defaultReadFile = (p: string): Promise<Buffer> => fsPromises.readFile(p) as Promise<Buffer>;
+const defaultStat = async (p: string): Promise<{ size: number }> => {
+  const s = await fsPromises.stat(p);
+  return { size: s.size };
+};
+
+const etagFromBytes = (bytes: Uint8Array): string =>
+  createHash('sha256').update(bytes).digest('hex').slice(0, ETAG_PREFIX_LEN);
+
+const flowDirOf = (flowPath: string): string => {
+  const d = dirname(flowPath);
+  return d === '.' ? '' : d;
+};
+
+const flowBaseAbs = (repoPath: string, flowPath: string): string => {
+  const fd = flowDirOf(flowPath);
+  return fd === '' ? repoPath : join(repoPath, fd);
+};
+
+export function createFilesManifestBuilder(deps: FilesManifestDeps): FilesManifestBuilder {
+  const readdir = deps.readdir ?? defaultReaddir;
+  const readFile = deps.readFile ?? defaultReadFile;
+  const stat = deps.stat ?? defaultStat;
+
+  // absPath -> entry. absPath chosen as the key because (a) it's the only
+  // identifier that uniquely names a file across flows and (b) it lets
+  // `recordNodeRemoved(nodeDirAbsPath)` drop everything under a single prefix
+  // without a secondary index.
+  const entries = new Map<string, InternalEntry>();
+
+  const walkNodeDir = async (flowBase: string, nodeId: string): Promise<void> => {
+    const nodeDir = join(flowBase, 'nodes', nodeId);
+    let level1: Dirent[];
+    try {
+      level1 = await readdir(nodeDir);
+    } catch {
+      return;
+    }
+    for (const d of level1) {
+      if (d.isSymbolicLink()) continue;
+      if (d.isFile()) {
+        await ingestFile(nodeDir, nodeId, d.name, [d.name]);
+        continue;
+      }
+      if (!d.isDirectory()) continue;
+      // Depth 2: one level of subdirectory. Files inside are included; nested
+      // directories below this are NOT descended into.
+      const subDir = join(nodeDir, d.name);
+      let level2: Dirent[];
+      try {
+        level2 = await readdir(subDir);
+      } catch {
+        continue;
+      }
+      for (const inner of level2) {
+        if (inner.isSymbolicLink()) continue;
+        if (!inner.isFile()) continue;
+        await ingestFile(subDir, nodeId, inner.name, [d.name, inner.name]);
+      }
+    }
+  };
+
+  const ingestFile = async (
+    fileDir: string,
+    nodeId: string,
+    fileName: string,
+    relSegments: string[],
+  ): Promise<void> => {
+    const absPath = join(fileDir, fileName);
+    // Per-file relPath is relative to the flow folder. Always forward-slash
+    // separated so the wire payload is platform-neutral.
+    const relPath = `nodes/${nodeId}/${relSegments.join('/')}`;
+    let size = 0;
+    let etag = '';
+    try {
+      const st = await stat(absPath);
+      size = st.size;
+    } catch {
+      return;
+    }
+    try {
+      const bytes = await readFile(absPath);
+      etag = etagFromBytes(bytes);
+    } catch {
+      return;
+    }
+    entries.set(absPath, { nodeId, relPath, size, etag });
+  };
+
+  return {
+    async init() {
+      entries.clear();
+      for (const entry of deps.registry.list()) {
+        const flowBase = flowBaseAbs(entry.repoPath, entry.flowPath);
+        const nodesRoot = join(flowBase, 'nodes');
+        let nodeDirs: Dirent[];
+        try {
+          nodeDirs = await readdir(nodesRoot);
+        } catch {
+          continue;
+        }
+        for (const d of nodeDirs) {
+          if (d.isSymbolicLink()) continue;
+          if (!d.isDirectory()) continue;
+          if (!NODE_ID_RE.test(d.name)) continue;
+          await walkNodeDir(flowBase, d.name);
+        }
+      }
+    },
+    build() {
+      const all = [...entries.values()].map(
+        ({ nodeId, relPath, size, etag }): FilesManifestEntry => ({
+          nodeId,
+          relPath,
+          size,
+          etag,
+        }),
+      );
+      if (all.length > MAX_MANIFEST_ENTRIES) {
+        console.warn(
+          `[share] files-manifest truncated: ${all.length} entries exceeds cap ${MAX_MANIFEST_ENTRIES}`,
+        );
+        return { entries: all.slice(0, MAX_MANIFEST_ENTRIES) };
+      }
+      return { entries: all };
+    },
+    recordFileWritten({ absPath, nodeId, relPath, size, etag }) {
+      entries.set(absPath, { nodeId, relPath, size, etag });
+    },
+    recordFileWrittenFromBytes({ absPath, nodeId, relPath, bytes }) {
+      entries.set(absPath, {
+        nodeId,
+        relPath,
+        size: bytes.byteLength,
+        etag: etagFromBytes(bytes),
+      });
+    },
+    recordNodeRemoved(nodeDirAbsPath) {
+      // Match either the directory itself or anything strictly under it. A
+      // bare prefix check would falsely match a sibling directory whose name
+      // starts with `nodeDirAbsPath` (e.g. `/a/node-x` vs `/a/node-xx-data`).
+      const prefix = nodeDirAbsPath.endsWith(pathSep) ? nodeDirAbsPath : nodeDirAbsPath + pathSep;
+      for (const key of entries.keys()) {
+        if (key === nodeDirAbsPath || key.startsWith(prefix)) entries.delete(key);
+      }
+    },
+    size() {
+      return entries.size;
+    },
+  };
+}

package/src/share-ratelimit.ts

@@ -0,0 +1,69 @@
+/**
+ * Per-peer token-bucket rate limiter for inbound share frames.
+ *
+ * Each peer holds a bucket of `burst` tokens, refilling at `ratePerSec`
+ * tokens/second. check() deducts a token if available and returns ok; if
+ * empty, it returns the milliseconds the caller should wait before retrying.
+ * Time is injected via `now()` so tests can drive the bucket synchronously.
+ */
+
+export interface RateLimitOk {
+  ok: true;
+}
+
+export interface RateLimitDenied {
+  ok: false;
+  retryAfterMs: number;
+}
+
+export type RateLimitResult = RateLimitOk | RateLimitDenied;
+
+export interface RateLimiter {
+  check(peerId: string): RateLimitResult;
+}
+
+export interface RateLimiterOpts {
+  ratePerSec: number;
+  burst?: number;
+  now?: () => number;
+}
+
+interface Bucket {
+  tokens: number;
+  lastRefillMs: number;
+}
+
+export function createRateLimiter(opts: RateLimiterOpts): RateLimiter {
+  const ratePerSec = opts.ratePerSec;
+  const capacity = opts.burst ?? opts.ratePerSec;
+  const nowFn = opts.now ?? Date.now;
+  const buckets = new Map<string, Bucket>();
+
+  const refill = (bucket: Bucket, now: number) => {
+    const elapsedSec = (now - bucket.lastRefillMs) / 1000;
+    if (elapsedSec <= 0) return;
+    const refilled = bucket.tokens + elapsedSec * ratePerSec;
+    bucket.tokens = refilled > capacity ? capacity : refilled;
+    bucket.lastRefillMs = now;
+  };
+
+  return {
+    check(peerId) {
+      const now = nowFn();
+      let bucket = buckets.get(peerId);
+      if (!bucket) {
+        bucket = { tokens: capacity, lastRefillMs: now };
+        buckets.set(peerId, bucket);
+      } else {
+        refill(bucket, now);
+      }
+      if (bucket.tokens >= 1) {
+        bucket.tokens -= 1;
+        return { ok: true };
+      }
+      const deficit = 1 - bucket.tokens;
+      const retryAfterMs = Math.ceil((deficit / ratePerSec) * 1000);
+      return { ok: false, retryAfterMs };
+    },
+  };
+}

package/src/share-rpc-schema.ts

@@ -0,0 +1,249 @@
+/**
+ * Shared RPC envelope schema for the live-share WebSocket protocol (phase 4).
+ *
+ * ⚠ BYTE-FOR-BYTE MIRROR: this file MUST stay identical to
+ *   /Users/tuongaz/dev/seeflow-viewer/src/lib/share-rpc-schema.ts.
+ * The integration check at apps/studio/integration/share-rpc-schema-sync.it.ts
+ * reads both files and asserts strict bytewise equality. Any edit here MUST
+ * be mirrored to the viewer copy in the same commit, or CI fails.
+ *
+ * Self-contained on purpose: NO imports from './operations.ts' or './schema.ts'.
+ * Those paths don't resolve in the viewer repo, and byte-for-byte mirroring
+ * requires identical imports across both files.
+ *
+ * Body-shape strictness:
+ *   - PositionBody + ReorderBody are tiny self-contained Zod shapes — inlined
+ *     here so obviously-malformed payloads (NaN x, unknown reorder op) reject
+ *     at the wire layer.
+ *   - Node/connector add + patch bodies are loose `z.record(z.unknown())`. The
+ *     strict re-validation runs at impl-dispatch time in operations.ts via
+ *     `NodePatchBodySchema` / `ConnectorPatchBodySchema` / the post-merge
+ *     `ResolvedFlowSchema` reparse (US-038's `handleRpcFrame`). Treat this file
+ *     as an envelope schema, not a substitute for the impl-layer guards.
+ *
+ * Op allowlist (9 ops): addNode, patchNode, moveNode, reorderNode, deleteNode,
+ * addConnector, patchConnector, deleteConnector, addBulk. Unknown ops are
+ * rejected by the discriminated union; the per-op `.strict()` rejects unknown
+ * top-level keys so a `{ op: 'addNode', nodeId, position }` cross-op shape
+ * fails fast instead of silently accepting the wrong wrapper.
+ *
+ * `addBulk` is the only op without `.strict()` because its body carries the
+ * optional `nodes` + `connectors` arrays alongside `op` + `flowId`; the
+ * 100-item-per-array cap mirrors operations.ts's `FlowBulkBodyShape`. The
+ * "at least one non-empty" refine from `FlowBulkBodySchema` is NOT applied
+ * here (discriminatedUnion requires bare ZodObjects, not ZodEffects) — the
+ * non-empty check runs in US-038's handler before dispatching to
+ * `addFlowBulkImpl`.
+ */
+
+import { z } from 'zod';
+
+const FlowIdSchema = z.string().min(1);
+const NodeIdSchema = z.string().min(1);
+const ConnectorIdSchema = z.string().min(1);
+const FrameIdSchema = z.string().min(1);
+
+const PositionSchema = z.object({
+  x: z.number().finite(),
+  y: z.number().finite(),
+});
+
+const ReorderSchema = z.discriminatedUnion('op', [
+  z.object({ op: z.literal('forward') }),
+  z.object({ op: z.literal('backward') }),
+  z.object({ op: z.literal('toFront') }),
+  z.object({ op: z.literal('toBack') }),
+  z.object({ op: z.literal('toIndex'), index: z.number().int().nonnegative() }),
+]);
+
+const NodeBodySchema = z.record(z.unknown());
+const ConnectorBodySchema = z.record(z.unknown());
+const NodePatchBodyShape = z.record(z.unknown());
+const ConnectorPatchBodyShape = z.record(z.unknown());
+
+const BULK_MAX_ITEMS = 100;
+const BulkArraySchema = z.array(z.record(z.unknown())).max(BULK_MAX_ITEMS);
+
+export const AddNodeOpSchema = z
+  .object({
+    op: z.literal('addNode'),
+    flowId: FlowIdSchema,
+    node: NodeBodySchema,
+  })
+  .strict();
+
+export const PatchNodeOpSchema = z
+  .object({
+    op: z.literal('patchNode'),
+    flowId: FlowIdSchema,
+    nodeId: NodeIdSchema,
+    patch: NodePatchBodyShape,
+  })
+  .strict();
+
+export const MoveNodeOpSchema = z
+  .object({
+    op: z.literal('moveNode'),
+    flowId: FlowIdSchema,
+    nodeId: NodeIdSchema,
+    position: PositionSchema,
+  })
+  .strict();
+
+export const ReorderNodeOpSchema = z
+  .object({
+    op: z.literal('reorderNode'),
+    flowId: FlowIdSchema,
+    nodeId: NodeIdSchema,
+    reorder: ReorderSchema,
+  })
+  .strict();
+
+export const DeleteNodeOpSchema = z
+  .object({
+    op: z.literal('deleteNode'),
+    flowId: FlowIdSchema,
+    nodeId: NodeIdSchema,
+  })
+  .strict();
+
+export const AddConnectorOpSchema = z
+  .object({
+    op: z.literal('addConnector'),
+    flowId: FlowIdSchema,
+    connector: ConnectorBodySchema,
+  })
+  .strict();
+
+export const PatchConnectorOpSchema = z
+  .object({
+    op: z.literal('patchConnector'),
+    flowId: FlowIdSchema,
+    connectorId: ConnectorIdSchema,
+    patch: ConnectorPatchBodyShape,
+  })
+  .strict();
+
+export const DeleteConnectorOpSchema = z
+  .object({
+    op: z.literal('deleteConnector'),
+    flowId: FlowIdSchema,
+    connectorId: ConnectorIdSchema,
+  })
+  .strict();
+
+export const AddBulkOpSchema = z
+  .object({
+    op: z.literal('addBulk'),
+    flowId: FlowIdSchema,
+    nodes: BulkArraySchema.optional(),
+    connectors: BulkArraySchema.optional(),
+  })
+  .strict();
+
+export const RpcOpSchema = z.discriminatedUnion('op', [
+  AddNodeOpSchema,
+  PatchNodeOpSchema,
+  MoveNodeOpSchema,
+  ReorderNodeOpSchema,
+  DeleteNodeOpSchema,
+  AddConnectorOpSchema,
+  PatchConnectorOpSchema,
+  DeleteConnectorOpSchema,
+  AddBulkOpSchema,
+]);
+
+export const RpcFrameSchema = z
+  .object({
+    v: z.literal(1),
+    type: z.literal('rpc'),
+    id: FrameIdSchema,
+    payload: RpcOpSchema,
+  })
+  .strict();
+
+/**
+ * Attribution metadata attached to every accepted edit. `peerId` is either the
+ * literal `'host'` (when the studio owner originates the edit) or matches a
+ * known remote peer's id. `displayName` is the human label rendered by the
+ * peer SPA's toast / cursor label.
+ *
+ * Cross-validation (peerId === 'host' OR peerId ∈ knownPeers) is enforced at
+ * runtime by the share controller — the wire schema can't reach the peer set,
+ * so it accepts any non-empty string and the controller drops broadcasts with
+ * peerIds that don't match.
+ */
+export const AttributionSchema = z
+  .object({
+    peerId: z.string().min(1),
+    displayName: z.string().min(1),
+  })
+  .strict();
+
+export type Attribution = z.infer<typeof AttributionSchema>;
+
+export const RpcResultPayloadSchema = z.discriminatedUnion('ok', [
+  z
+    .object({
+      ok: z.literal(true),
+      result: z.unknown().optional(),
+      attributedTo: AttributionSchema.optional(),
+    })
+    .strict(),
+  z.object({ ok: z.literal(false), reason: z.string() }).strict(),
+]);
+
+export const RpcResultFrameSchema = z
+  .object({
+    v: z.literal(1),
+    type: z.literal('rpc-result'),
+    id: FrameIdSchema,
+    payload: RpcResultPayloadSchema,
+  })
+  .strict();
+
+export type RpcOp = z.infer<typeof RpcOpSchema>;
+export type RpcFrame = z.infer<typeof RpcFrameSchema>;
+export type RpcResultFrame = z.infer<typeof RpcResultFrameSchema>;
+
+/**
+ * `node-patched` broadcast frame (US-039 host → US-042 peer apply).
+ *
+ * Host emits this after every accepted `rpc` op so all peers (including the
+ * originator) converge on the canonical post-op snapshot. Payload shape mirrors
+ * `computeNodePatchedDiff` in `apps/studio/src/share.ts`:
+ *
+ *   { flowId, op, version, diff: { kind, ...fields } }
+ *
+ * where `op` is one of the 9 RpcOp strings and `diff.kind` is one of
+ *   'move' | 'patch' | 'add' | 'delete' | 'reorder' | 'bulk'
+ * (the latter only for `addBulk`). The diff body is intentionally loose at
+ * the wire layer — peers switch on `op` + `diff.kind` to drive the apply.
+ * Strict diff shapes live host-side in `computeNodePatchedDiff`.
+ *
+ * `version` is a per-flow monotonic counter. Peers track
+ * `lastAppliedVersion[flowId]` and drop frames where `version <=` the last
+ * applied — guards against out-of-order delivery on reconnect.
+ */
+const NodePatchedDiffSchema = z.record(z.unknown());
+
+export const NodePatchedFramePayloadSchema = z
+  .object({
+    flowId: FlowIdSchema,
+    op: z.string().min(1),
+    version: z.number().int().nonnegative(),
+    diff: NodePatchedDiffSchema,
+    attributedTo: AttributionSchema,
+  })
+  .passthrough();
+
+export const NodePatchedFrameSchema = z
+  .object({
+    v: z.literal(1),
+    type: z.literal('node-patched'),
+    payload: NodePatchedFramePayloadSchema,
+  })
+  .passthrough();
+
+export type NodePatchedFramePayload = z.infer<typeof NodePatchedFramePayloadSchema>;
+export type NodePatchedFrame = z.infer<typeof NodePatchedFrameSchema>;