@thevinci/web 1.0.0

package/server/lib/event-stream/upstream-reader.js

@@ -0,0 +1,226 @@
+import { parseSseEventEnvelope } from './protocol.js';
+
+export const DEFAULT_UPSTREAM_STALL_TIMEOUT_MS = 20_000;
+export const UPSTREAM_STALL_TIMEOUT_CONCURRENT_MS = DEFAULT_UPSTREAM_STALL_TIMEOUT_MS * 3;
+export const DEFAULT_UPSTREAM_RECONNECT_DELAY_MS = 250;
+
+function resolveTimeoutMs(value, fallback) {
+  const resolved = typeof value === 'function' ? value() : value;
+  return Number.isFinite(resolved) ? resolved : fallback;
+}
+
+function waitForReconnectDelay(ms, signal) {
+  if (signal?.aborted) {
+    return Promise.resolve();
+  }
+
+  return new Promise((resolve) => {
+    let settled = false;
+    const finish = () => {
+      if (settled) return;
+      settled = true;
+      signal?.removeEventListener('abort', onAbort);
+      resolve();
+    };
+    const timeout = setTimeout(finish, Math.max(0, ms));
+    const onAbort = () => {
+      clearTimeout(timeout);
+      finish();
+    };
+    signal?.addEventListener('abort', onAbort, { once: true });
+  });
+}
+
+function normalizeHeaders(headers) {
+  if (!headers || typeof headers !== 'object') {
+    return {};
+  }
+
+  return { ...headers };
+}
+
+async function cancelResponseBody(response) {
+  if (response?.body && typeof response.body.cancel === 'function') {
+    await response.body.cancel().catch(() => {});
+  }
+}
+
+export function createUpstreamSseReader({
+  buildUrl,
+  getHeaders = () => ({}),
+  fetchImpl = fetch,
+  parseBlock = parseSseEventEnvelope,
+  initialLastEventId = '',
+  signal,
+  stallTimeoutMs = DEFAULT_UPSTREAM_STALL_TIMEOUT_MS,
+  reconnectDelayMs = DEFAULT_UPSTREAM_RECONNECT_DELAY_MS,
+  onEvent,
+  onConnect,
+  onDisconnect,
+  onError,
+}) {
+  let running = null;
+  let stopped = false;
+  let activeController = null;
+  let lastEventId = typeof initialLastEventId === 'string' ? initialLastEventId : '';
+
+  const stop = () => {
+    stopped = true;
+    if (activeController && !activeController.signal.aborted) {
+      activeController.abort();
+    }
+  };
+
+  signal?.addEventListener('abort', stop, { once: true });
+
+  const start = () => {
+    if (running) {
+      return running;
+    }
+
+    stopped = false;
+    running = (async () => {
+      while (!stopped && !signal?.aborted) {
+        const controller = new AbortController();
+        activeController = controller;
+        const abortActive = () => controller.abort();
+        signal?.addEventListener('abort', abortActive, { once: true });
+
+        let abortReason = null;
+        let stallTimer = null;
+        const clearStallTimer = () => {
+          if (stallTimer) {
+            clearTimeout(stallTimer);
+            stallTimer = null;
+          }
+        };
+        const resetStallTimer = () => {
+          clearStallTimer();
+          const currentStallTimeoutMs = resolveTimeoutMs(stallTimeoutMs, DEFAULT_UPSTREAM_STALL_TIMEOUT_MS);
+          if (currentStallTimeoutMs <= 0) {
+            return;
+          }
+
+          stallTimer = setTimeout(() => {
+            abortReason = 'upstream_stalled';
+            controller.abort();
+          }, currentStallTimeoutMs);
+        };
+
+        try {
+          const url = buildUrl();
+          const headers = {
+            Accept: 'text/event-stream',
+            'Cache-Control': 'no-cache',
+            Connection: 'keep-alive',
+            ...normalizeHeaders(getHeaders()),
+          };
+          if (lastEventId) {
+            headers['Last-Event-ID'] = lastEventId;
+          }
+
+          const response = await fetchImpl(url.toString(), {
+            headers,
+            signal: controller.signal,
+          });
+
+          if (!response?.ok || !response.body) {
+            onError?.({
+              type: 'upstream_unavailable',
+              status: response?.status ?? 0,
+              response,
+            });
+            await cancelResponseBody(response);
+            await waitForReconnectDelay(reconnectDelayMs, signal);
+            continue;
+          }
+
+          onConnect?.({ response, lastEventId });
+
+          const decoder = new TextDecoder();
+          const reader = response.body.getReader();
+          let buffer = '';
+
+          resetStallTimer();
+
+          while (!stopped && !signal?.aborted) {
+            const { value, done } = await reader.read();
+            if (done) {
+              break;
+            }
+
+            resetStallTimer();
+            buffer += decoder.decode(value, { stream: true }).replace(/\r\n/g, '\n');
+
+            let separatorIndex = buffer.indexOf('\n\n');
+            while (separatorIndex !== -1 && !stopped && !signal?.aborted) {
+              const block = buffer.slice(0, separatorIndex);
+              buffer = buffer.slice(separatorIndex + 2);
+              const envelope = parseBlock(block);
+              if (envelope?.payload) {
+                if (typeof envelope.eventId === 'string' && envelope.eventId.length > 0) {
+                  lastEventId = envelope.eventId;
+                }
+                onEvent?.({
+                  block,
+                  envelope,
+                  payload: envelope.payload,
+                  eventId: envelope.eventId,
+                  directory: envelope.directory,
+                });
+              }
+              separatorIndex = buffer.indexOf('\n\n');
+            }
+          }
+
+          if (!stopped && !signal?.aborted && buffer.trim().length > 0) {
+            const block = buffer.trim();
+            const envelope = parseBlock(block);
+            if (envelope?.payload) {
+              if (typeof envelope.eventId === 'string' && envelope.eventId.length > 0) {
+                lastEventId = envelope.eventId;
+              }
+              onEvent?.({
+                block,
+                envelope,
+                payload: envelope.payload,
+                eventId: envelope.eventId,
+                directory: envelope.directory,
+              });
+            }
+          }
+        } catch (error) {
+          if (!stopped && !signal?.aborted && abortReason !== 'upstream_stalled') {
+            onError?.({
+              type: 'stream_error',
+              error,
+            });
+          }
+        } finally {
+          clearStallTimer();
+          signal?.removeEventListener('abort', abortActive);
+          if (activeController === controller) {
+            activeController = null;
+          }
+          onDisconnect?.({ reason: abortReason ?? (stopped || signal?.aborted ? 'stopped' : 'closed') });
+        }
+
+        if (!stopped && !signal?.aborted) {
+          await waitForReconnectDelay(reconnectDelayMs, signal);
+        }
+      }
+    })().finally(() => {
+      running = null;
+    });
+
+    return running;
+  };
+
+  return {
+    start,
+    stop,
+    getLastEventId() {
+      return lastEventId;
+    },
+  };
+}

package/server/lib/event-stream/upstream-reader.test.js

@@ -0,0 +1,276 @@
+import { describe, expect, it } from 'vitest';
+
+import { createUpstreamSseReader } from './upstream-reader.js';
+
+function createSseResponse({ blocks = [], signal, holdOpen = false }) {
+  const encoder = new TextEncoder();
+  let index = 0;
+
+  return {
+    ok: true,
+    status: 200,
+    body: {
+      getReader() {
+        return {
+          async read() {
+            if (index < blocks.length) {
+              return { value: encoder.encode(blocks[index++]), done: false };
+            }
+
+            if (!holdOpen) {
+              return { value: undefined, done: true };
+            }
+
+            return new Promise((_resolve, reject) => {
+              const onAbort = () => {
+                signal.removeEventListener('abort', onAbort);
+                const error = new Error('Aborted');
+                error.name = 'AbortError';
+                reject(error);
+              };
+              signal.addEventListener('abort', onAbort, { once: true });
+            });
+          },
+        };
+      },
+    },
+  };
+}
+
+function createTrackedSignal() {
+  const listeners = new Set();
+  return {
+    signal: {
+      aborted: false,
+      addEventListener(type, listener) {
+        if (type === 'abort') {
+          listeners.add(listener);
+        }
+      },
+      removeEventListener(type, listener) {
+        if (type === 'abort') {
+          listeners.delete(listener);
+        }
+      },
+    },
+    getListenerCount() {
+      return listeners.size;
+    },
+  };
+}
+
+describe('createUpstreamSseReader', () => {
+  it('emits parsed events and tracks the latest event id', async () => {
+    const events = [];
+    let reader;
+
+    reader = createUpstreamSseReader({
+      buildUrl: () => 'http://127.0.0.1:4096/global/event',
+      reconnectDelayMs: 0,
+      fetchImpl: async (_url, options) => createSseResponse({
+        signal: options.signal,
+        blocks: [
+          'id: evt-1\r\ndata: {"type":"server.connected","properties":{"directory":"/tmp/project"}}\r\n\r\n',
+        ],
+      }),
+      onEvent(event) {
+        events.push(event);
+        reader.stop();
+      },
+    });
+
+    await reader.start();
+
+    expect(events).toHaveLength(1);
+    expect(events[0].eventId).toBe('evt-1');
+    expect(events[0].directory).toBe('/tmp/project');
+    expect(events[0].payload).toEqual({
+      type: 'server.connected',
+      properties: {
+        directory: '/tmp/project',
+      },
+    });
+    expect(reader.getLastEventId()).toBe('evt-1');
+  });
+
+  it('reconnects a stalled stream with Last-Event-ID', async () => {
+    const fetchLastEventIds = [];
+    const events = [];
+    let attempt = 0;
+    let reader;
+
+    reader = createUpstreamSseReader({
+      buildUrl: () => 'http://127.0.0.1:4096/global/event',
+      stallTimeoutMs: 10,
+      reconnectDelayMs: 0,
+      fetchImpl: async (_url, options) => {
+        fetchLastEventIds.push(options.headers['Last-Event-ID'] ?? null);
+        attempt += 1;
+
+        if (attempt === 1) {
+          return createSseResponse({
+            signal: options.signal,
+            holdOpen: true,
+            blocks: [
+              'id: evt-1\ndata: {"type":"server.connected","properties":{}}\n\n',
+            ],
+          });
+        }
+
+        return createSseResponse({
+          signal: options.signal,
+          blocks: [
+            'id: evt-2\ndata: {"type":"session.updated","properties":{}}\n\n',
+          ],
+        });
+      },
+      onEvent(event) {
+        events.push(event.eventId);
+        if (event.eventId === 'evt-2') {
+          reader.stop();
+        }
+      },
+    });
+
+    await reader.start();
+
+    expect(events).toEqual(['evt-1', 'evt-2']);
+    expect(fetchLastEventIds.slice(0, 2)).toEqual([null, 'evt-1']);
+    expect(reader.getLastEventId()).toBe('evt-2');
+  });
+
+  it('resolves the stall timeout for each upstream read window', async () => {
+    const events = [];
+    let attempt = 0;
+    let currentTimeout = 10;
+    let reader;
+
+    reader = createUpstreamSseReader({
+      buildUrl: () => 'http://127.0.0.1:4096/global/event',
+      stallTimeoutMs: () => currentTimeout,
+      reconnectDelayMs: 0,
+      fetchImpl: async (_url, options) => {
+        attempt += 1;
+
+        if (attempt === 1) {
+          currentTimeout = 60;
+          return createSseResponse({
+            signal: options.signal,
+            holdOpen: true,
+            blocks: [
+              'id: evt-1\ndata: {"type":"server.connected","properties":{}}\n\n',
+            ],
+          });
+        }
+
+        return createSseResponse({
+          signal: options.signal,
+          blocks: [
+            'id: evt-2\ndata: {"type":"session.updated","properties":{}}\n\n',
+          ],
+        });
+      },
+      onEvent(event) {
+        events.push(event.eventId);
+        if (event.eventId === 'evt-2') {
+          reader.stop();
+        }
+      },
+    });
+
+    await reader.start();
+
+    expect(events).toEqual(['evt-1', 'evt-2']);
+    expect(attempt).toBe(2);
+  });
+
+  it('reports unavailable upstream responses and continues reconnecting until stopped', async () => {
+    const errors = [];
+    let attempt = 0;
+    let unavailableBodyCanceled = false;
+    let reader;
+
+    reader = createUpstreamSseReader({
+      buildUrl: () => 'http://127.0.0.1:4096/global/event',
+      reconnectDelayMs: 0,
+      fetchImpl: async (_url, options) => {
+        attempt += 1;
+        if (attempt === 1) {
+          return {
+            ok: false,
+            status: 503,
+            body: {
+              cancel: async () => {
+                unavailableBodyCanceled = true;
+              },
+            },
+          };
+        }
+
+        return createSseResponse({
+          signal: options.signal,
+          blocks: [
+            'id: evt-1\ndata: {"type":"server.connected","properties":{}}\n\n',
+          ],
+        });
+      },
+      onError(error) {
+        errors.push(error);
+      },
+      onEvent() {
+        reader.stop();
+      },
+    });
+
+    await reader.start();
+
+    expect(errors).toEqual([
+      expect.objectContaining({
+        type: 'upstream_unavailable',
+        status: 503,
+      }),
+    ]);
+    expect(unavailableBodyCanceled).toBe(true);
+    expect(attempt).toBe(2);
+  });
+
+  it('removes reconnect delay abort listeners after normal timeout completion', async () => {
+    const tracked = createTrackedSignal();
+    let attempt = 0;
+    let reader;
+
+    reader = createUpstreamSseReader({
+      buildUrl: () => 'http://127.0.0.1:4096/global/event',
+      reconnectDelayMs: 1,
+      signal: tracked.signal,
+      fetchImpl: async (_url, options) => {
+        attempt += 1;
+        if (attempt === 1) {
+          return {
+            ok: false,
+            status: 503,
+            body: {
+              cancel: async () => {},
+            },
+          };
+        }
+
+        return createSseResponse({
+          signal: options.signal,
+          blocks: [
+            'id: evt-1\ndata: {"type":"server.connected","properties":{}}\n\n',
+          ],
+        });
+      },
+      onEvent() {
+        reader.stop();
+      },
+    });
+
+    await reader.start();
+
+    expect(attempt).toBe(2);
+    // The top-level stop listener remains; the reconnect-delay listener should be removed.
+    expect(tracked.getListenerCount()).toBe(1);
+  });
+});

package/server/lib/fs/DOCUMENTATION.md

@@ -0,0 +1,36 @@
+# FS Module Documentation
+
+## Purpose
+Own filesystem API behavior for the web server runtime, including workspace-bound file operations, directory listing, reveal, and background command execution jobs.
+
+## Entrypoints and structure
+- `packages/web/server/lib/fs/routes.js`: route registration and runtime-owned state for `/api/fs/*` endpoints.
+- `packages/web/server/lib/fs/search.js`: fuzzy filesystem search runtime used by non-FS routes (for example project icon discovery).
+
+## Public exports
+- `registerFsRoutes(app, dependencies)` from `routes.js`
+  - Registers all filesystem routes:
+    - `GET /api/fs/home`
+    - `POST /api/fs/mkdir`
+    - `GET /api/fs/read`
+    - `GET /api/fs/raw`
+    - `POST /api/fs/write`
+    - `POST /api/fs/delete`
+    - `POST /api/fs/rename`
+    - `POST /api/fs/reveal`
+    - `POST /api/fs/exec`
+    - `GET /api/fs/exec/:jobId`
+    - `GET /api/fs/list`
+  - Owns exec job queue state (`execJobs`) and lifecycle/TTL pruning.
+  - Enforces workspace boundary checks with active project + worktree fallback support.
+- `createFsSearchRuntime({ fsPromises, path, spawn, resolveGitBinaryForSpawn })` from `search.js`
+  - Returns `{ searchFilesystemFiles(rootPath, options) }`.
+  - Supports fuzzy matching, hidden-file handling, and optional `git check-ignore` filtering.
+
+## Composition contract with `index.js`
+- `index.js` provides composition-time dependencies only (platform primitives + callbacks such as `resolveProjectDirectory`, `normalizeDirectoryPath`, and `buildAugmentedPath`).
+- `index.js` no longer owns FS route handlers or FS exec job state.
+
+## Notes for contributors
+- Keep filesystem policy (workspace root checks, error mapping, exec timeout behavior) inside this module, not in the composition root.
+- If adding new `/api/fs/*` endpoints, add them in `routes.js` and extend this document.