@codeyam/codeyam-cli 0.1.0-staging.a890816 → 0.1.0-staging.ad88eeb

package/codeyam-cli/src/webserver/scripts/codeyam-preload.mjs

@@ -0,0 +1,414 @@
+/**
+ * Node.js preload module for CodeYam editor proxy.
+ *
+ * Loaded via NODE_OPTIONS='--import /path/to/codeyam-preload.mjs' in the
+ * dev server process. Patches global fetch and http/https.request to:
+ *
+ * 1. **Direct mock matching** — reads the active scenario data from disk and
+ *    returns mock responses for ANY matching URL (internal API routes, external
+ *    APIs like Stripe/weather, etc.) without a network round-trip.
+ *
+ * 2. **Proxy redirect** — for dev-server-targeting requests that don't match
+ *    a mock route, redirects them through the CodeYam proxy.
+ *
+ * Must be .mjs (native ESM) because --import requires it.
+ * No TypeScript, no build step, no dependencies.
+ */
+
+import http from 'node:http';
+import https from 'node:https';
+import fs from 'node:fs';
+import path from 'node:path';
+import { Readable } from 'node:stream';
+import { EventEmitter } from 'node:events';
+
+let proxyConfig = null;
+let configLoaded = false;
+
+/**
+ * Lazily load the proxy config from .codeyam/proxy-config.json.
+ * Returns null if the file doesn't exist yet (proxy still starting).
+ */
+function getProxyConfig() {
+  if (configLoaded && proxyConfig) return proxyConfig;
+
+  // Try env var first (most reliable)
+  const proxyUrl = process.env.CODEYAM_PROXY_URL;
+  const devServerUrl = process.env.CODEYAM_DEV_SERVER_URL;
+  if (proxyUrl && devServerUrl) {
+    proxyConfig = { proxyUrl, devServerUrl };
+    configLoaded = true;
+    return proxyConfig;
+  }
+
+  // Fall back to config file
+  try {
+    const cwd = process.cwd();
+    const configPath = path.join(cwd, '.codeyam', 'proxy-config.json');
+    if (fs.existsSync(configPath)) {
+      const raw = fs.readFileSync(configPath, 'utf-8');
+      proxyConfig = JSON.parse(raw);
+      configLoaded = true;
+      return proxyConfig;
+    }
+  } catch {
+    // File doesn't exist yet or is invalid — that's fine
+  }
+
+  return null;
+}
+
+// --- Scenario data for direct mock matching (SSR + external APIs) ---
+
+let scenarioCache = { data: null, timestamp: 0 };
+const SCENARIO_CACHE_TTL = 500;
+
+/**
+ * Read the active scenario's mock data from disk, with brief caching.
+ * Returns null if no active scenario or data file is missing.
+ *
+ * For application/user scenarios (seed-based): only returns externalApis
+ * as routes. DB-backed routes are served by the real app from seeded data.
+ */
+function readScenarioData() {
+  const now = Date.now();
+  if (scenarioCache.data !== null && now - scenarioCache.timestamp < SCENARIO_CACHE_TTL) {
+    return scenarioCache.data;
+  }
+
+  try {
+    const cwd = process.cwd();
+    const activePath = path.join(cwd, '.codeyam', 'active-scenario.json');
+    if (!fs.existsSync(activePath)) {
+      scenarioCache = { data: null, timestamp: now };
+      return null;
+    }
+
+    const active = JSON.parse(fs.readFileSync(activePath, 'utf-8'));
+    if (!active.scenarioId) {
+      scenarioCache = { data: null, timestamp: now };
+      return null;
+    }
+
+    const dataPath = path.join(cwd, '.codeyam', 'editor-scenarios', `${active.scenarioId}.json`);
+    if (!fs.existsSync(dataPath)) {
+      scenarioCache = { data: null, timestamp: now };
+      return null;
+    }
+
+    const rawData = JSON.parse(fs.readFileSync(dataPath, 'utf-8'));
+
+    // Type-aware: for seed-based scenarios, only intercept externalApis
+    const scenarioType = active.type || rawData.type || null;
+    let mockData;
+
+    if ((scenarioType === 'application' || scenarioType === 'user') && rawData.seed) {
+      if (rawData.externalApis && typeof rawData.externalApis === 'object') {
+        mockData = { routes: rawData.externalApis };
+      } else {
+        mockData = null;
+      }
+    } else {
+      mockData = rawData;
+    }
+
+    scenarioCache = { data: mockData, timestamp: now };
+    return mockData;
+  } catch {
+    scenarioCache = { data: null, timestamp: now };
+    return null;
+  }
+}
+
+/**
+ * Match a request URL against scenario mock data routes.
+ * Mirrors the logic in editorPreloadHelpers.ts matchScenarioRoute().
+ *
+ * Supports:
+ * - Path-only routes:    "/api/drinks"
+ * - Full URL routes:     "https://api.stripe.com/v1/charges"
+ * - Method-prefixed:     "POST /api/drinks"
+ */
+function matchScenarioRoute(requestUrl, method, scenarioData) {
+  const routes = scenarioData.routes;
+  if (!routes || typeof routes !== 'object') return null;
+
+  let pathname;
+  try {
+    pathname = new URL(requestUrl).pathname;
+  } catch {
+    pathname = requestUrl.split('?')[0];
+  }
+
+  const upperMethod = method.toUpperCase();
+
+  for (const [routeKey, routeValue] of Object.entries(routes)) {
+    if (!routeValue || typeof routeValue !== 'object') continue;
+
+    // Method-prefixed: "POST /api/drinks"
+    const spaceIdx = routeKey.indexOf(' ');
+    if (spaceIdx !== -1) {
+      const routeMethod = routeKey.slice(0, spaceIdx).toUpperCase();
+      const routePath = routeKey.slice(spaceIdx + 1);
+      if (routeMethod === upperMethod && routePath === pathname) {
+        return {
+          body: routeValue.body ?? routeValue,
+          status: typeof routeValue.status === 'number' ? routeValue.status : 200,
+        };
+      }
+      continue;
+    }
+
+    // Full URL: "https://api.stripe.com/v1/charges"
+    if (routeKey.startsWith('http://') || routeKey.startsWith('https://')) {
+      try {
+        const routeParsed = new URL(routeKey);
+        const reqParsed = new URL(requestUrl);
+        if (
+          routeParsed.hostname === reqParsed.hostname &&
+          routeParsed.port === reqParsed.port &&
+          routeParsed.pathname === reqParsed.pathname
+        ) {
+          return {
+            body: routeValue.body ?? routeValue,
+            status: typeof routeValue.status === 'number' ? routeValue.status : 200,
+          };
+        }
+      } catch {
+        // Invalid URL in route key — skip
+      }
+      continue;
+    }
+
+    // Path-only: "/api/drinks"
+    if (routeKey === pathname) {
+      return {
+        body: routeValue.body ?? routeValue,
+        status: typeof routeValue.status === 'number' ? routeValue.status : 200,
+      };
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Create a mock http.ClientRequest that delivers a mock response.
+ * Used to intercept http.request/http.get calls for matched routes.
+ */
+function createMockRequest(mockResult, callback) {
+  const body = JSON.stringify(mockResult.body);
+
+  const mockResponse = new Readable({
+    read() {
+      this.push(body);
+      this.push(null);
+    },
+  });
+  mockResponse.statusCode = mockResult.status;
+  mockResponse.statusMessage = 'OK';
+  mockResponse.headers = { 'content-type': 'application/json', 'x-codeyam-mock': 'preload' };
+  mockResponse.rawHeaders = ['content-type', 'application/json', 'x-codeyam-mock', 'preload'];
+
+  const mockReq = new EventEmitter();
+  mockReq.write = () => true;
+  mockReq.end = () => {
+    process.nextTick(() => {
+      if (callback) callback(mockResponse);
+      mockReq.emit('response', mockResponse);
+    });
+  };
+  mockReq.abort = () => {};
+  mockReq.destroy = () => {};
+  mockReq.setTimeout = () => mockReq;
+  mockReq.setNoDelay = () => {};
+  mockReq.setSocketKeepAlive = () => {};
+
+  return mockReq;
+}
+
+/**
+ * Check if a URL string targets the dev server and return the redirected URL.
+ */
+function redirectUrl(urlStr, config) {
+  try {
+    const parsed = new URL(urlStr);
+    const dev = new URL(config.devServerUrl);
+    const proxy = new URL(config.proxyUrl);
+
+    if (parsed.hostname === dev.hostname && parsed.port === dev.port) {
+      parsed.hostname = proxy.hostname;
+      parsed.port = proxy.port;
+      parsed.protocol = proxy.protocol;
+      return parsed.toString();
+    }
+  } catch {
+    // Invalid URL — pass through
+  }
+  return urlStr;
+}
+
+/**
+ * Check if http.request options target the dev server and patch them.
+ */
+function redirectOptions(options, config) {
+  const dev = new URL(config.devServerUrl);
+  const proxy = new URL(config.proxyUrl);
+  const devPort = parseInt(dev.port, 10);
+  const proxyPort = parseInt(proxy.port, 10);
+
+  if (options.hostname && parseInt(String(options.port), 10) === devPort && options.hostname === dev.hostname) {
+    return {
+      ...options,
+      hostname: proxy.hostname,
+      port: proxyPort,
+      headers: options.headers
+        ? { ...options.headers, host: `${proxy.hostname}:${proxyPort}` }
+        : options.headers,
+    };
+  }
+
+  if (options.host) {
+    const expectedHost = `${dev.hostname}:${devPort}`;
+    if (options.host === expectedHost) {
+      return {
+        ...options,
+        host: `${proxy.hostname}:${proxyPort}`,
+      };
+    }
+  }
+
+  return options;
+}
+
+// --- Patch global fetch ---
+
+const originalFetch = globalThis.fetch;
+
+if (originalFetch) {
+  globalThis.fetch = function patchedFetch(input, init) {
+    // Try direct mock matching first (handles both internal and external URLs)
+    const scenarioData = readScenarioData();
+    if (scenarioData) {
+      let url = '';
+      let method = (init?.method || 'GET').toUpperCase();
+
+      if (typeof input === 'string') {
+        url = input;
+      } else if (input instanceof URL) {
+        url = input.toString();
+      } else if (input instanceof Request) {
+        url = input.url;
+        method = input.method || method;
+      }
+
+      if (url) {
+        const match = matchScenarioRoute(url, method, scenarioData);
+        if (match) {
+          return Promise.resolve(
+            new Response(JSON.stringify(match.body), {
+              status: match.status,
+              headers: {
+                'Content-Type': 'application/json',
+                'X-CodeYam-Mock': 'preload',
+              },
+            }),
+          );
+        }
+      }
+    }
+
+    // Fall back to proxy redirect for dev-server-targeting requests
+    const config = getProxyConfig();
+    if (!config) return originalFetch.call(this, input, init);
+
+    if (typeof input === 'string') {
+      const redirected = redirectUrl(input, config);
+      if (redirected !== input) {
+        return originalFetch.call(this, redirected, init);
+      }
+    } else if (input instanceof URL) {
+      const redirected = redirectUrl(input.toString(), config);
+      if (redirected !== input.toString()) {
+        return originalFetch.call(this, redirected, init);
+      }
+    } else if (input instanceof Request) {
+      const redirected = redirectUrl(input.url, config);
+      if (redirected !== input.url) {
+        return originalFetch.call(this, new Request(redirected, input), init);
+      }
+    }
+
+    return originalFetch.call(this, input, init);
+  };
+}
+
+// --- Patch http.request and https.request ---
+
+function patchModule(mod, protocol) {
+  const originalRequest = mod.request;
+  const originalGet = mod.get;
+
+  mod.request = function patchedRequest(urlOrOptions, optionsOrCallback, callback) {
+    // Try direct mock matching first (handles external API calls from SSR)
+    const scenarioData = readScenarioData();
+    if (scenarioData) {
+      let urlStr = '';
+      let method = 'GET';
+      let cb = typeof optionsOrCallback === 'function' ? optionsOrCallback : callback;
+
+      if (typeof urlOrOptions === 'string' || urlOrOptions instanceof URL) {
+        urlStr = typeof urlOrOptions === 'string' ? urlOrOptions : urlOrOptions.toString();
+        if (typeof optionsOrCallback === 'object' && optionsOrCallback !== null) {
+          method = optionsOrCallback.method || method;
+        }
+      } else if (urlOrOptions && typeof urlOrOptions === 'object') {
+        // Build URL from options
+        const hostname = urlOrOptions.hostname || urlOrOptions.host?.split(':')[0] || 'localhost';
+        const port = urlOrOptions.port || (protocol === 'https' ? 443 : 80);
+        const urlPath = urlOrOptions.path || '/';
+        urlStr = `${protocol}://${hostname}:${port}${urlPath}`;
+        method = urlOrOptions.method || method;
+        cb = typeof optionsOrCallback === 'function' ? optionsOrCallback : callback;
+      }
+
+      if (urlStr) {
+        const match = matchScenarioRoute(urlStr, method, scenarioData);
+        if (match) {
+          return createMockRequest(match, cb);
+        }
+      }
+    }
+
+    // Fall back to proxy redirect for dev-server-targeting requests
+    const config = getProxyConfig();
+    if (!config) return originalRequest.call(this, urlOrOptions, optionsOrCallback, callback);
+
+    // http.request(url, options, callback)
+    if (typeof urlOrOptions === 'string' || urlOrOptions instanceof URL) {
+      const urlStr = typeof urlOrOptions === 'string' ? urlOrOptions : urlOrOptions.toString();
+      const redirected = redirectUrl(urlStr, config);
+      if (redirected !== urlStr) {
+        // Redirect to proxy — force http since proxy is always http
+        return http.request.call(http, redirected, optionsOrCallback, callback);
+      }
+      return originalRequest.call(this, urlOrOptions, optionsOrCallback, callback);
+    }
+
+    // http.request(options, callback)
+    const redirected = redirectOptions(urlOrOptions, config);
+    if (redirected !== urlOrOptions) {
+      return http.request.call(http, redirected, optionsOrCallback);
+    }
+    return originalRequest.call(this, urlOrOptions, optionsOrCallback, callback);
+  };
+
+  mod.get = function patchedGet(urlOrOptions, optionsOrCallback, callback) {
+    const req = mod.request(urlOrOptions, optionsOrCallback, callback);
+    req.end();
+    return req;
+  };
+}
+
+patchModule(http, 'http');
+patchModule(https, 'https');

package/codeyam-cli/src/webserver/scripts/journalCapture.ts

@@ -7,7 +7,90 @@
  * Args: { url, outputPath, viewportWidth?, viewportHeight? }
  */
 
-import { chromium } from 'playwright';
+import { chromium, Page } from 'playwright';
+
+/**
+ * Wait until DOM mutations stop for `quietMs`, or until `maxWaitMs` elapsed.
+ * Uses MutationObserver to detect when React has finished rendering,
+ * including data-driven re-renders from useEffect fetches.
+ *
+ * Same pattern used in takeElementScreenshot.ts (background package).
+ *
+ * Uses string-based evaluate to avoid tsx/esbuild __name decorator issues
+ * when the function is serialized for browser execution.
+ */
+async function waitForDomIdle(
+  page: Page,
+  quietMs: number,
+  maxWaitMs: number,
+): Promise<void> {
+  await page.evaluate(`
+    new Promise(function(resolve) {
+      var timer;
+      var startTime = Date.now();
+
+      function finish() {
+        clearTimeout(timer);
+        observer.disconnect();
+        resolve();
+      }
+
+      function reset() {
+        clearTimeout(timer);
+        if (Date.now() - startTime >= ${maxWaitMs}) { finish(); return; }
+        timer = setTimeout(finish, ${quietMs});
+      }
+
+      var observer = new MutationObserver(reset);
+      observer.observe(document, {
+        childList: true,
+        subtree: true,
+        attributes: true
+      });
+
+      setTimeout(finish, ${maxWaitMs});
+      reset();
+    })
+  `);
+}
+
+/**
+ * Stabilize the page before taking a screenshot.
+ *
+ * Handles client-side rendered content (React useEffect + fetch pattern):
+ * 1. Wait for DOM to settle after initial hydration
+ * 2. Wait for any network requests triggered by client JS (useEffect fetches)
+ * 3. Wait for DOM to settle again after data-driven re-renders
+ * 4. Wait for web fonts to load
+ */
+async function stabilizePage(page: Page): Promise<void> {
+  // 1. Wait for React hydration / initial client rendering to settle
+  await waitForDomIdle(page, DOM_IDLE_QUIET_MS, DOM_IDLE_MAX_WAIT_MS);
+
+  // 2. Wait for any network requests triggered by client-side JS
+  //    (e.g. useEffect → fetch('/api/data')). This catches in-flight requests
+  //    that started during or after hydration.
+  try {
+    await page.waitForLoadState('networkidle', {
+      timeout: POST_LOAD_NETWORK_TIMEOUT_MS,
+    });
+  } catch {
+    // Timeout is OK — some pages have persistent connections (WebSocket, SSE)
+  }
+
+  // 3. Wait for data-driven re-renders to settle
+  await waitForDomIdle(page, DOM_IDLE_QUIET_MS, DOM_IDLE_MAX_WAIT_MS);
+
+  // 4. Wait for web fonts (string-based to avoid tsx __name decorator issues)
+  try {
+    await page.waitForFunction(
+      `document.fonts ? document.fonts.status === 'loaded' : true`,
+      { timeout: FONTS_TIMEOUT_MS },
+    );
+  } catch {
+    // Font loading timeout — continue anyway
+  }
+}
 
 interface CaptureArgs {
   url: string;
@@ -21,7 +104,10 @@ interface CaptureArgs {
 
 const MAX_RETRIES = 2;
 const GOTO_TIMEOUT_MS = 15_000;
-const SETTLE_DELAY_MS = 500;
+const DOM_IDLE_QUIET_MS = 500;
+const DOM_IDLE_MAX_WAIT_MS = 5_000;
+const POST_LOAD_NETWORK_TIMEOUT_MS = 5_000;
+const FONTS_TIMEOUT_MS = 5_000;
 
 async function main() {
   const args = process.argv.slice(2);
@@ -84,8 +170,12 @@ async function main() {
           );
         }
 
-        // Brief settle time for any client-side rendering/animations
-        await page.waitForTimeout(SETTLE_DELAY_MS);
+        // Wait for client-side rendering to complete (React hydration,
+        // useEffect data fetches, font loading, etc.)
+        await stabilizePage(page);
+
+        const pageTitle = await page.title();
+        console.log(`[JournalCapture] Page title: ${pageTitle}`);
 
         if (selector) {
           // Element-level screenshot — captures just the wrapper element

package/codeyam-cli/src/webserver/server.js

@@ -199,11 +199,13 @@ async function checkPreviewHealth(url) {
         };
     }
 }
-app.post('/api/dev-mode-refresh', async (_req, res) => {
+// Unified preview control — refresh, navigate, switch scenario, or any combination.
+// Handles log clearing, HMR settle, SSR health check, scenario switching, and navigation.
+app.post('/api/dev-mode-preview', express.json(), async (req, res) => {
+    const { scenarioSlug, scenarioId, path: navPath } = req.body || {};
     let sessionsNotified = 0;
     let logCleared = false;
-    let previewUrl = null;
-    // Read config and resolve log path
+    // 1. Read config and resolve log path
     let logPath = null;
     try {
         const configPath = path.join(process.cwd(), '.codeyam', 'config.json');
@@ -215,19 +217,69 @@ app.post('/api/dev-mode-refresh', async (_req, res) => {
     catch {
         // Config not available — not critical
     }
-    // Extract preview URL from log BEFORE clearing it
+    // 2. Extract preview URL from log BEFORE clearing it
+    let previewUrl = null;
     if (logPath && fs.existsSync(logPath)) {
         previewUrl = getPreviewUrl(logPath);
     }
-    // Broadcast WebSocket refresh to browser (iframe starts reloading)
+    // Fall back to cached URL if log didn't have it
+    if (!previewUrl)
+        previewUrl = cachedPreviewUrl;
+    // 3. Switch scenario if provided (accepts scenarioId alone or scenarioSlug+scenarioId)
+    const effectiveScenarioId = scenarioId || null;
+    let effectiveSlug = scenarioSlug || null;
+    let scenarioUrl = null;
+    // Look up the scenario's URL from the database so we can navigate the preview
+    if (effectiveScenarioId && !navPath) {
+        try {
+            const { getDatabase } = await import('../../../packages/database/index.js');
+            const db = getDatabase();
+            const row = await db
+                .selectFrom('editor_scenarios')
+                .select(['url'])
+                .where('id', '=', effectiveScenarioId)
+                .executeTakeFirst();
+            if (row?.url) {
+                scenarioUrl = row.url;
+            }
+        }
+        catch {
+            // DB not available — non-critical
+        }
+    }
+    // Switch active scenario using the shared logic that handles type detection,
+    // active-scenario.json writes, and seed adapter execution for app/user scenarios.
+    if (effectiveScenarioId) {
+        try {
+            const { switchActiveScenario } = await import('../utils/editorScenarioSwitch.js');
+            const projectRoot = process.env.CODEYAM_ROOT_PATH || process.cwd();
+            const switchResult = await switchActiveScenario({
+                scenarioId: effectiveScenarioId,
+                scenarioSlug: effectiveSlug || undefined,
+                projectRoot,
+            });
+            effectiveSlug = switchResult.scenarioSlug;
+            // Invalidate proxy cache so it picks up new scenario data
+            const { invalidateScenarioCache } = await import('./editorProxy.js');
+            invalidateScenarioCache();
+        }
+        catch {
+            // Scenario switch failed — not critical, continue with refresh
+        }
+    }
+    // 4. Broadcast WebSocket refresh to browser (iframe starts reloading)
+    // Use explicit path, or fall back to scenario URL from DB, so the preview
+    // navigates to the correct page (e.g. /drinks/1 for a detail page scenario).
     try {
         const { broadcastPreviewRefresh } = await import('./terminalServer.js');
-        sessionsNotified = broadcastPreviewRefresh();
+        const { resolvePreviewNavPath } = await import('../utils/editorScenarios.js');
+        const effectivePath = resolvePreviewNavPath(navPath, scenarioUrl);
+        sessionsNotified = broadcastPreviewRefresh(effectivePath, effectiveScenarioId);
     }
     catch {
         // Terminal server not loaded yet — ignore
     }
-    // Clear the server log so post-refresh log reads only show fresh output
+    // 5. Clear the server log so post-refresh log reads only show fresh output
     if (logPath && fs.existsSync(logPath)) {
         try {
             fs.writeFileSync(logPath, '');
@@ -237,12 +289,23 @@ app.post('/api/dev-mode-refresh', async (_req, res) => {
             // Log file not writable — not critical
         }
     }
-    // Wait for Vite HMR to settle and the page to re-render
+    // 6. Wait for HMR to settle and the page to re-render
     await new Promise((resolve) => setTimeout(resolve, 2000));
-    // Fetch the preview URL to detect SSR errors
+    // 7. Compute the full URL (with navigation path if provided)
+    let fullUrl = previewUrl;
+    if (navPath && previewUrl) {
+        try {
+            fullUrl = new URL(navPath, previewUrl).href;
+        }
+        catch {
+            fullUrl = previewUrl;
+        }
+    }
+    // 8. Fetch the preview URL to detect SSR errors
     let preview = null;
-    if (previewUrl) {
-        preview = await checkPreviewHealth(previewUrl);
+    const healthCheckUrl = fullUrl || previewUrl;
+    if (healthCheckUrl) {
+        preview = await checkPreviewHealth(healthCheckUrl);
         // If unhealthy, write error to log so Claude can see it via tail
         if (!preview.healthy && preview.error && logPath) {
             try {
@@ -257,7 +320,25 @@ app.post('/api/dev-mode-refresh', async (_req, res) => {
             }
         }
     }
-    res.json({ success: true, sessionsNotified, logCleared, preview });
+    res.json({
+        success: true,
+        sessionsNotified,
+        logCleared,
+        url: fullUrl || null,
+        preview,
+    });
+});
+// Show results panel in the editor Build tab — triggered by Claude at end of step 12.
+app.post('/api/editor-show-results', async (_req, res) => {
+    const { broadcastShowResults } = await import('./terminalServer.js');
+    const sessionsNotified = broadcastShowResults();
+    res.json({ success: true, sessionsNotified });
+});
+// Hide results panel — triggered by Claude before making changes.
+app.post('/api/editor-hide-results', async (_req, res) => {
+    const { broadcastHideResults } = await import('./terminalServer.js');
+    const sessionsNotified = broadcastHideResults();
+    res.json({ success: true, sessionsNotified });
 });
 // Handle Remix routes (API routes, page routes, etc.)
 if (process.env.NODE_ENV === 'development') {