@codeyam/codeyam-cli 0.1.0-staging.a890816 → 0.1.0-staging.ac9ca58

package/codeyam-cli/src/webserver/scripts/codeyam-preload.mjs

@@ -0,0 +1,414 @@
+/**
+ * Node.js preload module for CodeYam editor proxy.
+ *
+ * Loaded via NODE_OPTIONS='--import /path/to/codeyam-preload.mjs' in the
+ * dev server process. Patches global fetch and http/https.request to:
+ *
+ * 1. **Direct mock matching** — reads the active scenario data from disk and
+ *    returns mock responses for ANY matching URL (internal API routes, external
+ *    APIs like Stripe/weather, etc.) without a network round-trip.
+ *
+ * 2. **Proxy redirect** — for dev-server-targeting requests that don't match
+ *    a mock route, redirects them through the CodeYam proxy.
+ *
+ * Must be .mjs (native ESM) because --import requires it.
+ * No TypeScript, no build step, no dependencies.
+ */
+
+import http from 'node:http';
+import https from 'node:https';
+import fs from 'node:fs';
+import path from 'node:path';
+import { Readable } from 'node:stream';
+import { EventEmitter } from 'node:events';
+
+let proxyConfig = null;
+let configLoaded = false;
+
+/**
+ * Lazily load the proxy config from .codeyam/proxy-config.json.
+ * Returns null if the file doesn't exist yet (proxy still starting).
+ */
+function getProxyConfig() {
+  if (configLoaded && proxyConfig) return proxyConfig;
+
+  // Try env var first (most reliable)
+  const proxyUrl = process.env.CODEYAM_PROXY_URL;
+  const devServerUrl = process.env.CODEYAM_DEV_SERVER_URL;
+  if (proxyUrl && devServerUrl) {
+    proxyConfig = { proxyUrl, devServerUrl };
+    configLoaded = true;
+    return proxyConfig;
+  }
+
+  // Fall back to config file
+  try {
+    const cwd = process.cwd();
+    const configPath = path.join(cwd, '.codeyam', 'proxy-config.json');
+    if (fs.existsSync(configPath)) {
+      const raw = fs.readFileSync(configPath, 'utf-8');
+      proxyConfig = JSON.parse(raw);
+      configLoaded = true;
+      return proxyConfig;
+    }
+  } catch {
+    // File doesn't exist yet or is invalid — that's fine
+  }
+
+  return null;
+}
+
+// --- Scenario data for direct mock matching (SSR + external APIs) ---
+
+let scenarioCache = { data: null, timestamp: 0 };
+const SCENARIO_CACHE_TTL = 500;
+
+/**
+ * Read the active scenario's mock data from disk, with brief caching.
+ * Returns null if no active scenario or data file is missing.
+ *
+ * For application/user scenarios (seed-based): only returns externalApis
+ * as routes. DB-backed routes are served by the real app from seeded data.
+ */
+function readScenarioData() {
+  const now = Date.now();
+  if (scenarioCache.data !== null && now - scenarioCache.timestamp < SCENARIO_CACHE_TTL) {
+    return scenarioCache.data;
+  }
+
+  try {
+    const cwd = process.cwd();
+    const activePath = path.join(cwd, '.codeyam', 'active-scenario.json');
+    if (!fs.existsSync(activePath)) {
+      scenarioCache = { data: null, timestamp: now };
+      return null;
+    }
+
+    const active = JSON.parse(fs.readFileSync(activePath, 'utf-8'));
+    if (!active.scenarioId) {
+      scenarioCache = { data: null, timestamp: now };
+      return null;
+    }
+
+    const dataPath = path.join(cwd, '.codeyam', 'editor-scenarios', `${active.scenarioId}.json`);
+    if (!fs.existsSync(dataPath)) {
+      scenarioCache = { data: null, timestamp: now };
+      return null;
+    }
+
+    const rawData = JSON.parse(fs.readFileSync(dataPath, 'utf-8'));
+
+    // Type-aware: for seed-based scenarios, only intercept externalApis
+    const scenarioType = active.type || rawData.type || null;
+    let mockData;
+
+    if ((scenarioType === 'application' || scenarioType === 'user') && rawData.seed) {
+      if (rawData.externalApis && typeof rawData.externalApis === 'object') {
+        mockData = { routes: rawData.externalApis };
+      } else {
+        mockData = null;
+      }
+    } else {
+      mockData = rawData;
+    }
+
+    scenarioCache = { data: mockData, timestamp: now };
+    return mockData;
+  } catch {
+    scenarioCache = { data: null, timestamp: now };
+    return null;
+  }
+}
+
+/**
+ * Match a request URL against scenario mock data routes.
+ * Mirrors the logic in editorPreloadHelpers.ts matchScenarioRoute().
+ *
+ * Supports:
+ * - Path-only routes:    "/api/drinks"
+ * - Full URL routes:     "https://api.stripe.com/v1/charges"
+ * - Method-prefixed:     "POST /api/drinks"
+ */
+function matchScenarioRoute(requestUrl, method, scenarioData) {
+  const routes = scenarioData.routes;
+  if (!routes || typeof routes !== 'object') return null;
+
+  let pathname;
+  try {
+    pathname = new URL(requestUrl).pathname;
+  } catch {
+    pathname = requestUrl.split('?')[0];
+  }
+
+  const upperMethod = method.toUpperCase();
+
+  for (const [routeKey, routeValue] of Object.entries(routes)) {
+    if (!routeValue || typeof routeValue !== 'object') continue;
+
+    // Method-prefixed: "POST /api/drinks"
+    const spaceIdx = routeKey.indexOf(' ');
+    if (spaceIdx !== -1) {
+      const routeMethod = routeKey.slice(0, spaceIdx).toUpperCase();
+      const routePath = routeKey.slice(spaceIdx + 1);
+      if (routeMethod === upperMethod && routePath === pathname) {
+        return {
+          body: routeValue.body ?? routeValue,
+          status: typeof routeValue.status === 'number' ? routeValue.status : 200,
+        };
+      }
+      continue;
+    }
+
+    // Full URL: "https://api.stripe.com/v1/charges"
+    if (routeKey.startsWith('http://') || routeKey.startsWith('https://')) {
+      try {
+        const routeParsed = new URL(routeKey);
+        const reqParsed = new URL(requestUrl);
+        if (
+          routeParsed.hostname === reqParsed.hostname &&
+          routeParsed.port === reqParsed.port &&
+          routeParsed.pathname === reqParsed.pathname
+        ) {
+          return {
+            body: routeValue.body ?? routeValue,
+            status: typeof routeValue.status === 'number' ? routeValue.status : 200,
+          };
+        }
+      } catch {
+        // Invalid URL in route key — skip
+      }
+      continue;
+    }
+
+    // Path-only: "/api/drinks"
+    if (routeKey === pathname) {
+      return {
+        body: routeValue.body ?? routeValue,
+        status: typeof routeValue.status === 'number' ? routeValue.status : 200,
+      };
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Create a mock http.ClientRequest that delivers a mock response.
+ * Used to intercept http.request/http.get calls for matched routes.
+ */
+function createMockRequest(mockResult, callback) {
+  const body = JSON.stringify(mockResult.body);
+
+  const mockResponse = new Readable({
+    read() {
+      this.push(body);
+      this.push(null);
+    },
+  });
+  mockResponse.statusCode = mockResult.status;
+  mockResponse.statusMessage = 'OK';
+  mockResponse.headers = { 'content-type': 'application/json', 'x-codeyam-mock': 'preload' };
+  mockResponse.rawHeaders = ['content-type', 'application/json', 'x-codeyam-mock', 'preload'];
+
+  const mockReq = new EventEmitter();
+  mockReq.write = () => true;
+  mockReq.end = () => {
+    process.nextTick(() => {
+      if (callback) callback(mockResponse);
+      mockReq.emit('response', mockResponse);
+    });
+  };
+  mockReq.abort = () => {};
+  mockReq.destroy = () => {};
+  mockReq.setTimeout = () => mockReq;
+  mockReq.setNoDelay = () => {};
+  mockReq.setSocketKeepAlive = () => {};
+
+  return mockReq;
+}
+
+/**
+ * Check if a URL string targets the dev server and return the redirected URL.
+ */
+function redirectUrl(urlStr, config) {
+  try {
+    const parsed = new URL(urlStr);
+    const dev = new URL(config.devServerUrl);
+    const proxy = new URL(config.proxyUrl);
+
+    if (parsed.hostname === dev.hostname && parsed.port === dev.port) {
+      parsed.hostname = proxy.hostname;
+      parsed.port = proxy.port;
+      parsed.protocol = proxy.protocol;
+      return parsed.toString();
+    }
+  } catch {
+    // Invalid URL — pass through
+  }
+  return urlStr;
+}
+
+/**
+ * Check if http.request options target the dev server and patch them.
+ */
+function redirectOptions(options, config) {
+  const dev = new URL(config.devServerUrl);
+  const proxy = new URL(config.proxyUrl);
+  const devPort = parseInt(dev.port, 10);
+  const proxyPort = parseInt(proxy.port, 10);
+
+  if (options.hostname && parseInt(String(options.port), 10) === devPort && options.hostname === dev.hostname) {
+    return {
+      ...options,
+      hostname: proxy.hostname,
+      port: proxyPort,
+      headers: options.headers
+        ? { ...options.headers, host: `${proxy.hostname}:${proxyPort}` }
+        : options.headers,
+    };
+  }
+
+  if (options.host) {
+    const expectedHost = `${dev.hostname}:${devPort}`;
+    if (options.host === expectedHost) {
+      return {
+        ...options,
+        host: `${proxy.hostname}:${proxyPort}`,
+      };
+    }
+  }
+
+  return options;
+}
+
+// --- Patch global fetch ---
+
+const originalFetch = globalThis.fetch;
+
+if (originalFetch) {
+  globalThis.fetch = function patchedFetch(input, init) {
+    // Try direct mock matching first (handles both internal and external URLs)
+    const scenarioData = readScenarioData();
+    if (scenarioData) {
+      let url = '';
+      let method = (init?.method || 'GET').toUpperCase();
+
+      if (typeof input === 'string') {
+        url = input;
+      } else if (input instanceof URL) {
+        url = input.toString();
+      } else if (input instanceof Request) {
+        url = input.url;
+        method = input.method || method;
+      }
+
+      if (url) {
+        const match = matchScenarioRoute(url, method, scenarioData);
+        if (match) {
+          return Promise.resolve(
+            new Response(JSON.stringify(match.body), {
+              status: match.status,
+              headers: {
+                'Content-Type': 'application/json',
+                'X-CodeYam-Mock': 'preload',
+              },
+            }),
+          );
+        }
+      }
+    }
+
+    // Fall back to proxy redirect for dev-server-targeting requests
+    const config = getProxyConfig();
+    if (!config) return originalFetch.call(this, input, init);
+
+    if (typeof input === 'string') {
+      const redirected = redirectUrl(input, config);
+      if (redirected !== input) {
+        return originalFetch.call(this, redirected, init);
+      }
+    } else if (input instanceof URL) {
+      const redirected = redirectUrl(input.toString(), config);
+      if (redirected !== input.toString()) {
+        return originalFetch.call(this, redirected, init);
+      }
+    } else if (input instanceof Request) {
+      const redirected = redirectUrl(input.url, config);
+      if (redirected !== input.url) {
+        return originalFetch.call(this, new Request(redirected, input), init);
+      }
+    }
+
+    return originalFetch.call(this, input, init);
+  };
+}
+
+// --- Patch http.request and https.request ---
+
+function patchModule(mod, protocol) {
+  const originalRequest = mod.request;
+  const originalGet = mod.get;
+
+  mod.request = function patchedRequest(urlOrOptions, optionsOrCallback, callback) {
+    // Try direct mock matching first (handles external API calls from SSR)
+    const scenarioData = readScenarioData();
+    if (scenarioData) {
+      let urlStr = '';
+      let method = 'GET';
+      let cb = typeof optionsOrCallback === 'function' ? optionsOrCallback : callback;
+
+      if (typeof urlOrOptions === 'string' || urlOrOptions instanceof URL) {
+        urlStr = typeof urlOrOptions === 'string' ? urlOrOptions : urlOrOptions.toString();
+        if (typeof optionsOrCallback === 'object' && optionsOrCallback !== null) {
+          method = optionsOrCallback.method || method;
+        }
+      } else if (urlOrOptions && typeof urlOrOptions === 'object') {
+        // Build URL from options
+        const hostname = urlOrOptions.hostname || urlOrOptions.host?.split(':')[0] || 'localhost';
+        const port = urlOrOptions.port || (protocol === 'https' ? 443 : 80);
+        const urlPath = urlOrOptions.path || '/';
+        urlStr = `${protocol}://${hostname}:${port}${urlPath}`;
+        method = urlOrOptions.method || method;
+        cb = typeof optionsOrCallback === 'function' ? optionsOrCallback : callback;
+      }
+
+      if (urlStr) {
+        const match = matchScenarioRoute(urlStr, method, scenarioData);
+        if (match) {
+          return createMockRequest(match, cb);
+        }
+      }
+    }
+
+    // Fall back to proxy redirect for dev-server-targeting requests
+    const config = getProxyConfig();
+    if (!config) return originalRequest.call(this, urlOrOptions, optionsOrCallback, callback);
+
+    // http.request(url, options, callback)
+    if (typeof urlOrOptions === 'string' || urlOrOptions instanceof URL) {
+      const urlStr = typeof urlOrOptions === 'string' ? urlOrOptions : urlOrOptions.toString();
+      const redirected = redirectUrl(urlStr, config);
+      if (redirected !== urlStr) {
+        // Redirect to proxy — force http since proxy is always http
+        return http.request.call(http, redirected, optionsOrCallback, callback);
+      }
+      return originalRequest.call(this, urlOrOptions, optionsOrCallback, callback);
+    }
+
+    // http.request(options, callback)
+    const redirected = redirectOptions(urlOrOptions, config);
+    if (redirected !== urlOrOptions) {
+      return http.request.call(http, redirected, optionsOrCallback);
+    }
+    return originalRequest.call(this, urlOrOptions, optionsOrCallback, callback);
+  };
+
+  mod.get = function patchedGet(urlOrOptions, optionsOrCallback, callback) {
+    const req = mod.request(urlOrOptions, optionsOrCallback, callback);
+    req.end();
+    return req;
+  };
+}
+
+patchModule(http, 'http');
+patchModule(https, 'https');

package/codeyam-cli/src/webserver/scripts/journalCapture.ts

@@ -7,7 +7,90 @@
  * Args: { url, outputPath, viewportWidth?, viewportHeight? }
  */
 
-import { chromium } from 'playwright';
+import { chromium, Page } from 'playwright';
+
+/**
+ * Wait until DOM mutations stop for `quietMs`, or until `maxWaitMs` elapsed.
+ * Uses MutationObserver to detect when React has finished rendering,
+ * including data-driven re-renders from useEffect fetches.
+ *
+ * Same pattern used in takeElementScreenshot.ts (background package).
+ *
+ * Uses string-based evaluate to avoid tsx/esbuild __name decorator issues
+ * when the function is serialized for browser execution.
+ */
+async function waitForDomIdle(
+  page: Page,
+  quietMs: number,
+  maxWaitMs: number,
+): Promise<void> {
+  await page.evaluate(`
+    new Promise(function(resolve) {
+      var timer;
+      var startTime = Date.now();
+
+      function finish() {
+        clearTimeout(timer);
+        observer.disconnect();
+        resolve();
+      }
+
+      function reset() {
+        clearTimeout(timer);
+        if (Date.now() - startTime >= ${maxWaitMs}) { finish(); return; }
+        timer = setTimeout(finish, ${quietMs});
+      }
+
+      var observer = new MutationObserver(reset);
+      observer.observe(document, {
+        childList: true,
+        subtree: true,
+        attributes: true
+      });
+
+      setTimeout(finish, ${maxWaitMs});
+      reset();
+    })
+  `);
+}
+
+/**
+ * Stabilize the page before taking a screenshot.
+ *
+ * Handles client-side rendered content (React useEffect + fetch pattern):
+ * 1. Wait for DOM to settle after initial hydration
+ * 2. Wait for any network requests triggered by client JS (useEffect fetches)
+ * 3. Wait for DOM to settle again after data-driven re-renders
+ * 4. Wait for web fonts to load
+ */
+async function stabilizePage(page: Page): Promise<void> {
+  // 1. Wait for React hydration / initial client rendering to settle
+  await waitForDomIdle(page, DOM_IDLE_QUIET_MS, DOM_IDLE_MAX_WAIT_MS);
+
+  // 2. Wait for any network requests triggered by client-side JS
+  //    (e.g. useEffect → fetch('/api/data')). This catches in-flight requests
+  //    that started during or after hydration.
+  try {
+    await page.waitForLoadState('networkidle', {
+      timeout: POST_LOAD_NETWORK_TIMEOUT_MS,
+    });
+  } catch {
+    // Timeout is OK — some pages have persistent connections (WebSocket, SSE)
+  }
+
+  // 3. Wait for data-driven re-renders to settle
+  await waitForDomIdle(page, DOM_IDLE_QUIET_MS, DOM_IDLE_MAX_WAIT_MS);
+
+  // 4. Wait for web fonts (string-based to avoid tsx __name decorator issues)
+  try {
+    await page.waitForFunction(
+      `document.fonts ? document.fonts.status === 'loaded' : true`,
+      { timeout: FONTS_TIMEOUT_MS },
+    );
+  } catch {
+    // Font loading timeout — continue anyway
+  }
+}
 
 interface CaptureArgs {
   url: string;
@@ -21,7 +104,10 @@ interface CaptureArgs {
 
 const MAX_RETRIES = 2;
 const GOTO_TIMEOUT_MS = 15_000;
-const SETTLE_DELAY_MS = 500;
+const DOM_IDLE_QUIET_MS = 500;
+const DOM_IDLE_MAX_WAIT_MS = 5_000;
+const POST_LOAD_NETWORK_TIMEOUT_MS = 5_000;
+const FONTS_TIMEOUT_MS = 5_000;
 
 async function main() {
   const args = process.argv.slice(2);
@@ -65,6 +151,31 @@ async function main() {
       );
     });
 
+    // Capture non-2xx API responses with their body — these reveal
+    // the actual server error (e.g. "Collection not found") rather than
+    // just a generic 404 status on the page.
+    const apiErrorPromises: Promise<void>[] = [];
+    page.on('response', (resp) => {
+      const respUrl = resp.url();
+      if (resp.status() >= 400 && respUrl.includes('/api/')) {
+        const p = resp
+          .text()
+          .then((body) => {
+            const truncated =
+              body.length > 500 ? body.slice(0, 500) + '…' : body;
+            console.log(
+              `[JournalCapture] API response error: ${resp.request().method()} ${respUrl} → ${resp.status()} — ${truncated}`,
+            );
+          })
+          .catch(() => {
+            console.log(
+              `[JournalCapture] API response error: ${resp.request().method()} ${respUrl} → ${resp.status()} — (body unavailable)`,
+            );
+          });
+        apiErrorPromises.push(p);
+      }
+    });
+
     let lastError: Error | null = null;
 
     for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
@@ -82,10 +193,42 @@ async function main() {
           console.log(
             `[JournalCapture] Page loaded: status=${response.status()}, url=${response.url()}`,
           );
+          if (response.status() >= 400) {
+            console.log(
+              `[JournalCapture] HTTP error: status=${response.status()}, url=${response.url()}`,
+            );
+          }
+        }
+
+        // Wait for client-side rendering to complete (React hydration,
+        // useEffect data fetches, font loading, etc.)
+        await stabilizePage(page);
+
+        // Flush any pending API error response reads so they appear
+        // before the RESULT line in stdout
+        if (apiErrorPromises.length > 0) {
+          await Promise.all(apiErrorPromises);
         }
 
-        // Brief settle time for any client-side rendering/animations
-        await page.waitForTimeout(SETTLE_DELAY_MS);
+        const pageTitle = await page.title();
+        console.log(`[JournalCapture] Page title: ${pageTitle}`);
+
+        // Capture visible text for diagnostic output — helps debug when
+        // multiple scenarios produce identical screenshots
+        try {
+          const visibleText = await page.evaluate(`
+            (function() {
+              var el = document.getElementById('codeyam-capture') || document.body;
+              var text = el ? el.innerText : '';
+              return text.substring(0, 500).replace(/\\n+/g, ' | ').replace(/\\s+/g, ' ').trim();
+            })()
+          `);
+          console.log(
+            '[JournalCapture] Visible text: ' + (visibleText || '(empty page)'),
+          );
+        } catch (e) {
+          console.log('[JournalCapture] Visible text: (extraction failed)');
+        }
 
         if (selector) {
           // Element-level screenshot — captures just the wrapper element