jamdesk 1.0.13 → 1.0.14

package/vendored/lib/docs.ts

@@ -85,13 +85,29 @@ export function getImagesDir(): string {
   return path.join(process.cwd(), 'content', 'images');
 }
 
-// Cache the config to avoid re-parsing 368KB+ JSON for every page during SSR
+// Cache the config to avoid re-parsing 368KB+ JSON for every page during SSR.
+// In development, reads from the project source (not public/docs.json) and uses
+// mtime-based invalidation so docs.json edits are picked up on browser refresh
+// without writing to public/ (which triggers Turbopack recompilation hangs).
 let cachedConfig: DocsConfig | null = null;
+let cachedMtimeMs = 0;
+let deprecationWarningsLogged = false;
 
 export function getDocsConfig(): DocsConfig {
-  if (cachedConfig) return cachedConfig;
+  // In dev, read from project source so we pick up edits without touching public/
+  const isDev = process.env.NODE_ENV === 'development';
+  const docsPath = isDev ? getProjectDocsConfigPath() : getDocsConfigPath();
+
+  if (cachedConfig) {
+    if (isDev) {
+      // Check mtime to detect file changes (cheap stat call vs 368KB readFileSync)
+      const mtimeMs = fs.statSync(docsPath).mtimeMs;
+      if (mtimeMs === cachedMtimeMs) return cachedConfig;
+    } else {
+      return cachedConfig;
+    }
+  }
 
-  const docsPath = getDocsConfigPath();
   const fileContents = fs.readFileSync(docsPath, 'utf8');
   const rawConfig = JSON.parse(fileContents);
 
@@ -99,13 +115,15 @@ export function getDocsConfig(): DocsConfig {
   const { config, warnings } = normalizeConfig(rawConfig);
 
   // Log deprecation warnings (only once due to caching)
-  if (warnings.length > 0) {
+  if (warnings.length > 0 && !deprecationWarningsLogged) {
+    deprecationWarningsLogged = true;
     console.warn('\n⚠️  Deprecated docs.json fields detected:');
     warnings.forEach((w) => console.warn(`   - ${w}`));
     console.warn('');
   }
 
   cachedConfig = config;
+  cachedMtimeMs = fs.statSync(docsPath).mtimeMs;
   return cachedConfig;
 }
 

package/vendored/lib/embedding-chunker.ts

@@ -0,0 +1,173 @@
+/**
+ * Embedding Chunker
+ *
+ * Splits MDX page content into chunks suitable for embedding generation.
+ * Each chunk is ~2000 chars (~500 tokens), split at section boundaries
+ * first, then at sentence boundaries for oversized sections.
+ */
+import { extractSections } from './static-artifacts.js';
+
+export interface EmbeddingChunk {
+  /** Unique ID: `${pageSlug}#${sectionIndex}` */
+  id: string;
+  /** Page slug derived from file path (without extension) */
+  pageSlug: string;
+  /** Heading of the section this chunk belongs to */
+  sectionHeading: string;
+  /** Plain text content, stripped of markdown and JSX */
+  content: string;
+  /** Page title from frontmatter, or slug-derived fallback */
+  pageTitle: string;
+}
+
+/**
+ * Light markdown cleanup for embedding content.
+ * Keeps code blocks and inline code (critical for docs Q&A) but strips
+ * formatting noise (bold/italic markers, JSX tags, image syntax, frontmatter).
+ *
+ * Code blocks are protected from tag stripping to preserve JSX/HTML inside code.
+ * MDX component TAGS are stripped but their content is kept — so
+ * `<CodeGroup>```js\ncode\n```</CodeGroup>` preserves the code block.
+ */
+function stripForEmbedding(text: string): string {
+  // Protect fenced code blocks and inline code from tag stripping.
+  // Code can contain <Component> or <tag> syntax that should NOT be stripped.
+  const preserved: string[] = [];
+  let safe = text
+    .replace(/```[\s\S]*?```/g, (m) => {
+      preserved.push(m.replace(/```[^\n]*\n/g, '```\n')); // strip language hints + titles
+      return `\x00${preserved.length - 1}\x00`;
+    })
+    .replace(/`[^`\n]+`/g, (m) => {
+      preserved.push(m);
+      return `\x00${preserved.length - 1}\x00`;
+    });
+
+  let result = safe
+    .replace(/^---\n[\s\S]*?\n---\n?/, '')   // YAML frontmatter
+    .replace(/^import\s+.*$/gm, '')           // MDX import statements
+    .replace(/#{1,6}\s/g, '')                 // heading markers
+    .replace(/\*\*([^*]+)\*\*/g, '$1')        // bold
+    .replace(/\*([^*]+)\*/g, '$1')            // italic
+    .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1')  // links → text only
+    .replace(/!\[([^\]]*)\]\([^)]+\)/g, '')   // images
+    .replace(/<[^>]+>/g, '')                   // ALL tags (MDX + HTML) — content kept
+    .replace(/\n{3,}/g, '\n\n');               // collapse excessive newlines
+
+  // Restore protected code
+  result = result.replace(/\x00(\d+)\x00/g, (_, i) => preserved[parseInt(i)]);
+
+  return result.trim();
+}
+
+/**
+ * Split text at sentence boundaries, keeping chunks under maxChars.
+ *
+ * Sentences end with `.`, `!`, or `?` followed by a space or end of string.
+ * If no sentence boundary is found within maxChars, splits at maxChars.
+ */
+function splitAtSentenceBoundaries(text: string, maxChars: number): string[] {
+  const chunks: string[] = [];
+  let remaining = text;
+
+  while (remaining.length > maxChars) {
+    // Look for the last sentence boundary within maxChars
+    const window = remaining.slice(0, maxChars);
+    const sentenceEndRegex = /[.!?](?:\s|$)/g;
+    let lastBoundary = -1;
+    let match;
+
+    while ((match = sentenceEndRegex.exec(window)) !== null) {
+      // Include the punctuation mark in the split position
+      lastBoundary = match.index + 1;
+    }
+
+    if (lastBoundary > 0) {
+      chunks.push(remaining.slice(0, lastBoundary).trim());
+      remaining = remaining.slice(lastBoundary).trim();
+    } else {
+      // No sentence boundary found — hard split at maxChars
+      chunks.push(remaining.slice(0, maxChars).trim());
+      remaining = remaining.slice(maxChars).trim();
+    }
+  }
+
+  if (remaining.trim()) {
+    chunks.push(remaining.trim());
+  }
+
+  return chunks;
+}
+
+/**
+ * Derive a human-readable title from a slug.
+ * e.g. "getting-started" → "Getting Started"
+ */
+function titleFromSlug(slug: string): string {
+  const lastSegment = slug.split('/').pop() || slug;
+  return lastSegment
+    .replace(/[-_]/g, ' ')
+    .replace(/\b\w/g, c => c.toUpperCase());
+}
+
+/**
+ * Detect if a page is an API reference page and return a prefix label.
+ * API pages get a prefix like "API Reference — POST /post\n" so the
+ * embedding model clusters them distinctly from guides/tutorials.
+ */
+function getEmbeddingPrefix(
+  slug: string,
+  frontmatter: Record<string, unknown>,
+): string {
+  const apiMethod = (frontmatter.api as string) || (frontmatter.openapi as string);
+  if (apiMethod) return `API Reference — ${apiMethod}\n`;
+  if (slug.startsWith('apis/')) return 'API Reference\n';
+  return '';
+}
+
+/**
+ * Chunk a documentation page into embedding-sized pieces.
+ *
+ * @param page - Page with file path, raw MDX content, and frontmatter
+ * @param maxChars - Maximum characters per chunk (default 2000, ~500 tokens)
+ * @returns Array of embedding chunks with unique IDs
+ */
+export function chunkPageForEmbedding(
+  page: { path: string; content: string; frontmatter: Record<string, unknown> },
+  maxChars = 2000,
+): EmbeddingChunk[] {
+  const slug = page.path.replace(/\.mdx?$/, '').replace(/\\/g, '/');
+  const pageTitle = (page.frontmatter.title as string) || titleFromSlug(slug);
+  const embeddingPrefix = getEmbeddingPrefix(slug, page.frontmatter);
+
+  // Normalize Windows line endings before extracting sections
+  const normalizedContent = page.content.replace(/\r\n/g, '\n');
+
+  const sections = extractSections(normalizedContent);
+  const chunks: EmbeddingChunk[] = [];
+  let chunkIndex = 0;
+
+  for (const section of sections) {
+    const cleanContent = stripForEmbedding(section.content);
+    if (!cleanContent.trim()) continue;
+
+    // Small sections stay whole; oversized ones split at sentence boundaries
+    const pieces = cleanContent.length <= maxChars
+      ? [cleanContent]
+      : splitAtSentenceBoundaries(cleanContent, maxChars);
+
+    for (const piece of pieces) {
+      if (!piece.trim()) continue;
+      chunks.push({
+        id: `${slug}#${chunkIndex}`,
+        pageSlug: slug,
+        sectionHeading: section.heading,
+        content: embeddingPrefix + piece,
+        pageTitle,
+      });
+      chunkIndex++;
+    }
+  }
+
+  return chunks;
+}

package/vendored/lib/generate-starter-questions.ts

@@ -0,0 +1,98 @@
+/**
+ * Auto-generate chat starter questions from documentation structure.
+ *
+ * Sends page titles and API methods to Claude Haiku, which returns
+ * 4 contextually relevant questions new users would likely ask.
+ * Non-fatal: returns empty array on any error so builds aren't blocked.
+ *
+ * Uses temperature 0 for deterministic output — same docs produce the
+ * same questions, avoiding unnecessary cache invalidation between builds.
+ */
+import { getAnthropicClient } from './anthropic-client.js';
+
+const MAX_QUESTIONS = 4;
+/** Cap summary lines to avoid excessive prompt size with large doc sets */
+const MAX_SUMMARY_LINES = 100;
+/** Abort if Haiku takes longer than this (don't block the build) */
+const TIMEOUT_MS = 10_000;
+
+type PageInfo = { path: string; frontmatter: Record<string, unknown> };
+
+/**
+ * Build a concise summary of the documentation structure for the prompt.
+ * Includes page titles and API methods. For large doc sets (> MAX_SUMMARY_LINES),
+ * samples evenly across the list to avoid bias toward alphabetically-first pages.
+ */
+export function buildDocSummary(pages: PageInfo[]): string {
+  let sampled = pages;
+  if (pages.length > MAX_SUMMARY_LINES) {
+    // Sample evenly: spread indices across the full range so the last page
+    // is always included (avoids bias toward alphabetically-first pages)
+    const lastIdx = pages.length - 1;
+    const step = lastIdx / (MAX_SUMMARY_LINES - 1);
+    sampled = Array.from({ length: MAX_SUMMARY_LINES }, (_, i) =>
+      pages[Math.min(Math.round(i * step), lastIdx)],
+    );
+  }
+
+  return sampled.map(page => {
+    const title = (page.frontmatter.title as string)
+      || page.path.replace(/\.mdx?$/, '').split('/').pop() || 'Untitled';
+    const apiMethod = (page.frontmatter.api as string) || (page.frontmatter.openapi as string) || '';
+    return apiMethod ? `- ${title} (${apiMethod})` : `- ${title}`;
+  }).join('\n');
+}
+
+/**
+ * Generate starter questions for the chat empty state using Haiku.
+ * Returns up to 4 questions, or empty array if generation fails.
+ */
+export async function generateStarterQuestions(pages: PageInfo[]): Promise<string[]> {
+  const anthropic = getAnthropicClient();
+  if (!anthropic || pages.length === 0) return [];
+
+  const summary = buildDocSummary(pages);
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), TIMEOUT_MS);
+
+  try {
+    const response = await anthropic.messages.create(
+      {
+        model: 'claude-haiku-4-5-20251001',
+        max_tokens: 256,
+        temperature: 0,
+        messages: [{
+          role: 'user',
+          content: `You are generating starter questions for a documentation chatbot. Given this documentation structure, generate exactly ${MAX_QUESTIONS} questions that a new user would likely ask first.
+
+Rules:
+- Each question must be under 50 characters
+- Cover different topics (getting started, API usage, features, integration)
+- Be specific to THIS documentation, not generic
+- Use natural language ("How do I..." not "Explain the...")
+- Return ONLY a JSON array of strings, no other text
+
+Documentation pages:
+${summary}`,
+        }],
+      },
+      { signal: controller.signal },
+    );
+
+    let text = response.content[0]?.type === 'text' ? response.content[0].text : '';
+    // Strip markdown code fences — Haiku sometimes wraps JSON in ```json ... ```
+    text = text.replace(/^```(?:json)?\s*\n?/i, '').replace(/\n?```\s*$/, '').trim();
+    const parsed = JSON.parse(text);
+    if (!Array.isArray(parsed)) return [];
+
+    return parsed
+      // Accept string items directly, or objects with a "question" field
+      .map((q) => typeof q === 'string' ? q : (q?.question as string) || null)
+      .filter((q): q is string => typeof q === 'string' && q.length > 0)
+      .slice(0, MAX_QUESTIONS);
+  } catch {
+    return [];
+  } finally {
+    clearTimeout(timeout);
+  }
+}

package/vendored/lib/isr-build-executor.ts

@@ -193,7 +193,8 @@ export const ISR_PHASES = {
   copy_content: { label: 'Preparing content...', weight: 10 },
   // ISR mode: nextjs_build is instant (pages compiled on-demand by Vercel)
   nextjs_build: { label: 'Building documentation...', weight: 5 },
-  r2_upload: { label: 'Uploading to CDN...', weight: 45 },
+  r2_upload: { label: 'Uploading to CDN...', weight: 40 },
+  embeddings: { label: 'Generating AI embeddings...', weight: 5 },
   vercel_purge: { label: 'Refreshing cache...', weight: 20 },
   cleanup: { label: 'Cleaning up...', weight: 5 },
 } as const;

package/vendored/lib/middleware-helpers.ts

@@ -287,6 +287,7 @@ export const INTERNAL_API_ROUTES = [
   '/api/assets',     // Asset serving from R2 (app/api/assets/[...path])
   '/api/ev',         // Analytics events (app/api/ev)
   '/api/isr-health', // Health check endpoint (app/api/isr-health)
+  '/api/chat',       // Chat endpoint (app/api/chat/[project])
   '/api/mcp',        // MCP endpoint (app/api/mcp/[project])
   '/api/og',         // OG image generation (app/api/og)
   '/api/r2',         // R2 content serving (app/api/r2/[project]/[...path])
@@ -394,6 +395,26 @@ export function getMcpApiPath(projectSlug: string): string {
   return `/api/mcp/${projectSlug}`;
 }
 
+/**
+ * Check if this is a chat request that needs routing.
+ *
+ * @param pathname - Request pathname
+ * @returns true if this is a chat request
+ */
+export function isChatRequest(pathname: string): boolean {
+  return pathname === '/_chat' || pathname === '/docs/_chat';
+}
+
+/**
+ * Get the chat API path for a project.
+ *
+ * @param projectSlug - Project identifier
+ * @returns Chat API route path
+ */
+export function getChatApiPath(projectSlug: string): string {
+  return `/api/chat/${projectSlug}`;
+}
+
 /**
  * Check if path needs trailing slash normalization.
  *

package/vendored/lib/route-helpers.ts

@@ -0,0 +1,96 @@
+/**
+ * Shared route helpers for resolving project URLs, docs paths, and tracking analytics.
+ * Used by the MCP route and the chat API route.
+ */
+import { redis } from './redis';
+import { isCustomDomain, parseRedisConfig } from './domain-helpers';
+
+/**
+ * Resolve docsPath for a project ('' or '/docs').
+ * Queries Redis for projectCfg: or domainCfg: keys to determine hostAtDocs.
+ * Defaults to '/docs' (hostAtDocs=true) for safety — most Jamdesk sites use this.
+ */
+export async function getDocsPath(projectSlug: string, originalHost: string): Promise<string> {
+  if (!redis) return '/docs';
+
+  try {
+    const key = isCustomDomain(originalHost) ? `domainCfg:${originalHost}` : `projectCfg:${projectSlug}`;
+    const cfg = parseRedisConfig(await redis.get(key));
+    if (cfg) {
+      return cfg.hostAtDocs !== false ? '/docs' : '';
+    }
+  } catch {
+    // Fall through to default
+  }
+
+  return '/docs';
+}
+
+/**
+ * Resolve base URL for a project.
+ * Uses resolved host for custom/forwarded domains, falls back to project.jamdesk.app.
+ */
+export function getBaseUrl(projectSlug: string, resolvedHost: string): string {
+  const hostname = resolvedHost.split(':')[0];
+  return isCustomDomain(hostname)
+    ? `https://${hostname}`
+    : `https://${projectSlug}.jamdesk.app`;
+}
+
+/**
+ * Fire-and-forget analytics tracking.
+ */
+export async function trackServerAnalytics(params: {
+  projectSlug: string;
+  type: string;
+  query: string;
+  resultsCount: number;
+  source: string;
+}): Promise<void> {
+  const trackingUrl = process.env.TRACKING_URL || 'https://us-central1-jamdesk-prod.cloudfunctions.net/trackSearchAnalytics';
+  try {
+    await fetch(trackingUrl, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        ...params,
+        sessionId: `${params.source}-${Date.now()}`,
+      }),
+    });
+  } catch {
+    // Silent failure - don't break caller if analytics fails
+  }
+}
+
+/**
+ * Fire-and-forget chat analytics tracking.
+ * Sends enriched chat event data (token usage, quality signals) to the
+ * trackChatAnalytics Firebase Function for storage in Firestore chatEvents.
+ */
+export async function trackChatAnalytics(params: {
+  projectSlug: string;
+  query: string;
+  resultsCount: number;
+  inputTokens: number;
+  outputTokens: number;
+  model: string;
+  hadExplicitCitations: boolean;
+  hasClarification: boolean;
+  durationMs: number;
+  userAgent?: string;
+}): Promise<void> {
+  const trackingUrl = process.env.CHAT_TRACKING_URL || 'https://us-central1-jamdesk-prod.cloudfunctions.net/trackChatAnalytics';
+  const headers: Record<string, string> = { 'Content-Type': 'application/json' };
+  if (params.userAgent) {
+    headers['User-Agent'] = params.userAgent;
+  }
+  try {
+    await fetch(trackingUrl, {
+      method: 'POST',
+      headers,
+      body: JSON.stringify(params),
+    });
+  } catch {
+    // Silent failure
+  }
+}

package/vendored/lib/snippet-loader-isr.ts

@@ -52,13 +52,119 @@ function normalizeSnippetPath(importPath: string): string {
     .replace(/^\//, '');
 }
 
+/**
+ * Extract the call expression from an arrow function body.
+ * Handles both concise body (() => expr) and block body (() => { expr; }).
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function extractCallFromArrow(arrow: any): any | undefined {
+  if (arrow.body.type === 'CallExpression') {
+    return arrow.body;
+  }
+
+  if (
+    arrow.body.type === 'BlockStatement' &&
+    arrow.body.body.length === 1 &&
+    arrow.body.body[0].type === 'ExpressionStatement' &&
+    arrow.body.body[0].expression.type === 'CallExpression'
+  ) {
+    return arrow.body.body[0].expression;
+  }
+
+  return undefined;
+}
+
+/**
+ * Check if a call expression is window.open(stringLiteral, '_self').
+ * Returns the URL string if it matches, undefined otherwise.
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function extractWindowOpenSelfUrl(call: any): string | undefined {
+  const callee = call.callee;
+  if (
+    callee.type !== 'MemberExpression' ||
+    callee.object?.name !== 'window' ||
+    callee.property?.name !== 'open' ||
+    call.arguments.length < 2
+  ) return undefined;
+
+  const urlArg = call.arguments[0];
+  const targetArg = call.arguments[1];
+  if (urlArg.type !== 'StringLiteral') return undefined;
+  if (targetArg.type !== 'StringLiteral' || targetArg.value !== '_self') return undefined;
+
+  return urlArg.value;
+}
+
+/**
+ * Babel plugin that transforms onClick navigation patterns for RSC compatibility.
+ *
+ * Converts: <span onClick={() => window.open('url', '_self')} ...>
+ * To:       <a href="url" ...>
+ *
+ * Uses AST instead of regex for robustness (handles any attribute ordering,
+ * nesting depth, and whitespace). Matches the behavior of the static-mode
+ * regex in compile-snippets.cjs:158-174.
+ *
+ * Only targets <span> elements with the specific window.open(url, '_self')
+ * pattern (matching static-mode behavior) — other elements and event handlers
+ * are left unchanged (documented ISR limitation in SNIPPETS.md).
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function onClickToHrefPlugin({ types: t }: { types: any }) {
+  return {
+    visitor: {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      JSXElement(path: any) {
+        const opening = path.node.openingElement;
+
+        if (opening.name.type !== 'JSXIdentifier' || opening.name.name !== 'span') return;
+
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const onClickIndex = opening.attributes.findIndex((attr: any) =>
+          attr.type === 'JSXAttribute' &&
+          attr.name?.type === 'JSXIdentifier' &&
+          attr.name.name === 'onClick'
+        );
+        if (onClickIndex === -1) return;
+
+        const onClickAttr = opening.attributes[onClickIndex];
+        if (onClickAttr.value?.type !== 'JSXExpressionContainer') return;
+
+        const arrow = onClickAttr.value.expression;
+        if (arrow.type !== 'ArrowFunctionExpression') return;
+
+        const call = extractCallFromArrow(arrow);
+        if (!call) return;
+
+        const url = extractWindowOpenSelfUrl(call);
+        if (url === undefined) return;
+
+        // Transform: <span onClick={...}> → <a href="url">
+        opening.name.name = 'a';
+        if (path.node.closingElement) {
+          path.node.closingElement.name.name = 'a';
+        }
+
+        opening.attributes.splice(onClickIndex, 1);
+        opening.attributes.unshift(
+          t.jsxAttribute(t.jsxIdentifier('href'), t.stringLiteral(url))
+        );
+      },
+    },
+  };
+}
+
 /**
  * Transpile JSX source code to JavaScript.
  */
 function transpileJsx(source: string): string {
   const result = transform(source, {
     presets: ['react', 'typescript'],
-    plugins: [['transform-react-jsx', { runtime: 'automatic', importSource: 'react' }]],
+    plugins: [
+      onClickToHrefPlugin,
+      ['transform-react-jsx', { runtime: 'automatic', importSource: 'react' }],
+    ],
     filename: 'snippet.tsx',
   });
 

package/vendored/lib/static-artifacts.ts

@@ -291,7 +291,7 @@ function inferPageType(slug: string): string {
 /**
  * Strip markdown formatting from text.
  */
-function stripMarkdown(text: string): string {
+export function stripMarkdown(text: string): string {
   return text
     .replace(/```[\s\S]*?```/g, '')    // code blocks
     .replace(/`[^`]+`/g, '')           // inline code
@@ -299,6 +299,7 @@ function stripMarkdown(text: string): string {
     .replace(/\*\*([^*]+)\*\*/g, '$1') // bold
     .replace(/\*([^*]+)\*/g, '$1')     // italic
     .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1') // links
+    .replace(/<[A-Z][^>]*(?:\/>|>[\s\S]*?<\/[A-Z][^>]*>)/g, '') // MDX components (e.g. <Callout>, <Steps>)
     .replace(/<[^>]+>/g, '')           // HTML tags
     .replace(/!\[([^\]]*)\]\([^)]+\)/g, '') // images
     .replace(/\n+/g, ' ')              // newlines
@@ -308,7 +309,7 @@ function stripMarkdown(text: string): string {
 /**
  * Extract sections from MDX content.
  */
-function extractSections(content: string): Array<{ heading: string; content: string }> {
+export function extractSections(content: string): Array<{ heading: string; content: string }> {
   const sections: Array<{ heading: string; content: string }> = [];
   const lines = content.split('\n');
   let currentHeading = '';

package/vendored/lib/validate-config.ts

@@ -21,6 +21,7 @@ import { LIB_DIR } from './paths.js';
 // Types
 export interface DocsConfig {
   name: string;
+  projectId?: string;
   theme?: 'jam' | 'nebula' | 'pulsar';
   navigation: NavigationConfig;
   integrations?: IntegrationsConfig;