mkdnsite 0.0.1 → 1.0.1

package/src/content/r2.ts

@@ -0,0 +1,214 @@
+import { parseFrontmatter } from './frontmatter.ts'
+import type { ContentSource, ContentPage, NavNode } from './types.ts'
+import type { ContentCache } from './cache.ts'
+import { MemoryContentCache } from './cache.ts'
+import { buildNavTree } from './nav-builder.ts'
+
+/** Parsed R2 file ready for use */
+interface ParsedFile {
+  /** Relative path from content root, e.g. 'docs/getting-started.md' */
+  path: string
+  meta: Record<string, unknown>
+  body: string
+  raw: string
+}
+
+export interface R2ContentSourceConfig {
+  /** R2 bucket binding */
+  bucket: R2Bucket
+  /** Key prefix for content (e.g. 'sites/abc123/') */
+  basePath?: string
+  /** Optional cache layer (defaults to in-memory). Pass KVContentCache for durable caching. */
+  cache?: ContentCache
+}
+
+// ─── R2 type stubs (not available outside Cloudflare Workers) ────────────────
+
+interface R2Bucket {
+  get: (key: string) => Promise<R2Object | null>
+  list: (options?: R2ListOptions) => Promise<R2ObjectList>
+}
+
+interface R2Object {
+  key: string
+  uploaded: Date
+  size: number
+  text: () => Promise<string>
+}
+
+interface R2ObjectList {
+  objects: R2Object[]
+  truncated: boolean
+  cursor?: string
+}
+
+interface R2ListOptions {
+  prefix?: string
+  cursor?: string
+  limit?: number
+}
+
+// ─── Helpers ─────────────────────────────────────────────────────────────────
+
+async function listAllMdFiles (bucket: R2Bucket, prefix: string): Promise<R2Object[]> {
+  const objects: R2Object[] = []
+  let cursor: string | undefined
+  do {
+    const result = await bucket.list({ prefix, cursor })
+    objects.push(...result.objects.filter(o => o.key.endsWith('.md')))
+    cursor = result.truncated ? result.cursor : undefined
+  } while (cursor != null)
+  return objects
+}
+
+/** Convert a URL slug to a relative content key (no basePath prefix) */
+function slugToRelKey (slug: string): string {
+  const stripped = slug.replace(/^\/+|\.md$/g, '').replace(/\/+$/, '')
+  return stripped === '' ? 'index' : stripped
+}
+
+function keyToRelPath (key: string, basePath: string): string {
+  return basePath !== '' && key.startsWith(basePath)
+    ? key.slice(basePath.length)
+    : key
+}
+
+// ─── R2ContentSource ─────────────────────────────────────────────────────────
+
+/**
+ * Content source that reads .md files from a Cloudflare R2 bucket.
+ *
+ * Lazy prefetch: on first access, lists all .md keys, fetches them in parallel,
+ * parses frontmatter, and caches for TTL_MS (5 minutes).
+ */
+export class R2ContentSource implements ContentSource {
+  private readonly bucket: R2Bucket
+  private readonly basePath: string
+  private readonly cache: ContentCache
+
+  private rawCache: Map<string, ParsedFile> | null = null
+  private rawCacheExpiresAt: number = 0
+  private initPromise: Promise<Map<string, ParsedFile>> | null = null
+
+  constructor (config: R2ContentSourceConfig) {
+    this.bucket = config.bucket
+    this.cache = config.cache ?? new MemoryContentCache()
+    // Normalise basePath to end with '/' or be empty
+    const raw = config.basePath ?? ''
+    this.basePath = raw !== '' && !raw.endsWith('/') ? raw + '/' : raw
+  }
+
+  // ─── Public API ────────────────────────────────────────────────────────────
+
+  async getPage (slug: string): Promise<ContentPage | null> {
+    const key = slugToRelKey(slug)
+
+    // Try cache first for each candidate
+    for (const candidate of [key, key + '/index', key + '/README', key + '/readme']) {
+      const cached = await this.cache.getPage(candidate)
+      if (cached != null) return cached
+    }
+
+    // Fall through to prefetch
+    const pages = await this.ensurePrefetched()
+
+    for (const candidate of [key, key + '/index', key + '/README', key + '/readme']) {
+      const file = pages.get(candidate)
+      if (file != null && file.meta.draft !== true) {
+        const page = this.toContentPage(file)
+        await this.cache.setPage(candidate, page)
+        return page
+      }
+    }
+    return null
+  }
+
+  async getNavTree (): Promise<NavNode> {
+    const cached = await this.cache.getNav()
+    if (cached != null) return cached
+
+    const pages = await this.ensurePrefetched()
+    const files = Array.from(pages.values())
+    const nav = buildNavTree(files)
+    await this.cache.setNav(nav)
+    return nav
+  }
+
+  async listPages (): Promise<ContentPage[]> {
+    const cached = await this.cache.getPageList()
+    if (cached != null) return cached
+
+    const pages = await this.ensurePrefetched()
+    const result = Array.from(pages.values())
+      .filter(f => f.meta.draft !== true)
+      .map(f => this.toContentPage(f))
+    await this.cache.setPageList(result)
+    return result
+  }
+
+  async refresh (): Promise<void> {
+    this.rawCache = null
+    this.rawCacheExpiresAt = 0
+    this.initPromise = null
+    await this.cache.clear()
+  }
+
+  // ─── Internal ──────────────────────────────────────────────────────────────
+
+  private async ensurePrefetched (): Promise<Map<string, ParsedFile>> {
+    if (this.rawCache != null && Date.now() < this.rawCacheExpiresAt) {
+      return this.rawCache
+    }
+    if (this.initPromise != null) return await this.initPromise
+    this.initPromise = this.prefetchAll()
+    const result = await this.initPromise
+    this.initPromise = null
+    return result
+  }
+
+  private async prefetchAll (): Promise<Map<string, ParsedFile>> {
+    const objects = await listAllMdFiles(this.bucket, this.basePath)
+
+    const fetched = await Promise.all(
+      objects.map(async (obj) => {
+        try {
+          const r2obj = await this.bucket.get(obj.key)
+          if (r2obj == null) return null
+          const raw = await r2obj.text()
+          const { meta, body } = parseFrontmatter(raw)
+          const relPath = keyToRelPath(obj.key, this.basePath)
+          return { path: relPath, meta: meta as Record<string, unknown>, body, raw } satisfies ParsedFile
+        } catch {
+          return null
+        }
+      })
+    )
+
+    const pages = new Map<string, ParsedFile>()
+    for (const file of fetched) {
+      if (file == null) continue
+      const key = file.path.replace(/\.md$/, '')
+      pages.set(key, file)
+    }
+
+    this.rawCache = pages
+    this.rawCacheExpiresAt = Date.now() + 5 * 60 * 1000
+    return pages
+  }
+
+  private toContentPage (file: ParsedFile): ContentPage {
+    const key = file.path.replace(/\.md$/, '')
+    const isIndex = /(?:^|\/)(?:index|README|readme)$/.test(key)
+    const cleanKey = isIndex
+      ? key.replace(/\/(?:index|README|readme)$/, '')
+      : key
+    const slug = cleanKey === '' || cleanKey === 'index' ? '/' : '/' + cleanKey
+    return {
+      slug,
+      sourcePath: file.path,
+      meta: file.meta,
+      body: file.body,
+      raw: file.raw
+    }
+  }
+}

package/src/handler.ts

@@ -7,11 +7,42 @@ import { negotiateFormat } from './negotiate/accept.ts'
 import { markdownHeaders, htmlHeaders, estimateTokens } from './negotiate/headers.ts'
 import { renderPage, render404 } from './render/page-shell.ts'
 import { generateLlmsTxt } from './discovery/llmstxt.ts'
+import { createSearchIndex } from './search/index.ts'
+import type { SearchIndex } from './search/index.ts'
+import type { ContentCache } from './content/cache.ts'
+import type { ResponseCache } from './cache/response.ts'
+import { createMcpServer } from './mcp/server.ts'
+import { buildCsp } from './security/csp.ts'
+import { createMcpHandler } from './mcp/transport.ts'
+import type { TrafficAnalytics, AnalyticsResponseFormat } from './analytics/types.ts'
+import { classifyTraffic } from './analytics/classify.ts'
 
 export interface HandlerOptions {
   source: ContentSource
   renderer: MarkdownRenderer
   config: MkdnSiteConfig
+  /** Optional traffic analytics backend. When provided, every request is logged. */
+  analytics?: TrafficAnalytics
+  /** Site identifier for multi-tenant analytics isolation (e.g. mkdn.io). */
+  siteId?: string
+  /**
+   * Optional content cache.
+   * When provided, the search index is loaded from / stored in the cache
+   * so cold-start rebuilds are avoided across isolates (e.g. Cloudflare Workers + KV).
+   */
+  contentCache?: ContentCache
+  /**
+   * Optional response cache.
+   * When provided, rendered HTML and markdown responses are cached to skip SSR
+   * on repeat requests. Works with MemoryResponseCache (single-process) or
+   * KVResponseCache (Cloudflare Workers with KV — cross-isolate sharing).
+   */
+  responseCache?: ResponseCache
+  /**
+   * Optional secret token for authenticating POST /_refresh requests.
+   * When set, requests without `Authorization: Bearer <refreshToken>` are rejected with 401.
+   */
+  refreshToken?: string
 }
 
 /**
@@ -26,42 +57,221 @@ export interface HandlerOptions {
  * - Deno.serve()
  */
 export function createHandler (opts: HandlerOptions): (request: Request) => Promise<Response> {
-  const { source, renderer, config } = opts
+  const { source, renderer, config, analytics, siteId, contentCache, responseCache, refreshToken } = opts
 
   let llmsTxtCache: string | null = null
+  let mcpHandlerFn: ((req: Request) => Promise<Response>) | null = null
+  let mcpInitPromise: Promise<(req: Request) => Promise<Response>> | null = null
+  let searchIndexPromise: Promise<SearchIndex> | null = null
+
+  // Shared search index used by both /api/search and MCP
+  async function ensureSearchIndex (): Promise<SearchIndex> {
+    if (searchIndexPromise != null) return await searchIndexPromise
+    searchIndexPromise = (async () => {
+      const si = createSearchIndex()
+      // Try to restore from cache (avoids full rebuild on cold starts)
+      if (contentCache != null) {
+        const cached = await contentCache.getSearchIndex()
+        if (cached != null && cached !== '') {
+          try {
+            si.deserialize(cached)
+            return si
+          } catch {
+            // Corrupt / incompatible — fall through to rebuild
+          }
+        }
+      }
+      await si.rebuild(source)
+      // Persist to cache for next cold start
+      if (contentCache != null) {
+        try {
+          await contentCache.setSearchIndex(si.serialize())
+        } catch {
+          // Non-fatal — cache write failure shouldn't break search
+        }
+      }
+      return si
+    })()
+    return await searchIndexPromise
+  }
+
+  async function ensureMcpHandler (): Promise<(req: Request) => Promise<Response>> {
+    if (mcpInitPromise != null) return await mcpInitPromise
+    mcpInitPromise = (async () => {
+      const si = await ensureSearchIndex()
+      const mcpServer = createMcpServer({ source, searchIndex: si })
+      return createMcpHandler(mcpServer)
+    })()
+    mcpHandlerFn = await mcpInitPromise
+    return mcpHandlerFn
+  }
+
+  // Eagerly init search index when client search is enabled
+  if (config.client.search) {
+    void ensureSearchIndex()
+  }
 
   return async function handler (request: Request): Promise<Response> {
+    const start = Date.now()
     const url = new URL(request.url)
     const pathname = decodeURIComponent(url.pathname)
 
+    const { response, cacheHit } = await handleRequest(request, url, pathname)
+
+    if (analytics != null) {
+      const format = resolveAnalyticsFormat(request, pathname, response)
+      try {
+        analytics.logRequest({
+          timestamp: start,
+          path: pathname,
+          method: request.method,
+          format,
+          trafficType: classifyTraffic(request, format),
+          statusCode: response.status,
+          latencyMs: Date.now() - start,
+          userAgent: request.headers.get('User-Agent') ?? '',
+          contentLength: readContentLength(response),
+          cacheHit,
+          siteId
+        })
+      } catch {
+        // analytics must never break the response path
+      }
+    }
+
+    return response
+  }
+
+  // ---- Analytics format resolution ----
+
+  function resolveAnalyticsFormat (
+    request: Request,
+    pathname: string,
+    response: Response
+  ): AnalyticsResponseFormat {
+    // MCP: check endpoint first (MCP responses may have various Content-Types)
+    const mcpEndpoint = config.mcp.endpoint ?? '/mcp'
+    if (
+      config.mcp.enabled &&
+      (pathname === mcpEndpoint || pathname.startsWith(mcpEndpoint + '/'))
+    ) {
+      return 'mcp'
+    }
+
+    // Prefer response Content-Type when available
+    const contentType = response.headers.get('Content-Type') ?? ''
+    if (contentType.includes('text/markdown')) return 'markdown'
+    if (contentType.includes('text/html')) return 'html'
+    if (contentType.includes('application/json')) return 'api'
+
+    // Fallback: infer from request when Content-Type is missing (e.g. static files)
+    const accept = request.headers.get('Accept') ?? ''
+    if (
+      accept.includes('text/markdown') ||
+      accept.includes('text/x-markdown') ||
+      accept.includes('application/markdown') ||
+      pathname.endsWith('.md')
+    ) {
+      return 'markdown'
+    }
+    return 'other'
+  }
+
+  // ---- Inner request handler (no analytics instrumentation) ----
+
+  async function handleRequest (
+    request: Request,
+    url: URL,
+    pathname: string
+  ): Promise<{ response: Response, cacheHit: boolean }> {
+    function ok (response: Response): { response: Response, cacheHit: boolean } {
+      return { response, cacheHit: false }
+    }
+
     // ---- Special routes ----
 
     if (pathname === '/_health') {
-      return new Response('ok', { status: 200 })
+      return ok(new Response('ok', { status: 200 }))
     }
 
     if (pathname === '/llms.txt' && config.llmsTxt.enabled) {
       if (llmsTxtCache == null) {
         llmsTxtCache = await generateLlmsTxt(source, config)
       }
-      return new Response(llmsTxtCache, {
+      return ok(textResponse(llmsTxtCache, {
         status: 200,
         headers: {
           'Content-Type': 'text/markdown; charset=utf-8',
           'Cache-Control': 'public, max-age=3600'
         }
-      })
+      }))
     }
 
     if (pathname === '/_refresh' && request.method === 'POST') {
+      // Optional Bearer token auth
+      if (refreshToken != null && refreshToken !== '') {
+        const authHeader = request.headers.get('Authorization') ?? ''
+        const token = authHeader.startsWith('Bearer ') ? authHeader.slice(7) : ''
+        if (!timingSafeEqual(token, refreshToken)) {
+          return ok(new Response('Unauthorized', { status: 401 }))
+        }
+      }
+
       await source.refresh()
       llmsTxtCache = null
-      return new Response('cache cleared', { status: 200 })
+
+      // Clear cached search index
+      if (contentCache != null) {
+        try { await contentCache.setSearchIndex('') } catch { /* non-fatal */ }
+      }
+
+      // Clear response cache (supports ?path= for single-entry invalidation)
+      if (responseCache != null) {
+        const pathParam = url.searchParams.get('path')
+        if (pathParam != null) {
+          // Invalidate both html and markdown variants
+          await responseCache.delete(pathParam + ':html')
+          await responseCache.delete(pathParam + ':markdown')
+        } else {
+          await responseCache.clear()
+        }
+      }
+
+      // Reset search index
+      if (searchIndexPromise != null) {
+        searchIndexPromise = null
+        void ensureSearchIndex()
+      }
+
+      return ok(new Response('cache cleared', { status: 200 }))
+    }
+
+    // ---- Search API ----
+    if (pathname === '/api/search' && config.client.search) {
+      const query = (url.searchParams.get('q') ?? '').slice(0, 200)
+      const rawLimit = parseInt(url.searchParams.get('limit') ?? '10', 10)
+      const limit = Math.min(isNaN(rawLimit) ? 10 : rawLimit, 50)
+      const si = await ensureSearchIndex()
+      const results = si.search(query, limit)
+      return ok(textResponse(JSON.stringify(results), {
+        status: 200,
+        headers: { 'Content-Type': 'application/json' }
+      }))
+    }
+
+    // ---- MCP endpoint ----
+    if (
+      config.mcp.enabled &&
+      (pathname === (config.mcp.endpoint ?? '/mcp') ||
+        pathname.startsWith((config.mcp.endpoint ?? '/mcp') + '/'))
+    ) {
+      const mcp = await ensureMcpHandler()
+      return ok(await mcp(request))
     }
 
     // ---- Static files passthrough ----
     if (config.staticDir != null && hasStaticExtension(pathname)) {
-      return await serveStatic(pathname, config.staticDir)
+      return ok(await serveStatic(pathname, config.staticDir))
     }
 
     // ---- Content negotiation + page serving ----
@@ -83,38 +293,69 @@ export function createHandler (opts: HandlerOptions): (request: Request) => Prom
     if (page == null) {
       const format = negotiateFormat(request.headers.get('Accept'))
       if (format === 'markdown') {
-        return new Response(
+        return ok(textResponse(
           '# 404 — Page Not Found\n\nThe requested page does not exist.\n',
           {
             status: 404,
             headers: { 'Content-Type': 'text/markdown; charset=utf-8' }
           }
-        )
+        ))
       }
-      return new Response(render404(config), {
+      return ok(textResponse(render404(config), {
         status: 404,
-        headers: htmlHeaders()
-      })
+        headers: htmlHeadersWithCsp(config)
+      }))
     }
 
     const format = forceMarkdown
       ? 'markdown'
       : negotiateFormat(request.headers.get('Accept'))
 
+    // ---- Response cache check (content pages only) ----
+    const cacheEnabled = config.cache?.enabled === true && responseCache != null
+    const cacheKey = slug + ':' + format
+
+    if (cacheEnabled && responseCache != null) {
+      const cached = await responseCache.get(cacheKey)
+      if (cached != null) {
+        // 304 Not Modified support
+        const ifNoneMatch = request.headers.get('If-None-Match')
+        const etag = cached.headers.ETag
+        if (ifNoneMatch != null && etag != null && ifNoneMatch === etag) {
+          return { response: new Response(null, { status: 304, headers: notModifiedHeaders(cached.headers) }), cacheHit: true }
+        }
+        return { response: textResponse(cached.body, { status: cached.status, headers: cached.headers }), cacheHit: true }
+      }
+    }
+
     if (format === 'markdown') {
       const tokens = config.negotiation.includeTokenCount
         ? estimateTokens(page.body)
         : null
 
-      return new Response(page.body, {
-        status: 200,
-        headers: markdownHeaders(tokens, config.negotiation.contentSignals)
-      })
+      const headers = markdownHeaders(tokens, config.negotiation.contentSignals, config.cache, slug)
+
+      // 304 Not Modified support for non-cached path
+      const ifNoneMatch = request.headers.get('If-None-Match')
+      if (ifNoneMatch != null && headers.ETag != null && ifNoneMatch === headers.ETag) {
+        return ok(new Response(null, { status: 304, headers: notModifiedHeaders(headers) }))
+      }
+
+      const response = textResponse(page.body, { status: 200, headers })
+      if (cacheEnabled && responseCache != null) {
+        await responseCache.set(cacheKey, {
+          body: page.body,
+          status: 200,
+          headers,
+          timestamp: Date.now()
+        }, config.cache?.maxAgeMarkdown)
+      }
+      return ok(response)
     }
 
     // ---- Render HTML via React SSR ----
     const renderedHtml = renderer.renderToHtml(page.body, config.theme.components)
-    const nav = config.theme.showNav
+    const nav = (config.theme.showNav || config.theme.prevNext === true)
       ? await source.getNavTree()
       : undefined
 
@@ -123,16 +364,58 @@ export function createHandler (opts: HandlerOptions): (request: Request) => Prom
       meta: page.meta,
       config,
       nav,
-      currentSlug: page.slug
+      currentSlug: page.slug,
+      body: page.body
     })
 
-    return new Response(fullPage, {
-      status: 200,
-      headers: htmlHeaders()
-    })
+    const htmlHdrs = htmlHeadersWithCsp(config, slug)
+
+    // 304 Not Modified support for non-cached path
+    const ifNoneMatch = request.headers.get('If-None-Match')
+    if (ifNoneMatch != null && htmlHdrs.ETag != null && ifNoneMatch === htmlHdrs.ETag) {
+      return ok(new Response(null, { status: 304, headers: notModifiedHeaders(htmlHdrs) }))
+    }
+
+    const htmlResponse = textResponse(fullPage, { status: 200, headers: htmlHdrs })
+    if (cacheEnabled && responseCache != null) {
+      await responseCache.set(cacheKey, {
+        body: fullPage,
+        status: 200,
+        headers: htmlHdrs,
+        timestamp: Date.now()
+      }, config.cache?.maxAge)
+    }
+    return ok(htmlResponse)
   }
 }
 
+/**
+ * Read content length from the response Content-Length header.
+ * Returns 0 when the header is absent or unparseable.
+ */
+function readContentLength (response: Response): number {
+  const header = response.headers.get('Content-Length')
+  if (header == null) return 0
+  const parsed = parseInt(header, 10)
+  return isNaN(parsed) ? 0 : parsed
+}
+
+/** Create a Response with Content-Length set from the body string. */
+function textResponse (body: string, init: ResponseInit): Response {
+  const encoded = new TextEncoder().encode(body)
+  const headers = new Headers(init.headers)
+  headers.set('Content-Length', String(encoded.byteLength))
+  return new Response(encoded, { ...init, headers })
+}
+
+function htmlHeadersWithCsp (config: MkdnSiteConfig, slug?: string): Record<string, string> {
+  const headers = htmlHeaders(config.cache, slug)
+  if (config.csp?.enabled !== false) {
+    headers['Content-Security-Policy'] = buildCsp(config)
+  }
+  return headers
+}
+
 const STATIC_EXTENSIONS = new Set([
   '.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg', '.ico',
   '.mp4', '.webm', '.ogg', '.mp3', '.wav',
@@ -172,6 +455,43 @@ const MIME_TYPES: Record<string, string> = {
   '.eot': 'application/vnd.ms-fontobject'
 }
 
+/**
+ * Extract headers required by RFC 7232 for 304 Not Modified responses.
+ * Must include ETag, Cache-Control, Vary (and Content-Type if set).
+ */
+function notModifiedHeaders (original: Record<string, string>): Record<string, string> {
+  const headers: Record<string, string> = {}
+  if (original.ETag != null) headers.ETag = original.ETag
+  if (original['Cache-Control'] != null) headers['Cache-Control'] = original['Cache-Control']
+  if (original.Vary != null) headers.Vary = original.Vary
+  return headers
+}
+
+/**
+ * Constant-time string comparison to prevent timing side-channel attacks.
+ * Uses TextEncoder (Web API, available on all runtimes) instead of Buffer
+ * for Deno compatibility. Always compares full length regardless of mismatch.
+ */
+function timingSafeEqual (a: string, b: string): boolean {
+  const encoder = new TextEncoder()
+  const aBuf = encoder.encode(a)
+  const bBuf = encoder.encode(b)
+  if (aBuf.length !== bBuf.length) {
+    // Still do a full comparison to avoid leaking length info via timing
+    let mismatch = 1
+    for (let i = 0; i < bBuf.length; i++) {
+      mismatch |= bBuf[i] ^ bBuf[i]
+    }
+    void mismatch
+    return false
+  }
+  let mismatch = 0
+  for (let i = 0; i < aBuf.length; i++) {
+    mismatch |= aBuf[i] ^ bBuf[i]
+  }
+  return mismatch === 0
+}
+
 async function serveStatic (pathname: string, staticDir: string): Promise<Response> {
   try {
     const filePath = `${staticDir}${pathname}`