mkdnsite 0.0.1

package/src/content/filesystem.ts

@@ -0,0 +1,210 @@
+import { readdir, readFile, stat } from 'node:fs/promises'
+import { join, relative, extname, basename, dirname } from 'node:path'
+import { parseFrontmatter } from './frontmatter.ts'
+import type { ContentSource, ContentPage, NavNode } from './types.ts'
+
+export class FilesystemSource implements ContentSource {
+  private readonly rootDir: string
+  private readonly cache = new Map<string, ContentPage>()
+  private navCache: NavNode | null = null
+  private allPagesCache: ContentPage[] | null = null
+
+  constructor (rootDir: string) {
+    this.rootDir = rootDir
+  }
+
+  async getPage (slug: string): Promise<ContentPage | null> {
+    const stripped = slug.replace(/^\/+|\/+$/g, '')
+    const normalized = stripped !== '' ? stripped : 'index'
+
+    if (this.cache.has(normalized)) {
+      return this.cache.get(normalized) ?? null
+    }
+
+    const candidates = [
+      join(this.rootDir, `${normalized}.md`),
+      join(this.rootDir, normalized, 'index.md'),
+      join(this.rootDir, normalized, 'README.md'),
+      join(this.rootDir, normalized, 'readme.md')
+    ]
+
+    if (normalized === 'index') {
+      candidates.unshift(
+        join(this.rootDir, 'index.md'),
+        join(this.rootDir, 'README.md'),
+        join(this.rootDir, 'readme.md')
+      )
+    }
+
+    for (const filePath of candidates) {
+      try {
+        const raw = await readFile(filePath, 'utf-8')
+        const parsed = parseFrontmatter(raw)
+        const fileStat = await stat(filePath)
+
+        const page: ContentPage = {
+          slug: `/${normalized === 'index' ? '' : normalized}`,
+          sourcePath: filePath,
+          meta: parsed.meta,
+          body: parsed.body,
+          raw: parsed.raw,
+          modifiedAt: fileStat.mtime
+        }
+
+        this.cache.set(normalized, page)
+        return page
+      } catch {
+        continue
+      }
+    }
+
+    return null
+  }
+
+  async getNavTree (): Promise<NavNode> {
+    if (this.navCache != null) return this.navCache
+    this.navCache = await this.buildNavTree(this.rootDir, '/')
+    return this.navCache
+  }
+
+  async listPages (): Promise<ContentPage[]> {
+    if (this.allPagesCache != null) return this.allPagesCache
+
+    const pages: ContentPage[] = []
+    await this.walkDir(this.rootDir, pages)
+    pages.sort((a, b) => a.slug.localeCompare(b.slug))
+    this.allPagesCache = pages
+    return pages
+  }
+
+  async refresh (): Promise<void> {
+    this.cache.clear()
+    this.navCache = null
+    this.allPagesCache = null
+  }
+
+  private async walkDir (dir: string, pages: ContentPage[]): Promise<void> {
+    let entries
+    try {
+      entries = await readdir(dir, { withFileTypes: true })
+    } catch {
+      return
+    }
+
+    for (const entry of entries) {
+      const fullPath = join(dir, entry.name)
+
+      if (entry.isDirectory()) {
+        if (entry.name.startsWith('.') || entry.name === 'node_modules') {
+          continue
+        }
+        await this.walkDir(fullPath, pages)
+      } else if (entry.isFile() && extname(entry.name) === '.md') {
+        const relPath = relative(this.rootDir, fullPath)
+        const slug = this.filePathToSlug(relPath)
+
+        try {
+          const raw = await readFile(fullPath, 'utf-8')
+          const parsed = parseFrontmatter(raw)
+          const fileStat = await stat(fullPath)
+
+          if (parsed.meta.draft === true) continue
+
+          pages.push({
+            slug,
+            sourcePath: fullPath,
+            meta: parsed.meta,
+            body: parsed.body,
+            raw: parsed.raw,
+            modifiedAt: fileStat.mtime
+          })
+        } catch {
+          continue
+        }
+      }
+    }
+  }
+
+  private filePathToSlug (relPath: string): string {
+    let slug = relPath.replace(/\.md$/, '')
+
+    const base = basename(slug)
+    if (base === 'index' || base === 'README' || base === 'readme') {
+      slug = dirname(slug)
+      if (slug === '.') slug = ''
+    }
+
+    slug = slug.replace(/\\/g, '/')
+    return `/${slug}`
+  }
+
+  private async buildNavTree (dir: string, slugPrefix: string): Promise<NavNode> {
+    const entries = await readdir(dir, { withFileTypes: true })
+    const children: NavNode[] = []
+
+    let dirTitle = basename(dir)
+    let dirOrder = 0
+
+    for (const name of ['index.md', 'README.md', 'readme.md']) {
+      try {
+        const raw = await readFile(join(dir, name), 'utf-8')
+        const parsed = parseFrontmatter(raw)
+        if (parsed.meta.title != null) dirTitle = parsed.meta.title
+        if (parsed.meta.order != null) dirOrder = parsed.meta.order
+        break
+      } catch {
+        continue
+      }
+    }
+
+    for (const entry of entries) {
+      if (entry.name.startsWith('.') || entry.name === 'node_modules') continue
+
+      const fullPath = join(dir, entry.name)
+      const childSlug = `${slugPrefix}${slugPrefix.endsWith('/') ? '' : '/'}${entry.name}`
+
+      if (entry.isDirectory()) {
+        const subtree = await this.buildNavTree(fullPath, childSlug)
+        children.push(subtree)
+      } else if (
+        entry.isFile() &&
+        extname(entry.name) === '.md' &&
+        entry.name !== 'index.md' &&
+        entry.name !== 'README.md' &&
+        entry.name !== 'readme.md'
+      ) {
+        const raw = await readFile(fullPath, 'utf-8')
+        const parsed = parseFrontmatter(raw)
+
+        if (parsed.meta.draft === true) continue
+
+        const name = basename(entry.name, '.md')
+        const slug = `${slugPrefix}${slugPrefix.endsWith('/') ? '' : '/'}${name}`
+
+        children.push({
+          title: parsed.meta.title ?? titleCase(name),
+          slug,
+          order: parsed.meta.order ?? 999,
+          children: [],
+          isSection: false
+        })
+      }
+    }
+
+    children.sort((a, b) => (a.order !== 0 || b.order !== 0) ? a.order - b.order : a.title.localeCompare(b.title))
+
+    return {
+      title: titleCase(dirTitle),
+      slug: slugPrefix,
+      order: dirOrder,
+      children,
+      isSection: true
+    }
+  }
+}
+
+function titleCase (str: string): string {
+  return str
+    .replace(/[-_]/g, ' ')
+    .replace(/\b\w/g, c => c.toUpperCase())
+}

package/src/content/frontmatter.ts

@@ -0,0 +1,66 @@
+import matter from 'gray-matter'
+import type { MarkdownMeta } from './types.ts'
+
+export interface ParsedMarkdown {
+  meta: MarkdownMeta
+  body: string
+  raw: string
+}
+
+/**
+ * Parse YAML frontmatter from a markdown string.
+ * Uses gray-matter for full YAML support, falls back to regex parser.
+ */
+export function parseFrontmatter (raw: string): ParsedMarkdown {
+  if (!raw.startsWith('---')) {
+    return { meta: {}, body: raw, raw }
+  }
+
+  try {
+    const result = matter(raw)
+    return {
+      meta: result.data as MarkdownMeta,
+      body: result.content,
+      raw
+    }
+  } catch {
+    return parseFrontmatterSimple(raw)
+  }
+}
+
+/**
+ * Lightweight fallback parser for environments without gray-matter.
+ * Handles simple key: value pairs (no nested YAML).
+ */
+function parseFrontmatterSimple (raw: string): ParsedMarkdown {
+  const match = raw.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n([\s\S]*)$/)
+  if (match == null) {
+    return { meta: {}, body: raw, raw }
+  }
+
+  const [, frontmatter, body] = match
+  const meta: MarkdownMeta = {}
+
+  for (const line of frontmatter.split('\n')) {
+    const colonIdx = line.indexOf(':')
+    if (colonIdx === -1) continue
+
+    const key = line.slice(0, colonIdx).trim()
+    const rawValue = line.slice(colonIdx + 1).trim()
+
+    if (rawValue === 'true') {
+      (meta as Record<string, unknown>)[key] = true
+    } else if (rawValue === 'false') {
+      (meta as Record<string, unknown>)[key] = false
+    } else if (rawValue.startsWith('[') && rawValue.endsWith(']')) {
+      (meta as Record<string, unknown>)[key] = rawValue
+        .slice(1, -1)
+        .split(',')
+        .map(s => s.trim().replace(/^["']|["']$/g, ''))
+    } else {
+      (meta as Record<string, unknown>)[key] = rawValue
+    }
+  }
+
+  return { meta, body, raw }
+}

package/src/content/github.ts

@@ -0,0 +1,211 @@
+import { parseFrontmatter } from './frontmatter.ts'
+import type {
+  ContentSource,
+  ContentPage,
+  NavNode,
+  GitHubSourceConfig
+} from './types.ts'
+
+/**
+ * Content source that reads .md files from a public GitHub repository.
+ *
+ * Uses the GitHub Contents API (or raw.githubusercontent.com for performance).
+ * Supports caching for production use (bring your own cache backend).
+ *
+ * Rate limits:
+ * - Unauthenticated: 60 requests/hour
+ * - Authenticated (token): 5,000 requests/hour
+ *
+ * For the hosted service (mkdn.io), this source will be paired with:
+ * - Upstash Redis cache for rendered pages
+ * - GitHub webhook for cache invalidation on push
+ * - Optional R2 mirror for high-traffic sites
+ */
+export class GitHubSource implements ContentSource {
+  private readonly config: Required<GitHubSourceConfig>
+  private readonly cache = new Map<string, ContentPage>()
+  private navCache: NavNode | null = null
+
+  constructor (config: GitHubSourceConfig) {
+    this.config = {
+      owner: config.owner,
+      repo: config.repo,
+      ref: config.ref ?? 'main',
+      path: config.path ?? '',
+      token: config.token ?? ''
+    }
+  }
+
+  async getPage (slug: string): Promise<ContentPage | null> {
+    const stripped = slug.replace(/^\/+|\/+$/g, '')
+    const normalized = stripped !== '' ? stripped : 'index'
+
+    if (this.cache.has(normalized)) {
+      return this.cache.get(normalized) ?? null
+    }
+
+    const candidates = [
+      `${normalized}.md`,
+      `${normalized}/index.md`,
+      `${normalized}/README.md`,
+      `${normalized}/readme.md`
+    ]
+
+    if (normalized === 'index') {
+      candidates.unshift(
+        'index.md',
+        'README.md',
+        'readme.md'
+      )
+    }
+
+    for (const filePath of candidates) {
+      const raw = await this.fetchFile(filePath)
+      if (raw != null) {
+        const parsed = parseFrontmatter(raw)
+        const page: ContentPage = {
+          slug: `/${normalized === 'index' ? '' : normalized}`,
+          sourcePath: filePath,
+          meta: parsed.meta,
+          body: parsed.body,
+          raw: parsed.raw
+        }
+
+        this.cache.set(normalized, page)
+        return page
+      }
+    }
+
+    return null
+  }
+
+  async getNavTree (): Promise<NavNode> {
+    if (this.navCache != null) return this.navCache
+
+    // Fetch the repo tree via GitHub API
+    const tree = await this.fetchRepoTree()
+    this.navCache = this.buildNavFromTree(tree)
+    return this.navCache
+  }
+
+  async listPages (): Promise<ContentPage[]> {
+    const tree = await this.fetchRepoTree()
+    const mdFiles = tree.filter(f => f.path.endsWith('.md'))
+
+    const pages: ContentPage[] = []
+    for (const file of mdFiles) {
+      const slug = file.path
+        .replace(/\.md$/, '')
+        .replace(/\/index$/, '')
+        .replace(/\/README$/, '')
+        .replace(/\/readme$/, '')
+      const page = await this.getPage(slug)
+      if (page != null) pages.push(page)
+    }
+
+    return pages.sort((a, b) => a.slug.localeCompare(b.slug))
+  }
+
+  async refresh (): Promise<void> {
+    this.cache.clear()
+    this.navCache = null
+  }
+
+  /**
+   * Fetch a file from the GitHub repo via raw.githubusercontent.com
+   * (faster than the API, no rate limit headers, but no metadata).
+   */
+  private async fetchFile (filePath: string): Promise<string | null> {
+    const { owner, repo, ref, path: basePath } = this.config
+    const fullPath = basePath !== '' ? `${basePath}/${filePath}` : filePath
+    const url = `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${fullPath}`
+
+    const headers: Record<string, string> = {}
+    if (this.config.token !== '') {
+      headers.Authorization = `token ${this.config.token}`
+    }
+
+    try {
+      const response = await fetch(url, { headers })
+      if (!response.ok) return null
+      return await response.text()
+    } catch {
+      return null
+    }
+  }
+
+  /**
+   * Fetch the full repo tree via the GitHub Git Trees API.
+   * Uses ?recursive=1 to get all files in one request.
+   */
+  private async fetchRepoTree (): Promise<GitHubTreeEntry[]> {
+    const { owner, repo, ref } = this.config
+    const url = `https://api.github.com/repos/${owner}/${repo}/git/trees/${ref}?recursive=1`
+
+    const headers: Record<string, string> = {
+      Accept: 'application/vnd.github.v3+json'
+    }
+    if (this.config.token !== '') {
+      headers.Authorization = `token ${this.config.token}`
+    }
+
+    try {
+      const response = await fetch(url, { headers })
+      if (!response.ok) return []
+
+      const data = await response.json() as { tree: GitHubTreeEntry[] }
+      const basePath = this.config.path
+
+      return data.tree
+        .filter(entry =>
+          entry.type === 'blob' &&
+          entry.path.endsWith('.md') &&
+          (basePath === '' || entry.path.startsWith(basePath + '/'))
+        )
+        .map(entry => ({
+          ...entry,
+          path: basePath !== '' ? entry.path.slice(basePath.length + 1) : entry.path
+        }))
+    } catch {
+      return []
+    }
+  }
+
+  private buildNavFromTree (entries: GitHubTreeEntry[]): NavNode {
+    // TODO: Build proper tree structure from flat file list
+    // For now, return a flat list
+    const children: NavNode[] = entries
+      .filter(e => e.path.endsWith('.md') && e.path !== 'index.md' && e.path !== 'README.md' && e.path !== 'readme.md')
+      .map(e => {
+        const name = e.path.replace(/\.md$/, '').split('/').pop() ?? e.path
+        return {
+          title: titleCase(name),
+          slug: '/' + e.path.replace(/\.md$/, '').replace(/\/index$/, ''),
+          order: 999,
+          children: [],
+          isSection: false
+        }
+      })
+
+    return {
+      title: 'Root',
+      slug: '/',
+      order: 0,
+      children,
+      isSection: true
+    }
+  }
+}
+
+interface GitHubTreeEntry {
+  path: string
+  type: string
+  sha: string
+  size?: number
+}
+
+function titleCase (str: string): string {
+  return str
+    .replace(/[-_]/g, ' ')
+    .replace(/\b\w/g, c => c.toUpperCase())
+}

package/src/content/types.ts

@@ -0,0 +1,86 @@
+/**
+ * Parsed frontmatter metadata from a markdown file.
+ */
+export interface MarkdownMeta {
+  title?: string
+  description?: string
+  date?: string
+  updated?: string
+  draft?: boolean
+  order?: number
+  tags?: string[]
+  layout?: string
+  [key: string]: unknown
+}
+
+/**
+ * A single content page derived from a .md file.
+ */
+export interface ContentPage {
+  /** URL path slug (e.g. /docs/getting-started) */
+  slug: string
+  /** Source path (filesystem path, R2 key, GitHub path, etc.) */
+  sourcePath: string
+  /** Parsed frontmatter metadata */
+  meta: MarkdownMeta
+  /** Raw markdown body (without frontmatter) */
+  body: string
+  /** Original raw content (with frontmatter) */
+  raw: string
+  /** Last modified timestamp */
+  modifiedAt?: Date
+}
+
+/**
+ * A node in the site navigation tree.
+ */
+export interface NavNode {
+  title: string
+  slug: string
+  order: number
+  children: NavNode[]
+  isSection: boolean
+}
+
+/**
+ * Content source interface.
+ *
+ * Implementations provide content from different backends:
+ * - FilesystemSource: local .md files (dev, self-hosted)
+ * - GitHubSource: public GitHub repo (hosted service)
+ * - R2Source: Cloudflare R2 bucket (hosted service)
+ * - S3Source: AWS S3 bucket
+ * - MemorySource: in-memory (testing)
+ *
+ * The interface is intentionally minimal so new backends
+ * are easy to implement.
+ */
+export interface ContentSource {
+  /** Get a single page by its URL slug. Returns null if not found. */
+  getPage: (slug: string) => Promise<ContentPage | null>
+
+  /** Get the full navigation tree. */
+  getNavTree: () => Promise<NavNode>
+
+  /** List all pages (for llms.txt, sitemap, etc.) */
+  listPages: () => Promise<ContentPage[]>
+
+  /** Refresh/invalidate any caches. */
+  refresh: () => Promise<void>
+}
+
+/**
+ * Configuration for GitHub-based content source.
+ */
+export interface GitHubSourceConfig {
+  /** GitHub owner (user or org) */
+  owner: string
+  /** Repository name */
+  repo: string
+  /** Branch or tag (default: main) */
+  ref?: string
+  /** Subdirectory within the repo to treat as content root */
+  path?: string
+  /** GitHub personal access token (for private repos or higher rate limits) */
+  token?: string
+}

package/src/discovery/llmstxt.ts

@@ -0,0 +1,70 @@
+import type { ContentSource, ContentPage } from '../content/types.ts'
+import type { MkdnSiteConfig } from '../config/schema.ts'
+
+export async function generateLlmsTxt (
+  source: ContentSource,
+  config: MkdnSiteConfig
+): Promise<string> {
+  const pages = await source.listPages()
+  const baseUrl = config.site.url ?? ''
+
+  const lines: string[] = []
+
+  lines.push(`# ${config.site.title}`)
+  lines.push('')
+
+  const desc = config.llmsTxt.description ?? config.site.description
+  if (desc != null) {
+    lines.push(`> ${desc}`)
+    lines.push('')
+  }
+
+  const groups = groupBySection(pages)
+
+  for (const [section, sectionPages] of Object.entries(groups)) {
+    const sectionTitle = config.llmsTxt.sections?.[section] ??
+      (section === '_root' ? 'Pages' : titleCase(section))
+
+    lines.push(`## ${sectionTitle}`)
+    lines.push('')
+
+    for (const page of sectionPages) {
+      const title = page.meta.title ?? slugToTitle(page.slug)
+      const url = `${baseUrl}${page.slug}`
+      const pageDesc = page.meta.description
+
+      lines.push(pageDesc != null
+        ? `- [${title}](${url}): ${pageDesc}`
+        : `- [${title}](${url})`
+      )
+    }
+
+    lines.push('')
+  }
+
+  return lines.join('\n')
+}
+
+function groupBySection (pages: ContentPage[]): Record<string, ContentPage[]> {
+  const groups: Record<string, ContentPage[]> = {}
+
+  for (const page of pages) {
+    const segments = page.slug.replace(/^\//, '').split('/')
+    const section = segments.length > 1 ? segments[0] : '_root'
+
+    if (groups[section] == null) groups[section] = []
+    groups[section].push(page)
+  }
+
+  return groups
+}
+
+function titleCase (str: string): string {
+  return str.replace(/[-_]/g, ' ').replace(/\b\w/g, c => c.toUpperCase())
+}
+
+function slugToTitle (slug: string): string {
+  const name = slug.split('/').pop() ?? slug
+  const cleaned = name.replace(/^\//, '')
+  return titleCase(cleaned !== '' ? cleaned : 'Home')
+}