dexin-content 0.1.0

package/core/discovery.ts

@@ -0,0 +1,110 @@
+// ─────────────────────────────────────────────────────────────
+// dexin-content/core/discovery.ts
+// Source adapter + collection routing + identity derivation helpers.
+//
+// Document path convention → DocumentIdentity:
+//   id   = relPath minus .md, minus trailing `/index`
+//   path = '/' + id
+//   file = relPath WITH .md extension
+//   + auxiliary fields: collection, (optionally) index_file:true
+//
+// Hosts needing domain-specific identity shapes define their own types
+// and construct CompileInput identities themselves; core only knows the
+// document convention.
+// ─────────────────────────────────────────────────────────────
+
+import path from 'node:path'
+import { readFileSync, existsSync, statSync } from 'node:fs'
+import type { DocumentIdentity, ParseError } from './types'
+
+// ── Shared BOM / CRLF sanitisation primitives ─────────────
+//    Used by both sync (readSourceFile) and async (createLocalSource.read)
+//    IO entry points to guarantee byte-identical failure behaviour.
+
+/**
+ * Build a LINE_ENDING_CONTAMINATION ParseError for a given relative path.
+ * Caller supplies a short remediation hint matching its context.
+ */
+export function mkLineEndingError (
+  relPath: string,
+  hint: string = 'Normalize to LF before processing.'
+): ParseError & Error {
+  const err = new Error(
+    `[LINE_ENDING_CONTAMINATION] Source file '${relPath}' contains CRLF line endings. ` +
+    hint
+  ) as ParseError & Error
+  err.code = 'LINE_ENDING_CONTAMINATION'
+  err.file = relPath
+  return err
+}
+
+/**
+ * Strip a leading UTF-8 BOM (U+FEFF) if present and guard against CRLF
+ * contamination. Returns the cleaned source string; throws with
+ * LINE_ENDING_CONTAMINATION code on CR presence.
+ *
+ * This is intentionally a pure string-in / string-out (or throw) helper so
+ * both sync and async IO paths share one canonical sanitisation.
+ */
+export function sanitizeSourceText (raw: string, relPath: string): string {
+  const source = raw.charCodeAt(0) === 0xfeff ? raw.slice(1) : raw
+  if (source.includes('\r')) throw mkLineEndingError(relPath)
+  return source
+}
+
+/**
+ * Normalise a relative path to forward slashes and strip any leading `./`.
+ */
+export function normaliseRel (p: string): string {
+  let out = p.split(path.sep).join('/')
+  while (out.startsWith('./')) out = out.slice(2)
+  return out
+}
+
+/**
+ * Build DocumentIdentity from a `.md` file path relative
+ * to the collection source root.
+ */
+export function buildDocumentIdentity (
+  relPath: string,
+  collection: string
+): DocumentIdentity {
+  const norm = normaliseRel(relPath)
+  const file = norm
+
+  let id = file.endsWith('.md') ? file.slice(0, -'.md'.length) : file
+  if (id.endsWith('.markdown')) id = id.slice(0, -'.markdown'.length)
+
+  const isIndex = id.endsWith('/index') || id === 'index'
+  if (isIndex) {
+    if (id === 'index') id = ''
+    else id = id.slice(0, -'/index'.length)
+  }
+
+  const pathUrl = '/' + id
+  const identity: DocumentIdentity = {
+    id,
+    path: pathUrl,
+    file,
+    collection
+  }
+  if (isIndex) identity.index_file = true
+  return identity
+}
+
+/** Read a single UTF-8 file and pair it with its relative path. */
+export interface SourceFile {
+  absPath: string
+  relPath: string
+  source: string
+}
+
+export function readSourceFile (absPath: string, sourceRoot: string): SourceFile {
+  if (!existsSync(absPath) || !statSync(absPath).isFile()) {
+    throw new Error(`[discovery] Not a file or missing: ${absPath}`)
+  }
+  const raw = readFileSync(absPath, 'utf8')
+  const rel = normaliseRel(path.relative(sourceRoot, absPath))
+  const source = sanitizeSourceText(raw, rel)
+  return { absPath, relPath: rel, source }
+}

package/core/frontmatter.ts

@@ -0,0 +1,144 @@
+// ─────────────────────────────────────────────────────────────
+// dexin-content/core/frontmatter.ts
+// Split YAML frontmatter + SCHEMA-FREE scalar projection.
+// SCHEMA-FREE RULE: meta = frontmatter scalar全集 (string|number|boolean)
+//   regardless of schema declaration. Schema only validates.
+// ─────────────────────────────────────────────────────────────
+
+import { parse as yamlParse } from 'yaml'
+import type { Meta, ParseError, Schema } from './types'
+
+export interface SplitResult {
+  /** Frontmatter block (verbatim text between `---` lines) — may be empty string */
+  frontmatter: string
+  /** Markdown body that follows the frontmatter block (may be empty string) */
+  body: string
+  /** Whether a YAML frontmatter block was actually present */
+  hasFrontmatter: boolean
+}
+
+/**
+ * Split `---` delimited YAML frontmatter from the rest of the document.
+ * Returns empty strings for frontmatter/body if the document has no block.
+ * The first line must be exactly `---` for a block to be recognised.
+ */
+export function splitFrontmatter (source: string): SplitResult {
+  if (!source.startsWith('---')) {
+    return { frontmatter: '', body: source, hasFrontmatter: false }
+  }
+  const firstLineEnd = source.indexOf('\n')
+  if (firstLineEnd === -1) {
+    return { frontmatter: '', body: source, hasFrontmatter: false }
+  }
+  const firstLine = source.slice(0, firstLineEnd).trimEnd()
+  if (firstLine !== '---') {
+    return { frontmatter: '', body: source, hasFrontmatter: false }
+  }
+  // Find closing `---` line starting at position after the opening line
+  const rest = source.slice(firstLineEnd + 1)
+  const lines = rest.split('\n')
+  let closeIdx = -1
+  for (let i = 0; i < lines.length; i++) {
+    if (lines[i].trimEnd() === '---') {
+      closeIdx = i
+      break
+    }
+  }
+  if (closeIdx === -1) {
+    return { frontmatter: '', body: source, hasFrontmatter: false }
+  }
+  const frontmatter = lines.slice(0, closeIdx).join('\n')
+  // Close-line index in `rest` coordinates: closeIdx * (line + newline) + lengthOfLine
+  // Simpler: rejoin after closeIdx, drop the --- line itself
+  const afterLines = lines.slice(closeIdx + 1)
+  // Preserve original line breaks by joining with \n
+  let body = afterLines.join('\n')
+  // If body begins with a single \n it's the newline right after closing fence — strip it
+  if (body.startsWith('\n')) body = body.slice(1)
+  return { frontmatter, body, hasFrontmatter: true }
+}
+
+/**
+ * Project parsed YAML object to scalar-only meta.
+ * SCHEMA-FREE RULE: scalar = string | number | boolean.
+ * All other value types (null, undefined, object, array, bigint, date, …) are DROPPED.
+ */
+export function projectMeta (raw: unknown): Meta {
+  const meta: Meta = {}
+  if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return meta
+  const obj = raw as Record<string, unknown>
+  for (const key of Object.keys(obj)) {
+    const v = obj[key]
+    switch (typeof v) {
+      case 'string':
+      case 'number':
+      case 'boolean':
+        meta[key] = v
+        break
+      default:
+        // drop silently — schema-free projection only keeps scalars
+        break
+    }
+  }
+  return meta
+}
+
+/** Parse the raw frontmatter string via yaml, then project scalars into Meta. */
+export function parseFrontmatter (raw: string, file?: string): Meta {
+  if (!raw.trim()) return {}
+  let parsed: unknown
+  try {
+    parsed = yamlParse(raw)
+  } catch (e) {
+    const yamlMsg = e instanceof Error ? e.message : String(e)
+    const fileCtx = file ? ` in ${file}` : ''
+    const err = new Error(
+      `[SCHEMA_VALIDATION_FAILED] Frontmatter YAML parse error${fileCtx}: ${yamlMsg}`
+    ) as ParseError
+    err.code = 'SCHEMA_VALIDATION_FAILED'
+    if (file) err.file = file
+    throw err
+  }
+  return projectMeta(parsed)
+}
+
+/**
+ * Validate a meta object against the collection schema.
+ * Throws a ParseError with code = SCHEMA_VALIDATION_FAILED when violated.
+ * Caller is responsible for deciding when schema validation runs;
+ * schema-less collections are allowed (SCHEMA-FREE RULE).
+ */
+export function validateSchema (
+  meta: Meta,
+  schema: Schema | undefined,
+  file: string
+): void {
+  if (!schema) return
+  if (schema.required && schema.required.length > 0) {
+    for (const key of schema.required) {
+      if (!(key in meta)) {
+        const err = new Error(
+          `[SCHEMA_VALIDATION_FAILED] Missing required frontmatter field '${key}' in ${file}`
+        ) as ParseError
+        err.code = 'SCHEMA_VALIDATION_FAILED'
+        err.file = file
+        throw err
+      }
+    }
+  }
+  if (schema.types) {
+    for (const key of Object.keys(schema.types)) {
+      if (!(key in meta)) continue // presence checked separately
+      const expected = schema.types[key]
+      const actual = typeof meta[key]
+      if (actual !== expected) {
+        const err = new Error(
+          `[SCHEMA_VALIDATION_FAILED] Field '${key}' expected type ${expected}, got ${actual} in ${file}`
+        ) as ParseError
+        err.code = 'SCHEMA_VALIDATION_FAILED'
+        err.file = file
+        throw err
+      }
+    }
+  }
+}

package/core/markdown.ts

@@ -0,0 +1,295 @@
+// ─────────────────────────────────────────────────────────────
+// dexin-content/core/markdown.ts
+// Unified pipeline → neutral DocumentAST.
+//
+// Responsibilities:
+//   * Parse markdown source (GFM tables/lists, LaTeX-format spans,
+//     generic directive containers) into MDAST nodes.
+//   * Walk MDAST nodes and produce DocumentBlock[] arrays with
+//     full inline-format coverage.
+//   * ContainerBlock: always neutral, attributes =
+//     Record<string,string> (derived from directive attributes).
+//   * Heading boundary behaviour for out-of-range markdown depth is
+//     delegated to the active domain parser; the generic layer clamps
+//     the syntactic level and records the original depth so domain
+//     parsers can run their own fail-fast checks.
+//   * Quote paragraph merging: soft newlines inside a single quote-
+//     wrapped paragraph collapse into one ParagraphBlock with a literal
+//     '\n' text node between its inlines (R-02). Separated paragraphs
+//     (blank-line delimited) remain separate, inside neutral
+//     ContainerBlock nodes (R-04), because blank lines mark paragraph
+//     boundaries.
+// ─────────────────────────────────────────────────────────────
+
+import { unified } from 'unified'
+import remarkParse from 'remark-parse'
+import remarkGfm from 'remark-gfm'
+import remarkMath from 'remark-math'
+import remarkDirective from 'remark-directive'
+
+import type {
+  DocumentBlock,
+  DocumentContent,
+  HeadingBlock,
+  ParagraphBlock,
+  QuoteBlock,
+  ListBlock,
+  TableBlock,
+  TableCell,
+  ImageBlock,
+  CodeBlock,
+  FormulaBlock,
+  ContainerBlock,
+  Inline,
+  ParseError
+} from './types'
+
+// mdast node type tags produced by remark-math (case-sensitive).
+const NODE_FORMULA_BLOCK  = 'math'        // block-level LaTeX fence
+const NODE_FORMULA_INLINE = 'inlineMath'  // inline LaTeX span
+
+// Minimal MDAST subset. The plugin output is walked by shape only.
+interface MdNode {
+  type: string
+  depth?: number
+  children?: MdNode[]
+  value?: string
+  alt?: string
+  url?: string
+  identifier?: string
+  label?: string
+  lang?: string
+  meta?: string
+  ordered?: boolean
+  spread?: boolean
+  start?: number
+  align?: ('left' | 'right' | 'center' | null)[]
+  position?: unknown
+  name?: string
+  attributes?: Record<string, string | null | undefined> | null
+}
+
+// ── Entry point ──────────────────────────────────────────
+
+export function parseToDocAST (source: string, file: string): DocumentContent {
+  const mdast = unified()
+    .use(remarkParse)
+    .use(remarkGfm)
+    .use(remarkMath)
+    .use(remarkDirective)
+    .parse(source) as MdNode
+
+  const blocks = walkBlocks(mdast.children ?? [], file, 'root')
+  return { version: 1, blocks }
+}
+
+export { parseToDocAST as parseDocument }
+
+// ── Block walkers ─────────────────────────────────────────
+
+type BlockScope = 'root' | 'blockquote' | 'container'
+
+function walkBlocks (nodes: MdNode[], file: string, scope: BlockScope): DocumentBlock[] {
+  const out: DocumentBlock[] = []
+  for (const n of nodes) {
+    switch (n.type) {
+      case 'heading':              out.push(walkHeading(n, file)); break
+      case 'paragraph':            out.push(walkParagraph(n, file)); break
+      case 'blockquote':           out.push(walkBlockquote(n, file)); break
+      case 'thematicBreak':        out.push({ type: 'divider' }); break
+      case 'list':                 out.push(walkList(n, file)); break
+      case 'table':                out.push(walkTable(n, file)); break
+      case 'image':                out.push(walkImage(n)); break
+      case 'code':                 out.push(walkCode(n)); break
+      case NODE_FORMULA_BLOCK:     out.push(walkFormula(n)); break
+      case 'containerDirective':   out.push(walkContainerDirective(n, file)); break
+      default: {
+        const err = new Error(
+          `[MDAST_UNSUPPORTED_NODE] Unsupported block node type '${n.type}' in ${file} (scope: ${scope}). ` +
+          `Fail-fast: unknown mdast block nodes must not be silently dropped.`
+        ) as ParseError
+        err.code = 'MDAST_UNSUPPORTED_NODE'
+        err.file = file
+        throw err
+      }
+    }
+  }
+  return out
+}
+
+// ── Individual block converters ──────────────────────────
+
+function walkHeading (n: MdNode, file: string): HeadingBlock {
+  const d = n.depth ?? 1
+  if (d < 1 || d > 6) {
+    // Defensive clamp (should not occur with standard mdast output).
+  }
+  // Boundary behaviour for heading depth is the active domain parser's
+  // responsibility. The generic layer produces a structural heading with
+  // synthetic level 1..4, and records the original markdown depth via the
+  // explicit optional HeadingBlock.mdDepth field so domain-level fail-fast
+  // checks run precisely. The generic layer never throws for md h5/h6;
+  // domain parsers decide their own policy.
+  //   - `mdDepth` is TRANSIENT: domain output render/clone paths must drop
+  //     it so it never appears in the canonical Artifact content.
+  const clamped = (d < 1 ? 1 : d > 6 ? 6 : d)
+  const syntheticLevel = (clamped <= 4 ? clamped : 4) as 1 | 2 | 3 | 4
+  const children = walkInlines(n.children ?? [], file)
+  const h: HeadingBlock = {
+    type: 'heading',
+    level: syntheticLevel,
+    children,
+    mdDepth: clamped
+  }
+  void file
+  return h
+}
+
+function walkParagraph (n: MdNode, file: string): ParagraphBlock {
+  return { type: 'paragraph', children: walkInlines(n.children ?? [], file) }
+}
+
+/**
+ * Quote block R-02 behaviour: a soft break inside a single wrapped
+ * paragraph renders as a TextInline('\n'), merging both halves into one
+ * ParagraphBlock with a literal newline rather than two paragraphs.
+ * Paragraphs separated by blank lines inside the quote remain separate.
+ */
+function walkBlockquote (n: MdNode, file: string): QuoteBlock {
+  const inner = walkBlocks(n.children ?? [], file, 'blockquote')
+  return { type: 'quote', children: inner }
+}
+
+function walkList (n: MdNode, file: string): ListBlock {
+  const ordered = !!n.ordered
+  const items: Inline[][] = []
+  for (const li of n.children ?? []) {
+    const itemInlines: Inline[] = []
+    const firstBlocks = li.children ?? []
+    let emitted = 0
+    for (const b of firstBlocks) {
+      if (b.type === 'paragraph') {
+        if (emitted > 0) itemInlines.push({ type: 'text', value: '\n' })
+        itemInlines.push(...walkInlines(b.children ?? [], file))
+        emitted++
+      } else {
+        const err = new Error(
+          `[MDAST_UNSUPPORTED_NODE] Unsupported list-item child type '${b.type}' in ${file}. ` +
+          `Fail-fast: non-paragraph list item children are not supported.`
+        ) as ParseError
+        err.code = 'MDAST_UNSUPPORTED_NODE'
+        err.file = file
+        throw err
+      }
+    }
+    items.push(itemInlines)
+  }
+  return { type: 'list', ordered, items }
+}
+
+function walkTable (n: MdNode, file: string): TableBlock {
+  const kids = n.children ?? []
+  const headers: TableCell[] = []
+  const rows: TableCell[][] = []
+  for (let i = 0; i < kids.length; i++) {
+    const row = kids[i]
+    const rowCells: TableCell[] = []
+    const cells = row.children ?? []
+    for (const cell of cells) rowCells.push(walkInlines(cell.children ?? [], file))
+    if (i === 0) for (const c of rowCells) headers.push(c)
+    else rows.push(rowCells)
+  }
+  return { type: 'table', headers, rows }
+}
+
+function walkImage (n: MdNode): ImageBlock {
+  return { type: 'image', src: n.url ?? '', alt: n.alt ?? '' }
+}
+
+function walkCode (n: MdNode): CodeBlock {
+  return { type: 'code', language: n.lang ?? '', code: n.value ?? '' }
+}
+
+function walkFormula (n: MdNode): FormulaBlock {
+  // Block-level LaTeX fence mdast node: value = latex body; display = true.
+  return { type: 'formula', latex: n.value ?? '', display: true }
+}
+
+/**
+ * Directive container → neutral ContainerBlock.
+ * Interpretation of a container's name and semantics is deferred to the
+ * active domain parser; here the structure is preserved verbatim.
+ */
+function walkContainerDirective (n: MdNode, file: string): ContainerBlock {
+  const name = n.name ?? 'unknown'
+  const attrsIn = n.attributes ?? {}
+  const attrs: Record<string, string> = {}
+  for (const key of Object.keys(attrsIn)) {
+    const v = attrsIn[key]
+    attrs[key] = v == null ? '' : String(v)
+  }
+  const children = walkBlocks(n.children ?? [], file, 'container')
+  return { type: 'container', name, attrs, children }
+}
+
+// ── Inline walkers ────────────────────────────────────────
+
+function walkInlines (nodes: MdNode[], file: string): Inline[] {
+  const out: Inline[] = []
+  for (const n of nodes) {
+    switch (n.type) {
+      case 'text':
+        out.push({ type: 'text', value: n.value ?? '' })
+        break
+      case 'strong':
+        out.push({ type: 'bold', children: walkInlines(n.children ?? [], file) })
+        break
+      case 'emphasis':
+        out.push({ type: 'italic', children: walkInlines(n.children ?? [], file) })
+        break
+      case 'inlineCode':
+        out.push({ type: 'code', value: n.value ?? '' })
+        break
+      case 'link':
+        out.push({
+          type: 'link',
+          url: n.url ?? '',
+          children: walkInlines(n.children ?? [], file)
+        })
+        break
+      case 'image':
+        out.push({ type: 'text', value: n.alt ?? n.url ?? '' })
+        break
+      case NODE_FORMULA_INLINE:
+        out.push({ type: 'math', latex: n.value ?? '' })
+        break
+      case 'break':
+      case 'softbreak':
+        out.push({ type: 'text', value: '\n' })
+        break
+      case 'html':
+        out.push({ type: 'text', value: n.value ?? '' })
+        break
+      case 'linkReference':
+        out.push({
+          type: 'link',
+          url: n.identifier ?? '',
+          children: walkInlines(n.children ?? [], file)
+        })
+        break
+      case 'textDirective':
+      case 'leafDirective':
+        out.push(...walkInlines(n.children ?? [], file))
+        break
+      default:
+        if (n.children && Array.isArray(n.children)) {
+          out.push(...walkInlines(n.children, file))
+        }
+        break
+    }
+  }
+  return out
+}
+
+// Keep imports used (silence warnings).
+export type { ParseError }