dexin-content 0.1.1 → 0.2.0

package/core/{compiler.ts → compiler/compiler.ts}

@@ -1,9 +1,9 @@
 // ─────────────────────────────────────────────────────────────
-// dexin-content/core/compiler.ts
+// dexin-content/core/compiler/compiler.ts
 //
 // Pipeline:
 //   read source → split frontmatter → project meta (SCHEMA-FREE) →
-//   validate schema (if any) → parse Markdown to DocumentAST →
+//   validate schema (if any) → parse Markdown to AST →
 //   dispatch to domain parser → assemble PositiveArtifact (5 fields).
 //
 // The compiler does NOT hard-code domain parsers internally. Instead they
@@ -13,20 +13,20 @@
 
 import type {
   DomainParser,
-  DocumentContent,
   Artifact,
   ParseContext,
-  DocumentIdentity,
+  Identity,
   Meta,
   ParseError,
   Schema
-} from './types'
+} from '../types'
+import type { LessonContent } from '../types/lessonAST'
 import {
   parseFrontmatter,
   splitFrontmatter,
   validateSchema
-} from './frontmatter'
-import { parseDocument } from './markdown'
+} from '../parser/frontmatter'
+import { parseDocument } from '../parser/markdown'
 
 export type ArtifactKind = 'positive' | 'error'
 
@@ -45,15 +45,15 @@ export class DomainParserRegistry {
 
 /**
  * Inputs required by `compile`. This is deliberately a plain object so the
- * host runner can mix-and-match per-collection identity derivation rules.
+ * host runner can mix-and-match per-fixture identity derivation rules.
  */
 export interface CompileInput {
-  /** Stable document id — used in the output Artifact.id field */
-  id: string
+  /** Fixture id (e.g. "L1-minimal") — used in output Artifact.fixture field */
+  fixture: string
   /** The domain tag to compile under. Determines parser lookup. */
   domain: string
-  /** Identity shape computed by host/discovery layer (document path convention). */
-  identity: DocumentIdentity
+  /** Identity shape computed by host/discovery layer (6.3.15 for docs). */
+  identity: Identity
   /** Raw Markdown source WITH frontmatter. */
   source: string
   /** File path used in error messages (relative, short form). */
@@ -65,8 +65,8 @@ export interface CompileInput {
 export interface CompileResult {
   kind: ArtifactKind
   artifact: Artifact
-  /** Parsed DocumentAST; undefined if parsing failed before this stage. */
-  docAST?: DocumentContent
+  /** Parsed AST; undefined if parsing failed before this stage. */
+  docAST?: LessonContent
   /** Parsed meta (SCHEMA-FREE scalar projection). Always present. */
   meta: Meta
   error?: Error & ParseError
@@ -87,15 +87,18 @@ export function compile (
     const { frontmatter, body } = splitFrontmatter(input.source)
     meta = parseFrontmatter(frontmatter, input.file)
 
-    // 2. Schema validation (schema is validator-only — doesn't change meta).
+    // 2. Schema validation (§6.3.13: schema is validator-only — doesn't change meta).
     validateSchema(meta, input.schema, input.file)
 
-    // 3. Markdown → DocumentAST (neutral).
+    // 3. Markdown → AST (neutral).
     //    Note: the GENERIC layer does NOT throw for md h5/h6 — it produces a
     //    structural HeadingBlock with synthetic level=4 and attaches the
     //    original md depth to the explicit HeadingBlock.mdDepth optional
     //    field. Fail-fast for out-of-range heading depths is the ACTIVE
-    //    DOMAIN PARSER's responsibility.
+    //    DOMAIN PARSER's responsibility per 6.3.10 frozen:
+    //      * document domain         → DOC_HEADING_DEPTH_UNDEFINED for md h5/h6
+    //      * structured-ext-A domain → HEADING_DEPTH_UNDEFINED for md h6
+    //                                   (md h5 allowed → level 4)
     const docAST = parseDocument(body, input.file)
 
     // 4. Domain parser dispatch.
@@ -112,18 +115,18 @@ export function compile (
       schema: input.schema,
       meta
     }
-    const content = parser.parse(docAST, ctx)
+    const parsed = parser.parse(docAST, ctx)
 
     return {
       kind: 'positive',
       meta,
       docAST,
       artifact: {
-        id: input.id,
+        fixture: input.fixture,
         domain: input.domain,
         identity: input.identity,
         meta,
-        content: { version: 1, blocks: content.blocks }
+        content: { version: 1, blocks: parsed.blocks }
       }
     }
   } catch (e) {
@@ -133,7 +136,7 @@ export function compile (
       meta,
       error: err,
       artifact: {
-        id: input.id,
+        fixture: input.fixture,
         domain: input.domain,
         identity: input.identity,
         meta,

package/core/compiler/discovery.ts

@@ -0,0 +1,116 @@
+// ─────────────────────────────────────────────────────────────
+// dexin-content/core/discovery.ts
+// Source adapter + collection routing + identity derivation helpers.
+//
+// Two identity conventions are supported. Both are neutral; the host
+// runner decides which one applies per collection.
+//
+//   DOCUMENT path convention → DocumentIdentity
+//     id   = relPath minus .md, minus trailing `/index`
+//     path = '/' + id
+//     file = relPath WITH .md extension
+//     + auxiliary fields: collection, (optionally) index_file:true
+//
+//   STRUCTURED path convention → StructuredIdentity
+//     slug / topic_slug / chapter_slug derived from collection layout.
+//     P0 fixture layout exercises: <topic>/<chapter>.md → slugs === chapter.
+// ─────────────────────────────────────────────────────────────
+
+import path from 'node:path'
+import { readFileSync, existsSync, statSync } from 'node:fs'
+import type { DocumentIdentity, StructuredIdentity, DomainName, ParseError } from '../types'
+
+export type IdentityKind = 'document' | 'structured'
+
+export interface CollectionConfig {
+  /** Collection name carried in identity.collection (document convention). */
+  name: string
+  /** Absolute path to the source root of this collection. */
+  sourceRoot: string
+  /** Domain tag this collection routes its source through. */
+  domain: DomainName | string
+}
+
+/**
+ * Normalise a relative path to forward slashes and strip any leading `./`.
+ */
+export function normaliseRel (p: string): string {
+  let out = p.split(path.sep).join('/')
+  while (out.startsWith('./')) out = out.slice(2)
+  return out
+}
+
+/**
+ * Build DocumentIdentity (6.3.15 frozen) from a `.md` file path relative
+ * to the collection source root.
+ */
+export function buildDocumentIdentity (
+  relPath: string,
+  collection: string
+): DocumentIdentity {
+  const norm = normaliseRel(relPath)
+  const file = norm
+
+  let id = file.endsWith('.md') ? file.slice(0, -'.md'.length) : file
+  if (id.endsWith('.markdown')) id = id.slice(0, -'.markdown'.length)
+
+  const isIndex = id.endsWith('/index') || id === 'index'
+  if (isIndex) {
+    if (id === 'index') id = ''
+    else id = id.slice(0, -'/index'.length)
+  }
+
+  const pathUrl = '/' + id
+  const identity: DocumentIdentity = {
+    id,
+    path: pathUrl,
+    file,
+    collection
+  }
+  if (isIndex) identity.index_file = true
+  return identity
+}
+
+/**
+ * Build StructuredIdentity (slug / topic_slug / chapter_slug) using
+ * explicit hierarchical tokens passed by the host. Runner supplies
+ * values directly based on fixture routing — no path-parsing guesswork
+ * happens here for the P0 corpus.
+ */
+export function buildStructuredIdentity (
+  params: { topic_slug: string; chapter_slug: string; slug?: string }
+): StructuredIdentity {
+  const { topic_slug, chapter_slug } = params
+  return {
+    slug: params.slug ?? chapter_slug,
+    topic_slug,
+    chapter_slug
+  }
+}
+
+/** Read a single UTF-8 file and pair it with its relative path. */
+export interface SourceFile {
+  absPath: string
+  relPath: string
+  source: string
+}
+
+export function readSourceFile (absPath: string, sourceRoot: string): SourceFile {
+  if (!existsSync(absPath) || !statSync(absPath).isFile()) {
+    throw new Error(`[discovery] Not a file or missing: ${absPath}`)
+  }
+  const raw = readFileSync(absPath, 'utf8')
+  const source = raw.charCodeAt(0) === 0xfeff ? raw.slice(1) : raw
+  if (source.includes('\r')) {
+    const rel = normaliseRel(path.relative(sourceRoot, absPath))
+    const err = new Error(
+      `[LINE_ENDING_CONTAMINATION] Source file '${rel}' contains CRLF line endings. ` +
+      `Normalize to LF before processing (e.g. sed -i 's/\\r$//' <file>).`
+    ) as ParseError
+    err.code = 'LINE_ENDING_CONTAMINATION'
+    err.file = rel
+    throw err
+  }
+  const rel = normaliseRel(path.relative(sourceRoot, absPath))
+  return { absPath, relPath: rel, source }
+}

package/core/{frontmatter.ts → parser/frontmatter.ts}

@@ -1,144 +1,145 @@
-// ─────────────────────────────────────────────────────────────
-// dexin-content/core/frontmatter.ts
-// Split YAML frontmatter + SCHEMA-FREE scalar projection.
-// SCHEMA-FREE RULE: meta = frontmatter scalar全集 (string|number|boolean)
-//   regardless of schema declaration. Schema only validates.
-// ─────────────────────────────────────────────────────────────
-
-import { parse as yamlParse } from 'yaml'
-import type { Meta, ParseError, Schema } from './types'
-
-export interface SplitResult {
-  /** Frontmatter block (verbatim text between `---` lines) — may be empty string */
-  frontmatter: string
-  /** Markdown body that follows the frontmatter block (may be empty string) */
-  body: string
-  /** Whether a YAML frontmatter block was actually present */
-  hasFrontmatter: boolean
-}
-
-/**
- * Split `---` delimited YAML frontmatter from the rest of the document.
- * Returns empty strings for frontmatter/body if the document has no block.
- * The first line must be exactly `---` for a block to be recognised.
- */
-export function splitFrontmatter (source: string): SplitResult {
-  if (!source.startsWith('---')) {
-    return { frontmatter: '', body: source, hasFrontmatter: false }
-  }
-  const firstLineEnd = source.indexOf('\n')
-  if (firstLineEnd === -1) {
-    return { frontmatter: '', body: source, hasFrontmatter: false }
-  }
-  const firstLine = source.slice(0, firstLineEnd).trimEnd()
-  if (firstLine !== '---') {
-    return { frontmatter: '', body: source, hasFrontmatter: false }
-  }
-  // Find closing `---` line starting at position after the opening line
-  const rest = source.slice(firstLineEnd + 1)
-  const lines = rest.split('\n')
-  let closeIdx = -1
-  for (let i = 0; i < lines.length; i++) {
-    if ((lines[i] ?? '').trimEnd() === '---') {
-      closeIdx = i
-      break
-    }
-  }
-  if (closeIdx === -1) {
-    return { frontmatter: '', body: source, hasFrontmatter: false }
-  }
-  const frontmatter = lines.slice(0, closeIdx).join('\n')
-  // Close-line index in `rest` coordinates: closeIdx * (line + newline) + lengthOfLine
-  // Simpler: rejoin after closeIdx, drop the --- line itself
-  const afterLines = lines.slice(closeIdx + 1)
-  // Preserve original line breaks by joining with \n
-  let body = afterLines.join('\n')
-  // If body begins with a single \n it's the newline right after closing fence — strip it
-  if (body.startsWith('\n')) body = body.slice(1)
-  return { frontmatter, body, hasFrontmatter: true }
-}
-
-/**
- * Project parsed YAML object to scalar-only meta.
- * SCHEMA-FREE RULE: scalar = string | number | boolean.
- * All other value types (null, undefined, object, array, bigint, date, …) are DROPPED.
- */
-export function projectMeta (raw: unknown): Meta {
-  const meta: Meta = {}
-  if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return meta
-  const obj = raw as Record<string, unknown>
-  for (const key of Object.keys(obj)) {
-    const v = obj[key]
-    switch (typeof v) {
-      case 'string':
-      case 'number':
-      case 'boolean':
-        meta[key] = v
-        break
-      default:
-        // drop silently — schema-free projection only keeps scalars
-        break
-    }
-  }
-  return meta
-}
-
-/** Parse the raw frontmatter string via yaml, then project scalars into Meta. */
-export function parseFrontmatter (raw: string, file?: string): Meta {
-  if (!raw.trim()) return {}
-  let parsed: unknown
-  try {
-    parsed = yamlParse(raw)
-  } catch (e) {
-    const yamlMsg = e instanceof Error ? e.message : String(e)
-    const fileCtx = file ? ` in ${file}` : ''
-    const err = new Error(
-      `[SCHEMA_VALIDATION_FAILED] Frontmatter YAML parse error${fileCtx}: ${yamlMsg}`
-    ) as ParseError
-    err.code = 'SCHEMA_VALIDATION_FAILED'
-    if (file) err.file = file
-    throw err
-  }
-  return projectMeta(parsed)
-}
-
-/**
- * Validate a meta object against the collection schema.
- * Throws a ParseError with code = SCHEMA_VALIDATION_FAILED when violated.
- * Caller is responsible for deciding when schema validation runs;
- * schema-less collections are allowed (SCHEMA-FREE RULE).
- */
-export function validateSchema (
-  meta: Meta,
-  schema: Schema | undefined,
-  file: string
-): void {
-  if (!schema) return
-  if (schema.required && schema.required.length > 0) {
-    for (const key of schema.required) {
-      if (!(key in meta)) {
-        const err = new Error(
-          `[SCHEMA_VALIDATION_FAILED] Missing required frontmatter field '${key}' in ${file}`
-        ) as ParseError
-        err.code = 'SCHEMA_VALIDATION_FAILED'
-        err.file = file
-        throw err
-      }
-    }
-  }
-  if (schema.types) {
-    for (const key of Object.keys(schema.types)) {
-      if (!(key in meta)) continue // presence checked separately
-      const expected = schema.types[key]
-      const actual = typeof meta[key]
-      if (actual !== expected) {
-        const err = new Error(
-          `[SCHEMA_VALIDATION_FAILED] Field '${key}' expected type ${expected}, got ${actual} in ${file}`
-        ) as ParseError
-        err.code = 'SCHEMA_VALIDATION_FAILED'
-        err.file = file
-        throw err
-      }
-    }
-  }
-}
+// ─────────────────────────────────────────────────────────────
+// dexin-content/core/frontmatter.ts
+// Split YAML frontmatter + SCHEMA-FREE scalar projection.
+// 6.3.13 frozen: meta = frontmatter scalar全集 (string|number|boolean)
+//   regardless of schema declaration. Schema only validates.
+// ─────────────────────────────────────────────────────────────
+
+import { parse as yamlParse } from 'yaml'
+import type { Meta, ParseError, Schema } from '../types'
+
+export interface SplitResult {
+  /** Frontmatter block (verbatim text between `---` lines) — may be empty string */
+  frontmatter: string
+  /** Markdown body that follows the frontmatter block (may be empty string) */
+  body: string
+  /** Whether a YAML frontmatter block was actually present */
+  hasFrontmatter: boolean
+}
+
+/**
+ * Split `---` delimited YAML frontmatter from the rest of the document.
+ * Returns empty strings for frontmatter/body if the document has no block.
+ * The first line must be exactly `---` for a block to be recognised.
+ */
+export function splitFrontmatter (source: string): SplitResult {
+  if (!source.startsWith('---')) {
+    return { frontmatter: '', body: source, hasFrontmatter: false }
+  }
+  const firstLineEnd = source.indexOf('\n')
+  if (firstLineEnd === -1) {
+    return { frontmatter: '', body: source, hasFrontmatter: false }
+  }
+  const firstLine = source.slice(0, firstLineEnd).trimEnd()
+  if (firstLine !== '---') {
+    return { frontmatter: '', body: source, hasFrontmatter: false }
+  }
+  // Find closing `---` line starting at position after the opening line
+  const rest = source.slice(firstLineEnd + 1)
+  const lines = rest.split('\n')
+  let closeIdx = -1
+  for (let i = 0; i < lines.length; i++) {
+    if (lines[i].trimEnd() === '---') {
+      closeIdx = i
+      break
+    }
+  }
+  if (closeIdx === -1) {
+    return { frontmatter: '', body: source, hasFrontmatter: false }
+  }
+  const frontmatter = lines.slice(0, closeIdx).join('\n')
+  // Close-line index in `rest` coordinates: closeIdx * (line + newline) + lengthOfLine
+  // Simpler: rejoin after closeIdx, drop the --- line itself
+  const afterLines = lines.slice(closeIdx + 1)
+  // Preserve original line breaks by joining with \n
+  let body = afterLines.join('\n')
+  // If body begins with a single \n it's the newline right after closing fence — strip it
+  if (body.startsWith('\n')) body = body.slice(1)
+  return { frontmatter, body, hasFrontmatter: true }
+}
+
+/**
+ * Project parsed YAML object to scalar-only meta.
+ * SCHEMA-FREE RULE (6.3.13): scalar = string | number | boolean.
+ * All other value types (null, undefined, object, array, bigint, date, …) are DROPPED.
+ */
+export function projectMeta (raw: unknown): Meta {
+  const meta: Meta = {}
+  if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return meta
+  const obj = raw as Record<string, unknown>
+  for (const key of Object.keys(obj)) {
+    const v = obj[key]
+    switch (typeof v) {
+      case 'string':
+      case 'number':
+      case 'boolean':
+        meta[key] = v
+        break
+      default:
+        // drop silently — schema-free projection only keeps scalars
+        break
+    }
+  }
+  return meta
+}
+
+/** Parse the raw frontmatter string via yaml, then project scalars into Meta. */
+export function parseFrontmatter (raw: string, file?: string): Meta {
+  if (!raw.trim()) return {}
+  let parsed: unknown
+  try {
+    parsed = yamlParse(raw)
+  } catch (e) {
+    const yamlMsg = e instanceof Error ? e.message : String(e)
+    const fileCtx = file ? ` in ${file}` : ''
+    const err = new Error(
+      `[SCHEMA_VALIDATION_FAILED] Frontmatter YAML parse error${fileCtx}: ${yamlMsg}`
+    ) as ParseError
+    err.code = 'SCHEMA_VALIDATION_FAILED'
+    if (file) err.file = file
+    throw err
+  }
+  return projectMeta(parsed)
+}
+
+/**
+ * Validate a meta object against the collection schema.
+ * Throws a ParseError with code = SCHEMA_VALIDATION_FAILED when violated.
+ * Caller is responsible for deciding when schema validation runs;
+ * schema-less collections are allowed (SCHEMA-FREE RULE).
+ */
+export function validateSchema (
+  meta: Meta,
+  schema: Schema | undefined,
+  file: string
+): void {
+  if (!schema) return
+  if (schema.required && schema.required.length > 0) {
+    for (const key of schema.required) {
+      if (!(key in meta)) {
+        const err = new Error(
+          `[SCHEMA_VALIDATION_FAILED] Missing required frontmatter field '${key}' in ${file}`
+        ) as ParseError
+        err.code = 'SCHEMA_VALIDATION_FAILED'
+        err.file = file
+        err.message = err.message
+        throw err
+      }
+    }
+  }
+  if (schema.types) {
+    for (const key of Object.keys(schema.types)) {
+      if (!(key in meta)) continue // presence checked separately
+      const expected = schema.types[key]
+      const actual = typeof meta[key]
+      if (actual !== expected) {
+        const err = new Error(
+          `[SCHEMA_VALIDATION_FAILED] Field '${key}' expected type ${expected}, got ${actual} in ${file}`
+        ) as ParseError
+        err.code = 'SCHEMA_VALIDATION_FAILED'
+        err.file = file
+        throw err
+      }
+    }
+  }
+}