dexin-content 0.1.2 → 0.2.0

package/{collection.ts → core/artifact/collection.ts}

@@ -1,289 +1,297 @@
-// ─────────────────────────────────────────────────────────────
-// dexin-content/collection.ts
-//
-// Collection declaration + source adaptation + discovery +
-// batch compile orchestration.
-//
-// The batch compileCollections() wraps the formal per-file
-// core/compiler.ts compile() — it discovers files, builds
-// CompileInput per file, calls compile, and writes PositiveArtifact
-// docs + ContentIndex to the store.
-//
-// grep-zero: no business vocabulary appears anywhere in this
-// module — including error messages.
-// ─────────────────────────────────────────────────────────────
-
-import { readdir, readFile } from 'node:fs/promises'
-import { join } from 'node:path'
-import { createHash } from 'node:crypto'
-import type {
-  PositiveArtifact,
-  DomainName
-} from './core/types'
-import type { DomainParserRegistry, CompileInput } from './core/compiler'
-import { compile } from './core/compiler'
-import { buildDocumentIdentity, sanitizeSourceText } from './core/discovery'
-import { splitFrontmatter } from './core/frontmatter'
-import type { ArtifactStore, ContentIndex, IndexEntry } from './store'
-
-// ── Collection types ──
-
-export interface CollectionDefinition {
-  /** Collection name (appears in identity.collection + IndexEntry.collection). */
-  name: string
-  /** Directory under content root; .md files collected recursively. */
-  source: string
-  /** Domain tag; defaults to 'document' when omitted. */
-  domain?: DomainName | string
-  /** Frontmatter schema (reuses core Schema; validator-only, no meta projection). */
-  schema?: { required?: string[]; types?: Record<string, 'string' | 'number' | 'boolean'> }
-}
-
-export interface ResolvedCollection extends CollectionDefinition {
-  domain: string
-  sourceRoot: string
-}
-
-/** Declaration passthrough (declarative API semantic); resolution in resolveCollections. */
-export function defineCollection (def: CollectionDefinition): CollectionDefinition {
-  return def
-}
-
-/** Resolve definitions: fill domain default + bind absolute sourceRoot. */
-export function resolveCollections (
-  defs: CollectionDefinition[],
-  contentRoot: string
-): ResolvedCollection[] {
-  return defs.map(d => ({
-    ...d,
-    domain: d.domain ?? 'document',
-    sourceRoot: join(contentRoot, d.source)
-  }))
-}
-
-// ── SourceAdapter ──
-
-export interface SourceAdapter {
-  list(): Promise<string[]>
-  read(relPath: string): Promise<string>
-}
-
-/**
- * Local filesystem source. Reads .md files recursively from root.
- * CRLF protection (R-d): read() throws LINE_ENDING_CONTAMINATION
- * on CRLF so the batch path enjoys the same guard as the synchronous
- * single-file read path.
- *
- * Sanitisation (BOM strip + CRLF guard) is delegated to the shared
- * `sanitizeSourceText` pure helper in core/discovery — byte-identical
- * to the synchronous readSourceFile() path.
- */
-export function createLocalSource (root: string): SourceAdapter {
-  async function walk (dir: string, base: string): Promise<string[]> {
-    const entries = await readdir(dir, { withFileTypes: true })
-    const out: string[] = []
-    for (const e of entries) {
-      const rel = base ? `${base}/${e.name}` : e.name
-      if (e.isDirectory()) out.push(...await walk(join(dir, e.name), rel))
-      else if (e.isFile() && e.name.endsWith('.md')) out.push(rel)
-    }
-    return out.sort()
-  }
-  return {
-    list: () => walk(root, ''),
-    read: async (rel) => {
-      const raw = await readFile(join(root, rel), 'utf8')
-      return sanitizeSourceText(raw, rel)
-    }
-  }
-}
-
-/** Virtual in-memory source (test use). */
-export function createMemorySource (files: Record<string, string>): SourceAdapter {
-  return {
-    list: async () => Object.keys(files).sort(),
-    read: async (rel) => {
-      if (!(rel in files)) throw new Error(`[MemorySource] Not found: ${rel}`)
-      return files[rel]!
-    }
-  }
-}
-
-// ── Discovery ──
-
-export interface ContentFile {
-  id: string
-  path: string
-  collection: string
-  file: string
-  /**
-   * Raw source (YAML frontmatter + Markdown body), already BOM/CRLF sanitised.
-   * Cached at discovery phase so compileCollections() avoids a second IO read
-   * per file; the split into frontmatter/body happens only when needed below.
-   */
-  raw: string
-}
-
-/**
- * Scan all collections via source. Files not belonging to any
- * collection → throw (fail-fast, no silent drop).
- *
- * Identity derivation: document path convention via
- * buildDocumentIdentity (id = rel minus .md minus trailing /index,
- * path = '/' + id, file = rel, collection = name). Non-document
- * domains are not supported in the batch path; hosts requiring
- * custom identity shapes must construct CompileInput per file
- * themselves — a thrown error makes the gap explicit.
- */
-export async function discover (
-  source: SourceAdapter,
-  collections: ResolvedCollection[]
-): Promise<ContentFile[]> {
-  const byDir = new Map(collections.map(c => [c.source, c]))
-  const all = await source.list()
-  const files: ContentFile[] = []
-  for (const rel of all) {
-    const top = rel.split('/')[0]!
-    const col = byDir.get(top)
-    if (!col) {
-      throw new Error(
-        `[discover] File '${rel}' belongs to no collection (top dir '${top}' undefined).`
-      )
-    }
-    if (col.domain !== 'document') {
-      throw new Error(
-        `[discover] Domain '${col.domain}' discovery not yet implemented in batch path; ` +
-        `only 'document' domain is supported here. Non-document domains require ` +
-        `explicit identity construction.`
-      )
-    }
-    const identity = buildDocumentIdentity(rel, col.name)
-    const raw = await source.read(rel)
-    files.push({
-      id: identity.id,
-      path: identity.path,
-      collection: col.name,
-      file: identity.file,
-      raw
-    })
-  }
-  return files
-}
-
-// ── Batch compile orchestration ──
-
-function checksum (s: string): string {
-  return createHash('md5').update(s, 'utf-8').digest('hex').slice(0, 12)
-}
-
-/**
- * Batch compile: discover → per-file compile → write docs + index to store.
- *
- * For each ContentFile:
- *   1. Build CompileInput (id = file.id, identity = buildDocumentIdentity,
- *      source = raw, schema = col.schema)
- *   2. Call formal compile(input, registry)
- *   3. On positive → collect PositiveArtifact + IndexEntry
- *      On error → collect error message (fail-fast: throw after all files processed)
- *
- * Store receives only positive artifacts. compile failures are aggregated
- * and thrown as a single Error.
- */
-export async function compileCollections (
-  source: SourceAdapter,
-  collections: ResolvedCollection[],
-  registry: DomainParserRegistry,
-  store: ArtifactStore
-): Promise<{ docs: PositiveArtifact[]; index: ContentIndex }> {
-  const files = await discover(source, collections)
-  const docs: PositiveArtifact[] = []
-  const entries: IndexEntry[] = []
-  const errors: string[] = []
-
-  for (const f of files) {
-    const col = collections.find(c => c.name === f.collection)!
-    // Reuse discovery-cached source; no second per-file IO read.
-    // `body` (post-frontmatter) is derived once solely for the
-    // change-detection checksum — compile() takes the full raw
-    // source per its stable CompileInput contract.
-    const { body } = splitFrontmatter(f.raw)
-    const input: CompileInput = {
-      id: f.id,
-      domain: col.domain,
-      identity: buildDocumentIdentity(f.file, col.name),
-      source: f.raw,
-      file: f.file,
-      schema: col.schema
-    }
-    const result = compile(input, registry)
-    if (result.kind !== 'positive') {
-      errors.push(`  ✗ ${f.file}: ${result.error?.message ?? 'unknown'}`)
-      continue
-    }
-    const artifact = result.artifact as PositiveArtifact
-    docs.push(artifact)
-    entries.push({
-      id: f.id,
-      path: f.path,
-      collection: f.collection,
-      domain: col.domain,
-      file: f.file,
-      meta: result.meta,
-      checksum: checksum(body)
-    })
-  }
-
-  if (errors.length > 0) {
-    throw new Error(`Compile failed (${errors.length} error(s)):\n${errors.join('\n')}`)
-  }
-
-  const index: ContentIndex = {
-    generator: 'dexin-content',
-    builtAt: new Date().toISOString(),
-    entries
-  }
-
-  await store.writeIndex(index)
-  for (const doc of docs) {
-    await store.writeDoc(doc)
-  }
-
-  return { docs, index }
-}
-
-// ── Incremental recompile ──
-
-/**
- * Detect changed/removed docs by checksum diff against the previous index.
- *
- * Wraps compileCollections (which writes index + all docs to store), then
- * reports which ids changed (checksum differs from prev) or were removed
- * (present in prev, absent in next). Used by Dev flows (memory store)
- * to drive incremental re-render signals. Writing all docs is acceptable
- * for ephemeral in-memory Dev stores. Watcher / preview server
- * integrations are out of scope.
- */
-export async function recompileChanged (
-  source: SourceAdapter,
-  collections: ResolvedCollection[],
-  registry: DomainParserRegistry,
-  store: ArtifactStore
-): Promise<{ changed: string[]; removed: string[] }> {
-  const prev = await store.readIndex()
-  const prevMap = new Map((prev?.entries ?? []).map(e => [e.id, e]))
-
-  const { index } = await compileCollections(source, collections, registry, store)
-
-  const nextIds = new Set(index.entries.map(e => e.id))
-  const changed: string[] = []
-  const removed: string[] = []
-
-  for (const e of index.entries) {
-    if (prevMap.get(e.id)?.checksum !== e.checksum) changed.push(e.id)
-  }
-  for (const id of prevMap.keys()) {
-    if (!nextIds.has(id)) removed.push(id)
-  }
-
-  return { changed, removed }
-}
+// ─────────────────────────────────────────────────────────────
+// dexin-content/collection.ts
+//
+// Collection declaration + source adaptation + discovery +
+// batch compile orchestration (API-FREEZE §3.1–§3.4).
+//
+// The batch compileCollections() wraps the formal per-file
+// core/compiler.ts compile() — it discovers files, builds
+// CompileInput per file, calls compile, and writes PositiveArtifact
+// docs + ContentIndex to the store.
+//
+// grep-zero: no business vocabulary appears anywhere in this
+// module — including error messages.
+// ─────────────────────────────────────────────────────────────
+
+import { readdir, readFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { createHash } from 'node:crypto'
+import type {
+  PositiveArtifact,
+  Identity,
+  DomainName,
+  Meta,
+  ParseError
+} from '../types'
+import type { DomainParserRegistry, CompileInput } from '../compiler/compiler'
+import { compile } from '../compiler/compiler'
+import { buildDocumentIdentity } from '../compiler/discovery'
+import { splitFrontmatter, parseFrontmatter, validateSchema } from '../parser/frontmatter'
+import type { ArtifactStore, ContentIndex, IndexEntry } from './store'
+
+// ── Collection types (API-FREEZE §3.1) ──
+
+export interface CollectionDefinition {
+  /** Collection name (appears in identity.collection + IndexEntry.collection). */
+  name: string
+  /** Directory under content root; .md files collected recursively. */
+  source: string
+  /** Domain tag; defaults to 'document' when omitted. */
+  domain?: DomainName | string
+  /** Frontmatter schema (reuses core Schema; validator-only, no meta projection). */
+  schema?: { required?: string[]; types?: Record<string, 'string' | 'number' | 'boolean'> }
+}
+
+export interface ResolvedCollection extends CollectionDefinition {
+  domain: string
+  sourceRoot: string
+}
+
+/** Declaration passthrough (declarative API semantic); resolution in resolveCollections. */
+export function defineCollection (def: CollectionDefinition): CollectionDefinition {
+  return def
+}
+
+/** Resolve definitions: fill domain default + bind absolute sourceRoot. */
+export function resolveCollections (
+  defs: CollectionDefinition[],
+  contentRoot: string
+): ResolvedCollection[] {
+  return defs.map(d => ({
+    ...d,
+    domain: d.domain ?? 'document',
+    sourceRoot: join(contentRoot, d.source)
+  }))
+}
+
+// ── SourceAdapter (API-FREEZE §3.2) ──
+
+export interface SourceAdapter {
+  list(): Promise<string[]>
+  read(relPath: string): Promise<string>
+}
+
+/**
+ * Local filesystem source. Reads .md files recursively from root.
+ * CRLF protection (R-d): read() throws LINE_ENDING_CONTAMINATION
+ * on CRLF so the batch path enjoys the same guard as run-p0.
+ */
+export function createLocalSource (root: string): SourceAdapter {
+  async function walk (dir: string, base: string): Promise<string[]> {
+    const entries = await readdir(dir, { withFileTypes: true })
+    const out: string[] = []
+    for (const e of entries) {
+      const rel = base ? `${base}/${e.name}` : e.name
+      if (e.isDirectory()) out.push(...await walk(join(dir, e.name), rel))
+      else if (e.isFile() && e.name.endsWith('.md')) out.push(rel)
+    }
+    return out.sort()
+  }
+  return {
+    list: () => walk(root, ''),
+    read: async (rel) => {
+      const raw = await readFile(join(root, rel), 'utf8')
+      const src = raw.charCodeAt(0) === 0xfeff ? raw.slice(1) : raw
+      if (src.includes('\r')) {
+        const err = new Error(
+          `[LINE_ENDING_CONTAMINATION] Source file '${rel}' contains CRLF line endings. ` +
+          `Normalize to LF before processing.`
+        ) as ParseError
+        err.code = 'LINE_ENDING_CONTAMINATION'
+        err.file = rel
+        throw err
+      }
+      return src
+    }
+  }
+}
+
+/** Virtual in-memory source (test / fixture use). */
+export function createMemorySource (files: Record<string, string>): SourceAdapter {
+  return {
+    list: async () => Object.keys(files).sort(),
+    read: async (rel) => {
+      if (!(rel in files)) throw new Error(`[MemorySource] Not found: ${rel}`)
+      return files[rel]
+    }
+  }
+}
+
+// ── Discovery (API-FREEZE §3.3) ──
+
+export interface ContentFile {
+  id: string
+  path: string
+  collection: string
+  file: string
+  frontmatter: string
+  body: string
+}
+
+/**
+ * Scan all collections via source. Files not belonging to any
+ * collection → throw (fail-fast, no silent drop).
+ *
+ * Identity derivation: document-domain path convention via
+ * buildDocumentIdentity (id = rel minus .md minus trailing /index,
+ * path = '/' + id, file = rel, collection = name). Non-document
+ * domains are not yet supported in the batch path (T5 lands the
+ * structured-ext-A domain discovery rules); a thrown error makes
+ * the gap explicit.
+ */
+export async function discover (
+  source: SourceAdapter,
+  collections: ResolvedCollection[]
+): Promise<ContentFile[]> {
+  const byDir = new Map(collections.map(c => [c.source, c]))
+  const all = await source.list()
+  const files: ContentFile[] = []
+  for (const rel of all) {
+    const top = rel.split('/')[0]
+    const col = byDir.get(top)
+    if (!col) {
+      throw new Error(
+        `[discover] File '${rel}' belongs to no collection (top dir '${top}' undefined).`
+      )
+    }
+    if (col.domain !== 'document') {
+      throw new Error(
+        `[discover] Domain '${col.domain}' discovery not yet implemented in batch path; ` +
+        `only 'document' domain is supported here. Non-document domains require ` +
+        `explicit identity construction (see run-p0 FixtureSpec pattern).`
+      )
+    }
+    const identity = buildDocumentIdentity(rel, col.name)
+    const raw = await source.read(rel)
+    const { frontmatter, body } = splitFrontmatter(raw)
+    files.push({
+      id: identity.id,
+      path: identity.path,
+      collection: col.name,
+      file: identity.file,
+      frontmatter,
+      body
+    })
+  }
+  return files
+}
+
+// ── Batch compile orchestration (API-FREEZE §3.3) ──
+
+function checksum (s: string): string {
+  return createHash('md5').update(s, 'utf-8').digest('hex').slice(0, 12)
+}
+
+/**
+ * Batch compile: discover → per-file compile → write docs + index to store.
+ *
+ * For each ContentFile:
+ *   1. Build CompileInput (fixture = file.id, identity = buildDocumentIdentity,
+ *      source = raw, schema = col.schema)
+ *   2. Call formal compile(input, registry)
+ *   3. On positive → collect PositiveArtifact + IndexEntry
+ *      On error → collect error message (fail-fast: throw after all files processed)
+ *
+ * Store receives only positive artifacts. compile failures are aggregated
+ * and thrown as a single Error (prototype-aligned behaviour).
+ */
+export async function compileCollections (
+  source: SourceAdapter,
+  collections: ResolvedCollection[],
+  registry: DomainParserRegistry,
+  store: ArtifactStore
+): Promise<{ docs: PositiveArtifact[]; index: ContentIndex }> {
+  const files = await discover(source, collections)
+  const docs: PositiveArtifact[] = []
+  const entries: IndexEntry[] = []
+  const errors: string[] = []
+
+  for (const f of files) {
+    const col = collections.find(c => c.name === f.collection)!
+    const raw = await source.read(f.file)
+    const input: CompileInput = {
+      fixture: f.id,
+      domain: col.domain,
+      identity: buildDocumentIdentity(f.file, col.name),
+      source: raw,
+      file: f.file,
+      schema: col.schema
+    }
+    const result = compile(input, registry)
+    if (result.kind !== 'positive') {
+      errors.push(`  ✗ ${f.file}: ${result.error?.message ?? 'unknown'}`)
+      continue
+    }
+    const artifact = result.artifact as PositiveArtifact
+    docs.push(artifact)
+    entries.push({
+      id: f.id,
+      path: f.path,
+      collection: f.collection,
+      domain: col.domain,
+      file: f.file,
+      meta: result.meta,
+      checksum: checksum(f.body)
+    })
+  }
+
+  if (errors.length > 0) {
+    throw new Error(`Compile failed (${errors.length} error(s)):\n${errors.join('\n')}`)
+  }
+
+  const index: ContentIndex = {
+    generator: 'dexin-content',
+    builtAt: new Date().toISOString(),
+    entries
+  }
+
+  await store.writeIndex(index)
+  for (const doc of docs) {
+    await store.writeDoc(doc)
+  }
+
+  return { docs, index }
+}
+
+// ── Incremental recompile (API-FREEZE §3.3, prototype L663 移植) ──
+
+/**
+ * Detect changed/removed docs by checksum diff against the previous index.
+ *
+ * Wraps compileCollections (which writes index + all docs to store), then
+ * reports which ids changed (checksum differs from prev) or were removed
+ * (present in prev, absent in next). Used by the Dev flow (memory store)
+ * to drive incremental re-render signals.
+ *
+ * Adapts prototype's recompileChanged to the formal layer: prototype's
+ * `compile(source, collections, domains)` returned { docs, index } without
+ * writing to store; our compileCollections writes index + all docs to
+ * store AND returns { docs, index }. The changed/removed detection is
+ * preserved; for in-memory Dev stores writing all docs is acceptable
+ * (ephemeral). Per B1-SPEC T5.4 "最小形态", watcher + preview server
+ * are explicitly out of scope.
+ */
+export async function recompileChanged (
+  source: SourceAdapter,
+  collections: ResolvedCollection[],
+  registry: DomainParserRegistry,
+  store: ArtifactStore
+): Promise<{ changed: string[]; removed: string[] }> {
+  const prev = await store.readIndex()
+  const prevMap = new Map((prev?.entries ?? []).map(e => [e.id, e]))
+
+  const { index } = await compileCollections(source, collections, registry, store)
+
+  const nextIds = new Set(index.entries.map(e => e.id))
+  const changed: string[] = []
+  const removed: string[] = []
+
+  for (const e of index.entries) {
+    if (prevMap.get(e.id)?.checksum !== e.checksum) changed.push(e.id)
+  }
+  for (const id of prevMap.keys()) {
+    if (!nextIds.has(id)) removed.push(id)
+  }
+
+  return { changed, removed }
+}