dexin-content 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 得心实验室 (Dexin Labs — https://gitee.com/cuizhn)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,112 @@
+# dexin-content
+
+得心实验室的**结构化内容编译与运行时 package**。
+
+负责与具体业务领域无关的通用内容管线：
+
+- **Compile**：将带 frontmatter 的 Markdown 编译成中性 Content AST，再经宿主注入的 `DomainParser` 产出稳定的 PositiveArtifact
+- **Store**：Artifact 与索引（ContentIndex）的 FS/Memory 抽象
+- **Query**：按 id / URL path / collection / 条件过滤对 Artifact 做只读查询
+- **Collection**：集合声明、发现、批量编译与增量变化检测
+- **Diff**：Canonical JSON 序列化与对象级差异原语（Golden/Regress 管线使用）
+
+## 目录范围
+
+dexin-content 是**通用层**：
+- `core/` 编译核心（types / compiler / markdown / frontmatter / discovery）
+- 根层 `collection.ts`、`store.ts`、`query.ts`、`diff.ts` 是稳定能力
+- `scripts/` 仓内脚本（不在发布包 files 内，仅供本地开发/测试；也不作为公开 API）
+
+宿主应用负责域扩展（DomainParser）、编排脚本与内容文件所有权，通过 `exports` 里的子路径消费本 package。
+
+## 安装
+
+> 需要 Node.js ≥ 20。Package 以 TypeScript 源码形式分发（`*.ts`），宿主应通过 `tsx`、Nuxt/Vite 等支持 TS 解析的工具链使用。
+
+```bash
+# Gitee 源（当前发布前）
+npm install git+https://gitee.com/cuizhn/dexin-content.git
+
+# 发布到 npm 后
+npm install dexin-content
+```
+
+## 公开 API 边界
+
+唯一公共 API 边界是 `package.json` 中 `exports` 字段。**只使用下列路径**：
+
+| 子路径 | 暴露内容 |
+|---|---|
+| `.` | 门面：核心类型 + `DomainParserRegistry` / `compile` + `ArtifactStore` / `ContentQuery` + `defineCollection / resolveCollections / discover / compileCollections` |
+| `./core/types` | 类型：`Inline` / `DocumentBlock` / `DocumentContent` / `DocumentIdentity` / `PositiveArtifact` / `Artifact` / `Meta` / `ParseError` / `DomainParser` / `Schema` / `ParseContext` |
+| `./core/compiler` | `compile`、`CompileInput`、`CompileResult`、`DomainParserRegistry` |
+| `./core/discovery` | `buildDocumentIdentity` / `normaliseRel` / `readSourceFile` / `sanitizeSourceText` / `mkLineEndingError` |
+| `./core/frontmatter` | `splitFrontmatter` / `parseFrontmatter` / `validateSchema` / `projectMeta` |
+| `./store` | `ArtifactStore` 接口 + `createFsArtifactStore` / `createMemoryArtifactStore` + `ContentIndex` / `IndexEntry` |
+| `./collection` | `defineCollection` / `resolveCollections` / `compileCollections` / `recompileChanged` / `createLocalSource` / `createMemorySource` / `SourceAdapter` / `ResolvedCollection` / `CollectionDefinition` |
+| `./query` | `ContentQuery` (byId / byPath / list / collection) + `QueryOptions` |
+| `./diff` | `toCanonicalJSON` / `sortKeysDeep` / `stripUnderscoreKeysGolden` / `underscorePrefixedPaths` / `firstDiff` / `shortStr` |
+
+任何未列入上表的内部路径均不保证稳定。
+
+## 快速示例
+
+```ts
+import {
+  defineCollection,
+  resolveCollections,
+  createLocalSource,
+  compileCollections,
+} from 'dexin-content/collection'
+import { createFsArtifactStore } from 'dexin-content/store'
+import { ContentQuery } from 'dexin-content/query'
+import { DomainParserRegistry } from 'dexin-content/core/compiler'
+import type {
+  DomainParser,
+  DocumentContent,
+  ParseContext,
+} from 'dexin-content/core/types'
+
+// 1) 注入你的域 parser；parser 决定中性 AST 如何映射到目标 Artifact 内容结构
+const documentParser: DomainParser = {
+  domain: 'document',
+  parse(content: DocumentContent, _ctx: ParseContext) {
+    return { version: 1, blocks: content.blocks as unknown[] }
+  },
+}
+const registry = new DomainParserRegistry()
+registry.register(documentParser)
+
+// 2) 声明集合 + 编译
+const collections = resolveCollections(
+  [defineCollection({ name: 'legal', source: 'legal', domain: 'document' })],
+  '/path/to/content/root',
+)
+const source = createLocalSource('/path/to/content/root')
+const store = createFsArtifactStore('/path/to/output/dexin-content')
+await compileCollections(source, collections, registry, store)
+
+// 3) 运行时只读查询（不解析任何 Markdown）
+const query = new ContentQuery(store)
+const entry = await query.byPath('/legal/privacy')
+```
+
+## 开发
+
+```bash
+# 依赖
+npm install
+
+# 类型检查（唯一静态门禁）
+npm run typecheck
+
+# 仓内自检测（需先 build 输出 output/dexin-content/）
+npm run runtime:check
+
+# 预发布 tarball 内容检查
+npm run pack
+```
+
+## License
+
+MIT © 得心实验室。详见 [LICENSE](./LICENSE)。

package/collection.ts

@@ -0,0 +1,289 @@
+// ─────────────────────────────────────────────────────────────
+// dexin-content/collection.ts
+//
+// Collection declaration + source adaptation + discovery +
+// batch compile orchestration.
+//
+// The batch compileCollections() wraps the formal per-file
+// core/compiler.ts compile() — it discovers files, builds
+// CompileInput per file, calls compile, and writes PositiveArtifact
+// docs + ContentIndex to the store.
+//
+// grep-zero: no business vocabulary appears anywhere in this
+// module — including error messages.
+// ─────────────────────────────────────────────────────────────
+
+import { readdir, readFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { createHash } from 'node:crypto'
+import type {
+  PositiveArtifact,
+  DomainName
+} from './core/types'
+import type { DomainParserRegistry, CompileInput } from './core/compiler'
+import { compile } from './core/compiler'
+import { buildDocumentIdentity, sanitizeSourceText } from './core/discovery'
+import { splitFrontmatter } from './core/frontmatter'
+import type { ArtifactStore, ContentIndex, IndexEntry } from './store'
+
+// ── Collection types ──
+
+export interface CollectionDefinition {
+  /** Collection name (appears in identity.collection + IndexEntry.collection). */
+  name: string
+  /** Directory under content root; .md files collected recursively. */
+  source: string
+  /** Domain tag; defaults to 'document' when omitted. */
+  domain?: DomainName | string
+  /** Frontmatter schema (reuses core Schema; validator-only, no meta projection). */
+  schema?: { required?: string[]; types?: Record<string, 'string' | 'number' | 'boolean'> }
+}
+
+export interface ResolvedCollection extends CollectionDefinition {
+  domain: string
+  sourceRoot: string
+}
+
+/** Declaration passthrough (declarative API semantic); resolution in resolveCollections. */
+export function defineCollection (def: CollectionDefinition): CollectionDefinition {
+  return def
+}
+
+/** Resolve definitions: fill domain default + bind absolute sourceRoot. */
+export function resolveCollections (
+  defs: CollectionDefinition[],
+  contentRoot: string
+): ResolvedCollection[] {
+  return defs.map(d => ({
+    ...d,
+    domain: d.domain ?? 'document',
+    sourceRoot: join(contentRoot, d.source)
+  }))
+}
+
+// ── SourceAdapter ──
+
+export interface SourceAdapter {
+  list(): Promise<string[]>
+  read(relPath: string): Promise<string>
+}
+
+/**
+ * Local filesystem source. Reads .md files recursively from root.
+ * CRLF protection (R-d): read() throws LINE_ENDING_CONTAMINATION
+ * on CRLF so the batch path enjoys the same guard as the synchronous
+ * single-file read path.
+ *
+ * Sanitisation (BOM strip + CRLF guard) is delegated to the shared
+ * `sanitizeSourceText` pure helper in core/discovery — byte-identical
+ * to the synchronous readSourceFile() path.
+ */
+export function createLocalSource (root: string): SourceAdapter {
+  async function walk (dir: string, base: string): Promise<string[]> {
+    const entries = await readdir(dir, { withFileTypes: true })
+    const out: string[] = []
+    for (const e of entries) {
+      const rel = base ? `${base}/${e.name}` : e.name
+      if (e.isDirectory()) out.push(...await walk(join(dir, e.name), rel))
+      else if (e.isFile() && e.name.endsWith('.md')) out.push(rel)
+    }
+    return out.sort()
+  }
+  return {
+    list: () => walk(root, ''),
+    read: async (rel) => {
+      const raw = await readFile(join(root, rel), 'utf8')
+      return sanitizeSourceText(raw, rel)
+    }
+  }
+}
+
+/** Virtual in-memory source (test use). */
+export function createMemorySource (files: Record<string, string>): SourceAdapter {
+  return {
+    list: async () => Object.keys(files).sort(),
+    read: async (rel) => {
+      if (!(rel in files)) throw new Error(`[MemorySource] Not found: ${rel}`)
+      return files[rel]
+    }
+  }
+}
+
+// ── Discovery ──
+
+export interface ContentFile {
+  id: string
+  path: string
+  collection: string
+  file: string
+  /**
+   * Raw source (YAML frontmatter + Markdown body), already BOM/CRLF sanitised.
+   * Cached at discovery phase so compileCollections() avoids a second IO read
+   * per file; the split into frontmatter/body happens only when needed below.
+   */
+  raw: string
+}
+
+/**
+ * Scan all collections via source. Files not belonging to any
+ * collection → throw (fail-fast, no silent drop).
+ *
+ * Identity derivation: document path convention via
+ * buildDocumentIdentity (id = rel minus .md minus trailing /index,
+ * path = '/' + id, file = rel, collection = name). Non-document
+ * domains are not supported in the batch path; hosts requiring
+ * custom identity shapes must construct CompileInput per file
+ * themselves — a thrown error makes the gap explicit.
+ */
+export async function discover (
+  source: SourceAdapter,
+  collections: ResolvedCollection[]
+): Promise<ContentFile[]> {
+  const byDir = new Map(collections.map(c => [c.source, c]))
+  const all = await source.list()
+  const files: ContentFile[] = []
+  for (const rel of all) {
+    const top = rel.split('/')[0]
+    const col = byDir.get(top)
+    if (!col) {
+      throw new Error(
+        `[discover] File '${rel}' belongs to no collection (top dir '${top}' undefined).`
+      )
+    }
+    if (col.domain !== 'document') {
+      throw new Error(
+        `[discover] Domain '${col.domain}' discovery not yet implemented in batch path; ` +
+        `only 'document' domain is supported here. Non-document domains require ` +
+        `explicit identity construction.`
+      )
+    }
+    const identity = buildDocumentIdentity(rel, col.name)
+    const raw = await source.read(rel)
+    files.push({
+      id: identity.id,
+      path: identity.path,
+      collection: col.name,
+      file: identity.file,
+      raw
+    })
+  }
+  return files
+}
+
+// ── Batch compile orchestration ──
+
+function checksum (s: string): string {
+  return createHash('md5').update(s, 'utf-8').digest('hex').slice(0, 12)
+}
+
+/**
+ * Batch compile: discover → per-file compile → write docs + index to store.
+ *
+ * For each ContentFile:
+ *   1. Build CompileInput (id = file.id, identity = buildDocumentIdentity,
+ *      source = raw, schema = col.schema)
+ *   2. Call formal compile(input, registry)
+ *   3. On positive → collect PositiveArtifact + IndexEntry
+ *      On error → collect error message (fail-fast: throw after all files processed)
+ *
+ * Store receives only positive artifacts. compile failures are aggregated
+ * and thrown as a single Error.
+ */
+export async function compileCollections (
+  source: SourceAdapter,
+  collections: ResolvedCollection[],
+  registry: DomainParserRegistry,
+  store: ArtifactStore
+): Promise<{ docs: PositiveArtifact[]; index: ContentIndex }> {
+  const files = await discover(source, collections)
+  const docs: PositiveArtifact[] = []
+  const entries: IndexEntry[] = []
+  const errors: string[] = []
+
+  for (const f of files) {
+    const col = collections.find(c => c.name === f.collection)!
+    // Reuse discovery-cached source; no second per-file IO read.
+    // `body` (post-frontmatter) is derived once solely for the
+    // change-detection checksum — compile() takes the full raw
+    // source per its stable CompileInput contract.
+    const { body } = splitFrontmatter(f.raw)
+    const input: CompileInput = {
+      id: f.id,
+      domain: col.domain,
+      identity: buildDocumentIdentity(f.file, col.name),
+      source: f.raw,
+      file: f.file,
+      schema: col.schema
+    }
+    const result = compile(input, registry)
+    if (result.kind !== 'positive') {
+      errors.push(`  ✗ ${f.file}: ${result.error?.message ?? 'unknown'}`)
+      continue
+    }
+    const artifact = result.artifact as PositiveArtifact
+    docs.push(artifact)
+    entries.push({
+      id: f.id,
+      path: f.path,
+      collection: f.collection,
+      domain: col.domain,
+      file: f.file,
+      meta: result.meta,
+      checksum: checksum(body)
+    })
+  }
+
+  if (errors.length > 0) {
+    throw new Error(`Compile failed (${errors.length} error(s)):\n${errors.join('\n')}`)
+  }
+
+  const index: ContentIndex = {
+    generator: 'dexin-content',
+    builtAt: new Date().toISOString(),
+    entries
+  }
+
+  await store.writeIndex(index)
+  for (const doc of docs) {
+    await store.writeDoc(doc)
+  }
+
+  return { docs, index }
+}
+
+// ── Incremental recompile ──
+
+/**
+ * Detect changed/removed docs by checksum diff against the previous index.
+ *
+ * Wraps compileCollections (which writes index + all docs to store), then
+ * reports which ids changed (checksum differs from prev) or were removed
+ * (present in prev, absent in next). Used by Dev flows (memory store)
+ * to drive incremental re-render signals. Writing all docs is acceptable
+ * for ephemeral in-memory Dev stores. Watcher / preview server
+ * integrations are out of scope.
+ */
+export async function recompileChanged (
+  source: SourceAdapter,
+  collections: ResolvedCollection[],
+  registry: DomainParserRegistry,
+  store: ArtifactStore
+): Promise<{ changed: string[]; removed: string[] }> {
+  const prev = await store.readIndex()
+  const prevMap = new Map((prev?.entries ?? []).map(e => [e.id, e]))
+
+  const { index } = await compileCollections(source, collections, registry, store)
+
+  const nextIds = new Set(index.entries.map(e => e.id))
+  const changed: string[] = []
+  const removed: string[] = []
+
+  for (const e of index.entries) {
+    if (prevMap.get(e.id)?.checksum !== e.checksum) changed.push(e.id)
+  }
+  for (const id of prevMap.keys()) {
+    if (!nextIds.has(id)) removed.push(id)
+  }
+
+  return { changed, removed }
+}

package/core/compiler.ts

@@ -0,0 +1,174 @@
+// ─────────────────────────────────────────────────────────────
+// dexin-content/core/compiler.ts
+//
+// Pipeline:
+//   read source → split frontmatter → project meta (SCHEMA-FREE) →
+//   validate schema (if any) → parse Markdown to DocumentAST →
+//   dispatch to domain parser → assemble PositiveArtifact (5 fields).
+//
+// The compiler does NOT hard-code domain parsers internally. Instead they
+// are injected via `DomainParserRegistry` by the host runner, satisfying
+// the dependency inversion principle.
+// ─────────────────────────────────────────────────────────────
+
+import type {
+  DomainParser,
+  DocumentContent,
+  Artifact,
+  ParseContext,
+  DocumentIdentity,
+  Meta,
+  ParseError,
+  Schema
+} from './types'
+import {
+  parseFrontmatter,
+  splitFrontmatter,
+  validateSchema
+} from './frontmatter'
+import { parseDocument } from './markdown'
+
+export type ArtifactKind = 'positive' | 'error'
+
+export class DomainParserRegistry {
+  private parsers = new Map<string, DomainParser>()
+  register (parser: DomainParser): void {
+    if (this.parsers.has(parser.domain)) {
+      throw new Error(`[compiler] Domain parser already registered for '${parser.domain}'`)
+    }
+    this.parsers.set(parser.domain, parser)
+  }
+  get (domain: string): DomainParser | undefined {
+    return this.parsers.get(domain)
+  }
+}
+
+/**
+ * Inputs required by `compile`. This is deliberately a plain object so the
+ * host runner can mix-and-match per-collection identity derivation rules.
+ */
+export interface CompileInput {
+  /** Stable document id — used in the output Artifact.id field */
+  id: string
+  /** The domain tag to compile under. Determines parser lookup. */
+  domain: string
+  /** Identity shape computed by host/discovery layer (document path convention). */
+  identity: DocumentIdentity
+  /** Raw Markdown source WITH frontmatter. */
+  source: string
+  /** File path used in error messages (relative, short form). */
+  file: string
+  /** Optional collection schema for frontmatter validation. */
+  schema?: Schema
+}
+
+export interface CompileResult {
+  kind: ArtifactKind
+  artifact: Artifact
+  /** Parsed DocumentAST; undefined if parsing failed before this stage. */
+  docAST?: DocumentContent
+  /** Parsed meta (SCHEMA-FREE scalar projection). Always present. */
+  meta: Meta
+  error?: Error & ParseError
+}
+
+/**
+ * Run the compile pipeline once. Fail-fast errors are returned wrapped as
+ * CompileResult with kind = 'error' — no throw out of this function, so the
+ * runner can log + diff against L5 descriptor.
+ */
+export function compile (
+  input: CompileInput,
+  registry: DomainParserRegistry
+): CompileResult {
+  let meta: Meta = {}
+  try {
+    // 1. Frontmatter split + parse + SCHEMA-FREE scalar projection.
+    const { frontmatter, body } = splitFrontmatter(input.source)
+    meta = parseFrontmatter(frontmatter, input.file)
+
+    // 2. Schema validation (schema is validator-only — doesn't change meta).
+    validateSchema(meta, input.schema, input.file)
+
+    // 3. Markdown → DocumentAST (neutral).
+    //    Note: the GENERIC layer does NOT throw for md h5/h6 — it produces a
+    //    structural HeadingBlock with synthetic level=4 and attaches the
+    //    original md depth to the explicit HeadingBlock.mdDepth optional
+    //    field. Fail-fast for out-of-range heading depths is the ACTIVE
+    //    DOMAIN PARSER's responsibility.
+    const docAST = parseDocument(body, input.file)
+
+    // 4. Domain parser dispatch.
+    const parser = registry.get(input.domain)
+    if (!parser) {
+      throw mkError(
+        `COMPILER_NO_DOMAIN_PARSER`,
+        `[compiler] No domain parser registered for '${input.domain}' — file: ${input.file}`,
+        input.file
+      )
+    }
+    const ctx: ParseContext = {
+      file: input.file,
+      schema: input.schema,
+      meta
+    }
+    const content = parser.parse(docAST, ctx)
+
+    return {
+      kind: 'positive',
+      meta,
+      docAST,
+      artifact: {
+        id: input.id,
+        domain: input.domain,
+        identity: input.identity,
+        meta,
+        content: { version: 1, blocks: content.blocks }
+      }
+    }
+  } catch (e) {
+    const err = normaliseError(e, input.file)
+    return {
+      kind: 'error',
+      meta,
+      error: err,
+      artifact: {
+        id: input.id,
+        domain: input.domain,
+        identity: input.identity,
+        meta,
+        kind: 'COMPILE_ERROR',
+        error: { code: err.code, message: err.message }
+      }
+    }
+  }
+}
+
+// ── Error helpers ────────────────────────────────────────
+
+function mkError (code: string, message: string, file: string): ParseError & Error {
+  const e = new Error(message) as ParseError & Error
+  e.code = code
+  e.file = file
+  return e
+}
+
+function normaliseError (e: unknown, file: string): ParseError & Error {
+  if (e && typeof e === 'object') {
+    const obj = e as Record<string, unknown>
+    if ('code' in obj && 'message' in obj) {
+      // already a ParseError — ensure .file set
+      const err = e as ParseError & Error
+      if (!err.file) err.file = file
+      return err
+    }
+    if (e instanceof Error) {
+      const err = e as ParseError & Error
+      const obj = e as unknown as Record<string, unknown>
+      err.code = (obj.code as string) ?? 'UNKNOWN_ERROR'
+      err.file = file
+      return err
+    }
+  }
+  return mkError('UNKNOWN_ERROR', String(e), file)
+}