dexin-content 0.1.0

package/core/types.ts

@@ -0,0 +1,162 @@
+// ─────────────────────────────────────────────────────────────
+// dexin-content/core/types.ts — Generic shared types.
+// ─────────────────────────────────────────────────────────────
+// This file defines the shared type system used across domains.
+// It MUST remain business-agnostic. All names, literals, and
+// comment text are written in generic, domain-neutral vocabulary.
+
+// ── Inline content (shared across the neutral AST and all domains) ──
+
+export type Inline =
+  | TextInline
+  | BoldInline
+  | ItalicInline
+  | CodeInline
+  | LinkInline
+  | FormulaInline
+
+export interface TextInline { type: 'text'; value: string }
+export interface BoldInline { type: 'bold'; children: Inline[] }
+export interface ItalicInline { type: 'italic'; children: Inline[] }
+export interface CodeInline { type: 'code'; value: string }
+export interface LinkInline { type: 'link'; url: string; children: Inline[] }
+/** Inline LaTeX-format formula span (single-paragraph). */
+export interface FormulaInline { type: 'math'; latex: string }
+
+// ── Document-level blocks (neutral AST — no domain mapping) ──
+
+/**
+ * HeadingBlock in the neutral AST: level is clamped to 1–4
+ * (h1→1, h2→2, h3→3, h4→4). `mdDepth` is the ORIGINAL markdown heading
+ * depth (1..6 inclusive) captured at parse time. It is OPTIONAL and exists
+ * ONLY so active domain parsers can run their own fail-fast checks for
+ * out-of-range depths. `mdDepth` MUST NOT leak into the serialised
+ * canonical Artifact (the `content` output): domain parsers' clone/render
+ * paths must drop it.
+ */
+export interface HeadingBlock {
+  type: 'heading'
+  level: 1 | 2 | 3 | 4
+  children: Inline[]
+  /** Original markdown heading depth (1..6). Transient, non-canonical. */
+  mdDepth?: number
+}
+export interface ParagraphBlock { type: 'paragraph'; children: Inline[] }
+export interface QuoteBlock { type: 'quote'; children: DocumentBlock[] }
+export interface DividerBlock { type: 'divider' }
+export interface ListBlock { type: 'list'; ordered: boolean; items: Inline[][] }
+export type TableCell = Inline[]
+export interface TableBlock { type: 'table'; headers: TableCell[]; rows: TableCell[][] }
+export interface ImageBlock { type: 'image'; src: string; alt: string; caption?: string }
+export interface CodeBlock { type: 'code'; language: string; code: string }
+/** Block-level LaTeX-format formula node; display=true for centered fences. */
+export interface FormulaBlock { type: 'formula'; latex: string; display: boolean }
+
+/**
+ * Neutral ContainerBlock:
+ * Produced for every directive container encountered in the markdown body.
+ * The active domain decides whether an unknown name is acceptable;
+ * the generic layer always preserves the structure as-is.
+ */
+export interface ContainerBlock {
+  type: 'container'
+  name: string
+  attrs: Record<string, string>
+  children: DocumentBlock[]
+}
+
+export type DocumentBlock =
+  | ParagraphBlock
+  | HeadingBlock
+  | QuoteBlock
+  | DividerBlock
+  | ListBlock
+  | TableBlock
+  | ImageBlock
+  | CodeBlock
+  | FormulaBlock
+  | ContainerBlock
+
+export interface DocumentContent {
+  version: 1
+  blocks: DocumentBlock[]
+}
+
+// ── Artifact shapes (shared five-field header) ──
+
+/**
+ * Generic domain tag. Literal string values are chosen by the host runner
+ * and flow through to the Artifact output untouched. Core itself does not
+ * interpret them beyond registry lookup keys.
+ */
+export type DomainName = string & { readonly __brand?: unique symbol }
+
+/**
+ * Plain-document identity:
+ *   id    — relative to content root; strip `.md` and trailing `/index`
+ *   path  — URL semantics: `'/' + id`
+ *   file  — physical path relative to content root, INCLUDING `.md` extension
+ *
+ * Domain-specific routing fields may be attached by hosts via the index
+ * signature; core never interprets them.
+ */
+export interface DocumentIdentity {
+  id: string
+  path: string        // URL semantic: '/' + id
+  file: string        // physical path relative to content root WITH ext
+  collection: string
+  [k: string]: unknown   // auxiliary fields (e.g. index_file:true)
+}
+
+/** Frontmatter scalar-only projection (SCHEMA-FREE RULE) */
+export type Meta = Record<string, string | number | boolean>
+
+export interface ArtifactHeader {
+  id: string               // Stable document id (unique within a store)
+  domain: DomainName | string
+  identity: DocumentIdentity
+  meta: Meta
+}
+
+export interface PositiveArtifact extends ArtifactHeader {
+  content: { version: 1; blocks: unknown[] }
+}
+
+/** Compile-failure output — captures first fail-fast error. */
+export interface ErrorArtifact extends ArtifactHeader {
+  kind: 'COMPILE_ERROR'
+  error: { code: string; message: string }
+}
+
+export type Artifact = PositiveArtifact | ErrorArtifact
+
+// ── Unified error object ──
+
+export interface ParseError extends Error {
+  code: string
+  message: string
+  file: string
+  loc?: { container?: string; headingDepth?: number; invalidEnumValue?: string; allowed?: string }
+}
+
+// ── Parser contract — domains implement; core dispatches ──
+
+export interface ParseContext {
+  /** Relative path to the source file, used in error messages. */
+  file: string
+  /** Collection-level schema (required fields + scalar type checks). */
+  schema?: Schema
+  /** Frontmatter already projected as scalar-only meta. */
+  meta: Meta
+}
+
+export interface Schema {
+  required?: string[]
+  types?: Record<string, 'string' | 'number' | 'boolean'>
+}
+
+export interface DomainParser {
+  domain: DomainName | string
+  /** Convert neutral AST → domain-specific Artifact content.blocks shape. May throw ParseError. */
+  parse(content: DocumentContent, ctx: ParseContext): { version: 1; blocks: unknown[] }
+}

package/diff.ts

@@ -0,0 +1,124 @@
+// ─────────────────────────────────────────────────────────────
+// dexin-content/diff.ts — 共享 Canonical/Diff 原语
+//
+// 统一 CANONICALIZE 语义：
+//   * Golden 侧递归剥离 `_` 前缀键（CANONICALIZE Rule 1）
+//   * Candidate 侧 `_` 前缀键路径扫描（CANONICALIZE Rule 2）
+//   * 对象级 firstDiff（键序不敏感，首差异 JSON-path）
+//   * Canonical JSON 序列化（2 空格、键字典序、LF、末换行、
+//     Unicode 不转义、U+2028/U+2029 转义）
+// ─────────────────────────────────────────────────────────────
+
+export interface DiffResult {
+  path: string
+  expected: unknown
+  actual: unknown
+}
+
+// ── 对象与键工具 ──────────────────────────────────────────
+
+export function isPlainObject (v: unknown): v is Record<string, unknown> {
+  return !!v && typeof v === 'object' && !Array.isArray(v)
+}
+
+// ── 1) Canonical JSON 序列化 ─────────────────────────────
+//    规则：2 空格缩进 + 对象每层级键字典序 + LF 换行 + 尾部换行 +
+//          Unicode 原文保留（不 \u 转义，除 JSON 强制字符）+
+//          行分隔符 U+2028 / 段分隔符 U+2029 转义（老 JS 引擎兼容）。
+
+export function toCanonicalJSON (value: unknown): string {
+  return JSON.stringify(sortKeysDeep(value), null, 2)
+    .split('\r\n').join('\n')
+    .split('\u2028').join('\\u2028')
+    .split('\u2029').join('\\u2029') + '\n'
+}
+
+export function sortKeysDeep (v: unknown): unknown {
+  if (Array.isArray(v)) return v.map(sortKeysDeep)
+  if (isPlainObject(v)) {
+    const keys = Object.keys(v).sort()
+    const out: Record<string, unknown> = {}
+    for (const k of keys) out[k] = sortKeysDeep((v as Record<string, unknown>)[k])
+    return out
+  }
+  return v
+}
+
+// ── 2) Golden 侧 _ 键递归剥离（CANONICALIZE §3 Rule 1 v1.1） ─
+//    只剥离 Golden。Candidate 若含 _ 前缀键，属于 Rule 2 违规，
+//    应走 underscorePrefixedPaths 判 FAIL，绝不在 diff 中掩盖。
+
+export function stripUnderscoreKeysGolden (obj: unknown): unknown {
+  if (Array.isArray(obj)) return obj.map(stripUnderscoreKeysGolden)
+  if (isPlainObject(obj)) {
+    const out: Record<string, unknown> = {}
+    for (const k of Object.keys(obj)) {
+      if (k.startsWith('_')) continue
+      out[k] = stripUnderscoreKeysGolden((obj as Record<string, unknown>)[k])
+    }
+    return out
+  }
+  return obj
+}
+
+// ── 3) Candidate/B3 侧全层级 _ 前缀键扫描 ─────────────────
+//    返回 JSON-path 字符串数组（空 = 干净）。
+
+export function underscorePrefixedPaths (obj: unknown): string[] {
+  const out: string[] = []
+  walk(obj, '$')
+  return out
+  function walk (v: unknown, at: string): void {
+    if (Array.isArray(v)) {
+      for (let i = 0; i < v.length; i++) walk(v[i], `${at}[${i}]`)
+      return
+    }
+    if (isPlainObject(v)) {
+      for (const k of Object.keys(v)) {
+        if (k.startsWith('_')) out.push(`${at}.${k}`)
+        walk((v as Record<string, unknown>)[k], `${at}.${k}`)
+      }
+    }
+  }
+}
+
+// ── 4) 对象级 firstDiff（键序不敏感） ─────────────────────
+//    返回 null（相等）或首个差异（JSON-path + 两边值）。
+//    相等判定使用 Object.is —— NaN 对 NaN 视为相等。
+
+export function firstDiff (actual: unknown, expected: unknown, root: string): DiffResult | null {
+  if (isPlainObject(expected) && isPlainObject(actual)) {
+    const keys = Array.from(
+      new Set([...Object.keys(expected), ...Object.keys(actual)])
+    ).sort()
+    for (const k of keys) {
+      const e = (expected as Record<string, unknown>)[k]
+      const a = (actual as Record<string, unknown>)[k]
+      if (!(k in (expected as object))) {
+        return { path: `${root}.${k}`, expected: '<absent>', actual: shortStr(a) }
+      }
+      if (!(k in (actual as object))) {
+        return { path: `${root}.${k}`, expected: shortStr(e), actual: '<absent>' }
+      }
+      const sub = firstDiff(a, e, `${root}.${k}`)
+      if (sub) return sub
+    }
+    return null
+  }
+  if (Array.isArray(expected) && Array.isArray(actual)) {
+    if (expected.length !== actual.length) {
+      return { path: root + '.length', expected: expected.length, actual: actual.length }
+    }
+    for (let i = 0; i < expected.length; i++) {
+      const sub = firstDiff(actual[i], expected[i], `${root}[${i}]`)
+      if (sub) return sub
+    }
+    return null
+  }
+  if (Object.is(actual, expected)) return null
+  return { path: root, expected, actual }
+}
+
+export function shortStr (v: unknown): string {
+  try { return JSON.stringify(v) } catch { return String(v) }
+}

package/index.ts

@@ -0,0 +1,12 @@
+// dexin-content — Universal content compilation layer
+// Re-exports core types and interfaces for host consumption.
+// Domain parsers are injected by the host via DomainParserRegistry.
+
+export type { DomainName, DocumentIdentity, Schema, Artifact, ParseError, DomainParser, ParseContext, DocumentContent, DocumentBlock } from './core/types'
+export { DomainParserRegistry, compile } from './core/compiler'
+export type { ArtifactStore } from './store'
+export { createFsArtifactStore, createMemoryArtifactStore } from './store'
+export type { CollectionDefinition, SourceAdapter, ResolvedCollection } from './collection'
+export { defineCollection, resolveCollections, discover, compileCollections } from './collection'
+export type { ContentQuery as ContentQueryType } from './query'
+export { ContentQuery } from './query'

package/package.json

@@ -0,0 +1,68 @@
+{
+  "name": "dexin-content",
+  "version": "0.1.0",
+  "description": "Structured content compilation & runtime layer — neutral Markdown → Content AST → PositiveArtifact + store/query/collection pipeline.",
+  "keywords": [
+    "markdown",
+    "content",
+    "compiler",
+    "ast",
+    "cms",
+    "remark",
+    "dexinlabs"
+  ],
+  "type": "module",
+  "license": "MIT",
+  "author": "得心实验室 <https://gitee.com/cuizhn>",
+  "repository": {
+    "type": "git",
+    "url": "git+https://gitee.com/cuizhn/dexin-content.git"
+  },
+  "homepage": "https://gitee.com/cuizhn/dexin-content#readme",
+  "bugs": {
+    "url": "https://gitee.com/cuizhn/dexin-content/issues"
+  },
+  "engines": {
+    "node": ">=20.0.0"
+  },
+  "files": [
+    "index.ts",
+    "collection.ts",
+    "store.ts",
+    "query.ts",
+    "diff.ts",
+    "core/**/*.ts",
+    "README.md",
+    "LICENSE"
+  ],
+  "exports": {
+    ".": "./index.ts",
+    "./core/types": "./core/types.ts",
+    "./core/compiler": "./core/compiler.ts",
+    "./core/frontmatter": "./core/frontmatter.ts",
+    "./core/discovery": "./core/discovery.ts",
+    "./store": "./store.ts",
+    "./collection": "./collection.ts",
+    "./query": "./query.ts",
+    "./diff": "./diff.ts"
+  },
+  "scripts": {
+    "typecheck": "tsc --noEmit",
+    "runtime:check": "tsx scripts/runtime-check.ts",
+    "pack": "npm pack --dry-run"
+  },
+  "dependencies": {
+    "yaml": "^2.5.1",
+    "remark-parse": "^11.0.0",
+    "remark-gfm": "^4.0.0",
+    "remark-math": "^6.0.0",
+    "remark-directive": "^3.0.0",
+    "unified": "^11.0.0"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "@types/mdast": "^4.0.4",
+    "typescript": "^5.5.0",
+    "tsx": "^4.7.0"
+  }
+}

package/query.ts

@@ -0,0 +1,56 @@
+// ─────────────────────────────────────────────────────────────
+// dexin-content/query.ts
+//
+// Runtime query — read-only over Artifact store.
+//
+// Query NEVER parses Markdown. No index → throw. byPath/byId
+// return PositiveArtifact (with content.blocks); collection/list
+// return IndexEntry[] (metadata + routing only).
+//
+// grep-zero: no business vocabulary appears anywhere in this
+// module — including error messages.
+// ─────────────────────────────────────────────────────────────
+
+import type { PositiveArtifact } from './core/types'
+import type { ArtifactStore, IndexEntry } from './store'
+
+export interface QueryOptions {
+  where?: (entry: IndexEntry) => boolean
+  sort?: (a: IndexEntry, b: IndexEntry) => number
+  limit?: number
+}
+
+export class ContentQuery {
+  constructor (private store: ArtifactStore) {}
+
+  /** List all entries (optionally filtered to one collection). */
+  async list (collection?: string, opts: QueryOptions = {}): Promise<IndexEntry[]> {
+    const index = await this.store.readIndex()
+    if (!index) throw new Error('[Query] No index — run compile first (runtime never reparses source).')
+    let rows = collection ? index.entries.filter(e => e.collection === collection) : [...index.entries]
+    if (opts.where) rows = rows.filter(opts.where)
+    if (opts.sort) rows.sort(opts.sort)
+    if (opts.limit != null) rows = rows.slice(0, opts.limit)
+    return rows
+  }
+
+  /** Fetch single doc by URL path. Returns null if not found. */
+  async byPath (path: string): Promise<PositiveArtifact | null> {
+    const norm = path.replace(/\/+$/, '') || '/'
+    const index = await this.store.readIndex()
+    if (!index) throw new Error('[Query] No index — run compile first.')
+    const entry = index.entries.find(e => e.path === norm)
+    if (!entry) return null
+    return this.store.readDoc(entry.id)
+  }
+
+  /** Fetch single doc by stable id (= PositiveArtifact.id). */
+  async byId (id: string): Promise<PositiveArtifact | null> {
+    return this.store.readDoc(id)
+  }
+
+  /** List metadata entries for a collection (host runtime usage). */
+  async collection (name: string, opts: QueryOptions = {}): Promise<IndexEntry[]> {
+    return this.list(name, opts)
+  }
+}

package/store.ts

@@ -0,0 +1,128 @@
+// ─────────────────────────────────────────────────────────────
+// dexin-content/store.ts
+//
+// Artifact storage abstraction.
+//
+// The stored Doc shape IS the formal-layer PositiveArtifact
+// (core/types.ts contract). No second Doc alias is introduced;
+// the Store reads/writes the exact shape the compiler emits.
+//
+// The Index (ContentIndex / IndexEntry) is the routing overlay layer:
+//   IndexEntry.id maps to PositiveArtifact.id.
+//   IndexEntry.path/file/collection are path-derived routing info
+//   independent of the domain-specific identity inside the Artifact.
+//
+// grep-zero: no business vocabulary appears anywhere in this
+// module — including error messages.
+// ─────────────────────────────────────────────────────────────
+
+import { mkdir, writeFile, readFile, access } from 'node:fs/promises'
+import { dirname } from 'node:path'
+import type { PositiveArtifact, Meta } from './core/types'
+import { toCanonicalJSON } from './diff'
+
+// ── Index types ──
+
+export interface IndexEntry {
+  /** Stable doc id; equals PositiveArtifact.id. */
+  id: string
+  /** URL semantic path: '/' + id. */
+  path: string
+  /** Owning collection name. */
+  collection: string
+  /** Domain tag (consistent with Artifact.domain). */
+  domain: string
+  /** Physical path relative to content root, INCLUDING .md. */
+  file: string
+  /** Scalar-only frontmatter projection. */
+  meta: Meta
+  /** First 12 hex chars of body md5 (change detection). */
+  checksum: string
+}
+
+export interface ContentIndex {
+  generator: 'dexin-content'
+  builtAt: string
+  entries: IndexEntry[]
+}
+
+// ── Store interface ──
+
+export interface ArtifactStore {
+  writeIndex(index: ContentIndex): Promise<void>
+  writeDoc(doc: PositiveArtifact): Promise<void>
+  readIndex(): Promise<ContentIndex | null>
+  readDoc(id: string): Promise<PositiveArtifact | null>
+}
+
+// ── File-system store ──
+
+export interface FsOps {
+  mkdir(p: string): Promise<void>
+  writeFile(p: string, data: string): Promise<void>
+  readFile(p: string): Promise<string>
+  exists(p: string): Promise<boolean>
+}
+
+const defaultFsOps: FsOps = {
+  async mkdir (p) { await mkdir(p, { recursive: true }) },
+  async writeFile (p, data) { await writeFile(p, data, 'utf8') },
+  async readFile (p) { return readFile(p, 'utf8') },
+  async exists (p) { try { await access(p); return true } catch { return false } }
+}
+
+/**
+ * File-system Artifact Store.
+ * Layout: <baseDir>/index.json + <baseDir>/docs/<id>.json
+ * Docs are serialised via toCanonicalJSON (2-space, key-lexicographic,
+ * LF, trailing newline) to match the canonical Artifact form.
+ */
+export function createFsArtifactStore (
+  baseDir: string,
+  fsOps: FsOps = defaultFsOps
+): ArtifactStore {
+  const idxPath = () => `${baseDir}/index.json`
+  const docPath = (id: string) => `${baseDir}/docs/${id}.json`
+
+  return {
+    async writeIndex (index) {
+      await fsOps.mkdir(baseDir)
+      await fsOps.writeFile(idxPath(), toCanonicalJSON(index))
+    },
+
+    async writeDoc (doc) {
+      const p = docPath(doc.id)
+      await fsOps.mkdir(dirname(p))
+      await fsOps.writeFile(p, toCanonicalJSON(doc))
+    },
+
+    async readIndex () {
+      if (!(await fsOps.exists(idxPath()))) return null
+      return JSON.parse(await fsOps.readFile(idxPath())) as ContentIndex
+    },
+
+    async readDoc (id) {
+      if (!(await fsOps.exists(docPath(id)))) return null
+      return JSON.parse(await fsOps.readFile(docPath(id))) as PositiveArtifact
+    }
+  }
+}
+
+// ── Memory store (Dev / test) ──
+
+export interface MemoryStore extends ArtifactStore {
+  snapshot(): { index: ContentIndex | null; docs: Map<string, PositiveArtifact> }
+}
+
+export function createMemoryArtifactStore (): MemoryStore {
+  let index: ContentIndex | null = null
+  const docs = new Map<string, PositiveArtifact>()
+
+  return {
+    async writeIndex (i) { index = i },
+    async writeDoc (d) { docs.set(d.id, d) },
+    async readIndex () { return index },
+    async readDoc (id) { return docs.get(id) ?? null },
+    snapshot () { return { index, docs } }
+  }
+}