@nebutra/document-pipeline 0.1.0

package/README.md

@@ -0,0 +1,29 @@
+# @nebutra/document-pipeline
+
+Status: WIP — Not yet integrated into any production app.
+
+`@nebutra/document-pipeline` owns document parser routing, metadata-preserving
+chunks, content-store ingestion, parser health checks, doctor output, and debug
+inspection. File truth remains in content-store; this package translates
+documents into indexed content units.
+
+It does not own Thread/Turn/Item state, prompt generation, model calls,
+sub-agent scheduling, or approval lifecycle. Complex parsing and OCR are
+sidecar-backed capability ports, not runtime logic.
+
+## Commands
+
+```bash
+pnpm docs:doctor
+pnpm docs:debug <job_id>
+pnpm docs:ingest <path>
+pnpm docs:inspect <chunk_id>
+```
+
+## Examples
+
+Executable examples live under `examples/`:
+
+- `ingest-markdown.ts`
+- `parse-html.ts`
+- `sidecar-gate.ts`

package/dist/chunk-HZF62UJB.js

@@ -0,0 +1,227 @@
+// src/index.ts
+import { mkdir, readFile, writeFile } from "fs/promises";
+import { dirname, extname, join } from "path";
+import { ContentStore } from "@nebutra/content-store";
+import { CapabilityError } from "@nebutra/errors";
+function debugPath(root = process.cwd()) {
+  return join(root, ".nebutra", "debug", "document-pipeline.jsonl");
+}
+async function appendDebug(root, entry) {
+  const path = debugPath(root);
+  await mkdir(dirname(path), { recursive: true });
+  await writeFile(path, `${JSON.stringify({ at: (/* @__PURE__ */ new Date()).toISOString(), ...entry })}
+`, {
+    flag: "a"
+  });
+}
+async function readDocumentPipelineDebug(root = process.cwd(), limit = 20) {
+  try {
+    const raw = await readFile(debugPath(root), "utf8");
+    return raw.trim().split("\n").filter(Boolean).slice(-limit).map((line) => JSON.parse(line));
+  } catch {
+    return [];
+  }
+}
+function requireTenant(explicit, fallback) {
+  const tenantId = explicit ?? fallback;
+  if (!tenantId) {
+    throw new CapabilityError("document-pipeline", "Document ingestion requires tenant context", {
+      suggestion: "Pass tenantId on the request or construct DocumentPipeline with a tenantId default.",
+      statusCode: 400
+    });
+  }
+  return tenantId;
+}
+var DocumentPipeline = class _DocumentPipeline {
+  #tenantId;
+  #root;
+  #debugRoot;
+  #contentStore;
+  #sidecarParser;
+  constructor(options = {}) {
+    this.#tenantId = options.tenantId;
+    this.#root = options.root ?? join(process.cwd(), ".nebutra", "document-pipeline");
+    this.#debugRoot = options.debugRoot ?? process.cwd();
+    this.#contentStore = options.contentStore;
+    this.#sidecarParser = options.sidecarParser;
+  }
+  static async open(root = ".nebutra/document-pipeline", options = {}) {
+    const tenantId = options.tenantId ?? "local";
+    const contentStore = await ContentStore.open(join(root, "content"), { tenantId });
+    return new _DocumentPipeline({ ...options, tenantId, root, contentStore });
+  }
+  async parse(request) {
+    const tenantId = requireTenant(request.tenantId, this.#tenantId);
+    const required = { ...request, tenantId };
+    const parser = this.#chooseParser(required);
+    if (parser === "sidecar") {
+      if (!this.#sidecarParser) {
+        throw new CapabilityError("document-pipeline", "Parser sidecar is not configured", {
+          suggestion: "Configure the parser sidecar for PDFs, office files, OCR, and layout-aware extraction.",
+          statusCode: 503,
+          metadata: { path: request.source.path }
+        });
+      }
+      const parsed2 = await this.#sidecarParser.parse(required);
+      await appendDebug(this.#debugRoot, {
+        type: "parse",
+        tenantId,
+        parser: "sidecar",
+        path: parsed2.path
+      });
+      return parsed2;
+    }
+    const raw = await readSource(request.source);
+    const parsed = parseNativeDocument(required, raw, parser);
+    await appendDebug(this.#debugRoot, {
+      type: "parse",
+      tenantId,
+      parser,
+      path: parsed.path,
+      chunks: parsed.chunks.length
+    });
+    return parsed;
+  }
+  async ingest(request) {
+    const tenantId = requireTenant(request.tenantId, this.#tenantId);
+    const contentStore = await this.#resolveContentStore(tenantId);
+    const parsed = await this.parse({ ...request, tenantId });
+    const targetPath = request.targetPath ?? parsed.path;
+    const content = serializeForContentStore(parsed, request.metadata);
+    await contentStore.write(targetPath, content);
+    await appendDebug(this.#debugRoot, {
+      type: "ingest",
+      tenantId,
+      path: targetPath,
+      parser: parsed.parser,
+      chunks: parsed.chunks.length
+    });
+    return {
+      tenantId,
+      path: targetPath,
+      parser: parsed.parser,
+      chunkCount: parsed.chunks.length,
+      contentIndexed: true
+    };
+  }
+  async ingestFile(path, options = {}) {
+    return this.ingest({ ...options, source: { type: "file", path } });
+  }
+  async doctor() {
+    const contentStore = await this.#resolveContentStore(this.#tenantId ?? "local");
+    const sidecar = this.#sidecarParser ? await this.#sidecarParser.doctor() : { ok: false };
+    return {
+      capability: "document-pipeline",
+      nativeParsers: ["markdown", "html", "text"],
+      sidecarConfigured: sidecar.ok,
+      contentStore: await contentStore.doctor(),
+      ...sidecar.ok ? {} : {
+        suggestion: "Native parsers are active; configure the parser sidecar before ingesting PDFs, office files, OCR, or complex layout documents."
+      }
+    };
+  }
+  #chooseParser(request) {
+    if (request.parser && request.parser !== "auto") return request.parser;
+    const mime = request.source.mimeType ?? mimeFromPath(request.source.path);
+    if (mime === "text/markdown") return "markdown";
+    if (mime === "text/html") return "html";
+    if (mime.startsWith("text/")) return "text";
+    return "sidecar";
+  }
+  async #resolveContentStore(tenantId) {
+    if (this.#contentStore) return this.#contentStore;
+    return ContentStore.open(join(this.#root, "content"), { tenantId });
+  }
+};
+async function readSource(source) {
+  return source.type === "inline" ? source.content : readFile(source.path, "utf8");
+}
+function mimeFromPath(path) {
+  const extension = extname(path).toLowerCase();
+  if (extension === ".md" || extension === ".mdx") return "text/markdown";
+  if (extension === ".html" || extension === ".htm") return "text/html";
+  if (extension === ".txt" || extension === ".csv" || extension === ".json") return "text/plain";
+  if (extension === ".pdf") return "application/pdf";
+  if (extension === ".docx")
+    return "application/vnd.openxmlformats-officedocument.wordprocessingml.document";
+  if (extension === ".pptx")
+    return "application/vnd.openxmlformats-officedocument.presentationml.presentation";
+  if (extension === ".png" || extension === ".jpg" || extension === ".jpeg") return "image";
+  return "application/octet-stream";
+}
+function parseNativeDocument(request, raw, parser) {
+  const mimeType = request.source.mimeType ?? mimeFromPath(request.source.path);
+  const { frontmatter, body } = parser === "markdown" ? parseFrontmatter(raw) : { frontmatter: {}, body: raw };
+  const text = parser === "html" ? htmlToText(body) : body;
+  const chunks = chunkParagraphs(text).map((chunk, index) => ({
+    id: `${request.tenantId}:${request.source.path}:${index}`,
+    text: chunk,
+    metadata: {
+      tenantId: request.tenantId,
+      sourcePath: request.source.path,
+      chunkIndex: index,
+      mimeType,
+      parser,
+      ...request.metadata ?? {}
+    }
+  }));
+  return {
+    tenantId: request.tenantId,
+    path: request.source.path,
+    mimeType,
+    parser,
+    chunks: chunks.length > 0 ? chunks : [
+      {
+        id: `${request.tenantId}:${request.source.path}:0`,
+        text,
+        metadata: {
+          tenantId: request.tenantId,
+          sourcePath: request.source.path,
+          chunkIndex: 0,
+          mimeType,
+          parser,
+          ...request.metadata ?? {}
+        }
+      }
+    ],
+    frontmatter
+  };
+}
+function serializeForContentStore(parsed, metadata) {
+  const frontmatter = {
+    ...parsed.frontmatter,
+    ...metadata ?? {},
+    source_mime: parsed.mimeType,
+    parser: parsed.parser
+  };
+  const frontmatterLines = Object.entries(frontmatter).sort(([left], [right]) => left.localeCompare(right)).map(([key, value]) => `${key}: ${value}`);
+  const body = parsed.chunks.map((chunk) => chunk.text).join("\n\n");
+  return frontmatterLines.length > 0 ? `---
+${frontmatterLines.join("\n")}
+---
+${body}` : body;
+}
+function parseFrontmatter(content) {
+  if (!content.startsWith("---\n")) return { frontmatter: {}, body: content };
+  const end = content.indexOf("\n---", 4);
+  if (end < 0) return { frontmatter: {}, body: content };
+  const raw = content.slice(4, end).trim();
+  const frontmatter = {};
+  for (const line of raw.split("\n")) {
+    const idx = line.indexOf(":");
+    if (idx > 0) frontmatter[line.slice(0, idx).trim()] = line.slice(idx + 1).trim();
+  }
+  return { frontmatter, body: content.slice(end + 4).trimStart() };
+}
+function chunkParagraphs(content) {
+  return content.split(/\n\s*\n/).map((part) => part.trim()).filter(Boolean);
+}
+function htmlToText(html) {
+  return html.replace(/<script[\s\S]*?<\/script>/gi, " ").replace(/<style[\s\S]*?<\/style>/gi, " ").replace(/<\/(h[1-6]|p|li|tr|div)>/gi, "\n\n").replace(/<[^>]+>/g, " ").replace(/&nbsp;/g, " ").replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/[ \t]+/g, " ").replace(/\n\s+/g, "\n").trim();
+}
+
+export {
+  readDocumentPipelineDebug,
+  DocumentPipeline
+};
+//# sourceMappingURL=chunk-HZF62UJB.js.map

package/dist/chunk-HZF62UJB.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/index.ts"],"sourcesContent":["import { mkdir, readFile, writeFile } from \"node:fs/promises\";\nimport { dirname, extname, join } from \"node:path\";\nimport { ContentStore } from \"@nebutra/content-store\";\nimport { CapabilityError } from \"@nebutra/errors\";\n\nexport type ParserChoice = \"auto\" | \"markdown\" | \"html\" | \"text\" | \"sidecar\";\nexport type ChunkerChoice = \"paragraph\" | \"recursive\" | \"hierarchical\";\n\nexport interface InlineSource {\n  readonly type: \"inline\";\n  readonly path: string;\n  readonly content: string;\n  readonly mimeType?: string;\n}\n\nexport interface FileSource {\n  readonly type: \"file\";\n  readonly path: string;\n  readonly mimeType?: string;\n}\n\nexport type DocumentSource = InlineSource | FileSource;\n\nexport interface IngestRequest {\n  readonly tenantId?: string;\n  readonly source: DocumentSource;\n  readonly parser?: ParserChoice;\n  readonly chunker?: ChunkerChoice;\n  readonly targetPath?: string;\n  readonly metadata?: Record<string, string>;\n}\n\nexport interface ParseRequest {\n  readonly tenantId?: string;\n  readonly source: DocumentSource;\n  readonly parser?: ParserChoice;\n  readonly chunker?: ChunkerChoice;\n  readonly metadata?: Record<string, string>;\n}\n\nexport interface ParsedChunk {\n  readonly id: string;\n  readonly text: string;\n  readonly metadata: {\n    readonly tenantId: string;\n    readonly sourcePath: string;\n    readonly chunkIndex: number;\n    readonly mimeType: string;\n    readonly parser: ParserChoice;\n    readonly heading?: string;\n  } & Record<string, string | number>;\n}\n\nexport interface ParsedDocument {\n  readonly tenantId: string;\n  readonly path: string;\n  readonly mimeType: string;\n  readonly parser: ParserChoice;\n  readonly chunks: readonly ParsedChunk[];\n  readonly frontmatter: Record<string, string>;\n}\n\nexport interface IngestResult {\n  readonly tenantId: string;\n  readonly path: string;\n  readonly parser: ParserChoice;\n  readonly chunkCount: number;\n  readonly contentIndexed: true;\n}\n\nexport interface DocumentPipelineDoctorReport {\n  readonly capability: \"document-pipeline\";\n  readonly nativeParsers: readonly ParserChoice[];\n  readonly sidecarConfigured: boolean;\n  readonly contentStore: Awaited<ReturnType<ContentStore[\"doctor\"]>>;\n  readonly suggestion?: string;\n}\n\nexport interface DocumentParser {\n  parse(request: RequiredTenant<ParseRequest>): Promise<ParsedDocument>;\n  doctor(): Promise<{ readonly ok: boolean; readonly suggestion?: string }>;\n}\n\nexport interface DocumentPipelineOptions {\n  readonly tenantId?: string;\n  readonly root?: string;\n  readonly debugRoot?: string;\n  readonly contentStore?: ContentStore;\n  readonly sidecarParser?: DocumentParser;\n}\n\ntype RequiredTenant<T extends { readonly tenantId?: string }> = Omit<T, \"tenantId\"> & {\n  readonly tenantId: string;\n};\n\nfunction debugPath(root = process.cwd()): string {\n  return join(root, \".nebutra\", \"debug\", \"document-pipeline.jsonl\");\n}\n\nasync function appendDebug(root: string, entry: Record<string, unknown>): Promise<void> {\n  const path = debugPath(root);\n  await mkdir(dirname(path), { recursive: true });\n  await writeFile(path, `${JSON.stringify({ at: new Date().toISOString(), ...entry })}\\n`, {\n    flag: \"a\",\n  });\n}\n\nexport async function readDocumentPipelineDebug(\n  root = process.cwd(),\n  limit = 20,\n): Promise<unknown[]> {\n  try {\n    const raw = await readFile(debugPath(root), \"utf8\");\n    return raw\n      .trim()\n      .split(\"\\n\")\n      .filter(Boolean)\n      .slice(-limit)\n      .map((line) => JSON.parse(line) as unknown);\n  } catch {\n    return [];\n  }\n}\n\nfunction requireTenant(explicit: string | undefined, fallback: string | undefined): string {\n  const tenantId = explicit ?? fallback;\n  if (!tenantId) {\n    throw new CapabilityError(\"document-pipeline\", \"Document ingestion requires tenant context\", {\n      suggestion:\n        \"Pass tenantId on the request or construct DocumentPipeline with a tenantId default.\",\n      statusCode: 400,\n    });\n  }\n  return tenantId;\n}\n\nexport class DocumentPipeline {\n  readonly #tenantId: string | undefined;\n  readonly #root: string;\n  readonly #debugRoot: string;\n  readonly #contentStore: ContentStore | undefined;\n  readonly #sidecarParser: DocumentParser | undefined;\n\n  constructor(options: DocumentPipelineOptions = {}) {\n    this.#tenantId = options.tenantId;\n    this.#root = options.root ?? join(process.cwd(), \".nebutra\", \"document-pipeline\");\n    this.#debugRoot = options.debugRoot ?? process.cwd();\n    this.#contentStore = options.contentStore;\n    this.#sidecarParser = options.sidecarParser;\n  }\n\n  static async open(\n    root = \".nebutra/document-pipeline\",\n    options: Omit<DocumentPipelineOptions, \"root\" | \"contentStore\"> = {},\n  ): Promise<DocumentPipeline> {\n    const tenantId = options.tenantId ?? \"local\";\n    const contentStore = await ContentStore.open(join(root, \"content\"), { tenantId });\n    return new DocumentPipeline({ ...options, tenantId, root, contentStore });\n  }\n\n  async parse(request: ParseRequest): Promise<ParsedDocument> {\n    const tenantId = requireTenant(request.tenantId, this.#tenantId);\n    const required: RequiredTenant<ParseRequest> = { ...request, tenantId };\n    const parser = this.#chooseParser(required);\n    if (parser === \"sidecar\") {\n      if (!this.#sidecarParser) {\n        throw new CapabilityError(\"document-pipeline\", \"Parser sidecar is not configured\", {\n          suggestion:\n            \"Configure the parser sidecar for PDFs, office files, OCR, and layout-aware extraction.\",\n          statusCode: 503,\n          metadata: { path: request.source.path },\n        });\n      }\n      const parsed = await this.#sidecarParser.parse(required);\n      await appendDebug(this.#debugRoot, {\n        type: \"parse\",\n        tenantId,\n        parser: \"sidecar\",\n        path: parsed.path,\n      });\n      return parsed;\n    }\n\n    const raw = await readSource(request.source);\n    const parsed = parseNativeDocument(required, raw, parser);\n    await appendDebug(this.#debugRoot, {\n      type: \"parse\",\n      tenantId,\n      parser,\n      path: parsed.path,\n      chunks: parsed.chunks.length,\n    });\n    return parsed;\n  }\n\n  async ingest(request: IngestRequest): Promise<IngestResult> {\n    const tenantId = requireTenant(request.tenantId, this.#tenantId);\n    const contentStore = await this.#resolveContentStore(tenantId);\n    const parsed = await this.parse({ ...request, tenantId });\n    const targetPath = request.targetPath ?? parsed.path;\n    const content = serializeForContentStore(parsed, request.metadata);\n    await contentStore.write(targetPath, content);\n    await appendDebug(this.#debugRoot, {\n      type: \"ingest\",\n      tenantId,\n      path: targetPath,\n      parser: parsed.parser,\n      chunks: parsed.chunks.length,\n    });\n    return {\n      tenantId,\n      path: targetPath,\n      parser: parsed.parser,\n      chunkCount: parsed.chunks.length,\n      contentIndexed: true,\n    };\n  }\n\n  async ingestFile(\n    path: string,\n    options: Omit<IngestRequest, \"source\"> = {},\n  ): Promise<IngestResult> {\n    return this.ingest({ ...options, source: { type: \"file\", path } });\n  }\n\n  async doctor(): Promise<DocumentPipelineDoctorReport> {\n    const contentStore = await this.#resolveContentStore(this.#tenantId ?? \"local\");\n    const sidecar = this.#sidecarParser ? await this.#sidecarParser.doctor() : { ok: false };\n    return {\n      capability: \"document-pipeline\",\n      nativeParsers: [\"markdown\", \"html\", \"text\"],\n      sidecarConfigured: sidecar.ok,\n      contentStore: await contentStore.doctor(),\n      ...(sidecar.ok\n        ? {}\n        : {\n            suggestion:\n              \"Native parsers are active; configure the parser sidecar before ingesting PDFs, office files, OCR, or complex layout documents.\",\n          }),\n    };\n  }\n\n  #chooseParser(request: RequiredTenant<ParseRequest>): ParserChoice {\n    if (request.parser && request.parser !== \"auto\") return request.parser;\n    const mime = request.source.mimeType ?? mimeFromPath(request.source.path);\n    if (mime === \"text/markdown\") return \"markdown\";\n    if (mime === \"text/html\") return \"html\";\n    if (mime.startsWith(\"text/\")) return \"text\";\n    return \"sidecar\";\n  }\n\n  async #resolveContentStore(tenantId: string): Promise<ContentStore> {\n    if (this.#contentStore) return this.#contentStore;\n    return ContentStore.open(join(this.#root, \"content\"), { tenantId });\n  }\n}\n\nasync function readSource(source: DocumentSource): Promise<string> {\n  return source.type === \"inline\" ? source.content : readFile(source.path, \"utf8\");\n}\n\nfunction mimeFromPath(path: string): string {\n  const extension = extname(path).toLowerCase();\n  if (extension === \".md\" || extension === \".mdx\") return \"text/markdown\";\n  if (extension === \".html\" || extension === \".htm\") return \"text/html\";\n  if (extension === \".txt\" || extension === \".csv\" || extension === \".json\") return \"text/plain\";\n  if (extension === \".pdf\") return \"application/pdf\";\n  if (extension === \".docx\")\n    return \"application/vnd.openxmlformats-officedocument.wordprocessingml.document\";\n  if (extension === \".pptx\")\n    return \"application/vnd.openxmlformats-officedocument.presentationml.presentation\";\n  if (extension === \".png\" || extension === \".jpg\" || extension === \".jpeg\") return \"image\";\n  return \"application/octet-stream\";\n}\n\nfunction parseNativeDocument(\n  request: RequiredTenant<ParseRequest>,\n  raw: string,\n  parser: ParserChoice,\n): ParsedDocument {\n  const mimeType = request.source.mimeType ?? mimeFromPath(request.source.path);\n  const { frontmatter, body } =\n    parser === \"markdown\" ? parseFrontmatter(raw) : { frontmatter: {}, body: raw };\n  const text = parser === \"html\" ? htmlToText(body) : body;\n  const chunks = chunkParagraphs(text).map((chunk, index) => ({\n    id: `${request.tenantId}:${request.source.path}:${index}`,\n    text: chunk,\n    metadata: {\n      tenantId: request.tenantId,\n      sourcePath: request.source.path,\n      chunkIndex: index,\n      mimeType,\n      parser,\n      ...(request.metadata ?? {}),\n    },\n  }));\n  return {\n    tenantId: request.tenantId,\n    path: request.source.path,\n    mimeType,\n    parser,\n    chunks:\n      chunks.length > 0\n        ? chunks\n        : [\n            {\n              id: `${request.tenantId}:${request.source.path}:0`,\n              text,\n              metadata: {\n                tenantId: request.tenantId,\n                sourcePath: request.source.path,\n                chunkIndex: 0,\n                mimeType,\n                parser,\n                ...(request.metadata ?? {}),\n              },\n            },\n          ],\n    frontmatter,\n  };\n}\n\nfunction serializeForContentStore(\n  parsed: ParsedDocument,\n  metadata: Record<string, string> | undefined,\n): string {\n  const frontmatter = {\n    ...parsed.frontmatter,\n    ...(metadata ?? {}),\n    source_mime: parsed.mimeType,\n    parser: parsed.parser,\n  };\n  const frontmatterLines = Object.entries(frontmatter)\n    .sort(([left], [right]) => left.localeCompare(right))\n    .map(([key, value]) => `${key}: ${value}`);\n  const body = parsed.chunks.map((chunk) => chunk.text).join(\"\\n\\n\");\n  return frontmatterLines.length > 0 ? `---\\n${frontmatterLines.join(\"\\n\")}\\n---\\n${body}` : body;\n}\n\nfunction parseFrontmatter(content: string): {\n  readonly frontmatter: Record<string, string>;\n  readonly body: string;\n} {\n  if (!content.startsWith(\"---\\n\")) return { frontmatter: {}, body: content };\n  const end = content.indexOf(\"\\n---\", 4);\n  if (end < 0) return { frontmatter: {}, body: content };\n  const raw = content.slice(4, end).trim();\n  const frontmatter: Record<string, string> = {};\n  for (const line of raw.split(\"\\n\")) {\n    const idx = line.indexOf(\":\");\n    if (idx > 0) frontmatter[line.slice(0, idx).trim()] = line.slice(idx + 1).trim();\n  }\n  return { frontmatter, body: content.slice(end + 4).trimStart() };\n}\n\nfunction chunkParagraphs(content: string): string[] {\n  return content\n    .split(/\\n\\s*\\n/)\n    .map((part) => part.trim())\n    .filter(Boolean);\n}\n\nfunction htmlToText(html: string): string {\n  return html\n    .replace(/<script[\\s\\S]*?<\\/script>/gi, \" \")\n    .replace(/<style[\\s\\S]*?<\\/style>/gi, \" \")\n    .replace(/<\\/(h[1-6]|p|li|tr|div)>/gi, \"\\n\\n\")\n    .replace(/<[^>]+>/g, \" \")\n    .replace(/&nbsp;/g, \" \")\n    .replace(/&amp;/g, \"&\")\n    .replace(/&lt;/g, \"<\")\n    .replace(/&gt;/g, \">\")\n    .replace(/[ \\t]+/g, \" \")\n    .replace(/\\n\\s+/g, \"\\n\")\n    .trim();\n}\n"],"mappings":";AAAA,SAAS,OAAO,UAAU,iBAAiB;AAC3C,SAAS,SAAS,SAAS,YAAY;AACvC,SAAS,oBAAoB;AAC7B,SAAS,uBAAuB;AA4FhC,SAAS,UAAU,OAAO,QAAQ,IAAI,GAAW;AAC/C,SAAO,KAAK,MAAM,YAAY,SAAS,yBAAyB;AAClE;AAEA,eAAe,YAAY,MAAc,OAA+C;AACtF,QAAM,OAAO,UAAU,IAAI;AAC3B,QAAM,MAAM,QAAQ,IAAI,GAAG,EAAE,WAAW,KAAK,CAAC;AAC9C,QAAM,UAAU,MAAM,GAAG,KAAK,UAAU,EAAE,KAAI,oBAAI,KAAK,GAAE,YAAY,GAAG,GAAG,MAAM,CAAC,CAAC;AAAA,GAAM;AAAA,IACvF,MAAM;AAAA,EACR,CAAC;AACH;AAEA,eAAsB,0BACpB,OAAO,QAAQ,IAAI,GACnB,QAAQ,IACY;AACpB,MAAI;AACF,UAAM,MAAM,MAAM,SAAS,UAAU,IAAI,GAAG,MAAM;AAClD,WAAO,IACJ,KAAK,EACL,MAAM,IAAI,EACV,OAAO,OAAO,EACd,MAAM,CAAC,KAAK,EACZ,IAAI,CAAC,SAAS,KAAK,MAAM,IAAI,CAAY;AAAA,EAC9C,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AACF;AAEA,SAAS,cAAc,UAA8B,UAAsC;AACzF,QAAM,WAAW,YAAY;AAC7B,MAAI,CAAC,UAAU;AACb,UAAM,IAAI,gBAAgB,qBAAqB,8CAA8C;AAAA,MAC3F,YACE;AAAA,MACF,YAAY;AAAA,IACd,CAAC;AAAA,EACH;AACA,SAAO;AACT;AAEO,IAAM,mBAAN,MAAM,kBAAiB;AAAA,EACnB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAET,YAAY,UAAmC,CAAC,GAAG;AACjD,SAAK,YAAY,QAAQ;AACzB,SAAK,QAAQ,QAAQ,QAAQ,KAAK,QAAQ,IAAI,GAAG,YAAY,mBAAmB;AAChF,SAAK,aAAa,QAAQ,aAAa,QAAQ,IAAI;AACnD,SAAK,gBAAgB,QAAQ;AAC7B,SAAK,iBAAiB,QAAQ;AAAA,EAChC;AAAA,EAEA,aAAa,KACX,OAAO,8BACP,UAAkE,CAAC,GACxC;AAC3B,UAAM,WAAW,QAAQ,YAAY;AACrC,UAAM,eAAe,MAAM,aAAa,KAAK,KAAK,MAAM,SAAS,GAAG,EAAE,SAAS,CAAC;AAChF,WAAO,IAAI,kBAAiB,EAAE,GAAG,SAAS,UAAU,MAAM,aAAa,CAAC;AAAA,EAC1E;AAAA,EAEA,MAAM,MAAM,SAAgD;AAC1D,UAAM,WAAW,cAAc,QAAQ,UAAU,KAAK,SAAS;AAC/D,UAAM,WAAyC,EAAE,GAAG,SAAS,SAAS;AACtE,UAAM,SAAS,KAAK,cAAc,QAAQ;AAC1C,QAAI,WAAW,WAAW;AACxB,UAAI,CAAC,KAAK,gBAAgB;AACxB,cAAM,IAAI,gBAAgB,qBAAqB,oCAAoC;AAAA,UACjF,YACE;AAAA,UACF,YAAY;AAAA,UACZ,UAAU,EAAE,MAAM,QAAQ,OAAO,KAAK;AAAA,QACxC,CAAC;AAAA,MACH;AACA,YAAMA,UAAS,MAAM,KAAK,eAAe,MAAM,QAAQ;AACvD,YAAM,YAAY,KAAK,YAAY;AAAA,QACjC,MAAM;AAAA,QACN;AAAA,QACA,QAAQ;AAAA,QACR,MAAMA,QAAO;AAAA,MACf,CAAC;AACD,aAAOA;AAAA,IACT;AAEA,UAAM,MAAM,MAAM,WAAW,QAAQ,MAAM;AAC3C,UAAM,SAAS,oBAAoB,UAAU,KAAK,MAAM;AACxD,UAAM,YAAY,KAAK,YAAY;AAAA,MACjC,MAAM;AAAA,MACN;AAAA,MACA;AAAA,MACA,MAAM,OAAO;AAAA,MACb,QAAQ,OAAO,OAAO;AAAA,IACxB,CAAC;AACD,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,OAAO,SAA+C;AAC1D,UAAM,WAAW,cAAc,QAAQ,UAAU,KAAK,SAAS;AAC/D,UAAM,eAAe,MAAM,KAAK,qBAAqB,QAAQ;AAC7D,UAAM,SAAS,MAAM,KAAK,MAAM,EAAE,GAAG,SAAS,SAAS,CAAC;AACxD,UAAM,aAAa,QAAQ,cAAc,OAAO;AAChD,UAAM,UAAU,yBAAyB,QAAQ,QAAQ,QAAQ;AACjE,UAAM,aAAa,MAAM,YAAY,OAAO;AAC5C,UAAM,YAAY,KAAK,YAAY;AAAA,MACjC,MAAM;AAAA,MACN;AAAA,MACA,MAAM;AAAA,MACN,QAAQ,OAAO;AAAA,MACf,QAAQ,OAAO,OAAO;AAAA,IACxB,CAAC;AACD,WAAO;AAAA,MACL;AAAA,MACA,MAAM;AAAA,MACN,QAAQ,OAAO;AAAA,MACf,YAAY,OAAO,OAAO;AAAA,MAC1B,gBAAgB;AAAA,IAClB;AAAA,EACF;AAAA,EAEA,MAAM,WACJ,MACA,UAAyC,CAAC,GACnB;AACvB,WAAO,KAAK,OAAO,EAAE,GAAG,SAAS,QAAQ,EAAE,MAAM,QAAQ,KAAK,EAAE,CAAC;AAAA,EACnE;AAAA,EAEA,MAAM,SAAgD;AACpD,UAAM,eAAe,MAAM,KAAK,qBAAqB,KAAK,aAAa,OAAO;AAC9E,UAAM,UAAU,KAAK,iBAAiB,MAAM,KAAK,eAAe,OAAO,IAAI,EAAE,IAAI,MAAM;AACvF,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,eAAe,CAAC,YAAY,QAAQ,MAAM;AAAA,MAC1C,mBAAmB,QAAQ;AAAA,MAC3B,cAAc,MAAM,aAAa,OAAO;AAAA,MACxC,GAAI,QAAQ,KACR,CAAC,IACD;AAAA,QACE,YACE;AAAA,MACJ;AAAA,IACN;AAAA,EACF;AAAA,EAEA,cAAc,SAAqD;AACjE,QAAI,QAAQ,UAAU,QAAQ,WAAW,OAAQ,QAAO,QAAQ;AAChE,UAAM,OAAO,QAAQ,OAAO,YAAY,aAAa,QAAQ,OAAO,IAAI;AACxE,QAAI,SAAS,gBAAiB,QAAO;AACrC,QAAI,SAAS,YAAa,QAAO;AACjC,QAAI,KAAK,WAAW,OAAO,EAAG,QAAO;AACrC,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,qBAAqB,UAAyC;AAClE,QAAI,KAAK,cAAe,QAAO,KAAK;AACpC,WAAO,aAAa,KAAK,KAAK,KAAK,OAAO,SAAS,GAAG,EAAE,SAAS,CAAC;AAAA,EACpE;AACF;AAEA,eAAe,WAAW,QAAyC;AACjE,SAAO,OAAO,SAAS,WAAW,OAAO,UAAU,SAAS,OAAO,MAAM,MAAM;AACjF;AAEA,SAAS,aAAa,MAAsB;AAC1C,QAAM,YAAY,QAAQ,IAAI,EAAE,YAAY;AAC5C,MAAI,cAAc,SAAS,cAAc,OAAQ,QAAO;AACxD,MAAI,cAAc,WAAW,cAAc,OAAQ,QAAO;AAC1D,MAAI,cAAc,UAAU,cAAc,UAAU,cAAc,QAAS,QAAO;AAClF,MAAI,cAAc,OAAQ,QAAO;AACjC,MAAI,cAAc;AAChB,WAAO;AACT,MAAI,cAAc;AAChB,WAAO;AACT,MAAI,cAAc,UAAU,cAAc,UAAU,cAAc,QAAS,QAAO;AAClF,SAAO;AACT;AAEA,SAAS,oBACP,SACA,KACA,QACgB;AAChB,QAAM,WAAW,QAAQ,OAAO,YAAY,aAAa,QAAQ,OAAO,IAAI;AAC5E,QAAM,EAAE,aAAa,KAAK,IACxB,WAAW,aAAa,iBAAiB,GAAG,IAAI,EAAE,aAAa,CAAC,GAAG,MAAM,IAAI;AAC/E,QAAM,OAAO,WAAW,SAAS,WAAW,IAAI,IAAI;AACpD,QAAM,SAAS,gBAAgB,IAAI,EAAE,IAAI,CAAC,OAAO,WAAW;AAAA,IAC1D,IAAI,GAAG,QAAQ,QAAQ,IAAI,QAAQ,OAAO,IAAI,IAAI,KAAK;AAAA,IACvD,MAAM;AAAA,IACN,UAAU;AAAA,MACR,UAAU,QAAQ;AAAA,MAClB,YAAY,QAAQ,OAAO;AAAA,MAC3B,YAAY;AAAA,MACZ;AAAA,MACA;AAAA,MACA,GAAI,QAAQ,YAAY,CAAC;AAAA,IAC3B;AAAA,EACF,EAAE;AACF,SAAO;AAAA,IACL,UAAU,QAAQ;AAAA,IAClB,MAAM,QAAQ,OAAO;AAAA,IACrB;AAAA,IACA;AAAA,IACA,QACE,OAAO,SAAS,IACZ,SACA;AAAA,MACE;AAAA,QACE,IAAI,GAAG,QAAQ,QAAQ,IAAI,QAAQ,OAAO,IAAI;AAAA,QAC9C;AAAA,QACA,UAAU;AAAA,UACR,UAAU,QAAQ;AAAA,UAClB,YAAY,QAAQ,OAAO;AAAA,UAC3B,YAAY;AAAA,UACZ;AAAA,UACA;AAAA,UACA,GAAI,QAAQ,YAAY,CAAC;AAAA,QAC3B;AAAA,MACF;AAAA,IACF;AAAA,IACN;AAAA,EACF;AACF;AAEA,SAAS,yBACP,QACA,UACQ;AACR,QAAM,cAAc;AAAA,IAClB,GAAG,OAAO;AAAA,IACV,GAAI,YAAY,CAAC;AAAA,IACjB,aAAa,OAAO;AAAA,IACpB,QAAQ,OAAO;AAAA,EACjB;AACA,QAAM,mBAAmB,OAAO,QAAQ,WAAW,EAChD,KAAK,CAAC,CAAC,IAAI,GAAG,CAAC,KAAK,MAAM,KAAK,cAAc,KAAK,CAAC,EACnD,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM,GAAG,GAAG,KAAK,KAAK,EAAE;AAC3C,QAAM,OAAO,OAAO,OAAO,IAAI,CAAC,UAAU,MAAM,IAAI,EAAE,KAAK,MAAM;AACjE,SAAO,iBAAiB,SAAS,IAAI;AAAA,EAAQ,iBAAiB,KAAK,IAAI,CAAC;AAAA;AAAA,EAAU,IAAI,KAAK;AAC7F;AAEA,SAAS,iBAAiB,SAGxB;AACA,MAAI,CAAC,QAAQ,WAAW,OAAO,EAAG,QAAO,EAAE,aAAa,CAAC,GAAG,MAAM,QAAQ;AAC1E,QAAM,MAAM,QAAQ,QAAQ,SAAS,CAAC;AACtC,MAAI,MAAM,EAAG,QAAO,EAAE,aAAa,CAAC,GAAG,MAAM,QAAQ;AACrD,QAAM,MAAM,QAAQ,MAAM,GAAG,GAAG,EAAE,KAAK;AACvC,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,IAAI,MAAM,IAAI,GAAG;AAClC,UAAM,MAAM,KAAK,QAAQ,GAAG;AAC5B,QAAI,MAAM,EAAG,aAAY,KAAK,MAAM,GAAG,GAAG,EAAE,KAAK,CAAC,IAAI,KAAK,MAAM,MAAM,CAAC,EAAE,KAAK;AAAA,EACjF;AACA,SAAO,EAAE,aAAa,MAAM,QAAQ,MAAM,MAAM,CAAC,EAAE,UAAU,EAAE;AACjE;AAEA,SAAS,gBAAgB,SAA2B;AAClD,SAAO,QACJ,MAAM,SAAS,EACf,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,OAAO,OAAO;AACnB;AAEA,SAAS,WAAW,MAAsB;AACxC,SAAO,KACJ,QAAQ,+BAA+B,GAAG,EAC1C,QAAQ,6BAA6B,GAAG,EACxC,QAAQ,8BAA8B,MAAM,EAC5C,QAAQ,YAAY,GAAG,EACvB,QAAQ,WAAW,GAAG,EACtB,QAAQ,UAAU,GAAG,EACrB,QAAQ,SAAS,GAAG,EACpB,QAAQ,SAAS,GAAG,EACpB,QAAQ,WAAW,GAAG,EACtB,QAAQ,UAAU,IAAI,EACtB,KAAK;AACV;","names":["parsed"]}

package/dist/cli.d.ts

@@ -0,0 +1,2 @@
+
+export {  }

package/dist/cli.js

@@ -0,0 +1,39 @@
+import {
+  DocumentPipeline,
+  readDocumentPipelineDebug
+} from "./chunk-HZF62UJB.js";
+
+// src/cli.ts
+var command = process.argv[2] ?? "doctor";
+var pipeline = await DocumentPipeline.open(
+  process.env.DOCUMENT_PIPELINE_ROOT ?? ".nebutra/document-pipeline",
+  {
+    tenantId: process.env.NEBUTRA_TENANT_ID ?? "local"
+  }
+);
+if (command === "doctor") {
+  process.stdout.write(`${JSON.stringify(await pipeline.doctor(), null, 2)}
+`);
+} else if (command === "ingest") {
+  const path = process.argv[3];
+  if (!path) {
+    process.stderr.write("Usage: pnpm docs:ingest <path>\n");
+    process.exitCode = 1;
+  } else {
+    process.stdout.write(
+      `${JSON.stringify({ capability: "document-pipeline", result: await pipeline.ingestFile(path) }, null, 2)}
+`
+    );
+  }
+} else if (command === "inspect" || command === "debug") {
+  const entries = await readDocumentPipelineDebug();
+  process.stdout.write(
+    `${JSON.stringify({ capability: "document-pipeline", entries }, null, 2)}
+`
+  );
+} else {
+  process.stderr.write(`Unknown document-pipeline command: ${command}
+`);
+  process.exitCode = 1;
+}
+//# sourceMappingURL=cli.js.map

package/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/cli.ts"],"sourcesContent":["import { DocumentPipeline, readDocumentPipelineDebug } from \"./index\";\n\nconst command = process.argv[2] ?? \"doctor\";\nconst pipeline = await DocumentPipeline.open(\n  process.env.DOCUMENT_PIPELINE_ROOT ?? \".nebutra/document-pipeline\",\n  {\n    tenantId: process.env.NEBUTRA_TENANT_ID ?? \"local\",\n  },\n);\n\nif (command === \"doctor\") {\n  process.stdout.write(`${JSON.stringify(await pipeline.doctor(), null, 2)}\\n`);\n} else if (command === \"ingest\") {\n  const path = process.argv[3];\n  if (!path) {\n    process.stderr.write(\"Usage: pnpm docs:ingest <path>\\n\");\n    process.exitCode = 1;\n  } else {\n    process.stdout.write(\n      `${JSON.stringify({ capability: \"document-pipeline\", result: await pipeline.ingestFile(path) }, null, 2)}\\n`,\n    );\n  }\n} else if (command === \"inspect\" || command === \"debug\") {\n  const entries = await readDocumentPipelineDebug();\n  process.stdout.write(\n    `${JSON.stringify({ capability: \"document-pipeline\", entries }, null, 2)}\\n`,\n  );\n} else {\n  process.stderr.write(`Unknown document-pipeline command: ${command}\\n`);\n  process.exitCode = 1;\n}\n"],"mappings":";;;;;;AAEA,IAAM,UAAU,QAAQ,KAAK,CAAC,KAAK;AACnC,IAAM,WAAW,MAAM,iBAAiB;AAAA,EACtC,QAAQ,IAAI,0BAA0B;AAAA,EACtC;AAAA,IACE,UAAU,QAAQ,IAAI,qBAAqB;AAAA,EAC7C;AACF;AAEA,IAAI,YAAY,UAAU;AACxB,UAAQ,OAAO,MAAM,GAAG,KAAK,UAAU,MAAM,SAAS,OAAO,GAAG,MAAM,CAAC,CAAC;AAAA,CAAI;AAC9E,WAAW,YAAY,UAAU;AAC/B,QAAM,OAAO,QAAQ,KAAK,CAAC;AAC3B,MAAI,CAAC,MAAM;AACT,YAAQ,OAAO,MAAM,kCAAkC;AACvD,YAAQ,WAAW;AAAA,EACrB,OAAO;AACL,YAAQ,OAAO;AAAA,MACb,GAAG,KAAK,UAAU,EAAE,YAAY,qBAAqB,QAAQ,MAAM,SAAS,WAAW,IAAI,EAAE,GAAG,MAAM,CAAC,CAAC;AAAA;AAAA,IAC1G;AAAA,EACF;AACF,WAAW,YAAY,aAAa,YAAY,SAAS;AACvD,QAAM,UAAU,MAAM,0BAA0B;AAChD,UAAQ,OAAO;AAAA,IACb,GAAG,KAAK,UAAU,EAAE,YAAY,qBAAqB,QAAQ,GAAG,MAAM,CAAC,CAAC;AAAA;AAAA,EAC1E;AACF,OAAO;AACL,UAAQ,OAAO,MAAM,sCAAsC,OAAO;AAAA,CAAI;AACtE,UAAQ,WAAW;AACrB;","names":[]}

package/dist/index.d.ts

@@ -0,0 +1,96 @@
+import { ContentStore } from '@nebutra/content-store';
+
+type ParserChoice = "auto" | "markdown" | "html" | "text" | "sidecar";
+type ChunkerChoice = "paragraph" | "recursive" | "hierarchical";
+interface InlineSource {
+    readonly type: "inline";
+    readonly path: string;
+    readonly content: string;
+    readonly mimeType?: string;
+}
+interface FileSource {
+    readonly type: "file";
+    readonly path: string;
+    readonly mimeType?: string;
+}
+type DocumentSource = InlineSource | FileSource;
+interface IngestRequest {
+    readonly tenantId?: string;
+    readonly source: DocumentSource;
+    readonly parser?: ParserChoice;
+    readonly chunker?: ChunkerChoice;
+    readonly targetPath?: string;
+    readonly metadata?: Record<string, string>;
+}
+interface ParseRequest {
+    readonly tenantId?: string;
+    readonly source: DocumentSource;
+    readonly parser?: ParserChoice;
+    readonly chunker?: ChunkerChoice;
+    readonly metadata?: Record<string, string>;
+}
+interface ParsedChunk {
+    readonly id: string;
+    readonly text: string;
+    readonly metadata: {
+        readonly tenantId: string;
+        readonly sourcePath: string;
+        readonly chunkIndex: number;
+        readonly mimeType: string;
+        readonly parser: ParserChoice;
+        readonly heading?: string;
+    } & Record<string, string | number>;
+}
+interface ParsedDocument {
+    readonly tenantId: string;
+    readonly path: string;
+    readonly mimeType: string;
+    readonly parser: ParserChoice;
+    readonly chunks: readonly ParsedChunk[];
+    readonly frontmatter: Record<string, string>;
+}
+interface IngestResult {
+    readonly tenantId: string;
+    readonly path: string;
+    readonly parser: ParserChoice;
+    readonly chunkCount: number;
+    readonly contentIndexed: true;
+}
+interface DocumentPipelineDoctorReport {
+    readonly capability: "document-pipeline";
+    readonly nativeParsers: readonly ParserChoice[];
+    readonly sidecarConfigured: boolean;
+    readonly contentStore: Awaited<ReturnType<ContentStore["doctor"]>>;
+    readonly suggestion?: string;
+}
+interface DocumentParser {
+    parse(request: RequiredTenant<ParseRequest>): Promise<ParsedDocument>;
+    doctor(): Promise<{
+        readonly ok: boolean;
+        readonly suggestion?: string;
+    }>;
+}
+interface DocumentPipelineOptions {
+    readonly tenantId?: string;
+    readonly root?: string;
+    readonly debugRoot?: string;
+    readonly contentStore?: ContentStore;
+    readonly sidecarParser?: DocumentParser;
+}
+type RequiredTenant<T extends {
+    readonly tenantId?: string;
+}> = Omit<T, "tenantId"> & {
+    readonly tenantId: string;
+};
+declare function readDocumentPipelineDebug(root?: string, limit?: number): Promise<unknown[]>;
+declare class DocumentPipeline {
+    #private;
+    constructor(options?: DocumentPipelineOptions);
+    static open(root?: string, options?: Omit<DocumentPipelineOptions, "root" | "contentStore">): Promise<DocumentPipeline>;
+    parse(request: ParseRequest): Promise<ParsedDocument>;
+    ingest(request: IngestRequest): Promise<IngestResult>;
+    ingestFile(path: string, options?: Omit<IngestRequest, "source">): Promise<IngestResult>;
+    doctor(): Promise<DocumentPipelineDoctorReport>;
+}
+
+export { type ChunkerChoice, type DocumentParser, DocumentPipeline, type DocumentPipelineDoctorReport, type DocumentPipelineOptions, type DocumentSource, type FileSource, type IngestRequest, type IngestResult, type InlineSource, type ParseRequest, type ParsedChunk, type ParsedDocument, type ParserChoice, readDocumentPipelineDebug };

package/dist/index.js

@@ -0,0 +1,9 @@
+import {
+  DocumentPipeline,
+  readDocumentPipelineDebug
+} from "./chunk-HZF62UJB.js";
+export {
+  DocumentPipeline,
+  readDocumentPipelineDebug
+};
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/examples/ingest-markdown.ts

@@ -0,0 +1,15 @@
+import { DocumentPipeline } from "@nebutra/document-pipeline";
+
+const pipeline = await DocumentPipeline.open(".nebutra/document-pipeline-example", {
+  tenantId: "local",
+});
+
+const result = await pipeline.ingest({
+  tenantId: "local",
+  source: {
+    type: "inline",
+    path: "research/note.md",
+    content: "---\nschema: research_note\n---\nretrieval augmented generation notes",
+  },
+});
+process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);

package/examples/parse-html.ts

@@ -0,0 +1,15 @@
+import { DocumentPipeline } from "@nebutra/document-pipeline";
+
+const pipeline = await DocumentPipeline.open(".nebutra/document-pipeline-example", {
+  tenantId: "local",
+});
+
+const result = await pipeline.parse({
+  tenantId: "local",
+  source: {
+    type: "inline",
+    path: "landing.html",
+    content: "<h1>Launch</h1><p>Clear operator language</p>",
+  },
+});
+process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);

package/examples/sidecar-gate.ts

@@ -0,0 +1,14 @@
+import { DocumentPipeline } from "@nebutra/document-pipeline";
+
+const pipeline = await DocumentPipeline.open(".nebutra/document-pipeline-example", {
+  tenantId: "local",
+});
+
+try {
+  await pipeline.parse({
+    tenantId: "local",
+    source: { type: "file", path: "paper.pdf" },
+  });
+} catch (error) {
+  process.stdout.write(`${JSON.stringify(error, null, 2)}\n`);
+}

package/package.json

@@ -0,0 +1,58 @@
+{
+  "name": "@nebutra/document-pipeline",
+  "version": "0.1.0",
+  "description": "Document parse, chunk, embed-port, and content-store ingestion pipeline",
+  "private": false,
+  "license": "MIT",
+  "type": "module",
+  "nebutra": {
+    "status": "wip",
+    "productionReady": false,
+    "surface": "execution-capability",
+    "requires": [
+      "@nebutra/content-store for file-truth indexing",
+      "@nebutra/sandbox-runtime for parser sidecars"
+    ],
+    "gaps": [
+      "Complex PDF, office, and OCR parsing require a configured parser sidecar",
+      "Embedding is delegated to content-store/model ports rather than owned here",
+      "Async ingestion jobs are not yet backed by a durable queue"
+    ],
+    "featureId": "document-pipeline",
+    "category": "ai",
+    "summary": "Parser router, metadata-preserving chunks, content-store ingest, doctor, and debug tools"
+  },
+  "main": "./src/index.ts",
+  "types": "./src/index.ts",
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "dependencies": {
+    "@nebutra/content-store": "0.1.0",
+    "@nebutra/errors": "0.1.0",
+    "@nebutra/sandbox-runtime": "0.1.0"
+  },
+  "devDependencies": {
+    "@types/node": "^22.19.15",
+    "tsup": "^8.5.1",
+    "typescript": "^5.9.3",
+    "vitest": "^4.1.4"
+  },
+  "homepage": "https://github.com/Nebutra/Nebutra-Sailor/tree/main/packages/ai/document-pipeline#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Nebutra/Nebutra-Sailor.git",
+    "directory": "packages/ai/document-pipeline"
+  },
+  "bugs": {
+    "url": "https://github.com/Nebutra/Nebutra-Sailor/issues"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsup",
+    "test": "vitest run",
+    "typecheck": "tsc --noEmit"
+  }
+}

package/src/cli.ts

@@ -0,0 +1,31 @@
+import { DocumentPipeline, readDocumentPipelineDebug } from "./index";
+
+const command = process.argv[2] ?? "doctor";
+const pipeline = await DocumentPipeline.open(
+  process.env.DOCUMENT_PIPELINE_ROOT ?? ".nebutra/document-pipeline",
+  {
+    tenantId: process.env.NEBUTRA_TENANT_ID ?? "local",
+  },
+);
+
+if (command === "doctor") {
+  process.stdout.write(`${JSON.stringify(await pipeline.doctor(), null, 2)}\n`);
+} else if (command === "ingest") {
+  const path = process.argv[3];
+  if (!path) {
+    process.stderr.write("Usage: pnpm docs:ingest <path>\n");
+    process.exitCode = 1;
+  } else {
+    process.stdout.write(
+      `${JSON.stringify({ capability: "document-pipeline", result: await pipeline.ingestFile(path) }, null, 2)}\n`,
+    );
+  }
+} else if (command === "inspect" || command === "debug") {
+  const entries = await readDocumentPipelineDebug();
+  process.stdout.write(
+    `${JSON.stringify({ capability: "document-pipeline", entries }, null, 2)}\n`,
+  );
+} else {
+  process.stderr.write(`Unknown document-pipeline command: ${command}\n`);
+  process.exitCode = 1;
+}