@nebutra/document-pipeline 0.1.1 → 0.1.3

package/dist/{chunk-GC4W5KJ2.js → chunk-UF5MSWGW.js}

@@ -205,11 +205,11 @@ function serializeForContentStore(parsed, metadata) {
   return serializeContentFrontmatter(frontmatter, body);
 }
 function htmlToText(html) {
-  return html.replace(/<script[\s\S]*?<\/script>/gi, " ").replace(/<style[\s\S]*?<\/style>/gi, " ").replace(/<\/(h[1-6]|p|li|tr|div)>/gi, "\n\n").replace(/<[^>]+>/g, " ").replace(/&nbsp;/g, " ").replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/[ \t]+/g, " ").replace(/\n\s+/g, "\n").trim();
+  return html.replace(/<script[\s\S]{0,200000}?<\/script>/gi, " ").replace(/<style[\s\S]{0,200000}?<\/style>/gi, " ").replace(/<\/(h[1-6]|p|li|tr|div)>/gi, "\n\n").replace(/<[^>]{0,2000}>/g, " ").replace(/&nbsp;/g, " ").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&amp;/g, "&").replace(/[ \t]+/g, " ").replace(/\n\s+/g, "\n").trim();
 }
 
 export {
   readDocumentPipelineDebug,
   DocumentPipeline
 };
-//# sourceMappingURL=chunk-GC4W5KJ2.js.map
+//# sourceMappingURL=chunk-UF5MSWGW.js.map

package/dist/{chunk-GC4W5KJ2.js.map → chunk-UF5MSWGW.js.map}

@@ -1 +1 @@
-{"version":3,"sources":["../src/index.ts"],"sourcesContent":["import { readFile } from \"node:fs/promises\";\nimport { extname, join } from \"node:path\";\nimport { requireCapabilityTenant } from \"@nebutra/capability-kit\";\nimport { appendCapabilityDebug, readCapabilityDebug } from \"@nebutra/capability-kit/debug\";\nimport {\n  ContentStore,\n  parseContentFrontmatter,\n  serializeContentFrontmatter,\n  splitContentParagraphs,\n} from \"@nebutra/content-store\";\nimport { CapabilityError } from \"@nebutra/errors\";\n\nexport type ParserChoice = \"auto\" | \"markdown\" | \"html\" | \"text\" | \"sidecar\";\nexport type ChunkerChoice = \"paragraph\" | \"recursive\" | \"hierarchical\";\n\nexport interface InlineSource {\n  readonly type: \"inline\";\n  readonly path: string;\n  readonly content: string;\n  readonly mimeType?: string;\n}\n\nexport interface FileSource {\n  readonly type: \"file\";\n  readonly path: string;\n  readonly mimeType?: string;\n}\n\nexport type DocumentSource = InlineSource | FileSource;\n\nexport interface IngestRequest {\n  readonly tenantId?: string;\n  readonly source: DocumentSource;\n  readonly parser?: ParserChoice;\n  readonly chunker?: ChunkerChoice;\n  readonly targetPath?: string;\n  readonly metadata?: Record<string, string>;\n}\n\nexport interface ParseRequest {\n  readonly tenantId?: string;\n  readonly source: DocumentSource;\n  readonly parser?: ParserChoice;\n  readonly chunker?: ChunkerChoice;\n  readonly metadata?: Record<string, string>;\n}\n\nexport interface ParsedChunk {\n  readonly id: string;\n  readonly text: string;\n  readonly metadata: {\n    readonly tenantId: string;\n    readonly sourcePath: string;\n    readonly chunkIndex: number;\n    readonly mimeType: string;\n    readonly parser: ParserChoice;\n    readonly heading?: string;\n  } & Record<string, string | number>;\n}\n\nexport interface ParsedDocument {\n  readonly tenantId: string;\n  readonly path: string;\n  readonly mimeType: string;\n  readonly parser: ParserChoice;\n  readonly chunks: readonly ParsedChunk[];\n  readonly frontmatter: Record<string, string>;\n}\n\nexport interface IngestResult {\n  readonly tenantId: string;\n  readonly path: string;\n  readonly parser: ParserChoice;\n  readonly chunkCount: number;\n  readonly contentIndexed: true;\n}\n\nexport interface DocumentPipelineDoctorReport {\n  readonly capability: \"document-pipeline\";\n  readonly nativeParsers: readonly ParserChoice[];\n  readonly sidecarConfigured: boolean;\n  readonly contentStore: Awaited<ReturnType<ContentStore[\"doctor\"]>>;\n  readonly suggestion?: string;\n}\n\nexport interface DocumentParser {\n  parse(request: RequiredTenant<ParseRequest>): Promise<ParsedDocument>;\n  doctor(): Promise<{ readonly ok: boolean; readonly suggestion?: string }>;\n}\n\nexport interface DocumentPipelineOptions {\n  readonly tenantId?: string;\n  readonly root?: string;\n  readonly debugRoot?: string;\n  readonly contentStore?: ContentStore;\n  readonly sidecarParser?: DocumentParser;\n}\n\ntype RequiredTenant<T extends { readonly tenantId?: string }> = Omit<T, \"tenantId\"> & {\n  readonly tenantId: string;\n};\n\nexport async function readDocumentPipelineDebug(\n  root = process.cwd(),\n  limit = 20,\n): Promise<unknown[]> {\n  return readCapabilityDebug(\"document-pipeline\", { root, limit });\n}\n\nexport class DocumentPipeline {\n  readonly #tenantId: string | undefined;\n  readonly #root: string;\n  readonly #debugRoot: string;\n  readonly #contentStore: ContentStore | undefined;\n  readonly #sidecarParser: DocumentParser | undefined;\n\n  constructor(options: DocumentPipelineOptions = {}) {\n    this.#tenantId = options.tenantId;\n    this.#root = options.root ?? join(process.cwd(), \".nebutra\", \"document-pipeline\");\n    this.#debugRoot = options.debugRoot ?? process.cwd();\n    this.#contentStore = options.contentStore;\n    this.#sidecarParser = options.sidecarParser;\n  }\n\n  static async open(\n    root = \".nebutra/document-pipeline\",\n    options: Omit<DocumentPipelineOptions, \"root\" | \"contentStore\"> = {},\n  ): Promise<DocumentPipeline> {\n    const tenantId = options.tenantId ?? \"local\";\n    const contentStore = await ContentStore.open(join(root, \"content\"), { tenantId });\n    return new DocumentPipeline({ ...options, tenantId, root, contentStore });\n  }\n\n  async parse(request: ParseRequest): Promise<ParsedDocument> {\n    const tenantId = requireCapabilityTenant({\n      explicit: request.tenantId,\n      fallback: this.#tenantId,\n      onMissing: () =>\n        new CapabilityError(\"document-pipeline\", \"Document ingestion requires tenant context\", {\n          suggestion:\n            \"Pass tenantId on the request or construct DocumentPipeline with a tenantId default.\",\n          statusCode: 400,\n        }),\n    });\n    const required: RequiredTenant<ParseRequest> = { ...request, tenantId };\n    const parser = this.#chooseParser(required);\n    if (parser === \"sidecar\") {\n      if (!this.#sidecarParser) {\n        throw new CapabilityError(\"document-pipeline\", \"Parser sidecar is not configured\", {\n          suggestion:\n            \"Configure the parser sidecar for PDFs, office files, OCR, and layout-aware extraction.\",\n          statusCode: 503,\n          metadata: { path: request.source.path },\n        });\n      }\n      const parsed = await this.#sidecarParser.parse(required);\n      await appendCapabilityDebug(\n        \"document-pipeline\",\n        {\n          type: \"parse\",\n          tenantId,\n          parser: \"sidecar\",\n          path: parsed.path,\n        },\n        { root: this.#debugRoot },\n      );\n      return parsed;\n    }\n\n    const raw = await readSource(request.source);\n    const parsed = parseNativeDocument(required, raw, parser);\n    await appendCapabilityDebug(\n      \"document-pipeline\",\n      {\n        type: \"parse\",\n        tenantId,\n        parser,\n        path: parsed.path,\n        chunks: parsed.chunks.length,\n      },\n      { root: this.#debugRoot },\n    );\n    return parsed;\n  }\n\n  async ingest(request: IngestRequest): Promise<IngestResult> {\n    const tenantId = requireCapabilityTenant({\n      explicit: request.tenantId,\n      fallback: this.#tenantId,\n      onMissing: () =>\n        new CapabilityError(\"document-pipeline\", \"Document ingestion requires tenant context\", {\n          suggestion:\n            \"Pass tenantId on the request or construct DocumentPipeline with a tenantId default.\",\n          statusCode: 400,\n        }),\n    });\n    const contentStore = await this.#resolveContentStore(tenantId);\n    const parsed = await this.parse({ ...request, tenantId });\n    const targetPath = request.targetPath ?? parsed.path;\n    const content = serializeForContentStore(parsed, request.metadata);\n    await contentStore.write(targetPath, content);\n    await appendCapabilityDebug(\n      \"document-pipeline\",\n      {\n        type: \"ingest\",\n        tenantId,\n        path: targetPath,\n        parser: parsed.parser,\n        chunks: parsed.chunks.length,\n      },\n      { root: this.#debugRoot },\n    );\n    return {\n      tenantId,\n      path: targetPath,\n      parser: parsed.parser,\n      chunkCount: parsed.chunks.length,\n      contentIndexed: true,\n    };\n  }\n\n  async ingestFile(\n    path: string,\n    options: Omit<IngestRequest, \"source\"> = {},\n  ): Promise<IngestResult> {\n    return this.ingest({ ...options, source: { type: \"file\", path } });\n  }\n\n  async doctor(): Promise<DocumentPipelineDoctorReport> {\n    const contentStore = await this.#resolveContentStore(this.#tenantId ?? \"local\");\n    const sidecar = this.#sidecarParser ? await this.#sidecarParser.doctor() : { ok: false };\n    return {\n      capability: \"document-pipeline\",\n      nativeParsers: [\"markdown\", \"html\", \"text\"],\n      sidecarConfigured: sidecar.ok,\n      contentStore: await contentStore.doctor(),\n      ...(sidecar.ok\n        ? {}\n        : {\n            suggestion:\n              \"Native parsers are active; configure the parser sidecar before ingesting PDFs, office files, OCR, or complex layout documents.\",\n          }),\n    };\n  }\n\n  #chooseParser(request: RequiredTenant<ParseRequest>): ParserChoice {\n    if (request.parser && request.parser !== \"auto\") return request.parser;\n    const mime = request.source.mimeType ?? mimeFromPath(request.source.path);\n    if (mime === \"text/markdown\") return \"markdown\";\n    if (mime === \"text/html\") return \"html\";\n    if (mime.startsWith(\"text/\")) return \"text\";\n    return \"sidecar\";\n  }\n\n  async #resolveContentStore(tenantId: string): Promise<ContentStore> {\n    if (this.#contentStore) return this.#contentStore;\n    return ContentStore.open(join(this.#root, \"content\"), { tenantId });\n  }\n}\n\nasync function readSource(source: DocumentSource): Promise<string> {\n  return source.type === \"inline\" ? source.content : readFile(source.path, \"utf8\");\n}\n\nfunction mimeFromPath(path: string): string {\n  const extension = extname(path).toLowerCase();\n  if (extension === \".md\" || extension === \".mdx\") return \"text/markdown\";\n  if (extension === \".html\" || extension === \".htm\") return \"text/html\";\n  if (extension === \".txt\" || extension === \".csv\" || extension === \".json\") return \"text/plain\";\n  if (extension === \".pdf\") return \"application/pdf\";\n  if (extension === \".docx\")\n    return \"application/vnd.openxmlformats-officedocument.wordprocessingml.document\";\n  if (extension === \".pptx\")\n    return \"application/vnd.openxmlformats-officedocument.presentationml.presentation\";\n  if (extension === \".png\" || extension === \".jpg\" || extension === \".jpeg\") return \"image\";\n  return \"application/octet-stream\";\n}\n\nfunction parseNativeDocument(\n  request: RequiredTenant<ParseRequest>,\n  raw: string,\n  parser: ParserChoice,\n): ParsedDocument {\n  const mimeType = request.source.mimeType ?? mimeFromPath(request.source.path);\n  const { frontmatter, body } =\n    parser === \"markdown\" ? parseContentFrontmatter(raw) : { frontmatter: {}, body: raw };\n  const text = parser === \"html\" ? htmlToText(body) : body;\n  const chunks = splitContentParagraphs(text).map((chunk, index) => ({\n    id: `${request.tenantId}:${request.source.path}:${index}`,\n    text: chunk,\n    metadata: {\n      tenantId: request.tenantId,\n      sourcePath: request.source.path,\n      chunkIndex: index,\n      mimeType,\n      parser,\n      ...(request.metadata ?? {}),\n    },\n  }));\n  return {\n    tenantId: request.tenantId,\n    path: request.source.path,\n    mimeType,\n    parser,\n    chunks:\n      chunks.length > 0\n        ? chunks\n        : [\n            {\n              id: `${request.tenantId}:${request.source.path}:0`,\n              text,\n              metadata: {\n                tenantId: request.tenantId,\n                sourcePath: request.source.path,\n                chunkIndex: 0,\n                mimeType,\n                parser,\n                ...(request.metadata ?? {}),\n              },\n            },\n          ],\n    frontmatter,\n  };\n}\n\nfunction serializeForContentStore(\n  parsed: ParsedDocument,\n  metadata: Record<string, string> | undefined,\n): string {\n  const frontmatter = {\n    ...parsed.frontmatter,\n    ...(metadata ?? {}),\n    source_mime: parsed.mimeType,\n    parser: parsed.parser,\n  };\n  const body = parsed.chunks.map((chunk) => chunk.text).join(\"\\n\\n\");\n  return serializeContentFrontmatter(frontmatter, body);\n}\n\nfunction htmlToText(html: string): string {\n  return html\n    .replace(/<script[\\s\\S]*?<\\/script>/gi, \" \")\n    .replace(/<style[\\s\\S]*?<\\/style>/gi, \" \")\n    .replace(/<\\/(h[1-6]|p|li|tr|div)>/gi, \"\\n\\n\")\n    .replace(/<[^>]+>/g, \" \")\n    .replace(/&nbsp;/g, \" \")\n    .replace(/&amp;/g, \"&\")\n    .replace(/&lt;/g, \"<\")\n    .replace(/&gt;/g, \">\")\n    .replace(/[ \\t]+/g, \" \")\n    .replace(/\\n\\s+/g, \"\\n\")\n    .trim();\n}\n"],"mappings":";AAAA,SAAS,gBAAgB;AACzB,SAAS,SAAS,YAAY;AAC9B,SAAS,+BAA+B;AACxC,SAAS,uBAAuB,2BAA2B;AAC3D;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,uBAAuB;AA4FhC,eAAsB,0BACpB,OAAO,QAAQ,IAAI,GACnB,QAAQ,IACY;AACpB,SAAO,oBAAoB,qBAAqB,EAAE,MAAM,MAAM,CAAC;AACjE;AAEO,IAAM,mBAAN,MAAM,kBAAiB;AAAA,EACnB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAET,YAAY,UAAmC,CAAC,GAAG;AACjD,SAAK,YAAY,QAAQ;AACzB,SAAK,QAAQ,QAAQ,QAAQ,KAAK,QAAQ,IAAI,GAAG,YAAY,mBAAmB;AAChF,SAAK,aAAa,QAAQ,aAAa,QAAQ,IAAI;AACnD,SAAK,gBAAgB,QAAQ;AAC7B,SAAK,iBAAiB,QAAQ;AAAA,EAChC;AAAA,EAEA,aAAa,KACX,OAAO,8BACP,UAAkE,CAAC,GACxC;AAC3B,UAAM,WAAW,QAAQ,YAAY;AACrC,UAAM,eAAe,MAAM,aAAa,KAAK,KAAK,MAAM,SAAS,GAAG,EAAE,SAAS,CAAC;AAChF,WAAO,IAAI,kBAAiB,EAAE,GAAG,SAAS,UAAU,MAAM,aAAa,CAAC;AAAA,EAC1E;AAAA,EAEA,MAAM,MAAM,SAAgD;AAC1D,UAAM,WAAW,wBAAwB;AAAA,MACvC,UAAU,QAAQ;AAAA,MAClB,UAAU,KAAK;AAAA,MACf,WAAW,MACT,IAAI,gBAAgB,qBAAqB,8CAA8C;AAAA,QACrF,YACE;AAAA,QACF,YAAY;AAAA,MACd,CAAC;AAAA,IACL,CAAC;AACD,UAAM,WAAyC,EAAE,GAAG,SAAS,SAAS;AACtE,UAAM,SAAS,KAAK,cAAc,QAAQ;AAC1C,QAAI,WAAW,WAAW;AACxB,UAAI,CAAC,KAAK,gBAAgB;AACxB,cAAM,IAAI,gBAAgB,qBAAqB,oCAAoC;AAAA,UACjF,YACE;AAAA,UACF,YAAY;AAAA,UACZ,UAAU,EAAE,MAAM,QAAQ,OAAO,KAAK;AAAA,QACxC,CAAC;AAAA,MACH;AACA,YAAMA,UAAS,MAAM,KAAK,eAAe,MAAM,QAAQ;AACvD,YAAM;AAAA,QACJ;AAAA,QACA;AAAA,UACE,MAAM;AAAA,UACN;AAAA,UACA,QAAQ;AAAA,UACR,MAAMA,QAAO;AAAA,QACf;AAAA,QACA,EAAE,MAAM,KAAK,WAAW;AAAA,MAC1B;AACA,aAAOA;AAAA,IACT;AAEA,UAAM,MAAM,MAAM,WAAW,QAAQ,MAAM;AAC3C,UAAM,SAAS,oBAAoB,UAAU,KAAK,MAAM;AACxD,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,QACE,MAAM;AAAA,QACN;AAAA,QACA;AAAA,QACA,MAAM,OAAO;AAAA,QACb,QAAQ,OAAO,OAAO;AAAA,MACxB;AAAA,MACA,EAAE,MAAM,KAAK,WAAW;AAAA,IAC1B;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,OAAO,SAA+C;AAC1D,UAAM,WAAW,wBAAwB;AAAA,MACvC,UAAU,QAAQ;AAAA,MAClB,UAAU,KAAK;AAAA,MACf,WAAW,MACT,IAAI,gBAAgB,qBAAqB,8CAA8C;AAAA,QACrF,YACE;AAAA,QACF,YAAY;AAAA,MACd,CAAC;AAAA,IACL,CAAC;AACD,UAAM,eAAe,MAAM,KAAK,qBAAqB,QAAQ;AAC7D,UAAM,SAAS,MAAM,KAAK,MAAM,EAAE,GAAG,SAAS,SAAS,CAAC;AACxD,UAAM,aAAa,QAAQ,cAAc,OAAO;AAChD,UAAM,UAAU,yBAAyB,QAAQ,QAAQ,QAAQ;AACjE,UAAM,aAAa,MAAM,YAAY,OAAO;AAC5C,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,QACE,MAAM;AAAA,QACN;AAAA,QACA,MAAM;AAAA,QACN,QAAQ,OAAO;AAAA,QACf,QAAQ,OAAO,OAAO;AAAA,MACxB;AAAA,MACA,EAAE,MAAM,KAAK,WAAW;AAAA,IAC1B;AACA,WAAO;AAAA,MACL;AAAA,MACA,MAAM;AAAA,MACN,QAAQ,OAAO;AAAA,MACf,YAAY,OAAO,OAAO;AAAA,MAC1B,gBAAgB;AAAA,IAClB;AAAA,EACF;AAAA,EAEA,MAAM,WACJ,MACA,UAAyC,CAAC,GACnB;AACvB,WAAO,KAAK,OAAO,EAAE,GAAG,SAAS,QAAQ,EAAE,MAAM,QAAQ,KAAK,EAAE,CAAC;AAAA,EACnE;AAAA,EAEA,MAAM,SAAgD;AACpD,UAAM,eAAe,MAAM,KAAK,qBAAqB,KAAK,aAAa,OAAO;AAC9E,UAAM,UAAU,KAAK,iBAAiB,MAAM,KAAK,eAAe,OAAO,IAAI,EAAE,IAAI,MAAM;AACvF,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,eAAe,CAAC,YAAY,QAAQ,MAAM;AAAA,MAC1C,mBAAmB,QAAQ;AAAA,MAC3B,cAAc,MAAM,aAAa,OAAO;AAAA,MACxC,GAAI,QAAQ,KACR,CAAC,IACD;AAAA,QACE,YACE;AAAA,MACJ;AAAA,IACN;AAAA,EACF;AAAA,EAEA,cAAc,SAAqD;AACjE,QAAI,QAAQ,UAAU,QAAQ,WAAW,OAAQ,QAAO,QAAQ;AAChE,UAAM,OAAO,QAAQ,OAAO,YAAY,aAAa,QAAQ,OAAO,IAAI;AACxE,QAAI,SAAS,gBAAiB,QAAO;AACrC,QAAI,SAAS,YAAa,QAAO;AACjC,QAAI,KAAK,WAAW,OAAO,EAAG,QAAO;AACrC,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,qBAAqB,UAAyC;AAClE,QAAI,KAAK,cAAe,QAAO,KAAK;AACpC,WAAO,aAAa,KAAK,KAAK,KAAK,OAAO,SAAS,GAAG,EAAE,SAAS,CAAC;AAAA,EACpE;AACF;AAEA,eAAe,WAAW,QAAyC;AACjE,SAAO,OAAO,SAAS,WAAW,OAAO,UAAU,SAAS,OAAO,MAAM,MAAM;AACjF;AAEA,SAAS,aAAa,MAAsB;AAC1C,QAAM,YAAY,QAAQ,IAAI,EAAE,YAAY;AAC5C,MAAI,cAAc,SAAS,cAAc,OAAQ,QAAO;AACxD,MAAI,cAAc,WAAW,cAAc,OAAQ,QAAO;AAC1D,MAAI,cAAc,UAAU,cAAc,UAAU,cAAc,QAAS,QAAO;AAClF,MAAI,cAAc,OAAQ,QAAO;AACjC,MAAI,cAAc;AAChB,WAAO;AACT,MAAI,cAAc;AAChB,WAAO;AACT,MAAI,cAAc,UAAU,cAAc,UAAU,cAAc,QAAS,QAAO;AAClF,SAAO;AACT;AAEA,SAAS,oBACP,SACA,KACA,QACgB;AAChB,QAAM,WAAW,QAAQ,OAAO,YAAY,aAAa,QAAQ,OAAO,IAAI;AAC5E,QAAM,EAAE,aAAa,KAAK,IACxB,WAAW,aAAa,wBAAwB,GAAG,IAAI,EAAE,aAAa,CAAC,GAAG,MAAM,IAAI;AACtF,QAAM,OAAO,WAAW,SAAS,WAAW,IAAI,IAAI;AACpD,QAAM,SAAS,uBAAuB,IAAI,EAAE,IAAI,CAAC,OAAO,WAAW;AAAA,IACjE,IAAI,GAAG,QAAQ,QAAQ,IAAI,QAAQ,OAAO,IAAI,IAAI,KAAK;AAAA,IACvD,MAAM;AAAA,IACN,UAAU;AAAA,MACR,UAAU,QAAQ;AAAA,MAClB,YAAY,QAAQ,OAAO;AAAA,MAC3B,YAAY;AAAA,MACZ;AAAA,MACA;AAAA,MACA,GAAI,QAAQ,YAAY,CAAC;AAAA,IAC3B;AAAA,EACF,EAAE;AACF,SAAO;AAAA,IACL,UAAU,QAAQ;AAAA,IAClB,MAAM,QAAQ,OAAO;AAAA,IACrB;AAAA,IACA;AAAA,IACA,QACE,OAAO,SAAS,IACZ,SACA;AAAA,MACE;AAAA,QACE,IAAI,GAAG,QAAQ,QAAQ,IAAI,QAAQ,OAAO,IAAI;AAAA,QAC9C;AAAA,QACA,UAAU;AAAA,UACR,UAAU,QAAQ;AAAA,UAClB,YAAY,QAAQ,OAAO;AAAA,UAC3B,YAAY;AAAA,UACZ;AAAA,UACA;AAAA,UACA,GAAI,QAAQ,YAAY,CAAC;AAAA,QAC3B;AAAA,MACF;AAAA,IACF;AAAA,IACN;AAAA,EACF;AACF;AAEA,SAAS,yBACP,QACA,UACQ;AACR,QAAM,cAAc;AAAA,IAClB,GAAG,OAAO;AAAA,IACV,GAAI,YAAY,CAAC;AAAA,IACjB,aAAa,OAAO;AAAA,IACpB,QAAQ,OAAO;AAAA,EACjB;AACA,QAAM,OAAO,OAAO,OAAO,IAAI,CAAC,UAAU,MAAM,IAAI,EAAE,KAAK,MAAM;AACjE,SAAO,4BAA4B,aAAa,IAAI;AACtD;AAEA,SAAS,WAAW,MAAsB;AACxC,SAAO,KACJ,QAAQ,+BAA+B,GAAG,EAC1C,QAAQ,6BAA6B,GAAG,EACxC,QAAQ,8BAA8B,MAAM,EAC5C,QAAQ,YAAY,GAAG,EACvB,QAAQ,WAAW,GAAG,EACtB,QAAQ,UAAU,GAAG,EACrB,QAAQ,SAAS,GAAG,EACpB,QAAQ,SAAS,GAAG,EACpB,QAAQ,WAAW,GAAG,EACtB,QAAQ,UAAU,IAAI,EACtB,KAAK;AACV;","names":["parsed"]}
+{"version":3,"sources":["../src/index.ts"],"sourcesContent":["import { readFile } from \"node:fs/promises\";\nimport { extname, join } from \"node:path\";\nimport { requireCapabilityTenant } from \"@nebutra/capability-kit\";\nimport { appendCapabilityDebug, readCapabilityDebug } from \"@nebutra/capability-kit/debug\";\nimport {\n  ContentStore,\n  parseContentFrontmatter,\n  serializeContentFrontmatter,\n  splitContentParagraphs,\n} from \"@nebutra/content-store\";\nimport { CapabilityError } from \"@nebutra/errors\";\n\nexport type ParserChoice = \"auto\" | \"markdown\" | \"html\" | \"text\" | \"sidecar\";\nexport type ChunkerChoice = \"paragraph\" | \"recursive\" | \"hierarchical\";\n\nexport interface InlineSource {\n  readonly type: \"inline\";\n  readonly path: string;\n  readonly content: string;\n  readonly mimeType?: string;\n}\n\nexport interface FileSource {\n  readonly type: \"file\";\n  readonly path: string;\n  readonly mimeType?: string;\n}\n\nexport type DocumentSource = InlineSource | FileSource;\n\nexport interface IngestRequest {\n  readonly tenantId?: string;\n  readonly source: DocumentSource;\n  readonly parser?: ParserChoice;\n  readonly chunker?: ChunkerChoice;\n  readonly targetPath?: string;\n  readonly metadata?: Record<string, string>;\n}\n\nexport interface ParseRequest {\n  readonly tenantId?: string;\n  readonly source: DocumentSource;\n  readonly parser?: ParserChoice;\n  readonly chunker?: ChunkerChoice;\n  readonly metadata?: Record<string, string>;\n}\n\nexport interface ParsedChunk {\n  readonly id: string;\n  readonly text: string;\n  readonly metadata: {\n    readonly tenantId: string;\n    readonly sourcePath: string;\n    readonly chunkIndex: number;\n    readonly mimeType: string;\n    readonly parser: ParserChoice;\n    readonly heading?: string;\n  } & Record<string, string | number>;\n}\n\nexport interface ParsedDocument {\n  readonly tenantId: string;\n  readonly path: string;\n  readonly mimeType: string;\n  readonly parser: ParserChoice;\n  readonly chunks: readonly ParsedChunk[];\n  readonly frontmatter: Record<string, string>;\n}\n\nexport interface IngestResult {\n  readonly tenantId: string;\n  readonly path: string;\n  readonly parser: ParserChoice;\n  readonly chunkCount: number;\n  readonly contentIndexed: true;\n}\n\nexport interface DocumentPipelineDoctorReport {\n  readonly capability: \"document-pipeline\";\n  readonly nativeParsers: readonly ParserChoice[];\n  readonly sidecarConfigured: boolean;\n  readonly contentStore: Awaited<ReturnType<ContentStore[\"doctor\"]>>;\n  readonly suggestion?: string;\n}\n\nexport interface DocumentParser {\n  parse(request: RequiredTenant<ParseRequest>): Promise<ParsedDocument>;\n  doctor(): Promise<{ readonly ok: boolean; readonly suggestion?: string }>;\n}\n\nexport interface DocumentPipelineOptions {\n  readonly tenantId?: string;\n  readonly root?: string;\n  readonly debugRoot?: string;\n  readonly contentStore?: ContentStore;\n  readonly sidecarParser?: DocumentParser;\n}\n\ntype RequiredTenant<T extends { readonly tenantId?: string }> = Omit<T, \"tenantId\"> & {\n  readonly tenantId: string;\n};\n\nexport async function readDocumentPipelineDebug(\n  root = process.cwd(),\n  limit = 20,\n): Promise<unknown[]> {\n  return readCapabilityDebug(\"document-pipeline\", { root, limit });\n}\n\nexport class DocumentPipeline {\n  readonly #tenantId: string | undefined;\n  readonly #root: string;\n  readonly #debugRoot: string;\n  readonly #contentStore: ContentStore | undefined;\n  readonly #sidecarParser: DocumentParser | undefined;\n\n  constructor(options: DocumentPipelineOptions = {}) {\n    this.#tenantId = options.tenantId;\n    this.#root = options.root ?? join(process.cwd(), \".nebutra\", \"document-pipeline\");\n    this.#debugRoot = options.debugRoot ?? process.cwd();\n    this.#contentStore = options.contentStore;\n    this.#sidecarParser = options.sidecarParser;\n  }\n\n  static async open(\n    root = \".nebutra/document-pipeline\",\n    options: Omit<DocumentPipelineOptions, \"root\" | \"contentStore\"> = {},\n  ): Promise<DocumentPipeline> {\n    const tenantId = options.tenantId ?? \"local\";\n    const contentStore = await ContentStore.open(join(root, \"content\"), { tenantId });\n    return new DocumentPipeline({ ...options, tenantId, root, contentStore });\n  }\n\n  async parse(request: ParseRequest): Promise<ParsedDocument> {\n    const tenantId = requireCapabilityTenant({\n      explicit: request.tenantId,\n      fallback: this.#tenantId,\n      onMissing: () =>\n        new CapabilityError(\"document-pipeline\", \"Document ingestion requires tenant context\", {\n          suggestion:\n            \"Pass tenantId on the request or construct DocumentPipeline with a tenantId default.\",\n          statusCode: 400,\n        }),\n    });\n    const required: RequiredTenant<ParseRequest> = { ...request, tenantId };\n    const parser = this.#chooseParser(required);\n    if (parser === \"sidecar\") {\n      if (!this.#sidecarParser) {\n        throw new CapabilityError(\"document-pipeline\", \"Parser sidecar is not configured\", {\n          suggestion:\n            \"Configure the parser sidecar for PDFs, office files, OCR, and layout-aware extraction.\",\n          statusCode: 503,\n          metadata: { path: request.source.path },\n        });\n      }\n      const parsed = await this.#sidecarParser.parse(required);\n      await appendCapabilityDebug(\n        \"document-pipeline\",\n        {\n          type: \"parse\",\n          tenantId,\n          parser: \"sidecar\",\n          path: parsed.path,\n        },\n        { root: this.#debugRoot },\n      );\n      return parsed;\n    }\n\n    const raw = await readSource(request.source);\n    const parsed = parseNativeDocument(required, raw, parser);\n    await appendCapabilityDebug(\n      \"document-pipeline\",\n      {\n        type: \"parse\",\n        tenantId,\n        parser,\n        path: parsed.path,\n        chunks: parsed.chunks.length,\n      },\n      { root: this.#debugRoot },\n    );\n    return parsed;\n  }\n\n  async ingest(request: IngestRequest): Promise<IngestResult> {\n    const tenantId = requireCapabilityTenant({\n      explicit: request.tenantId,\n      fallback: this.#tenantId,\n      onMissing: () =>\n        new CapabilityError(\"document-pipeline\", \"Document ingestion requires tenant context\", {\n          suggestion:\n            \"Pass tenantId on the request or construct DocumentPipeline with a tenantId default.\",\n          statusCode: 400,\n        }),\n    });\n    const contentStore = await this.#resolveContentStore(tenantId);\n    const parsed = await this.parse({ ...request, tenantId });\n    const targetPath = request.targetPath ?? parsed.path;\n    const content = serializeForContentStore(parsed, request.metadata);\n    await contentStore.write(targetPath, content);\n    await appendCapabilityDebug(\n      \"document-pipeline\",\n      {\n        type: \"ingest\",\n        tenantId,\n        path: targetPath,\n        parser: parsed.parser,\n        chunks: parsed.chunks.length,\n      },\n      { root: this.#debugRoot },\n    );\n    return {\n      tenantId,\n      path: targetPath,\n      parser: parsed.parser,\n      chunkCount: parsed.chunks.length,\n      contentIndexed: true,\n    };\n  }\n\n  async ingestFile(\n    path: string,\n    options: Omit<IngestRequest, \"source\"> = {},\n  ): Promise<IngestResult> {\n    return this.ingest({ ...options, source: { type: \"file\", path } });\n  }\n\n  async doctor(): Promise<DocumentPipelineDoctorReport> {\n    const contentStore = await this.#resolveContentStore(this.#tenantId ?? \"local\");\n    const sidecar = this.#sidecarParser ? await this.#sidecarParser.doctor() : { ok: false };\n    return {\n      capability: \"document-pipeline\",\n      nativeParsers: [\"markdown\", \"html\", \"text\"],\n      sidecarConfigured: sidecar.ok,\n      contentStore: await contentStore.doctor(),\n      ...(sidecar.ok\n        ? {}\n        : {\n            suggestion:\n              \"Native parsers are active; configure the parser sidecar before ingesting PDFs, office files, OCR, or complex layout documents.\",\n          }),\n    };\n  }\n\n  #chooseParser(request: RequiredTenant<ParseRequest>): ParserChoice {\n    if (request.parser && request.parser !== \"auto\") return request.parser;\n    const mime = request.source.mimeType ?? mimeFromPath(request.source.path);\n    if (mime === \"text/markdown\") return \"markdown\";\n    if (mime === \"text/html\") return \"html\";\n    if (mime.startsWith(\"text/\")) return \"text\";\n    return \"sidecar\";\n  }\n\n  async #resolveContentStore(tenantId: string): Promise<ContentStore> {\n    if (this.#contentStore) return this.#contentStore;\n    return ContentStore.open(join(this.#root, \"content\"), { tenantId });\n  }\n}\n\nasync function readSource(source: DocumentSource): Promise<string> {\n  return source.type === \"inline\" ? source.content : readFile(source.path, \"utf8\");\n}\n\nfunction mimeFromPath(path: string): string {\n  const extension = extname(path).toLowerCase();\n  if (extension === \".md\" || extension === \".mdx\") return \"text/markdown\";\n  if (extension === \".html\" || extension === \".htm\") return \"text/html\";\n  if (extension === \".txt\" || extension === \".csv\" || extension === \".json\") return \"text/plain\";\n  if (extension === \".pdf\") return \"application/pdf\";\n  if (extension === \".docx\")\n    return \"application/vnd.openxmlformats-officedocument.wordprocessingml.document\";\n  if (extension === \".pptx\")\n    return \"application/vnd.openxmlformats-officedocument.presentationml.presentation\";\n  if (extension === \".png\" || extension === \".jpg\" || extension === \".jpeg\") return \"image\";\n  return \"application/octet-stream\";\n}\n\nfunction parseNativeDocument(\n  request: RequiredTenant<ParseRequest>,\n  raw: string,\n  parser: ParserChoice,\n): ParsedDocument {\n  const mimeType = request.source.mimeType ?? mimeFromPath(request.source.path);\n  const { frontmatter, body } =\n    parser === \"markdown\" ? parseContentFrontmatter(raw) : { frontmatter: {}, body: raw };\n  const text = parser === \"html\" ? htmlToText(body) : body;\n  const chunks = splitContentParagraphs(text).map((chunk, index) => ({\n    id: `${request.tenantId}:${request.source.path}:${index}`,\n    text: chunk,\n    metadata: {\n      tenantId: request.tenantId,\n      sourcePath: request.source.path,\n      chunkIndex: index,\n      mimeType,\n      parser,\n      ...(request.metadata ?? {}),\n    },\n  }));\n  return {\n    tenantId: request.tenantId,\n    path: request.source.path,\n    mimeType,\n    parser,\n    chunks:\n      chunks.length > 0\n        ? chunks\n        : [\n            {\n              id: `${request.tenantId}:${request.source.path}:0`,\n              text,\n              metadata: {\n                tenantId: request.tenantId,\n                sourcePath: request.source.path,\n                chunkIndex: 0,\n                mimeType,\n                parser,\n                ...(request.metadata ?? {}),\n              },\n            },\n          ],\n    frontmatter,\n  };\n}\n\nfunction serializeForContentStore(\n  parsed: ParsedDocument,\n  metadata: Record<string, string> | undefined,\n): string {\n  const frontmatter = {\n    ...parsed.frontmatter,\n    ...(metadata ?? {}),\n    source_mime: parsed.mimeType,\n    parser: parsed.parser,\n  };\n  const body = parsed.chunks.map((chunk) => chunk.text).join(\"\\n\\n\");\n  return serializeContentFrontmatter(frontmatter, body);\n}\n\nfunction htmlToText(html: string): string {\n  return html\n    .replace(/<script[\\s\\S]{0,200000}?<\\/script>/gi, \" \")\n    .replace(/<style[\\s\\S]{0,200000}?<\\/style>/gi, \" \")\n    .replace(/<\\/(h[1-6]|p|li|tr|div)>/gi, \"\\n\\n\")\n    .replace(/<[^>]{0,2000}>/g, \" \")\n    .replace(/&nbsp;/g, \" \")\n    .replace(/&lt;/g, \"<\")\n    .replace(/&gt;/g, \">\")\n    .replace(/&amp;/g, \"&\")\n    .replace(/[ \\t]+/g, \" \")\n    .replace(/\\n\\s+/g, \"\\n\")\n    .trim();\n}\n"],"mappings":";AAAA,SAAS,gBAAgB;AACzB,SAAS,SAAS,YAAY;AAC9B,SAAS,+BAA+B;AACxC,SAAS,uBAAuB,2BAA2B;AAC3D;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,uBAAuB;AA4FhC,eAAsB,0BACpB,OAAO,QAAQ,IAAI,GACnB,QAAQ,IACY;AACpB,SAAO,oBAAoB,qBAAqB,EAAE,MAAM,MAAM,CAAC;AACjE;AAEO,IAAM,mBAAN,MAAM,kBAAiB;AAAA,EACnB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAET,YAAY,UAAmC,CAAC,GAAG;AACjD,SAAK,YAAY,QAAQ;AACzB,SAAK,QAAQ,QAAQ,QAAQ,KAAK,QAAQ,IAAI,GAAG,YAAY,mBAAmB;AAChF,SAAK,aAAa,QAAQ,aAAa,QAAQ,IAAI;AACnD,SAAK,gBAAgB,QAAQ;AAC7B,SAAK,iBAAiB,QAAQ;AAAA,EAChC;AAAA,EAEA,aAAa,KACX,OAAO,8BACP,UAAkE,CAAC,GACxC;AAC3B,UAAM,WAAW,QAAQ,YAAY;AACrC,UAAM,eAAe,MAAM,aAAa,KAAK,KAAK,MAAM,SAAS,GAAG,EAAE,SAAS,CAAC;AAChF,WAAO,IAAI,kBAAiB,EAAE,GAAG,SAAS,UAAU,MAAM,aAAa,CAAC;AAAA,EAC1E;AAAA,EAEA,MAAM,MAAM,SAAgD;AAC1D,UAAM,WAAW,wBAAwB;AAAA,MACvC,UAAU,QAAQ;AAAA,MAClB,UAAU,KAAK;AAAA,MACf,WAAW,MACT,IAAI,gBAAgB,qBAAqB,8CAA8C;AAAA,QACrF,YACE;AAAA,QACF,YAAY;AAAA,MACd,CAAC;AAAA,IACL,CAAC;AACD,UAAM,WAAyC,EAAE,GAAG,SAAS,SAAS;AACtE,UAAM,SAAS,KAAK,cAAc,QAAQ;AAC1C,QAAI,WAAW,WAAW;AACxB,UAAI,CAAC,KAAK,gBAAgB;AACxB,cAAM,IAAI,gBAAgB,qBAAqB,oCAAoC;AAAA,UACjF,YACE;AAAA,UACF,YAAY;AAAA,UACZ,UAAU,EAAE,MAAM,QAAQ,OAAO,KAAK;AAAA,QACxC,CAAC;AAAA,MACH;AACA,YAAMA,UAAS,MAAM,KAAK,eAAe,MAAM,QAAQ;AACvD,YAAM;AAAA,QACJ;AAAA,QACA;AAAA,UACE,MAAM;AAAA,UACN;AAAA,UACA,QAAQ;AAAA,UACR,MAAMA,QAAO;AAAA,QACf;AAAA,QACA,EAAE,MAAM,KAAK,WAAW;AAAA,MAC1B;AACA,aAAOA;AAAA,IACT;AAEA,UAAM,MAAM,MAAM,WAAW,QAAQ,MAAM;AAC3C,UAAM,SAAS,oBAAoB,UAAU,KAAK,MAAM;AACxD,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,QACE,MAAM;AAAA,QACN;AAAA,QACA;AAAA,QACA,MAAM,OAAO;AAAA,QACb,QAAQ,OAAO,OAAO;AAAA,MACxB;AAAA,MACA,EAAE,MAAM,KAAK,WAAW;AAAA,IAC1B;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,OAAO,SAA+C;AAC1D,UAAM,WAAW,wBAAwB;AAAA,MACvC,UAAU,QAAQ;AAAA,MAClB,UAAU,KAAK;AAAA,MACf,WAAW,MACT,IAAI,gBAAgB,qBAAqB,8CAA8C;AAAA,QACrF,YACE;AAAA,QACF,YAAY;AAAA,MACd,CAAC;AAAA,IACL,CAAC;AACD,UAAM,eAAe,MAAM,KAAK,qBAAqB,QAAQ;AAC7D,UAAM,SAAS,MAAM,KAAK,MAAM,EAAE,GAAG,SAAS,SAAS,CAAC;AACxD,UAAM,aAAa,QAAQ,cAAc,OAAO;AAChD,UAAM,UAAU,yBAAyB,QAAQ,QAAQ,QAAQ;AACjE,UAAM,aAAa,MAAM,YAAY,OAAO;AAC5C,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,QACE,MAAM;AAAA,QACN;AAAA,QACA,MAAM;AAAA,QACN,QAAQ,OAAO;AAAA,QACf,QAAQ,OAAO,OAAO;AAAA,MACxB;AAAA,MACA,EAAE,MAAM,KAAK,WAAW;AAAA,IAC1B;AACA,WAAO;AAAA,MACL;AAAA,MACA,MAAM;AAAA,MACN,QAAQ,OAAO;AAAA,MACf,YAAY,OAAO,OAAO;AAAA,MAC1B,gBAAgB;AAAA,IAClB;AAAA,EACF;AAAA,EAEA,MAAM,WACJ,MACA,UAAyC,CAAC,GACnB;AACvB,WAAO,KAAK,OAAO,EAAE,GAAG,SAAS,QAAQ,EAAE,MAAM,QAAQ,KAAK,EAAE,CAAC;AAAA,EACnE;AAAA,EAEA,MAAM,SAAgD;AACpD,UAAM,eAAe,MAAM,KAAK,qBAAqB,KAAK,aAAa,OAAO;AAC9E,UAAM,UAAU,KAAK,iBAAiB,MAAM,KAAK,eAAe,OAAO,IAAI,EAAE,IAAI,MAAM;AACvF,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,eAAe,CAAC,YAAY,QAAQ,MAAM;AAAA,MAC1C,mBAAmB,QAAQ;AAAA,MAC3B,cAAc,MAAM,aAAa,OAAO;AAAA,MACxC,GAAI,QAAQ,KACR,CAAC,IACD;AAAA,QACE,YACE;AAAA,MACJ;AAAA,IACN;AAAA,EACF;AAAA,EAEA,cAAc,SAAqD;AACjE,QAAI,QAAQ,UAAU,QAAQ,WAAW,OAAQ,QAAO,QAAQ;AAChE,UAAM,OAAO,QAAQ,OAAO,YAAY,aAAa,QAAQ,OAAO,IAAI;AACxE,QAAI,SAAS,gBAAiB,QAAO;AACrC,QAAI,SAAS,YAAa,QAAO;AACjC,QAAI,KAAK,WAAW,OAAO,EAAG,QAAO;AACrC,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,qBAAqB,UAAyC;AAClE,QAAI,KAAK,cAAe,QAAO,KAAK;AACpC,WAAO,aAAa,KAAK,KAAK,KAAK,OAAO,SAAS,GAAG,EAAE,SAAS,CAAC;AAAA,EACpE;AACF;AAEA,eAAe,WAAW,QAAyC;AACjE,SAAO,OAAO,SAAS,WAAW,OAAO,UAAU,SAAS,OAAO,MAAM,MAAM;AACjF;AAEA,SAAS,aAAa,MAAsB;AAC1C,QAAM,YAAY,QAAQ,IAAI,EAAE,YAAY;AAC5C,MAAI,cAAc,SAAS,cAAc,OAAQ,QAAO;AACxD,MAAI,cAAc,WAAW,cAAc,OAAQ,QAAO;AAC1D,MAAI,cAAc,UAAU,cAAc,UAAU,cAAc,QAAS,QAAO;AAClF,MAAI,cAAc,OAAQ,QAAO;AACjC,MAAI,cAAc;AAChB,WAAO;AACT,MAAI,cAAc;AAChB,WAAO;AACT,MAAI,cAAc,UAAU,cAAc,UAAU,cAAc,QAAS,QAAO;AAClF,SAAO;AACT;AAEA,SAAS,oBACP,SACA,KACA,QACgB;AAChB,QAAM,WAAW,QAAQ,OAAO,YAAY,aAAa,QAAQ,OAAO,IAAI;AAC5E,QAAM,EAAE,aAAa,KAAK,IACxB,WAAW,aAAa,wBAAwB,GAAG,IAAI,EAAE,aAAa,CAAC,GAAG,MAAM,IAAI;AACtF,QAAM,OAAO,WAAW,SAAS,WAAW,IAAI,IAAI;AACpD,QAAM,SAAS,uBAAuB,IAAI,EAAE,IAAI,CAAC,OAAO,WAAW;AAAA,IACjE,IAAI,GAAG,QAAQ,QAAQ,IAAI,QAAQ,OAAO,IAAI,IAAI,KAAK;AAAA,IACvD,MAAM;AAAA,IACN,UAAU;AAAA,MACR,UAAU,QAAQ;AAAA,MAClB,YAAY,QAAQ,OAAO;AAAA,MAC3B,YAAY;AAAA,MACZ;AAAA,MACA;AAAA,MACA,GAAI,QAAQ,YAAY,CAAC;AAAA,IAC3B;AAAA,EACF,EAAE;AACF,SAAO;AAAA,IACL,UAAU,QAAQ;AAAA,IAClB,MAAM,QAAQ,OAAO;AAAA,IACrB;AAAA,IACA;AAAA,IACA,QACE,OAAO,SAAS,IACZ,SACA;AAAA,MACE;AAAA,QACE,IAAI,GAAG,QAAQ,QAAQ,IAAI,QAAQ,OAAO,IAAI;AAAA,QAC9C;AAAA,QACA,UAAU;AAAA,UACR,UAAU,QAAQ;AAAA,UAClB,YAAY,QAAQ,OAAO;AAAA,UAC3B,YAAY;AAAA,UACZ;AAAA,UACA;AAAA,UACA,GAAI,QAAQ,YAAY,CAAC;AAAA,QAC3B;AAAA,MACF;AAAA,IACF;AAAA,IACN;AAAA,EACF;AACF;AAEA,SAAS,yBACP,QACA,UACQ;AACR,QAAM,cAAc;AAAA,IAClB,GAAG,OAAO;AAAA,IACV,GAAI,YAAY,CAAC;AAAA,IACjB,aAAa,OAAO;AAAA,IACpB,QAAQ,OAAO;AAAA,EACjB;AACA,QAAM,OAAO,OAAO,OAAO,IAAI,CAAC,UAAU,MAAM,IAAI,EAAE,KAAK,MAAM;AACjE,SAAO,4BAA4B,aAAa,IAAI;AACtD;AAEA,SAAS,WAAW,MAAsB;AACxC,SAAO,KACJ,QAAQ,wCAAwC,GAAG,EACnD,QAAQ,sCAAsC,GAAG,EACjD,QAAQ,8BAA8B,MAAM,EAC5C,QAAQ,mBAAmB,GAAG,EAC9B,QAAQ,WAAW,GAAG,EACtB,QAAQ,SAAS,GAAG,EACpB,QAAQ,SAAS,GAAG,EACpB,QAAQ,UAAU,GAAG,EACrB,QAAQ,WAAW,GAAG,EACtB,QAAQ,UAAU,IAAI,EACtB,KAAK;AACV;","names":["parsed"]}

package/dist/cli.js

@@ -1,7 +1,7 @@
 import {
   DocumentPipeline,
   readDocumentPipelineDebug
-} from "./chunk-GC4W5KJ2.js";
+} from "./chunk-UF5MSWGW.js";
 
 // src/cli.ts
 var command = process.argv[2] ?? "doctor";

package/dist/index.js

@@ -1,7 +1,7 @@
 import {
   DocumentPipeline,
   readDocumentPipelineDebug
-} from "./chunk-GC4W5KJ2.js";
+} from "./chunk-UF5MSWGW.js";
 export {
   DocumentPipeline,
   readDocumentPipelineDebug

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nebutra/document-pipeline",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "description": "Document parse, chunk, embed-port, and content-store ingestion pipeline",
   "private": false,
   "license": "MIT",
@@ -22,19 +22,23 @@
     "category": "ai",
     "summary": "Parser router, metadata-preserving chunks, content-store ingest, doctor, and debug tools"
   },
-  "main": "./src/index.ts",
-  "types": "./src/index.ts",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
   "exports": {
-    ".": "./src/index.ts"
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    }
   },
   "dependencies": {
-    "@nebutra/capability-kit": "0.2.1",
-    "@nebutra/content-store": "0.1.1",
-    "@nebutra/errors": "0.1.1",
-    "@nebutra/sandbox-runtime": "0.1.1"
+    "@nebutra/capability-kit": "0.2.4",
+    "@nebutra/errors": "0.1.2",
+    "@nebutra/sandbox-runtime": "0.1.4",
+    "@nebutra/content-store": "0.1.4"
   },
   "devDependencies": {
-    "@types/node": "^22.19.15",
+    "@types/node": "^25.9.1",
     "tsup": "^8.5.1",
     "typescript": "^5.9.3",
     "vitest": "^4.1.4"
@@ -51,6 +55,12 @@
   "publishConfig": {
     "access": "public"
   },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE",
+    "CHANGELOG.md"
+  ],
   "scripts": {
     "build": "tsup",
     "test": "vitest run",

package/.nebutra/debug/content-store.jsonl

@@ -1,6 +0,0 @@
-{"at":"2026-05-19T08:15:10.701Z","type":"reindex","tenantId":"tenant_a","files":0}
-{"at":"2026-05-19T08:15:10.749Z","type":"write","tenantId":"tenant_a","path":"research/BRAND.md"}
-{"at":"2026-05-19T08:15:10.827Z","type":"reindex","tenantId":"tenant_a","files":0}
-{"at":"2026-05-19T08:15:10.866Z","type":"reindex","tenantId":"tenant_a","files":0}
-{"at":"2026-05-19T08:15:10.894Z","type":"reindex","tenantId":"tenant_a","files":0}
-{"at":"2026-05-19T08:15:10.898Z","type":"write","tenantId":"tenant_a","path":"notes/a.md"}

package/.nebutra/debug/document-pipeline.jsonl

@@ -1,3 +0,0 @@
-{"at":"2026-05-19T08:15:10.708Z","type":"parse","tenantId":"tenant_a","parser":"markdown","path":"research/BRAND.md","chunks":1}
-{"at":"2026-05-19T08:15:10.750Z","type":"ingest","tenantId":"tenant_a","path":"research/BRAND.md","parser":"markdown","chunks":1}
-{"at":"2026-05-19T08:15:10.832Z","type":"parse","tenantId":"tenant_a","parser":"html","path":"page.html","chunks":1}

package/.turbo/turbo-build.log

@@ -1,22 +0,0 @@
-
-> @nebutra/document-pipeline@0.1.1 build /home/runner/work/Nebutra-Sailor/Nebutra-Sailor/packages/ai/document-pipeline
-> tsup
-
-CLI Building entry: src/cli.ts, src/index.ts
-CLI Using tsconfig: tsconfig.json
-CLI tsup v8.5.1
-CLI Using tsup config: /home/runner/work/Nebutra-Sailor/Nebutra-Sailor/packages/ai/document-pipeline/tsup.config.ts
-CLI Target: es2022
-CLI Cleaning output folder
-ESM Build start
-ESM dist/cli.js                1.10 KB
-ESM dist/index.js              180.00 B
-ESM dist/chunk-GC4W5KJ2.js     7.81 KB
-ESM dist/cli.js.map            1.93 KB
-ESM dist/index.js.map          71.00 B
-ESM dist/chunk-GC4W5KJ2.js.map 16.66 KB
-ESM ⚡️ Build success in 109ms
-DTS Build start
-DTS ⚡️ Build success in 10840ms
-DTS dist/cli.d.ts   13.00 B
-DTS dist/index.d.ts 3.44 KB

package/.turbo/turbo-test.log

@@ -1,14 +0,0 @@
-
-> @nebutra/document-pipeline@0.1.1 test /home/runner/work/Nebutra-Sailor/Nebutra-Sailor/packages/ai/document-pipeline
-> vitest run
-
-
- RUN  v4.1.4 /home/runner/work/Nebutra-Sailor/Nebutra-Sailor/packages/ai/document-pipeline
-
- ✓ src/index.test.ts (5 tests) 372ms
-
- Test Files  1 passed (1)
-      Tests  5 passed (5)
-   Start at  08:15:09
-   Duration  1.31s (transform 563ms, setup 0ms, import 647ms, tests 372ms, environment 0ms)
-

package/.turbo/turbo-typecheck.log

@@ -1,4 +0,0 @@
-
-> @nebutra/document-pipeline@0.1.1 typecheck /home/runner/work/Nebutra-Sailor/Nebutra-Sailor/packages/ai/document-pipeline
-> tsc --noEmit
-

package/examples/ingest-markdown.ts

@@ -1,15 +0,0 @@
-import { DocumentPipeline } from "@nebutra/document-pipeline";
-
-const pipeline = await DocumentPipeline.open(".nebutra/document-pipeline-example", {
-  tenantId: "local",
-});
-
-const result = await pipeline.ingest({
-  tenantId: "local",
-  source: {
-    type: "inline",
-    path: "research/note.md",
-    content: "---\nschema: research_note\n---\nretrieval augmented generation notes",
-  },
-});
-process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);

package/examples/parse-html.ts

@@ -1,15 +0,0 @@
-import { DocumentPipeline } from "@nebutra/document-pipeline";
-
-const pipeline = await DocumentPipeline.open(".nebutra/document-pipeline-example", {
-  tenantId: "local",
-});
-
-const result = await pipeline.parse({
-  tenantId: "local",
-  source: {
-    type: "inline",
-    path: "landing.html",
-    content: "<h1>Launch</h1><p>Clear operator language</p>",
-  },
-});
-process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);

package/examples/sidecar-gate.ts

@@ -1,14 +0,0 @@
-import { DocumentPipeline } from "@nebutra/document-pipeline";
-
-const pipeline = await DocumentPipeline.open(".nebutra/document-pipeline-example", {
-  tenantId: "local",
-});
-
-try {
-  await pipeline.parse({
-    tenantId: "local",
-    source: { type: "file", path: "paper.pdf" },
-  });
-} catch (error) {
-  process.stdout.write(`${JSON.stringify(error, null, 2)}\n`);
-}

package/src/cli.ts

@@ -1,31 +0,0 @@
-import { DocumentPipeline, readDocumentPipelineDebug } from "./index";
-
-const command = process.argv[2] ?? "doctor";
-const pipeline = await DocumentPipeline.open(
-  process.env.DOCUMENT_PIPELINE_ROOT ?? ".nebutra/document-pipeline",
-  {
-    tenantId: process.env.NEBUTRA_TENANT_ID ?? "local",
-  },
-);
-
-if (command === "doctor") {
-  process.stdout.write(`${JSON.stringify(await pipeline.doctor(), null, 2)}\n`);
-} else if (command === "ingest") {
-  const path = process.argv[3];
-  if (!path) {
-    process.stderr.write("Usage: pnpm docs:ingest <path>\n");
-    process.exitCode = 1;
-  } else {
-    process.stdout.write(
-      `${JSON.stringify({ capability: "document-pipeline", result: await pipeline.ingestFile(path) }, null, 2)}\n`,
-    );
-  }
-} else if (command === "inspect" || command === "debug") {
-  const entries = await readDocumentPipelineDebug();
-  process.stdout.write(
-    `${JSON.stringify({ capability: "document-pipeline", entries }, null, 2)}\n`,
-  );
-} else {
-  process.stderr.write(`Unknown document-pipeline command: ${command}\n`);
-  process.exitCode = 1;
-}

package/src/index.test.ts

@@ -1,93 +0,0 @@
-import { mkdtemp, rm, writeFile } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { ContentStore } from "@nebutra/content-store";
-import { afterEach, describe, expect, it } from "vitest";
-import { DocumentPipeline, readDocumentPipelineDebug } from "./index";
-
-let root: string | undefined;
-let store: ContentStore | undefined;
-
-afterEach(async () => {
-  if (store) await store.close();
-  if (root) await rm(root, { recursive: true, force: true });
-  root = undefined;
-  store = undefined;
-});
-
-describe("DocumentPipeline", () => {
-  it("ingests frontmatter-aware markdown into content-store", async () => {
-    root = await mkdtemp(join(tmpdir(), "document-pipeline-"));
-    store = await ContentStore.open(join(root, "content"), { tenantId: "tenant_a" });
-    const pipeline = new DocumentPipeline({ tenantId: "tenant_a", contentStore: store });
-
-    const result = await pipeline.ingest({
-      tenantId: "tenant_a",
-      source: {
-        type: "inline",
-        path: "research/BRAND.md",
-        content: "---\nschema: note\n---\nretrieval note",
-      },
-    });
-
-    expect(result).toMatchObject({ path: "research/BRAND.md", chunkCount: 1 });
-    await expect(
-      store.search().query("retrieval").filter({ schema: "note" }).topK(5),
-    ).resolves.toHaveLength(1);
-  });
-
-  it("parses html through the lightweight deterministic parser", async () => {
-    root = await mkdtemp(join(tmpdir(), "document-pipeline-"));
-    store = await ContentStore.open(join(root, "content"), { tenantId: "tenant_a" });
-    const pipeline = new DocumentPipeline({ tenantId: "tenant_a", contentStore: store });
-
-    const parsed = await pipeline.parse({
-      tenantId: "tenant_a",
-      source: { type: "inline", path: "page.html", content: "<h1>Title</h1><p>Useful body</p>" },
-    });
-
-    expect(parsed.chunks[0]?.text).toContain("Title");
-    expect(parsed.chunks[0]?.metadata.mimeType).toBe("text/html");
-  });
-
-  it("routes complex binary formats to parser sidecars with a fix suggestion", async () => {
-    root = await mkdtemp(join(tmpdir(), "document-pipeline-"));
-    const pdfPath = join(root, "paper.pdf");
-    await writeFile(pdfPath, "%PDF-1.7\n", "utf8");
-    store = await ContentStore.open(join(root, "content"), { tenantId: "tenant_a" });
-    const pipeline = new DocumentPipeline({ tenantId: "tenant_a", contentStore: store });
-
-    await expect(
-      pipeline.parse({ tenantId: "tenant_a", source: { type: "file", path: pdfPath } }),
-    ).rejects.toMatchObject({
-      capability: "document-pipeline",
-      suggestion: expect.stringContaining("parser sidecar"),
-    });
-  });
-
-  it("requires tenant context for persistent ingestion", async () => {
-    const pipeline = new DocumentPipeline();
-    await expect(
-      pipeline.ingest({ source: { type: "inline", path: "note.md", content: "hello" } }),
-    ).rejects.toMatchObject({
-      capability: "document-pipeline",
-      suggestion: expect.stringContaining("tenantId"),
-    });
-  });
-
-  it("writes inspectable debug records", async () => {
-    root = await mkdtemp(join(tmpdir(), "document-pipeline-"));
-    store = await ContentStore.open(join(root, "content"), { tenantId: "tenant_a" });
-    const pipeline = new DocumentPipeline({
-      tenantId: "tenant_a",
-      contentStore: store,
-      debugRoot: root,
-    });
-    await pipeline.ingest({
-      tenantId: "tenant_a",
-      source: { type: "inline", path: "notes/a.md", content: "debuggable chunk" },
-    });
-
-    expect(await readDocumentPipelineDebug(root)).toEqual(expect.any(Array));
-  });
-});