@veryfront/ext-document-kreuzberg 0.1.993 → 0.1.995

package/esm/extensions/ext-document-kreuzberg/src/extraction-config.d.ts

@@ -1,2 +1,2 @@
-export declare function extractionConfigForMimeType(mimeType: string): Record<string, unknown> | undefined;
+export declare function extractionConfigForMimeType(mimeType: string): Record<string, unknown>;
 //# sourceMappingURL=extraction-config.d.ts.map

package/esm/extensions/ext-document-kreuzberg/src/extraction-config.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"extraction-config.d.ts","sourceRoot":"","sources":["../../../../src/extensions/ext-document-kreuzberg/src/extraction-config.ts"],"names":[],"mappings":"AAcA,wBAAgB,2BAA2B,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAIjG"}
+{"version":3,"file":"extraction-config.d.ts","sourceRoot":"","sources":["../../../../src/extensions/ext-document-kreuzberg/src/extraction-config.ts"],"names":[],"mappings":"AAmBA,wBAAgB,2BAA2B,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAIrF"}

package/esm/extensions/ext-document-kreuzberg/src/extraction-config.js

@@ -1,4 +1,8 @@
-const PDF_TEXT_ONLY_EXTRACTION_CONFIG = {
+const MARKDOWN_EXTRACTION_CONFIG = {
+    outputFormat: "markdown",
+};
+const PDF_MARKDOWN_EXTRACTION_CONFIG = {
+    ...MARKDOWN_EXTRACTION_CONFIG,
     images: { extractImages: false },
     pdfOptions: {
         extractImages: false,
@@ -12,6 +16,6 @@ function normalizeMimeType(mimeType) {
 }
 export function extractionConfigForMimeType(mimeType) {
     return normalizeMimeType(mimeType) === "application/pdf"
-        ? PDF_TEXT_ONLY_EXTRACTION_CONFIG
-        : undefined;
+        ? PDF_MARKDOWN_EXTRACTION_CONFIG
+        : MARKDOWN_EXTRACTION_CONFIG;
 }

package/esm/extensions/ext-document-kreuzberg/src/native-progress-extraction-worker.js

@@ -23,6 +23,12 @@ function decodeXmlText(value) {
 function extractPptxSlideText(xml) {
   return Array.from(xml.matchAll(/<a:t(?:\s[^>]*)?>([\s\S]*?)<\/a:t>/g)).map((match) => decodeXmlText(match[1] ?? "").trim()).filter(Boolean).join(" ").trim();
 }
+function normalizePptxText(value) {
+  return value.replace(/\s+/g, " ").trim();
+}
+function cleanExtractedMarkdown(value) {
+  return value.replace(/!\[[^\]\n]*\]\(\s*\)/g, "").replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
+}
 function slideNumber(path) {
   return Number(path.match(/\/slide(\d+)\.xml$/)?.[1] ?? 0);
 }
@@ -30,6 +36,72 @@ function getXmlAttribute(tag, name) {
   const escapedName = name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
   return tag.match(new RegExp(`\\s${escapedName}=(["'])(.*?)\\1`))?.[2];
 }
+function pptxShapeRole(shapeXml) {
+  const placeholderTag = shapeXml.match(/<(?:\w+:)?ph\b[^>]*>/)?.[0] ?? "";
+  const placeholderType = getXmlAttribute(placeholderTag, "type");
+  if (placeholderType === "title" || placeholderType === "ctrTitle") return "title";
+  if (placeholderType === "subTitle") return "subtitle";
+  const propertyTag = shapeXml.match(/<(?:\w+:)?cNvPr\b[^>]*>/)?.[0] ?? "";
+  const name = getXmlAttribute(propertyTag, "name")?.toLowerCase() ?? "";
+  if (name.includes("subtitle")) return "subtitle";
+  if (name.includes("title")) return "title";
+  return "body";
+}
+function pptxTextShapes(xml) {
+  const shapes = Array.from(xml.matchAll(/<p:sp\b[\s\S]*?<\/p:sp>/g)).map((match) => {
+    const shapeXml = match[0];
+    return {
+      role: pptxShapeRole(shapeXml),
+      text: extractPptxSlideText(shapeXml)
+    };
+  }).filter((shape) => shape.text.length > 0);
+  const hasExplicitHeading = shapes.some(
+    (shape) => shape.role === "title" || shape.role === "subtitle"
+  );
+  if (!hasExplicitHeading && shapes.length === 1) {
+    return [{ ...shapes[0], role: "title" }];
+  }
+  return shapes;
+}
+function pptxShapeRoleQueues(xml) {
+  const roles = /* @__PURE__ */ new Map();
+  for (const shape of pptxTextShapes(xml)) {
+    const key = normalizePptxText(shape.text);
+    if (!key) continue;
+    const queue = roles.get(key) ?? [];
+    queue.push(shape.role);
+    roles.set(key, queue);
+  }
+  return roles;
+}
+function normalizePptxMarkdownHeadings(markdown, xml) {
+  const roles = pptxShapeRoleQueues(xml);
+  if (roles.size === 0) return markdown;
+  return cleanExtractedMarkdown(
+    markdown.split("\n").map((line) => {
+      const heading = line.match(/^#{1,6}\s+(.+?)\s*$/);
+      if (!heading) return line;
+      const text = heading[1] ?? "";
+      const queue = roles.get(normalizePptxText(text));
+      const role = queue?.shift();
+      if (!role) return line;
+      if (role === "title") return `# ${text}`;
+      if (role === "subtitle") return `## ${text}`;
+      return text;
+    }).join("\n")
+  );
+}
+function formatPptxShapeText(shape) {
+  if (shape.role === "title") return `# ${shape.text}`;
+  if (shape.role === "subtitle") return `## ${shape.text}`;
+  return shape.text;
+}
+function formatPptxSlideText(xml) {
+  const shapes = pptxTextShapes(xml);
+  const nonShapeText = extractPptxSlideText(xml.replace(/<p:sp\b[\s\S]*?<\/p:sp>/g, ""));
+  if (shapes.length === 0) return nonShapeText || extractPptxSlideText(xml);
+  return [...shapes.map(formatPptxShapeText), nonShapeText].filter(Boolean).join("\n\n").trim();
+}
 function normalizeZipPath(path) {
   const parts = [];
   for (const part of path.split("/")) {
@@ -89,6 +161,42 @@ async function pptxSlidePaths(zip) {
     ...fallback.filter((path) => !ordered.includes(path))
   ];
 }
+async function buildSingleSlidePptx(slideXml) {
+  const zip = new JSZip();
+  zip.file(
+    "[Content_Types].xml",
+    `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">
+  <Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>
+  <Default Extension="xml" ContentType="application/xml"/>
+  <Override PartName="/ppt/presentation.xml" ContentType="application/vnd.openxmlformats-officedocument.presentationml.presentation.main+xml"/>
+  <Override PartName="/ppt/slides/slide1.xml" ContentType="application/vnd.openxmlformats-officedocument.presentationml.slide+xml"/>
+</Types>`
+  );
+  zip.file(
+    "_rels/.rels",
+    `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
+  <Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="ppt/presentation.xml"/>
+</Relationships>`
+  );
+  zip.file(
+    "ppt/presentation.xml",
+    `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<p:presentation xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">
+  <p:sldIdLst><p:sldId id="256" r:id="rId1"/></p:sldIdLst>
+</p:presentation>`
+  );
+  zip.file(
+    "ppt/_rels/presentation.xml.rels",
+    `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
+  <Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/slide" Target="slides/slide1.xml"/>
+</Relationships>`
+  );
+  zip.file("ppt/slides/slide1.xml", slideXml);
+  return await zip.generateAsync({ type: "uint8array" });
+}
 async function extractPdfByPage(buffer) {
   const { extractBytes } = await loadKreuzbergNative();
   const source = await PDFDocument.load(buffer, { ignoreEncryption: true });
@@ -119,17 +227,37 @@ async function extractPdfByPage(buffer) {
 async function extractPptxBySlide(buffer, mimeType) {
   const zip = await JSZip.loadAsync(buffer);
   const slidePaths = await pptxSlidePaths(zip);
+  const config = extractionConfigForMimeType(mimeType);
+  let nativeExtractor;
+  try {
+    nativeExtractor = await loadKreuzbergNative();
+  } catch (error) {
+    if (!slidePaths.length) throw error;
+  }
   if (!slidePaths.length) {
-    const { extractBytes } = await loadKreuzbergNative();
-    const result = await extractBytes(new Uint8Array(buffer), mimeType);
-    postProgress({ unit: "file", current: 1, total: 1, characters: result.content.length });
-    return result.content;
+    const result = await nativeExtractor.extractBytes(
+      new Uint8Array(buffer),
+      mimeType,
+      config
+    );
+    const content = cleanExtractedMarkdown(result.content);
+    postProgress({ unit: "file", current: 1, total: 1, characters: content.length });
+    return content;
   }
   const slides = [];
   for (const [index, path] of slidePaths.entries()) {
     const file = zip.file(path);
     const xml = file ? await file.async("text") : "";
-    const content = extractPptxSlideText(xml);
+    let content;
+    if (nativeExtractor) {
+      try {
+        const slideBytes = await buildSingleSlidePptx(xml);
+        const result = await nativeExtractor.extractBytes(slideBytes, mimeType, config);
+        content = normalizePptxMarkdownHeadings(cleanExtractedMarkdown(result.content), xml);
+      } catch {
+      }
+    }
+    content ||= formatPptxSlideText(xml);
     slides.push(content);
     postProgress({
       unit: "slide",
@@ -142,7 +270,11 @@ async function extractPptxBySlide(buffer, mimeType) {
 }
 async function extractWholeFile(buffer, mimeType) {
   const { extractBytes } = await loadKreuzbergNative();
-  const result = await extractBytes(new Uint8Array(buffer), mimeType);
+  const result = await extractBytes(
+    new Uint8Array(buffer),
+    mimeType,
+    extractionConfigForMimeType(mimeType)
+  );
   postProgress({ unit: "file", current: 1, total: 1, characters: result.content.length });
   return result.content;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@veryfront/ext-document-kreuzberg",
-  "version": "0.1.993",
+  "version": "0.1.995",
   "description": "Veryfront first-party extension package for ext-document-kreuzberg",
   "keywords": [
     "veryfront",
@@ -55,7 +55,7 @@
     "pdf-lib": "1.17.1"
   },
   "peerDependencies": {
-    "veryfront": "^0.1.993"
+    "veryfront": "^0.1.995"
   },
   "type": "module",
   "types": "./esm/extensions/ext-document-kreuzberg/src/index.d.ts",