@statewalker/content-extractors 0.2.2

package/dist/index.js

@@ -0,0 +1,13 @@
+import { collectBytes, collectText } from "./collect-bytes.js";
+import { htmlToMarkdown } from "./html-to-markdown.js";
+import { docxExtractor } from "./extractors/docx-extractor.js";
+import { htmlExtractor } from "./extractors/html-extractor.js";
+import { markdownExtractor } from "./extractors/markdown-extractor.js";
+import { pdfExtractor } from "./extractors/pdf-extractor.js";
+import { textExtractor } from "./extractors/text-extractor.js";
+import { xlsxExtractor } from "./extractors/xlsx-extractor.js";
+import { ExtractorRegistry } from "./extractor-registry.js";
+import { createDefaultRegistry } from "./extractors/index.js";
+import { markdownToHtml } from "./markdown-to-html.js";
+import { getMimeType } from "./mime-utils.js";
+export { ExtractorRegistry, collectBytes, collectText, createDefaultRegistry, docxExtractor, getMimeType, htmlExtractor, htmlToMarkdown, markdownExtractor, markdownToHtml, pdfExtractor, textExtractor, xlsxExtractor };

package/dist/markdown-to-html.d.ts

@@ -0,0 +1,10 @@
+//#region src/markdown-to-html.d.ts
+/**
+ * Provides the reverse of `htmlToMarkdown` for cases where extracted
+ * markdown needs to be rendered back to HTML (e.g. previews, email
+ * output). Configured with pass-through raw HTML and auto-linking
+ * so embedded markup and bare URLs survive the round-trip.
+ */
+export declare function markdownToHtml(markdown: string): string;
+//#endregion
+//# sourceMappingURL=markdown-to-html.d.ts.map

package/dist/markdown-to-html.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"markdown-to-html.d.ts","names":[],"sources":["../src/markdown-to-html.ts"],"mappings":";;;;;;;wBAegB,eAAe"}

package/dist/markdown-to-html.js

@@ -0,0 +1,22 @@
+import MarkdownIt from "markdown-it";
+//#region src/markdown-to-html.ts
+const md = new MarkdownIt({
+	html: true,
+	linkify: true,
+	typographer: true,
+	breaks: false
+});
+/**
+* Provides the reverse of `htmlToMarkdown` for cases where extracted
+* markdown needs to be rendered back to HTML (e.g. previews, email
+* output). Configured with pass-through raw HTML and auto-linking
+* so embedded markup and bare URLs survive the round-trip.
+*/
+function markdownToHtml(markdown) {
+	if (!markdown || markdown.trim() === "") return "";
+	return md.render(markdown);
+}
+//#endregion
+export { markdownToHtml };
+
+//# sourceMappingURL=markdown-to-html.js.map

package/dist/markdown-to-html.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"markdown-to-html.js","names":[],"sources":["../src/markdown-to-html.ts"],"sourcesContent":["import MarkdownIt from \"markdown-it\";\n\nconst md = new MarkdownIt({\n  html: true,\n  linkify: true,\n  typographer: true,\n  breaks: false,\n});\n\n/**\n * Provides the reverse of `htmlToMarkdown` for cases where extracted\n * markdown needs to be rendered back to HTML (e.g. previews, email\n * output). Configured with pass-through raw HTML and auto-linking\n * so embedded markup and bare URLs survive the round-trip.\n */\nexport function markdownToHtml(markdown: string): string {\n  if (!markdown || markdown.trim() === \"\") {\n    return \"\";\n  }\n  return md.render(markdown);\n}\n"],"mappings":";;AAEA,MAAM,KAAK,IAAI,WAAW;CACxB,MAAM;CACN,SAAS;CACT,aAAa;CACb,QAAQ;AACV,CAAC;;;;;;;AAQD,SAAgB,eAAe,UAA0B;CACvD,IAAI,CAAC,YAAY,SAAS,KAAK,MAAM,IACnC,OAAO;CAET,OAAO,GAAG,OAAO,QAAQ;AAC3B"}

package/dist/mime-utils.d.ts

@@ -0,0 +1,10 @@
+//#region src/mime-utils.d.ts
+/**
+ * Lightweight alternative to a full MIME-sniffing library -- covers only
+ * the extensions this package can actually extract, keeping the dependency
+ * footprint small. Used by callers who have a filename but no
+ * Content-Type header and need a MIME type for the registry fallback path.
+ */
+export declare function getMimeType(path: string): string | undefined;
+//#endregion
+//# sourceMappingURL=mime-utils.d.ts.map

package/dist/mime-utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mime-utils.d.ts","names":[],"sources":["../src/mime-utils.ts"],"mappings":";;;;;;;wBAmBgB,YAAY"}

package/dist/mime-utils.js

@@ -0,0 +1,29 @@
+//#region src/mime-utils.ts
+const MIME_MAP = {
+	".md": "text/markdown",
+	".markdown": "text/markdown",
+	".txt": "text/plain",
+	".html": "text/html",
+	".htm": "text/html",
+	".json": "application/json",
+	".pdf": "application/pdf",
+	".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+	".docm": "application/vnd.ms-word.document.macroEnabled.12",
+	".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+};
+/**
+* Lightweight alternative to a full MIME-sniffing library -- covers only
+* the extensions this package can actually extract, keeping the dependency
+* footprint small. Used by callers who have a filename but no
+* Content-Type header and need a MIME type for the registry fallback path.
+*/
+function getMimeType(path) {
+	const dotIndex = path.lastIndexOf(".");
+	if (dotIndex === -1) return void 0;
+	const ext = path.slice(dotIndex).toLowerCase();
+	return MIME_MAP[ext];
+}
+//#endregion
+export { getMimeType };
+
+//# sourceMappingURL=mime-utils.js.map

package/dist/mime-utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mime-utils.js","names":[],"sources":["../src/mime-utils.ts"],"sourcesContent":["const MIME_MAP: Record<string, string> = {\n  \".md\": \"text/markdown\",\n  \".markdown\": \"text/markdown\",\n  \".txt\": \"text/plain\",\n  \".html\": \"text/html\",\n  \".htm\": \"text/html\",\n  \".json\": \"application/json\",\n  \".pdf\": \"application/pdf\",\n  \".docx\": \"application/vnd.openxmlformats-officedocument.wordprocessingml.document\",\n  \".docm\": \"application/vnd.ms-word.document.macroEnabled.12\",\n  \".xlsx\": \"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet\",\n};\n\n/**\n * Lightweight alternative to a full MIME-sniffing library -- covers only\n * the extensions this package can actually extract, keeping the dependency\n * footprint small. Used by callers who have a filename but no\n * Content-Type header and need a MIME type for the registry fallback path.\n */\nexport function getMimeType(path: string): string | undefined {\n  const dotIndex = path.lastIndexOf(\".\");\n  if (dotIndex === -1) return undefined;\n  const ext = path.slice(dotIndex).toLowerCase();\n  return MIME_MAP[ext];\n}\n"],"mappings":";AAAA,MAAM,WAAmC;CACvC,OAAO;CACP,aAAa;CACb,QAAQ;CACR,SAAS;CACT,QAAQ;CACR,SAAS;CACT,QAAQ;CACR,SAAS;CACT,SAAS;CACT,SAAS;AACX;;;;;;;AAQA,SAAgB,YAAY,MAAkC;CAC5D,MAAM,WAAW,KAAK,YAAY,GAAG;CACrC,IAAI,aAAa,IAAI,OAAO,KAAA;CAC5B,MAAM,MAAM,KAAK,MAAM,QAAQ,CAAC,CAAC,YAAY;CAC7C,OAAO,SAAS;AAClB"}

package/dist/types.d.ts

@@ -0,0 +1,29 @@
+//#region src/types.d.ts
+/**
+ * The lowest-level extraction contract: raw bytes in, text out.
+ * Individual format handlers (PDF, DOCX, HTML, etc.) implement this type
+ * so the registry can treat every file format uniformly without knowing
+ * the parsing details.
+ */
+export type ContentExtractor = (content: AsyncIterable<Uint8Array> | Iterable<Uint8Array>) => Promise<string | unknown>;
+/**
+ * Decouples post-processing from extraction so callers stay free of
+ * LLM or AI dependencies. Consumers can inject an LLM-backed normalizer
+ * to clean up, restructure, or enrich extracted markdown without the
+ * core package ever knowing about the LLM.
+ */
+export interface ContentNormalizer {
+  /**
+   * Refines raw extracted markdown -- fixing formatting artifacts, improving
+   * structure, or enriching content. The optional context hint lets the
+   * normalizer tailor its output (e.g. summarize for a specific audience).
+   */
+  normalize(params: {
+    markdown: string;
+    context?: string;
+  }): Promise<{
+    markdown: string;
+  }>;
+}
+//#endregion
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","names":[],"sources":["../src/types.ts"],"mappings":";;;;;;;YAMY,oBACV,SAAS,cAAc,cAAc,SAAS,gBAC3C;;;;;;;iBAQY;;;;;;EAMf,UAAU;IAAU;IAAkB;MAAqB;IAAU"}

package/dist/types.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -0,0 +1,71 @@
+{
+  "name": "@statewalker/content-extractors",
+  "version": "0.2.2",
+  "private": false,
+  "type": "module",
+  "description": "Content extractors: turn PDF, DOCX, XLSX, Markdown, and HTML into markdown text.",
+  "homepage": "https://github.com/statewalker/statewalker-content",
+  "author": {
+    "name": "Mikhail Kotelnikov",
+    "email": "mikhail.kotelnikov@gmail.com"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+ssh://git@github.com/statewalker/statewalker-content.git"
+  },
+  "exports": {
+    ".": {
+      "source": "./src/index.ts",
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    },
+    "./types": {
+      "source": "./src/types.ts",
+      "types": "./dist/types.d.ts",
+      "import": "./dist/types.js",
+      "default": "./dist/types.js"
+    },
+    "./extractors": {
+      "source": "./src/extractors/index.ts",
+      "types": "./dist/extractors/index.d.ts",
+      "import": "./dist/extractors/index.js",
+      "default": "./dist/extractors/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "dependencies": {
+    "@joplin/turndown-plugin-gfm": "^1.0.68",
+    "@types/turndown": "^5.0.6",
+    "exceljs": "^4.4.0",
+    "mammoth": "^1.13.0",
+    "markdown-it": "^15.0.2",
+    "turndown": "^7.2.4",
+    "unpdf": "^1.8.1"
+  },
+  "devDependencies": {
+    "@types/node": "^26.6.4",
+    "rimraf": "^6.1.3",
+    "tsdown": "^0.23.0",
+    "typescript": "^7.0.2",
+    "vitest": "^5.0.3"
+  },
+  "sideEffects": false,
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsdown",
+    "dev": "tsdown --watch",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "typecheck": "tsc --noEmit",
+    "clean": "rimraf dist",
+    "lint": "biome check --write .",
+    "format": "biome format --write ."
+  }
+}

package/src/collect-bytes.ts

@@ -0,0 +1,39 @@
+/**
+ * Bridges the streaming world (chunked iterables from fetch, file reads,
+ * etc.) with libraries that require a single contiguous buffer (pdfjs-dist,
+ * mammoth). Returns the original chunk when only one arrives, avoiding
+ * an extra copy in the common single-read case.
+ */
+export async function collectBytes(
+  content: AsyncIterable<Uint8Array> | Iterable<Uint8Array>,
+): Promise<Uint8Array> {
+  const chunks: Uint8Array[] = [];
+  let totalLength = 0;
+  for await (const chunk of content) {
+    chunks.push(chunk);
+    totalLength += chunk.length;
+  }
+  const first = chunks[0];
+  if (chunks.length === 1 && first) {
+    return first;
+  }
+  const result = new Uint8Array(totalLength);
+  let offset = 0;
+  for (const chunk of chunks) {
+    result.set(chunk, offset);
+    offset += chunk.length;
+  }
+  return result;
+}
+
+/**
+ * Shorthand for text-based extractors (plain text, markdown, HTML) that
+ * need the full content as a string. Avoids duplicating the
+ * collect-then-decode pattern across every text extractor.
+ */
+export async function collectText(
+  content: AsyncIterable<Uint8Array> | Iterable<Uint8Array>,
+): Promise<string> {
+  const bytes = await collectBytes(content);
+  return new TextDecoder().decode(bytes);
+}

package/src/extractor-registry.ts

@@ -0,0 +1,69 @@
+import type { ContentExtractor } from "./types.js";
+
+/**
+ * Central routing table that decouples file-format knowledge from the
+ * extraction pipeline. New formats can be added without touching pipeline
+ * code. Longest-suffix matching ensures compound extensions (e.g.
+ * `*.tar.gz`) win over shorter ones, and MIME-type fallback handles
+ * files that arrive without a recognizable extension.
+ */
+export class ExtractorRegistry {
+  private patternMap = new Map<string, ContentExtractor>();
+  private mimeMap = new Map<string, ContentExtractor>();
+
+  /**
+   * Associates a suffix pattern with an extractor so the registry can
+   * route files to the right handler based on their filename.
+   */
+  registerByPattern(pattern: string, extractor: ContentExtractor): void {
+    this.patternMap.set(pattern, extractor);
+  }
+
+  /**
+   * Provides a fallback route for files that lack a recognizable extension
+   * (e.g. temp files or HTTP responses identified only by Content-Type).
+   */
+  registerByMime(mimeType: string, extractor: ContentExtractor): void {
+    this.mimeMap.set(mimeType, extractor);
+  }
+
+  /**
+   * Resolves by filename suffix, preferring the most specific match so
+   * compound extensions like `*.test.html` beat `*.html` when both exist.
+   */
+  getByPath(path: string): ContentExtractor | undefined {
+    const filename = path.includes("/") ? path.slice(path.lastIndexOf("/") + 1) : path;
+    const lowerFilename = filename.toLowerCase();
+
+    let bestMatch: ContentExtractor | undefined;
+    let bestLength = 0;
+
+    for (const [pattern, extractor] of this.patternMap) {
+      // Pattern format: *.ext or *.compound.ext
+      const suffix = pattern.startsWith("*") ? pattern.slice(1) : pattern;
+      if (lowerFilename.endsWith(suffix) && suffix.length > bestLength) {
+        bestMatch = extractor;
+        bestLength = suffix.length;
+      }
+    }
+
+    return bestMatch;
+  }
+
+  /** Direct MIME lookup -- used internally by `get` as the fallback path. */
+  getByMime(mimeType: string): ContentExtractor | undefined {
+    return this.mimeMap.get(mimeType);
+  }
+
+  /**
+   * Single entry point for the pipeline -- tries the most reliable signal
+   * (file extension) first, then falls back to MIME type so callers never
+   * need to worry about the resolution strategy.
+   */
+  get(path: string, mimeType?: string): ContentExtractor | undefined {
+    const byPath = this.getByPath(path);
+    if (byPath) return byPath;
+    if (mimeType) return this.getByMime(mimeType);
+    return undefined;
+  }
+}

package/src/extractors/docx-extractor.ts

@@ -0,0 +1,21 @@
+import mammoth from "mammoth";
+import { collectBytes } from "../collect-bytes.js";
+import { htmlToMarkdown } from "../html-to-markdown.js";
+import type { ContentExtractor } from "../types.js";
+
+/**
+ * Unlocks Word document content for text-based pipelines via a two-stage
+ * conversion (DOCX -> HTML -> markdown). Supplies both `buffer` and
+ * `arrayBuffer` to mammoth so the same code runs in Node.js and browsers
+ * without environment detection.
+ */
+export const docxExtractor: ContentExtractor = async (content) => {
+  const bytes = await collectBytes(content);
+  const arrayBuffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
+  // mammoth uses "buffer" in Node.js and "arrayBuffer" in browsers
+  const result = await mammoth.convertToHtml({
+    buffer: arrayBuffer,
+    arrayBuffer,
+  } as Parameters<typeof mammoth.convertToHtml>[0]);
+  return htmlToMarkdown(result.value);
+};

package/src/extractors/html-extractor.ts

@@ -0,0 +1,13 @@
+import { collectText } from "../collect-bytes.js";
+import { htmlToMarkdown } from "../html-to-markdown.js";
+import type { ContentExtractor } from "../types.js";
+
+/**
+ * Converts HTML files to markdown so the pipeline outputs a uniform
+ * text format. Delegates to `htmlToMarkdown` for the actual conversion,
+ * keeping this extractor focused on the bytes-to-string bridging step.
+ */
+export const htmlExtractor: ContentExtractor = async (content) => {
+  const html = await collectText(content);
+  return htmlToMarkdown(html);
+};

package/src/extractors/index.ts

@@ -0,0 +1,34 @@
+export { docxExtractor } from "./docx-extractor.js";
+export { htmlExtractor } from "./html-extractor.js";
+export { markdownExtractor } from "./markdown-extractor.js";
+export { pdfExtractor } from "./pdf-extractor.js";
+export { textExtractor } from "./text-extractor.js";
+export { xlsxExtractor } from "./xlsx-extractor.js";
+
+import { ExtractorRegistry } from "../extractor-registry.js";
+import { docxExtractor } from "./docx-extractor.js";
+import { htmlExtractor } from "./html-extractor.js";
+import { markdownExtractor } from "./markdown-extractor.js";
+import { pdfExtractor } from "./pdf-extractor.js";
+import { textExtractor } from "./text-extractor.js";
+import { xlsxExtractor } from "./xlsx-extractor.js";
+
+/**
+ * Provides a batteries-included registry so most consumers can start
+ * extracting immediately without manual format registration. Custom
+ * registries are still supported for apps that need to override or
+ * limit the built-in set.
+ */
+export function createDefaultRegistry(): ExtractorRegistry {
+  const registry = new ExtractorRegistry();
+  registry.registerByPattern("*.md", markdownExtractor);
+  registry.registerByPattern("*.markdown", markdownExtractor);
+  registry.registerByPattern("*.txt", textExtractor);
+  registry.registerByPattern("*.html", htmlExtractor);
+  registry.registerByPattern("*.htm", htmlExtractor);
+  registry.registerByPattern("*.pdf", pdfExtractor);
+  registry.registerByPattern("*.docx", docxExtractor);
+  registry.registerByPattern("*.docm", docxExtractor);
+  registry.registerByPattern("*.xlsx", xlsxExtractor);
+  return registry;
+}

package/src/extractors/markdown-extractor.ts

@@ -0,0 +1,12 @@
+import { collectText } from "../collect-bytes.js";
+import type { ContentExtractor } from "../types.js";
+
+/**
+ * Passthrough extractor for markdown files. Functionally identical to
+ * `textExtractor` but registered under its own extension patterns so
+ * the registry can distinguish `.md` from `.txt` -- allowing future
+ * markdown-specific processing without breaking existing registrations.
+ */
+export const markdownExtractor: ContentExtractor = async (content) => {
+  return collectText(content);
+};

package/src/extractors/pdf-extractor.ts

@@ -0,0 +1,23 @@
+import { extractText } from "unpdf";
+import { collectBytes } from "../collect-bytes.js";
+import type { ContentExtractor } from "../types.js";
+
+/**
+ * Extracts PDF content page-by-page, formatting each page as a markdown
+ * section with a `## Page N` header. This preserves page boundaries in
+ * the indexed content, making search results more precise.
+ */
+export const pdfExtractor: ContentExtractor = async (content) => {
+  const bytes = await collectBytes(content);
+  const { text: pages } = await extractText(bytes);
+
+  const sections: string[] = [];
+  for (let i = 0; i < pages.length; i++) {
+    const pageText = pages[i]?.trim();
+    if (pageText) {
+      sections.push(`## Page ${i + 1}\n\n${pageText}`);
+    }
+  }
+
+  return sections.join("\n\n");
+};

package/src/extractors/text-extractor.ts

@@ -0,0 +1,11 @@
+import { collectText } from "../collect-bytes.js";
+import type { ContentExtractor } from "../types.js";
+
+/**
+ * Identity extractor for plain text -- no conversion needed, just
+ * decode bytes to a string. Exists as a distinct entry so `.txt` files
+ * route through the same pipeline as every other format.
+ */
+export const textExtractor: ContentExtractor = async (content) => {
+  return collectText(content);
+};

package/src/extractors/xlsx-extractor.ts

@@ -0,0 +1,57 @@
+import ExcelJS from "exceljs";
+import { collectBytes } from "../collect-bytes.js";
+import type { ContentExtractor } from "../types.js";
+
+/**
+ * Converts Excel spreadsheets into markdown tables so the content
+ * is searchable and consumable by text-based pipelines. Each worksheet
+ * becomes a separate section with a level-2 heading. Empty sheets are
+ * skipped to avoid noise.
+ */
+export const xlsxExtractor: ContentExtractor = async (content) => {
+  const bytes = await collectBytes(content);
+  const workbook = new ExcelJS.Workbook();
+  const arrayBuffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
+  await workbook.xlsx.load(arrayBuffer as ArrayBuffer);
+
+  const sections: string[] = [];
+
+  workbook.eachSheet((sheet) => {
+    const rows: string[][] = [];
+
+    sheet.eachRow((row) => {
+      const cells: string[] = [];
+      row.eachCell({ includeEmpty: true }, (cell) => {
+        try {
+          const v = cell.value;
+          cells.push(v == null ? "" : String(v));
+        } catch {
+          cells.push("");
+        }
+      });
+      rows.push(cells);
+    });
+
+    if (rows.length === 0) return;
+
+    // Normalize column count to the widest row
+    const colCount = Math.max(...rows.map((r) => r.length));
+    const padded = rows.map((r) => {
+      while (r.length < colCount) r.push("");
+      return r;
+    });
+
+    // First row as headers, rest as data
+    const [headers, ...dataRows] = padded;
+    if (!headers) return;
+
+    const headerLine = `| ${headers.join(" | ")} |`;
+    const separator = `| ${headers.map(() => "---").join(" | ")} |`;
+    const bodyLines = dataRows.map((r) => `| ${r.join(" | ")} |`);
+
+    const table = [headerLine, separator, ...bodyLines].join("\n");
+    sections.push(`## ${sheet.name}\n\n${table}`);
+  });
+
+  return sections.join("\n\n");
+};

package/src/html-to-markdown.ts

@@ -0,0 +1,22 @@
+/// <reference path="./turndown-plugin-gfm.d.ts" />
+import { gfm } from "@joplin/turndown-plugin-gfm";
+import TurndownService from "turndown";
+
+/**
+ * Normalizes HTML into markdown so downstream consumers get a uniform
+ * text format regardless of whether the source was HTML, DOCX (via
+ * mammoth's HTML output), or any other HTML-producing extractor.
+ * Uses Turndown with GFM tables and strikethrough support.
+ */
+export function htmlToMarkdown(html: string): string {
+  const turndownService = new TurndownService({
+    headingStyle: "atx",
+    hr: "---",
+    bulletListMarker: "*",
+    codeBlockStyle: "fenced",
+    emDelimiter: "*",
+    strongDelimiter: "**",
+  });
+  turndownService.use(gfm);
+  return turndownService.turndown(html);
+}

package/src/index.ts

@@ -0,0 +1,7 @@
+export * from "./collect-bytes.js";
+export * from "./extractor-registry.js";
+export * from "./extractors/index.js";
+export * from "./html-to-markdown.js";
+export * from "./markdown-to-html.js";
+export * from "./mime-utils.js";
+export * from "./types.js";

package/src/markdown-to-html.ts

@@ -0,0 +1,21 @@
+import MarkdownIt from "markdown-it";
+
+const md = new MarkdownIt({
+  html: true,
+  linkify: true,
+  typographer: true,
+  breaks: false,
+});
+
+/**
+ * Provides the reverse of `htmlToMarkdown` for cases where extracted
+ * markdown needs to be rendered back to HTML (e.g. previews, email
+ * output). Configured with pass-through raw HTML and auto-linking
+ * so embedded markup and bare URLs survive the round-trip.
+ */
+export function markdownToHtml(markdown: string): string {
+  if (!markdown || markdown.trim() === "") {
+    return "";
+  }
+  return md.render(markdown);
+}

package/src/mime-utils.ts

@@ -0,0 +1,25 @@
+const MIME_MAP: Record<string, string> = {
+  ".md": "text/markdown",
+  ".markdown": "text/markdown",
+  ".txt": "text/plain",
+  ".html": "text/html",
+  ".htm": "text/html",
+  ".json": "application/json",
+  ".pdf": "application/pdf",
+  ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+  ".docm": "application/vnd.ms-word.document.macroEnabled.12",
+  ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+};
+
+/**
+ * Lightweight alternative to a full MIME-sniffing library -- covers only
+ * the extensions this package can actually extract, keeping the dependency
+ * footprint small. Used by callers who have a filename but no
+ * Content-Type header and need a MIME type for the registry fallback path.
+ */
+export function getMimeType(path: string): string | undefined {
+  const dotIndex = path.lastIndexOf(".");
+  if (dotIndex === -1) return undefined;
+  const ext = path.slice(dotIndex).toLowerCase();
+  return MIME_MAP[ext];
+}

package/src/turndown-plugin-gfm.d.ts

@@ -0,0 +1,7 @@
+declare module "@joplin/turndown-plugin-gfm" {
+  import type TurndownService from "turndown";
+  export function gfm(service: TurndownService): void;
+  export function tables(service: TurndownService): void;
+  export function strikethrough(service: TurndownService): void;
+  export function taskListItems(service: TurndownService): void;
+}

package/src/types.ts

@@ -0,0 +1,24 @@
+/**
+ * The lowest-level extraction contract: raw bytes in, text out.
+ * Individual format handlers (PDF, DOCX, HTML, etc.) implement this type
+ * so the registry can treat every file format uniformly without knowing
+ * the parsing details.
+ */
+export type ContentExtractor = (
+  content: AsyncIterable<Uint8Array> | Iterable<Uint8Array>,
+) => Promise<string | unknown>;
+
+/**
+ * Decouples post-processing from extraction so callers stay free of
+ * LLM or AI dependencies. Consumers can inject an LLM-backed normalizer
+ * to clean up, restructure, or enrich extracted markdown without the
+ * core package ever knowing about the LLM.
+ */
+export interface ContentNormalizer {
+  /**
+   * Refines raw extracted markdown -- fixing formatting artifacts, improving
+   * structure, or enriching content. The optional context hint lets the
+   * normalizer tailor its output (e.g. summarize for a specific audience).
+   */
+  normalize(params: { markdown: string; context?: string }): Promise<{ markdown: string }>;
+}