@happyvertical/documents 0.80.0 → 0.80.1

package/dist/cli/claude-context.js

@@ -1,21 +1,21 @@
 #!/usr/bin/env node
-import { existsSync, mkdirSync, copyFileSync } from "node:fs";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
-const Dirname = dirname(fileURLToPath(import.meta.url));
-const pkgRoot = join(Dirname, "../..");
-const targetDir = join(process.cwd(), ".claude");
-if (!existsSync(targetDir)) {
-  mkdirSync(targetDir, { recursive: true });
-}
-const pkgName = "documents";
-const agentMdSrc = existsSync(join(pkgRoot, "AGENT.md")) ? join(pkgRoot, "AGENT.md") : join(pkgRoot, "CLAUDE.md");
-const metaSrc = existsSync(join(pkgRoot, "metadata.json")) ? join(pkgRoot, "metadata.json") : join(pkgRoot, ".claude-meta.json");
-if (existsSync(agentMdSrc)) {
-  copyFileSync(agentMdSrc, join(targetDir, `have-${pkgName}.md`));
-}
-if (existsSync(metaSrc)) {
-  copyFileSync(metaSrc, join(targetDir, `have-${pkgName}.meta.json`));
-}
+import { copyFileSync, existsSync, mkdirSync } from "node:fs";
+//#region src/cli/claude-context.ts
+/**
+* CLI script to install agent context for @happyvertical/documents
+* Run the published context installer binary for this package.
+*/
+var pkgRoot = join(dirname(fileURLToPath(import.meta.url)), "../..");
+var targetDir = join(process.cwd(), ".claude");
+if (!existsSync(targetDir)) mkdirSync(targetDir, { recursive: true });
+var pkgName = "documents";
+var agentMdSrc = existsSync(join(pkgRoot, "AGENT.md")) ? join(pkgRoot, "AGENT.md") : join(pkgRoot, "CLAUDE.md");
+var metaSrc = existsSync(join(pkgRoot, "metadata.json")) ? join(pkgRoot, "metadata.json") : join(pkgRoot, ".claude-meta.json");
+if (existsSync(agentMdSrc)) copyFileSync(agentMdSrc, join(targetDir, `have-${pkgName}.md`));
+if (existsSync(metaSrc)) copyFileSync(metaSrc, join(targetDir, `have-${pkgName}.meta.json`));
 console.log(`✓ Installed @happyvertical/${pkgName} context to .claude/`);
-//# sourceMappingURL=claude-context.js.map
+//#endregion
+
+//# sourceMappingURL=claude-context.js.map

package/dist/cli/claude-context.js.map

@@ -1 +1 @@
-{"version":3,"file":"claude-context.js","sources":["../../src/cli/claude-context.ts"],"sourcesContent":["#!/usr/bin/env node\n/**\n * CLI script to install agent context for @happyvertical/documents\n * Run the published context installer binary for this package.\n */\nimport { copyFileSync, existsSync, mkdirSync } from 'node:fs';\nimport { dirname, join } from 'node:path';\nimport { fileURLToPath } from 'node:url';\n\nconst Dirname = dirname(fileURLToPath(import.meta.url));\nconst pkgRoot = join(Dirname, '../..');\nconst targetDir = join(process.cwd(), '.claude');\n\nif (!existsSync(targetDir)) {\n  mkdirSync(targetDir, { recursive: true });\n}\n\nconst pkgName = 'documents';\nconst agentMdSrc = existsSync(join(pkgRoot, 'AGENT.md'))\n  ? join(pkgRoot, 'AGENT.md')\n  : join(pkgRoot, 'CLAUDE.md');\nconst metaSrc = existsSync(join(pkgRoot, 'metadata.json'))\n  ? join(pkgRoot, 'metadata.json')\n  : join(pkgRoot, '.claude-meta.json');\n\nif (existsSync(agentMdSrc)) {\n  copyFileSync(agentMdSrc, join(targetDir, `have-${pkgName}.md`));\n}\n\nif (existsSync(metaSrc)) {\n  copyFileSync(metaSrc, join(targetDir, `have-${pkgName}.meta.json`));\n}\n\nconsole.log(`✓ Installed @happyvertical/${pkgName} context to .claude/`);\n"],"names":[],"mappings":";;;;AASA,MAAM,UAAU,QAAQ,cAAc,YAAY,GAAG,CAAC;AACtD,MAAM,UAAU,KAAK,SAAS,OAAO;AACrC,MAAM,YAAY,KAAK,QAAQ,IAAA,GAAO,SAAS;AAE/C,IAAI,CAAC,WAAW,SAAS,GAAG;AAC1B,YAAU,WAAW,EAAE,WAAW,KAAA,CAAM;AAC1C;AAEA,MAAM,UAAU;AAChB,MAAM,aAAa,WAAW,KAAK,SAAS,UAAU,CAAC,IACnD,KAAK,SAAS,UAAU,IACxB,KAAK,SAAS,WAAW;AAC7B,MAAM,UAAU,WAAW,KAAK,SAAS,eAAe,CAAC,IACrD,KAAK,SAAS,eAAe,IAC7B,KAAK,SAAS,mBAAmB;AAErC,IAAI,WAAW,UAAU,GAAG;AAC1B,eAAa,YAAY,KAAK,WAAW,QAAQ,OAAO,KAAK,CAAC;AAChE;AAEA,IAAI,WAAW,OAAO,GAAG;AACvB,eAAa,SAAS,KAAK,WAAW,QAAQ,OAAO,YAAY,CAAC;AACpE;AAEA,QAAQ,IAAI,8BAA8B,OAAO,sBAAsB;"}
+{"version":3,"file":"claude-context.js","names":[],"sources":["../../src/cli/claude-context.ts"],"sourcesContent":["#!/usr/bin/env node\n/**\n * CLI script to install agent context for @happyvertical/documents\n * Run the published context installer binary for this package.\n */\nimport { copyFileSync, existsSync, mkdirSync } from 'node:fs';\nimport { dirname, join } from 'node:path';\nimport { fileURLToPath } from 'node:url';\n\nconst Dirname = dirname(fileURLToPath(import.meta.url));\nconst pkgRoot = join(Dirname, '../..');\nconst targetDir = join(process.cwd(), '.claude');\n\nif (!existsSync(targetDir)) {\n  mkdirSync(targetDir, { recursive: true });\n}\n\nconst pkgName = 'documents';\nconst agentMdSrc = existsSync(join(pkgRoot, 'AGENT.md'))\n  ? join(pkgRoot, 'AGENT.md')\n  : join(pkgRoot, 'CLAUDE.md');\nconst metaSrc = existsSync(join(pkgRoot, 'metadata.json'))\n  ? join(pkgRoot, 'metadata.json')\n  : join(pkgRoot, '.claude-meta.json');\n\nif (existsSync(agentMdSrc)) {\n  copyFileSync(agentMdSrc, join(targetDir, `have-${pkgName}.md`));\n}\n\nif (existsSync(metaSrc)) {\n  copyFileSync(metaSrc, join(targetDir, `have-${pkgName}.meta.json`));\n}\n\nconsole.log(`✓ Installed @happyvertical/${pkgName} context to .claude/`);\n"],"mappings":";;;;;;;;;AAUA,IAAM,UAAU,KADA,QAAQ,cAAc,OAAO,KAAK,GAAG,CAChC,GAAS,OAAO;AACrC,IAAM,YAAY,KAAK,QAAQ,IAAI,GAAG,SAAS;AAE/C,IAAI,CAAC,WAAW,SAAS,GACvB,UAAU,WAAW,EAAE,WAAW,KAAK,CAAC;AAG1C,IAAM,UAAU;AAChB,IAAM,aAAa,WAAW,KAAK,SAAS,UAAU,CAAC,IACnD,KAAK,SAAS,UAAU,IACxB,KAAK,SAAS,WAAW;AAC7B,IAAM,UAAU,WAAW,KAAK,SAAS,eAAe,CAAC,IACrD,KAAK,SAAS,eAAe,IAC7B,KAAK,SAAS,mBAAmB;AAErC,IAAI,WAAW,UAAU,GACvB,aAAa,YAAY,KAAK,WAAW,QAAQ,QAAQ,IAAI,CAAC;AAGhE,IAAI,WAAW,OAAO,GACpB,aAAa,SAAS,KAAK,WAAW,QAAQ,QAAQ,WAAW,CAAC;AAGpE,QAAQ,IAAI,8BAA8B,QAAQ,qBAAqB"}

package/dist/index.js

@@ -1,322 +1,358 @@
 import os from "node:os";
 import path from "node:path";
 import { URL as URL$1 } from "node:url";
-import { getMimeType, downloadFileWithCache, getCached, setCached } from "@happyvertical/files";
+import { downloadFileWithCache, getCached, getMimeType, setCached } from "@happyvertical/files";
 import { makeSlug } from "@happyvertical/utils";
 import { scrapeDocument } from "@happyvertical/spider";
 import { promises } from "node:fs";
 import { getPDFReader } from "@happyvertical/pdf";
 import { v4 } from "uuid";
-class Document {
-  /**
-   * Flag indicating if document is from a remote source
-   */
-  isRemote = false;
-  /**
-   * Configuration options
-   */
-  options;
-  /**
-   * Local file path where document is stored
-   */
-  _localPath = "";
-  /**
-   * Directory used for caching files
-   */
-  _cacheDir = "";
-  /**
-   * Document URL
-   */
-  url;
-  /**
-   * Document MIME type
-   */
-  type;
-  /**
-   * Document parts (hierarchical structure)
-   */
-  parts = [];
-  /**
-   * Document-level metadata
-   */
-  metadata = {};
-  /**
-   * Get the local file path where document is stored
-   */
-  get localPath() {
-    return this._localPath;
-  }
-  /**
-   * Get the directory used for caching files
-   */
-  get cacheDir() {
-    return this._cacheDir;
-  }
-  /**
-   * Creates a new Document instance
-   *
-   * @param url - Document URL or file path
-   * @param options - Document configuration options
-   */
-  constructor(url, options = {}) {
-    this.url = new URL$1(url);
-    this.options = options;
-    this.type = options.type || getMimeType(this.url.toString()) || "text/plain";
-    this._cacheDir = options.cacheDir || path.resolve(os.tmpdir(), ".cache", "have-sdk", "documents");
-    if (this.url.protocol.startsWith("file")) {
-      this._localPath = decodeURIComponent(this.url.pathname);
-      this.isRemote = false;
-    } else if (this.url.protocol.startsWith("http")) {
-      let pathname = this.url.pathname;
-      if (pathname.endsWith("/")) {
-        pathname = pathname.slice(0, -1);
-      }
-      if (!pathname.match(/\.[a-z0-9]+$/i)) {
-        if (this.type === "application/pdf" || options.type === "application/pdf") {
-          pathname += ".pdf";
-        }
-      }
-      this._localPath = path.join(
-        this._cacheDir,
-        makeSlug(this.url.hostname),
-        pathname
-      );
-      this.isRemote = true;
-    }
-  }
-  /**
-   * Creates and initializes a Document instance
-   *
-   * Downloads remote files and prepares the document for processing.
-   *
-   * @param url - Document URL or file path
-   * @param options - Document configuration options
-   * @returns Promise resolving to the initialized Document
-   */
-  static async create(url, options = {}) {
-    const document = new Document(url, options);
-    await document.initialize();
-    return document;
-  }
-  /**
-   * Initializes the document, downloading it if it's remote
-   *
-   * @returns Promise that resolves when initialization is complete
-   */
-  async initialize() {
-    if (this.isRemote) {
-      if (!this.url) {
-        throw new Error("Cannot initialize remote document: URL is required");
-      }
-      await downloadFileWithCache(this.url.toString(), this._localPath);
-    }
-  }
-  /**
-   * Checks if the document is a text-based file that can be read directly
-   *
-   * @returns Boolean indicating if the file is text-based
-   */
-  isTextFile() {
-    if (!this.type) return false;
-    return this.type.startsWith("text/") || this.type === "application/json" || this.type === "application/xml" || this.type === "application/javascript" || this.type === "application/typescript" || [
-      ".txt",
-      ".md",
-      ".json",
-      ".xml",
-      ".html",
-      ".css",
-      ".js",
-      ".ts",
-      ".yaml",
-      ".yml"
-    ].some((ext) => this.localPath.toLowerCase().endsWith(ext));
-  }
-  /**
-   * Converts the document to the standard Document interface
-   *
-   * @returns Document object with URL, type, parts, and metadata
-   */
-  toDocument() {
-    return {
-      url: this.url.toString(),
-      type: this.type,
-      parts: this.parts,
-      metadata: this.metadata
-    };
-  }
-}
+//#region src/document.ts
+/**
+* Base document handler with multi-part support
+*
+* Provides functionality for downloading, caching, and structuring documents
+* into hierarchical parts. Specific format processing (PDF, HTML, Markdown)
+* is handled by specialized processors.
+*/
+var Document = class Document {
+	/**
+	* Flag indicating if document is from a remote source
+	*/
+	isRemote = false;
+	/**
+	* Configuration options
+	*/
+	options;
+	/**
+	* Local file path where document is stored
+	*/
+	_localPath = "";
+	/**
+	* Directory used for caching files
+	*/
+	_cacheDir = "";
+	/**
+	* Document URL
+	*/
+	url;
+	/**
+	* Document MIME type
+	*/
+	type;
+	/**
+	* Document parts (hierarchical structure)
+	*/
+	parts = [];
+	/**
+	* Document-level metadata
+	*/
+	metadata = {};
+	/**
+	* Get the local file path where document is stored
+	*/
+	get localPath() {
+		return this._localPath;
+	}
+	/**
+	* Get the directory used for caching files
+	*/
+	get cacheDir() {
+		return this._cacheDir;
+	}
+	/**
+	* Creates a new Document instance
+	*
+	* @param url - Document URL or file path
+	* @param options - Document configuration options
+	*/
+	constructor(url, options = {}) {
+		this.url = new URL$1(url);
+		this.options = options;
+		this.type = options.type || getMimeType(this.url.toString()) || "text/plain";
+		this._cacheDir = options.cacheDir || path.resolve(os.tmpdir(), ".cache", "have-sdk", "documents");
+		if (this.url.protocol.startsWith("file")) {
+			this._localPath = decodeURIComponent(this.url.pathname);
+			this.isRemote = false;
+		} else if (this.url.protocol.startsWith("http")) {
+			let pathname = this.url.pathname;
+			if (pathname.endsWith("/")) pathname = pathname.slice(0, -1);
+			if (!pathname.match(/\.[a-z0-9]+$/i)) {
+				if (this.type === "application/pdf" || options.type === "application/pdf") pathname += ".pdf";
+			}
+			this._localPath = path.join(this._cacheDir, makeSlug(this.url.hostname), pathname);
+			this.isRemote = true;
+		}
+	}
+	/**
+	* Creates and initializes a Document instance
+	*
+	* Downloads remote files and prepares the document for processing.
+	*
+	* @param url - Document URL or file path
+	* @param options - Document configuration options
+	* @returns Promise resolving to the initialized Document
+	*/
+	static async create(url, options = {}) {
+		const document = new Document(url, options);
+		await document.initialize();
+		return document;
+	}
+	/**
+	* Initializes the document, downloading it if it's remote
+	*
+	* @returns Promise that resolves when initialization is complete
+	*/
+	async initialize() {
+		if (this.isRemote) {
+			if (!this.url) throw new Error("Cannot initialize remote document: URL is required");
+			await downloadFileWithCache(this.url.toString(), this._localPath);
+		}
+	}
+	/**
+	* Checks if the document is a text-based file that can be read directly
+	*
+	* @returns Boolean indicating if the file is text-based
+	*/
+	isTextFile() {
+		if (!this.type) return false;
+		return this.type.startsWith("text/") || this.type === "application/json" || this.type === "application/xml" || this.type === "application/javascript" || this.type === "application/typescript" || [
+			".txt",
+			".md",
+			".json",
+			".xml",
+			".html",
+			".css",
+			".js",
+			".ts",
+			".yaml",
+			".yml"
+		].some((ext) => this.localPath.toLowerCase().endsWith(ext));
+	}
+	/**
+	* Converts the document to the standard Document interface
+	*
+	* @returns Document object with URL, type, parts, and metadata
+	*/
+	toDocument() {
+		return {
+			url: this.url.toString(),
+			type: this.type,
+			parts: this.parts,
+			metadata: this.metadata
+		};
+	}
+};
+//#endregion
+//#region src/utils.ts
+/**
+* Utility functions for document processing
+*/
+/**
+* Extract a human-readable title from a URL
+*
+* Takes a URL and extracts the filename from the pathname, then formats it
+* into a readable title by removing the extension and converting separators
+* to spaces. Also decodes URL-encoded characters like %20.
+*
+* @param url - URL string to extract title from
+* @param defaultTitle - Default title to use if extraction fails
+* @returns Formatted title string
+*
+* @example
+* ```typescript
+* getTitleFromUrl('file:///path/to/My%20Document.pdf')
+* // Returns: 'My Document'
+*
+* getTitleFromUrl('https://example.com/research_paper.pdf')
+* // Returns: 'research paper'
+* ```
+*/
 function getTitleFromUrl(url, defaultTitle = "Document") {
-  try {
-    const urlObj = new URL(url);
-    const pathname = urlObj.pathname;
-    const filename = pathname.split("/").pop() || defaultTitle;
-    const decodedFilename = decodeURIComponent(filename);
-    return decodedFilename.replace(/\.(pdf|html?|md|txt)$/i, "").replace(/[-_]/g, " ").trim();
-  } catch {
-    return defaultTitle;
-  }
-}
-class PDFProcessor {
-  /**
-   * Check if this processor supports the given MIME type or extension.
-   * Accepts `'application/pdf'`, `'.pdf'`, or `'pdf'` (case-insensitive).
-   *
-   * @param type - MIME type or file extension to check
-   * @returns `true` if this processor can handle the given type
-   */
-  supports(type) {
-    return type === "application/pdf" || type.endsWith(".pdf") || type.toLowerCase() === "pdf";
-  }
-  /**
-   * Process a PDF document
-   *
-   * Extracts text and optionally images/OCR from the PDF, structuring
-   * it into hierarchical document parts.
-   *
-   * @param url - PDF URL or file path
-   * @param options - Processing options
-   * @returns Promise resolving to structured Document
-   */
-  async process(url, options = {}) {
-    const baseDoc = await Document.create(url, options);
-    const cacheKey = `${baseDoc.localPath}.processed_pdf`;
-    const cached = await getCached(cacheKey);
-    if (cached) {
-      try {
-        const parsed = JSON.parse(cached);
-        return {
-          url: baseDoc.url.toString(),
-          type: baseDoc.type,
-          parts: parsed.parts,
-          metadata: parsed.metadata || {}
-        };
-      } catch (error) {
-        console.warn("Cached PDF data corrupted, reprocessing", error);
-      }
-    }
-    const fileBuffer = await promises.readFile(baseDoc.localPath);
-    const header = fileBuffer.subarray(0, 5).toString("utf-8");
-    if (header !== "%PDF-") {
-      try {
-        await promises.unlink(baseDoc.localPath);
-      } catch (unlinkError) {
-        console.warn(
-          `Failed to delete poisoned cache file: ${baseDoc.localPath}`,
-          unlinkError
-        );
-      }
-      const content = fileBuffer.toString(
-        "utf-8",
-        0,
-        Math.min(1e3, fileBuffer.length)
-      );
-      if (content.includes("<!DOCTYPE html>") || content.includes("<html")) {
-        throw new Error(
-          `Downloaded file is HTML, not PDF. The server returned HTML content for ${url}. This commonly occurs with WordPress Download Manager URLs that return tracking pages. Expected PDF magic bytes (%PDF-) but got: ${header}. The poisoned cache file has been removed - please try again.`
-        );
-      } else {
-        throw new Error(
-          `Downloaded file is not a valid PDF. Expected %PDF- magic bytes but got: ${header}. The invalid cache file has been removed - please try again.`
-        );
-      }
-    }
-    const reader = await getPDFReader();
-    const extractedText = await reader.extractText(baseDoc.localPath);
-    const mainPart = {
-      id: v4(),
-      title: getTitleFromUrl(url, "PDF Document"),
-      content: extractedText || "",
-      type: "text",
-      metadata: {
-        source: "pdf",
-        filePath: baseDoc.localPath
-      }
-    };
-    if (options.extractImages === true) {
-      mainPart.images = await this.extractImages(
-        baseDoc.localPath,
-        options.runOcr !== false
-      );
-    }
-    const document = {
-      url: baseDoc.url.toString(),
-      type: baseDoc.type,
-      parts: [mainPart],
-      metadata: {
-        processor: "pdf",
-        extractedAt: (/* @__PURE__ */ new Date()).toISOString(),
-        hasImages: (mainPart.images?.length || 0) > 0
-      }
-    };
-    await setCached(cacheKey, JSON.stringify(document));
-    return document;
-  }
-  /**
-   * Extract images from PDF
-   *
-   * This is a placeholder for future image extraction functionality.
-   * Will use @happyvertical/pdf's image extraction capabilities when available.
-   *
-   * @param filePath - Local PDF file path
-   * @param runOcr - Whether to run OCR on extracted images
-   * @returns Promise resolving to array of DocumentImages
-   */
-  async extractImages(_filePath, _runOcr) {
-    return [];
-  }
+	try {
+		const filename = new URL(url).pathname.split("/").pop() || defaultTitle;
+		return decodeURIComponent(filename).replace(/\.(pdf|html?|md|txt)$/i, "").replace(/[-_]/g, " ").trim();
+	} catch {
+		return defaultTitle;
+	}
 }
-const processors = [new PDFProcessor()];
+//#endregion
+//#region src/processors/pdf.ts
+/**
+* PDF Document Processor
+*
+* Handles PDF documents with support for:
+* - Text extraction from PDF content via `@happyvertical/pdf`
+* - PDF header validation (detects HTML cache poisoning from document management systems)
+* - Processed document caching via `@happyvertical/files`
+*
+* Image extraction and OCR are stubbed for future implementation.
+*/
+var PDFProcessor = class {
+	/**
+	* Check if this processor supports the given MIME type or extension.
+	* Accepts `'application/pdf'`, `'.pdf'`, or `'pdf'` (case-insensitive).
+	*
+	* @param type - MIME type or file extension to check
+	* @returns `true` if this processor can handle the given type
+	*/
+	supports(type) {
+		return type === "application/pdf" || type.endsWith(".pdf") || type.toLowerCase() === "pdf";
+	}
+	/**
+	* Process a PDF document
+	*
+	* Extracts text and optionally images/OCR from the PDF, structuring
+	* it into hierarchical document parts.
+	*
+	* @param url - PDF URL or file path
+	* @param options - Processing options
+	* @returns Promise resolving to structured Document
+	*/
+	async process(url, options = {}) {
+		const baseDoc = await Document.create(url, options);
+		const cacheKey = `${baseDoc.localPath}.processed_pdf`;
+		const cached = await getCached(cacheKey);
+		if (cached) try {
+			const parsed = JSON.parse(cached);
+			return {
+				url: baseDoc.url.toString(),
+				type: baseDoc.type,
+				parts: parsed.parts,
+				metadata: parsed.metadata || {}
+			};
+		} catch (error) {
+			console.warn("Cached PDF data corrupted, reprocessing", error);
+		}
+		const fileBuffer = await promises.readFile(baseDoc.localPath);
+		const header = fileBuffer.subarray(0, 5).toString("utf-8");
+		if (header !== "%PDF-") {
+			try {
+				await promises.unlink(baseDoc.localPath);
+			} catch (unlinkError) {
+				console.warn(`Failed to delete poisoned cache file: ${baseDoc.localPath}`, unlinkError);
+			}
+			const content = fileBuffer.toString("utf-8", 0, Math.min(1e3, fileBuffer.length));
+			if (content.includes("<!DOCTYPE html>") || content.includes("<html")) throw new Error(`Downloaded file is HTML, not PDF. The server returned HTML content for ${url}. This commonly occurs with WordPress Download Manager URLs that return tracking pages. Expected PDF magic bytes (%PDF-) but got: ${header}. The poisoned cache file has been removed - please try again.`);
+			else throw new Error(`Downloaded file is not a valid PDF. Expected %PDF- magic bytes but got: ${header}. The invalid cache file has been removed - please try again.`);
+		}
+		const extractedText = await (await getPDFReader()).extractText(baseDoc.localPath);
+		const mainPart = {
+			id: v4(),
+			title: getTitleFromUrl(url, "PDF Document"),
+			content: extractedText || "",
+			type: "text",
+			metadata: {
+				source: "pdf",
+				filePath: baseDoc.localPath
+			}
+		};
+		if (options.extractImages === true) mainPart.images = await this.extractImages(baseDoc.localPath, options.runOcr !== false);
+		const document = {
+			url: baseDoc.url.toString(),
+			type: baseDoc.type,
+			parts: [mainPart],
+			metadata: {
+				processor: "pdf",
+				extractedAt: (/* @__PURE__ */ new Date()).toISOString(),
+				hasImages: (mainPart.images?.length || 0) > 0
+			}
+		};
+		await setCached(cacheKey, JSON.stringify(document));
+		return document;
+	}
+	/**
+	* Extract images from PDF
+	*
+	* This is a placeholder for future image extraction functionality.
+	* Will use @happyvertical/pdf's image extraction capabilities when available.
+	*
+	* @param filePath - Local PDF file path
+	* @param runOcr - Whether to run OCR on extracted images
+	* @returns Promise resolving to array of DocumentImages
+	*/
+	async extractImages(_filePath, _runOcr) {
+		return [];
+	}
+};
+//#endregion
+//#region src/factory.ts
+/**
+* Available document processors
+*/
+var processors = [new PDFProcessor()];
+/**
+* Fetch a document from a URL with automatic format detection
+*
+* This factory function:
+* 1. Detects the document format (PDF, HTML, Markdown, etc.)
+* 2. Selects the appropriate processor
+* 3. Processes the document into structured parts
+* 4. Returns a Document object with hierarchical content
+*
+* @param url - Document URL or file path (file://, http://, https://)
+* @param options - Fetch and processing options
+* @returns Promise resolving to structured Document
+*
+* @example
+* ```typescript
+* // Fetch a PDF with image extraction and OCR
+* const doc = await fetchDocument('https://example.com/report.pdf', {
+*   extractImages: true,
+*   runOcr: true
+* });
+*
+* // Access document parts
+* for (const part of doc.parts) {
+*   console.log(part.title);
+*   console.log(part.content);
+*
+*   // Check for images
+*   if (part.images) {
+*     for (const image of part.images) {
+*       console.log(image.url);
+*       console.log(image.ocrText); // Text extracted via OCR
+*     }
+*   }
+* }
+* ```
+*/
 async function fetchDocument(url, options = {}) {
-  const isWebUrl = url.startsWith("http://") || url.startsWith("https://");
-  if (isWebUrl && !options.type) {
-    try {
-      const scraper = options.scraper === "crawlee" ? "basic" : options.scraper ?? "basic";
-      const spider = options.spider ?? options.spiderAdapter ?? (options.scraper === "crawlee" ? "crawlee" : "dom");
-      const scraped = await scrapeDocument(url, {
-        scraper,
-        spider,
-        cache: options.cache,
-        cacheExpiry: options.cacheExpiry,
-        headers: options.headers,
-        timeout: options.timeout,
-        maxDuration: options.maxDuration,
-        maxInteractions: options.maxInteractions
-      });
-      const hasDocLink = scraped.metadata.strategy === "wordpress-pdf-link" || scraped.metadata.strategy === "civicweb-pdf-link" || scraped.metadata.strategy === "docushare-pdf-link";
-      if (hasDocLink && scraped.metadata.isPdf && !scraped.metadata.complete) {
-        url = scraped.url;
-        options.type = "application/pdf";
-      }
-    } catch (error) {
-      console.warn(
-        `Spider detection failed for ${url}, falling back to direct download:`,
-        error
-      );
-    }
-  }
-  let type = options.type ?? "";
-  if (!type) {
-    const urlLower = url.toLowerCase();
-    if (urlLower.endsWith(".pdf") || urlLower.includes(".pdf?") || urlLower.includes(".pdf#")) {
-      type = "application/pdf";
-    } else {
-      type = getMimeType(url) ?? "";
-    }
-  }
-  const processor = processors.find((p) => p.supports(type));
-  if (!processor) {
-    throw new Error(
-      `No processor available for document type: ${type}. Supported types: PDF (.pdf, application/pdf)`
-    );
-  }
-  return processor.process(url, options);
+	if ((url.startsWith("http://") || url.startsWith("https://")) && !options.type) try {
+		const scraper = options.scraper === "crawlee" ? "basic" : options.scraper ?? "basic";
+		const spider = options.spider ?? options.spiderAdapter ?? (options.scraper === "crawlee" ? "crawlee" : "dom");
+		const scraped = await scrapeDocument(url, {
+			scraper,
+			spider,
+			cache: options.cache,
+			cacheExpiry: options.cacheExpiry,
+			headers: options.headers,
+			timeout: options.timeout,
+			maxDuration: options.maxDuration,
+			maxInteractions: options.maxInteractions
+		});
+		if ((scraped.metadata.strategy === "wordpress-pdf-link" || scraped.metadata.strategy === "civicweb-pdf-link" || scraped.metadata.strategy === "docushare-pdf-link") && scraped.metadata.isPdf && !scraped.metadata.complete) {
+			url = scraped.url;
+			options.type = "application/pdf";
+		}
+	} catch (error) {
+		console.warn(`Spider detection failed for ${url}, falling back to direct download:`, error);
+	}
+	let type = options.type ?? "";
+	if (!type) {
+		const urlLower = url.toLowerCase();
+		if (urlLower.endsWith(".pdf") || urlLower.includes(".pdf?") || urlLower.includes(".pdf#")) type = "application/pdf";
+		else type = getMimeType(url) ?? "";
+	}
+	const processor = processors.find((p) => p.supports(type));
+	if (!processor) throw new Error(`No processor available for document type: ${type}. Supported types: PDF (.pdf, application/pdf)`);
+	return processor.process(url, options);
 }
-const PACKAGE_VERSION_INITIALIZED = true;
-export {
-  Document,
-  PACKAGE_VERSION_INITIALIZED,
-  PDFProcessor,
-  fetchDocument,
-  getTitleFromUrl
-};
-//# sourceMappingURL=index.js.map
+//#endregion
+//#region src/index.ts
+/** @internal */
+var PACKAGE_VERSION_INITIALIZED = true;
+//#endregion
+export { Document, PACKAGE_VERSION_INITIALIZED, PDFProcessor, fetchDocument, getTitleFromUrl };
+
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sources":["../src/document.ts","../src/utils.ts","../src/processors/pdf.ts","../src/factory.ts","../src/index.ts"],"sourcesContent":["import os from 'node:os';\nimport path from 'node:path';\nimport { URL } from 'node:url';\nimport { downloadFileWithCache, getMimeType } from '@happyvertical/files';\nimport { makeSlug } from '@happyvertical/utils';\nimport type {\n  DocumentPart,\n  Document as DocumentType,\n  FetchDocumentOptions,\n} from './types';\n\n/**\n * Base document handler with multi-part support\n *\n * Provides functionality for downloading, caching, and structuring documents\n * into hierarchical parts. Specific format processing (PDF, HTML, Markdown)\n * is handled by specialized processors.\n */\nexport class Document {\n  /**\n   * Flag indicating if document is from a remote source\n   */\n  protected isRemote = false;\n\n  /**\n   * Configuration options\n   */\n  protected options: FetchDocumentOptions;\n\n  /**\n   * Local file path where document is stored\n   */\n  private _localPath = '';\n\n  /**\n   * Directory used for caching files\n   */\n  private _cacheDir = '';\n\n  /**\n   * Document URL\n   */\n  public url: URL;\n\n  /**\n   * Document MIME type\n   */\n  public type: string;\n\n  /**\n   * Document parts (hierarchical structure)\n   */\n  public parts: DocumentPart[] = [];\n\n  /**\n   * Document-level metadata\n   */\n  public metadata: Record<string, any> = {};\n\n  /**\n   * Get the local file path where document is stored\n   */\n  public get localPath(): string {\n    return this._localPath;\n  }\n\n  /**\n   * Get the directory used for caching files\n   */\n  public get cacheDir(): string {\n    return this._cacheDir;\n  }\n\n  /**\n   * Creates a new Document instance\n   *\n   * @param url - Document URL or file path\n   * @param options - Document configuration options\n   */\n  constructor(url: string, options: FetchDocumentOptions = {}) {\n    this.url = new URL(url);\n    this.options = options;\n    this.type =\n      options.type || getMimeType(this.url.toString()) || 'text/plain';\n\n    this._cacheDir =\n      options.cacheDir ||\n      path.resolve(os.tmpdir(), '.cache', 'have-sdk', 'documents');\n\n    if (this.url.protocol.startsWith('file')) {\n      // Decode URL-encoded characters in the pathname only (e.g., %20 -> space).\n      // Note: Query parameters and hash fragments are not decoded here.\n      this._localPath = decodeURIComponent(this.url.pathname);\n      this.isRemote = false;\n    } else if (this.url.protocol.startsWith('http')) {\n      // Generate cache path from URL pathname\n      // Query parameters (?) and fragments (#) are automatically excluded from url.pathname\n      let pathname = this.url.pathname;\n\n      // Remove trailing slash (directory-style URLs)\n      if (pathname.endsWith('/')) {\n        pathname = pathname.slice(0, -1);\n      }\n\n      // Add file extension if missing and we know the type\n      // This is crucial for URLs like /download/file/?wpdmdl=123 which have no extension\n      if (!pathname.match(/\\.[a-z0-9]+$/i)) {\n        // Add appropriate extension based on MIME type\n        if (\n          this.type === 'application/pdf' ||\n          options.type === 'application/pdf'\n        ) {\n          pathname += '.pdf';\n        }\n        // Future: Add other common extensions (html, json, etc.)\n      }\n\n      this._localPath = path.join(\n        this._cacheDir,\n        makeSlug(this.url.hostname),\n        pathname,\n      );\n      this.isRemote = true;\n    }\n  }\n\n  /**\n   * Creates and initializes a Document instance\n   *\n   * Downloads remote files and prepares the document for processing.\n   *\n   * @param url - Document URL or file path\n   * @param options - Document configuration options\n   * @returns Promise resolving to the initialized Document\n   */\n  static async create(\n    url: string,\n    options: FetchDocumentOptions = {},\n  ): Promise<Document> {\n    const document = new Document(url, options);\n    await document.initialize();\n    return document;\n  }\n\n  /**\n   * Initializes the document, downloading it if it's remote\n   *\n   * @returns Promise that resolves when initialization is complete\n   */\n  async initialize(): Promise<void> {\n    if (this.isRemote) {\n      if (!this.url) {\n        throw new Error('Cannot initialize remote document: URL is required');\n      }\n      await downloadFileWithCache(this.url.toString(), this._localPath);\n    }\n  }\n\n  /**\n   * Checks if the document is a text-based file that can be read directly\n   *\n   * @returns Boolean indicating if the file is text-based\n   */\n  public isTextFile(): boolean {\n    if (!this.type) return false;\n\n    return (\n      this.type.startsWith('text/') ||\n      this.type === 'application/json' ||\n      this.type === 'application/xml' ||\n      this.type === 'application/javascript' ||\n      this.type === 'application/typescript' ||\n      [\n        '.txt',\n        '.md',\n        '.json',\n        '.xml',\n        '.html',\n        '.css',\n        '.js',\n        '.ts',\n        '.yaml',\n        '.yml',\n      ].some((ext) => this.localPath.toLowerCase().endsWith(ext))\n    );\n  }\n\n  /**\n   * Converts the document to the standard Document interface\n   *\n   * @returns Document object with URL, type, parts, and metadata\n   */\n  public toDocument(): DocumentType {\n    return {\n      url: this.url.toString(),\n      type: this.type,\n      parts: this.parts,\n      metadata: this.metadata,\n    };\n  }\n}\n\nexport default Document;\n","/**\n * Utility functions for document processing\n */\n\n/**\n * Extract a human-readable title from a URL\n *\n * Takes a URL and extracts the filename from the pathname, then formats it\n * into a readable title by removing the extension and converting separators\n * to spaces. Also decodes URL-encoded characters like %20.\n *\n * @param url - URL string to extract title from\n * @param defaultTitle - Default title to use if extraction fails\n * @returns Formatted title string\n *\n * @example\n * ```typescript\n * getTitleFromUrl('file:///path/to/My%20Document.pdf')\n * // Returns: 'My Document'\n *\n * getTitleFromUrl('https://example.com/research_paper.pdf')\n * // Returns: 'research paper'\n * ```\n */\nexport function getTitleFromUrl(\n  url: string,\n  defaultTitle = 'Document',\n): string {\n  try {\n    const urlObj = new URL(url);\n    const pathname = urlObj.pathname;\n    const filename = pathname.split('/').pop() || defaultTitle;\n\n    // Decode URL-encoded characters (e.g., %20 -> space)\n    const decodedFilename = decodeURIComponent(filename);\n\n    // Remove extension and convert separators to spaces\n    return decodedFilename\n      .replace(/\\.(pdf|html?|md|txt)$/i, '')\n      .replace(/[-_]/g, ' ')\n      .trim();\n  } catch {\n    return defaultTitle;\n  }\n}\n","import { promises as fs } from 'node:fs';\nimport { getCached, setCached } from '@happyvertical/files';\nimport { getPDFReader } from '@happyvertical/pdf';\nimport { v4 as uuidv4 } from 'uuid';\nimport { Document as BaseDocument } from '../document';\nimport type {\n  Document,\n  DocumentImage,\n  DocumentPart,\n  DocumentProcessor,\n  FetchDocumentOptions,\n} from '../types';\nimport { getTitleFromUrl } from '../utils';\n\n/**\n * PDF Document Processor\n *\n * Handles PDF documents with support for:\n * - Text extraction from PDF content via `@happyvertical/pdf`\n * - PDF header validation (detects HTML cache poisoning from document management systems)\n * - Processed document caching via `@happyvertical/files`\n *\n * Image extraction and OCR are stubbed for future implementation.\n */\nexport class PDFProcessor implements DocumentProcessor {\n  /**\n   * Check if this processor supports the given MIME type or extension.\n   * Accepts `'application/pdf'`, `'.pdf'`, or `'pdf'` (case-insensitive).\n   *\n   * @param type - MIME type or file extension to check\n   * @returns `true` if this processor can handle the given type\n   */\n  supports(type: string): boolean {\n    return (\n      type === 'application/pdf' ||\n      type.endsWith('.pdf') ||\n      type.toLowerCase() === 'pdf'\n    );\n  }\n\n  /**\n   * Process a PDF document\n   *\n   * Extracts text and optionally images/OCR from the PDF, structuring\n   * it into hierarchical document parts.\n   *\n   * @param url - PDF URL or file path\n   * @param options - Processing options\n   * @returns Promise resolving to structured Document\n   */\n  async process(\n    url: string,\n    options: FetchDocumentOptions = {},\n  ): Promise<Document> {\n    // Create and initialize base document\n    const baseDoc = await BaseDocument.create(url, options);\n\n    // Check cache for processed document\n    const cacheKey = `${baseDoc.localPath}.processed_pdf`;\n    const cached = await getCached(cacheKey);\n    if (cached) {\n      try {\n        const parsed = JSON.parse(cached);\n        return {\n          url: baseDoc.url.toString(),\n          type: baseDoc.type,\n          parts: parsed.parts,\n          metadata: parsed.metadata || {},\n        };\n      } catch (error) {\n        // Cache corrupted, continue with fresh processing\n        console.warn('Cached PDF data corrupted, reprocessing', error);\n      }\n    }\n\n    // Validate that the downloaded file is actually a PDF (issue #460, #463)\n    // WordPress Download Manager and some other servers may return HTML\n    // with Content-Type: application/pdf, causing PDF extraction to fail\n    const fileBuffer = await fs.readFile(baseDoc.localPath);\n    const header = fileBuffer.subarray(0, 5).toString('utf-8');\n\n    if (header !== '%PDF-') {\n      // File is not a valid PDF - delete poisoned cache file (issue #463)\n      try {\n        await fs.unlink(baseDoc.localPath);\n      } catch (unlinkError) {\n        console.warn(\n          `Failed to delete poisoned cache file: ${baseDoc.localPath}`,\n          unlinkError,\n        );\n      }\n\n      // Check if it's HTML to provide helpful error message\n      const content = fileBuffer.toString(\n        'utf-8',\n        0,\n        Math.min(1000, fileBuffer.length),\n      );\n      if (content.includes('<!DOCTYPE html>') || content.includes('<html')) {\n        throw new Error(\n          `Downloaded file is HTML, not PDF. The server returned HTML content for ${url}. ` +\n            'This commonly occurs with WordPress Download Manager URLs that return tracking pages. ' +\n            `Expected PDF magic bytes (%PDF-) but got: ${header}. ` +\n            'The poisoned cache file has been removed - please try again.',\n        );\n      } else {\n        throw new Error(\n          `Downloaded file is not a valid PDF. Expected %PDF- magic bytes but got: ${header}. ` +\n            'The invalid cache file has been removed - please try again.',\n        );\n      }\n    }\n\n    // Get PDF reader and extract content\n    const reader = await getPDFReader();\n    const extractedText = await reader.extractText(baseDoc.localPath);\n\n    // Create main document part\n    const mainPart: DocumentPart = {\n      id: uuidv4(),\n      title: getTitleFromUrl(url, 'PDF Document'),\n      content: extractedText || '',\n      type: 'text',\n      metadata: {\n        source: 'pdf',\n        filePath: baseDoc.localPath,\n      },\n    };\n\n    // Extract images if enabled\n    if (options.extractImages === true) {\n      mainPart.images = await this.extractImages(\n        baseDoc.localPath,\n        options.runOcr !== false,\n      );\n    }\n\n    const document: Document = {\n      url: baseDoc.url.toString(),\n      type: baseDoc.type,\n      parts: [mainPart],\n      metadata: {\n        processor: 'pdf',\n        extractedAt: new Date().toISOString(),\n        hasImages: (mainPart.images?.length || 0) > 0,\n      },\n    };\n\n    // Cache the processed document\n    await setCached(cacheKey, JSON.stringify(document));\n\n    return document;\n  }\n\n  /**\n   * Extract images from PDF\n   *\n   * This is a placeholder for future image extraction functionality.\n   * Will use @happyvertical/pdf's image extraction capabilities when available.\n   *\n   * @param filePath - Local PDF file path\n   * @param runOcr - Whether to run OCR on extracted images\n   * @returns Promise resolving to array of DocumentImages\n   */\n  private async extractImages(\n    _filePath: string,\n    _runOcr: boolean,\n  ): Promise<DocumentImage[]> {\n    // TODO: Implement image extraction using @happyvertical/pdf\n    // For now, return empty array as placeholder\n\n    // Future implementation will:\n    // 1. Use getPDFReader() to extract images from PDF\n    // 2. Save images to cache directory\n    // 3. If runOcr is true, use @happyvertical/ocr to extract text from images\n    // 4. Return array of DocumentImage objects with metadata\n\n    return [];\n  }\n}\n\nexport default PDFProcessor;\n","import { getMimeType } from '@happyvertical/files';\nimport { scrapeDocument } from '@happyvertical/spider';\nimport { PDFProcessor } from './processors/pdf';\nimport type { Document, FetchDocumentOptions } from './types';\n\n/**\n * Available document processors\n */\nconst processors = [new PDFProcessor()];\n\n/**\n * Fetch a document from a URL with automatic format detection\n *\n * This factory function:\n * 1. Detects the document format (PDF, HTML, Markdown, etc.)\n * 2. Selects the appropriate processor\n * 3. Processes the document into structured parts\n * 4. Returns a Document object with hierarchical content\n *\n * @param url - Document URL or file path (file://, http://, https://)\n * @param options - Fetch and processing options\n * @returns Promise resolving to structured Document\n *\n * @example\n * ```typescript\n * // Fetch a PDF with image extraction and OCR\n * const doc = await fetchDocument('https://example.com/report.pdf', {\n *   extractImages: true,\n *   runOcr: true\n * });\n *\n * // Access document parts\n * for (const part of doc.parts) {\n *   console.log(part.title);\n *   console.log(part.content);\n *\n *   // Check for images\n *   if (part.images) {\n *     for (const image of part.images) {\n *       console.log(image.url);\n *       console.log(image.ocrText); // Text extracted via OCR\n *     }\n *   }\n * }\n * ```\n */\nexport async function fetchDocument(\n  url: string,\n  options: FetchDocumentOptions = {},\n): Promise<Document> {\n  // For web URLs (http/https), use spider package to detect special cases\n  // (WordPress Download Manager, CivicWeb, DocuShare, etc.)\n  const isWebUrl = url.startsWith('http://') || url.startsWith('https://');\n\n  if (isWebUrl && !options.type) {\n    try {\n      const scraper =\n        options.scraper === 'crawlee' ? 'basic' : (options.scraper ?? 'basic');\n      const spider =\n        options.spider ??\n        options.spiderAdapter ??\n        (options.scraper === 'crawlee' ? 'crawlee' : 'dom');\n\n      // Use spider to detect WordPress, CivicWeb, DocuShare, and other document management systems\n      const scraped = await scrapeDocument(url, {\n        scraper,\n        spider,\n        cache: options.cache,\n        cacheExpiry: options.cacheExpiry,\n        headers: options.headers,\n        timeout: options.timeout,\n        maxDuration: options.maxDuration,\n        maxInteractions: options.maxInteractions,\n      });\n\n      // Check if spider detected a document management system with PDF link\n      const hasDocLink =\n        scraped.metadata.strategy === 'wordpress-pdf-link' ||\n        scraped.metadata.strategy === 'civicweb-pdf-link' ||\n        scraped.metadata.strategy === 'docushare-pdf-link';\n\n      if (hasDocLink && scraped.metadata.isPdf && !scraped.metadata.complete) {\n        // Spider detected a document management page and extracted the PDF URL\n        // Use the extracted URL for PDF processing\n        url = scraped.url;\n        options.type = 'application/pdf';\n      }\n    } catch (error) {\n      // If spider fails, continue with direct download\n      // This ensures backward compatibility\n      console.warn(\n        `Spider detection failed for ${url}, falling back to direct download:`,\n        error,\n      );\n    }\n  }\n\n  // Determine type - check URL extension first, then MIME type\n  // This handles servers that return incorrect Content-Type headers (e.g., application/octet-stream for PDFs)\n  let type = options.type ?? '';\n\n  if (!type) {\n    // Extract file extension from URL\n    const urlLower = url.toLowerCase();\n\n    // Check for common document extensions in URL\n    if (\n      urlLower.endsWith('.pdf') ||\n      urlLower.includes('.pdf?') ||\n      urlLower.includes('.pdf#')\n    ) {\n      type = 'application/pdf';\n    } else {\n      // Fall back to MIME type detection\n      type = getMimeType(url) ?? '';\n    }\n  }\n\n  // Find appropriate processor\n  const processor = processors.find((p) => p.supports(type));\n\n  if (!processor) {\n    throw new Error(\n      `No processor available for document type: ${type}. Supported types: PDF (.pdf, application/pdf)`,\n    );\n  }\n\n  // Process document\n  return processor.process(url, options);\n}\n\nexport default fetchDocument;\n","/**\n * @happyvertical/documents - Document processing with multi-part structure\n *\n * Provides document processing for PDFs with support for:\n * - Hierarchical document parts\n * - Automatic format detection from URL or MIME type\n * - Document management system detection (WordPress, CivicWeb, DocuShare)\n * - File caching for performance\n *\n * @example\n * ```typescript\n * import { fetchDocument } from '@happyvertical/documents';\n *\n * const doc = await fetchDocument('https://example.com/report.pdf');\n *\n * for (const part of doc.parts) {\n *   console.log(part.title);\n *   console.log(part.content);\n * }\n * ```\n */\n\n// Base classes\nexport { Document } from './document';\n// Main factory function\nexport { fetchDocument } from './factory';\n\n// Processors\nexport { PDFProcessor } from './processors/pdf';\n// Types\nexport type {\n  Document as DocumentType,\n  DocumentImage,\n  DocumentPart,\n  DocumentProcessor,\n  FetchDocumentOptions,\n} from './types';\n// Utilities\nexport { getTitleFromUrl } from './utils';\n\n/** @internal */\nexport const PACKAGE_VERSION_INITIALIZED = true;\n"],"names":["URL","BaseDocument","fs","uuidv4"],"mappings":";;;;;;;;;AAkBO,MAAM,SAAS;AAAA;AAAA;AAAA;AAAA,EAIV,WAAW;AAAA;AAAA;AAAA;AAAA,EAKX;AAAA;AAAA;AAAA;AAAA,EAKF,aAAa;AAAA;AAAA;AAAA;AAAA,EAKb,YAAY;AAAA;AAAA;AAAA;AAAA,EAKb;AAAA;AAAA;AAAA;AAAA,EAKA;AAAA;AAAA;AAAA;AAAA,EAKA,QAAwB,CAAA;AAAA;AAAA;AAAA;AAAA,EAKxB,WAAgC,CAAA;AAAA;AAAA;AAAA;AAAA,EAKvC,IAAW,YAAoB;AAC7B,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA,EAKA,IAAW,WAAmB;AAC5B,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,YAAY,KAAa,UAAgC,IAAI;AAC3D,SAAK,MAAM,IAAIA,MAAI,GAAG;AACtB,SAAK,UAAU;AACf,SAAK,OACH,QAAQ,QAAQ,YAAY,KAAK,IAAI,SAAA,CAAU,KAAK;AAEtD,SAAK,YACH,QAAQ,YACR,KAAK,QAAQ,GAAG,OAAA,GAAU,UAAU,YAAY,WAAW;AAE7D,QAAI,KAAK,IAAI,SAAS,WAAW,MAAM,GAAG;AAGxC,WAAK,aAAa,mBAAmB,KAAK,IAAI,QAAQ;AACtD,WAAK,WAAW;AAAA,IAClB,WAAW,KAAK,IAAI,SAAS,WAAW,MAAM,GAAG;AAG/C,UAAI,WAAW,KAAK,IAAI;AAGxB,UAAI,SAAS,SAAS,GAAG,GAAG;AAC1B,mBAAW,SAAS,MAAM,GAAG,EAAE;AAAA,MACjC;AAIA,UAAI,CAAC,SAAS,MAAM,eAAe,GAAG;AAEpC,YACE,KAAK,SAAS,qBACd,QAAQ,SAAS,mBACjB;AACA,sBAAY;AAAA,QACd;AAAA,MAEF;AAEA,WAAK,aAAa,KAAK;AAAA,QACrB,KAAK;AAAA,QACL,SAAS,KAAK,IAAI,QAAQ;AAAA,QAC1B;AAAA,MAAA;AAEF,WAAK,WAAW;AAAA,IAClB;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWA,aAAa,OACX,KACA,UAAgC,IACb;AACnB,UAAM,WAAW,IAAI,SAAS,KAAK,OAAO;AAC1C,UAAM,SAAS,WAAA;AACf,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,aAA4B;AAChC,QAAI,KAAK,UAAU;AACjB,UAAI,CAAC,KAAK,KAAK;AACb,cAAM,IAAI,MAAM,oDAAoD;AAAA,MACtE;AACA,YAAM,sBAAsB,KAAK,IAAI,SAAA,GAAY,KAAK,UAAU;AAAA,IAClE;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOO,aAAsB;AAC3B,QAAI,CAAC,KAAK,KAAM,QAAO;AAEvB,WACE,KAAK,KAAK,WAAW,OAAO,KAC5B,KAAK,SAAS,sBACd,KAAK,SAAS,qBACd,KAAK,SAAS,4BACd,KAAK,SAAS,4BACd;AAAA,MACE;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA,EACA,KAAK,CAAC,QAAQ,KAAK,UAAU,YAAA,EAAc,SAAS,GAAG,CAAC;AAAA,EAE9D;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOO,aAA2B;AAChC,WAAO;AAAA,MACL,KAAK,KAAK,IAAI,SAAA;AAAA,MACd,MAAM,KAAK;AAAA,MACX,OAAO,KAAK;AAAA,MACZ,UAAU,KAAK;AAAA,IAAA;AAAA,EAEnB;AACF;AChLO,SAAS,gBACd,KACA,eAAe,YACP;AACR,MAAI;AACF,UAAM,SAAS,IAAI,IAAI,GAAG;AAC1B,UAAM,WAAW,OAAO;AACxB,UAAM,WAAW,SAAS,MAAM,GAAG,EAAE,SAAS;AAG9C,UAAM,kBAAkB,mBAAmB,QAAQ;AAGnD,WAAO,gBACJ,QAAQ,0BAA0B,EAAE,EACpC,QAAQ,SAAS,GAAG,EACpB,KAAA;AAAA,EACL,QAAQ;AACN,WAAO;AAAA,EACT;AACF;ACpBO,MAAM,aAA0C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQrD,SAAS,MAAuB;AAC9B,WACE,SAAS,qBACT,KAAK,SAAS,MAAM,KACpB,KAAK,kBAAkB;AAAA,EAE3B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,MAAM,QACJ,KACA,UAAgC,IACb;AAEnB,UAAM,UAAU,MAAMC,SAAa,OAAO,KAAK,OAAO;AAGtD,UAAM,WAAW,GAAG,QAAQ,SAAS;AACrC,UAAM,SAAS,MAAM,UAAU,QAAQ;AACvC,QAAI,QAAQ;AACV,UAAI;AACF,cAAM,SAAS,KAAK,MAAM,MAAM;AAChC,eAAO;AAAA,UACL,KAAK,QAAQ,IAAI,SAAA;AAAA,UACjB,MAAM,QAAQ;AAAA,UACd,OAAO,OAAO;AAAA,UACd,UAAU,OAAO,YAAY,CAAA;AAAA,QAAC;AAAA,MAElC,SAAS,OAAO;AAEd,gBAAQ,KAAK,2CAA2C,KAAK;AAAA,MAC/D;AAAA,IACF;AAKA,UAAM,aAAa,MAAMC,SAAG,SAAS,QAAQ,SAAS;AACtD,UAAM,SAAS,WAAW,SAAS,GAAG,CAAC,EAAE,SAAS,OAAO;AAEzD,QAAI,WAAW,SAAS;AAEtB,UAAI;AACF,cAAMA,SAAG,OAAO,QAAQ,SAAS;AAAA,MACnC,SAAS,aAAa;AACpB,gBAAQ;AAAA,UACN,yCAAyC,QAAQ,SAAS;AAAA,UAC1D;AAAA,QAAA;AAAA,MAEJ;AAGA,YAAM,UAAU,WAAW;AAAA,QACzB;AAAA,QACA;AAAA,QACA,KAAK,IAAI,KAAM,WAAW,MAAM;AAAA,MAAA;AAElC,UAAI,QAAQ,SAAS,iBAAiB,KAAK,QAAQ,SAAS,OAAO,GAAG;AACpE,cAAM,IAAI;AAAA,UACR,0EAA0E,GAAG,qIAE9B,MAAM;AAAA,QAAA;AAAA,MAGzD,OAAO;AACL,cAAM,IAAI;AAAA,UACR,2EAA2E,MAAM;AAAA,QAAA;AAAA,MAGrF;AAAA,IACF;AAGA,UAAM,SAAS,MAAM,aAAA;AACrB,UAAM,gBAAgB,MAAM,OAAO,YAAY,QAAQ,SAAS;AAGhE,UAAM,WAAyB;AAAA,MAC7B,IAAIC,GAAA;AAAA,MACJ,OAAO,gBAAgB,KAAK,cAAc;AAAA,MAC1C,SAAS,iBAAiB;AAAA,MAC1B,MAAM;AAAA,MACN,UAAU;AAAA,QACR,QAAQ;AAAA,QACR,UAAU,QAAQ;AAAA,MAAA;AAAA,IACpB;AAIF,QAAI,QAAQ,kBAAkB,MAAM;AAClC,eAAS,SAAS,MAAM,KAAK;AAAA,QAC3B,QAAQ;AAAA,QACR,QAAQ,WAAW;AAAA,MAAA;AAAA,IAEvB;AAEA,UAAM,WAAqB;AAAA,MACzB,KAAK,QAAQ,IAAI,SAAA;AAAA,MACjB,MAAM,QAAQ;AAAA,MACd,OAAO,CAAC,QAAQ;AAAA,MAChB,UAAU;AAAA,QACR,WAAW;AAAA,QACX,cAAa,oBAAI,KAAA,GAAO,YAAA;AAAA,QACxB,YAAY,SAAS,QAAQ,UAAU,KAAK;AAAA,MAAA;AAAA,IAC9C;AAIF,UAAM,UAAU,UAAU,KAAK,UAAU,QAAQ,CAAC;AAElD,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,MAAc,cACZ,WACA,SAC0B;AAU1B,WAAO,CAAA;AAAA,EACT;AACF;AC3KA,MAAM,aAAa,CAAC,IAAI,cAAc;AAsCtC,eAAsB,cACpB,KACA,UAAgC,IACb;AAGnB,QAAM,WAAW,IAAI,WAAW,SAAS,KAAK,IAAI,WAAW,UAAU;AAEvE,MAAI,YAAY,CAAC,QAAQ,MAAM;AAC7B,QAAI;AACF,YAAM,UACJ,QAAQ,YAAY,YAAY,UAAW,QAAQ,WAAW;AAChE,YAAM,SACJ,QAAQ,UACR,QAAQ,kBACP,QAAQ,YAAY,YAAY,YAAY;AAG/C,YAAM,UAAU,MAAM,eAAe,KAAK;AAAA,QACxC;AAAA,QACA;AAAA,QACA,OAAO,QAAQ;AAAA,QACf,aAAa,QAAQ;AAAA,QACrB,SAAS,QAAQ;AAAA,QACjB,SAAS,QAAQ;AAAA,QACjB,aAAa,QAAQ;AAAA,QACrB,iBAAiB,QAAQ;AAAA,MAAA,CAC1B;AAGD,YAAM,aACJ,QAAQ,SAAS,aAAa,wBAC9B,QAAQ,SAAS,aAAa,uBAC9B,QAAQ,SAAS,aAAa;AAEhC,UAAI,cAAc,QAAQ,SAAS,SAAS,CAAC,QAAQ,SAAS,UAAU;AAGtE,cAAM,QAAQ;AACd,gBAAQ,OAAO;AAAA,MACjB;AAAA,IACF,SAAS,OAAO;AAGd,cAAQ;AAAA,QACN,+BAA+B,GAAG;AAAA,QAClC;AAAA,MAAA;AAAA,IAEJ;AAAA,EACF;AAIA,MAAI,OAAO,QAAQ,QAAQ;AAE3B,MAAI,CAAC,MAAM;AAET,UAAM,WAAW,IAAI,YAAA;AAGrB,QACE,SAAS,SAAS,MAAM,KACxB,SAAS,SAAS,OAAO,KACzB,SAAS,SAAS,OAAO,GACzB;AACA,aAAO;AAAA,IACT,OAAO;AAEL,aAAO,YAAY,GAAG,KAAK;AAAA,IAC7B;AAAA,EACF;AAGA,QAAM,YAAY,WAAW,KAAK,CAAC,MAAM,EAAE,SAAS,IAAI,CAAC;AAEzD,MAAI,CAAC,WAAW;AACd,UAAM,IAAI;AAAA,MACR,6CAA6C,IAAI;AAAA,IAAA;AAAA,EAErD;AAGA,SAAO,UAAU,QAAQ,KAAK,OAAO;AACvC;ACxFO,MAAM,8BAA8B;"}
+{"version":3,"file":"index.js","names":[],"sources":["../src/document.ts","../src/utils.ts","../src/processors/pdf.ts","../src/factory.ts","../src/index.ts"],"sourcesContent":["import os from 'node:os';\nimport path from 'node:path';\nimport { URL } from 'node:url';\nimport { downloadFileWithCache, getMimeType } from '@happyvertical/files';\nimport { makeSlug } from '@happyvertical/utils';\nimport type {\n  DocumentPart,\n  Document as DocumentType,\n  FetchDocumentOptions,\n} from './types';\n\n/**\n * Base document handler with multi-part support\n *\n * Provides functionality for downloading, caching, and structuring documents\n * into hierarchical parts. Specific format processing (PDF, HTML, Markdown)\n * is handled by specialized processors.\n */\nexport class Document {\n  /**\n   * Flag indicating if document is from a remote source\n   */\n  protected isRemote = false;\n\n  /**\n   * Configuration options\n   */\n  protected options: FetchDocumentOptions;\n\n  /**\n   * Local file path where document is stored\n   */\n  private _localPath = '';\n\n  /**\n   * Directory used for caching files\n   */\n  private _cacheDir = '';\n\n  /**\n   * Document URL\n   */\n  public url: URL;\n\n  /**\n   * Document MIME type\n   */\n  public type: string;\n\n  /**\n   * Document parts (hierarchical structure)\n   */\n  public parts: DocumentPart[] = [];\n\n  /**\n   * Document-level metadata\n   */\n  public metadata: Record<string, any> = {};\n\n  /**\n   * Get the local file path where document is stored\n   */\n  public get localPath(): string {\n    return this._localPath;\n  }\n\n  /**\n   * Get the directory used for caching files\n   */\n  public get cacheDir(): string {\n    return this._cacheDir;\n  }\n\n  /**\n   * Creates a new Document instance\n   *\n   * @param url - Document URL or file path\n   * @param options - Document configuration options\n   */\n  constructor(url: string, options: FetchDocumentOptions = {}) {\n    this.url = new URL(url);\n    this.options = options;\n    this.type =\n      options.type || getMimeType(this.url.toString()) || 'text/plain';\n\n    this._cacheDir =\n      options.cacheDir ||\n      path.resolve(os.tmpdir(), '.cache', 'have-sdk', 'documents');\n\n    if (this.url.protocol.startsWith('file')) {\n      // Decode URL-encoded characters in the pathname only (e.g., %20 -> space).\n      // Note: Query parameters and hash fragments are not decoded here.\n      this._localPath = decodeURIComponent(this.url.pathname);\n      this.isRemote = false;\n    } else if (this.url.protocol.startsWith('http')) {\n      // Generate cache path from URL pathname\n      // Query parameters (?) and fragments (#) are automatically excluded from url.pathname\n      let pathname = this.url.pathname;\n\n      // Remove trailing slash (directory-style URLs)\n      if (pathname.endsWith('/')) {\n        pathname = pathname.slice(0, -1);\n      }\n\n      // Add file extension if missing and we know the type\n      // This is crucial for URLs like /download/file/?wpdmdl=123 which have no extension\n      if (!pathname.match(/\\.[a-z0-9]+$/i)) {\n        // Add appropriate extension based on MIME type\n        if (\n          this.type === 'application/pdf' ||\n          options.type === 'application/pdf'\n        ) {\n          pathname += '.pdf';\n        }\n        // Future: Add other common extensions (html, json, etc.)\n      }\n\n      this._localPath = path.join(\n        this._cacheDir,\n        makeSlug(this.url.hostname),\n        pathname,\n      );\n      this.isRemote = true;\n    }\n  }\n\n  /**\n   * Creates and initializes a Document instance\n   *\n   * Downloads remote files and prepares the document for processing.\n   *\n   * @param url - Document URL or file path\n   * @param options - Document configuration options\n   * @returns Promise resolving to the initialized Document\n   */\n  static async create(\n    url: string,\n    options: FetchDocumentOptions = {},\n  ): Promise<Document> {\n    const document = new Document(url, options);\n    await document.initialize();\n    return document;\n  }\n\n  /**\n   * Initializes the document, downloading it if it's remote\n   *\n   * @returns Promise that resolves when initialization is complete\n   */\n  async initialize(): Promise<void> {\n    if (this.isRemote) {\n      if (!this.url) {\n        throw new Error('Cannot initialize remote document: URL is required');\n      }\n      await downloadFileWithCache(this.url.toString(), this._localPath);\n    }\n  }\n\n  /**\n   * Checks if the document is a text-based file that can be read directly\n   *\n   * @returns Boolean indicating if the file is text-based\n   */\n  public isTextFile(): boolean {\n    if (!this.type) return false;\n\n    return (\n      this.type.startsWith('text/') ||\n      this.type === 'application/json' ||\n      this.type === 'application/xml' ||\n      this.type === 'application/javascript' ||\n      this.type === 'application/typescript' ||\n      [\n        '.txt',\n        '.md',\n        '.json',\n        '.xml',\n        '.html',\n        '.css',\n        '.js',\n        '.ts',\n        '.yaml',\n        '.yml',\n      ].some((ext) => this.localPath.toLowerCase().endsWith(ext))\n    );\n  }\n\n  /**\n   * Converts the document to the standard Document interface\n   *\n   * @returns Document object with URL, type, parts, and metadata\n   */\n  public toDocument(): DocumentType {\n    return {\n      url: this.url.toString(),\n      type: this.type,\n      parts: this.parts,\n      metadata: this.metadata,\n    };\n  }\n}\n\nexport default Document;\n","/**\n * Utility functions for document processing\n */\n\n/**\n * Extract a human-readable title from a URL\n *\n * Takes a URL and extracts the filename from the pathname, then formats it\n * into a readable title by removing the extension and converting separators\n * to spaces. Also decodes URL-encoded characters like %20.\n *\n * @param url - URL string to extract title from\n * @param defaultTitle - Default title to use if extraction fails\n * @returns Formatted title string\n *\n * @example\n * ```typescript\n * getTitleFromUrl('file:///path/to/My%20Document.pdf')\n * // Returns: 'My Document'\n *\n * getTitleFromUrl('https://example.com/research_paper.pdf')\n * // Returns: 'research paper'\n * ```\n */\nexport function getTitleFromUrl(\n  url: string,\n  defaultTitle = 'Document',\n): string {\n  try {\n    const urlObj = new URL(url);\n    const pathname = urlObj.pathname;\n    const filename = pathname.split('/').pop() || defaultTitle;\n\n    // Decode URL-encoded characters (e.g., %20 -> space)\n    const decodedFilename = decodeURIComponent(filename);\n\n    // Remove extension and convert separators to spaces\n    return decodedFilename\n      .replace(/\\.(pdf|html?|md|txt)$/i, '')\n      .replace(/[-_]/g, ' ')\n      .trim();\n  } catch {\n    return defaultTitle;\n  }\n}\n","import { promises as fs } from 'node:fs';\nimport { getCached, setCached } from '@happyvertical/files';\nimport { getPDFReader } from '@happyvertical/pdf';\nimport { v4 as uuidv4 } from 'uuid';\nimport { Document as BaseDocument } from '../document';\nimport type {\n  Document,\n  DocumentImage,\n  DocumentPart,\n  DocumentProcessor,\n  FetchDocumentOptions,\n} from '../types';\nimport { getTitleFromUrl } from '../utils';\n\n/**\n * PDF Document Processor\n *\n * Handles PDF documents with support for:\n * - Text extraction from PDF content via `@happyvertical/pdf`\n * - PDF header validation (detects HTML cache poisoning from document management systems)\n * - Processed document caching via `@happyvertical/files`\n *\n * Image extraction and OCR are stubbed for future implementation.\n */\nexport class PDFProcessor implements DocumentProcessor {\n  /**\n   * Check if this processor supports the given MIME type or extension.\n   * Accepts `'application/pdf'`, `'.pdf'`, or `'pdf'` (case-insensitive).\n   *\n   * @param type - MIME type or file extension to check\n   * @returns `true` if this processor can handle the given type\n   */\n  supports(type: string): boolean {\n    return (\n      type === 'application/pdf' ||\n      type.endsWith('.pdf') ||\n      type.toLowerCase() === 'pdf'\n    );\n  }\n\n  /**\n   * Process a PDF document\n   *\n   * Extracts text and optionally images/OCR from the PDF, structuring\n   * it into hierarchical document parts.\n   *\n   * @param url - PDF URL or file path\n   * @param options - Processing options\n   * @returns Promise resolving to structured Document\n   */\n  async process(\n    url: string,\n    options: FetchDocumentOptions = {},\n  ): Promise<Document> {\n    // Create and initialize base document\n    const baseDoc = await BaseDocument.create(url, options);\n\n    // Check cache for processed document\n    const cacheKey = `${baseDoc.localPath}.processed_pdf`;\n    const cached = await getCached(cacheKey);\n    if (cached) {\n      try {\n        const parsed = JSON.parse(cached);\n        return {\n          url: baseDoc.url.toString(),\n          type: baseDoc.type,\n          parts: parsed.parts,\n          metadata: parsed.metadata || {},\n        };\n      } catch (error) {\n        // Cache corrupted, continue with fresh processing\n        console.warn('Cached PDF data corrupted, reprocessing', error);\n      }\n    }\n\n    // Validate that the downloaded file is actually a PDF (issue #460, #463)\n    // WordPress Download Manager and some other servers may return HTML\n    // with Content-Type: application/pdf, causing PDF extraction to fail\n    const fileBuffer = await fs.readFile(baseDoc.localPath);\n    const header = fileBuffer.subarray(0, 5).toString('utf-8');\n\n    if (header !== '%PDF-') {\n      // File is not a valid PDF - delete poisoned cache file (issue #463)\n      try {\n        await fs.unlink(baseDoc.localPath);\n      } catch (unlinkError) {\n        console.warn(\n          `Failed to delete poisoned cache file: ${baseDoc.localPath}`,\n          unlinkError,\n        );\n      }\n\n      // Check if it's HTML to provide helpful error message\n      const content = fileBuffer.toString(\n        'utf-8',\n        0,\n        Math.min(1000, fileBuffer.length),\n      );\n      if (content.includes('<!DOCTYPE html>') || content.includes('<html')) {\n        throw new Error(\n          `Downloaded file is HTML, not PDF. The server returned HTML content for ${url}. ` +\n            'This commonly occurs with WordPress Download Manager URLs that return tracking pages. ' +\n            `Expected PDF magic bytes (%PDF-) but got: ${header}. ` +\n            'The poisoned cache file has been removed - please try again.',\n        );\n      } else {\n        throw new Error(\n          `Downloaded file is not a valid PDF. Expected %PDF- magic bytes but got: ${header}. ` +\n            'The invalid cache file has been removed - please try again.',\n        );\n      }\n    }\n\n    // Get PDF reader and extract content\n    const reader = await getPDFReader();\n    const extractedText = await reader.extractText(baseDoc.localPath);\n\n    // Create main document part\n    const mainPart: DocumentPart = {\n      id: uuidv4(),\n      title: getTitleFromUrl(url, 'PDF Document'),\n      content: extractedText || '',\n      type: 'text',\n      metadata: {\n        source: 'pdf',\n        filePath: baseDoc.localPath,\n      },\n    };\n\n    // Extract images if enabled\n    if (options.extractImages === true) {\n      mainPart.images = await this.extractImages(\n        baseDoc.localPath,\n        options.runOcr !== false,\n      );\n    }\n\n    const document: Document = {\n      url: baseDoc.url.toString(),\n      type: baseDoc.type,\n      parts: [mainPart],\n      metadata: {\n        processor: 'pdf',\n        extractedAt: new Date().toISOString(),\n        hasImages: (mainPart.images?.length || 0) > 0,\n      },\n    };\n\n    // Cache the processed document\n    await setCached(cacheKey, JSON.stringify(document));\n\n    return document;\n  }\n\n  /**\n   * Extract images from PDF\n   *\n   * This is a placeholder for future image extraction functionality.\n   * Will use @happyvertical/pdf's image extraction capabilities when available.\n   *\n   * @param filePath - Local PDF file path\n   * @param runOcr - Whether to run OCR on extracted images\n   * @returns Promise resolving to array of DocumentImages\n   */\n  private async extractImages(\n    _filePath: string,\n    _runOcr: boolean,\n  ): Promise<DocumentImage[]> {\n    // TODO: Implement image extraction using @happyvertical/pdf\n    // For now, return empty array as placeholder\n\n    // Future implementation will:\n    // 1. Use getPDFReader() to extract images from PDF\n    // 2. Save images to cache directory\n    // 3. If runOcr is true, use @happyvertical/ocr to extract text from images\n    // 4. Return array of DocumentImage objects with metadata\n\n    return [];\n  }\n}\n\nexport default PDFProcessor;\n","import { getMimeType } from '@happyvertical/files';\nimport { scrapeDocument } from '@happyvertical/spider';\nimport { PDFProcessor } from './processors/pdf';\nimport type { Document, FetchDocumentOptions } from './types';\n\n/**\n * Available document processors\n */\nconst processors = [new PDFProcessor()];\n\n/**\n * Fetch a document from a URL with automatic format detection\n *\n * This factory function:\n * 1. Detects the document format (PDF, HTML, Markdown, etc.)\n * 2. Selects the appropriate processor\n * 3. Processes the document into structured parts\n * 4. Returns a Document object with hierarchical content\n *\n * @param url - Document URL or file path (file://, http://, https://)\n * @param options - Fetch and processing options\n * @returns Promise resolving to structured Document\n *\n * @example\n * ```typescript\n * // Fetch a PDF with image extraction and OCR\n * const doc = await fetchDocument('https://example.com/report.pdf', {\n *   extractImages: true,\n *   runOcr: true\n * });\n *\n * // Access document parts\n * for (const part of doc.parts) {\n *   console.log(part.title);\n *   console.log(part.content);\n *\n *   // Check for images\n *   if (part.images) {\n *     for (const image of part.images) {\n *       console.log(image.url);\n *       console.log(image.ocrText); // Text extracted via OCR\n *     }\n *   }\n * }\n * ```\n */\nexport async function fetchDocument(\n  url: string,\n  options: FetchDocumentOptions = {},\n): Promise<Document> {\n  // For web URLs (http/https), use spider package to detect special cases\n  // (WordPress Download Manager, CivicWeb, DocuShare, etc.)\n  const isWebUrl = url.startsWith('http://') || url.startsWith('https://');\n\n  if (isWebUrl && !options.type) {\n    try {\n      const scraper =\n        options.scraper === 'crawlee' ? 'basic' : (options.scraper ?? 'basic');\n      const spider =\n        options.spider ??\n        options.spiderAdapter ??\n        (options.scraper === 'crawlee' ? 'crawlee' : 'dom');\n\n      // Use spider to detect WordPress, CivicWeb, DocuShare, and other document management systems\n      const scraped = await scrapeDocument(url, {\n        scraper,\n        spider,\n        cache: options.cache,\n        cacheExpiry: options.cacheExpiry,\n        headers: options.headers,\n        timeout: options.timeout,\n        maxDuration: options.maxDuration,\n        maxInteractions: options.maxInteractions,\n      });\n\n      // Check if spider detected a document management system with PDF link\n      const hasDocLink =\n        scraped.metadata.strategy === 'wordpress-pdf-link' ||\n        scraped.metadata.strategy === 'civicweb-pdf-link' ||\n        scraped.metadata.strategy === 'docushare-pdf-link';\n\n      if (hasDocLink && scraped.metadata.isPdf && !scraped.metadata.complete) {\n        // Spider detected a document management page and extracted the PDF URL\n        // Use the extracted URL for PDF processing\n        url = scraped.url;\n        options.type = 'application/pdf';\n      }\n    } catch (error) {\n      // If spider fails, continue with direct download\n      // This ensures backward compatibility\n      console.warn(\n        `Spider detection failed for ${url}, falling back to direct download:`,\n        error,\n      );\n    }\n  }\n\n  // Determine type - check URL extension first, then MIME type\n  // This handles servers that return incorrect Content-Type headers (e.g., application/octet-stream for PDFs)\n  let type = options.type ?? '';\n\n  if (!type) {\n    // Extract file extension from URL\n    const urlLower = url.toLowerCase();\n\n    // Check for common document extensions in URL\n    if (\n      urlLower.endsWith('.pdf') ||\n      urlLower.includes('.pdf?') ||\n      urlLower.includes('.pdf#')\n    ) {\n      type = 'application/pdf';\n    } else {\n      // Fall back to MIME type detection\n      type = getMimeType(url) ?? '';\n    }\n  }\n\n  // Find appropriate processor\n  const processor = processors.find((p) => p.supports(type));\n\n  if (!processor) {\n    throw new Error(\n      `No processor available for document type: ${type}. Supported types: PDF (.pdf, application/pdf)`,\n    );\n  }\n\n  // Process document\n  return processor.process(url, options);\n}\n\nexport default fetchDocument;\n","/**\n * @happyvertical/documents - Document processing with multi-part structure\n *\n * Provides document processing for PDFs with support for:\n * - Hierarchical document parts\n * - Automatic format detection from URL or MIME type\n * - Document management system detection (WordPress, CivicWeb, DocuShare)\n * - File caching for performance\n *\n * @example\n * ```typescript\n * import { fetchDocument } from '@happyvertical/documents';\n *\n * const doc = await fetchDocument('https://example.com/report.pdf');\n *\n * for (const part of doc.parts) {\n *   console.log(part.title);\n *   console.log(part.content);\n * }\n * ```\n */\n\n// Base classes\nexport { Document } from './document';\n// Main factory function\nexport { fetchDocument } from './factory';\n\n// Processors\nexport { PDFProcessor } from './processors/pdf';\n// Types\nexport type {\n  Document as DocumentType,\n  DocumentImage,\n  DocumentPart,\n  DocumentProcessor,\n  FetchDocumentOptions,\n} from './types';\n// Utilities\nexport { getTitleFromUrl } from './utils';\n\n/** @internal */\nexport const PACKAGE_VERSION_INITIALIZED = true;\n"],"mappings":";;;;;;;;;;;;;;;;;AAkBA,IAAa,WAAb,MAAa,SAAS;;;;CAIpB,WAAqB;;;;CAKrB;;;;CAKA,aAAqB;;;;CAKrB,YAAoB;;;;CAKpB;;;;CAKA;;;;CAKA,QAA+B,CAAC;;;;CAKhC,WAAuC,CAAC;;;;CAKxC,IAAW,YAAoB;EAC7B,OAAO,KAAK;CACd;;;;CAKA,IAAW,WAAmB;EAC5B,OAAO,KAAK;CACd;;;;;;;CAQA,YAAY,KAAa,UAAgC,CAAC,GAAG;EAC3D,KAAK,MAAM,IAAI,MAAI,GAAG;EACtB,KAAK,UAAU;EACf,KAAK,OACH,QAAQ,QAAQ,YAAY,KAAK,IAAI,SAAS,CAAC,KAAK;EAEtD,KAAK,YACH,QAAQ,YACR,KAAK,QAAQ,GAAG,OAAO,GAAG,UAAU,YAAY,WAAW;EAE7D,IAAI,KAAK,IAAI,SAAS,WAAW,MAAM,GAAG;GAGxC,KAAK,aAAa,mBAAmB,KAAK,IAAI,QAAQ;GACtD,KAAK,WAAW;EAClB,OAAO,IAAI,KAAK,IAAI,SAAS,WAAW,MAAM,GAAG;GAG/C,IAAI,WAAW,KAAK,IAAI;GAGxB,IAAI,SAAS,SAAS,GAAG,GACvB,WAAW,SAAS,MAAM,GAAG,EAAE;GAKjC,IAAI,CAAC,SAAS,MAAM,eAAe;QAG/B,KAAK,SAAS,qBACd,QAAQ,SAAS,mBAEjB,YAAY;GAAA;GAKhB,KAAK,aAAa,KAAK,KACrB,KAAK,WACL,SAAS,KAAK,IAAI,QAAQ,GAC1B,QACF;GACA,KAAK,WAAW;EAClB;CACF;;;;;;;;;;CAWA,aAAa,OACX,KACA,UAAgC,CAAC,GACd;EACnB,MAAM,WAAW,IAAI,SAAS,KAAK,OAAO;EAC1C,MAAM,SAAS,WAAW;EAC1B,OAAO;CACT;;;;;;CAOA,MAAM,aAA4B;EAChC,IAAI,KAAK,UAAU;GACjB,IAAI,CAAC,KAAK,KACR,MAAM,IAAI,MAAM,oDAAoD;GAEtE,MAAM,sBAAsB,KAAK,IAAI,SAAS,GAAG,KAAK,UAAU;EAClE;CACF;;;;;;CAOA,aAA6B;EAC3B,IAAI,CAAC,KAAK,MAAM,OAAO;EAEvB,OACE,KAAK,KAAK,WAAW,OAAO,KAC5B,KAAK,SAAS,sBACd,KAAK,SAAS,qBACd,KAAK,SAAS,4BACd,KAAK,SAAS,4BACd;GACE;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;EACF,CAAC,CAAC,MAAM,QAAQ,KAAK,UAAU,YAAY,CAAC,CAAC,SAAS,GAAG,CAAC;CAE9D;;;;;;CAOA,aAAkC;EAChC,OAAO;GACL,KAAK,KAAK,IAAI,SAAS;GACvB,MAAM,KAAK;GACX,OAAO,KAAK;GACZ,UAAU,KAAK;EACjB;CACF;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;AChLA,SAAgB,gBACd,KACA,eAAe,YACP;CACR,IAAI;EAGF,MAAM,WADW,IADE,IAAI,GACN,CAAA,CAAO,SACE,MAAM,GAAG,CAAC,CAAC,IAAI,KAAK;EAM9C,OAHwB,mBAAmB,QAGpC,CAAA,CACJ,QAAQ,0BAA0B,EAAE,CAAC,CACrC,QAAQ,SAAS,GAAG,CAAC,CACrB,KAAK;CACV,QAAQ;EACN,OAAO;CACT;AACF;;;;;;;;;;;;;ACpBA,IAAa,eAAb,MAAuD;;;;;;;;CAQrD,SAAS,MAAuB;EAC9B,OACE,SAAS,qBACT,KAAK,SAAS,MAAM,KACpB,KAAK,YAAY,MAAM;CAE3B;;;;;;;;;;;CAYA,MAAM,QACJ,KACA,UAAgC,CAAC,GACd;EAEnB,MAAM,UAAU,MAAM,SAAa,OAAO,KAAK,OAAO;EAGtD,MAAM,WAAW,GAAG,QAAQ,UAAU;EACtC,MAAM,SAAS,MAAM,UAAU,QAAQ;EACvC,IAAI,QACF,IAAI;GACF,MAAM,SAAS,KAAK,MAAM,MAAM;GAChC,OAAO;IACL,KAAK,QAAQ,IAAI,SAAS;IAC1B,MAAM,QAAQ;IACd,OAAO,OAAO;IACd,UAAU,OAAO,YAAY,CAAC;GAChC;EACF,SAAS,OAAO;GAEd,QAAQ,KAAK,2CAA2C,KAAK;EAC/D;EAMF,MAAM,aAAa,MAAM,SAAG,SAAS,QAAQ,SAAS;EACtD,MAAM,SAAS,WAAW,SAAS,GAAG,CAAC,CAAC,CAAC,SAAS,OAAO;EAEzD,IAAI,WAAW,SAAS;GAEtB,IAAI;IACF,MAAM,SAAG,OAAO,QAAQ,SAAS;GACnC,SAAS,aAAa;IACpB,QAAQ,KACN,yCAAyC,QAAQ,aACjD,WACF;GACF;GAGA,MAAM,UAAU,WAAW,SACzB,SACA,GACA,KAAK,IAAI,KAAM,WAAW,MAAM,CAClC;GACA,IAAI,QAAQ,SAAS,iBAAiB,KAAK,QAAQ,SAAS,OAAO,GACjE,MAAM,IAAI,MACR,0EAA0E,IAAI,oIAE/B,OAAO,+DAExD;QAEA,MAAM,IAAI,MACR,2EAA2E,OAAO,8DAEpF;EAEJ;EAIA,MAAM,gBAAgB,OAAM,MADP,aAAa,EAAA,CACC,YAAY,QAAQ,SAAS;EAGhE,MAAM,WAAyB;GAC7B,IAAI,GAAO;GACX,OAAO,gBAAgB,KAAK,cAAc;GAC1C,SAAS,iBAAiB;GAC1B,MAAM;GACN,UAAU;IACR,QAAQ;IACR,UAAU,QAAQ;GACpB;EACF;EAGA,IAAI,QAAQ,kBAAkB,MAC5B,SAAS,SAAS,MAAM,KAAK,cAC3B,QAAQ,WACR,QAAQ,WAAW,KACrB;EAGF,MAAM,WAAqB;GACzB,KAAK,QAAQ,IAAI,SAAS;GAC1B,MAAM,QAAQ;GACd,OAAO,CAAC,QAAQ;GAChB,UAAU;IACR,WAAW;IACX,8BAAa,IAAI,KAAK,EAAA,CAAE,YAAY;IACpC,YAAY,SAAS,QAAQ,UAAU,KAAK;GAC9C;EACF;EAGA,MAAM,UAAU,UAAU,KAAK,UAAU,QAAQ,CAAC;EAElD,OAAO;CACT;;;;;;;;;;;CAYA,MAAc,cACZ,WACA,SAC0B;EAU1B,OAAO,CAAC;CACV;AACF;;;;;;AC3KA,IAAM,aAAa,CAAC,IAAI,aAAa,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsCtC,eAAsB,cACpB,KACA,UAAgC,CAAC,GACd;CAKnB,KAFiB,IAAI,WAAW,SAAS,KAAK,IAAI,WAAW,UAAU,MAEvD,CAAC,QAAQ,MACvB,IAAI;EACF,MAAM,UACJ,QAAQ,YAAY,YAAY,UAAW,QAAQ,WAAW;EAChE,MAAM,SACJ,QAAQ,UACR,QAAQ,kBACP,QAAQ,YAAY,YAAY,YAAY;EAG/C,MAAM,UAAU,MAAM,eAAe,KAAK;GACxC;GACA;GACA,OAAO,QAAQ;GACf,aAAa,QAAQ;GACrB,SAAS,QAAQ;GACjB,SAAS,QAAQ;GACjB,aAAa,QAAQ;GACrB,iBAAiB,QAAQ;EAC3B,CAAC;EAQD,KAJE,QAAQ,SAAS,aAAa,wBAC9B,QAAQ,SAAS,aAAa,uBAC9B,QAAQ,SAAS,aAAa,yBAEd,QAAQ,SAAS,SAAS,CAAC,QAAQ,SAAS,UAAU;GAGtE,MAAM,QAAQ;GACd,QAAQ,OAAO;EACjB;CACF,SAAS,OAAO;EAGd,QAAQ,KACN,+BAA+B,IAAI,qCACnC,KACF;CACF;CAKF,IAAI,OAAO,QAAQ,QAAQ;CAE3B,IAAI,CAAC,MAAM;EAET,MAAM,WAAW,IAAI,YAAY;EAGjC,IACE,SAAS,SAAS,MAAM,KACxB,SAAS,SAAS,OAAO,KACzB,SAAS,SAAS,OAAO,GAEzB,OAAO;OAGP,OAAO,YAAY,GAAG,KAAK;CAE/B;CAGA,MAAM,YAAY,WAAW,MAAM,MAAM,EAAE,SAAS,IAAI,CAAC;CAEzD,IAAI,CAAC,WACH,MAAM,IAAI,MACR,6CAA6C,KAAK,+CACpD;CAIF,OAAO,UAAU,QAAQ,KAAK,OAAO;AACvC;;;;ACxFA,IAAa,8BAA8B"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@happyvertical/documents",
-  "version": "0.80.0",
+  "version": "0.80.1",
   "description": "Multi-part document processing with support for PDF, HTML, and Markdown",
   "type": "module",
   "main": "dist/index.js",
@@ -40,14 +40,14 @@
     "@happyvertical/pdf": "^0.65",
     "@happyvertical/spider": "^1.1",
     "uuid": "^13.0.0",
-    "@happyvertical/files": "0.80.0",
-    "@happyvertical/utils": "0.80.0"
+    "@happyvertical/files": "0.80.1",
+    "@happyvertical/utils": "0.80.1"
   },
   "devDependencies": {
     "@types/node": "25.0.10",
-    "typescript": "^5.9.3",
-    "vite": "7.3.2",
-    "vitest": "^4.1.5"
+    "typescript": "5.9.3",
+    "vite": "8.1.4",
+    "vitest": "4.1.10"
   },
   "keywords": [
     "documents",