@ryangarber/markitdown-ts 0.1.5 → 0.2.1

package/dist/cli.mjs

@@ -1,18 +1,28 @@
+#!/usr/bin/env node
 import { readFile } from "node:fs/promises";
-import { Command } from "@commander-js/extra-typings";
-import turndownPluginGfm from "@joplin/turndown-plugin-gfm";
-import TurndownService from "turndown";
+import { basename } from "node:path";
+import { argument, commandLine, flag, message, object, option, optional, string, withDefault } from "@optique/core";
+import { url } from "@optique/core/message";
+import { defineProgram } from "@optique/core/program";
+import { path, runSync } from "@optique/run";
 import { parseHTML } from "linkedom";
+import TurndownPluginGfm from "@truto/turndown-plugin-gfm";
+import TurndownService from "turndown";
+import { fileTypeFromBuffer } from "file-type";
+import { Mime } from "mime";
+import otherTypes from "mime/types/other.js";
+import standardTypes from "mime/types/standard.js";
 import mammoth from "mammoth";
 import { extractText, getDocumentProxy, getMeta } from "unpdf";
 import { pptxToHtml } from "@jvmr/pptx-to-html";
 import { DOMParser, Document, Element, Node } from "@xmldom/xmldom";
 import readExcelFile from "read-excel-file/universal";
 import { unzipSync } from "fflate";
-//#region package.json
-var version = "0.1.5";
-//#endregion
-//#region src/bytes.ts
+// package.json
+var version = "0.2.0";
+var bugs = { "url": "https://github.com/ryangarber/markitdown-ts/issues" };
+var author = "Ryan Garber";
+// src/utils/bytes.ts
 async function toUint8Array(source) {
 	if (source instanceof Uint8Array) return source;
 	if (isBlob(source)) return new Uint8Array(await source.arrayBuffer());
@@ -28,12 +38,11 @@ function decodeText(source) {
 function isBlob(value) {
 	return typeof Blob !== "undefined" && value instanceof Blob;
 }
-//#endregion
-//#region src/custom-turndown.ts
-var CustomTurnDown = class {
+// src/utils/turndown.ts
+var Turndown = class {
 	convertSoup(document) {
 		const service = new TurndownService({ headingStyle: "atx" });
-		service.use(turndownPluginGfm.gfm);
+		service.use(TurndownPluginGfm);
 		service.addRule("anchor tags", {
 			filter: ["a"],
 			replacement(content, node) {
@@ -68,48 +77,166 @@ var CustomTurnDown = class {
 				return `![${alt}](${source}${title ? ` "${title}"` : ""})`;
 			}
 		});
-		return service.turndown(document);
+		return service.turndown(serializeNode(document));
 	}
 };
-//#endregion
-//#region src/html-document.ts
-function parseHtmlDocument(html) {
-	return parseHTML(html).document;
+/**
+* Turndown and its GFM plugin rely on DOM APIs (e.g. `HTMLTableElement.rows`, `Element.closest`)
+* that linkedom does not implement, which silently drops content such as tables. Serializing
+* lets Turndown re-parse the markup with its own spec-compliant DOM.
+*/
+function serializeNode(node) {
+	if (typeof node === "string") return node;
+	if (node.nodeType === node.DOCUMENT_NODE) return node.documentElement?.outerHTML ?? "";
+	if (node.nodeType === node.ELEMENT_NODE) return node.outerHTML;
+	if (node.nodeType === node.TEXT_NODE) return escapeHtml$1(node.textContent ?? "");
+	return [...node.childNodes].map(serializeNode).join("");
+}
+function escapeHtml$1(value) {
+	return value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+}
+// src/utils/html.ts
+function parseHtmlDocument(html, transform) {
+	const document = parseHTML(wrapFragment(html)).document;
+	return Object.fromEntries(Object.entries(transform(document)).map(([key, value]) => {
+		if (value.type === "node") return [key, (Array.isArray(value.value) ? value.value : [value.value]).map((node) => new Turndown().convertSoup(node).split(/\n+/).map((line) => line.trim()).filter(Boolean).join("\n\n")).join("\n\n")];
+		return [key, value.value];
+	}));
+}
+/**
+* linkedom only parses complete documents: given a fragment (e.g. mammoth output or an RSS
+* description), it promotes the first element to `documentElement` and drops its siblings
+* from the tree, so wrap fragments in a full document first.
+*/
+function wrapFragment(html) {
+	if (/<html[\s>]/i.test(html)) return html;
+	if (/<(head|body)[\s>]/i.test(html)) return `<html>${html}</html>`;
+	return `<html><head></head><body>${html}</body></html>`;
+}
+// src/utils/mime.ts
+const mime = new Mime(standardTypes, otherTypes);
+function extensionFromMime(contentType) {
+	if (!contentType) return null;
+	return extensionNormalized(mime.getExtension(contentType));
+}
+async function extensionFromBuffer(buffer) {
+	const type = await fileTypeFromBuffer(buffer);
+	if (!type) return null;
+	return extensionNormalized(type.ext);
+}
+/**
+* Sniffs text formats that have no magic number. Returns null for binary content, and `txt`
+* for text that is not recognized as anything more specific.
+*/
+function extensionFromText(buffer) {
+	let text;
+	try {
+		text = new TextDecoder("utf-8", { fatal: true }).decode(buffer);
+	} catch {
+		return null;
+	}
+	if (hasBinaryControlCharacters(text)) return null;
+	const start = text.trimStart().slice(0, 1024).toLowerCase();
+	if (start.startsWith("<")) {
+		if (/<rss[\s>]/.test(start)) return "rss";
+		if (/<feed[\s>]/.test(start)) return "atom";
+		if (/^<!doctype html|<(html|head|body)[\s>]/.test(start)) return "html";
+		if (start.startsWith("<?xml")) return "xml";
+	}
+	if (start.startsWith("{") && /"nbformat"\s*:/.test(text) && /"cells"\s*:/.test(text)) return "ipynb";
+	return "txt";
+}
+function hasBinaryControlCharacters(text) {
+	for (let index = 0; index < text.length; index++) {
+		const code = text.charCodeAt(index);
+		if (code < 32 && (code < 9 || code > 13) && code !== 27) return true;
+	}
+	return false;
+}
+function extensionFromPathname(pathname) {
+	const filename = pathname.split("/").pop() ?? "";
+	const dot = filename.lastIndexOf(".");
+	return dot > 0 ? extensionNormalized(filename.slice(dot)) : null;
+}
+/** Reads the filename from a Content-Disposition header, preferring the RFC 5987 form. */
+function filenameFromDisposition(disposition) {
+	if (!disposition) return null;
+	const extended = disposition.match(/filename\*\s*=\s*[\w-]*'[^']*'([^;]+)/i)?.[1]?.trim();
+	if (extended) try {
+		return decodeURIComponent(extended);
+	} catch {
+		return extended;
+	}
+	const plain = disposition.match(/filename\s*=\s*(?:"([^"]*)"|([^;]+))/i);
+	return (plain?.[1] ?? plain?.[2])?.trim() || null;
+}
+/**
+* Lists candidate extensions for the content, most reliable first: explicit extensions, then
+* names, then declared MIME types, then the content itself.
+*/
+async function detectExtensions(buffer, { extension, filename, urls = [], contentType } = {}) {
+	const candidates = [
+		extensionNormalized(extension),
+		filename ? extensionFromPathname(filename) : null,
+		...urls.map((url) => url ? extensionFromUrl(url) : null),
+		extensionFromMime(contentType),
+		await extensionFromBuffer(buffer) ?? extensionFromText(buffer)
+	];
+	return [...new Set(candidates.filter((candidate) => !!candidate))];
+}
+function extensionFromUrl(url) {
+	try {
+		return extensionFromPathname(decodeURIComponent(new URL(url).pathname));
+	} catch {
+		return null;
+	}
+}
+function extensionNormalized(extension, withDot = false) {
+	if (!extension) return "";
+	const normalized = extension.trim().toLowerCase().replace(/^[.-]+/, "");
+	return withDot ? `.${normalized}` : normalized;
+}
+function isTextExtension(extension) {
+	const type = mime.getType(extension);
+	if (!type) return false;
+	return type.startsWith("text/");
 }
-//#endregion
-//#region src/converters/bingserp.ts
+// src/converters/bingserp.ts
 const BING_SEARCH_URL = /^https:\/\/www\.bing\.com\/search\?q=/i;
 const BASE64_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
 var BingSerpConverter = class {
 	async convert(source, options = {}) {
-		const extension = options.file_extension?.toLowerCase() ?? "";
-		const url = options.url ?? "";
-		if (extension !== ".html" && extension !== ".htm" || !BING_SEARCH_URL.test(url)) return null;
-		const document = parseHtmlDocument(decodeText(source));
-		const query = new URL(url).searchParams.get("q") ?? "";
-		document.querySelectorAll(".tptt").forEach((element) => {
-			if (element.textContent) element.textContent += " ";
-		});
-		document.querySelectorAll(".algoSlug_icon").forEach((element) => {
-			element.remove();
-		});
-		const turndown = new CustomTurnDown();
-		const markdown = `## A Bing search for '${query}' found the following results:\n\n${[...document.querySelectorAll(".b_algo")].map((result) => {
-			result.querySelectorAll("a[href]").forEach((anchor) => {
-				const href = anchor.getAttribute("href");
-				if (!href) return;
-				try {
-					const encoded = new URL(href, url).searchParams.get("u");
-					if (encoded) anchor.setAttribute("href", decodeBingUrl(encoded));
-				} catch {}
+		if (options.extension !== "html" && options.extension !== "htm" || !BING_SEARCH_URL.test(options.url ?? "")) return null;
+		let query = new URL(options.url ?? "").searchParams.get("q") || null;
+		if (query) query = `Search results for "${query}"`;
+		return parseHtmlDocument(decodeText(source), (document) => {
+			document.querySelectorAll(".tptt").forEach((element) => {
+				if (element.textContent) element.textContent += " ";
 			});
-			return turndown.convertSoup(result).split(/\n+/).map((line) => line.trim()).filter(Boolean).join("\n");
-		}).join("\n\n")}`;
-		return {
-			title: document.title || null,
-			markdown,
-			text_content: markdown
-		};
+			document.querySelectorAll(".algoSlug_icon").forEach((element) => {
+				element.remove();
+			});
+			return {
+				title: {
+					type: "metadata",
+					value: query || document.title || null
+				},
+				markdown: {
+					type: "node",
+					value: [...document.querySelectorAll(".b_algo")].map((result) => {
+						result.querySelectorAll("a[href]").forEach((anchor) => {
+							const href = anchor.getAttribute("href");
+							if (!href) return;
+							try {
+								const encoded = new URL(href, options.url ?? "").searchParams.get("u");
+								if (encoded) anchor.setAttribute("href", decodeBingUrl(encoded));
+							} catch {}
+						});
+						return result;
+					})
+				}
+			};
+		});
 	}
 };
 function decodeBingUrl(encoded) {
@@ -129,52 +256,46 @@ function decodeBingUrl(encoded) {
 	}
 	return new TextDecoder().decode(new Uint8Array(bytes));
 }
-//#endregion
-//#region src/converters/html.ts
+// src/converters/html.ts
 var HtmlConverter = class {
 	async convert(source, options) {
-		const extension = options.file_extension?.toLowerCase() ?? "";
-		if (extension !== ".html" && extension !== ".htm") return null;
+		if (options.extension !== "html" && options.extension !== "htm") return null;
 		return this.convertHtml(decodeText(source));
 	}
 	async convertHtml(html) {
-		const document = parseHtmlDocument(html);
-		const title = document.title;
-		document.querySelectorAll("script, style, title").forEach((element) => {
-			element.remove();
+		return parseHtmlDocument(html, (document) => {
+			const title = document.title || null;
+			document.querySelectorAll("script, style, title").forEach((element) => {
+				element.remove();
+			});
+			return {
+				title: {
+					type: "metadata",
+					value: title
+				},
+				markdown: {
+					type: "node",
+					value: document
+				}
+			};
 		});
-		const markdown = new CustomTurnDown().convertSoup(document);
-		return {
-			title: title || null,
-			markdown,
-			text_content: markdown
-		};
 	}
 };
-//#endregion
-//#region src/converters/docx.ts
+// src/converters/docx.ts
 var DocxConverter = class extends HtmlConverter {
 	async convert(source, options) {
-		if (options.file_extension?.toLowerCase() !== ".docx") return null;
+		if (options.extension !== "docx") return null;
 		let mammothInput = { arrayBuffer: toArrayBuffer(source) };
 		if (typeof process !== "undefined" && process.versions !== null && process.versions.node !== null) mammothInput = { buffer: Buffer.from(source) };
-		const mammothOptions = {
-			styleMap: options.styleMap,
-			includeEmbeddedStyleMap: options.includeEmbeddedStyleMap,
-			includeDefaultStyleMap: options.includeDefaultStyleMap,
-			ignoreEmptyParagraphs: options.ignoreEmptyParagraphs,
-			idPrefix: options.idPrefix
-		};
-		if (options.transformDocument) mammothOptions.transformDocument = options.transformDocument;
+		const mammothOptions = Object.fromEntries(Object.entries(options.mammoth ?? {}).flatMap(([key, value]) => value !== void 0 ? [[key, value]] : []));
 		const html = await mammoth.convertToHtml(mammothInput, mammothOptions);
 		return this.convertHtml(html.value);
 	}
 };
-//#endregion
-//#region src/converters/ipynb.ts
+// src/converters/ipynb.ts
 var IpynbConverter = class {
 	async convert(source, options = {}) {
-		if (options.file_extension?.toLowerCase() !== ".ipynb") return null;
+		if (options.extension !== "ipynb") return null;
 		const notebook = JSON.parse(decodeText(source));
 		const output = [];
 		let title = notebook.metadata?.title ?? null;
@@ -189,99 +310,49 @@ var IpynbConverter = class {
 		const markdown = output.join("\n\n");
 		return {
 			title,
-			markdown,
-			text_content: markdown
+			markdown
 		};
 	}
 };
-//#endregion
-//#region src/converters/pdf.ts
+// src/converters/pdf.ts
 var PdfConverter = class {
 	async convert(source, options) {
-		if (options.file_extension?.toLowerCase() !== ".pdf") return null;
+		if (options.extension !== "pdf") return null;
 		const pdf = await getDocumentProxy(source.slice());
 		try {
 			const [{ text }, { info }] = await Promise.all([extractText(pdf, { mergePages: true }), getMeta(pdf)]);
 			return {
 				title: typeof info.Title === "string" && info.Title.trim() ? info.Title : null,
-				markdown: text,
-				text_content: text
+				markdown: text
 			};
 		} finally {
 			await pdf.loadingTask.destroy();
 		}
 	}
 };
-//#endregion
-//#region src/mime.ts
-const MIME_EXTENSIONS = {
-	"application/atom+xml": ".atom",
-	"application/json": ".json",
-	"application/pdf": ".pdf",
-	"application/zip": ".zip",
-	"application/rss+xml": ".rss",
-	"application/x-pdf": ".pdf",
-	"application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx",
-	"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",
-	"application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
-	"application/xml": ".xml",
-	"text/csv": ".csv",
-	"text/html": ".html",
-	"text/markdown": ".md",
-	"text/plain": ".txt",
-	"text/xml": ".xml"
-};
-const TEXT_EXTENSIONS = /* @__PURE__ */ new Set([
-	".csv",
-	".json",
-	".log",
-	".markdown",
-	".md",
-	".text",
-	".txt"
-]);
-function extensionFromMime(contentType) {
-	if (!contentType) return null;
-	const mime = contentType.split(";", 1)[0]?.trim().toLowerCase() ?? "";
-	return MIME_EXTENSIONS[mime] ?? (mime.startsWith("text/") ? ".txt" : null);
-}
-function isTextExtension(extension) {
-	return TEXT_EXTENSIONS.has(extension.toLowerCase());
-}
-function extensionFromPathname(pathname) {
-	const filename = pathname.split("/").pop() ?? "";
-	const dot = filename.lastIndexOf(".");
-	return dot > 0 ? filename.slice(dot).toLowerCase() : null;
-}
-//#endregion
-//#region src/converters/plain-text.ts
+// src/converters/plain-text.ts
 var PlainTextConverter = class {
 	async convert(source, options = {}) {
-		if (!isTextExtension(options.file_extension ?? "")) return null;
-		const content = decodeText(source);
+		if (!isTextExtension(options.extension ?? "")) return null;
 		return {
 			title: null,
-			markdown: content,
-			text_content: content
+			markdown: decodeText(source)
 		};
 	}
 };
-//#endregion
-//#region src/converters/pptx.ts
+// src/converters/pptx.ts
 var PptxConverter = class extends HtmlConverter {
 	async convert(source, options) {
-		if (options.file_extension?.toLowerCase() !== ".pptx") return null;
+		if (options.extension !== "pptx") return null;
 		const slides = await pptxToHtml(toArrayBuffer(source), { domParserFactory: createPptxDomParser });
 		const sections = [];
 		for (const [index, html] of slides.entries()) {
 			const result = await this.convertHtml(`<html><body>${html}</body></html>`);
 			sections.push(`<!-- Slide number: ${index + 1} -->\n${result?.markdown.trim() ?? ""}`);
 		}
-		const markdown = sections.join("\n\n");
 		return {
 			title: null,
-			markdown,
-			text_content: markdown
+			markdown: sections.join("\n\n")
 		};
 	}
 };
@@ -345,32 +416,33 @@ function defineGetter(target, name, get) {
 		get
 	});
 }
-//#endregion
-//#region src/converters/wikipedia.ts
+// src/converters/wikipedia.ts
 const WIKIPEDIA_URL = /^https?:\/\/[a-z]{2,3}\.wikipedia\.org\//i;
 var WikipediaConverter = class {
 	async convert(source, options = {}) {
-		const extension = options.file_extension?.toLowerCase() ?? "";
-		if (extension !== ".html" && extension !== ".htm" || !WIKIPEDIA_URL.test(options.url ?? "")) return null;
-		const document = parseHtmlDocument(decodeText(source));
-		document.querySelectorAll("script, style").forEach((element) => {
-			element.remove();
+		if (options.extension !== "html" && options.extension !== "htm" || !WIKIPEDIA_URL.test(options.url ?? "")) return null;
+		return parseHtmlDocument(decodeText(source), (document) => {
+			document.querySelectorAll("script, style").forEach((element) => {
+				element.remove();
+			});
+			const body = document.querySelector("div#mw-content-text");
+			return {
+				title: {
+					type: "metadata",
+					value: document.querySelector("span.mw-page-title-main")?.textContent?.trim() || document.title || null
+				},
+				markdown: {
+					type: "node",
+					value: body || document
+				}
+			};
 		});
-		const body = document.querySelector("div#mw-content-text");
-		const title = document.querySelector("span.mw-page-title-main")?.textContent?.trim() || document.title || null;
-		const markdown = body ? `# ${title ?? "Wikipedia"}\n\n${new CustomTurnDown().convertSoup(body)}` : new CustomTurnDown().convertSoup(document);
-		return {
-			title,
-			markdown,
-			text_content: markdown
-		};
 	}
 };
-//#endregion
-//#region src/converters/xlsx.ts
+// src/converters/xlsx.ts
 var XlsxConverter = class extends HtmlConverter {
 	async convert(source, options) {
-		if (options.file_extension?.toLowerCase() !== ".xlsx") return null;
+		if (options.extension !== "xlsx") return null;
 		const sheets = await readExcelFile(toArrayBuffer(source));
 		const sections = [];
 		for (const { sheet, data } of sheets) {
@@ -379,11 +451,9 @@ var XlsxConverter = class extends HtmlConverter {
 			const result = await this.convertHtml(html);
 			sections.push(`## ${sheet}\n${result?.markdown.trim() ?? ""}`);
 		}
-		const markdown = sections.join("\n\n");
 		return {
 			title: null,
-			markdown,
-			text_content: markdown
+			markdown: sections.join("\n\n")
 		};
 	}
 };
@@ -404,129 +474,126 @@ function cellText(value) {
 function escapeHtml(value) {
 	return value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
 }
-//#endregion
-//#region src/converters/xml-rss-atom.ts
+// src/converters/xml-rss-atom.ts
 var RSSConverter = class {
 	async convert(source, options = {}) {
-		const extension = options.file_extension?.toLowerCase() ?? "";
-		if (![
-			".xml",
-			".rss",
-			".atom"
-		].includes(extension)) return null;
+		if (options.extension !== "xml" && options.extension !== "rss" && options.extension !== "atom") return null;
 		const document = new DOMParser().parseFromString(decodeText(source), "text/xml");
-		if (document.getElementsByTagName("rss").length > 0) return this.parseRss(document);
-		if (document.getElementsByTagName("feed")[0]?.getElementsByTagName("entry").length) return this.parseAtom(document);
+		if (document.getElementsByTagName("rss").length > 0) return parseRss(document);
+		if (document.getElementsByTagName("feed")[0]?.getElementsByTagName("entry").length) return parseAtom(document);
 		return null;
 	}
-	parseAtom(document) {
-		const root = document.getElementsByTagName("feed")[0];
-		if (!root) return null;
-		const title = getText(root, "title");
-		const subtitle = getText(root, "subtitle");
-		const sections = [`# ${title ?? "Feed"}`];
-		if (subtitle) sections.push(subtitle);
-		for (const entry of [...root.getElementsByTagName("entry")]) {
-			const entryTitle = getText(entry, "title");
-			const updated = getText(entry, "updated");
-			const summary = getText(entry, "summary");
-			const content = getText(entry, "content");
-			const lines = [];
-			if (entryTitle) lines.push(`## ${entryTitle}`);
-			if (updated) lines.push(`Updated on: ${updated}`);
-			if (summary) lines.push(parseContent(summary));
-			if (content) lines.push(parseContent(content));
-			sections.push(lines.join("\n"));
-		}
-		const markdown = sections.join("\n\n");
-		return {
-			title,
-			markdown,
-			text_content: markdown
-		};
+};
+function parseAtom(document) {
+	const root = document.getElementsByTagName("feed")[0];
+	if (!root) return null;
+	const title = getText(root, "title");
+	const subtitle = getText(root, "subtitle");
+	const sections = [`# ${title ?? "Feed"}`];
+	if (subtitle) sections.push(subtitle);
+	for (const entry of [...root.getElementsByTagName("entry")]) {
+		const entryTitle = getText(entry, "title");
+		const updated = getText(entry, "updated");
+		const summary = getText(entry, "summary");
+		const content = getText(entry, "content");
+		const lines = [];
+		if (entryTitle) lines.push(`## ${entryTitle}`);
+		if (updated) lines.push(`Updated on: ${updated}`);
+		const node = summary || content;
+		if (node) lines.push(parseHtmlDocument(node, (document) => ({ markdown: {
+			type: "node",
+			value: document.documentElement
+		} })).markdown);
+		sections.push(lines.join("\n\n"));
 	}
-	parseRss(document) {
-		const channel = document.getElementsByTagName("rss")[0]?.getElementsByTagName("channel")[0];
-		if (!channel) return null;
-		const title = getText(channel, "title");
-		const description = getText(channel, "description");
-		const sections = title ? [`# ${title}`] : [];
-		if (description) sections.push(description);
-		for (const item of [...channel.getElementsByTagName("item")]) {
-			const itemTitle = getText(item, "title");
-			const published = getText(item, "pubDate");
-			const itemDescription = getText(item, "description");
-			const content = getText(item, "content:encoded");
-			const lines = [];
-			if (itemTitle) lines.push(`## ${itemTitle}`);
-			if (published) lines.push(`Published on: ${published}`);
-			if (itemDescription) lines.push(parseContent(itemDescription));
-			if (content) lines.push(parseContent(content));
-			sections.push(lines.join("\n"));
-		}
-		const markdown = sections.join("\n\n");
-		return {
-			title,
-			markdown,
-			text_content: markdown
-		};
+	return {
+		title,
+		markdown: sections.join("\n\n")
+	};
+}
+function parseRss(document) {
+	const channel = document.getElementsByTagName("rss")[0]?.getElementsByTagName("channel")[0];
+	if (!channel) return null;
+	const title = getText(channel, "title");
+	const description = getText(channel, "description");
+	const sections = title ? [`# ${title}`] : [];
+	if (description) sections.push(description);
+	for (const item of [...channel.getElementsByTagName("item")]) {
+		const itemTitle = getText(item, "title");
+		const published = getText(item, "pubDate");
+		const itemDescription = getText(item, "description");
+		const content = getText(item, "content:encoded");
+		const lines = [];
+		if (itemTitle) lines.push(`## ${itemTitle}`);
+		if (published) lines.push(`Published on: ${published}`);
+		const node = itemDescription || content;
+		if (node) lines.push(parseHtmlDocument(node, (document) => ({ markdown: {
+			type: "node",
+			value: document.documentElement
+		} })).markdown);
+		sections.push(lines.join("\n\n"));
 	}
-};
-function parseContent(content) {
-	return new CustomTurnDown().convertSoup(parseHtmlDocument(content).documentElement);
+	return {
+		title,
+		markdown: sections.join("\n\n")
+	};
 }
 function getText(element, tagName) {
 	return element.getElementsByTagName(tagName)[0]?.textContent?.trim() || null;
 }
-//#endregion
-//#region src/converters/youtube.ts
+// src/converters/youtube.ts
+const YOUTUBE_URL = /^https?:\/\/(?:www\.)?(?:youtube\.com|youtu\.be)\/(?:watch\?v=|shorts\/)?([a-zA-Z0-9_-]+)/i;
 var YouTubeConverter = class {
 	async convert(source, options = {}) {
-		const extension = options.file_extension?.toLowerCase() ?? "";
-		if (extension !== ".html" && extension !== ".htm" || !options.url?.startsWith("https://www.youtube.com/watch?")) return null;
-		const document = parseHtmlDocument(decodeText(source));
-		const metadata = { title: document.title };
-		document.querySelectorAll("meta").forEach((meta) => {
-			const content = meta.getAttribute("content");
-			const key = meta.getAttribute("itemprop") ?? meta.getAttribute("property") ?? meta.getAttribute("name");
-			if (key && content) metadata[key] = content;
-		});
-		try {
-			for (const script of document.querySelectorAll("script")) {
-				const content = script.textContent ?? "";
-				if (!content.includes("ytInitialData")) continue;
-				const start = content.indexOf("{");
-				const end = content.lastIndexOf("}");
-				if (start >= 0 && end > start) {
-					const description = findKey(JSON.parse(content.slice(start, end + 1)), "attributedDescriptionBodyText");
-					if (isRecord(description) && typeof description.content === "string") metadata.description = description.content;
+		if (options.extension !== "html" && options.extension !== "htm" || !YOUTUBE_URL.test(options.url ?? "")) return null;
+		return parseHtmlDocument(decodeText(source), (document) => {
+			const metadata = {};
+			document.querySelectorAll("meta").forEach((meta) => {
+				const content = meta.getAttribute("content");
+				const key = meta.getAttribute("itemprop") ?? meta.getAttribute("property") ?? meta.getAttribute("name");
+				if (key && content) metadata[key] = content;
+			});
+			try {
+				for (const script of document.querySelectorAll("script")) {
+					const content = script.textContent ?? "";
+					if (!content.includes("ytInitialData")) continue;
+					const start = content.indexOf("{");
+					const end = content.lastIndexOf("}");
+					if (start >= 0 && end > start) {
+						const description = findKey(JSON.parse(content.slice(start, end + 1)), "attributedDescriptionBodyText");
+						if (isRecord(description) && typeof description.content === "string") metadata.description = description.content;
+					}
+					break;
 				}
-				break;
-			}
-		} catch {}
-		const title = first(metadata, [
-			"title",
-			"og:title",
-			"name"
-		]) || document.title || null;
-		const lines = ["# YouTube"];
-		if (title) lines.push(`## ${title}`);
-		const stats = [];
-		const views = first(metadata, ["interactionCount"]);
-		const keywords = first(metadata, ["keywords"]);
-		const runtime = first(metadata, ["duration"]);
-		if (views) stats.push(`- **Views:** ${views}`);
-		if (keywords) stats.push(`- **Keywords:** ${keywords}`);
-		if (runtime) stats.push(`- **Runtime:** ${runtime}`);
-		if (stats.length > 0) lines.push(`### Video Metadata\n${stats.join("\n")}`);
-		const description = first(metadata, ["description", "og:description"]);
-		if (description) lines.push(`### Description\n${description}`);
-		const markdown = lines.join("\n\n");
-		return {
-			title,
-			markdown,
-			text_content: markdown
-		};
+			} catch {}
+			const title = first(metadata, [
+				"title",
+				"og:title",
+				"name"
+			]) || document.title || null;
+			const lines = ["# YouTube"];
+			if (title) lines.push(`## ${title}`);
+			const stats = [];
+			const views = first(metadata, ["interactionCount"]);
+			const keywords = first(metadata, ["keywords"]);
+			const runtime = first(metadata, ["duration"]);
+			if (views) stats.push(`- **Views:** ${views}`);
+			if (keywords) stats.push(`- **Keywords:** ${keywords}`);
+			if (runtime) stats.push(`- **Runtime:** ${runtime}`);
+			if (stats.length > 0) lines.push(`### Video Metadata\n\n${stats.join("\n")}`);
+			const description = first(metadata, ["description", "og:description"]);
+			if (description) lines.push(`### Description\n\n${description}`);
+			return {
+				title: {
+					type: "metadata",
+					value: title
+				},
+				markdown: {
+					type: "metadata",
+					value: lines.join("\n\n")
+				}
+			};
+		});
 	}
 };
 function first(metadata, keys) {
@@ -549,23 +616,20 @@ function findKey(value, key) {
 function isRecord(value) {
 	return typeof value === "object" && value !== null;
 }
-//#endregion
-//#region src/converters/zip.ts
+// src/converters/zip.ts
 var ZipConverter = class {
 	async convert(source, options = {}) {
-		if (options.file_extension?.toLowerCase() !== ".zip") return null;
+		if (options.extension !== "zip") return null;
 		const files = unzipSync(source, { filter: (file) => !file.name.endsWith("/") });
-		const converters = options._parent_converters ?? [];
+		const converters = options._converters ?? [];
 		const sections = [`Content from the zip file \`${options.url ?? "archive.zip"}\`:`];
 		for (const [name, bytes] of Object.entries(files)) {
 			const result = await convertEntry(bytes, name, converters, options);
 			if (result) sections.push(`## File: ${name}\n\n${result.markdown}`);
 		}
-		const markdown = sections.join("\n\n");
 		return {
 			title: null,
-			markdown,
-			text_content: markdown
+			markdown: sections.join("\n\n")
 		};
 	}
 };
@@ -574,9 +638,9 @@ async function convertEntry(source, name, converters, options) {
 	if (!extension) return null;
 	const entryOptions = {
 		...options,
-		file_extension: extension,
+		extension,
 		url: void 0,
-		_parent_converters: converters
+		_converters: converters
 	};
 	for (const converter of converters) try {
 		const result = await converter.convert(source, entryOptions);
@@ -584,42 +648,38 @@ async function convertEntry(source, name, converters, options) {
 	} catch {}
 	return null;
 }
-//#endregion
-//#region src/markitdown.ts
+// src/markitdown.ts
 var MarkItDown = class {
+	converters = [];
 	constructor() {
-		this.converters = [];
-		this.registerConverter(new PlainTextConverter());
-		this.registerConverter(new HtmlConverter());
-		this.registerConverter(new ZipConverter());
-		this.registerConverter(new RSSConverter());
-		this.registerConverter(new WikipediaConverter());
-		this.registerConverter(new YouTubeConverter());
-		this.registerConverter(new BingSerpConverter());
-		this.registerConverter(new DocxConverter());
-		this.registerConverter(new XlsxConverter());
-		this.registerConverter(new PptxConverter());
-		this.registerConverter(new PdfConverter());
-		this.registerConverter(new IpynbConverter());
+		const ordered = [
+			PlainTextConverter,
+			HtmlConverter,
+			ZipConverter,
+			RSSConverter,
+			WikipediaConverter,
+			YouTubeConverter,
+			BingSerpConverter,
+			DocxConverter,
+			XlsxConverter,
+			PptxConverter,
+			PdfConverter,
+			IpynbConverter
+		];
+		for (const converter of ordered) this.registerConverter(new converter());
 	}
 	/** Convert a URL, response, blob, array buffer, data view, or typed array. */
 	async convert(source, options = {}) {
 		if (typeof source === "string" || source instanceof URL) return this.convertUrl(source.toString(), options);
 		if (isResponse(source)) return this.convertResponse(source, options);
-		const extensions = /* @__PURE__ */ new Set();
-		this.addExtension(extensions, options.file_extension?.replace(/^\./, ""));
-		if (isBlob(source)) {
-			this.addExtension(extensions, extensionFromMime(source.type));
-			if ("name" in source && typeof source.name === "string") this.addExtension(extensions, extensionFromPathname(source.name));
-		}
-		if (extensions.size === 0) throw new Error("Could not determine the file type. Provide `file_extension` when converting bytes.");
-		return this.convertBytes(await toUint8Array(source), extensions, options);
-	}
-	/**
-	* @deprecated Pass the byte source directly to `convert`.
-	*/
-	async convertBuffer(source, options) {
-		return this.convert(source, options);
+		const bytes = await toUint8Array(source);
+		const extensions = await detectExtensions(bytes, {
+			extension: options.extension,
+			filename: isBlob(source) && "name" in source ? String(source.name) : null,
+			urls: [options.url],
+			contentType: isBlob(source) ? source.type : null
+		});
+		return this.convertBytes(bytes, extensions, options);
 	}
 	async convertUrl(source, { fetch: fetchImplementation = globalThis.fetch, ...options }) {
 		const url = new URL(source);
@@ -633,44 +693,39 @@ var MarkItDown = class {
 		});
 	}
 	async convertResponse(response, options) {
-		const extensions = /* @__PURE__ */ new Set();
-		this.addExtension(extensions, options.file_extension);
-		this.addExtension(extensions, extensionFromMime(response.headers.get("content-type")));
-		const filename = (response.headers.get("content-disposition") ?? "").match(/filename\*?=(?:UTF-8''|["']?)([^;"']+)/i)?.[1];
-		if (filename) this.addExtension(extensions, extensionFromPathname(decodeURIComponent(filename.trim())));
-		const responseUrl = options.url || response.url;
-		if (responseUrl) this.addExtension(extensions, extensionFromPathname(new URL(responseUrl).pathname));
-		if (extensions.size === 0) throw new Error("Could not determine the file type. Provide `file_extension` or a Content-Type header.");
-		return this.convertBytes(new Uint8Array(await response.arrayBuffer()), extensions, {
+		const bytes = new Uint8Array(await response.arrayBuffer());
+		const extensions = await detectExtensions(bytes, {
+			extension: options.extension,
+			filename: filenameFromDisposition(response.headers.get("content-disposition")),
+			urls: [options.url, response.url],
+			contentType: response.headers.get("content-type")
+		});
+		return this.convertBytes(bytes, extensions, {
 			...options,
-			url: responseUrl
+			url: options.url || response.url || void 0
 		});
 	}
 	async convertBytes(source, extensions, options) {
+		if (extensions.length === 0) throw new Error("Could not determine the file type. Provide `extension` for better results.");
 		let lastError;
 		for (const extension of extensions) for (const converter of this.converters) try {
 			const result = await converter.convert(source, {
 				...options,
-				file_extension: extension,
-				_parent_converters: this.converters
+				extension,
+				_converters: this.converters
 			});
 			if (result) {
 				result.markdown = result.markdown.replace(/\r\n|\r|\n/g, "\n").trim().replace(/\n{3,}/g, "\n\n");
-				result.text_content = result.markdown;
 				return result;
 			}
 		} catch (error) {
 			if (options.debug) console.error("Converter threw error:", error);
 			lastError = error;
 		}
-		const formats = [...extensions].join(", ");
+		const formats = extensions.join(", ");
 		if (lastError) throw new Error(`Could not convert ${formats} to Markdown: ${String(lastError)}`);
 		throw new Error(`Unsupported file type: ${formats}.`);
 	}
-	addExtension(extensions, extension) {
-		if (!extension) return;
-		extensions.add(extension.startsWith(".") ? extension.toLowerCase() : `.${extension.toLowerCase()}`);
-	}
 	registerConverter(converter) {
 		this.converters.unshift(converter);
 	}
@@ -678,13 +733,53 @@ var MarkItDown = class {
 function isResponse(value) {
 	return typeof Response !== "undefined" && value instanceof Response;
 }
-//#endregion
-//#region src/cli.ts
-new Command().name("markitdown").version(version).argument("<file>", "The file to convert to Markdown").action(async (file) => {
-	const extension = file.split(".").pop()?.toLowerCase();
-	const bytes = await readFile(file);
-	const result = await new MarkItDown().convert(new Uint8Array(bytes), { file_extension: extension });
-	console.log(result);
-}).parse();
-//#endregion
-export {};
+// src/cli.ts
+const parser = object({
+	file: argument(path({
+		mustExist: true,
+		type: "file",
+		metavar: "FILE"
+	}), { description: message`File to convert.` }),
+	extension: optional(option("-x", "--extension", string({ metavar: "EXT" }), { description: message`File type to convert from, when it can't be detected from the file.` })),
+	json: withDefault(flag("--json", { description: message`Print the title and Markdown as JSON instead of only the Markdown.` }), false)
+});
+const program = defineProgram({
+	parser,
+	metadata: {
+		name: "markitdown",
+		version,
+		brief: message`Convert files to Markdown`,
+		description: message`Converts PDF, Word, Excel, PowerPoint, HTML, notebooks, feeds, zip archives and text files to Markdown. The file type is detected from the file's name and contents.`,
+		examples: message`Convert a document:
+
+  ${commandLine("markitdown report.docx")}
+
+Convert a file without an extension as HTML:
+
+  ${commandLine("markitdown --extension html page")}`,
+		author: message`${author}`,
+		bugs: message`Report bugs at ${url(bugs.url)}.`
+	}
+});
+if (import.meta.main) {
+	const argv = process.argv.slice(2);
+	const parsed = runSync(program, {
+		args: argv.length ? argv : ["--help"],
+		aboveError: "none",
+		help: "both",
+		version: {
+			value: version,
+			option: true
+		},
+		completion: "both"
+	});
+	try {
+		const bytes = await readFile(parsed.file);
+		const result = await new MarkItDown().convert(new File([bytes], basename(parsed.file)), parsed.extension ? { extension: parsed.extension } : {});
+		console.log(parsed.json ? JSON.stringify(result, null, 2) : result?.markdown);
+	} catch (error) {
+		console.error(`Error: ${error.message}`);
+		process.exitCode = 1;
+	}
+}
+export { program };