@ryangarber/markitdown-ts 0.1.2 → 0.1.4

package/dist/cli.mjs

@@ -0,0 +1,611 @@
+import { readFile } from "node:fs/promises";
+import { Command } from "@commander-js/extra-typings";
+import turndownPluginGfm from "@joplin/turndown-plugin-gfm";
+import TurndownService from "turndown";
+import { parseHTML } from "linkedom";
+import mammoth from "mammoth";
+import { extractText, getDocumentProxy, getMeta } from "unpdf";
+import ExcelJS from "exceljs";
+import { DOMParser } from "@xmldom/xmldom";
+import { unzipSync } from "fflate";
+//#region package.json
+var version = "0.1.4";
+//#endregion
+//#region src/bytes.ts
+async function toUint8Array(source) {
+	if (source instanceof Uint8Array) return source;
+	if (isBlob(source)) return new Uint8Array(await source.arrayBuffer());
+	if (ArrayBuffer.isView(source)) return new Uint8Array(source.buffer, source.byteOffset, source.byteLength);
+	return new Uint8Array(source);
+}
+function toArrayBuffer(source) {
+	return source.slice().buffer;
+}
+function decodeText(source) {
+	return new TextDecoder().decode(source);
+}
+function isBlob(value) {
+	return typeof Blob !== "undefined" && value instanceof Blob;
+}
+//#endregion
+//#region src/custom-turndown.ts
+var CustomTurnDown = class {
+	convertSoup(document) {
+		const service = new TurndownService({ headingStyle: "atx" });
+		service.use(turndownPluginGfm.gfm);
+		service.addRule("anchor tags", {
+			filter: ["a"],
+			replacement(content, node) {
+				if (!content) return "";
+				const prefix = content.startsWith(" ") ? " " : "";
+				const suffix = content.endsWith(" ") ? " " : "";
+				const text = content.trim().replace(/\n\n.*/g, "");
+				if (!text) return "";
+				const element = node;
+				const href = element.getAttribute("href");
+				let title = element.getAttribute("title") ?? "";
+				if (!href) return `${prefix}${text}${suffix}`;
+				try {
+					const url = new URL(href);
+					if (url.protocol !== "https:" && url.protocol !== "http:") return `${prefix}${text}${suffix}`;
+				} catch {
+					if (!href.startsWith("/") && !href.startsWith("#") && !href.startsWith("./") && !href.startsWith("../")) return `${prefix}${text}${suffix}`;
+				}
+				if (text.replace(/\\_/g, "_") === href && !title) return `<${href}>`;
+				if (!title) title = href;
+				return `${prefix}[${text}](${href}${title ? ` "${title}"` : ""})${suffix}`;
+			}
+		});
+		service.addRule("img tags", {
+			filter: ["img"],
+			replacement(_content, node) {
+				const element = node;
+				const alt = element.getAttribute("alt") ?? "";
+				let source = element.getAttribute("src") ?? "";
+				const title = element.getAttribute("title");
+				if (source.startsWith("data:")) source = `${source.split(",", 1)[0]}...`;
+				return `![${alt}](${source}${title ? ` "${title}"` : ""})`;
+			}
+		});
+		return service.turndown(document);
+	}
+};
+//#endregion
+//#region src/html-document.ts
+function parseHtmlDocument(html) {
+	return parseHTML(html).document;
+}
+//#endregion
+//#region src/converters/bingserp.ts
+const BING_SEARCH_URL = /^https:\/\/www\.bing\.com\/search\?q=/i;
+const BASE64_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+var BingSerpConverter = class {
+	async convert(source, options = {}) {
+		const extension = options.file_extension?.toLowerCase() ?? "";
+		const url = options.url ?? "";
+		if (extension !== ".html" && extension !== ".htm" || !BING_SEARCH_URL.test(url)) return null;
+		const document = parseHtmlDocument(decodeText(source));
+		const query = new URL(url).searchParams.get("q") ?? "";
+		document.querySelectorAll(".tptt").forEach((element) => {
+			if (element.textContent) element.textContent += " ";
+		});
+		document.querySelectorAll(".algoSlug_icon").forEach((element) => {
+			element.remove();
+		});
+		const turndown = new CustomTurnDown();
+		const markdown = `## A Bing search for '${query}' found the following results:\n\n${[...document.querySelectorAll(".b_algo")].map((result) => {
+			result.querySelectorAll("a[href]").forEach((anchor) => {
+				const href = anchor.getAttribute("href");
+				if (!href) return;
+				try {
+					const encoded = new URL(href, url).searchParams.get("u");
+					if (encoded) anchor.setAttribute("href", decodeBingUrl(encoded));
+				} catch {}
+			});
+			return turndown.convertSoup(result).split(/\n+/).map((line) => line.trim()).filter(Boolean).join("\n");
+		}).join("\n\n")}`;
+		return {
+			title: document.title || null,
+			markdown,
+			text_content: markdown
+		};
+	}
+};
+function decodeBingUrl(encoded) {
+	const input = encoded.slice(2).replace(/-/g, "+").replace(/_/g, "/").replace(/=+$/, "");
+	const bytes = [];
+	let bits = 0;
+	let bitCount = 0;
+	for (const character of input) {
+		const value = BASE64_ALPHABET.indexOf(character);
+		if (value < 0) return encoded;
+		bits = bits << 6 | value;
+		bitCount += 6;
+		if (bitCount >= 8) {
+			bitCount -= 8;
+			bytes.push(bits >> bitCount & 255);
+		}
+	}
+	return new TextDecoder().decode(new Uint8Array(bytes));
+}
+//#endregion
+//#region src/converters/html.ts
+var HtmlConverter = class {
+	async convert(source, options) {
+		const extension = options.file_extension?.toLowerCase() ?? "";
+		if (extension !== ".html" && extension !== ".htm") return null;
+		return this.convertHtml(decodeText(source));
+	}
+	async convertHtml(html) {
+		const document = parseHtmlDocument(html);
+		const title = document.title;
+		document.querySelectorAll("script, style, title").forEach((element) => {
+			element.remove();
+		});
+		const markdown = new CustomTurnDown().convertSoup(document);
+		return {
+			title: title || null,
+			markdown,
+			text_content: markdown
+		};
+	}
+};
+//#endregion
+//#region src/converters/docx.ts
+var DocxConverter = class extends HtmlConverter {
+	async convert(source, options) {
+		if (options.file_extension?.toLowerCase() !== ".docx") return null;
+		let mammothInput = { arrayBuffer: toArrayBuffer(source) };
+		if (typeof process !== "undefined" && process.versions !== null && process.versions.node !== null) mammothInput = { buffer: Buffer.from(source) };
+		const mammothOptions = {
+			styleMap: options.styleMap,
+			includeEmbeddedStyleMap: options.includeEmbeddedStyleMap,
+			includeDefaultStyleMap: options.includeDefaultStyleMap,
+			ignoreEmptyParagraphs: options.ignoreEmptyParagraphs,
+			idPrefix: options.idPrefix
+		};
+		if (options.transformDocument) mammothOptions.transformDocument = options.transformDocument;
+		const html = await mammoth.convertToHtml(mammothInput, mammothOptions);
+		return this.convertHtml(html.value);
+	}
+};
+//#endregion
+//#region src/converters/ipynb.ts
+var IpynbConverter = class {
+	async convert(source, options = {}) {
+		if (options.file_extension?.toLowerCase() !== ".ipynb") return null;
+		const notebook = JSON.parse(decodeText(source));
+		const output = [];
+		let title = notebook.metadata?.title ?? null;
+		for (const cell of notebook.cells ?? []) {
+			const lines = cell.source ?? [];
+			if (cell.cell_type === "markdown") {
+				output.push(lines.join(""));
+				title ??= lines.find((line) => line.startsWith("# "))?.slice(2).trim() ?? null;
+			} else if (cell.cell_type === "code") output.push(`\`\`\`python\n${lines.join("")}\n\`\`\``);
+			else if (cell.cell_type === "raw") output.push(`\`\`\`\n${lines.join("")}\n\`\`\``);
+		}
+		const markdown = output.join("\n\n");
+		return {
+			title,
+			markdown,
+			text_content: markdown
+		};
+	}
+};
+//#endregion
+//#region src/converters/pdf.ts
+var PdfConverter = class {
+	async convert(source, options) {
+		if (options.file_extension?.toLowerCase() !== ".pdf") return null;
+		const pdf = await getDocumentProxy(source.slice());
+		try {
+			const [{ text }, { info }] = await Promise.all([extractText(pdf, { mergePages: true }), getMeta(pdf)]);
+			return {
+				title: typeof info.Title === "string" && info.Title.trim() ? info.Title : null,
+				markdown: text,
+				text_content: text
+			};
+		} finally {
+			await pdf.loadingTask.destroy();
+		}
+	}
+};
+//#endregion
+//#region src/mime.ts
+const MIME_EXTENSIONS = {
+	"application/atom+xml": ".atom",
+	"application/json": ".json",
+	"application/pdf": ".pdf",
+	"application/zip": ".zip",
+	"application/rss+xml": ".rss",
+	"application/x-pdf": ".pdf",
+	"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",
+	"application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
+	"application/xml": ".xml",
+	"text/csv": ".csv",
+	"text/html": ".html",
+	"text/markdown": ".md",
+	"text/plain": ".txt",
+	"text/xml": ".xml"
+};
+const TEXT_EXTENSIONS = /* @__PURE__ */ new Set([
+	".csv",
+	".json",
+	".log",
+	".markdown",
+	".md",
+	".text",
+	".txt"
+]);
+function extensionFromMime(contentType) {
+	if (!contentType) return null;
+	const mime = contentType.split(";", 1)[0]?.trim().toLowerCase() ?? "";
+	return MIME_EXTENSIONS[mime] ?? (mime.startsWith("text/") ? ".txt" : null);
+}
+function isTextExtension(extension) {
+	return TEXT_EXTENSIONS.has(extension.toLowerCase());
+}
+function extensionFromPathname(pathname) {
+	const filename = pathname.split("/").pop() ?? "";
+	const dot = filename.lastIndexOf(".");
+	return dot > 0 ? filename.slice(dot).toLowerCase() : null;
+}
+//#endregion
+//#region src/converters/plain-text.ts
+var PlainTextConverter = class {
+	async convert(source, options = {}) {
+		if (!isTextExtension(options.file_extension ?? "")) return null;
+		const content = decodeText(source);
+		return {
+			title: null,
+			markdown: content,
+			text_content: content
+		};
+	}
+};
+//#endregion
+//#region src/converters/wikipedia.ts
+const WIKIPEDIA_URL = /^https?:\/\/[a-z]{2,3}\.wikipedia\.org\//i;
+var WikipediaConverter = class {
+	async convert(source, options = {}) {
+		const extension = options.file_extension?.toLowerCase() ?? "";
+		if (extension !== ".html" && extension !== ".htm" || !WIKIPEDIA_URL.test(options.url ?? "")) return null;
+		const document = parseHtmlDocument(decodeText(source));
+		document.querySelectorAll("script, style").forEach((element) => {
+			element.remove();
+		});
+		const body = document.querySelector("div#mw-content-text");
+		const title = document.querySelector("span.mw-page-title-main")?.textContent?.trim() || document.title || null;
+		const markdown = body ? `# ${title ?? "Wikipedia"}\n\n${new CustomTurnDown().convertSoup(body)}` : new CustomTurnDown().convertSoup(document);
+		return {
+			title,
+			markdown,
+			text_content: markdown
+		};
+	}
+};
+//#endregion
+//#region src/converters/xlsx.ts
+const { Workbook } = ExcelJS;
+var XlsxConverter = class extends HtmlConverter {
+	async convert(source, options) {
+		if (options.file_extension?.toLowerCase() !== ".xlsx") return null;
+		const workbook = new Workbook();
+		await workbook.xlsx.load(toArrayBuffer(source));
+		const sections = [];
+		for (const worksheet of workbook.worksheets) {
+			const html = worksheetToHtml(worksheet);
+			if (!html) continue;
+			const result = await this.convertHtml(html);
+			sections.push(`## ${worksheet.name}\n${result?.markdown.trim() ?? ""}`);
+		}
+		const markdown = sections.join("\n\n");
+		return {
+			title: workbook.title || "Untitled",
+			markdown,
+			text_content: markdown
+		};
+	}
+};
+function worksheetToHtml(worksheet) {
+	const columnCount = worksheet.columnCount;
+	if (!columnCount || !worksheet.actualRowCount) return "";
+	const rows = [];
+	worksheet.eachRow({ includeEmpty: true }, (row) => {
+		const cells = [];
+		for (let column = 1; column <= columnCount; column++) cells.push(`<td>${escapeHtml(cellText(row.getCell(column)))}</td>`);
+		rows.push(`<tr>${cells.join("")}</tr>`);
+	});
+	return `<html><body><table>${rows.join("")}</table></body></html>`;
+}
+function cellText(cell) {
+	if (cell.isMerged && cell.master !== cell) return "";
+	return cell.text;
+}
+function escapeHtml(value) {
+	return value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
+}
+//#endregion
+//#region src/converters/xml-rss-atom.ts
+var RSSConverter = class {
+	async convert(source, options = {}) {
+		const extension = options.file_extension?.toLowerCase() ?? "";
+		if (![
+			".xml",
+			".rss",
+			".atom"
+		].includes(extension)) return null;
+		const document = new DOMParser().parseFromString(decodeText(source), "text/xml");
+		if (document.getElementsByTagName("rss").length > 0) return this.parseRss(document);
+		if (document.getElementsByTagName("feed")[0]?.getElementsByTagName("entry").length) return this.parseAtom(document);
+		return null;
+	}
+	parseAtom(document) {
+		const root = document.getElementsByTagName("feed")[0];
+		if (!root) return null;
+		const title = getText(root, "title");
+		const subtitle = getText(root, "subtitle");
+		const sections = [`# ${title ?? "Feed"}`];
+		if (subtitle) sections.push(subtitle);
+		for (const entry of [...root.getElementsByTagName("entry")]) {
+			const entryTitle = getText(entry, "title");
+			const updated = getText(entry, "updated");
+			const summary = getText(entry, "summary");
+			const content = getText(entry, "content");
+			const lines = [];
+			if (entryTitle) lines.push(`## ${entryTitle}`);
+			if (updated) lines.push(`Updated on: ${updated}`);
+			if (summary) lines.push(parseContent(summary));
+			if (content) lines.push(parseContent(content));
+			sections.push(lines.join("\n"));
+		}
+		const markdown = sections.join("\n\n");
+		return {
+			title,
+			markdown,
+			text_content: markdown
+		};
+	}
+	parseRss(document) {
+		const channel = document.getElementsByTagName("rss")[0]?.getElementsByTagName("channel")[0];
+		if (!channel) return null;
+		const title = getText(channel, "title");
+		const description = getText(channel, "description");
+		const sections = title ? [`# ${title}`] : [];
+		if (description) sections.push(description);
+		for (const item of [...channel.getElementsByTagName("item")]) {
+			const itemTitle = getText(item, "title");
+			const published = getText(item, "pubDate");
+			const itemDescription = getText(item, "description");
+			const content = getText(item, "content:encoded");
+			const lines = [];
+			if (itemTitle) lines.push(`## ${itemTitle}`);
+			if (published) lines.push(`Published on: ${published}`);
+			if (itemDescription) lines.push(parseContent(itemDescription));
+			if (content) lines.push(parseContent(content));
+			sections.push(lines.join("\n"));
+		}
+		const markdown = sections.join("\n\n");
+		return {
+			title,
+			markdown,
+			text_content: markdown
+		};
+	}
+};
+function parseContent(content) {
+	return new CustomTurnDown().convertSoup(parseHtmlDocument(content).documentElement);
+}
+function getText(element, tagName) {
+	return element.getElementsByTagName(tagName)[0]?.textContent?.trim() || null;
+}
+//#endregion
+//#region src/converters/youtube.ts
+var YouTubeConverter = class {
+	async convert(source, options = {}) {
+		const extension = options.file_extension?.toLowerCase() ?? "";
+		if (extension !== ".html" && extension !== ".htm" || !options.url?.startsWith("https://www.youtube.com/watch?")) return null;
+		const document = parseHtmlDocument(decodeText(source));
+		const metadata = { title: document.title };
+		document.querySelectorAll("meta").forEach((meta) => {
+			const content = meta.getAttribute("content");
+			const key = meta.getAttribute("itemprop") ?? meta.getAttribute("property") ?? meta.getAttribute("name");
+			if (key && content) metadata[key] = content;
+		});
+		try {
+			for (const script of document.querySelectorAll("script")) {
+				const content = script.textContent ?? "";
+				if (!content.includes("ytInitialData")) continue;
+				const start = content.indexOf("{");
+				const end = content.lastIndexOf("}");
+				if (start >= 0 && end > start) {
+					const description = findKey(JSON.parse(content.slice(start, end + 1)), "attributedDescriptionBodyText");
+					if (isRecord(description) && typeof description.content === "string") metadata.description = description.content;
+				}
+				break;
+			}
+		} catch {}
+		const title = first(metadata, [
+			"title",
+			"og:title",
+			"name"
+		]) || document.title || null;
+		const lines = ["# YouTube"];
+		if (title) lines.push(`## ${title}`);
+		const stats = [];
+		const views = first(metadata, ["interactionCount"]);
+		const keywords = first(metadata, ["keywords"]);
+		const runtime = first(metadata, ["duration"]);
+		if (views) stats.push(`- **Views:** ${views}`);
+		if (keywords) stats.push(`- **Keywords:** ${keywords}`);
+		if (runtime) stats.push(`- **Runtime:** ${runtime}`);
+		if (stats.length > 0) lines.push(`### Video Metadata\n${stats.join("\n")}`);
+		const description = first(metadata, ["description", "og:description"]);
+		if (description) lines.push(`### Description\n${description}`);
+		const markdown = lines.join("\n\n");
+		return {
+			title,
+			markdown,
+			text_content: markdown
+		};
+	}
+};
+function first(metadata, keys) {
+	for (const key of keys) if (metadata[key]) return metadata[key];
+	return null;
+}
+function findKey(value, key) {
+	if (Array.isArray(value)) for (const item of value) {
+		const match = findKey(item, key);
+		if (match !== void 0) return match;
+	}
+	else if (isRecord(value)) {
+		if (key in value) return value[key];
+		for (const item of Object.values(value)) {
+			const match = findKey(item, key);
+			if (match !== void 0) return match;
+		}
+	}
+}
+function isRecord(value) {
+	return typeof value === "object" && value !== null;
+}
+//#endregion
+//#region src/converters/zip.ts
+var ZipConverter = class {
+	async convert(source, options = {}) {
+		if (options.file_extension?.toLowerCase() !== ".zip") return null;
+		const files = unzipSync(source, { filter: (file) => !file.name.endsWith("/") });
+		const converters = options._parent_converters ?? [];
+		const sections = [`Content from the zip file \`${options.url ?? "archive.zip"}\`:`];
+		for (const [name, bytes] of Object.entries(files)) {
+			const result = await convertEntry(bytes, name, converters, options);
+			if (result) sections.push(`## File: ${name}\n\n${result.markdown}`);
+		}
+		const markdown = sections.join("\n\n");
+		return {
+			title: null,
+			markdown,
+			text_content: markdown
+		};
+	}
+};
+async function convertEntry(source, name, converters, options) {
+	const extension = extensionFromPathname(name);
+	if (!extension) return null;
+	const entryOptions = {
+		...options,
+		file_extension: extension,
+		url: void 0,
+		_parent_converters: converters
+	};
+	for (const converter of converters) try {
+		const result = await converter.convert(source, entryOptions);
+		if (result) return result;
+	} catch {}
+	return null;
+}
+//#endregion
+//#region src/markitdown.ts
+var MarkItDown = class {
+	constructor() {
+		this.converters = [];
+		this.registerConverter(new PlainTextConverter());
+		this.registerConverter(new HtmlConverter());
+		this.registerConverter(new ZipConverter());
+		this.registerConverter(new RSSConverter());
+		this.registerConverter(new WikipediaConverter());
+		this.registerConverter(new YouTubeConverter());
+		this.registerConverter(new BingSerpConverter());
+		this.registerConverter(new DocxConverter());
+		this.registerConverter(new XlsxConverter());
+		this.registerConverter(new PdfConverter());
+		this.registerConverter(new IpynbConverter());
+	}
+	/** Convert a URL, response, blob, array buffer, data view, or typed array. */
+	async convert(source, options = {}) {
+		if (typeof source === "string" || source instanceof URL) return this.convertUrl(source.toString(), options);
+		if (isResponse(source)) return this.convertResponse(source, options);
+		const extensions = /* @__PURE__ */ new Set();
+		this.addExtension(extensions, options.file_extension?.replace(/^\./, ""));
+		if (isBlob(source)) {
+			this.addExtension(extensions, extensionFromMime(source.type));
+			if ("name" in source && typeof source.name === "string") this.addExtension(extensions, extensionFromPathname(source.name));
+		}
+		if (extensions.size === 0) throw new Error("Could not determine the file type. Provide `file_extension` when converting bytes.");
+		return this.convertBytes(await toUint8Array(source), extensions, options);
+	}
+	/**
+	* @deprecated Pass the byte source directly to `convert`.
+	*/
+	async convertBuffer(source, options) {
+		return this.convert(source, options);
+	}
+	async convertUrl(source, { fetch: fetchImplementation = globalThis.fetch, ...options }) {
+		const url = new URL(source);
+		if (url.protocol !== "http:" && url.protocol !== "https:") throw new Error(`Unsupported URL protocol: ${url.protocol}`);
+		if (!fetchImplementation) throw new Error("No fetch implementation is available.");
+		const response = await fetchImplementation(url);
+		if (!response.ok) throw new Error(`Failed to fetch URL: ${url}, status: ${response.status}`);
+		return this.convertResponse(response, {
+			...options,
+			url: url.toString()
+		});
+	}
+	async convertResponse(response, options) {
+		const extensions = /* @__PURE__ */ new Set();
+		this.addExtension(extensions, options.file_extension);
+		this.addExtension(extensions, extensionFromMime(response.headers.get("content-type")));
+		const filename = (response.headers.get("content-disposition") ?? "").match(/filename\*?=(?:UTF-8''|["']?)([^;"']+)/i)?.[1];
+		if (filename) this.addExtension(extensions, extensionFromPathname(decodeURIComponent(filename.trim())));
+		const responseUrl = options.url || response.url;
+		if (responseUrl) this.addExtension(extensions, extensionFromPathname(new URL(responseUrl).pathname));
+		if (extensions.size === 0) throw new Error("Could not determine the file type. Provide `file_extension` or a Content-Type header.");
+		return this.convertBytes(new Uint8Array(await response.arrayBuffer()), extensions, {
+			...options,
+			url: responseUrl
+		});
+	}
+	async convertBytes(source, extensions, options) {
+		let lastError;
+		for (const extension of extensions) for (const converter of this.converters) try {
+			const result = await converter.convert(source, {
+				...options,
+				file_extension: extension,
+				_parent_converters: this.converters
+			});
+			if (result) {
+				result.markdown = result.markdown.replace(/\r\n|\r|\n/g, "\n").trim().replace(/\n{3,}/g, "\n\n");
+				result.text_content = result.markdown;
+				return result;
+			}
+		} catch (error) {
+			if (options.debug) console.error("Converter threw error:", error);
+			lastError = error;
+		}
+		const formats = [...extensions].join(", ");
+		if (lastError) throw new Error(`Could not convert ${formats} to Markdown: ${String(lastError)}`);
+		throw new Error(`Unsupported file type: ${formats}.`);
+	}
+	addExtension(extensions, extension) {
+		if (!extension) return;
+		extensions.add(extension.startsWith(".") ? extension.toLowerCase() : `.${extension.toLowerCase()}`);
+	}
+	registerConverter(converter) {
+		this.converters.unshift(converter);
+	}
+};
+function isResponse(value) {
+	return typeof Response !== "undefined" && value instanceof Response;
+}
+//#endregion
+//#region src/cli.ts
+new Command().name("markitdown").version(version).argument("<file>", "The file to convert to Markdown").action(async (file) => {
+	const extension = file.split(".").pop()?.toLowerCase();
+	const bytes = await readFile(file);
+	const result = await new MarkItDown().convert(new Uint8Array(bytes), { file_extension: extension });
+	console.log(result);
+}).parse();
+//#endregion
+export {};