markit-ai 0.1.0

package/dist/main.js

@@ -0,0 +1,182 @@
+#!/usr/bin/env node
+import { createRequire } from "node:module";
+import { Command } from "commander";
+import { convert } from "./commands/convert.js";
+import { onboard } from "./commands/onboard.js";
+import { formats } from "./commands/formats.js";
+import { init } from "./commands/init.js";
+import { configShow, configGet, configSet } from "./commands/config.js";
+import { pluginInstall, pluginRemove, pluginList } from "./commands/plugin.js";
+const require = createRequire(import.meta.url);
+const { version } = require("../package.json");
+const program = new Command();
+program
+    .name("markit")
+    .description("Convert anything to markdown.")
+    .version(`markit ${version}`, "-V, --version")
+    .option("--json", "Output as JSON")
+    .option("-q, --quiet", "Raw markdown only, no decoration")
+    .option("-p, --prompt <text>", "Extra instructions for image description")
+    .option("-o, --output <file>", "Write to file instead of stdout")
+    .addHelpText("after", `
+Examples:
+  $ markit report.pdf                  Convert a PDF to markdown
+  $ markit document.docx -o doc.md     Convert DOCX, write to file
+  $ markit https://example.com         Convert a web page
+  $ markit photo.jpg                    Extract EXIF + AI description
+  $ markit recording.mp3               Metadata + transcription
+  $ cat file.pdf | markit -            Read from stdin
+  $ markit init                        Create .markit/ config
+  $ markit config show                 Show LLM settings
+
+Docs: https://github.com/Michaelliv/markit`);
+program
+    .command("convert")
+    .alias("c")
+    .description("Convert a file or URL to markdown")
+    .argument("<source>", "File path, URL, or - for stdin")
+    .option("-o, --output <file>", "Write to file instead of stdout")
+    .action(async (source, opts, cmd) => {
+    const globals = cmd.optsWithGlobals();
+    await convert(source, {
+        json: globals.json,
+        quiet: globals.quiet,
+        output: opts.output,
+        prompt: opts.prompt,
+    });
+});
+program
+    .command("init")
+    .description("Create .markit/ config directory")
+    .action(async (_opts, cmd) => {
+    const globals = cmd.optsWithGlobals();
+    await init([], { json: globals.json, quiet: globals.quiet });
+});
+const configCmd = program
+    .command("config")
+    .description("Manage markit configuration");
+configCmd
+    .command("show")
+    .description("Show current configuration")
+    .action(async (_opts, cmd) => {
+    const globals = cmd.optsWithGlobals();
+    await configShow([], { json: globals.json, quiet: globals.quiet });
+});
+configCmd
+    .command("get <key>")
+    .description("Get a config value")
+    .action(async (key, _opts, cmd) => {
+    const globals = cmd.optsWithGlobals();
+    await configGet(key, { json: globals.json, quiet: globals.quiet });
+});
+configCmd
+    .command("set <key> [value]")
+    .description("Set a config value (secrets read from stdin if no value given)")
+    .action(async (key, value, _opts, cmd) => {
+    const globals = cmd.optsWithGlobals();
+    await configSet(key, value, { json: globals.json, quiet: globals.quiet });
+});
+const pluginCmd = program
+    .command("plugin")
+    .description("Manage plugins");
+pluginCmd
+    .command("install <source>")
+    .description("Install a plugin (npm:pkg, git:url, or local path)")
+    .action(async (source, _opts, cmd) => {
+    const globals = cmd.optsWithGlobals();
+    await pluginInstall(source, { json: globals.json, quiet: globals.quiet });
+});
+pluginCmd
+    .command("remove <name>")
+    .description("Remove an installed plugin")
+    .action(async (name, _opts, cmd) => {
+    const globals = cmd.optsWithGlobals();
+    await pluginRemove(name, { json: globals.json, quiet: globals.quiet });
+});
+pluginCmd
+    .command("list")
+    .description("List installed plugins")
+    .action(async (_opts, cmd) => {
+    const globals = cmd.optsWithGlobals();
+    await pluginList({ json: globals.json, quiet: globals.quiet });
+});
+program
+    .command("formats")
+    .description("List supported formats")
+    .action(async (_opts, cmd) => {
+    const globals = cmd.optsWithGlobals();
+    await formats([], { json: globals.json, quiet: globals.quiet });
+});
+program
+    .command("onboard")
+    .description("Add markit instructions to CLAUDE.md or AGENTS.md")
+    .action(async (_opts, cmd) => {
+    const globals = cmd.optsWithGlobals();
+    await onboard([], { json: globals.json, quiet: globals.quiet });
+});
+// Default behavior: if first arg isn't a known subcommand, treat it as a source to convert
+program.on("command:*", async (args) => {
+    const source = args[0];
+    if (!source) {
+        program.help();
+        return;
+    }
+    // Check for typos against known subcommands
+    const commands = ["convert", "formats", "onboard", "help", "init", "config", "plugin"];
+    const close = commands.filter((c) => levenshtein(source, c) <= 2 && source !== c);
+    if (close.length > 0 &&
+        !source.includes("/") &&
+        !source.includes(".") &&
+        !source.startsWith("http")) {
+        const { error } = await import("./utils/output.js");
+        error(`Unknown command '${source}'. Did you mean '${close[0]}'?`);
+        process.exit(1);
+    }
+    const globals = program.opts();
+    await convert(source, {
+        json: globals.json,
+        quiet: globals.quiet,
+        output: globals.output,
+        prompt: globals.prompt,
+    });
+});
+// No args → show concise help
+if (process.argv.length <= 2) {
+    console.log(`markit — convert anything to markdown
+
+Usage:  markit <file-or-url> [options]
+
+Examples:
+  $ markit report.pdf
+  $ markit document.docx -o doc.md
+  $ markit https://example.com
+
+Commands:
+  markit init        Create .markit/ config directory
+  markit config      Manage settings (LLM, API keys)
+  markit formats     List supported formats
+  markit onboard     Add instructions to CLAUDE.md
+
+Run markit --help for all options.
+Docs: https://github.com/Michaelliv/markit`);
+    process.exit(0);
+}
+program.parseAsync(process.argv).catch((err) => {
+    console.error("Fatal error:", err.message);
+    process.exit(1);
+});
+function levenshtein(a, b) {
+    const m = a.length;
+    const n = b.length;
+    const dp = Array.from({ length: m + 1 }, () => Array(n + 1).fill(0));
+    for (let i = 0; i <= m; i++)
+        dp[i][0] = i;
+    for (let j = 0; j <= n; j++)
+        dp[0][j] = j;
+    for (let i = 1; i <= m; i++) {
+        for (let j = 1; j <= n; j++) {
+            dp[i][j] = Math.min(dp[i - 1][j] + 1, dp[i][j - 1] + 1, dp[i - 1][j - 1] + (a[i - 1] !== b[j - 1] ? 1 : 0));
+        }
+    }
+    return dp[m][n];
+}

package/dist/markit.d.ts

@@ -0,0 +1,19 @@
+import type { ConversionResult, StreamInfo, MarkitOptions } from "./types.js";
+import type { PluginDef } from "./plugins/types.js";
+export declare class Markit {
+    private converters;
+    private options;
+    constructor(options?: MarkitOptions, plugins?: PluginDef[]);
+    /**
+     * Convert a local file to markdown.
+     */
+    convertFile(path: string): Promise<ConversionResult>;
+    /**
+     * Convert a URL to markdown.
+     */
+    convertUrl(url: string): Promise<ConversionResult>;
+    /**
+     * Convert a buffer with stream info to markdown.
+     */
+    convert(input: Buffer, streamInfo: StreamInfo): Promise<ConversionResult>;
+}

package/dist/markit.js

@@ -0,0 +1,124 @@
+import { readFileSync } from "node:fs";
+import { extname, basename } from "node:path";
+import { PdfConverter } from "./converters/pdf.js";
+import { DocxConverter } from "./converters/docx.js";
+import { PptxConverter } from "./converters/pptx.js";
+import { XlsxConverter } from "./converters/xlsx.js";
+import { EpubConverter } from "./converters/epub.js";
+import { IpynbConverter } from "./converters/ipynb.js";
+import { HtmlConverter } from "./converters/html.js";
+import { WikipediaConverter } from "./converters/wikipedia.js";
+import { RssConverter } from "./converters/rss.js";
+import { CsvConverter } from "./converters/csv.js";
+import { JsonConverter } from "./converters/json.js";
+import { YamlConverter } from "./converters/yaml.js";
+import { XmlConverter } from "./converters/xml.js";
+import { ZipConverter } from "./converters/zip.js";
+import { ImageConverter } from "./converters/image.js";
+import { AudioConverter } from "./converters/audio.js";
+import { PlainTextConverter } from "./converters/plain-text.js";
+export class Markit {
+    converters = [];
+    options;
+    constructor(options = {}, plugins = []) {
+        this.options = options;
+        // Plugin converters go first — they override builtins for the same format
+        const pluginConverters = plugins.flatMap((p) => p.converters);
+        // Built-in converters: specific formats first, generic last.
+        const specific = [
+            new PdfConverter(),
+            new DocxConverter(),
+            new PptxConverter(),
+            new XlsxConverter(),
+            new EpubConverter(),
+            new IpynbConverter(),
+            new WikipediaConverter(),
+            new RssConverter(),
+            new CsvConverter(),
+            new JsonConverter(),
+            new YamlConverter(),
+            new ImageConverter(),
+            new AudioConverter(),
+        ];
+        const generic = [
+            new XmlConverter(),
+            new HtmlConverter(),
+        ];
+        // ZIP gets all converters (plugin + builtin) for recursive extraction
+        const allNonZip = [...pluginConverters, ...specific, ...generic];
+        const zipConverter = new ZipConverter(allNonZip);
+        // Plugin converters first, then builtins, plain text last
+        this.converters = [
+            ...pluginConverters,
+            ...specific,
+            zipConverter,
+            ...generic,
+            new PlainTextConverter(),
+        ];
+    }
+    /**
+     * Convert a local file to markdown.
+     */
+    async convertFile(path) {
+        const buffer = readFileSync(path);
+        const streamInfo = {
+            localPath: path,
+            extension: extname(path).toLowerCase(),
+            filename: basename(path),
+        };
+        return this.convert(buffer, streamInfo);
+    }
+    /**
+     * Convert a URL to markdown.
+     */
+    async convertUrl(url) {
+        const response = await fetch(url, {
+            headers: {
+                Accept: "text/markdown, text/html;q=0.9, text/plain;q=0.8, */*;q=0.1",
+                "User-Agent": "mill/0.1.0",
+            },
+        });
+        if (!response.ok) {
+            throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
+        }
+        const contentType = response.headers.get("content-type") || "";
+        const [mimetype] = contentType.split(";");
+        // Derive extension from URL path
+        const urlPath = new URL(url).pathname;
+        const ext = extname(urlPath).toLowerCase();
+        const buffer = Buffer.from(await response.arrayBuffer());
+        const streamInfo = {
+            url,
+            mimetype: mimetype.trim(),
+            extension: ext || undefined,
+            filename: basename(urlPath) || undefined,
+        };
+        return this.convert(buffer, streamInfo);
+    }
+    /**
+     * Convert a buffer with stream info to markdown.
+     */
+    async convert(input, streamInfo) {
+        const errors = [];
+        for (const converter of this.converters) {
+            if (!converter.accepts(streamInfo))
+                continue;
+            try {
+                return await converter.convert(input, streamInfo, this.options);
+            }
+            catch (err) {
+                errors.push({
+                    converter: converter.name,
+                    error: err instanceof Error ? err : new Error(String(err)),
+                });
+            }
+        }
+        if (errors.length > 0) {
+            const details = errors
+                .map((e) => `  ${e.converter}: ${e.error.message}`)
+                .join("\n");
+            throw new Error(`Conversion failed:\n${details}`);
+        }
+        throw new Error(`Unsupported format: ${streamInfo.extension || streamInfo.mimetype || "unknown"}`);
+    }
+}

package/dist/mill.d.ts

@@ -0,0 +1,18 @@
+import type { ConversionResult, StreamInfo, ConvertOptions } from "./types.js";
+export declare class Mill {
+    private converters;
+    private options;
+    constructor(options?: ConvertOptions);
+    /**
+     * Convert a local file to markdown.
+     */
+    convertFile(path: string): Promise<ConversionResult>;
+    /**
+     * Convert a URL to markdown.
+     */
+    convertUrl(url: string): Promise<ConversionResult>;
+    /**
+     * Convert a buffer with stream info to markdown.
+     */
+    convert(input: Buffer, streamInfo: StreamInfo): Promise<ConversionResult>;
+}

package/dist/mill.js

@@ -0,0 +1,123 @@
+import { readFileSync } from "node:fs";
+import { extname, basename } from "node:path";
+import { PdfConverter } from "./converters/pdf.js";
+import { DocxConverter } from "./converters/docx.js";
+import { PptxConverter } from "./converters/pptx.js";
+import { XlsxConverter } from "./converters/xlsx.js";
+import { EpubConverter } from "./converters/epub.js";
+import { IpynbConverter } from "./converters/ipynb.js";
+import { HtmlConverter } from "./converters/html.js";
+import { WikipediaConverter } from "./converters/wikipedia.js";
+import { RssConverter } from "./converters/rss.js";
+import { CsvConverter } from "./converters/csv.js";
+import { JsonConverter } from "./converters/json.js";
+import { YamlConverter } from "./converters/yaml.js";
+import { XmlConverter } from "./converters/xml.js";
+import { ZipConverter } from "./converters/zip.js";
+import { ImageConverter } from "./converters/image.js";
+import { AudioConverter } from "./converters/audio.js";
+import { PlainTextConverter } from "./converters/plain-text.js";
+export class Mill {
+    converters = [];
+    options;
+    constructor(options = {}) {
+        this.options = options;
+        // Order matters: specific formats first, generic last.
+        // URL-specific converters (Wikipedia) before generic HTML.
+        // ZIP converter gets a reference to other converters for recursive conversion.
+        const specific = [
+            new PdfConverter(),
+            new DocxConverter(),
+            new PptxConverter(),
+            new XlsxConverter(),
+            new EpubConverter(),
+            new IpynbConverter(),
+            new WikipediaConverter(),
+            new RssConverter(),
+            new CsvConverter(),
+            new JsonConverter(),
+            new YamlConverter(),
+            new ImageConverter(),
+            new AudioConverter(),
+        ];
+        const generic = [
+            new XmlConverter(),
+            new HtmlConverter(),
+        ];
+        // ZIP gets all other converters for recursive extraction
+        const allNonZip = [...specific, ...generic];
+        const zipConverter = new ZipConverter(allNonZip);
+        // Plain text is the ultimate catch-all
+        this.converters = [
+            ...specific,
+            zipConverter,
+            ...generic,
+            new PlainTextConverter(),
+        ];
+    }
+    /**
+     * Convert a local file to markdown.
+     */
+    async convertFile(path) {
+        const buffer = readFileSync(path);
+        const streamInfo = {
+            localPath: path,
+            extension: extname(path).toLowerCase(),
+            filename: basename(path),
+        };
+        return this.convert(buffer, streamInfo);
+    }
+    /**
+     * Convert a URL to markdown.
+     */
+    async convertUrl(url) {
+        const response = await fetch(url, {
+            headers: {
+                Accept: "text/markdown, text/html;q=0.9, text/plain;q=0.8, */*;q=0.1",
+                "User-Agent": "mill/0.1.0",
+            },
+        });
+        if (!response.ok) {
+            throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
+        }
+        const contentType = response.headers.get("content-type") || "";
+        const [mimetype] = contentType.split(";");
+        // Derive extension from URL path
+        const urlPath = new URL(url).pathname;
+        const ext = extname(urlPath).toLowerCase();
+        const buffer = Buffer.from(await response.arrayBuffer());
+        const streamInfo = {
+            url,
+            mimetype: mimetype.trim(),
+            extension: ext || undefined,
+            filename: basename(urlPath) || undefined,
+        };
+        return this.convert(buffer, streamInfo);
+    }
+    /**
+     * Convert a buffer with stream info to markdown.
+     */
+    async convert(input, streamInfo) {
+        const errors = [];
+        for (const converter of this.converters) {
+            if (!converter.accepts(streamInfo))
+                continue;
+            try {
+                return await converter.convert(input, streamInfo, this.options);
+            }
+            catch (err) {
+                errors.push({
+                    converter: converter.name,
+                    error: err instanceof Error ? err : new Error(String(err)),
+                });
+            }
+        }
+        if (errors.length > 0) {
+            const details = errors
+                .map((e) => `  ${e.converter}: ${e.error.message}`)
+                .join("\n");
+            throw new Error(`Conversion failed:\n${details}`);
+        }
+        throw new Error(`Unsupported format: ${streamInfo.extension || streamInfo.mimetype || "unknown"}`);
+    }
+}

package/dist/plugins/api.d.ts

@@ -0,0 +1,7 @@
+import type { MarkitPluginAPI, PluginDef, PluginFunction } from "./types.js";
+export declare function createPluginAPI(pluginId: string): {
+    api: MarkitPluginAPI;
+    resolve: () => PluginDef;
+};
+export declare function isPluginFunction(val: any): val is PluginFunction;
+export declare function resolvePluginExport(exported: PluginFunction | PluginDef, pluginId: string): PluginDef;

package/dist/plugins/api.js

@@ -0,0 +1,44 @@
+export function createPluginAPI(pluginId) {
+    let name = pluginId;
+    let version = "0.0.0";
+    const converters = [];
+    const providers = [];
+    const formats = [];
+    const api = {
+        setName(n) {
+            name = n;
+        },
+        setVersion(v) {
+            version = v;
+        },
+        registerConverter(converter, format) {
+            converters.push(converter);
+            if (format) {
+                formats.push(format);
+            }
+        },
+        registerProvider(provider) {
+            providers.push(provider);
+        },
+    };
+    function resolve() {
+        return { name, version, converters, providers, formats };
+    }
+    return { api, resolve };
+}
+export function isPluginFunction(val) {
+    return typeof val === "function";
+}
+export function resolvePluginExport(exported, pluginId) {
+    if (isPluginFunction(exported)) {
+        const { api, resolve } = createPluginAPI(pluginId);
+        exported(api);
+        return resolve();
+    }
+    if (exported &&
+        typeof exported === "object" &&
+        "converters" in exported) {
+        return exported;
+    }
+    throw new Error(`Invalid plugin export from "${pluginId}": expected a function or { name, converters } object`);
+}

package/dist/plugins/index.d.ts

@@ -0,0 +1,4 @@
+export type { MarkitPluginAPI, PluginFunction, PluginDef } from "./types.js";
+export { createPluginAPI, resolvePluginExport } from "./api.js";
+export { loadPluginFromPath, loadAllPlugins } from "./loader.js";
+export { installPlugin, removePlugin, listInstalled, parsePluginSource, } from "./installer.js";

package/dist/plugins/index.js

@@ -0,0 +1,3 @@
+export { createPluginAPI, resolvePluginExport } from "./api.js";
+export { loadPluginFromPath, loadAllPlugins } from "./loader.js";
+export { installPlugin, removePlugin, listInstalled, parsePluginSource, } from "./installer.js";

package/dist/plugins/installer.d.ts

@@ -0,0 +1,25 @@
+export interface PluginSource {
+    type: "npm" | "git" | "local";
+    name: string;
+    ref?: string;
+    url?: string;
+    path?: string;
+    subpath?: string;
+}
+export interface InstalledPlugin {
+    source: string;
+    path: string;
+    name?: string;
+}
+export declare function parsePluginSource(source: string): PluginSource;
+export declare function installPlugin(source: string): Promise<{
+    path: string;
+    name: string;
+}>;
+export declare function removePlugin(name: string): boolean;
+export declare function listInstalled(): Array<{
+    name: string;
+    type: string;
+    source: string;
+    path: string;
+}>;