markit-ai 0.1.0

package/dist/plugins/installer.js

@@ -0,0 +1,176 @@
+import { execSync } from "node:child_process";
+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync, } from "node:fs";
+import { basename, join, resolve } from "node:path";
+import { findConfigDir } from "../config.js";
+const PLUGINS_FILE = "plugins.json";
+export function parsePluginSource(source) {
+    // npm:package@version
+    if (source.startsWith("npm:")) {
+        const rest = source.slice(4);
+        let name;
+        let ref;
+        if (rest.startsWith("@")) {
+            const lastAt = rest.lastIndexOf("@");
+            if (lastAt > 0 && lastAt !== rest.indexOf("@")) {
+                name = rest.slice(0, lastAt);
+                ref = rest.slice(lastAt + 1);
+            }
+            else {
+                name = rest;
+            }
+        }
+        else {
+            const atIdx = rest.indexOf("@");
+            if (atIdx > 0) {
+                name = rest.slice(0, atIdx);
+                ref = rest.slice(atIdx + 1);
+            }
+            else {
+                name = rest;
+            }
+        }
+        return { type: "npm", name, ref };
+    }
+    // git:url or https://...
+    if (source.startsWith("git:") ||
+        source.startsWith("https://") ||
+        source.startsWith("http://") ||
+        source.startsWith("ssh://")) {
+        let raw = source;
+        if (raw.startsWith("git:"))
+            raw = raw.slice(4);
+        let subpath;
+        const hashIdx = raw.indexOf("#");
+        if (hashIdx > 0) {
+            subpath = raw.slice(hashIdx + 1);
+            raw = raw.slice(0, hashIdx);
+        }
+        let ref;
+        const atIdx = raw.lastIndexOf("@");
+        if (atIdx > 0 && !raw.slice(atIdx).includes("/")) {
+            ref = raw.slice(atIdx + 1);
+            raw = raw.slice(0, atIdx);
+        }
+        let url = raw;
+        if (!url.startsWith("http://") &&
+            !url.startsWith("https://") &&
+            !url.startsWith("ssh://")) {
+            url = `https://${url}`;
+        }
+        if (!url.endsWith(".git"))
+            url += ".git";
+        const name = subpath ? basename(subpath) : basename(url, ".git");
+        return { type: "git", name, url, ref, subpath };
+    }
+    // Local path
+    const absPath = resolve(source);
+    const name = basename(absPath).replace(/\.(ts|js)$/, "");
+    return { type: "local", name, path: absPath };
+}
+function getPluginsDir() {
+    const configDir = findConfigDir();
+    const dir = configDir
+        ? join(configDir, "plugins")
+        : join(process.cwd(), ".markit", "plugins");
+    mkdirSync(dir, { recursive: true });
+    return dir;
+}
+function getPluginsJsonPath() {
+    const configDir = findConfigDir();
+    return configDir
+        ? join(configDir, PLUGINS_FILE)
+        : join(process.cwd(), ".markit", PLUGINS_FILE);
+}
+function readPluginsJson() {
+    const path = getPluginsJsonPath();
+    if (!existsSync(path))
+        return { plugins: [] };
+    return JSON.parse(readFileSync(path, "utf-8"));
+}
+function writePluginsJson(data) {
+    const path = getPluginsJsonPath();
+    mkdirSync(join(path, ".."), { recursive: true });
+    writeFileSync(path, `${JSON.stringify(data, null, 2)}\n`);
+}
+export async function installPlugin(source) {
+    const parsed = parsePluginSource(source);
+    const pluginsDir = getPluginsDir();
+    let installPath;
+    switch (parsed.type) {
+        case "npm": {
+            const npmDir = join(pluginsDir, "npm");
+            mkdirSync(npmDir, { recursive: true });
+            const spec = parsed.ref ? `${parsed.name}@${parsed.ref}` : parsed.name;
+            execSync(`npm install ${spec}`, { cwd: npmDir, stdio: "pipe" });
+            installPath = join(npmDir, "node_modules", parsed.name);
+            break;
+        }
+        case "git": {
+            const url = new URL(parsed.url);
+            const gitDir = join(pluginsDir, "git", url.hostname, url.pathname.replace(/\.git$/, ""));
+            if (existsSync(gitDir)) {
+                execSync("git pull", { cwd: gitDir, stdio: "pipe" });
+            }
+            else {
+                mkdirSync(join(gitDir, ".."), { recursive: true });
+                const refArg = parsed.ref ? `--branch ${parsed.ref}` : "";
+                execSync(`git clone ${refArg} ${parsed.url} ${gitDir}`, {
+                    stdio: "pipe",
+                });
+            }
+            if (existsSync(join(gitDir, "package.json"))) {
+                execSync("npm install", { cwd: gitDir, stdio: "pipe" });
+            }
+            installPath = parsed.subpath ? join(gitDir, parsed.subpath) : gitDir;
+            break;
+        }
+        case "local": {
+            if (!existsSync(parsed.path)) {
+                throw new Error(`Path does not exist: ${parsed.path}`);
+            }
+            installPath = parsed.path;
+            break;
+        }
+    }
+    const data = readPluginsJson();
+    const existing = data.plugins.findIndex((p) => p.source === source);
+    const entry = {
+        source,
+        path: installPath,
+        name: parsed.name,
+    };
+    if (existing >= 0) {
+        data.plugins[existing] = entry;
+    }
+    else {
+        data.plugins.push(entry);
+    }
+    writePluginsJson(data);
+    return { path: installPath, name: parsed.name };
+}
+export function removePlugin(name) {
+    const data = readPluginsJson();
+    const idx = data.plugins.findIndex((p) => p.name === name || p.source.includes(name));
+    if (idx < 0)
+        return false;
+    const plugin = data.plugins[idx];
+    const parsed = parsePluginSource(plugin.source);
+    if (parsed.type !== "local" && existsSync(plugin.path)) {
+        rmSync(plugin.path, { recursive: true, force: true });
+    }
+    data.plugins.splice(idx, 1);
+    writePluginsJson(data);
+    return true;
+}
+export function listInstalled() {
+    const data = readPluginsJson();
+    return data.plugins.map((p) => {
+        const parsed = parsePluginSource(p.source);
+        return {
+            name: p.name || parsed.name,
+            type: parsed.type,
+            source: p.source,
+            path: p.path,
+        };
+    });
+}

package/dist/plugins/loader.d.ts

@@ -0,0 +1,6 @@
+import type { PluginDef } from "./types.js";
+export declare function loadPluginFromPath(path: string): Promise<PluginDef>;
+/**
+ * Load all plugins from .markit/plugins.json
+ */
+export declare function loadAllPlugins(): Promise<PluginDef[]>;

package/dist/plugins/loader.js

@@ -0,0 +1,61 @@
+import { existsSync, readFileSync, statSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import { findConfigDir } from "../config.js";
+import { resolvePluginExport } from "./api.js";
+export async function loadPluginFromPath(path) {
+    let absPath = resolve(path);
+    // Directory → find entry point
+    if (existsSync(absPath) && statSync(absPath).isDirectory()) {
+        const candidates = [
+            join(absPath, "src", "index.ts"),
+            join(absPath, "src", "index.js"),
+            join(absPath, "index.ts"),
+            join(absPath, "index.js"),
+        ];
+        const pkgPath = join(absPath, "package.json");
+        if (existsSync(pkgPath)) {
+            try {
+                const pkg = JSON.parse(readFileSync(pkgPath, "utf-8"));
+                if (pkg.main)
+                    candidates.unshift(join(absPath, pkg.main));
+            }
+            catch { }
+        }
+        const found = candidates.find((c) => existsSync(c));
+        if (found) {
+            absPath = found;
+        }
+        else {
+            throw new Error(`No entry point found in ${absPath}`);
+        }
+    }
+    const mod = await import(pathToFileURL(absPath).href);
+    const pluginId = absPath.replace(/.*\//, "").replace(/\.(ts|js)$/, "");
+    return resolvePluginExport(mod.default, pluginId);
+}
+/**
+ * Load all plugins from .markit/plugins.json
+ */
+export async function loadAllPlugins() {
+    const plugins = [];
+    const configDir = findConfigDir();
+    if (!configDir)
+        return plugins;
+    const pluginsFile = join(configDir, "plugins.json");
+    if (!existsSync(pluginsFile))
+        return plugins;
+    try {
+        const data = JSON.parse(readFileSync(pluginsFile, "utf-8"));
+        const entries = data.plugins ?? [];
+        for (const entry of entries) {
+            const p = typeof entry === "string" ? entry : entry.path;
+            try {
+                plugins.push(await loadPluginFromPath(p));
+            }
+            catch { }
+        }
+    }
+    catch { }
+    return plugins;
+}

package/dist/plugins/types.d.ts

@@ -0,0 +1,25 @@
+import type { Converter } from "../types.js";
+import type { Provider } from "../providers/types.js";
+export interface FormatDef {
+    name: string;
+    extensions: string[];
+}
+export interface MarkitPluginAPI {
+    setName(name: string): void;
+    setVersion(version: string): void;
+    registerConverter(converter: Converter, format?: FormatDef): void;
+    registerProvider(provider: Provider): void;
+}
+export type PluginFunction = (api: MarkitPluginAPI) => void;
+export interface PluginDef {
+    name: string;
+    version: string;
+    converters: Converter[];
+    providers: Provider[];
+    formats: FormatDef[];
+}
+export interface InstalledPlugin {
+    source: string;
+    path: string;
+    name?: string;
+}

package/dist/plugins/types.js

@@ -0,0 +1 @@
+export {};

package/dist/providers/anthropic.d.ts

@@ -0,0 +1,2 @@
+import type { Provider } from "./types.js";
+export declare const anthropic: Provider;

package/dist/providers/anthropic.js

@@ -0,0 +1,47 @@
+export const anthropic = {
+    name: "anthropic",
+    envKeys: ["ANTHROPIC_API_KEY", "MARKIT_API_KEY"],
+    defaultBase: "https://api.anthropic.com",
+    defaultModel: "claude-haiku-4-5",
+    create(config, prompt) {
+        return {
+            describe: async (image, mimetype) => {
+                const res = await fetch(`${config.apiBase}/v1/messages`, {
+                    method: "POST",
+                    headers: {
+                        "Content-Type": "application/json",
+                        "x-api-key": config.apiKey,
+                        "anthropic-version": "2023-06-01",
+                    },
+                    body: JSON.stringify({
+                        model: config.model,
+                        max_tokens: 1024,
+                        messages: [
+                            {
+                                role: "user",
+                                content: [
+                                    {
+                                        type: "image",
+                                        source: {
+                                            type: "base64",
+                                            media_type: mimetype,
+                                            data: image.toString("base64"),
+                                        },
+                                    },
+                                    { type: "text", text: prompt },
+                                ],
+                            },
+                        ],
+                    }),
+                });
+                if (!res.ok) {
+                    const body = await res.text();
+                    throw new Error(`Anthropic API error ${res.status}: ${body}`);
+                }
+                const data = await res.json();
+                return data.content?.[0]?.text ?? "";
+            },
+            // Anthropic doesn't have a transcription API
+        };
+    },
+};

package/dist/providers/index.d.ts

@@ -0,0 +1,21 @@
+import type { Provider } from "./types.js";
+import type { MarkitOptions } from "../types.js";
+import type { MarkitConfig } from "../config.js";
+export type { Provider, ProviderConfig, ResolvedConfig } from "./types.js";
+/**
+ * Register a custom provider.
+ */
+export declare function registerProvider(provider: Provider): void;
+/**
+ * Get a provider by name.
+ */
+export declare function getProvider(name: string): Provider | undefined;
+/**
+ * List all registered provider names.
+ */
+export declare function listProviders(): string[];
+/**
+ * Build describe/transcribe functions from config.
+ * Resolves provider, API key, model, and base URL automatically.
+ */
+export declare function createLlmFunctions(config: MarkitConfig, prompt?: string): MarkitOptions;

package/dist/providers/index.js

@@ -0,0 +1,58 @@
+import { openai } from "./openai.js";
+import { anthropic } from "./anthropic.js";
+const providers = {
+    openai,
+    anthropic,
+};
+/**
+ * Register a custom provider.
+ */
+export function registerProvider(provider) {
+    providers[provider.name] = provider;
+}
+/**
+ * Get a provider by name.
+ */
+export function getProvider(name) {
+    return providers[name];
+}
+/**
+ * List all registered provider names.
+ */
+export function listProviders() {
+    return Object.keys(providers);
+}
+/**
+ * Resolve config + env vars into a ResolvedConfig for a provider.
+ */
+function resolve(provider, config) {
+    // API key: env vars (in provider priority order) > config file
+    const apiKey = provider.envKeys.reduce((found, key) => found || process.env[key], undefined) || config.llm?.apiKey;
+    if (!apiKey)
+        return null;
+    return {
+        apiKey,
+        apiBase: (config.llm?.apiBase || provider.defaultBase).replace(/\/+$/, ""),
+        model: process.env.MARKIT_MODEL || config.llm?.model || provider.defaultModel,
+        transcriptionModel: config.llm?.transcriptionModel || provider.defaultTranscriptionModel,
+    };
+}
+const BASE_PROMPT = "Describe this image in detail.";
+/**
+ * Build describe/transcribe functions from config.
+ * Resolves provider, API key, model, and base URL automatically.
+ */
+export function createLlmFunctions(config, prompt) {
+    const providerName = config.llm?.provider || "openai";
+    const provider = providers[providerName];
+    if (!provider) {
+        throw new Error(`Unknown provider '${providerName}'. Available: ${Object.keys(providers).join(", ")}`);
+    }
+    const resolved = resolve(provider, config);
+    if (!resolved)
+        return {};
+    const fullPrompt = prompt
+        ? `${BASE_PROMPT}\n\n${prompt}`
+        : BASE_PROMPT;
+    return provider.create(resolved, fullPrompt);
+}

package/dist/providers/openai.d.ts

@@ -0,0 +1,2 @@
+import type { Provider } from "./types.js";
+export declare const openai: Provider;

package/dist/providers/openai.js

@@ -0,0 +1,65 @@
+export const openai = {
+    name: "openai",
+    envKeys: ["OPENAI_API_KEY", "MARKIT_API_KEY"],
+    defaultBase: "https://api.openai.com/v1",
+    defaultModel: "gpt-4.1-nano",
+    defaultTranscriptionModel: "gpt-4o-mini-transcribe",
+    create(config, prompt) {
+        return {
+            describe: async (image, mimetype) => {
+                const res = await fetch(`${config.apiBase}/chat/completions`, {
+                    method: "POST",
+                    headers: {
+                        "Content-Type": "application/json",
+                        Authorization: `Bearer ${config.apiKey}`,
+                    },
+                    body: JSON.stringify({
+                        model: config.model,
+                        messages: [
+                            {
+                                role: "user",
+                                content: [
+                                    { type: "text", text: prompt },
+                                    { type: "image_url", image_url: { url: `data:${mimetype};base64,${image.toString("base64")}` } },
+                                ],
+                            },
+                        ],
+                        max_tokens: 1024,
+                    }),
+                });
+                if (!res.ok) {
+                    const body = await res.text();
+                    throw new Error(`OpenAI API error ${res.status}: ${body}`);
+                }
+                const data = await res.json();
+                return data.choices?.[0]?.message?.content ?? "";
+            },
+            transcribe: async (audio, mimetype) => {
+                const ext = mimeToExt(mimetype);
+                const file = new File([audio], `audio${ext}`, { type: mimetype });
+                const formData = new FormData();
+                formData.append("model", config.transcriptionModel || "gpt-4o-mini-transcribe");
+                formData.append("file", file);
+                const res = await fetch(`${config.apiBase}/audio/transcriptions`, {
+                    method: "POST",
+                    headers: { Authorization: `Bearer ${config.apiKey}` },
+                    body: formData,
+                });
+                if (!res.ok) {
+                    const body = await res.text();
+                    throw new Error(`Transcription API error ${res.status}: ${body}`);
+                }
+                const data = await res.json();
+                return data.text ?? "";
+            },
+        };
+    },
+};
+function mimeToExt(mime) {
+    const map = {
+        "audio/mpeg": ".mp3", "audio/wav": ".wav", "audio/mp4": ".m4a",
+        "video/mp4": ".mp4", "audio/ogg": ".ogg", "audio/flac": ".flac",
+        "audio/aac": ".aac",
+    };
+    return map[mime] || ".mp3";
+}

package/dist/providers/types.d.ts

@@ -0,0 +1,26 @@
+import type { MarkitOptions } from "../types.js";
+export interface ProviderConfig {
+    apiKey?: string;
+    apiBase?: string;
+    model?: string;
+    transcriptionModel?: string;
+}
+export interface Provider {
+    name: string;
+    /** Env vars to check for API key, in priority order */
+    envKeys: string[];
+    /** Default API base URL */
+    defaultBase: string;
+    /** Default model for image description */
+    defaultModel: string;
+    /** Default model for audio transcription (if supported) */
+    defaultTranscriptionModel?: string;
+    /** Build describe/transcribe functions from resolved config */
+    create(config: ResolvedConfig, prompt: string): MarkitOptions;
+}
+export interface ResolvedConfig {
+    apiKey: string;
+    apiBase: string;
+    model: string;
+    transcriptionModel?: string;
+}

package/dist/providers/types.js

@@ -0,0 +1 @@
+export {};

package/dist/types.d.ts

@@ -0,0 +1,28 @@
+export interface StreamInfo {
+    mimetype?: string;
+    extension?: string;
+    charset?: string;
+    filename?: string;
+    localPath?: string;
+    url?: string;
+}
+export interface ConversionResult {
+    markdown: string;
+    title?: string;
+}
+export interface MarkitOptions {
+    /** Describe an image, return markdown. Receives raw bytes and mimetype. */
+    describe?: (image: Buffer, mimetype: string) => Promise<string>;
+    /** Transcribe audio, return text. Receives raw bytes and mimetype. */
+    transcribe?: (audio: Buffer, mimetype: string) => Promise<string>;
+    /** Extra instructions appended to the image description prompt. */
+    prompt?: string;
+}
+export interface Converter {
+    /** Human-readable name for error messages */
+    name: string;
+    /** Quick check: can this converter handle the given stream? */
+    accepts(streamInfo: StreamInfo): boolean;
+    /** Convert the source to markdown */
+    convert(input: Buffer, streamInfo: StreamInfo, options?: MarkitOptions): Promise<ConversionResult>;
+}

package/dist/types.js

@@ -0,0 +1 @@
+export {};

package/dist/utils/exit-codes.d.ts

@@ -0,0 +1,4 @@
+export declare const EXIT_SUCCESS = 0;
+export declare const EXIT_ERROR = 1;
+export declare const EXIT_USER_ERROR = 2;
+export declare const EXIT_UNSUPPORTED = 3;

package/dist/utils/exit-codes.js

@@ -0,0 +1,4 @@
+export const EXIT_SUCCESS = 0;
+export const EXIT_ERROR = 1;
+export const EXIT_USER_ERROR = 2; // Invalid input, missing args
+export const EXIT_UNSUPPORTED = 3; // Unsupported format

package/dist/utils/output.d.ts

@@ -0,0 +1,22 @@
+export interface OutputOptions {
+    json?: boolean;
+    quiet?: boolean;
+}
+export declare const success: (msg: string) => void;
+export declare const info: (msg: string) => void;
+export declare const warn: (msg: string) => void;
+export declare const error: (msg: string) => void;
+export declare const bold: (s: string) => string;
+export declare const dim: (s: string) => string;
+export declare const cmd: (s: string) => string;
+export declare const bullet: (msg: string) => void;
+export declare const bulletDim: (msg: string) => void;
+export declare const hint: (msg: string) => void;
+export declare const nextStep: (command: string) => void;
+export declare const header: (title: string) => void;
+export declare function jsonOutput(data: object): void;
+export declare function output(options: OutputOptions, handlers: {
+    json?: () => object;
+    quiet?: () => void;
+    human: () => void;
+}): void;

package/dist/utils/output.js

@@ -0,0 +1,31 @@
+import chalk from "chalk";
+export const success = (msg) => console.log(chalk.green("✓"), msg);
+export const info = (msg) => console.log(chalk.blue("ℹ"), msg);
+export const warn = (msg) => console.log(chalk.yellow("⚠"), msg);
+export const error = (msg) => console.error(chalk.red("✗"), msg);
+export const bold = (s) => chalk.bold(s);
+export const dim = (s) => chalk.dim(s);
+export const cmd = (s) => chalk.cyan(s);
+export const bullet = (msg) => console.log(chalk.green("●"), msg);
+export const bulletDim = (msg) => console.log(chalk.dim("●"), msg);
+export const hint = (msg) => console.log(chalk.dim(`  ${msg}`));
+export const nextStep = (command) => console.log(`  ${cmd(command)}`);
+export const header = (title) => {
+    console.log();
+    console.log(chalk.bold(title));
+    console.log();
+};
+export function jsonOutput(data) {
+    console.log(JSON.stringify(data, null, 2));
+}
+export function output(options, handlers) {
+    if (options.json && handlers.json) {
+        jsonOutput(handlers.json());
+    }
+    else if (options.quiet && handlers.quiet) {
+        handlers.quiet();
+    }
+    else {
+        handlers.human();
+    }
+}

package/package.json

@@ -0,0 +1,70 @@
+{
+  "name": "markit-ai",
+  "version": "0.1.0",
+  "description": "Convert anything to markdown. PDFs, DOCX, HTML, URLs — everything gets milled.",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "bin": {
+    "markit": "./dist/main.js"
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "prepublishOnly": "npm run build",
+    "dev": "bun run src/main.ts",
+    "build:bun": "bun build --compile src/main.ts --outfile mill",
+    "build:bun-linux": "bun build --compile --target=bun-linux-x64 src/main.ts --outfile mill-linux-x64",
+    "build:bun-mac-arm": "bun build --compile --target=bun-darwin-arm64 src/main.ts --outfile mill-darwin-arm64",
+    "build:bun-mac-x64": "bun build --compile --target=bun-darwin-x64 src/main.ts --outfile mill-darwin-x64",
+    "test": "bun test",
+    "format": "bunx biome format --write src/",
+    "lint": "bunx biome lint src/",
+    "check": "bunx biome check src/"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Michaelliv/markit.git"
+  },
+  "homepage": "https://github.com/Michaelliv/markit#readme",
+  "bugs": {
+    "url": "https://github.com/Michaelliv/markit/issues"
+  },
+  "keywords": [
+    "markdown",
+    "pdf",
+    "docx",
+    "html",
+    "converter",
+    "ai",
+    "llm",
+    "agents"
+  ],
+  "license": "MIT",
+  "devDependencies": {
+    "@biomejs/biome": "^2.3.14",
+    "@types/jszip": "^3.4.1",
+    "@types/turndown": "^5.0.5",
+    "typescript": "^5.8.0"
+  },
+  "dependencies": {
+    "chalk": "^5.6.2",
+    "commander": "^14.0.3",
+    "exifr": "^7.1.3",
+    "fast-xml-parser": "^5.5.9",
+    "jszip": "^3.10.1",
+    "mammoth": "^1.9.0",
+    "music-metadata": "^11.12.3",
+    "rss-parser": "^3.13.0",
+    "turndown": "^7.2.0",
+    "unpdf": "^1.4.0"
+  }
+}