markit-ai 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Michael Liv
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,333 @@
+# markit
+
+Convert anything to markdown. PDF, DOCX, PPTX, XLSX, HTML, EPUB, Jupyter, RSS, images, audio, URLs, and more. Pluggable converters, built-in LLM providers for image description and audio transcription. Works as a CLI and as a library.
+
+```bash
+npm install -g markit-ai
+```
+
+---
+
+## Quick Start
+
+```bash
+# Documents
+markit report.pdf
+markit document.docx
+markit slides.pptx
+
+# Data
+markit data.csv
+markit config.json
+markit schema.yaml
+
+# Web
+markit https://example.com/article
+markit https://en.wikipedia.org/wiki/Markdown
+
+# Media (via LLMs. set OPENAI_API_KEY or ANTHROPIC_API_KEY)
+markit photo.jpg                          # EXIF metadata + AI description
+markit recording.mp3                      # Audio metadata + transcription
+markit photo.jpg -p "Extract all text"    # Custom instructions
+
+# Write to file
+markit report.pdf -o report.md
+
+# Pipe it
+markit report.pdf | pbcopy
+markit data.xlsx -q | napkin create "Imported Data"
+```
+
+---
+
+## Supported Formats
+
+| Format | Extensions | How |
+|--------|-----------|-----|
+| PDF | `.pdf` | Text extraction via unpdf |
+| Word | `.docx` | mammoth → turndown, preserves headings/tables |
+| PowerPoint | `.pptx` | XML parsing, slides + notes + tables |
+| Excel | `.xlsx` | Each sheet → markdown table |
+| HTML | `.html` `.htm` | turndown, scripts/styles stripped |
+| EPUB | `.epub` | Spine-ordered chapters, metadata header |
+| Jupyter | `.ipynb` | Markdown cells + code + outputs |
+| RSS/Atom | `.rss` `.atom` `.xml` | Feed items with dates and content |
+| CSV/TSV | `.csv` `.tsv` | Markdown tables |
+| JSON | `.json` | Pretty-printed code block |
+| YAML | `.yaml` `.yml` | Code block |
+| XML/SVG | `.xml` `.svg` | Code block |
+| Images | `.jpg` `.png` `.gif` `.webp` | EXIF metadata + optional AI description |
+| Audio | `.mp3` `.wav` `.m4a` `.flac` | Metadata + optional AI transcription |
+| ZIP | `.zip` | Recursive. converts each file inside |
+| URLs | `http://` `https://` | Fetches with `Accept: text/markdown` |
+| Wikipedia | `*.wikipedia.org` | Main content extraction |
+| Code | `.py` `.ts` `.go` `.rs` ... | Fenced code block |
+| Plain text | `.txt` `.md` `.rst` `.log` | Pass-through |
+
+Need more? [Write a plugin.](#plugins)
+
+---
+
+## AI Features
+
+Images and audio get metadata extraction for free. For AI-powered descriptions and transcription, set an API key:
+
+```bash
+# OpenAI (default provider)
+export OPENAI_API_KEY=sk-...
+markit photo.jpg
+
+# Anthropic
+markit config set llm.provider anthropic
+export ANTHROPIC_API_KEY=sk-ant-...
+markit photo.jpg
+
+# Any OpenAI-compatible API (Ollama, Groq, Together, etc.)
+markit config set llm.apiBase http://localhost:11434/v1
+```
+
+Focus the AI on what matters:
+
+```bash
+markit receipt.jpg -p "List all line items with prices as a table"
+markit diagram.png -p "Describe the architecture and data flow"
+markit whiteboard.jpg -p "Extract all text verbatim"
+```
+
+---
+
+## Plugins
+
+Extend markit with new formats, override builtins, or add LLM providers.
+
+### Install
+
+```bash
+markit plugin install npm:markit-plugin-dwg
+markit plugin install git:github.com/user/markit-plugin-ocr
+markit plugin install ./my-plugin.ts
+markit plugin list
+markit plugin remove dwg
+```
+
+### Write a Plugin
+
+A plugin is a function that receives an API and registers converters and/or providers:
+
+```typescript
+import type { MarkitPluginAPI } from "markit-ai";
+
+export default function(api: MarkitPluginAPI) {
+  api.setName("cad");
+  api.setVersion("1.0.0");
+
+  // Register a converter for a new format
+  api.registerConverter(
+    {
+      name: "dwg",
+      accepts: (info) => [".dwg", ".dxf"].includes(info.extension || ""),
+      convert: async (input, info) => {
+        // Your conversion logic
+        return { markdown: "..." };
+      },
+    },
+    // Optional: declare the format so it shows in `markit formats`
+    { name: "AutoCAD", extensions: [".dwg", ".dxf"] },
+  );
+}
+```
+
+Plugin converters run **before** builtins. so you can override any format:
+
+```typescript
+export default function(api: MarkitPluginAPI) {
+  api.setName("better-pdf");
+
+  // This replaces the built-in PDF converter
+  api.registerConverter({
+    name: "pdf",
+    accepts: (info) => info.extension === ".pdf",
+    convert: async (input, info) => {
+      // Your superior PDF extraction
+      return { markdown: "..." };
+    },
+  });
+}
+```
+
+Plugins can also register LLM providers:
+
+```typescript
+api.registerProvider({
+  name: "gemini",
+  envKeys: ["GOOGLE_API_KEY"],
+  defaultBase: "https://generativelanguage.googleapis.com/v1beta",
+  defaultModel: "gemini-2.0-flash",
+  create: (config, prompt) => ({
+    describe: async (image, mime) => { /* ... */ },
+  }),
+});
+```
+
+---
+
+## For Agents
+
+Every command supports `--json`. Raw markdown with `-q`.
+
+```bash
+markit report.pdf --json       # Structured output for parsing
+markit report.pdf -q           # Raw markdown, nothing else
+markit onboard                 # Add instructions to CLAUDE.md
+```
+
+---
+
+## SDK
+
+markit is also a library:
+
+```typescript
+import { Markit } from "markit-ai";
+
+const markit = new Markit();
+const { markdown } = await markit.convertFile("report.pdf");
+const { markdown } = await markit.convertUrl("https://example.com");
+const { markdown } = await markit.convert(buffer, { extension: ".docx" });
+```
+
+With AI features. pass plain functions, use any provider:
+
+```typescript
+import OpenAI from "openai";
+import { Markit } from "markit-ai";
+
+const openai = new OpenAI();
+
+const markit = new Markit({
+  describe: async (image, mime) => {
+    const res = await openai.chat.completions.create({
+      model: "gpt-4.1-nano",
+      messages: [{ role: "user", content: [
+        { type: "text", text: "Describe this image." },
+        { type: "image_url", image_url: { url: `data:${mime};base64,${image.toString("base64")}` } },
+      ]}],
+    });
+    return res.choices[0].message.content ?? "";
+  },
+  transcribe: async (audio, mime) => {
+    const res = await openai.audio.transcriptions.create({
+      model: "gpt-4o-mini-transcribe",
+      file: new File([audio], "audio.mp3", { type: mime }),
+    });
+    return res.text;
+  },
+});
+```
+
+Mix providers. Claude for vision, OpenAI for audio, whatever:
+
+```typescript
+const markit = new Markit({
+  describe: async (image, mime) => {
+    const res = await anthropic.messages.create({
+      model: "claude-haiku-4-5",
+      messages: [{ role: "user", content: [
+        { type: "image", source: { type: "base64", media_type: mime, data: image.toString("base64") } },
+        { type: "text", text: "Describe this image." },
+      ]}],
+    });
+    return res.content[0].text;
+  },
+  transcribe: async (audio, mime) => { /* Whisper, Deepgram, AssemblyAI, ... */ },
+});
+```
+
+Or use the built-in providers. no SDK needed:
+
+```typescript
+import { Markit, createLlmFunctions, loadConfig } from "markit-ai";
+
+const config = loadConfig(); // reads .markit/config.json + env vars
+const markit = new Markit(createLlmFunctions(config));
+```
+
+With plugins:
+
+```typescript
+import { Markit, createLlmFunctions, loadConfig, loadAllPlugins } from "markit-ai";
+
+const config = loadConfig();
+const plugins = await loadAllPlugins();
+const markit = new Markit(createLlmFunctions(config), plugins);
+```
+
+---
+
+## Configuration
+
+```bash
+markit init                              # Create .markit/config.json
+markit config show                       # Show resolved settings
+markit config get llm.model              # Get a value
+markit config set llm.provider anthropic # Switch provider
+markit config set llm.apiKey sk-...      # Set a value
+```
+
+`.markit/config.json`:
+
+```json
+{
+  "llm": {
+    "provider": "openai",
+    "apiBase": "https://api.openai.com/v1",
+    "apiKey": "sk-...",
+    "model": "gpt-4.1-nano",
+    "transcriptionModel": "gpt-4o-mini-transcribe"
+  }
+}
+```
+
+Env vars override config. Each provider checks its own env vars first:
+
+| Provider | Env vars | Default model |
+|----------|---------|---------------|
+| `openai` | `OPENAI_API_KEY`, `MARKIT_API_KEY` | `gpt-4.1-nano` |
+| `anthropic` | `ANTHROPIC_API_KEY`, `MARKIT_API_KEY` | `claude-haiku-4-5` |
+
+---
+
+## CLI Reference
+
+```bash
+markit <source>                          # Convert file or URL
+markit <source> -o output.md             # Write to file
+markit <source> -p "instructions"        # Custom AI prompt
+markit <source> --json                   # JSON output
+markit <source> -q                       # Raw markdown only
+cat file.pdf | markit -                  # Read from stdin
+markit formats                           # List supported formats
+markit init                              # Create .markit/ config
+markit config show                       # Show settings
+markit config get <key>                  # Get config value
+markit config set <key> <value>          # Set config value
+markit plugin install <source>           # Install plugin
+markit plugin list                       # List plugins
+markit plugin remove <name>              # Remove plugin
+markit onboard                           # Add to CLAUDE.md
+```
+
+---
+
+## Development
+
+```bash
+bun install
+bun run dev -- report.pdf
+bun test
+bun run check
+```
+
+## License
+
+MIT

package/dist/commands/config.d.ts

@@ -0,0 +1,4 @@
+import type { OutputOptions } from "../utils/output.js";
+export declare function configShow(_args: string[], options: OutputOptions): Promise<void>;
+export declare function configGet(key: string, options: OutputOptions): Promise<void>;
+export declare function configSet(key: string, value: string | undefined, options: OutputOptions): Promise<void>;

package/dist/commands/config.js

@@ -0,0 +1,133 @@
+import { output, success, error, dim, bold } from "../utils/output.js";
+import { loadConfig, saveConfig, findConfigDir } from "../config.js";
+import { getProvider, listProviders } from "../providers/index.js";
+import { EXIT_ERROR, EXIT_USER_ERROR } from "../utils/exit-codes.js";
+export async function configShow(_args, options) {
+    const config = loadConfig();
+    const configDir = findConfigDir();
+    const providerName = config.llm?.provider || "openai";
+    const provider = getProvider(providerName);
+    output(options, {
+        json: () => ({
+            configDir,
+            config,
+            providers: listProviders(),
+        }),
+        human: () => {
+            console.log();
+            console.log(bold("Configuration"));
+            console.log();
+            if (configDir) {
+                console.log(`  ${dim("config:")} ${configDir}/config.json`);
+            }
+            else {
+                console.log(`  ${dim("config:")} none (run 'markit init')`);
+            }
+            console.log();
+            console.log(bold("LLM Settings"));
+            console.log();
+            console.log(`  ${dim("provider:")} ${providerName}`);
+            if (provider) {
+                // Resolve API key
+                const apiKey = provider.envKeys.reduce((found, key) => found || process.env[key], undefined) || config.llm?.apiKey;
+                const keySource = provider.envKeys.find((k) => process.env[k]) || (config.llm?.apiKey ? "config" : undefined);
+                console.log(`  ${dim("api key:")} ${apiKey ? `***${apiKey.slice(-4)} (${keySource})` : dim("not set")}`);
+                console.log(`  ${dim("api base:")} ${config.llm?.apiBase || provider.defaultBase}`);
+                console.log(`  ${dim("model:")} ${config.llm?.model || provider.defaultModel}`);
+                if (provider.defaultTranscriptionModel) {
+                    console.log(`  ${dim("transcription:")} ${config.llm?.transcriptionModel || provider.defaultTranscriptionModel}`);
+                }
+                console.log(`  ${dim("env vars:")} ${provider.envKeys.join(", ")}`);
+            }
+            else {
+                console.log(`  ${dim("(unknown provider)")}`);
+            }
+            console.log();
+            console.log(dim(`  Available providers: ${listProviders().join(", ")}`));
+            console.log();
+        },
+    });
+}
+export async function configGet(key, options) {
+    const config = loadConfig();
+    const value = getNestedValue(config, key);
+    if (value === undefined) {
+        output(options, {
+            json: () => ({ key, value: null }),
+            human: () => error(`Key '${key}' not found`),
+        });
+        process.exit(EXIT_USER_ERROR);
+    }
+    output(options, {
+        json: () => ({ key, value }),
+        quiet: () => console.log(String(value)),
+        human: () => console.log(String(value)),
+    });
+}
+export async function configSet(key, value, options) {
+    if (!findConfigDir()) {
+        output(options, {
+            json: () => ({ success: false, error: "No .markit/ directory. Run 'markit init'" }),
+            human: () => error("No .markit/ directory. Run 'markit init' first."),
+        });
+        process.exit(EXIT_ERROR);
+    }
+    // Secrets: read from stdin instead of args (avoids shell history)
+    const isSecret = key.toLowerCase().includes("key") || key.toLowerCase().includes("secret") || key.toLowerCase().includes("token");
+    let resolved;
+    if (isSecret && !value) {
+        // Prompt from stdin
+        if (process.stdin.isTTY) {
+            process.stderr.write(`Enter value for ${key}: `);
+        }
+        const chunks = [];
+        for await (const chunk of process.stdin) {
+            chunks.push(chunk);
+        }
+        resolved = Buffer.concat(chunks).toString("utf-8").trim();
+        if (!resolved) {
+            error("No value provided");
+            process.exit(EXIT_USER_ERROR);
+        }
+    }
+    else if (isSecret && value) {
+        // Warn if secret passed as arg
+        console.error(dim("  hint: secrets in args leak to shell history. Use: markit config set llm.apiKey < keyfile"));
+        resolved = value;
+    }
+    else if (value === undefined) {
+        error("Missing value. Usage: markit config set <key> <value>");
+        process.exit(EXIT_USER_ERROR);
+    }
+    else {
+        resolved = value;
+    }
+    const config = loadConfig();
+    let parsed = resolved;
+    if (resolved === "true")
+        parsed = true;
+    else if (resolved === "false")
+        parsed = false;
+    else if (/^\d+$/.test(resolved))
+        parsed = parseInt(resolved);
+    setNestedValue(config, key, parsed);
+    saveConfig(config);
+    output(options, {
+        json: () => ({ success: true, key, value: parsed }),
+        human: () => success(`${key} = ${JSON.stringify(parsed)}`),
+    });
+}
+function getNestedValue(obj, path) {
+    return path.split(".").reduce((o, k) => o?.[k], obj);
+}
+function setNestedValue(obj, path, value) {
+    const keys = path.split(".");
+    let current = obj;
+    for (let i = 0; i < keys.length - 1; i++) {
+        if (!current[keys[i]] || typeof current[keys[i]] !== "object") {
+            current[keys[i]] = {};
+        }
+        current = current[keys[i]];
+    }
+    current[keys[keys.length - 1]] = value;
+}

package/dist/commands/convert.d.ts

@@ -0,0 +1,5 @@
+import type { OutputOptions } from "../utils/output.js";
+export declare function convert(source: string, options: OutputOptions & {
+    output?: string;
+    prompt?: string;
+}): Promise<void>;

package/dist/commands/convert.js

@@ -0,0 +1,110 @@
+import { writeFileSync } from "node:fs";
+import { Markit } from "../markit.js";
+import { loadConfig } from "../config.js";
+import { createLlmFunctions } from "../providers/index.js";
+import { loadAllPlugins } from "../plugins/loader.js";
+import { registerProvider } from "../providers/index.js";
+import { output, success, error, dim } from "../utils/output.js";
+import { EXIT_ERROR, EXIT_UNSUPPORTED } from "../utils/exit-codes.js";
+async function readStdin() {
+    const chunks = [];
+    for await (const chunk of process.stdin) {
+        chunks.push(chunk);
+    }
+    return Buffer.concat(chunks);
+}
+export async function convert(source, options) {
+    const config = loadConfig();
+    const plugins = await loadAllPlugins();
+    // Register any providers from plugins
+    for (const plugin of plugins) {
+        for (const provider of plugin.providers) {
+            registerProvider(provider);
+        }
+    }
+    const llmFunctions = createLlmFunctions(config, options.prompt);
+    const markit = new Markit(llmFunctions, plugins);
+    try {
+        let result;
+        const isStdin = source === "-";
+        const isUrl = source.startsWith("http:") ||
+            source.startsWith("https:") ||
+            source.startsWith("file:");
+        if (isStdin) {
+            // Check if stdin is a TTY (no piped input)
+            if (process.stdin.isTTY) {
+                error("No input on stdin. Pipe a file: cat report.pdf | markit -");
+                process.exit(EXIT_ERROR);
+            }
+            const buffer = await readStdin();
+            result = await markit.convert(buffer, {});
+        }
+        else if (isUrl) {
+            // Progress hint for URL fetches (stderr so it doesn't pollute piped output)
+            if (!options.json && !options.quiet) {
+                process.stderr.write(`ℹ Fetching ${source}...\n`);
+            }
+            result = await markit.convertUrl(source);
+        }
+        else {
+            result = await markit.convertFile(source);
+        }
+        const label = isStdin ? "stdin" : source;
+        // Write to file or stdout
+        if (options.output) {
+            writeFileSync(options.output, result.markdown);
+            output(options, {
+                json: () => ({
+                    success: true,
+                    source: label,
+                    output: options.output,
+                    title: result.title,
+                    length: result.markdown.length,
+                }),
+                human: () => {
+                    success(`Converted → ${options.output}`);
+                    if (result.title)
+                        console.log(dim(`  title: ${result.title}`));
+                    console.log(dim(`  ${result.markdown.length} chars`));
+                },
+            });
+        }
+        else {
+            output(options, {
+                json: () => ({
+                    success: true,
+                    source: label,
+                    title: result.title,
+                    markdown: result.markdown,
+                }),
+                quiet: () => process.stdout.write(result.markdown),
+                human: () => process.stdout.write(result.markdown),
+            });
+        }
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        if (msg.includes("Unsupported format")) {
+            output(options, {
+                json: () => ({ success: false, error: msg }),
+                human: () => {
+                    error(msg);
+                    console.log(dim("  Run 'markit formats' to see supported formats."));
+                },
+            });
+            process.exit(EXIT_UNSUPPORTED);
+        }
+        if (msg.includes("ENOENT") || msg.includes("no such file")) {
+            output(options, {
+                json: () => ({ success: false, error: `File not found: ${source}` }),
+                human: () => error(`File not found: ${source}`),
+            });
+            process.exit(EXIT_ERROR);
+        }
+        output(options, {
+            json: () => ({ success: false, error: msg }),
+            human: () => error(msg),
+        });
+        process.exit(EXIT_ERROR);
+    }
+}

package/dist/commands/formats.d.ts

@@ -0,0 +1,2 @@
+import type { OutputOptions } from "../utils/output.js";
+export declare function formats(_args: string[], options: OutputOptions): Promise<void>;

package/dist/commands/formats.js

@@ -0,0 +1,56 @@
+import { output, bold, dim } from "../utils/output.js";
+import { loadAllPlugins } from "../plugins/loader.js";
+const BUILTIN_FORMATS = [
+    { name: "PDF", extensions: [".pdf"], builtin: true },
+    { name: "Word", extensions: [".docx"], builtin: true },
+    { name: "PowerPoint", extensions: [".pptx"], builtin: true },
+    { name: "Excel", extensions: [".xlsx"], builtin: true },
+    { name: "HTML", extensions: [".html", ".htm"], builtin: true },
+    { name: "EPUB", extensions: [".epub"], builtin: true },
+    { name: "Jupyter", extensions: [".ipynb"], builtin: true },
+    { name: "RSS/Atom", extensions: [".rss", ".atom", ".xml"], builtin: true },
+    { name: "CSV", extensions: [".csv", ".tsv"], builtin: true },
+    { name: "JSON", extensions: [".json"], builtin: true },
+    { name: "YAML", extensions: [".yaml", ".yml"], builtin: true },
+    { name: "XML", extensions: [".xml", ".svg"], builtin: true },
+    { name: "Images", extensions: [".jpg", ".png", ".gif", ".webp"], builtin: true },
+    { name: "Audio", extensions: [".mp3", ".wav", ".m4a", ".flac"], builtin: true },
+    { name: "ZIP", extensions: [".zip"], builtin: true },
+    { name: "Plain text", extensions: [".txt", ".md", ".rst", ".log"], builtin: true },
+    { name: "Code", extensions: [".py", ".js", ".ts", ".go", ".rs", "..."], builtin: true },
+    { name: "URLs", extensions: ["http://", "https://"], builtin: true },
+    { name: "Wikipedia", extensions: ["*.wikipedia.org"], builtin: true },
+];
+export async function formats(_args, options) {
+    const plugins = await loadAllPlugins();
+    const pluginFormats = plugins.flatMap((p) => p.formats.map((f) => ({
+        name: f.name,
+        extensions: f.extensions,
+        builtin: false,
+        plugin: p.name,
+    })));
+    const allFormats = [...BUILTIN_FORMATS, ...pluginFormats];
+    output(options, {
+        json: () => ({ formats: allFormats }),
+        human: () => {
+            console.log();
+            console.log(bold("Supported formats"));
+            console.log();
+            for (const fmt of BUILTIN_FORMATS) {
+                const exts = fmt.extensions.join(", ");
+                const note = fmt.dep ? dim(` (requires: npm i ${fmt.dep})`) : "";
+                console.log(`  ${fmt.name.padEnd(14)} ${dim(exts)}${note}`);
+            }
+            if (pluginFormats.length > 0) {
+                console.log();
+                console.log(bold("Plugin formats"));
+                console.log();
+                for (const fmt of pluginFormats) {
+                    const exts = fmt.extensions.join(", ");
+                    console.log(`  ${fmt.name.padEnd(14)} ${dim(exts)} ${dim(`(${fmt.plugin})`)}`);
+                }
+            }
+            console.log();
+        },
+    });
+}

package/dist/commands/init.d.ts

@@ -0,0 +1,2 @@
+import type { OutputOptions } from "../utils/output.js";
+export declare function init(_args: string[], options: OutputOptions): Promise<void>;