markit-ai 0.1.0

package/dist/commands/init.js

@@ -0,0 +1,29 @@
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { output, success, hint, cmd } from "../utils/output.js";
+const DATA_DIR = ".markit";
+export async function init(_args, options) {
+    const root = join(process.cwd(), DATA_DIR);
+    if (existsSync(root)) {
+        output(options, {
+            json: () => ({ success: true, path: root, message: "already_exists" }),
+            human: () => success(`.markit/ already exists`),
+        });
+        return;
+    }
+    mkdirSync(root, { recursive: true });
+    const config = {
+        llm: {},
+    };
+    writeFileSync(join(root, "config.json"), `${JSON.stringify(config, null, 2)}\n`);
+    output(options, {
+        json: () => ({ success: true, path: root }),
+        human: () => {
+            success(`Created .markit/ in ${process.cwd()}`);
+            hint("Set your API key for image/audio AI features:");
+            console.log(`  ${cmd("export OPENAI_API_KEY=sk-...")}`);
+            hint("Or configure directly:");
+            console.log(`  ${cmd("markit config set llm.apiKey sk-...")}`);
+        },
+    });
+}

package/dist/commands/onboard.d.ts

@@ -0,0 +1,2 @@
+import type { OutputOptions } from "../utils/output.js";
+export declare function onboard(_args: string[], options: OutputOptions): Promise<void>;

package/dist/commands/onboard.js

@@ -0,0 +1,61 @@
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { output, success } from "../utils/output.js";
+const INSTRUCTIONS = `
+<mill>
+Use \`markit\` to convert files and URLs to markdown. Supports PDF, DOCX, HTML, XLSX, CSV, JSON, and plain text.
+
+<commands>
+- \`markit <file-or-url>\` - convert to markdown (stdout)
+- \`markit <file-or-url> -o output.md\` - convert to file
+- \`markit formats\` - list supported formats
+</commands>
+
+<rules>
+- Use \`--json\` flag to get structured output for parsing
+- Use \`-q\` to get raw markdown without formatting
+- Pipe output directly: \`markit report.pdf | other-tool\`
+</rules>
+</markit>
+`.trim();
+const MARKER = "<mill>";
+export async function onboard(_args, options) {
+    const cwd = process.cwd();
+    const claudeMd = join(cwd, "CLAUDE.md");
+    const agentsMd = join(cwd, "AGENTS.md");
+    let targetFile;
+    if (existsSync(claudeMd)) {
+        targetFile = claudeMd;
+    }
+    else if (existsSync(agentsMd)) {
+        targetFile = agentsMd;
+    }
+    else {
+        targetFile = claudeMd;
+    }
+    let existingContent = "";
+    if (existsSync(targetFile)) {
+        existingContent = readFileSync(targetFile, "utf-8");
+    }
+    if (existingContent.includes(MARKER)) {
+        output(options, {
+            json: () => ({
+                success: true,
+                file: targetFile,
+                message: "already_onboarded",
+            }),
+            human: () => success(`Already onboarded (${targetFile})`),
+        });
+        return;
+    }
+    if (existingContent) {
+        writeFileSync(targetFile, `${existingContent.trimEnd()}\n\n${INSTRUCTIONS}\n`);
+    }
+    else {
+        writeFileSync(targetFile, `${INSTRUCTIONS}\n`);
+    }
+    output(options, {
+        json: () => ({ success: true, file: targetFile }),
+        human: () => success(`Added markit instructions to ${targetFile}`),
+    });
+}

package/dist/commands/plugin.d.ts

@@ -0,0 +1,4 @@
+import type { OutputOptions } from "../utils/output.js";
+export declare function pluginInstall(source: string, options: OutputOptions): Promise<void>;
+export declare function pluginRemove(name: string, options: OutputOptions): Promise<void>;
+export declare function pluginList(options: OutputOptions): Promise<void>;

package/dist/commands/plugin.js

@@ -0,0 +1,58 @@
+import { installPlugin, removePlugin, listInstalled } from "../plugins/installer.js";
+import { output, success, error, dim, bold } from "../utils/output.js";
+import { EXIT_ERROR } from "../utils/exit-codes.js";
+export async function pluginInstall(source, options) {
+    try {
+        const result = await installPlugin(source);
+        output(options, {
+            json: () => ({ success: true, ...result }),
+            human: () => {
+                success(`Installed ${result.name}`);
+                console.log(dim(`  ${result.path}`));
+            },
+        });
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        output(options, {
+            json: () => ({ success: false, error: msg }),
+            human: () => error(msg),
+        });
+        process.exit(EXIT_ERROR);
+    }
+}
+export async function pluginRemove(name, options) {
+    const removed = removePlugin(name);
+    output(options, {
+        json: () => ({ success: removed, name }),
+        human: () => {
+            if (removed) {
+                success(`Removed ${name}`);
+            }
+            else {
+                error(`Plugin '${name}' not found`);
+            }
+        },
+    });
+    if (!removed)
+        process.exit(EXIT_ERROR);
+}
+export async function pluginList(options) {
+    const plugins = listInstalled();
+    output(options, {
+        json: () => ({ plugins }),
+        human: () => {
+            if (plugins.length === 0) {
+                console.log(dim("  No plugins installed"));
+                return;
+            }
+            console.log();
+            console.log(bold("Installed plugins"));
+            console.log();
+            for (const p of plugins) {
+                console.log(`  ${p.name.padEnd(20)} ${dim(p.type)} ${dim(p.source)}`);
+            }
+            console.log();
+        },
+    });
+}

package/dist/config.d.ts

@@ -0,0 +1,26 @@
+export interface MarkitConfig {
+    llm?: {
+        /** Provider name: "openai" (default), "anthropic", or any registered provider */
+        provider?: string;
+        /** API base URL (overrides provider default) */
+        apiBase?: string;
+        /** API key — prefer env vars over storing here */
+        apiKey?: string;
+        /** Model override (overrides provider default) */
+        model?: string;
+        /** Transcription model override */
+        transcriptionModel?: string;
+    };
+}
+/**
+ * Walk up from cwd to find .markit/ directory.
+ */
+export declare function findConfigDir(): string | null;
+/**
+ * Load config from .markit/config.json.
+ */
+export declare function loadConfig(): MarkitConfig;
+/**
+ * Save config to .markit/config.json. Creates .markit/ if needed.
+ */
+export declare function saveConfig(config: MarkitConfig): void;

package/dist/config.js

@@ -0,0 +1,42 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+const DATA_DIR = ".markit";
+const CONFIG_FILE = "config.json";
+/**
+ * Walk up from cwd to find .markit/ directory.
+ */
+export function findConfigDir() {
+    let dir = process.cwd();
+    while (true) {
+        if (existsSync(join(dir, DATA_DIR))) {
+            return join(dir, DATA_DIR);
+        }
+        const parent = join(dir, "..");
+        if (parent === dir)
+            return null;
+        dir = parent;
+    }
+}
+/**
+ * Load config from .markit/config.json.
+ */
+export function loadConfig() {
+    const configDir = findConfigDir();
+    if (!configDir)
+        return {};
+    const configFile = join(configDir, CONFIG_FILE);
+    if (!existsSync(configFile))
+        return {};
+    const raw = JSON.parse(readFileSync(configFile, "utf-8"));
+    return { ...raw, llm: { ...raw.llm } };
+}
+/**
+ * Save config to .markit/config.json. Creates .markit/ if needed.
+ */
+export function saveConfig(config) {
+    const configDir = findConfigDir();
+    const dir = configDir || join(process.cwd(), DATA_DIR);
+    if (!existsSync(dir))
+        mkdirSync(dir, { recursive: true });
+    writeFileSync(join(dir, CONFIG_FILE), `${JSON.stringify(config, null, 2)}\n`);
+}

package/dist/converters/audio.d.ts

@@ -0,0 +1,7 @@
+import type { Converter, ConversionResult, StreamInfo, MarkitOptions } from "../types.js";
+export declare class AudioConverter implements Converter {
+    name: string;
+    accepts(streamInfo: StreamInfo): boolean;
+    convert(input: Buffer, streamInfo: StreamInfo, options?: MarkitOptions): Promise<ConversionResult>;
+    private formatDuration;
+}

package/dist/converters/audio.js

@@ -0,0 +1,87 @@
+const EXTENSIONS = [".mp3", ".wav", ".m4a", ".mp4", ".ogg", ".flac", ".aac", ".wma"];
+const MIMETYPES = ["audio/", "video/mp4"];
+export class AudioConverter {
+    name = "audio";
+    accepts(streamInfo) {
+        if (streamInfo.extension && EXTENSIONS.includes(streamInfo.extension))
+            return true;
+        if (streamInfo.mimetype && MIMETYPES.some((m) => streamInfo.mimetype.startsWith(m)))
+            return true;
+        return false;
+    }
+    async convert(input, streamInfo, options) {
+        const sections = [];
+        // Extract audio metadata
+        try {
+            const mm = await import("music-metadata");
+            const metadata = await mm.parseBuffer(new Uint8Array(input), {
+                mimeType: streamInfo.mimetype,
+                size: input.length,
+            });
+            const { common, format } = metadata;
+            sections.push("## Metadata\n");
+            const fields = {
+                Title: common.title,
+                Artist: common.artist,
+                Album: common.album,
+                Genre: common.genre?.join(", "),
+                Track: common.track?.no ? `${common.track.no}${common.track.of ? ` of ${common.track.of}` : ""}` : undefined,
+                Year: common.year ? String(common.year) : undefined,
+                Duration: format.duration
+                    ? this.formatDuration(format.duration)
+                    : undefined,
+                Format: format.codec || format.container,
+                SampleRate: format.sampleRate ? `${format.sampleRate} Hz` : undefined,
+                Channels: format.numberOfChannels
+                    ? String(format.numberOfChannels)
+                    : undefined,
+                Bitrate: format.bitrate
+                    ? `${Math.round(format.bitrate / 1000)} kbps`
+                    : undefined,
+            };
+            for (const [key, value] of Object.entries(fields)) {
+                if (value)
+                    sections.push(`${key}: ${value}`);
+            }
+            if (common.lyrics?.length) {
+                sections.push(`\n## Lyrics\n\n${common.lyrics.join("\n")}`);
+            }
+        }
+        catch {
+            // Metadata parsing failed
+        }
+        // AI transcription
+        if (options?.transcribe) {
+            try {
+                const mimetype = streamInfo.mimetype || guessMimetype(streamInfo.extension);
+                const transcript = await options.transcribe(input, mimetype);
+                if (transcript) {
+                    sections.push(`\n## Transcript\n\n${transcript}`);
+                }
+            }
+            catch {
+                // Transcription failed
+            }
+        }
+        if (sections.length === 0) {
+            return { markdown: `*[audio: ${streamInfo.filename || "unknown"}]*` };
+        }
+        return { markdown: sections.join("\n").trim() };
+    }
+    formatDuration(seconds) {
+        const h = Math.floor(seconds / 3600);
+        const m = Math.floor((seconds % 3600) / 60);
+        const s = Math.round(seconds % 60);
+        if (h > 0)
+            return `${h}:${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")}`;
+        return `${m}:${String(s).padStart(2, "0")}`;
+    }
+}
+function guessMimetype(ext) {
+    const map = {
+        ".mp3": "audio/mpeg", ".wav": "audio/wav", ".m4a": "audio/mp4",
+        ".mp4": "video/mp4", ".ogg": "audio/ogg", ".flac": "audio/flac",
+        ".aac": "audio/aac", ".wma": "audio/x-ms-wma",
+    };
+    return map[ext || ""] || "audio/mpeg";
+}

package/dist/converters/csv.d.ts

@@ -0,0 +1,7 @@
+import type { Converter, ConversionResult, StreamInfo } from "../types.js";
+export declare class CsvConverter implements Converter {
+    name: string;
+    accepts(streamInfo: StreamInfo): boolean;
+    convert(input: Buffer, streamInfo: StreamInfo): Promise<ConversionResult>;
+    private parseRows;
+}

package/dist/converters/csv.js

@@ -0,0 +1,83 @@
+const EXTENSIONS = [".csv", ".tsv"];
+const MIMETYPES = ["text/csv", "text/tab-separated-values"];
+export class CsvConverter {
+    name = "csv";
+    accepts(streamInfo) {
+        if (streamInfo.extension && EXTENSIONS.includes(streamInfo.extension)) {
+            return true;
+        }
+        if (streamInfo.mimetype &&
+            MIMETYPES.some((m) => streamInfo.mimetype.startsWith(m))) {
+            return true;
+        }
+        return false;
+    }
+    async convert(input, streamInfo) {
+        const text = new TextDecoder(streamInfo.charset || "utf-8").decode(input);
+        const delimiter = streamInfo.extension === ".tsv" ? "\t" : ",";
+        const rows = this.parseRows(text, delimiter);
+        if (rows.length === 0) {
+            return { markdown: "" };
+        }
+        const [header, ...body] = rows;
+        const lines = [];
+        // Header
+        lines.push(`| ${header.join(" | ")} |`);
+        lines.push(`| ${header.map(() => "---").join(" | ")} |`);
+        // Body
+        for (const row of body) {
+            // Pad row to match header length
+            while (row.length < header.length)
+                row.push("");
+            lines.push(`| ${row.join(" | ")} |`);
+        }
+        return { markdown: lines.join("\n") };
+    }
+    parseRows(text, delimiter) {
+        const rows = [];
+        let current = [];
+        let cell = "";
+        let inQuotes = false;
+        for (let i = 0; i < text.length; i++) {
+            const ch = text[i];
+            if (inQuotes) {
+                if (ch === '"' && text[i + 1] === '"') {
+                    cell += '"';
+                    i++;
+                }
+                else if (ch === '"') {
+                    inQuotes = false;
+                }
+                else {
+                    cell += ch;
+                }
+            }
+            else if (ch === '"') {
+                inQuotes = true;
+            }
+            else if (ch === delimiter) {
+                current.push(cell.trim());
+                cell = "";
+            }
+            else if (ch === "\n" || (ch === "\r" && text[i + 1] === "\n")) {
+                current.push(cell.trim());
+                if (current.some((c) => c.length > 0))
+                    rows.push(current);
+                current = [];
+                cell = "";
+                if (ch === "\r")
+                    i++;
+            }
+            else {
+                cell += ch;
+            }
+        }
+        // Last row
+        if (cell || current.length > 0) {
+            current.push(cell.trim());
+            if (current.some((c) => c.length > 0))
+                rows.push(current);
+        }
+        return rows;
+    }
+}

package/dist/converters/docx.d.ts

@@ -0,0 +1,6 @@
+import type { Converter, ConversionResult, StreamInfo } from "../types.js";
+export declare class DocxConverter implements Converter {
+    name: string;
+    accepts(streamInfo: StreamInfo): boolean;
+    convert(input: Buffer, _streamInfo: StreamInfo): Promise<ConversionResult>;
+}

package/dist/converters/docx.js

@@ -0,0 +1,28 @@
+import mammoth from "mammoth";
+import TurndownService from "turndown";
+const EXTENSIONS = [".docx"];
+const MIMETYPES = [
+    "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+];
+export class DocxConverter {
+    name = "docx";
+    accepts(streamInfo) {
+        if (streamInfo.extension && EXTENSIONS.includes(streamInfo.extension)) {
+            return true;
+        }
+        if (streamInfo.mimetype &&
+            MIMETYPES.some((m) => streamInfo.mimetype.startsWith(m))) {
+            return true;
+        }
+        return false;
+    }
+    async convert(input, _streamInfo) {
+        const { value: html } = await mammoth.convertToHtml({ buffer: input });
+        const turndown = new TurndownService({
+            headingStyle: "atx",
+            codeBlockStyle: "fenced",
+        });
+        const markdown = turndown.turndown(html);
+        return { markdown: markdown.trim() };
+    }
+}

package/dist/converters/epub.d.ts

@@ -0,0 +1,8 @@
+import type { Converter, ConversionResult, StreamInfo } from "../types.js";
+export declare class EpubConverter implements Converter {
+    name: string;
+    accepts(streamInfo: StreamInfo): boolean;
+    convert(input: Buffer, _streamInfo: StreamInfo): Promise<ConversionResult>;
+    private getText;
+    private getTextArray;
+}

package/dist/converters/epub.js

@@ -0,0 +1,110 @@
+import JSZip from "jszip";
+import { XMLParser } from "fast-xml-parser";
+import TurndownService from "turndown";
+const EXTENSIONS = [".epub"];
+const MIMETYPES = ["application/epub", "application/epub+zip", "application/x-epub+zip"];
+export class EpubConverter {
+    name = "epub";
+    accepts(streamInfo) {
+        if (streamInfo.extension && EXTENSIONS.includes(streamInfo.extension))
+            return true;
+        if (streamInfo.mimetype && MIMETYPES.some((m) => streamInfo.mimetype.startsWith(m)))
+            return true;
+        return false;
+    }
+    async convert(input, _streamInfo) {
+        const zip = await JSZip.loadAsync(input);
+        const parser = new XMLParser({
+            ignoreAttributes: false,
+            attributeNamePrefix: "@_",
+            textNodeName: "#text",
+        });
+        // Find content.opf path from container.xml
+        const containerXml = await zip.file("META-INF/container.xml")?.async("string");
+        if (!containerXml)
+            throw new Error("Invalid EPUB: missing container.xml");
+        const container = parser.parse(containerXml);
+        const rootfile = container.container?.rootfiles?.rootfile;
+        const opfPath = Array.isArray(rootfile)
+            ? rootfile[0]["@_full-path"]
+            : rootfile?.["@_full-path"];
+        if (!opfPath)
+            throw new Error("Invalid EPUB: missing rootfile path");
+        // Parse content.opf
+        const opfXml = await zip.file(opfPath)?.async("string");
+        if (!opfXml)
+            throw new Error("Invalid EPUB: missing content.opf");
+        const opf = parser.parse(opfXml);
+        // Extract metadata
+        const meta = opf.package?.metadata ?? {};
+        const metadata = {
+            title: this.getText(meta["dc:title"]),
+            authors: this.getTextArray(meta["dc:creator"]).join(", ") || undefined,
+            language: this.getText(meta["dc:language"]),
+            publisher: this.getText(meta["dc:publisher"]),
+            date: this.getText(meta["dc:date"]),
+            description: this.getText(meta["dc:description"]),
+        };
+        // Build manifest map (id → href)
+        const manifestItems = opf.package?.manifest?.item;
+        const itemList = Array.isArray(manifestItems) ? manifestItems : manifestItems ? [manifestItems] : [];
+        const manifest = new Map();
+        for (const item of itemList) {
+            manifest.set(item["@_id"], item["@_href"]);
+        }
+        // Get spine order
+        const spineItems = opf.package?.spine?.itemref;
+        const spineList = Array.isArray(spineItems) ? spineItems : spineItems ? [spineItems] : [];
+        const spineOrder = spineList.map((s) => s["@_idref"]);
+        // Resolve file paths
+        const basePath = opfPath.includes("/") ? opfPath.substring(0, opfPath.lastIndexOf("/")) : "";
+        const turndown = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced" });
+        const sections = [];
+        // Add metadata header
+        const metaLines = [];
+        for (const [key, value] of Object.entries(metadata)) {
+            if (value)
+                metaLines.push(`**${key.charAt(0).toUpperCase() + key.slice(1)}:** ${value}`);
+        }
+        if (metaLines.length > 0)
+            sections.push(metaLines.join("\n"));
+        // Convert spine files
+        for (const idref of spineOrder) {
+            const href = manifest.get(idref);
+            if (!href)
+                continue;
+            const filePath = basePath ? `${basePath}/${href}` : href;
+            const html = await zip.file(filePath)?.async("string");
+            if (!html)
+                continue;
+            // Strip script/style, convert to markdown
+            const cleaned = html
+                .replace(/<script[\s\S]*?<\/script>/gi, "")
+                .replace(/<style[\s\S]*?<\/style>/gi, "");
+            const md = turndown.turndown(cleaned).trim();
+            if (md)
+                sections.push(md);
+        }
+        return {
+            markdown: sections.join("\n\n").trim(),
+            title: metadata.title,
+        };
+    }
+    getText(node) {
+        if (!node)
+            return undefined;
+        if (typeof node === "string")
+            return node;
+        if (node["#text"])
+            return String(node["#text"]);
+        if (Array.isArray(node))
+            return this.getText(node[0]);
+        return undefined;
+    }
+    getTextArray(node) {
+        if (!node)
+            return [];
+        const list = Array.isArray(node) ? node : [node];
+        return list.map((n) => this.getText(n)).filter(Boolean);
+    }
+}

package/dist/converters/html.d.ts

@@ -0,0 +1,6 @@
+import type { Converter, ConversionResult, StreamInfo } from "../types.js";
+export declare class HtmlConverter implements Converter {
+    name: string;
+    accepts(streamInfo: StreamInfo): boolean;
+    convert(input: Buffer, streamInfo: StreamInfo): Promise<ConversionResult>;
+}

package/dist/converters/html.js

@@ -0,0 +1,33 @@
+import TurndownService from "turndown";
+const EXTENSIONS = [".html", ".htm"];
+const MIMETYPES = ["text/html", "application/xhtml"];
+export class HtmlConverter {
+    name = "html";
+    accepts(streamInfo) {
+        if (streamInfo.extension && EXTENSIONS.includes(streamInfo.extension)) {
+            return true;
+        }
+        if (streamInfo.mimetype &&
+            MIMETYPES.some((m) => streamInfo.mimetype.startsWith(m))) {
+            return true;
+        }
+        return false;
+    }
+    async convert(input, streamInfo) {
+        const charset = streamInfo.charset || "utf-8";
+        const html = new TextDecoder(charset).decode(input);
+        const turndown = new TurndownService({
+            headingStyle: "atx",
+            codeBlockStyle: "fenced",
+        });
+        // Remove script and style tags before converting
+        const cleaned = html
+            .replace(/<script[\s\S]*?<\/script>/gi, "")
+            .replace(/<style[\s\S]*?<\/style>/gi, "");
+        const markdown = turndown.turndown(cleaned);
+        // Try to extract title
+        const titleMatch = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+        const title = titleMatch ? titleMatch[1].trim() : undefined;
+        return { markdown: markdown.trim(), title };
+    }
+}

package/dist/converters/image.d.ts

@@ -0,0 +1,6 @@
+import type { Converter, ConversionResult, StreamInfo, MarkitOptions } from "../types.js";
+export declare class ImageConverter implements Converter {
+    name: string;
+    accepts(streamInfo: StreamInfo): boolean;
+    convert(input: Buffer, streamInfo: StreamInfo, options?: MarkitOptions): Promise<ConversionResult>;
+}