packmd 0.0.1 → 1.0.0

package/README.md

@@ -0,0 +1,148 @@
+## packmd
+
+**PackMD** is a command‑line tool that converts any GitHub repository, local directory, or web page into a clean, token‑efficient Markdown digest – ready to paste into ChatGPT, Claude, or any LLM.
+
+### Installation
+
+**Global (recommended)**
+
+```bash
+npm install -g packmd
+```
+
+Now you can use `packmd` from anywhere:
+
+```bash
+packmd --help
+```
+
+**Run without installing (using npx)**
+
+```bash
+npx packmd <target> [options]
+```
+
+**From source (monorepo)**
+
+```bash
+git clone https://github.com/thelastofinusa/packmd.git
+cd packmd
+bun install
+cd packages/cli
+bun run build
+# run locally
+node dist/index.js <target> [options]
+```
+
+### Quick Start
+
+```bash
+# Generate a digest from a public GitHub repo
+packmd https://github.com/vercel/next.js -o next.md
+
+# Scrape a documentation website (requires Jina API key for higher limits)
+packmd https://react.dev --jina-api-key YOUR_KEY --copy
+
+# Digest the current directory (respects .gitignore)
+packmd .
+
+# Digest a local folder with custom options
+packmd ~/projects/my-app -m 300 -s 50 --exclude "*.test.js"
+```
+
+### Usage
+
+```
+packmd [target] [options]
+```
+
+| Argument | Description                                                                                                                                         |
+| -------- | --------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `target` | GitHub URL (e.g., `https://github.com/owner/repo`), owner/repo slug, local directory path, or any webpage URL. Defaults to current directory (`.`). |
+
+**Options**
+
+| Flag                          | Description                                                                                |
+| ----------------------------- | ------------------------------------------------------------------------------------------ |
+| `-o, --output <path>`         | Save the generated Markdown to a specific file (e.g., `digest.md`).                        |
+| `-c, --copy`                  | Copy the output directly to your clipboard (no file saved).                                |
+| `-t, --token <token>`         | GitHub Personal Access Token for private repositories or higher rate limits.               |
+| `-m, --max-files <number>`    | Maximum number of files to include (default: `200`).                                       |
+| `-s, --max-file-size <kb>`    | Maximum file size in KB (files larger are skipped) (default: `100`).                       |
+| `-i, --include <patterns...>` | Glob patterns to explicitly include (e.g., `"*.ts" "*.md"`).                               |
+| `-e, --exclude <patterns...>` | Glob patterns to explicitly ignore (e.g., `"test/**" "*.log"`).                            |
+| `--jina-api-key <key>`        | Jina Reader API key – raises the free rate limit from 20 to 500 requests per minute.       |
+| `--no-gitignore`              | Ignore `.gitignore` rules when scanning a local directory (by default they are respected). |
+| `-v, --version`               | Show the version number.                                                                   |
+| `-h, --help`                  | Show help.                                                                                 |
+
+> **Note:** When using `--output`, if the file already exists, you will be prompted to overwrite or enter a new name.
+
+### Examples
+
+**GitHub repository**
+
+```bash
+# Basic
+packmd facebook/react -o react.md
+
+# With token (for private repos)
+packmd my-org/private-repo -t ghp_xxxxx -o private.md
+
+# Limit to 300 files, exclude everything in the "examples" folder
+packmd vercel/next.js -m 300 -e "examples/**" -o next-limited.md
+```
+
+**Web page (via Jina Reader)**
+
+```bash
+# Simple scrape
+packmd https://docs.nestjs.com --copy
+
+# With Jina API key (higher rate limit)
+packmd https://tailwindcss.com/docs --jina-api-key jina_xxxxx -o tailwind.md
+```
+
+**Local directory**
+
+```bash
+# Digest the current folder
+packmd .
+
+# Digest a specific path
+packmd ~/code/my-project -o project.md
+
+# Override .gitignore and include only TypeScript files
+packmd ./src --no-gitignore -i "*.ts" "*.tsx"
+```
+
+**Combine options**
+
+```bash
+packmd https://github.com/expressjs/express -o express.md -m 100 -s 50 -e "test/**" "*.spec.js"
+```
+
+### How It Works
+
+The CLI uses the same core engine (`@packmd/core`) as the web app:
+
+1. **Input detection** – determines if the target is a GitHub repo, a web page, or a local path.
+2. **Fetching** – for GitHub, uses the GitHub API; for web pages, uses Jina Reader; for local, uses Node.js `fs`.
+3. **Filtering** – applies ignore rules, globs, size limits, and binary detection.
+4. **Processing** – builds a directory tree and downloads file contents (in parallel for GitHub).
+5. **Markdown generation** – creates a header with metadata, an ASCII tree, and fenced code blocks for each file.
+
+Progress is shown via a live spinner (`ora`). Output can be saved to a file or copied to your clipboard.
+
+### Notes
+
+- **GitHub rate limits** – Without a token, you are limited to 60 requests per hour. Use `-t` to raise this to 5,000 per hour.
+- **Jina rate limits** – Without an API key, you get 20 requests per minute. Provide `--jina-api-key` for 500 RPM.
+- The CLI respects `.gitignore` by default. Use `--no-gitignore` to scan everything.
+- **Binary files** (images, fonts, archives) are automatically skipped.
+
+---
+
+### License
+
+MIT © [Holiday](https://github.com/thelastofinusa)

package/dist/index.js

@@ -1,4 +1,419 @@
 #!/usr/bin/env node
 
 // src/index.ts
-console.log("Hello, World!");
+import { Command } from "commander";
+
+// src/actions/run.ts
+import color2 from "picocolors";
+import clipboard from "clipboardy";
+import fs4 from "fs/promises";
+import inquirer2 from "inquirer";
+import ora from "ora";
+
+// package.json
+var name = "packmd";
+var description = "Convert GitHub repositories and webpages into AI-ready Markdown digests";
+var version = "1.0.0";
+
+// src/utils/version-check.ts
+import fs from "fs/promises";
+import os from "os";
+import path from "path";
+import color from "picocolors";
+var REGISTRY_URL = `https://registry.npmjs.org/${name}/latest`;
+var CACHE_PATH = path.join(os.homedir(), `.${name}`, "version-check.json");
+var CACHE_TTL_MS = 24 * 60 * 60 * 1e3;
+var FETCH_TIMEOUT_MS = 1500;
+function compareVersions(a, b) {
+  const pa = a.split(".").map(Number);
+  const pb = b.split(".").map(Number);
+  for (let i = 0; i < Math.max(pa.length, pb.length); i++) {
+    const na = pa[i] || 0;
+    const nb = pb[i] || 0;
+    if (na > nb) return 1;
+    if (na < nb) return -1;
+  }
+  return 0;
+}
+async function readCache() {
+  try {
+    return JSON.parse(await fs.readFile(CACHE_PATH, "utf-8"));
+  } catch {
+    return null;
+  }
+}
+async function writeCache(data) {
+  try {
+    await fs.mkdir(path.dirname(CACHE_PATH), { recursive: true });
+    await fs.writeFile(CACHE_PATH, JSON.stringify(data), "utf-8");
+  } catch {
+  }
+}
+async function fetchLatestVersion() {
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
+  try {
+    const res = await fetch(REGISTRY_URL, {
+      signal: controller.signal,
+      headers: { Accept: "application/vnd.npm.install-v1+json" }
+      // lightweight abbreviated response
+    });
+    if (!res.ok) return null;
+    const data = await res.json();
+    return typeof data.version === "string" ? data.version : null;
+  } catch {
+    return null;
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+async function checkForUpdate(currentVersion) {
+  const cache = await readCache();
+  const isFresh = cache && Date.now() - cache.lastChecked < CACHE_TTL_MS;
+  const latestVersion = isFresh ? cache.latestVersion : await fetchLatestVersion();
+  if (!latestVersion) return null;
+  if (!isFresh) await writeCache({ lastChecked: Date.now(), latestVersion });
+  return compareVersions(latestVersion, currentVersion) > 0 ? latestVersion : null;
+}
+function printUpdateNotice(currentVersion, latestVersion) {
+  console.log();
+  console.log(
+    color.yellow(`  \u2191 Update available: `) + color.dim(currentVersion) + color.yellow(" \u2192 ") + color.green(latestVersion)
+  );
+  console.log(
+    color.dim(`  Run `) + color.cyan(`npm install -g ${name}@latest`) + color.dim(` to update.`)
+  );
+  console.log();
+}
+
+// src/handlers/github.ts
+import { fetchGithubRepo } from "@packmd/core";
+async function handleGitHub(target, options, spinner) {
+  const result = await fetchGithubRepo(target, {
+    token: options.token || process.env.GITHUB_TOKEN,
+    maxFiles: Number(options.maxFiles),
+    maxFileSizeKB: Number(options.maxFileSize),
+    includeGlobs: options.include || [],
+    excludeGlobs: options.exclude || [],
+    onProgress: (msg) => {
+      spinner.text = msg;
+    }
+  });
+  return result.markdown;
+}
+
+// src/handlers/local.ts
+import path4 from "path";
+import { DEFAULT_IGNORES, buildDigestHeader } from "@packmd/core";
+
+// src/utils/fs.ts
+import fs2 from "fs/promises";
+import path2 from "path";
+import { minimatch } from "minimatch";
+async function walkLocalDir(dir, base, excludeGlobs, includeGlobs, maxFileSizeKB, maxFiles, gitignore = null, collected = []) {
+  if (collected.length >= maxFiles) return collected;
+  const entries = await fs2.readdir(dir);
+  for (const name2 of entries) {
+    if (collected.length >= maxFiles) break;
+    const fullPath = path2.join(dir, name2);
+    const relativePath = path2.relative(base, fullPath).replace(/\\/g, "/");
+    const isExcluded = excludeGlobs.some(
+      (g) => minimatch(relativePath, g, { dot: true })
+    );
+    if (isExcluded) continue;
+    const stats = await fs2.stat(fullPath);
+    if (gitignore?.ignores(
+      stats.isDirectory() ? `${relativePath}/` : relativePath
+    )) {
+      continue;
+    }
+    if (stats.isDirectory()) {
+      await walkLocalDir(
+        fullPath,
+        base,
+        excludeGlobs,
+        includeGlobs,
+        maxFileSizeKB,
+        maxFiles,
+        gitignore,
+        collected
+      );
+    } else if (stats.isFile()) {
+      if (includeGlobs.length > 0) {
+        const isIncluded = includeGlobs.some(
+          (g) => minimatch(relativePath, g, { dot: true })
+        );
+        if (!isIncluded) continue;
+      }
+      const sizeKB = stats.size / 1024;
+      if (sizeKB <= maxFileSizeKB) {
+        const content = await fs2.readFile(fullPath, "utf-8");
+        collected.push({ path: relativePath, content });
+      }
+    }
+  }
+  return collected;
+}
+
+// src/utils/gitignore.ts
+import fs3 from "fs/promises";
+import path3 from "path";
+import ignore from "ignore";
+async function loadGitignore(startDir) {
+  const ig = ignore();
+  let found = false;
+  let dir = startDir;
+  while (true) {
+    try {
+      const content = await fs3.readFile(path3.join(dir, ".gitignore"), "utf-8");
+      ig.add(content);
+      found = true;
+    } catch {
+    }
+    try {
+      await fs3.access(path3.join(dir, ".git"));
+      break;
+    } catch {
+    }
+    const parent = path3.dirname(dir);
+    if (parent === dir) break;
+    dir = parent;
+  }
+  return found ? ig : null;
+}
+
+// src/handlers/local.ts
+async function handleLocalDir(target, options, spinner) {
+  const absolutePath = path4.resolve(process.cwd(), target);
+  const excludeGlobs = options.exclude || DEFAULT_IGNORES;
+  const includeGlobs = options.include || [];
+  const maxFiles = Number(options.maxFiles);
+  const maxFileSizeKB = Number(options.maxFileSize);
+  const gitignore = options.gitignore === false ? null : await loadGitignore(absolutePath);
+  const files = await walkLocalDir(
+    absolutePath,
+    absolutePath,
+    excludeGlobs,
+    includeGlobs,
+    maxFileSizeKB,
+    maxFiles,
+    gitignore
+  );
+  spinner.text = `Processing ${files.length} local files..`;
+  const totalChars = files.reduce((sum, f) => sum + f.content.length, 0);
+  const estTokens = Math.round(totalChars / 4);
+  const header = buildDigestHeader({
+    title: `Local Digest \u2014 \`${path4.basename(absolutePath)}\``,
+    meta: {
+      Path: `\`${absolutePath}\``,
+      Date: (/* @__PURE__ */ new Date()).toISOString().slice(0, 10),
+      Files: files.length,
+      "Est. tokens": `~${estTokens.toLocaleString()}`
+    }
+  });
+  const markdownParts = [header];
+  for (const file of files) {
+    markdownParts.push(`## File: ${file.path}
+\`\`\`
+${file.content}
+\`\`\``);
+  }
+  return markdownParts.join("\n\n");
+}
+
+// src/handlers/webpage.ts
+import { buildDigestHeader as buildDigestHeader2, scrapeWebPage } from "@packmd/core";
+async function handleWebpage(target, options) {
+  const scraped = await scrapeWebPage(target, {
+    jinaApiKey: options.jinaApiKey || process.env.JINA_API_KEY
+  });
+  const estTokens = Math.round((scraped.content?.length || 0) / 4);
+  const header = buildDigestHeader2({
+    icon: "\u{1F310}",
+    title: scraped.title || target,
+    meta: {
+      Source: `\`${target}\``,
+      Date: (/* @__PURE__ */ new Date()).toISOString().slice(0, 10),
+      "Est. tokens": `~${estTokens.toLocaleString()}`
+    }
+  });
+  return `${header}
+
+${scraped.content || ""}`;
+}
+
+// src/prompts/github.ts
+import inquirer from "inquirer";
+async function promptGithubOptions(currentOptions) {
+  const { customize } = await inquirer.prompt([
+    {
+      type: "confirm",
+      name: "customize",
+      message: "GitHub URL detected. Would you like to configure options?",
+      default: false
+    }
+  ]);
+  if (!customize) return currentOptions;
+  const answers = await inquirer.prompt([
+    {
+      type: "input",
+      name: "maxFiles",
+      message: "Maximum number of files to include:",
+      default: currentOptions.maxFiles || "200",
+      validate: (val) => !isNaN(Number(val)) || "Please enter a valid number"
+    },
+    {
+      type: "input",
+      name: "maxFileSize",
+      message: "Maximum file size threshold (in KB):",
+      default: currentOptions.maxFileSize || "100",
+      validate: (val) => !isNaN(Number(val)) || "Please enter a valid number"
+    },
+    {
+      type: "password",
+      name: "token",
+      message: "GitHub Token (Optional - for private or larger repos):",
+      default: currentOptions.token || ""
+    }
+  ]);
+  return { ...currentOptions, ...answers };
+}
+
+// src/actions/run.ts
+async function runAction(target, options) {
+  console.log(color2.cyan(`${name} \u2014 AI Markdown Packager`));
+  const updateCheck = checkForUpdate(version).catch(() => null);
+  let finalOptions = { ...options };
+  try {
+    let digest = "";
+    const spinner = ora();
+    if (target.startsWith("http://") || target.startsWith("https://")) {
+      const targetUrl = new URL(target);
+      const isGitHub = targetUrl.hostname.includes("github.com");
+      if (isGitHub) {
+        finalOptions = await promptGithubOptions(finalOptions);
+        spinner.start("Fetching repository. Please wait..");
+        digest = await handleGitHub(target, finalOptions, spinner);
+      } else {
+        spinner.start("Scraping webpage. Please wait..");
+        digest = await handleWebpage(target, finalOptions);
+      }
+    } else {
+      spinner.start("Scanning local directory. Please wait..");
+      digest = await handleLocalDir(target, finalOptions, spinner);
+    }
+    let outputPath = finalOptions.copy ? null : finalOptions.output || "pack.md";
+    if (outputPath) {
+      let finalPath = outputPath;
+      while (true) {
+        let fileExists = false;
+        try {
+          await fs4.access(finalPath);
+          fileExists = true;
+        } catch {
+          fileExists = false;
+        }
+        if (fileExists) {
+          const { overwrite } = await inquirer2.prompt([
+            {
+              type: "confirm",
+              name: "overwrite",
+              message: `File ${color2.cyan(finalPath)} already exists. Overwrite?`,
+              default: false
+            }
+          ]);
+          if (overwrite) {
+            break;
+          } else {
+            const { newFileName } = await inquirer2.prompt([
+              {
+                type: "input",
+                name: "newFileName",
+                message: "Please enter a new file name:",
+                default: "pack-new.md",
+                validate: (val) => val.trim().length > 0 || "File name cannot be empty."
+              }
+            ]);
+            finalPath = newFileName;
+          }
+        } else {
+          break;
+        }
+      }
+      await fs4.writeFile(finalPath, digest, { encoding: "utf-8", flag: "w" });
+      spinner.succeed("Markdown generated successfully!");
+    }
+    if (finalOptions.copy) {
+      await clipboard.write(digest);
+      spinner.succeed("Markdown compiled successfully.");
+    }
+    const latestVersion = await updateCheck;
+    if (latestVersion) printUpdateNotice(version, latestVersion);
+  } catch (error) {
+    console.error(
+      color2.red(`
+\u2716 ${error.message || "An unexpected error occurred"}`)
+    );
+    process.exit(1);
+  }
+}
+
+// src/utils/program.ts
+var cliOptions = [
+  {
+    flags: "-o, --output <path>",
+    description: "Specify a custom file path to save the generated Markdown digest"
+  },
+  {
+    flags: "-t, --token <token>",
+    description: "Provide a GitHub Personal Access Token for private repositories or extended rate limits"
+  },
+  {
+    flags: "-m, --max-files <number>",
+    description: "Limit the total number of files to include in the final digest",
+    defaultValue: "200"
+  },
+  {
+    flags: "-s, --max-file-size <number>",
+    description: "Set the maximum allowed size (in KB) for individual files",
+    defaultValue: "100"
+  },
+  {
+    flags: "-i, --include <patterns...>",
+    description: "Specify glob patterns to explicitly include certain files or directories"
+  },
+  {
+    flags: "-e, --exclude <patterns...>",
+    description: "Specify glob patterns to explicitly ignore certain files or directories"
+  },
+  {
+    flags: "-c, --copy",
+    description: "Automatically copy the generated Markdown digest to your clipboard",
+    defaultValue: false
+  },
+  {
+    flags: "--jina-api-key <key>",
+    description: "Provide a Jina API key for enhanced webpage scraping and parsing"
+  },
+  {
+    flags: "--no-gitignore",
+    description: "Don't respect .gitignore rules when scanning a local directory"
+  }
+];
+
+// src/index.ts
+var program = new Command();
+program.name(name).description(description).version(version, "-v, --version", "Output the version number").argument(
+  "[target]",
+  "GitHub URL, Webpage URL, or local directory path (defaults to current directory)",
+  "."
+);
+cliOptions.forEach(({ flags, description: description2, defaultValue }) => {
+  if (defaultValue !== void 0) {
+    program.option(flags, description2, defaultValue);
+  } else {
+    program.option(flags, description2);
+  }
+});
+program.action(runAction);
+program.parse();

package/package.json

@@ -1,6 +1,7 @@
 {
     "name": "packmd",
-    "version": "0.0.1",
+    "description": "Convert GitHub repositories and webpages into AI-ready Markdown digests",
+    "version": "1.0.0",
     "license": "MIT",
     "type": "module",
     "bin": {
@@ -13,13 +14,15 @@
     ],
     "scripts": {
         "build": "tsup",
-        "dev": "tsx src/index.ts",
-        "start": "node dist/index.js",
+        "start": "tsx src/index.js",
+        "start:node": "node dist/index.js",
+        "clean": "rm -rf dist node_modules .turbo",
         "prepublishOnly": "bun run build",
         "pushout": "npm publish",
         "pushout-inc": "npm version patch --no-git-tag-version && bun run pushout"
     },
     "devDependencies": {
+        "@types/inquirer": "^9.0.10",
         "@types/node": "^26.1.1",
         "tsup": "^8.5.1",
         "tsx": "^4.23.1",
@@ -27,7 +30,37 @@
     },
     "dependencies": {
         "chalk": "^5.6.2",
+        "clipboardy": "^5.3.2",
         "commander": "^15.0.0",
-        "ora": "^9.4.1"
-    }
+        "ignore": "^7.0.6",
+        "inquirer": "^14.0.2",
+        "minimatch": "^10.2.5",
+        "ora": "^9.4.1",
+        "@packmd/core": "^1.0.0",
+        "picocolors": "^1.1.1"
+    },
+    "keywords": [
+        "packmd",
+        "markdown",
+        "github",
+        "llm",
+        "codebase",
+        "digest",
+        "cli",
+        "generator",
+        "scraper",
+        "developer-tools",
+        "chatgpt",
+        "claude"
+    ],
+    "author": "Holiday (https://github.com/thelastofinusa)",
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/thelastofinusa/packmd.git",
+        "directory": "packages/cli"
+    },
+    "bugs": {
+        "url": "https://github.com/thelastofinusa/packmd/issues"
+    },
+    "homepage": "https://packmd.vercel.app/docs"
 }