ethan-agent-skills 0.1.0

package/lib/update.mjs

@@ -0,0 +1,144 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import kleur from 'kleur';
+import { discoverBundledSkills } from './discover.mjs';
+import {
+  CLIENT_DIRS,
+  copyDir,
+  LOCK_FILE,
+  PACKAGE_NAME,
+  pathExists,
+  readJSON,
+  resolveTarget,
+  rmDir,
+  writeJSON,
+} from './utils.mjs';
+
+export async function update(opts = {}) {
+  const targets = resolveTargets(opts);
+
+  for (const target of targets) {
+    const skills = await discoverBundledSkills(target.groups, { dedupe: true });
+    await syncToTarget(target.path, skills, opts);
+  }
+}
+
+function resolveTargets(opts) {
+  const client = String(opts.client || 'claude').toLowerCase();
+
+  if (opts.target) {
+    return [
+      {
+        name: 'custom',
+        path: resolveTarget(opts.target),
+        groups: groupsForClient(client),
+      },
+    ];
+  }
+
+  if (client === 'claude') {
+    return [{ name: 'claude', path: CLIENT_DIRS.claude, groups: groupsForClient(client) }];
+  }
+
+  if (client === 'codex') {
+    return [{ name: 'codex', path: CLIENT_DIRS.codex, groups: groupsForClient(client) }];
+  }
+
+  if (client === 'all') {
+    return [
+      { name: 'claude', path: CLIENT_DIRS.claude, groups: groupsForClient('claude') },
+      { name: 'codex', path: CLIENT_DIRS.codex, groups: groupsForClient('codex') },
+    ];
+  }
+
+  throw new Error(`Unsupported client "${opts.client}". Use claude, codex, or all.`);
+}
+
+function groupsForClient(client) {
+  if (client === 'claude') return ['common', 'claude'];
+  if (client === 'codex') return ['common', 'codex'];
+  if (client === 'all') return ['common', 'claude', 'codex'];
+  throw new Error(`Unsupported client "${client}". Use claude, codex, or all.`);
+}
+
+async function syncToTarget(targetDir, skills, opts) {
+  console.log(kleur.cyan(`\n-> Syncing ${skills.length} skill(s) to ${targetDir}`));
+
+  if (skills.length === 0) {
+    console.log(kleur.yellow('  No bundled skills found for this target.'));
+    return;
+  }
+
+  const lockPath = path.join(targetDir, LOCK_FILE);
+  const hadLock = await pathExists(lockPath);
+  const lock = normalizeLock(await readJSON(lockPath, {}));
+  const nextLock = structuredClone(lock);
+  let updated = 0;
+  let skipped = 0;
+  let conflicts = 0;
+
+  if (!opts.dryRun) {
+    await fs.mkdir(targetDir, { recursive: true });
+  }
+
+  for (const skill of skills) {
+    const destination = path.join(targetDir, skill.name);
+    const lockEntry = lock.skills[skill.name];
+    const exists = await pathExists(destination);
+
+    if (exists && !isManaged(lockEntry)) {
+      console.log(`  ${kleur.red('!')} ${skill.name} ${kleur.red('(unmanaged destination exists; skipped)')}`);
+      conflicts += 1;
+      continue;
+    }
+
+    if (!opts.force && exists && lockEntry?.version === skill.version) {
+      console.log(`  ${kleur.gray('=')} ${skill.name} ${kleur.gray(`(${skill.version}, up-to-date)`)}`);
+      skipped += 1;
+      continue;
+    }
+
+    const oldVersion = lockEntry?.version ?? 'none';
+
+    if (opts.dryRun) {
+      const marker = exists ? '~' : '+';
+      console.log(`  ${kleur.yellow(marker)} ${skill.name} ${oldVersion} -> ${skill.version} ${kleur.gray('(dry-run)')}`);
+      updated += 1;
+      continue;
+    }
+
+    await rmDir(destination);
+    await copyDir(skill.sourceDir, destination);
+    nextLock.skills[skill.name] = {
+      name: skill.name,
+      version: skill.version,
+      sourcePackage: PACKAGE_NAME,
+      sourceGroup: skill.group,
+      updatedAt: new Date().toISOString(),
+    };
+    updated += 1;
+    console.log(`  ${kleur.green('✓')} ${skill.name} ${kleur.gray(oldVersion)} -> ${kleur.green(skill.version)}`);
+  }
+
+  if (!opts.dryRun && (updated > 0 || hadLock)) {
+    nextLock.package = PACKAGE_NAME;
+    nextLock.lockfileVersion = 1;
+    nextLock.updatedAt = new Date().toISOString();
+    await writeJSON(lockPath, nextLock);
+  }
+
+  console.log(kleur.gray(`  ${updated} changed, ${skipped} skipped, ${conflicts} conflict(s).`));
+}
+
+function normalizeLock(lock) {
+  return {
+    package: lock?.package || PACKAGE_NAME,
+    lockfileVersion: lock?.lockfileVersion || 1,
+    updatedAt: lock?.updatedAt || null,
+    skills: lock?.skills && typeof lock.skills === 'object' ? lock.skills : {},
+  };
+}
+
+function isManaged(lockEntry) {
+  return lockEntry?.sourcePackage === PACKAGE_NAME;
+}

package/lib/utils.mjs

@@ -0,0 +1,120 @@
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+export const PACKAGE_NAME = 'ethan-agent-skills';
+export const LOCK_FILE = '.skills-lock.json';
+export const PACKAGE_ROOT = path.resolve(
+  path.dirname(fileURLToPath(import.meta.url)),
+  '..',
+);
+
+export const CLIENT_DIRS = {
+  claude: path.join(os.homedir(), '.claude', 'skills'),
+  codex: path.join(os.homedir(), '.codex', 'skills'),
+};
+
+export const SOURCE_GROUPS = {
+  common: path.join(PACKAGE_ROOT, 'skills'),
+  claude: path.join(PACKAGE_ROOT, 'claude', 'skills'),
+  codex: path.join(PACKAGE_ROOT, 'codex', 'skills'),
+  agents: path.join(PACKAGE_ROOT, 'agents', 'skills'),
+};
+
+export async function pathExists(targetPath) {
+  try {
+    await fs.access(targetPath);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export async function readJSON(file, fallback = null) {
+  try {
+    return JSON.parse(await fs.readFile(file, 'utf8'));
+  } catch {
+    return fallback;
+  }
+}
+
+export async function writeJSON(file, data) {
+  await fs.mkdir(path.dirname(file), { recursive: true });
+  const tempFile = `${file}.${process.pid}.tmp`;
+  await fs.writeFile(tempFile, `${JSON.stringify(data, null, 2)}\n`);
+  await fs.rename(tempFile, file);
+}
+
+export async function copyDir(src, dest) {
+  await fs.mkdir(dest, { recursive: true });
+
+  for (const entry of await fs.readdir(src, { withFileTypes: true })) {
+    const srcPath = path.join(src, entry.name);
+    const destPath = path.join(dest, entry.name);
+
+    if (entry.isDirectory()) {
+      await copyDir(srcPath, destPath);
+      continue;
+    }
+
+    if (entry.isFile() || entry.isSymbolicLink()) {
+      await fs.copyFile(srcPath, destPath);
+    }
+  }
+}
+
+export async function rmDir(dir) {
+  await fs.rm(dir, { recursive: true, force: true });
+}
+
+export async function readSkillVersion(skillDir) {
+  const meta = await readJSON(path.join(skillDir, 'skill.json'));
+  if (meta?.version) return String(meta.version);
+
+  try {
+    const md = await fs.readFile(path.join(skillDir, 'SKILL.md'), 'utf8');
+    const frontmatter = md.match(/^---\s*\n([\s\S]*?)\n---/);
+    const version = frontmatter?.[1]?.match(/^version:\s*['"]?([^'"\n]+)['"]?$/m);
+    if (version) return version[1].trim();
+  } catch {
+    // Missing or unreadable metadata falls back to the package baseline.
+  }
+
+  return '0.0.0';
+}
+
+export async function readSkillDescription(skillDir) {
+  const meta = await readJSON(path.join(skillDir, 'skill.json'));
+  if (meta?.description) return String(meta.description);
+
+  try {
+    const md = await fs.readFile(path.join(skillDir, 'SKILL.md'), 'utf8');
+    const frontmatter = md.match(/^---\s*\n([\s\S]*?)\n---/);
+    const description = frontmatter?.[1]?.match(/^description:\s*['"]?([^'"\n]+)['"]?$/m);
+    if (description) return description[1].trim();
+  } catch {
+    // Description is optional.
+  }
+
+  return '';
+}
+
+export function normalizeClient(client) {
+  const value = String(client || 'claude').toLowerCase();
+  if (value === 'claude' || value === 'codex' || value === 'all') {
+    return value;
+  }
+
+  throw new Error(`Unsupported client "${client}". Use claude, codex, or all.`);
+}
+
+export function expandHome(inputPath) {
+  if (inputPath === '~') return os.homedir();
+  if (inputPath?.startsWith('~/')) return path.join(os.homedir(), inputPath.slice(2));
+  return inputPath;
+}
+
+export function resolveTarget(inputPath) {
+  return path.resolve(expandHome(inputPath));
+}

package/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "ethan-agent-skills",
+  "version": "0.1.0",
+  "description": "Agent skills published from my_agent_skill",
+  "type": "module",
+  "bin": {
+    "skills": "bin/skills.mjs"
+  },
+  "files": [
+    "bin/",
+    "lib/",
+    "skills/",
+    "codex/",
+    "claude/",
+    "agents/",
+    "README.md",
+    "LICENSE"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "scripts": {
+    "test:local": "rm -rf /tmp/my-agent-skill-test-local && node bin/skills.mjs --help && node bin/skills.mjs --version && node bin/skills.mjs list && node bin/skills.mjs update --dry-run --target /tmp/my-agent-skill-test-local && node bin/skills.mjs update --target /tmp/my-agent-skill-test-local",
+    "pack:check": "npm pack --dry-run"
+  },
+  "dependencies": {
+    "commander": "^12.1.0",
+    "kleur": "^4.1.5"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/EthenZhang/my_agent_skill.git"
+  },
+  "keywords": [
+    "agent",
+    "claude",
+    "claude-code",
+    "codex",
+    "skills",
+    "cli"
+  ],
+  "license": "MIT",
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/skills/pdf-extract/SKILL.md

@@ -0,0 +1,214 @@
+---
+name: pdf-extract
+description: Extract text from PDF files and save as clean markdown documents. Use this skill whenever a PDF file needs to be converted to readable text, especially for insurance policies, financial documents, scanned PDFs, or mixed text/image PDFs. Trigger on requests like "extract this PDF", "convert PDF to markdown", "read this PDF", "process this PDF", "OCR this PDF", or when the user drops a PDF and asks for its contents. This skill handles both text-based PDFs and scanned/image-based PDFs using macOS-native tools (no poppler or tesseract needed).
+---
+
+# PDF Extract Skill
+
+Extract complete text from PDF files and save as clean markdown. Handles text-based PDFs, scanned/image pages, and mixed documents — all using macOS native frameworks (no external dependencies beyond pyobjc, which comes with the system Python on macOS).
+
+## When to Use
+
+- A user uploads a PDF and wants its content extracted to text or markdown
+- The PDF contains a mix of text and scanned pages (e.g., insurance policy booklets)
+- Scanned pages are embedded within otherwise text-based PDFs
+- Large PDFs where the extracted text exceeds normal read limits
+- Financial documents, insurance policies, contracts, or reports that need structured extraction
+
+## Workflow Overview
+
+1. **Try text extraction first** — use PDFKit to get native text
+2. **Detect scanned pages** — pages with little or no extracted text are likely scans
+3. **Render scanned pages as images** — convert them to PNGs at high resolution
+4. **Extract text from images** — use multimodal vision to OCR the rendered pages
+5. **Crop if needed** — isolate specific regions (tables, signatures) from page images
+6. **Assemble and save** — combine all extracted text into a clean markdown document
+
+## Step-by-Step Instructions
+
+### Step 1: Inspect the PDF
+
+First, check the PDF file path and try basic text extraction on the first page to gauge quality:
+
+```python
+import sys, Quartz, os
+sys.path.insert(0, '/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python/PyObjC')
+import objc
+
+pdf_path = "/path/to/file.pdf"
+url = Quartz.NSURL.fileURLWithPath_(pdf_path)
+doc = Quartz.PDFDocument.alloc().initWithURL_(doc)
+
+if doc is None:
+    # Cannot read PDF
+    exit()
+
+page_count = doc.pageCount()
+print(f"Pages: {page_count}")
+
+# Quick quality check on first page
+p1 = doc.pageAtIndex_(0)
+text = p1.string() if p1 else ""
+print(f"First page text length: {len(text)}")
+if len(text) < 50:
+    print("WARNING: Low text extraction — likely a scanned PDF")
+```
+
+### Step 2: Extract All Text-Based Pages
+
+Attempt to extract text from every page. Pages with meaningful text (>50 chars) are text-based; pages with very little text (<10 chars, often just whitespace or decorative elements) are scanned images.
+
+```python
+import sys, Quartz
+
+pdf_path = "/path/to/file.pdf"
+url = Quartz.NSURL.fileURLWithPath_(pdf_path)
+doc = Quartz.PDFDocument.alloc().initWithURL_(url)
+
+pages_text = []
+scanned_pages = []
+
+for i in range(doc.pageCount()):
+    page = doc.pageAtIndex_(i)
+    text = page.string() if page else ""
+    if len(text.strip()) < 10:
+        scanned_pages.append(i)
+    else:
+        pages_text.append({"page": i + 1, "text": text})
+
+print(f"Text pages: {len(pages_text)}, Scanned pages: {scanned_pages}")
+```
+
+### Step 3: Render Scanned Pages as PNG Images
+
+For each scanned page, render it to a PNG image at 6x scale (300+ DPI equivalent) for clarity. Lower scales (3x-4x) may not be readable for dense financial tables.
+
+```python
+import sys, Quartz
+
+pdf_path = "/path/to/file.pdf"
+url = Quartz.NSURL.fileURLWithPath_(pdf_path)
+doc = Quartz.PDFDocument.alloc().initWithURL_(url)
+output_dir = "/tmp/pdf_rendered/"
+os.makedirs(output_dir, exist_ok=True)
+
+scale = 6.0  # 6x scale for dense tables
+
+for page_idx in scanned_pages:
+    page = doc.pageAtIndex_(page_idx)
+    media_box = page.boundsForBox_(Quartz.kCGPDFMediaBox)
+    
+    pw = int(media_box.size.width * scale)
+    ph = int(media_box.size.height * scale)
+    
+    cs = Quartz.CGColorSpaceCreateDeviceRGB()
+    ctx = Quartz.CGBitmapContextCreate(
+        None, pw, ph, 8, pw * 4, cs,
+        Quartz.kCGImageAlphaPremultipliedLast
+    )
+    
+    # White background
+    Quartz.CGContextSetRGBFillColor(ctx, 1.0, 1.0, 1.0, 1.0)
+    Quartz.CGContextFillRect(ctx, Quartz.CGRectMake(0, 0, pw, ph))
+    Quartz.CGContextScaleCTM(ctx, scale, scale)
+    page.drawWithBox_toContext_(Quartz.kCGPDFMediaBox, ctx)
+    
+    cg_img = Quartz.CGBitmapContextCreateImage(ctx)
+    Quartz.CGImageDestinationAddImage(
+        Quartz.CGImageDestinationCreateWithURL(
+            Quartz.NSURL.fileURLWithPath_(f"{output_dir}page_{page_idx+1}.png"),
+            Quartz.kUTTypePNG, 1, None
+        ),
+        cg_img, None
+    )
+    # Finalize the destination
+    dest = Quartz.CGImageDestinationCreateWithURL(
+        Quartz.NSURL.fileURLWithPath_(f"{output_dir}page_{page_idx+1}.png"),
+        Quartz.kUTTypePNG, 1, None
+    )
+    Quartz.CGImageDestinationAddImage(dest, cg_img, None)
+    Quartz.CGImageDestinationFinalize(dest)
+    
+    print(f"Rendered page {page_idx+1} to {output_dir}page_{page_idx+1}.png ({pw}x{ph})")
+```
+
+> **Scale reference:** 3x works for simple text, 4x for most documents, 5x for dense forms, 6x for financial tables with small fonts. Start at 4x and go up if text isn't legible.
+
+### Step 4: Crop Specific Regions (Optional)
+
+When only part of a page is relevant (e.g., a specific table), crop the rendered image to that region. Coordinates use the Quartz coordinate system where the origin (0,0) is at the **top-left** of the page.
+
+```python
+import sys, Quartz
+
+# After rendering to cg_img (before saving), crop a region:
+# crop_rect = Quartz.CGRectMake(x * scale, y * scale, width * scale, height * scale)
+# cg_img = Quartz.CGImageCreateWithImageInRect(cg_img, crop_rect)
+```
+
+> **Finding coordinates:** Render the full page first, view the image, then estimate the crop rectangle. The page bounds are in PDF points (1 point = 1/72 inch). A standard A4 page is roughly 595x842 points. Multiply by `scale` for the pixel-space crop rectangle.
+
+### Step 5: Extract Text from Rendered Images
+
+Use multimodal vision to extract text from the rendered PNG images. This works best with the Read tool on the image files.
+
+### Step 6: Assemble and Save as Markdown
+
+Combine all extracted content into a structured markdown document. Follow the Obsidian wiki schema if the target vault uses it:
+
+```markdown
+---
+type: source
+tags: [category, subcategory]
+date: YYYY-MM-DD
+source_count: 1
+sources: [[original-filename.pdf]]
+status: active
+---
+
+# Document Title
+
+> Brief description of the document
+
+## Key Data
+
+[Extracted tables as markdown tables]
+
+## Main Content
+
+[Extracted body text, organized by sections]
+
+## Important Notes
+
+[Any specific details, warnings, or risks mentioned]
+```
+
+## Handling Large PDFs
+
+When the extracted text is very long and would exceed output limits:
+
+- Process pages in batches of 5-10
+- Save intermediate results to a temporary file
+- For text-based extraction, use chunked reading with offset/limit if reading from a file:
+
+```python
+# Write extracted text to a file first, then read in chunks
+with open("/tmp/extracted.txt", "w") as f:
+    for page_data in pages_text:
+        f.write(f"\n--- Page {page_data['page']} ---\n")
+        f.write(page_data["text"])
+```
+
+## Common Pitfalls
+
+- **Low resolution renders:** If OCR quality is poor, increase the scale from 4x to 6x. Dense tables with small fonts almost always need 6x.
+- **Page orientation:** Some PDFs have rotated pages. Check `media_box` dimensions to detect landscape pages.
+- **Watermarks/overlays:** Background watermarks can interfere with OCR. If pages have heavy watermarks, try cropping to the content region.
+- **Mixed content pages:** A page might have both text and scanned elements. The `< 10 chars` threshold detects pure scans, but pages with partial text need manual review.
+- **pyobjc availability:** On macOS, pyobjc is pre-installed with the system Python. Use `python3` from the system, not a Homebrew Python that may lack the Quartz bindings.
+
+## Dependencies
+
+- macOS (required — uses Quartz/PDFKit frameworks)
+- pyobjc (pre-installed on macOS with system Python)
+- No additional packages needed (no poppler, no tesseract, no PIL)

package/skills/pdf-extract/scripts/pdf_extract.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+"""
+pdf_extract.py — Extract text from PDF files on macOS using PDFKit + Quartz.
+
+Usage:
+    python3 pdf_extract.py <pdf_path> [--render-scanned] [--output-dir DIR] [--scale N]
+
+Options:
+    --render-scanned    Render scanned/blank pages as PNG images for OCR
+    --output-dir DIR    Directory for rendered page images (default: /tmp/pdf_rendered/)
+    --scale N           Render scale multiplier (default: 6.0)
+
+Output:
+    - Prints extracted text to stdout, page by page
+    - If --render-scanned is set, saves scanned pages as PNGs to output_dir
+    - Prints a JSON summary to stderr with page counts and scanned page indices
+"""
+
+import sys
+import os
+import json
+import argparse
+
+import Quartz
+
+
+def extract_text(pdf_path, render_scanned=False, output_dir=None, scale=6.0):
+    url = Quartz.NSURL.fileURLWithPath_(pdf_path)
+    doc = Quartz.PDFDocument.alloc().initWithURL_(url)
+
+    if doc is None:
+        print(f"ERROR: Cannot open PDF: {pdf_path}", file=sys.stderr)
+        sys.exit(1)
+
+    page_count = doc.pageCount()
+    pages_text = []
+    scanned_pages = []
+
+    for i in range(page_count):
+        page = doc.pageAtIndex_(i)
+        text = page.string() if page else ""
+
+        if len(text.strip()) < 10:
+            scanned_pages.append(i)
+            if render_scanned:
+                if output_dir is None:
+                    output_dir = "/tmp/pdf_rendered/"
+                os.makedirs(output_dir, exist_ok=True)
+                _render_page(page, i, output_dir, scale)
+        else:
+            pages_text.append({"page": i + 1, "text": text})
+
+    # Print extracted text
+    for pd in pages_text:
+        print(f"\n--- Page {pd['page']} ---")
+        print(pd["text"])
+
+    # Summary to stderr
+    summary = {
+        "total_pages": page_count,
+        "text_pages": len(pages_text),
+        "scanned_pages": scanned_pages,
+        "scanned_count": len(scanned_pages),
+    }
+    print(f"\n[Summary] {summary['text_pages']} text pages, {summary['scanned_count']} scanned pages: {scanned_pages}", file=sys.stderr)
+    json.dump(summary, sys.stderr)
+    print("", file=sys.stderr)
+
+    return summary
+
+
+def _render_page(page, page_idx, output_dir, scale):
+    media_box = page.boundsForBox_(Quartz.kCGPDFMediaBox)
+    pw = int(media_box.size.width * scale)
+    ph = int(media_box.size.height * scale)
+
+    cs = Quartz.CGColorSpaceCreateDeviceRGB()
+    ctx = Quartz.CGBitmapContextCreate(
+        None, pw, ph, 8, pw * 4, cs,
+        Quartz.kCGImageAlphaPremultipliedLast
+    )
+
+    Quartz.CGContextSetRGBFillColor(ctx, 1.0, 1.0, 1.0, 1.0)
+    Quartz.CGContextFillRect(ctx, Quartz.CGRectMake(0, 0, pw, ph))
+    Quartz.CGContextScaleCTM(ctx, scale, scale)
+    page.drawWithBox_toContext_(Quartz.kCGPDFMediaBox, ctx)
+
+    cg_img = Quartz.CGBitmapContextCreateImage(ctx)
+    out_path = os.path.join(output_dir, f"page_{page_idx + 1}.png")
+
+    dest = Quartz.CGImageDestinationCreateWithURL(
+        Quartz.NSURL.fileURLWithPath_(out_path),
+        Quartz.kUTTypePNG, 1, None
+    )
+    Quartz.CGImageDestinationAddImage(dest, cg_img, None)
+    Quartz.CGImageDestinationFinalize(dest)
+
+    print(f"  [Rendered] page {page_idx + 1} -> {out_path} ({pw}x{ph})", file=sys.stderr)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Extract text from PDF on macOS")
+    parser.add_argument("pdf_path", help="Path to the PDF file")
+    parser.add_argument("--render-scanned", action="store_true",
+                        help="Render scanned/blank pages as PNG images")
+    parser.add_argument("--output-dir", default="/tmp/pdf_rendered/",
+                        help="Output directory for rendered images")
+    parser.add_argument("--scale", type=float, default=6.0,
+                        help="Scale multiplier for rendering (default: 6.0)")
+    args = parser.parse_args()
+
+    extract_text(args.pdf_path, args.render_scanned, args.output_dir, args.scale)

package/skills/pdf-extract/skill.json

@@ -0,0 +1,5 @@
+{
+  "name": "pdf-extract",
+  "version": "0.1.0",
+  "description": ""
+}