@xberg-io/opencode-xberg 1.3.6 → 1.3.8

package/.opencode/ai-rulez-bundle.json

@@ -0,0 +1,51 @@
+{
+  "mcp": {
+    "xberg": {
+      "type": "local",
+      "command": [
+        "${PLUGIN_ROOT}/scripts/mcp-launch.sh",
+        "mcp",
+        "--transport",
+        "stdio"
+      ]
+    }
+  },
+  "skills": {
+    "batch-extraction": {
+      "name": "batch-extraction",
+      "description": "Use when extracting from many files at once with shared config, bounded parallelism, per-file overrides, and error recovery. Covers the `batch` command, `--file-configs`, `--max-concurrent`, and output layout."
+    },
+    "chunking": {
+      "name": "chunking",
+      "description": "Use when splitting extracted text into chunks for LLM context windows or RAG ingestion. Covers chunk size, overlap, markdown/yaml/semantic chunkers, tokenizer-based sizing, and the standalone `chunk` command."
+    },
+    "extracting-keywords": {
+      "name": "extracting-keywords",
+      "description": "Use when extracting keywords (YAKE/RAKE) from documents — and, secondarily, when detecting document language or generating embeddings for RAG and search. Covers the keyword config (and its feature gating), `--detect-language`, and the standalone `embed` command with real flags."
+    },
+    "extracting-tables": {
+      "name": "extracting-tables",
+      "description": "Use when extracting tabular data from PDFs, spreadsheets, or images. Covers layout-aware table detection, table model selection, output formats (markdown / JSON cells), and known limits."
+    },
+    "extracting-with-ocr": {
+      "name": "extracting-with-ocr",
+      "description": "Use when extracting text from scanned PDFs, photographed pages, or images that have no embedded text layer. Covers OCR backends, language packs, force-OCR, and performance tuning."
+    },
+    "picking-a-format": {
+      "name": "picking-a-format",
+      "description": "Use when choosing an output format for extracted documents — plain text, markdown, djot, HTML, JSON, or DocTags. Maps consumer (LLM, parser, archive) to the right `--format` / `--content-format` pair."
+    },
+    "xberg": {
+      "name": "xberg",
+      "description": "Extract text, tables, metadata, and images from 107 document formats (PDF, Office, images, HTML, email, archives, academic) using Xberg. Use when writing code that calls Xberg APIs in Python, Node.js/TypeScript, Rust, or CLI. Covers installation, extraction (sync/async), configuration (OCR, chunking, output format), batch processing, error handling, and plugins."
+    }
+  },
+  "commands": {
+    "xberg-extract": {
+      "description": "Extract a document locally with Xberg"
+    },
+    "xberg-ocr": {
+      "description": "Extract scanned documents with OCR"
+    }
+  }
+}

package/.opencode/ai-rulez-content.js

@@ -0,0 +1,167 @@
+// AI-RULEZ :: GENERATED FILE — DO NOT EDIT
+// Content-Hash: blake3:822c510deb007e4f72877ad42f29918d0499f210ae363e82d465a0f0331ab294
+// Source-Hash: blake3:3fca38eb564d0ed9c9194d070fd8dec96de727edc92e82954cb4b9af52cddbaf
+// Schema-Version: v1
+
+// Generated by ai-rulez. Registers the skills, commands, agents, and MCP servers bundled with
+// this plugin in OpenCode v2. OpenCode only scans its own config directories, so a plugin installed from
+// npm has to register its bundled content itself.
+import { readdir, readFile } from "node:fs/promises";
+import path from "node:path";
+
+const root = import.meta.dirname;
+
+// Returns the text after a leading YAML frontmatter block. The frontmatter fields the
+// helper needs (name, description) are resolved when the bundle is generated and read
+// from ai-rulez-bundle.json, so no YAML is parsed here.
+function body(text) {
+  const match = /^---\r?\n[\s\S]*?\r?\n---\r?\n?([\s\S]*)$/u.exec(text);
+  return match ? match[1] : text;
+}
+
+function skip(kind, name, error) {
+  console.warn(`ai-rulez: skipping bundled ${kind} ${name}: ${error?.message ?? error}`);
+}
+
+async function entries(dir) {
+  try {
+    return (await readdir(path.join(root, dir), { withFileTypes: true })).toSorted((a, b) =>
+      a.name.localeCompare(b.name),
+    );
+  } catch {
+    return [];
+  }
+}
+
+async function readBundle() {
+  try {
+    return JSON.parse(await readFile(path.join(root, "ai-rulez-bundle.json"), "utf8"));
+  } catch {
+    return {};
+  }
+}
+
+// Plugin install directory: the parent of the .opencode directory this file lives in.
+const pluginRoot = path.dirname(root);
+
+// Resolves ${PLUGIN_ROOT} to the install directory and ${NAME} to process.env at runtime,
+// so no install path or secret is baked into the package.
+function expand(value) {
+  return value.replaceAll(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}/gu, (_, name) =>
+    name === "PLUGIN_ROOT" ? pluginRoot : (process.env[name] ?? ""),
+  );
+}
+
+function mcpServer(server) {
+  if (server.type === "remote") {
+    const entry = { type: "remote", url: expand(server.url) };
+    if (server.disabled) entry.disabled = true;
+    return entry;
+  }
+  const entry = { type: "local", command: server.command.map(expand) };
+  if (server.disabled) entry.disabled = true;
+  if (server.environment) {
+    entry.environment = Object.fromEntries(
+      Object.entries(server.environment).map(([key, value]) => [key, expand(value)]),
+    );
+  }
+  return entry;
+}
+
+async function load() {
+  const bundle = await readBundle();
+  const skills = [];
+  for (const entry of await entries("skills")) {
+    if (!entry.isDirectory()) continue;
+    const file = path.join(root, "skills", entry.name, "SKILL.md");
+    try {
+      const info = bundle.skills?.[entry.name] ?? {};
+      skills.push({
+        id: entry.name,
+        name: info.name || entry.name,
+        description: info.description,
+        path: file,
+        content: body(await readFile(file, "utf8")),
+      });
+    } catch (error) {
+      skip("skill", entry.name, error);
+    }
+  }
+  const commands = [];
+  for (const entry of await entries("commands")) {
+    if (!entry.isFile() || !entry.name.endsWith(".md")) continue;
+    const name = entry.name.slice(0, -3);
+    try {
+      const template = body(await readFile(path.join(root, "commands", entry.name), "utf8"));
+      commands.push({ name, description: bundle.commands?.[name]?.description, template });
+    } catch (error) {
+      skip("command", name, error);
+    }
+  }
+  const agents = [];
+  for (const entry of await entries("agents")) {
+    if (!entry.isFile() || !entry.name.endsWith(".md")) continue;
+    const id = entry.name.slice(0, -3);
+    try {
+      // The bundle holds settings resolved at generation time (the model in particular).
+      const system = body(await readFile(path.join(root, "agents", entry.name), "utf8"));
+      agents.push({ id, system: system.trim(), settings: bundle.agents?.[id] ?? {} });
+    } catch (error) {
+      skip("agent", id, error);
+    }
+  }
+  return { skills, commands, agents, mcp: bundle.mcp ?? {} };
+}
+
+export async function registerBundledContent(ctx) {
+  const { skills, commands, agents, mcp } = await load();
+  if (skills.length > 0) {
+    await ctx.skill.transform((editor) => {
+      for (const skill of skills) if (!editor.get(skill.id)) editor.add(skill);
+    });
+  }
+  if (commands.length > 0) {
+    await ctx.command.transform((editor) => {
+      for (const command of commands) {
+        editor.add({
+          name: command.name,
+          description: command.description,
+          execute: async ({ sessionID, prompt, delivery }) => {
+            await ctx.session.prompt({
+              ...prompt,
+              sessionID,
+              text: command.template.replaceAll("$ARGUMENTS", () => String(prompt.text ?? "")),
+              delivery,
+            });
+          },
+        });
+      }
+    });
+  }
+  if (agents.length > 0) {
+    await ctx.agent.transform((editor) => {
+      for (const agent of agents) {
+        editor.update(agent.id, (item) => {
+          const settings = agent.settings;
+          if (settings.description) item.description = settings.description;
+          if (["subagent", "primary", "all"].includes(settings.mode)) item.mode = settings.mode;
+          if (typeof settings.hidden === "boolean") item.hidden = settings.hidden;
+          if (typeof settings.model === "string" && settings.model.includes("/")) {
+            const split = settings.model.indexOf("/");
+            item.model = { providerID: settings.model.slice(0, split), id: settings.model.slice(split + 1) };
+            if (settings.variant) item.model.variant = settings.variant;
+          }
+          for (const key of ["temperature", "top_p"]) {
+            if (typeof settings[key] === "number") item.request.body[key] = settings[key];
+          }
+          if (agent.system) item.system = agent.system;
+        });
+      }
+    });
+  }
+  if (Object.keys(mcp).length > 0) {
+    await ctx.mcp.transform((editor) => {
+      for (const [name, server] of Object.entries(mcp)) editor.set(name, mcpServer(server));
+    });
+  }
+}

package/.opencode/commands/xberg-extract.md

@@ -0,0 +1,14 @@
+---
+description: "Extract a document locally with Xberg"
+---
+
+<!--
+AI-RULEZ :: GENERATED FILE — DO NOT EDIT
+Content-Hash: blake3:519efaa10fd8e92073089cdc9bdebf7151a312a3f280d8d70987603bf31d9113
+Source-Hash: blake3:3fca38eb564d0ed9c9194d070fd8dec96de727edc92e82954cb4b9af52cddbaf
+Schema-Version: v1
+-->
+
+Extract this document: $ARGUMENTS
+
+Read the xberg and picking-a-format skills. Use `xberg extract <file> --format json` and choose --content-format for the consumer. Preserve input paths as separate arguments. Report extraction errors and content omissions.

package/.opencode/commands/xberg-ocr.md

@@ -0,0 +1,14 @@
+---
+description: "Extract scanned documents with OCR"
+---
+
+<!--
+AI-RULEZ :: GENERATED FILE — DO NOT EDIT
+Content-Hash: blake3:d3a65f45804e9541bb33d628e4a7d7b48f806adb438e1fc8b15c92baef538908
+Source-Hash: blake3:3fca38eb564d0ed9c9194d070fd8dec96de727edc92e82954cb4b9af52cddbaf
+Schema-Version: v1
+-->
+
+OCR this document: $ARGUMENTS
+
+Read the extracting-with-ocr skill. Confirm the document needs OCR and language packs are installed. Use `xberg extract <file> --force-ocr=true` when forced OCR is appropriate. Preserve paths as separate arguments and report OCR failures.

package/.opencode/plugins/xberg.js

@@ -1,19 +1,10 @@
 // AI-RULEZ :: GENERATED FILE — DO NOT EDIT
-// Content-Hash: blake3:0c4b20b2428ca03b76e22085bdcc856fbac85091cd3f6d157655d6c25c02714f
-// Source-Hash: blake3:0a63e6793dfbd7671ca2cfec5688efa8523d2e4087553f9f6503f2d74cef8b78
+// Content-Hash: blake3:3fe1a33d721dfa8b49134cbe88659999cf9e481c558eb8e4b5ac81bc672c88db
+// Source-Hash: blake3:3fca38eb564d0ed9c9194d070fd8dec96de727edc92e82954cb4b9af52cddbaf
 // Schema-Version: v1
 
-import { tool } from "@opencode-ai/plugin";
 import { spawn } from "node:child_process";
-
-const schema = tool.schema;
-
-const wireFormat = schema.enum(["text", "json", "toon"]).default("json").describe("CLI output format.");
-
-const contentFormat = schema
-  .enum(["plain", "markdown", "djot", "html", "json"])
-  .optional()
-  .describe("Document content rendering format.");
+import { registerBundledContent } from "../ai-rulez-content.js";
 
 function hasValue(value) {
   return value !== undefined && value !== null && value !== "";
@@ -37,14 +28,12 @@ function validateJson(value, name) {
   }
 }
 
-function runCli(args, context) {
-  const directory = context?.directory ?? context?.worktree ?? process.cwd();
-
+function runCli(args, context, directory) {
   return new Promise((resolve, reject) => {
     const child = spawn("xberg", args, {
       cwd: directory,
       env: process.env,
-      signal: context?.abort,
+      signal: context?.signal,
       stdio: ["ignore", "pipe", "pipe"],
     });
 
@@ -56,8 +45,7 @@ function runCli(args, context) {
     child.on("error", (error) => {
       if (error.code === "ENOENT") {
         resolve({
-          title: "xberg CLI not found",
-          output:
+          content:
             "Install the xberg CLI with `brew install xberg-io/tap/xberg`, or run it via `npx -y @xberg-io/xberg-cli` / `uvx --from xberg-cli xberg`.",
           metadata: { exitCode: 127, command: "xberg", subcommand: args[0] },
         });
@@ -71,8 +59,7 @@ function runCli(args, context) {
       const output = [stdoutText, stderrText && `stderr:\n${stderrText}`].filter(Boolean).join("\n\n");
 
       resolve({
-        title: exitCode === 0 ? `xberg ${args[0]}` : `xberg ${args[0]} failed`,
-        output: output || "(no output)",
+        content: output || "(no output)",
         metadata: {
           exitCode,
           signal,
@@ -84,49 +71,75 @@ function runCli(args, context) {
   });
 }
 
-export const XbergPlugin = () =>
-  Promise.resolve({
-    tool: {
-      xberg_extract: tool({
-        description: "Extract text, tables, metadata, and images from a local document with the xberg CLI.",
-        args: {
-          path: schema.string().min(1).describe("Path to the local document."),
-          format: wireFormat,
-          content_format: contentFormat,
-          mime_type: schema.string().min(1).optional().describe("Optional MIME type hint."),
-          config_json: schema.string().min(2).optional().describe("Optional ExtractionConfig JSON."),
-        },
-        async execute(args, context) {
-          validateJson(args.config_json, "config_json");
+const wireFormat = { type: "string", enum: ["text", "json", "toon"], default: "json" };
+const contentFormat = { type: "string", enum: ["plain", "markdown", "djot", "html", "json"] };
+
+function extractTool(directory) {
+  return {
+    name: "xberg_extract",
+    description: "Extract text, tables, metadata, and images from a local document with the xberg CLI.",
+    input: {
+      type: "object",
+      properties: {
+        path: { type: "string", minLength: 1 },
+        format: wireFormat,
+        content_format: contentFormat,
+        mime_type: { type: "string", minLength: 1 },
+        config_json: { type: "string", minLength: 2 },
+      },
+      required: ["path"],
+      additionalProperties: false,
+    },
+    execute(args, context) {
+      validateJson(args.config_json, "config_json");
+      const cliArgs = ["extract", args.path, "--format", args.format ?? "json"];
+      pushOption(cliArgs, "--content-format", args.content_format);
+      pushOption(cliArgs, "--mime-type", args.mime_type);
+      pushOption(cliArgs, "--config-json", args.config_json);
+      return runCli(cliArgs, context, directory);
+    },
+  };
+}
 
-          const cliArgs = ["extract", args.path, "--format", args.format];
-          pushOption(cliArgs, "--content-format", args.content_format);
-          pushOption(cliArgs, "--mime-type", args.mime_type);
-          pushOption(cliArgs, "--config-json", args.config_json);
+function detectTool(directory) {
+  return {
+    name: "xberg_detect",
+    description: "Detect the MIME type for a local file with the xberg CLI.",
+    input: {
+      type: "object",
+      properties: { path: { type: "string", minLength: 1 }, format: wireFormat },
+      required: ["path"],
+      additionalProperties: false,
+    },
+    execute(args, context) {
+      return runCli(["detect", args.path, "--format", args.format ?? "json"], context, directory);
+    },
+  };
+}
 
-          return await runCli(cliArgs, context);
-        },
-      }),
-      xberg_detect: tool({
-        description: "Detect the MIME type for a local file with the xberg CLI.",
-        args: {
-          path: schema.string().min(1).describe("Path to the local file."),
-          format: wireFormat,
-        },
-        async execute(args, context) {
-          return await runCli(["detect", args.path, "--format", args.format], context);
-        },
-      }),
-      xberg_formats: tool({
-        description: "List document formats supported by the xberg CLI.",
-        args: {
-          format: wireFormat,
-        },
-        async execute(args, context) {
-          return await runCli(["formats", "--format", args.format], context);
-        },
-      }),
+function formatsTool(directory) {
+  return {
+    name: "xberg_formats",
+    description: "List document formats supported by the xberg CLI.",
+    input: { type: "object", properties: { format: wireFormat }, additionalProperties: false },
+    execute(args, context) {
+      return runCli(["formats", "--format", args.format ?? "json"], context, directory);
     },
-  });
+  };
+}
 
-export default XbergPlugin;
+export default {
+  id: "xberg",
+  async setup(ctx) {
+    await registerBundledContent(ctx);
+    await ctx.tool.transform((editor) => {
+      for (const tool of [
+        extractTool(ctx.location.directory),
+        detectTool(ctx.location.directory),
+        formatsTool(ctx.location.directory),
+      ]) {
+        editor.add(tool);
+      }
+    });
+  },
+};

package/.opencode/skills/batch-extraction/SKILL.md

@@ -0,0 +1,168 @@
+---
+name: batch-extraction
+description: Use when extracting from many files at once with shared config, bounded parallelism, per-file overrides, and error recovery. Covers the `batch` command, `--file-configs`, `--max-concurrent`, and output layout.
+---
+
+<!--
+AI-RULEZ :: GENERATED FILE — DO NOT EDIT
+Content-Hash: blake3:3a70b47dd77fcd3f37e83bed3d32b1ad1ea9d6c1642ae8e0ff7b09b55939858a
+Source-Hash: blake3:3fca38eb564d0ed9c9194d070fd8dec96de727edc92e82954cb4b9af52cddbaf
+Schema-Version: v1
+-->
+
+# Batch extraction
+
+Use this when processing a directory or glob of documents in one pass.
+`xberg batch` shares one extraction config across every file, runs
+extractions concurrently, and returns one structured array — failures on
+individual files do not abort the run.
+
+## Basic usage
+
+```bash
+# Glob expands to many paths; results come back as a JSON array (default)
+xberg batch *.pdf
+
+# Mixed formats, markdown content for LLM ingestion
+xberg batch docs/*.docx --content-format markdown
+
+# Recurse with the shell, then extract
+xberg batch $(find ./corpus -name '*.pdf')
+```
+
+`batch` defaults to `--format json` (vs `--format text` for single
+`extract`). Each array entry is a full extraction result, so downstream
+code can index by position into the input path list.
+
+```bash
+xberg batch reports/*.pdf \
+  | jq '.[] | {chars: (.content | length), mime: .mime_type}'
+```
+
+## Parallelism
+
+`--max-concurrent` caps how many files extract at once. When omitted, the scheduler derives document concurrency from the total thread budget. Lower it on memory-constrained hosts or when OCR/ML
+models are active, since each in-flight extraction holds its own buffers.
+Layout-heavy batches are further limited (1 concurrent extraction for
+all-PDF-layout batches, 2 for mixed layout):
+
+```bash
+# Cap at 4 concurrent extractions
+xberg batch scans/*.pdf --ocr true --max-concurrent 4
+```
+
+`--max-threads` additionally caps *total* internal threads (Rayon, ONNX
+intra-op, the batch semaphore) for tightly constrained environments:
+
+```bash
+xberg batch *.pdf --max-concurrent 2 --max-threads 4
+```
+
+## Per-file config overrides
+
+A single shared config does not always fit. `--file-configs` points at a
+JSON file mapping each path to its own override object, merged on top of
+the shared config for that file only:
+
+```json
+{
+  "scan.pdf": { "force_ocr": true },
+  "report.pdf": { "output_format": "markdown" },
+  "data.xlsx": { "output_format": "json" }
+}
+```
+
+```bash
+xberg batch scan.pdf report.pdf data.xlsx --file-configs overrides.json
+```
+
+Keys are file paths (matching the paths passed on the command line);
+values are per-file extraction config objects in snake_case, the same
+shape as a config file.
+
+## Output layout
+
+For text/toon output with image extraction, `--output-dir` controls where
+referenced image files (e.g. `image_0.png`) are written; the directory
+must already exist. JSON output embeds image bytes inline and ignores
+`--output-dir`.
+
+```bash
+mkdir -p out/images
+xberg batch slides/*.pptx --extract-images true --output-dir out/images --format text
+```
+
+## Error recovery
+
+Batch extraction is fault-tolerant per file: one unreadable or corrupt
+document does not stop the rest. Inspect results for partial content and
+surfaced errors rather than relying on the process exit code alone. Pair
+with `--max-concurrent` to avoid exhausting memory when a few large files
+sit in a big batch.
+
+## Shared config
+
+Every `extract` flag also applies to `batch` (OCR, chunking, layout,
+content format, etc.) and is shared across all files unless a
+`--file-configs` entry overrides it:
+
+```bash
+xberg batch invoices/*.pdf \
+  --layout --layout-table-model slanet_wireless \
+  --content-format markdown --max-concurrent 8
+```
+
+A config file works too and auto-discovers from the cwd upward:
+
+```toml
+output_format = "markdown"
+
+[ocr]
+backend = "tesseract"
+language = "eng"
+```
+
+```bash
+xberg batch corpus/*.pdf --config xberg.toml
+```
+
+## Programmatic access
+
+From Python, `extract_batch` takes a list of `ExtractInput`s and returns one
+envelope whose `results` array holds a document per input:
+
+```python
+from xberg import ExtractInput, extract_batch, ExtractionConfig
+
+config = ExtractionConfig(output_format="markdown")
+
+inputs = [ExtractInput(uri=p) for p in ["a.pdf", "b.docx", "c.xlsx"]]
+output = await extract_batch(inputs, config)
+
+for doc in output.results:
+    print(len(doc.content))
+```
+
+Per-input overrides go on `ExtractInput.config` (a `FileExtractionConfig`).
+Node.js mirrors this with `extractBatch`; Rust uses `extract_batch(inputs, &config)`.
+See `references/python-api.md`, `references/nodejs-api.md`, and
+`references/rust-api.md` in the sibling `xberg` skill.
+
+## MCP
+
+When the `xberg` MCP server is registered, prefer the
+`extract_batch` tool over shelling out — it takes an array of input objects
+and a config object and returns structured results directly.
+
+## Common pitfalls
+
+- **Default format differs** — `batch` defaults to `--format json`,
+  `extract` to `--format text`. Set `--format` explicitly if a script
+  depends on one shape.
+- **`--output-dir` must exist** — the CLI does not create it.
+- **Memory blowups** — large batches with OCR/layout active may need an explicit
+  `--max-concurrent` ceiling.
+- **`--file-configs` path keys** — must match the paths as passed on the
+  command line, not absolute-resolved variants.
+
+See `references/cli-reference.md` for the full `batch` flag set.