@extraktor/cli 0.0.0-stage → 0.1.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Dennis Kortsch
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,3 +1,59 @@
-# Temporary Holding Version
+# Extraktor CLI
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+Read live web pages as Markdown and search the web from the terminal. Made for AI agents: each command prints Markdown that an agent can use directly. Run `extraktor --help` for every command and option.
+
+```sh
+npm install --global @extraktor/cli
+extraktor login                      # or: export EXTRAKTOR_API_KEY=ext_...
+extraktor extract https://example.com
+```
+
+Extraktor needs a Pro plan. Make API keys at <https://extraktor.app/developers>.
+
+## Tasks
+
+| Task | Command |
+| --- | --- |
+| Read a page, then answer or summarize | `extraktor extract <url>` |
+| Get facts from a page | `extraktor extract <url> --find "<text>"` |
+| Read or compare 2 to 5 pages | `extraktor extract <url> <url> ...` |
+| Quote a page with a link to each quote | `extraktor extract <url> --excerpts --focus "<topic>"` |
+| Save the complete page as Markdown | `extraktor extract <url> --save page.md` |
+| Get contact details | `extraktor extract <url> --contacts` |
+| Take a full-page screenshot | `extraktor extract <url> --screenshot` |
+| Get the design system as DESIGN.md | `extraktor extract <url> --design-file DESIGN.md` |
+| Audit the SEO of a page | `extraktor extract <url> --seo --keyword "<keyword>"` |
+| Find how agents can use a site | `extraktor extract <url> --agent-access` |
+| Find pages when you have no URL | `extraktor search "<query>"` |
+| See what Google shows for a keyword | `extraktor search "<query>" --serp` |
+
+A long page comes in parts of about 24,000 characters. Each part gives the command for the next part. The CLI keeps each page for 10 minutes, so `--find`, `--offset` and the same command again use no credit.
+
+## Add the MCP server to your agents
+
+```sh
+npx -y @extraktor/cli mcp add
+```
+
+The command finds Claude Code, Codex, Cursor, VS Code, Gemini CLI and Windsurf on this computer and adds the server `https://extraktor.app/mcp` to each one. It keeps the other servers and settings, and a second run changes nothing. Each agent opens a sign-in page when it first uses Extraktor. The output shows the sign-in step for each agent.
+
+Use `--agent <name>` to change only some agents, and `--json` for a JSON result. `extraktor mcp remove` removes the server. For the Claude app, open **Settings > Connectors > Add custom connector** and paste the server URL.
+
+## Exit codes
+
+| Code | Meaning |
+| --- | --- |
+| 0 | Success. |
+| 1 | The page, the search or the server failed. Read the message. |
+| 2 | The command or an option is not correct. The message tells the fix. |
+| 3 | Sign-in, a plan or credits are necessary. |
+
+Each error tells the next step. With `--json`, errors are JSON on standard output.
+
+## Privacy
+
+The CLI calls the Extraktor MCP server at `https://extraktor.app/mcp` (or `$EXTRAKTOR_URL`). It saves the API key in `~/.config/extraktor/credentials.json` and results for 10 minutes in `~/.cache/extraktor/results`. When a coding agent runs the CLI, the CLI sends the agent name (for example `claude-code`) in the `X-Extraktor-Agent` header, and no other data about the agent.
+
+## License
+
+[MIT](LICENSE)

package/dist/agent.js

@@ -0,0 +1,26 @@
+const is = (name, value) => (env) => env[name] === value;
+const has = (name) => (env) => Boolean(env[name]?.trim());
+const AGENTS = [
+    ["claude-code", is("CLAUDECODE", "1")],
+    ["codex", has("CODEX_THREAD_ID")],
+    ["cursor", is("CURSOR_AGENT", "1")],
+    ["gemini-cli", is("GEMINI_CLI", "1")],
+    ["opencode", is("OPENCODE", "1")],
+    ["amp", has("AMP_CURRENT_THREAD_ID")],
+    ["qwen-code", has("QWEN_CODE_SESSION_ID")],
+    ["auggie", is("AUGMENT_AGENT", "1")],
+    ["github-copilot", is("COPILOT_AGENT", "1")],
+    ["warp", has("OZ_RUN_ID")],
+];
+const AGENT_NAME = /^[a-z0-9][a-z0-9._-]{0,39}$/iu;
+/** The agent name, or null when no known agent runs the CLI. */
+export const detectAgent = (env) => {
+    for (const [name, detect] of AGENTS) {
+        if (detect(env)) {
+            return name;
+        }
+    }
+    // AI_AGENT is a shared convention: its value is the agent name.
+    const named = env.AI_AGENT?.trim();
+    return named && AGENT_NAME.test(named) ? named.toLowerCase() : null;
+};

package/dist/auth.js

@@ -0,0 +1,67 @@
+import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { homedir } from "node:os";
+import path from "node:path";
+import { CliError, EXIT } from "./errors.js";
+export const API_KEY_FORMAT = /^ext_[a-f0-9]{64}$/u;
+export const credentialsPath = (env = process.env) => {
+    const base = env.XDG_CONFIG_HOME ||
+        (process.platform === "win32" && env.APPDATA) ||
+        path.join(homedir(), ".config");
+    return path.join(base, "extraktor", "credentials.json");
+};
+const readCredentials = async (file) => {
+    let text;
+    try {
+        text = await readFile(file, "utf-8");
+    }
+    catch {
+        return {};
+    }
+    const { credentialsSchema } = await import("./schemas.js");
+    try {
+        return credentialsSchema.parse(JSON.parse(text));
+    }
+    catch {
+        throw new CliError(`The saved credentials are not correct: ${file}`, EXIT.access, 'Delete the file, then run "extraktor login" again.');
+    }
+};
+const writeCredentials = async (file, credentials) => {
+    if (Object.keys(credentials).length === 0) {
+        await rm(file, { force: true });
+        return;
+    }
+    await mkdir(path.dirname(file), { recursive: true, mode: 0o700 });
+    await writeFile(file, `${JSON.stringify(credentials, null, 2)}\n`, {
+        mode: 0o600,
+    });
+};
+/** The API key for this server: EXTRAKTOR_API_KEY first, then the saved key. */
+export const resolveApiKey = async (baseUrl, env = process.env) => {
+    if (env.EXTRAKTOR_API_KEY) {
+        return env.EXTRAKTOR_API_KEY.trim();
+    }
+    const credentials = await readCredentials(credentialsPath(env));
+    return credentials[baseUrl]?.apiKey ?? null;
+};
+export const saveApiKey = async (baseUrl, apiKey, env = process.env) => {
+    if (!API_KEY_FORMAT.test(apiKey)) {
+        throw new CliError("The API key is not correct. An Extraktor API key starts with ext_.", EXIT.usage);
+    }
+    const file = credentialsPath(env);
+    const credentials = await readCredentials(file);
+    await writeCredentials(file, {
+        ...credentials,
+        [baseUrl]: { apiKey, savedAt: new Date().toISOString() },
+    });
+    return file;
+};
+/** Deletes the saved key for this server. Returns false when there was none. */
+export const deleteSavedApiKey = async (baseUrl, env = process.env) => {
+    const file = credentialsPath(env);
+    const credentials = await readCredentials(file);
+    if (!credentials[baseUrl]) {
+        return false;
+    }
+    await writeCredentials(file, Object.fromEntries(Object.entries(credentials).filter(([server]) => server !== baseUrl)));
+    return true;
+};

package/dist/cache.js

@@ -0,0 +1,113 @@
+/**
+ * A short-lived local cache of tool results. Agents read long output in
+ * slices: they run the same command again with head, sed or grep. Without the
+ * cache, each slice is a new extraction that costs a credit and a page load.
+ */
+import { createHash } from "node:crypto";
+import { mkdir, readdir, readFile, rename, rm, stat, writeFile, } from "node:fs/promises";
+import { homedir } from "node:os";
+import path from "node:path";
+import { callTool } from "./mcp.js";
+export const CACHE_TTL_MS = 10 * 60_000;
+export const cacheDir = (env) => path.join(env.XDG_CACHE_HOME ||
+    (process.platform === "win32" && env.LOCALAPPDATA) ||
+    path.join(homedir(), ".cache"), "extraktor", "results");
+const cacheFile = (dir, { baseUrl, name, args }) => path.join(dir, `${createHash("sha256")
+    .update(JSON.stringify([baseUrl, name, args]))
+    .digest("hex")}.json`);
+const isFresh = async (file, now) => {
+    try {
+        const { mtimeMs } = await stat(file);
+        return now - mtimeMs < CACHE_TTL_MS;
+    }
+    catch {
+        return false;
+    }
+};
+const readCached = async (file) => {
+    if (!(await isFresh(file, Date.now()))) {
+        return null;
+    }
+    try {
+        const { toolResultSchema } = await import("./schemas.js");
+        return toolResultSchema.parse(JSON.parse(await readFile(file, "utf-8")));
+    }
+    catch {
+        return null;
+    }
+};
+/** Deletes results older than the cache time, so the directory stays small. */
+const pruneExpired = async (dir, now) => {
+    const files = await readdir(dir);
+    await Promise.all(files.map(async (name) => {
+        const file = path.join(dir, name);
+        if (!(await isFresh(file, now))) {
+            await rm(file, { force: true });
+        }
+    }));
+};
+const writeCached = async (dir, file, result) => {
+    await mkdir(dir, { recursive: true, mode: 0o700 });
+    const temporary = `${file}.${process.pid}.tmp`;
+    await writeFile(temporary, JSON.stringify(result), { mode: 0o600 });
+    await rename(temporary, file);
+    await pruneExpired(dir, Date.now());
+};
+const OUTPUT_KEYS = new Set([
+    "excerpts",
+    "summary",
+    "screenshot",
+    "contacts",
+    "messaging",
+    "seo",
+    "design",
+    "agentAccess",
+]);
+/**
+ * The page text part of an extract result with outputs: the result of the
+ * same page without outputs. A later read, --find or --offset of the page
+ * then uses it and makes no new request.
+ */
+const plainPage = (options, result) => {
+    if (options.name !== "extract" ||
+        !Object.entries(options.args).some(([key, value]) => value !== undefined && OUTPUT_KEYS.has(key))) {
+        return null;
+    }
+    const page = Object.fromEntries(Object.entries(result.structuredContent ?? {}).filter(([key]) => !OUTPUT_KEYS.has(key)));
+    return [
+        { ...options, args: { url: options.args.url } },
+        {
+            ...result,
+            // Keep the page warnings. Drop the image of a screenshot.
+            content: [
+                { type: "text", text: JSON.stringify(page) },
+                ...result.content.slice(1).filter((item) => item.type === "text"),
+            ],
+            structuredContent: page,
+        },
+    ];
+};
+/**
+ * Calls the tool, or returns the saved result of the same call from the last
+ * 10 minutes. Only successful results are saved.
+ */
+export const callToolCached = async (options, env, useCache) => {
+    const dir = cacheDir(env);
+    const file = cacheFile(dir, options);
+    const cached = useCache ? await readCached(file) : null;
+    if (cached) {
+        return cached;
+    }
+    const result = await callTool(options);
+    try {
+        await writeCached(dir, file, result);
+        const plain = plainPage(options, result);
+        if (plain) {
+            await writeCached(dir, cacheFile(dir, plain[0]), plain[1]);
+        }
+    }
+    catch {
+        // The cache only saves time. A full disk must not fail the command.
+    }
+    return result;
+};

package/dist/cli.js

@@ -0,0 +1,11 @@
+#!/usr/bin/env node
+import { text } from "node:stream/consumers";
+import { run } from "./run.js";
+const exitCode = await run(process.argv.slice(2), {
+    stdout: (output) => process.stdout.write(output),
+    stderr: (output) => process.stderr.write(output),
+    env: process.env,
+    readStdin: () => text(process.stdin),
+    cwd: process.cwd(),
+});
+process.exitCode = exitCode;

package/dist/errors.js

@@ -0,0 +1,44 @@
+/** Exit codes and the error type that every command reports. */
+export const EXIT = {
+    ok: 0,
+    /** The page, the search or the server did not complete the request. */
+    failure: 1,
+    /** The command or its options are not correct. */
+    usage: 2,
+    /** Sign-in, plan or credits are necessary. */
+    access: 3,
+};
+const DEFAULT_CODES = new Map([
+    [EXIT.failure, "FAILED"],
+    [EXIT.usage, "INVALID_USAGE"],
+    [EXIT.access, "ACCESS_REQUIRED"],
+]);
+/** A failure that the CLI reports to the user and stops on. */
+export class CliError extends Error {
+    name = "CliError";
+    exitCode;
+    guidance;
+    /** A stable code for --json output, for example PAGE_UNAVAILABLE. */
+    code;
+    constructor(message, exitCode, guidance, code) {
+        super(message);
+        this.exitCode = exitCode;
+        this.guidance = guidance;
+        this.code = code ?? DEFAULT_CODES.get(exitCode) ?? "FAILED";
+    }
+    /** The error as --json prints it. */
+    toJSON() {
+        const suffix = ` (${this.code})`;
+        const error = {
+            code: this.code,
+            message: this.message.endsWith(suffix)
+                ? this.message.slice(0, -suffix.length)
+                : this.message,
+            exitCode: this.exitCode,
+        };
+        if (this.guidance) {
+            error.guidance = this.guidance;
+        }
+        return { error };
+    }
+}