@x12i/sdk 0.1.0

package/dist/client.d.ts

@@ -0,0 +1,53 @@
+import { ParserRegistry, type ClassifyCandidate, type FetchResult, type ParsedResult, type Parser } from "@x12i/core";
+import { type FetchOptions } from "@x12i/fetcher";
+import { type IngestOptions, type IngestResult } from "@x12i/ingest";
+export interface WebAgentClientOptions {
+    /** Provide your own registry (e.g. to only register a subset of parsers). Defaults to every first-party parser. */
+    registry?: ParserRegistry;
+    /** Default fetch options applied to every call, overridable per-call. */
+    fetchOptions?: FetchOptions;
+    /**
+     * Reuse a single browser instance across calls instead of launching one
+     * per fetch. Recommended for long-lived processes (the API/MCP server).
+     * Call `.close()` when done.
+     */
+    reuseBrowser?: boolean;
+}
+/**
+ * The single entry point every other package (CLI, REST API, MCP server)
+ * wraps: download a URL to Markdown, and/or classify+parse it into typed
+ * JSON using the registered parsers.
+ */
+export declare class WebAgentClient {
+    readonly registry: ParserRegistry;
+    private readonly defaultFetchOptions;
+    private readonly reuseBrowser;
+    private browserPromise;
+    constructor(options?: WebAgentClientOptions);
+    /** Register an additional parser on this client's registry. */
+    use<T>(parser: Parser<T>): this;
+    /** Download a URL and return { url, html, markdown, ... }. */
+    fetch(url: string, options?: FetchOptions): Promise<FetchResult>;
+    /** Fetch a URL, then rank every registered parser's fit without running parse(). */
+    classify(url: string, options?: FetchOptions): Promise<ClassifyCandidate[]>;
+    /** Fetch a URL, classify it, and parse it with the best-fit parser. */
+    parse<T = unknown>(url: string, options?: FetchOptions): Promise<ParsedResult<T>>;
+    /** Fetch a URL and force a specific parser by type id, skipping classification. */
+    parseAs<T = unknown>(type: string, url: string, options?: FetchOptions): Promise<ParsedResult<T>>;
+    /** List every registered parser's type id + description. */
+    listParsers(): {
+        type: string;
+        description?: string;
+        version?: string;
+    }[];
+    /**
+     * Ingest a curated list / README URL: follow outbound GitHub skill links
+     * one level deep and extract targeted files (SKILL.md, .md, .sh, …).
+     * Uses GitHub raw/API — not Playwright.
+     */
+    ingest(rootUrl: string, options?: IngestOptions): Promise<IngestResult>;
+    private getBrowser;
+    /** Close the reused browser, if one was launched. No-op otherwise. */
+    close(): Promise<void>;
+}
+//# sourceMappingURL=client.d.ts.map

package/dist/client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,cAAc,EACd,KAAK,iBAAiB,EACtB,KAAK,WAAW,EAChB,KAAK,YAAY,EACjB,KAAK,MAAM,EACZ,MAAM,YAAY,CAAC;AACpB,OAAO,EAA2B,KAAK,YAAY,EAAE,MAAM,eAAe,CAAC;AAC3E,OAAO,EAAoB,KAAK,aAAa,EAAE,KAAK,YAAY,EAAE,MAAM,cAAc,CAAC;AA8CvF,MAAM,WAAW,qBAAqB;IACpC,mHAAmH;IACnH,QAAQ,CAAC,EAAE,cAAc,CAAC;IAC1B,yEAAyE;IACzE,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;OAIG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAED;;;;GAIG;AACH,qBAAa,cAAc;IACzB,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAC;IAClC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAe;IACnD,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAU;IACvC,OAAO,CAAC,cAAc,CAA+B;gBAEzC,OAAO,GAAE,qBAA0B;IAM/C,+DAA+D;IAC/D,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI;IAK/B,8DAA8D;IACxD,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,WAAW,CAAC;IAU1E,oFAAoF;IAC9E,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;IAKrF,uEAAuE;IACjE,KAAK,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IAK3F,mFAAmF;IAC7E,OAAO,CAAC,CAAC,GAAG,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IAK3G,4DAA4D;IAC5D,WAAW;;;;;IAIX;;;;OAIG;IACH,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,YAAY,CAAC;YAI7D,UAAU;IAKxB,sEAAsE;IAChE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAO7B"}

package/dist/client.js

@@ -0,0 +1,110 @@
+import { fetchUrl, launchBrowser } from "@x12i/fetcher";
+import { ingestRepository } from "@x12i/ingest";
+import { createDefaultRegistry } from "./registry.js";
+/**
+ * Per-site fetch defaults so client-rendered pages actually populate before
+ * we snapshot HTML. Caller-supplied options still win (spread after this).
+ */
+function siteFetchDefaults(url) {
+    try {
+        const parsed = new URL(url);
+        const host = parsed.hostname;
+        const path = parsed.pathname;
+        if ((host === "openrouter.ai" || host.endsWith(".openrouter.ai")) &&
+            (path === "/rankings" || path.startsWith("/rankings/"))) {
+            return {
+                // Live leaderboard rows are CSS-grid divs, not <table>s.
+                waitForSelector: "div.grid.grid-cols-12.items-center",
+                settleMs: 500,
+            };
+        }
+        if ((host === "linkedin.com" || host === "www.linkedin.com" || host.endsWith(".linkedin.com")) &&
+            path.startsWith("/company/")) {
+            return { waitForSelector: "h1", settleMs: 1000 };
+        }
+        if (host === "medium.com" ||
+            host.endsWith(".medium.com") ||
+            host === "towardsdatascience.com") {
+            return { waitForSelector: "article h1, h1", settleMs: 800 };
+        }
+        if (host === "g2.com" || host === "www.g2.com" || host.endsWith(".g2.com")) {
+            // G2 often serves a bot-challenge with no product <h1>; don't hard-wait.
+            return { settleMs: 1500 };
+        }
+    }
+    catch {
+        // ignore unparseable URLs
+    }
+    return {};
+}
+/**
+ * The single entry point every other package (CLI, REST API, MCP server)
+ * wraps: download a URL to Markdown, and/or classify+parse it into typed
+ * JSON using the registered parsers.
+ */
+export class WebAgentClient {
+    registry;
+    defaultFetchOptions;
+    reuseBrowser;
+    browserPromise;
+    constructor(options = {}) {
+        this.registry = options.registry ?? createDefaultRegistry();
+        this.defaultFetchOptions = options.fetchOptions ?? {};
+        this.reuseBrowser = options.reuseBrowser ?? false;
+    }
+    /** Register an additional parser on this client's registry. */
+    use(parser) {
+        this.registry.register(parser);
+        return this;
+    }
+    /** Download a URL and return { url, html, markdown, ... }. */
+    async fetch(url, options = {}) {
+        const browser = this.reuseBrowser ? await this.getBrowser() : undefined;
+        return fetchUrl(url, {
+            ...this.defaultFetchOptions,
+            ...siteFetchDefaults(url),
+            ...options,
+            browser: browser ?? options.browser,
+        });
+    }
+    /** Fetch a URL, then rank every registered parser's fit without running parse(). */
+    async classify(url, options = {}) {
+        const fetched = await this.fetch(url, options);
+        return this.registry.classify(fetched);
+    }
+    /** Fetch a URL, classify it, and parse it with the best-fit parser. */
+    async parse(url, options = {}) {
+        const fetched = await this.fetch(url, options);
+        return this.registry.parse(fetched);
+    }
+    /** Fetch a URL and force a specific parser by type id, skipping classification. */
+    async parseAs(type, url, options = {}) {
+        const fetched = await this.fetch(url, options);
+        return this.registry.parseAs(type, fetched);
+    }
+    /** List every registered parser's type id + description. */
+    listParsers() {
+        return this.registry.list();
+    }
+    /**
+     * Ingest a curated list / README URL: follow outbound GitHub skill links
+     * one level deep and extract targeted files (SKILL.md, .md, .sh, …).
+     * Uses GitHub raw/API — not Playwright.
+     */
+    ingest(rootUrl, options = {}) {
+        return ingestRepository(rootUrl, options);
+    }
+    async getBrowser() {
+        this.browserPromise ??= launchBrowser(this.defaultFetchOptions.executablePath);
+        return this.browserPromise;
+    }
+    /** Close the reused browser, if one was launched. No-op otherwise. */
+    async close() {
+        if (this.browserPromise) {
+            const browser = await this.browserPromise;
+            await browser.close();
+            this.browserPromise = undefined;
+        }
+    }
+}
+//# sourceMappingURL=client.js.map

package/dist/client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.js","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAqB,MAAM,eAAe,CAAC;AAC3E,OAAO,EAAE,gBAAgB,EAAyC,MAAM,cAAc,CAAC;AAEvF,OAAO,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAC;AAEtD;;;GAGG;AACH,SAAS,iBAAiB,CAAC,GAAW;IACpC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC;QAC7B,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC;QAC7B,IACE,CAAC,IAAI,KAAK,eAAe,IAAI,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC;YAC7D,CAAC,IAAI,KAAK,WAAW,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,EACvD,CAAC;YACD,OAAO;gBACL,yDAAyD;gBACzD,eAAe,EAAE,oCAAoC;gBACrD,QAAQ,EAAE,GAAG;aACd,CAAC;QACJ,CAAC;QACD,IACE,CAAC,IAAI,KAAK,cAAc,IAAI,IAAI,KAAK,kBAAkB,IAAI,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;YAC1F,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAC5B,CAAC;YACD,OAAO,EAAE,eAAe,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QACnD,CAAC;QACD,IACE,IAAI,KAAK,YAAY;YACrB,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC;YAC5B,IAAI,KAAK,wBAAwB,EACjC,CAAC;YACD,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC;QAC9D,CAAC;QACD,IAAI,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,YAAY,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YAC3E,yEAAyE;YACzE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAC5B,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,0BAA0B;IAC5B,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAeD;;;;GAIG;AACH,MAAM,OAAO,cAAc;IAChB,QAAQ,CAAiB;IACjB,mBAAmB,CAAe;IAClC,YAAY,CAAU;IAC/B,cAAc,CAA+B;IAErD,YAAY,UAAiC,EAAE;QAC7C,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,qBAAqB,EAAE,CAAC;QAC5D,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC;QACtD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,KAAK,CAAC;IACpD,CAAC;IAED,+DAA+D;IAC/D,GAAG,CAAI,MAAiB;QACtB,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC/B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,8DAA8D;IAC9D,KAAK,CAAC,KAAK,CAAC,GAAW,EAAE,UAAwB,EAAE;QACjD,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QACxE,OAAO,QAAQ,CAAC,GAAG,EAAE;YACnB,GAAG,IAAI,CAAC,mBAAmB;YAC3B,GAAG,iBAAiB,CAAC,GAAG,CAAC;YACzB,GAAG,OAAO;YACV,OAAO,EAAE,OAAO,IAAI,OAAO,CAAC,OAAO;SACpC,CAAC,CAAC;IACL,CAAC;IAED,oFAAoF;IACpF,KAAK,CAAC,QAAQ,CAAC,GAAW,EAAE,UAAwB,EAAE;QACpD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC/C,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IACzC,CAAC;IAED,uEAAuE;IACvE,KAAK,CAAC,KAAK,CAAc,GAAW,EAAE,UAAwB,EAAE;QAC9D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC/C,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAI,OAAO,CAAC,CAAC;IACzC,CAAC;IAED,mFAAmF;IACnF,KAAK,CAAC,OAAO,CAAc,IAAY,EAAE,GAAW,EAAE,UAAwB,EAAE;QAC9E,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC/C,OAAO,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAI,IAAI,EAAE,OAAO,CAAC,CAAC;IACjD,CAAC;IAED,4DAA4D;IAC5D,WAAW;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IAC9B,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,OAAe,EAAE,UAAyB,EAAE;QACjD,OAAO,gBAAgB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC5C,CAAC;IAEO,KAAK,CAAC,UAAU;QACtB,IAAI,CAAC,cAAc,KAAK,aAAa,CAAC,IAAI,CAAC,mBAAmB,CAAC,cAAc,CAAC,CAAC;QAC/E,OAAO,IAAI,CAAC,cAAc,CAAC;IAC7B,CAAC;IAED,sEAAsE;IACtE,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC;YAC1C,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;YACtB,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC;QAClC,CAAC;IACH,CAAC;CACF"}

package/dist/index.d.ts

@@ -0,0 +1,6 @@
+export * from "./client.js";
+export * from "./registry.js";
+export * from "@x12i/core";
+export type { FetchOptions } from "@x12i/fetcher";
+export type { IngestOptions, IngestResult, IngestedAsset, ExtractedFile, IngestProgressEvent, } from "@x12i/ingest";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,eAAe,CAAC;AAC9B,cAAc,YAAY,CAAC;AAC3B,YAAY,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAClD,YAAY,EACV,aAAa,EACb,YAAY,EACZ,aAAa,EACb,aAAa,EACb,mBAAmB,GACpB,MAAM,cAAc,CAAC"}

package/dist/index.js

@@ -0,0 +1,4 @@
+export * from "./client.js";
+export * from "./registry.js";
+export * from "@x12i/core";
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,eAAe,CAAC;AAC9B,cAAc,YAAY,CAAC"}

package/dist/registry.d.ts

@@ -0,0 +1,8 @@
+import { ParserRegistry } from "@x12i/core";
+/**
+ * A ParserRegistry pre-loaded with every first-party parser this monorepo
+ * ships. Add new site parsers here as they're built — this is the one
+ * place the CLI/API/MCP all pull their default parser set from.
+ */
+export declare function createDefaultRegistry(): ParserRegistry;
+//# sourceMappingURL=registry.d.ts.map

package/dist/registry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../src/registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAQ5C;;;;GAIG;AACH,wBAAgB,qBAAqB,IAAI,cAAc,CAOtD"}

package/dist/registry.js

@@ -0,0 +1,17 @@
+import { ParserRegistry } from "@x12i/core";
+import { openRouterRankingsParser } from "@x12i/parser-openrouter";
+import { g2ProductParser, linkedInCompanyParser, mediumArticleParser, } from "@x12i/parsers-web";
+/**
+ * A ParserRegistry pre-loaded with every first-party parser this monorepo
+ * ships. Add new site parsers here as they're built — this is the one
+ * place the CLI/API/MCP all pull their default parser set from.
+ */
+export function createDefaultRegistry() {
+    return new ParserRegistry().registerAll([
+        openRouterRankingsParser,
+        g2ProductParser,
+        linkedInCompanyParser,
+        mediumArticleParser,
+    ]);
+}
+//# sourceMappingURL=registry.js.map

package/dist/registry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"registry.js","sourceRoot":"","sources":["../src/registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AACnE,OAAO,EACL,eAAe,EACf,qBAAqB,EACrB,mBAAmB,GACpB,MAAM,mBAAmB,CAAC;AAE3B;;;;GAIG;AACH,MAAM,UAAU,qBAAqB;IACnC,OAAO,IAAI,cAAc,EAAE,CAAC,WAAW,CAAC;QACtC,wBAAwB;QACxB,eAAe;QACf,qBAAqB;QACrB,mBAAmB;KACpB,CAAC,CAAC;AACL,CAAC"}

package/package.json

@@ -0,0 +1,50 @@
+{
+  "name": "@x12i/sdk",
+  "version": "0.1.0",
+  "description": "Programmatic client: fetch(url) -> Markdown, parse(url) -> typed JSON via the parser registry. What the CLI, REST API, and MCP server are all thin wrappers around.",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "dependencies": {
+    "playwright": "^1.49.1",
+    "@x12i/core": "0.1.0",
+    "@x12i/parser-openrouter": "0.1.0",
+    "@x12i/ingest": "0.1.0",
+    "@x12i/fetcher": "0.1.0",
+    "@x12i/parsers-web": "0.1.0"
+  },
+  "devDependencies": {
+    "typescript": "^5.7.3",
+    "vitest": "^2.1.8"
+  },
+  "license": "MIT",
+  "files": [
+    "dist"
+  ],
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/x12i/web-agentic-tools.git",
+    "directory": "packages/sdk"
+  },
+  "homepage": "https://github.com/x12i/web-agentic-tools#readme",
+  "bugs": {
+    "url": "https://github.com/x12i/web-agentic-tools/issues"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "dev": "tsc -p tsconfig.json --watch",
+    "test": "vitest run --passWithNoTests",
+    "typecheck": "tsc -p tsconfig.json --noEmit",
+    "clean": "rm -rf dist .turbo"
+  }
+}