@aaroncql/pim-agent 0.0.1

package/src/extensions/web-fetch/JinaReaderClient.ts

@@ -0,0 +1,230 @@
+import ky, { HTTPError, TimeoutError, type KyInstance } from "ky";
+import type { WebFetchPage } from "./fetch";
+
+type JinaReaderFetch = (
+  input: Parameters<typeof fetch>[0],
+  init?: Parameters<typeof fetch>[1]
+) => ReturnType<typeof fetch>;
+
+type JinaReaderClientOptions = {
+  readonly endpoint?: string;
+  readonly apiKey?: string;
+  readonly fetch?: JinaReaderFetch;
+  readonly timeoutMs?: number;
+};
+
+type JinaReaderFetchInput = {
+  readonly url: string;
+  readonly signal?: AbortSignal;
+};
+
+class JinaReaderClientError extends Error {
+  public constructor(message: string) {
+    super(message);
+    this.name = "JinaReaderClientError";
+  }
+}
+
+export class JinaReaderClient {
+  private static readonly defaultTimeoutMs = 20_000;
+
+  private readonly endpoint: string;
+  private readonly headers: Headers;
+  private readonly ky: KyInstance;
+  private readonly timeoutMs: number;
+
+  public constructor(options: JinaReaderClientOptions = {}) {
+    this.endpoint = JinaReaderClient.normalizeEndpoint(
+      options.endpoint ?? "https://r.jina.ai"
+    );
+    this.headers = JinaReaderClient.buildHeaders(options.apiKey);
+    this.timeoutMs = options.timeoutMs ?? JinaReaderClient.defaultTimeoutMs;
+    this.ky = ky.create(
+      options.fetch === undefined
+        ? {}
+        : { fetch: options.fetch as typeof fetch }
+    );
+  }
+
+  public async fetchUrl(input: JinaReaderFetchInput): Promise<WebFetchPage> {
+    if (input.signal?.aborted) {
+      throw new JinaReaderClientError("Request aborted.");
+    }
+
+    let response: Response;
+
+    try {
+      response = await this.ky(`${this.endpoint}/${input.url}`, {
+        headers: this.headers,
+        timeout: this.timeoutMs,
+        ...(input.signal === undefined ? {} : { signal: input.signal }),
+      });
+    } catch (error) {
+      const aborted = input.signal?.aborted ?? false;
+
+      if (JinaReaderClient.isAbortError(error) || aborted) {
+        throw new JinaReaderClientError("Request aborted.");
+      }
+
+      if (error instanceof TimeoutError) {
+        throw new JinaReaderClientError(
+          `Request timed out after ${this.timeoutMs}ms.`
+        );
+      }
+
+      if (error instanceof HTTPError) {
+        throw new JinaReaderClientError(
+          `Request failed with HTTP ${error.response.status}: ${JinaReaderClient.excerpt(JinaReaderClient.stringifyErrorData(error.data))}`
+        );
+      }
+
+      throw new JinaReaderClientError(
+        `Request failed: ${describeError(error)}`
+      );
+    }
+
+    return JinaReaderClient.parseResponse(
+      input.url,
+      response,
+      await response.text()
+    );
+  }
+
+  private static buildHeaders(apiKey: string | undefined): Headers {
+    const headers = new Headers({ Accept: "application/json" });
+
+    if (apiKey !== undefined && apiKey.length > 0) {
+      headers.set("Authorization", `Bearer ${apiKey}`);
+    }
+
+    return headers;
+  }
+
+  private static normalizeEndpoint(endpoint: string): string {
+    return endpoint.replace(/\/+$/u, "");
+  }
+
+  private static parseResponse(
+    requestedUrl: string,
+    response: Response,
+    responseText: string
+  ): WebFetchPage {
+    const contentType = response.headers.get("content-type") ?? "";
+    const declaresJson =
+      contentType.includes("application/json") || contentType.includes("+json");
+    const parsedJson = JinaReaderClient.tryParseJson(responseText);
+
+    if (parsedJson === undefined) {
+      if (declaresJson) {
+        throw new JinaReaderClientError("Response contained malformed JSON.");
+      }
+
+      return JinaReaderClient.createPage({
+        title: "",
+        url: requestedUrl,
+        content: responseText,
+      });
+    }
+
+    return JinaReaderClient.parseJsonPayload(requestedUrl, parsedJson);
+  }
+
+  private static parseJsonPayload(
+    requestedUrl: string,
+    parsedJson: unknown
+  ): WebFetchPage {
+    const responseRecord = JinaReaderClient.asRecord(parsedJson);
+    const payload =
+      JinaReaderClient.asRecord(responseRecord?.["data"]) ?? responseRecord;
+
+    if (payload === undefined) {
+      throw new JinaReaderClientError("Response contained invalid payload.");
+    }
+
+    return JinaReaderClient.createPage({
+      title: JinaReaderClient.optionalString(payload["title"], "title") ?? "",
+      url:
+        JinaReaderClient.optionalString(payload["url"], "url") ?? requestedUrl,
+      content: JinaReaderClient.requiredString(payload["content"], "content"),
+    });
+  }
+
+  private static createPage(page: WebFetchPage): WebFetchPage {
+    if (page.content.trim().length === 0) {
+      throw new JinaReaderClientError("Response contained empty content.");
+    }
+
+    return page;
+  }
+
+  private static tryParseJson(text: string): unknown | undefined {
+    try {
+      return JSON.parse(text);
+    } catch {
+      return undefined;
+    }
+  }
+
+  private static requiredString(value: unknown, name: string): string {
+    if (typeof value !== "string") {
+      throw new JinaReaderClientError(
+        `Response contained invalid payload: expected string ${name}.`
+      );
+    }
+
+    return value;
+  }
+
+  private static optionalString(
+    value: unknown,
+    name: string
+  ): string | undefined {
+    if (value === undefined || value === null) {
+      return undefined;
+    }
+
+    if (typeof value !== "string") {
+      throw new JinaReaderClientError(
+        `Response contained invalid payload: expected string ${name}.`
+      );
+    }
+
+    return value;
+  }
+
+  private static asRecord(
+    value: unknown
+  ): Readonly<Record<string, unknown>> | undefined {
+    if (typeof value !== "object" || value === null || Array.isArray(value)) {
+      return undefined;
+    }
+
+    return value as Readonly<Record<string, unknown>>;
+  }
+
+  private static stringifyErrorData(data: unknown): string {
+    if (typeof data === "string") {
+      return data;
+    }
+
+    if (data === undefined) {
+      return "";
+    }
+
+    return JSON.stringify(data);
+  }
+
+  private static excerpt(text: string): string {
+    const excerpt = text.replaceAll(/\s+/gu, " ").trim().slice(0, 200);
+
+    return excerpt.length === 0 ? "empty response body" : excerpt;
+  }
+
+  private static isAbortError(error: unknown): boolean {
+    return error instanceof Error && error.name === "AbortError";
+  }
+}
+
+function describeError(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}

package/src/extensions/web-fetch/WebViewFetchClient.ts

@@ -0,0 +1,186 @@
+import { platform } from "node:os";
+import type { WebFetchPage } from "./fetch";
+import { createMarkdownSnapshotScript } from "./WebViewMarkdownSnapshot";
+
+export type WebViewLike = {
+  readonly url: string;
+  readonly title: string;
+  navigate: (url: string) => Promise<void>;
+  evaluate: <T = unknown>(script: string) => Promise<T>;
+  close: () => void;
+};
+
+export type WebViewFactory = () => WebViewLike;
+
+type WebViewFetchClientOptions = {
+  readonly factory?: WebViewFactory;
+  readonly timeoutMs?: number;
+};
+
+type WebViewFetchInput = {
+  readonly url: string;
+  readonly signal?: AbortSignal;
+};
+
+type WebViewSnapshot = {
+  readonly title: string;
+  readonly url: string;
+  readonly content: string;
+};
+
+class WebViewFetchClientError extends Error {
+  public constructor(message: string) {
+    super(message);
+    this.name = "WebViewFetchClientError";
+  }
+}
+
+export class WebViewFetchClient {
+  private static readonly defaultTimeoutMs = 20_000;
+
+  private readonly factory: WebViewFactory;
+  private readonly timeoutMs: number;
+
+  public constructor(options: WebViewFetchClientOptions = {}) {
+    this.factory = options.factory ?? WebViewFetchClient.defaultFactory;
+    this.timeoutMs = options.timeoutMs ?? WebViewFetchClient.defaultTimeoutMs;
+  }
+
+  private static defaultFactory(): WebViewLike {
+    return new Bun.WebView(
+      platform() === "darwin" ? undefined : { backend: "chrome" }
+    );
+  }
+
+  public async fetchHtml(input: WebViewFetchInput): Promise<WebFetchPage> {
+    return this.capturePage(input, WebViewFetchClient.htmlSnapshotScript());
+  }
+
+  public async fetchMarkdown(input: WebViewFetchInput): Promise<WebFetchPage> {
+    return this.capturePage(input, createMarkdownSnapshotScript());
+  }
+
+  private async capturePage(
+    input: WebViewFetchInput,
+    snapshotScript: string
+  ): Promise<WebFetchPage> {
+    const signal = input.signal;
+
+    if (signal?.aborted) {
+      throw new WebViewFetchClientError("Request aborted.");
+    }
+
+    let view: WebViewLike;
+
+    try {
+      view = this.factory();
+    } catch (error) {
+      throw new WebViewFetchClientError(
+        `Request failed: ${describeError(error)}`
+      );
+    }
+
+    let aborted = false;
+    let timedOut = false;
+    const onAbort = () => {
+      aborted = true;
+      WebViewFetchClient.safeClose(view);
+    };
+
+    signal?.addEventListener("abort", onAbort, { once: true });
+
+    const timeoutHandle = setTimeout(() => {
+      timedOut = true;
+      WebViewFetchClient.safeClose(view);
+    }, this.timeoutMs);
+
+    const checkInterrupted = (): void => {
+      if (timedOut) {
+        throw new WebViewFetchClientError(
+          `Request timed out after ${this.timeoutMs}ms.`
+        );
+      }
+      if (aborted) {
+        throw new WebViewFetchClientError("Request aborted.");
+      }
+    };
+
+    try {
+      await view.navigate(input.url);
+      checkInterrupted();
+
+      const snapshot = WebViewFetchClient.readSnapshot(
+        await view.evaluate<unknown>(snapshotScript)
+      );
+
+      if (snapshot.content.trim().length === 0) {
+        throw new WebViewFetchClientError("Response contained empty content.");
+      }
+
+      const finalUrl =
+        snapshot.url.length > 0
+          ? snapshot.url
+          : view.url.length > 0
+            ? view.url
+            : input.url;
+
+      return {
+        title: snapshot.title,
+        url: finalUrl,
+        content: snapshot.content,
+      };
+    } catch (error) {
+      checkInterrupted();
+
+      if (error instanceof WebViewFetchClientError) {
+        throw error;
+      }
+
+      throw new WebViewFetchClientError(
+        `Request failed: ${describeError(error)}`
+      );
+    } finally {
+      clearTimeout(timeoutHandle);
+      signal?.removeEventListener("abort", onAbort);
+      WebViewFetchClient.safeClose(view);
+    }
+  }
+
+  private static htmlSnapshotScript(): string {
+    return String.raw`(() => ({
+      title: document.title,
+      url: location.href,
+      content: document.documentElement.outerHTML,
+    }))()`;
+  }
+
+  private static readSnapshot(value: unknown): WebViewSnapshot {
+    if (typeof value !== "object" || value === null) {
+      throw new WebViewFetchClientError("Response contained invalid payload.");
+    }
+
+    const record = value as Record<string, unknown>;
+
+    if (typeof record["content"] !== "string") {
+      throw new WebViewFetchClientError("Response contained invalid payload.");
+    }
+
+    return {
+      content: record["content"],
+      title: typeof record["title"] === "string" ? record["title"] : "",
+      url: typeof record["url"] === "string" ? record["url"] : "",
+    };
+  }
+
+  private static safeClose(view: WebViewLike): void {
+    try {
+      view.close();
+    } catch {
+      // close() throws if already closed; treat as idempotent.
+    }
+  }
+}
+
+function describeError(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}

package/src/extensions/web-fetch/WebViewMarkdownSnapshot.test.ts

@@ -0,0 +1,119 @@
+import { describe, expect, test } from "bun:test";
+import {
+  createMarkdownSnapshotScript,
+  renderMarkdownSnapshotTree,
+  type MarkdownSnapshotElementNode,
+  type MarkdownSnapshotNode,
+} from "./WebViewMarkdownSnapshot";
+
+const text = (value: string): MarkdownSnapshotNode => ({
+  type: "text",
+  text: value,
+});
+
+const element = (
+  tagName: string,
+  children: readonly MarkdownSnapshotNode[] = [],
+  options: Partial<MarkdownSnapshotElementNode> = {}
+): MarkdownSnapshotElementNode => ({
+  type: "element",
+  tagName,
+  children,
+  ...options,
+});
+
+describe("createMarkdownSnapshotScript", () => {
+  test("builds a parseable browser snapshot script from stringified functions", () => {
+    const script = createMarkdownSnapshotScript();
+
+    expect(script).toStartWith("(function captureMarkdownSnapshot");
+    expect(script).toContain("function renderMarkdownSnapshotTree");
+    expect(script).toContain("function serializeNode");
+    expect(script).not.toContain("const renderMarkdownSnapshotTree =");
+    expect(() => new Function(script)).not.toThrow();
+  });
+});
+
+describe("renderMarkdownSnapshotTree", () => {
+  test("renders headings, inline formatting, code, and absolute links", () => {
+    const root = element("body", [
+      element("h1", [text("Hello")]),
+      element("p", [
+        text("Read "),
+        element("strong", [text("docs")]),
+        text(" at "),
+        element("a", [text("guide")], {
+          attributes: [{ name: "href", value: "/guide" }],
+        }),
+        text(" with "),
+        element("code", [text("x")], { textContent: "x" }),
+      ]),
+    ]);
+
+    expect(renderMarkdownSnapshotTree(root, "https://example.test/base/")).toBe(
+      [
+        "# Hello",
+        "",
+        "Read **docs** at [guide](https://example.test/guide) with `x`",
+      ].join("\n")
+    );
+  });
+
+  test("renders lists, blockquotes, tables, and fenced code blocks", () => {
+    const root = element("body", [
+      element("ul", [
+        element("li", [text("one")]),
+        element("li", [text("two")]),
+      ]),
+      element("blockquote", [element("p", [text("quoted")])]),
+      element("table", [
+        element("tbody", [
+          element("tr", [
+            element("th", [text("Name")]),
+            element("th", [text("Value")]),
+          ]),
+          element("tr", [
+            element("td", [text("A|B")]),
+            element("td", [text("2")]),
+          ]),
+        ]),
+      ]),
+      element("pre", [element("code", [text("const x = 1;")])], {
+        textContent: "const x = 1;",
+      }),
+    ]);
+
+    expect(renderMarkdownSnapshotTree(root, "https://example.test/")).toBe(
+      [
+        "- one",
+        "- two",
+        "",
+        "> quoted",
+        "",
+        "| Name | Value |",
+        "| --- | --- |",
+        "| A\\|B | 2 |",
+        "",
+        "```",
+        "const x = 1;",
+        "```",
+      ].join("\n")
+    );
+  });
+
+  test("skips hidden content and unsafe links", () => {
+    const root = element("body", [
+      element("p", [
+        text("shown"),
+        element("span", [text("hidden")], { ariaHidden: "true" }),
+        element("a", [text("unsafe")], {
+          attributes: [{ name: "href", value: "javascript:alert(1)" }],
+        }),
+      ]),
+    ]);
+
+    expect(renderMarkdownSnapshotTree(root, "https://example.test/")).toBe(
+      "shown unsafe"
+    );
+  });
+});