@bluefields/fetcher 0.2.0

package/dist/types.d.ts

@@ -0,0 +1,231 @@
+/**
+ * Fetcher + storage adapter contracts (chunk 8).
+ *
+ * The fetcher returns raw HTML + metadata. The storage adapter persists
+ * the bytes content-addressed in R2 (Review #2). Both have an in-memory
+ * test implementation and an env-gated production implementation.
+ *
+ * Per the four-layer rule: the fetcher returns raw bytes +
+ * metadata, nothing more. It does NOT extract, diff, or notify.
+ */
+/**
+ * What to wait for before considering the page "ready" for content
+ * extraction. Only honored by browser-based adapters (Patchright);
+ * HTTP adapter ignores this and returns as soon as the response body
+ * arrives.
+ */
+export type WaitCondition = 
+/** Wait until `document.readyState === 'complete'` (default for browser). */
+{
+    type: 'load';
+    timeoutMs?: number;
+}
+/** Wait until network is idle for 500ms (Patchright `networkidle`). */
+ | {
+    type: 'networkIdle';
+    timeoutMs?: number;
+}
+/** Wait for a CSS selector to appear in the DOM. */
+ | {
+    type: 'selector';
+    selector: string;
+    timeoutMs?: number;
+}
+/** Wait a fixed amount of time after navigation. */
+ | {
+    type: 'timeout';
+    ms: number;
+};
+export interface RequestCookie {
+    name: string;
+    value: string;
+    domain?: string;
+    path?: string;
+    expires?: number;
+    httpOnly?: boolean;
+    secure?: boolean;
+    sameSite?: 'Strict' | 'Lax' | 'None';
+}
+/**
+ * Steps run inside the browser between `goto` + `waitFor` and content
+ * extraction. Required for sites that need interaction (cookie banners,
+ * "load more" buttons, login forms, infinite scroll). Browser-only —
+ * the HTTP fetcher rejects when actions are present.
+ *
+ * Execution is sequential; one failure short-circuits and surfaces the
+ * action index in the error. Each action has its own timeoutMs where
+ * applicable; the overall request timeout still applies.
+ */
+export type PageAction = 
+/** Sleep for `milliseconds`, or wait for `selector` to appear. */
+{
+    type: 'wait';
+    milliseconds?: number;
+    selector?: string;
+    timeoutMs?: number;
+}
+/** Click a CSS selector. */
+ | {
+    type: 'click';
+    selector: string;
+    timeoutMs?: number;
+}
+/**
+ * Type `text` into the page. When `selector` is provided, fills that
+ * element directly (replaces its value); otherwise types into whatever
+ * is focused — pair with a prior `click` action.
+ */
+ | {
+    type: 'write';
+    text: string;
+    selector?: string;
+}
+/** Press a keyboard key — `Enter`, `Tab`, `Escape`, `ArrowDown`, etc. */
+ | {
+    type: 'press';
+    key: string;
+}
+/** Scroll the viewport. `amount` defaults to one viewport. */
+ | {
+    type: 'scroll';
+    direction: 'up' | 'down';
+    amount?: number;
+}
+/** Capture a screenshot at this point in the sequence. */
+ | {
+    type: 'screenshot';
+    fullPage?: boolean;
+}
+/** Capture the current page HTML at this point in the sequence. */
+ | {
+    type: 'scrape';
+}
+/** Run JavaScript in the page context and capture its return value. */
+ | {
+    type: 'executeJavascript';
+    script: string;
+};
+/**
+ * Results from result-yielding actions (screenshot, scrape,
+ * executeJavascript), collected in execution order. Arrays are empty when
+ * no such action ran. Surfaced to /scrape callers under `actions`.
+ */
+export interface ActionResults {
+    /** Base64-encoded PNG bytes, one entry per `screenshot` action. */
+    screenshots: string[];
+    /** Page HTML captured by each `scrape` action. */
+    scrapes: {
+        html: string;
+    }[];
+    /** Return value of each `executeJavascript` action. */
+    javascriptReturns: {
+        value: unknown;
+    }[];
+}
+export interface FetchOptions {
+    /** Hard timeout (ms). Default 30s per Review #4. */
+    timeoutMs?: number;
+    /** User-Agent string to send. */
+    userAgent?: string;
+    /** Test seam — override Date.now() for `fetchedAt`. */
+    nowFn?: () => Date;
+    /**
+     * Wait condition before reading page content. Browser adapters honor
+     * this; HTTP adapter ignores (returns as soon as response body arrives).
+     * Default: `{ type: 'load' }` (browser) or N/A (HTTP).
+     */
+    waitFor?: WaitCondition;
+    /**
+     * Capture a screenshot alongside the HTML. Only browser adapters
+     * (Patchright) can do this — HTTP adapter returns `screenshot: null`.
+     */
+    captureScreenshot?: boolean;
+    /**
+     * Extra HTTP request headers — Authorization, custom UA overrides, etc.
+     * `user-agent` here takes precedence over the top-level `userAgent`.
+     */
+    headers?: Record<string, string>;
+    /**
+     * Cookies sent with the request (Phase G). Browser adapters apply via
+     * BrowserContext.addCookies; HTTP adapter serializes them into a
+     * `Cookie:` header (domain/path filtering matches the request URL).
+     */
+    cookies?: RequestCookie[];
+    /**
+     * Sequential interaction steps run between navigation and content
+     * extraction (Phase F). Browser-only — the HTTP fetcher throws when
+     * actions are present.
+     */
+    actions?: PageAction[];
+}
+export interface FetchResult {
+    /** Raw HTML response body. */
+    html: string;
+    responseStatus: number;
+    /** Map of lowercased response header name → value. */
+    responseHeaders: Record<string, string>;
+    fetchedAt: Date;
+    fetchDurationMs: number;
+    /** Identifies the fetcher implementation that produced this row. */
+    fetcherId: string;
+    /** Egress IP of the fetcher (proxy or local depending on adapter). */
+    proxyEgressIp: string;
+    userAgent: string;
+    rawHtmlSizeBytes: number;
+    /**
+     * Bytes actually transferred over the wire (sum of response
+     * Content-Length) — a proxy for metered-egress cost. Browser adapters
+     * that abort image/media/font downloads count only what they kept;
+     * aborted requests never produce a response, so they're excluded.
+     * Optional: adapters that don't measure it leave it undefined.
+     */
+    bytesTransferred?: number;
+    /**
+     * Full-page PNG screenshot bytes (browser adapters only when
+     * `captureScreenshot: true`). null otherwise.
+     */
+    screenshot: Buffer | null;
+    /**
+     * Final URL after redirects. Useful for /scrape to surface to
+     * customers when the requested URL bounced. Same as the input URL
+     * for non-redirecting fetches.
+     */
+    finalUrl: string;
+    /**
+     * Results from result-yielding page actions (screenshot, scrape,
+     * executeJavascript). Present only when such actions ran.
+     */
+    actionResults?: ActionResults;
+}
+export interface FetcherAdapter {
+    fetch(url: string, options?: FetchOptions): Promise<FetchResult>;
+}
+/**
+ * A fetcher whose lazy resources (e.g. the Patchright browser pool) can be eagerly
+ * initialized at boot so the FIRST customer fetch doesn't pay the setup cost (perf
+ * round, L4 — a cold Chromium launch is ~0.5–1.5s of pure tail). `warmUp` is optional
+ * and a no-op for adapters with nothing to warm; call it once at app boot,
+ * fire-and-forget. A plain `FetcherAdapter` consumer that never calls it is unaffected.
+ */
+export interface WarmableFetcherAdapter extends FetcherAdapter {
+    warmUp?(): Promise<void>;
+}
+/** Output of `storeSnapshot`. */
+export interface StoredSnapshot {
+    contentHash: string;
+    /** Object key — Review #2: `content/<first2-of-sha>/<sha256>`. */
+    r2Key: string;
+}
+export interface StorageAdapter {
+    /** Persist `content`; idempotent (same content → same key). */
+    storeSnapshot(content: string): Promise<StoredSnapshot>;
+    /** Retrieve previously-stored content by hash. Throws on miss. */
+    getSnapshot(contentHash: string): Promise<string>;
+    /** True if the content has already been stored. */
+    hasSnapshot(contentHash: string): Promise<boolean>;
+}
+/**
+ * Content-addressed key per Review #2: 2-char prefix shard for filesystem
+ * fan-out, then the full sha256 hex.
+ */
+export declare function buildR2Key(contentHash: string): string;

package/dist/types.js

@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Fetcher + storage adapter contracts (chunk 8).
+ *
+ * The fetcher returns raw HTML + metadata. The storage adapter persists
+ * the bytes content-addressed in R2 (Review #2). Both have an in-memory
+ * test implementation and an env-gated production implementation.
+ *
+ * Per the four-layer rule: the fetcher returns raw bytes +
+ * metadata, nothing more. It does NOT extract, diff, or notify.
+ */
+/**
+ * Content-addressed key per Review #2: 2-char prefix shard for filesystem
+ * fan-out, then the full sha256 hex.
+ */
+export function buildR2Key(contentHash) {
+    if (!/^[0-9a-f]{64}$/.test(contentHash)) {
+        throw new Error(`contentHash must be a 64-char lowercase hex string, got ${contentHash}`);
+    }
+    return `content/${contentHash.slice(0, 2)}/${contentHash}`;
+}
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C;;;;;;;;;GASG;AAuLH;;;GAGG;AACH,MAAM,UAAU,UAAU,CAAC,WAAmB;IAC5C,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,2DAA2D,WAAW,EAAE,CAAC,CAAC;IAC5F,CAAC;IACD,OAAO,WAAW,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,WAAW,EAAE,CAAC;AAC7D,CAAC"}

package/package.json

@@ -0,0 +1,54 @@
+{
+  "name": "@bluefields/fetcher",
+  "version": "0.2.0",
+  "type": "module",
+  "main": "dist/index.js",
+  "dependencies": {
+    "drizzle-orm": "0.36.4",
+    "patchright": "^1.60.0",
+    "@bluefields/db": "0.2.0",
+    "@bluefields/primitives": "0.2.0",
+    "@bluefields/attest": "0.2.0"
+  },
+  "devDependencies": {
+    "@types/node": "22.9.0",
+    "vitest": "2.1.4"
+  },
+  "description": "Bluefield fetcher — tiered HTML capture (HTTP → headless browser) with SSRF guard and signed provenance.",
+  "license": "AGPL-3.0-or-later",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "sideEffects": false,
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "keywords": [
+    "scraping",
+    "fetch",
+    "browser",
+    "bluefield"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/danielvhofmann/bluefield.git",
+    "directory": "packages/fetcher"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "typecheck": "tsc --noEmit -p .",
+    "test": "vitest run",
+    "build": "tsc -p tsconfig.build.json"
+  }
+}