@octocrawl/sdk 0.3.0

package/dist/types/cjs/contracts/structured.d.ts

@@ -0,0 +1,185 @@
+import type { AdapterDescriptor, ExtractedEntity, ProductFactSource } from './extractor.js';
+export type JsonPrimitive = string | number | boolean | null;
+export type JsonValue = JsonPrimitive | JsonValue[] | {
+    [key: string]: JsonValue;
+};
+/**
+ * The JSON Schema subset accepted by scrape (`readSchema` in api.ts checks
+ * it): the structure extraction maps, the assertions it checks on the result
+ * and annotations it ignores. `default` is never filled in and `format` is
+ * not checked.
+ */
+export interface JsonSchema {
+    /** Root only: draft-07, 2019-09 or 2020-12. */
+    $schema?: string;
+    /** Root only. */
+    $id?: string;
+    $ref?: string;
+    type?: 'object' | 'array' | 'string' | 'number' | 'integer' | 'boolean' | 'null' | readonly ('object' | 'array' | 'string' | 'number' | 'integer' | 'boolean' | 'null')[];
+    properties?: Readonly<Record<string, JsonSchema>>;
+    required?: readonly string[];
+    items?: JsonSchema;
+    enum?: readonly JsonValue[];
+    const?: JsonValue;
+    /** A schema and `{ type: 'null' }`, or primitive types only. */
+    anyOf?: readonly JsonSchema[];
+    oneOf?: readonly JsonSchema[];
+    additionalProperties?: boolean | JsonSchema;
+    $defs?: Readonly<Record<string, JsonSchema>>;
+    definitions?: Readonly<Record<string, JsonSchema>>;
+    minimum?: number;
+    maximum?: number;
+    exclusiveMinimum?: number;
+    exclusiveMaximum?: number;
+    multipleOf?: number;
+    minLength?: number;
+    maxLength?: number;
+    pattern?: string;
+    minItems?: number;
+    maxItems?: number;
+    uniqueItems?: boolean;
+    title?: string;
+    description?: string;
+    $comment?: string;
+    default?: JsonValue;
+    examples?: readonly JsonValue[];
+    deprecated?: boolean;
+    readOnly?: boolean;
+    writeOnly?: boolean;
+    format?: string;
+}
+export interface JsonFormatRequest {
+    type: 'json';
+    schema: JsonSchema;
+    prompt?: string;
+    /** Page content leaves the process only when this is explicitly true. */
+    modelFallback?: boolean;
+}
+/** One CSS selector and the HTML attribute to read from every element it matches (the `attributes` format). */
+export interface AttributeSelector {
+    /** A non-empty selector of at most 200 characters, within what W2L matches (`invalidSelector`), like `includeTags`. */
+    selector: string;
+    /** An HTML attribute name: `^[A-Za-z_][A-Za-z0-9_:.-]*$`, at most 100 characters. */
+    attribute: string;
+}
+/** Firecrawl's `{ type: 'attributes', selectors }`: 1 to 50 selectors, at most one such entry per request. */
+export interface AttributesFormatRequest {
+    type: 'attributes';
+    selectors: readonly AttributeSelector[];
+}
+/** One column of a `list` format: what to read from each record. */
+export interface ListField {
+    /** The column's name: 1 to 64 characters, unique within the format. */
+    name: string;
+    /** A selector matched within the record (the first match, in document order); omitted, the record element itself. */
+    selector?: string;
+    /** An HTML attribute to read instead of the text (`href`, `src`, `data-sku`, ...); a link or source is made absolute. */
+    attribute?: string;
+}
+/**
+ * W2L's `{ type: 'list', itemSelector, fields }`: every element `itemSelector`
+ * matches is a record (one nested in another matched one is not a record of
+ * its own), and each field is read from it. At most one such entry per request.
+ * Without `itemSelector` W2L finds the page's list itself, and without
+ * `fields` the fields its items hold (`fields` needs an `itemSelector`):
+ * what it chose is the result's `itemSelector` and `detected.fields`.
+ */
+export interface ListFormatRequest {
+    type: 'list';
+    itemSelector?: string;
+    /** 1 to 50 fields. */
+    fields?: readonly ListField[];
+}
+/** A list format with its items and fields named: as asked, or as W2L found them on the page. */
+export interface ListSpec {
+    type: 'list';
+    itemSelector: string;
+    fields: readonly ListField[];
+}
+/** The window a screenshot is taken at, in CSS pixels: integers within the declared screen (`readFormats` checks 320..1920 by 240..1080, the desktop identity's 1920x1080 screen). */
+export interface ScreenshotViewport {
+    width: number;
+    height: number;
+}
+/**
+ * What a screenshot entry asks of the browser lane. Every field is optional:
+ * the string `screenshot` alone is a PNG of the declared viewport (1280x800
+ * for the desktop identity). `fullPage` captures the document's whole height
+ * at the viewport's width, without scrolling first; `quality` (1 to 100)
+ * gives a JPEG at that quality instead of a PNG; `viewport` lays the page out
+ * in that window, a size within the declared screen and not a change of the
+ * identity (User-Agent, client hints, locale, time zone, screen and scale
+ * factor stay as declared).
+ */
+export interface ScreenshotOptions {
+    fullPage?: boolean;
+    quality?: number;
+    viewport?: ScreenshotViewport;
+}
+/** Firecrawl's `{ type: 'screenshot', fullPage, quality, viewport }`: at most one screenshot entry per request; the strings `screenshot` and `screenshot@fullPage` (Firecrawl v1's full-page spelling) are its shorthands. */
+export interface ScreenshotFormatRequest extends ScreenshotOptions {
+    type: 'screenshot';
+}
+/**
+ * String json returns the canonical envelope; an object maps into a caller schema.
+ * `html` is the cleaned HTML the Markdown is written from (the main content,
+ * the whole page when onlyMainContent is false, or an includeTags selection);
+ * `rawHtml` is the page as the lane received it. `images` is every image URL
+ * of the whole document, an attributes entry reads named attributes off the
+ * elements its selectors match, and `screenshot` (or a screenshot entry)
+ * captures the rendered page as an image on the browser lane.
+ */
+export type ScrapeFormat = 'markdown' | 'links' | 'json' | 'html' | 'rawHtml' | 'images' | 'tables' | 'screenshot' | JsonFormatRequest | AttributesFormatRequest | ScreenshotFormatRequest | ListFormatRequest;
+export interface StructuredFieldEvidence {
+    path: string;
+    /**
+     * `fetch`: the value is the fetch's own (`finalUrl`, `requestedUrl`), not read from the page.
+     * `pdf`: a `Label: value` line of a PDF's text; its evidencePath is `page N "label"`.
+     */
+    source: ProductFactSource | 'hydration' | 'fetch' | 'pdf';
+    evidencePath?: string;
+    /** For a number read from the page's text: that text, whitespace collapsed (`1.299,00 €`), so the reading can be checked. */
+    text?: string;
+}
+export type StructuredIssueCode = 'adapter_unavailable' | 'subject_unverified' | 'field_unavailable'
+/** Page labels matching the field state different values, so none was chosen. */
+ | 'field_ambiguous' | 'missing_required' | 'schema_invalid' | 'model_unavailable' | 'model_provider_error' | 'model_output_invalid' | 'model_timeout'
+/** The page status is neither `success` nor `partial`, so no fields were read from it. */
+ | 'page_unsuccessful'
+/** The page is `partial` (a timeout ended the scrape): fields come from the content fetched so far, never a complete result. */
+ | 'page_partial';
+export interface StructuredExtractionIssue {
+    code: StructuredIssueCode;
+    message: string;
+    path?: string;
+}
+export interface StructuredModelUsage {
+    model: string;
+    attempts: number;
+    inputTokens: number | null;
+    outputTokens: number | null;
+    /** Unknown provider pricing stays unknown. */
+    externalCostUsd: number | null;
+    /**
+     * True when the request used strict structured outputs with a strict-safe
+     * schema W2L derived from the caller's; false when it sent the caller's
+     * schema without strict mode, which cannot express it (`strictReason`).
+     */
+    strict?: boolean;
+    strictReason?: string;
+}
+export interface CanonicalStructuredData {
+    adapter: AdapterDescriptor;
+    pageType: string;
+    entities: readonly ExtractedEntity[];
+}
+export interface StructuredExtractionResult {
+    status: 'complete' | 'incomplete' | 'invalid';
+    data: JsonValue | CanonicalStructuredData | null;
+    /** Present for a caller supplied schema; omitted for canonical entity JSON. */
+    schemaSha256?: string;
+    evidence: readonly StructuredFieldEvidence[];
+    issues: readonly StructuredExtractionIssue[];
+    modelUsage?: StructuredModelUsage | null;
+}
+//# sourceMappingURL=structured.d.ts.map

package/dist/types/cjs/contracts/tableMarkdown.d.ts

@@ -0,0 +1,58 @@
+/**
+ * Structural table assertions for benchmark ground truth.
+ *
+ * mustContain/mustNotContain are substring checks: an extraction that drops a
+ * column, shifts a row, or flattens the table into prose still contains every
+ * required string. These helpers parse GFM tables out of extracted markdown and
+ * score column/row geometry against an ExpectedTable annotation.
+ */
+/** A parsed GFM table. Rows and columns are logical (span spacer cells counted). */
+export interface GfmTable {
+    columns: number;
+    rows: number;
+    /** Trimmed cell text at a logical position; undefined outside bounds. */
+    cellAt: (row: number, col: number) => string | undefined;
+}
+/**
+ * Structural assertion for one table in the extracted markdown.
+ *
+ * Cell keys are zero-indexed "row,col". A position covered by a colspan/rowspan
+ * spacer cell is addressable but always empty, so annotations should address
+ * span origins, not continuations. Cell text is compared exactly against the
+ * trimmed markdown cell, so fixture cells must be unique strings.
+ */
+export interface ExpectedTable {
+    /** Logical column count after colspans are expanded. */
+    columns: number;
+    /** Logical row count after rowspans are expanded (header row included). */
+    rows: number;
+    /** Exact cell text by logical position. Use unique cell strings. */
+    cells?: Readonly<Record<string, string>>;
+    /** Cell texts that must land in the same table row. */
+    sameRow?: readonly string[];
+    /** Cell texts that must land in the same table column. */
+    sameColumn?: readonly string[];
+    /**
+     * Require the content to be a real GFM table. Set false for fixtures that
+     * accept any representation; presence is then enforced by mustContain alone.
+     * Defaults to true.
+     */
+    requireMarkdown?: boolean;
+}
+export interface ExpectedTableCheck {
+    pass: boolean;
+    /** Human-readable failures, one per violated constraint. */
+    issues: readonly string[];
+    /** The parsed table, or null when no GFM table was found. */
+    table: GfmTable | null;
+}
+/**
+ * Find the first GFM table in markdown: a header row, a delimiter row with the
+ * same cell count whose cells are hyphens (optionally colon-aligned), and zero
+ * or more body rows of the same cell count. Fenced code blocks are skipped.
+ * Returns null when no table is present.
+ */
+export declare function parseGfmTable(markdown: string): GfmTable | null;
+/** Score extracted markdown against an ExpectedTable annotation. */
+export declare function evaluateExpectedTable(markdown: string, spec: ExpectedTable): ExpectedTableCheck;
+//# sourceMappingURL=tableMarkdown.d.ts.map

package/dist/types/cjs/contracts/tokens.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Canonical token estimator for benchmark purposes.
+ *
+ * Deliberately simple and dependency-free so that ground-truth ranges and
+ * measured values are produced by the same function. Swapping this out
+ * invalidates every stored `expectedMainTokens` range — treat a change here
+ * as a suite version bump.
+ *
+ * Approximates BPE behaviour: ~4 chars/token for prose, with CJK counted
+ * closer to 1 char/token.
+ */
+export declare function estimateTokens(text: string): number;
+/**
+ * A successful HTTP extraction at or below this many tokens is thin enough
+ * that the ladder should offer it to a higher lane. This is NOT a content
+ * floor that hides loss — thin content is reported as success with its real
+ * token count, and the escalation is an addition, never a rewrite.
+ *
+ * Calibrated against the live-comparison measurements: producthunt.com's JS
+ * shell yields ~100-150 tokens over HTTP while the rendered page yields
+ * ~24k. A threshold of 200 separates that shell from a real server-rendered
+ * page without ever mistaking an honest short page (a terse buy-box, a 404)
+ * for a shell — those stay exactly what they are.
+ */
+export declare const QUALITY_ESCALATION_MAX_TOKENS = 200;
+/**
+ * Extraction confidence at or below this value marks the result as
+ * low-confidence, the second half of the quality signal. `confidenceOf`
+ * caps non-article pages at 0.75 and suspiciously small main regions at 0.4;
+ * a value of 0.3 sits below both, so only genuinely uncertain extractions
+ * escalate.
+ */
+export declare const QUALITY_ESCALATION_MAX_CONFIDENCE = 0.3;
+/**
+ * A rendered answer (the browser or a provider lane's) at or below this many
+ * main-content tokens, extracted at or below QUALITY_ESCALATION_MAX_CONFIDENCE,
+ * carries the `low_content_yield` warning: it is the run's answer, so the
+ * caveat is what tells the reader it holds little. Set from the browser lane's
+ * yield on the real-site set (research/parity/runs/2026-10-04-rendered-yield-
+ * calibration-b9dd658.md): IMF's datamapper renders 226 tokens of social and
+ * navigation links at confidence 0 while its figures are drawn by script; the
+ * thinnest real listing at confidence 0 there held 363 (ten quotes), and an
+ * honest short page (example.com, 267 tokens) is extracted at confidence 1.
+ * Listings are often extracted at confidence 0, so the tokens decide.
+ */
+export declare const RENDERED_LOW_YIELD_MAX_TOKENS = 300;
+//# sourceMappingURL=tokens.d.ts.map

package/dist/types/cjs/index.d.ts

@@ -0,0 +1,9 @@
+export { chunkUrls, MAP_ANSWER_MARGIN_MS, SDK_ORIGIN, W2L, W2LError, WaitTimeoutError } from './client.js';
+export { DEFAULT_WATCH_POLL_INTERVAL_MS, JobWatcher, MIN_WATCH_POLL_INTERVAL_MS, pageCursor, parseSseBlock } from './watcher.js';
+export type { WatchKind, WatchOptions, WatchTransport, WatcherClient, WatcherError, WatcherEvent, WatcherWebSocket, WatcherWebSocketConstructor } from './watcher.js';
+export { SDK_VERSION } from './version.js';
+export type { W2LOptions, RequestOptions, WaitOptions, CreateMonitorRequest, ReviseMonitorRequest, RunMonitorRequest, ScrapeOptions } from './client.js';
+export type { AppendToBatchOptions, ChunkedBatchJob, ChunkedBatchOptions, ChunkedBatchResult } from './client.js';
+export type { BatchCollected, BatchDocuments, CrawlCollected, CrawlDocuments, PageCollection, PagedListOptions, PaginationEnd, PaginationLimits, PaginationStop } from './client.js';
+export type { EvidenceRecord, PageTable, PdfPageMarkdown, PdfParser, ActiveCrawl, ActiveCrawlList, ActiveCrawlOptions, AgentHints, ApiErrorBody, ApiErrorCode, ApiErrorDetails, AttributeExtraction, AttributeSelector, AttributesFormatRequest, FetchWarning, JsonFormatRequest, PageMetadata, ScrapeFormat, ScreenshotEvidence, ScreenshotFormatRequest, ScreenshotOptions, ScreenshotViewport, RateLimitedBody, RequestAttribution, ScrapeMetadata, ScrapeRecord, ScrapeResponseMetadata, BatchAccepted, BatchErrorItem, BatchErrorStatus, BatchErrorsQuery, BatchErrorsResponse, BatchStartRequest, BatchStatusResponse, CompactScrapeResponse, ScrapeResponse, CrawlAccepted, DeliveryAttempt, DeliveryDestination, DeliveryDestinationInput, DeliveryDetail, DeliveryQuery, DeliveryState, WebhookDelivery, WebhookEventEnvelope, JobWebhookEnvelope, JobWebhookStatus, WebhookConfig, WebhookEvent, WebhookOption, WebhookPayload, WebhookPayloadFormat, FirecrawlWebhookPayload, DeliveryDestinationKind, CrawlReport, CrawlError, CrawlPage, CrawlPageList, CrawlPageQuery, CrawlStartRequest, JobStreamEventType, JobStreamFrame, JobStreamReport, SitemapMode, SitemapDiscovery, SitemapFileRecord, CrawlDiscovery, FetchResult, DocumentMonitorConfig, DocumentFields, FieldValue, MonitorFieldRule, MonitorRevision, MonitorRun, MonitorView, MonitorEvent, MonitorSnapshot, ScrapeRequest, } from './contracts/index.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/types/cjs/package.json

@@ -0,0 +1 @@
+{ "type": "commonjs" }

package/dist/types/cjs/version.d.ts

@@ -0,0 +1,8 @@
+/**
+ * The SDK's own version, pinned to packages/sdk/package.json by its test. It
+ * is what the SDK writes as the `origin` of every scrape, crawl and batch it
+ * starts (`js-sdk@<version>`), for W2L's own records; nothing goes to the
+ * target.
+ */
+export declare const SDK_VERSION = "0.3.0";
+//# sourceMappingURL=version.d.ts.map

package/dist/types/cjs/watcher.d.ts

@@ -0,0 +1,151 @@
+/**
+ * The job watcher: a crawl's or batch's pages as they are recorded, from the
+ * API's stream routes (WebSocket, then server-sent events) or, where neither
+ * answers, from the status and listing routes polled. Every transport
+ * delivers the same compact pages the items routes list; a document is
+ * emitted once per step id whatever route, reconnection or transport switch
+ * it arrived by. Purely client-side: the server records nothing about a
+ * watcher, and `close()` stops watching without touching the job.
+ */
+import { type BatchStatusResponse, type CrawlError, type CrawlPage, type CrawlPageList, type CrawlPageQuery, type CrawlReport, type JobStreamFrame, type JobStreamReport, type TaskStatus } from './contracts/index.js';
+export type WatchKind = 'crawl' | 'batch';
+export type WatchTransport = 'websocket' | 'sse' | 'poll';
+/** What the watcher reports as `error`: the API's code where it has one (`not_found`, `unauthorized`), else the watcher's own (`watcher_timeout`, `transport_unavailable`, `poll_failed`). */
+export interface WatcherError {
+    code: string;
+    message: string;
+}
+/** The events a watcher dispatches (CustomEvent `detail`) and yields from its async iterator. */
+export type WatcherEvent = {
+    type: 'document';
+    data: CrawlPage;
+} | {
+    type: 'snapshot';
+    data: JobStreamReport;
+} | {
+    type: 'done';
+    data: JobStreamReport;
+} | {
+    type: 'error';
+    error: WatcherError;
+};
+/** The part of a WebSocket the watcher uses; the platform's WebSocket satisfies it. */
+export interface WatcherWebSocket {
+    readonly readyState: number;
+    readonly protocol: string;
+    close(code?: number, reason?: string): void;
+    addEventListener(type: 'open' | 'message' | 'close' | 'error', listener: (event: {
+        data?: unknown;
+        code?: number;
+        reason?: string;
+    }) => void): void;
+}
+export type WatcherWebSocketConstructor = new (url: string, protocols?: string | string[]) => WatcherWebSocket;
+export interface WatchOptions {
+    /** Which kind of job the id names. Default `crawl`. */
+    kind?: WatchKind;
+    /**
+     * `auto` (default) tries a WebSocket, then the server-sent events route, then
+     * polling, switching once per level when a transport is unavailable or ends
+     * before `done`; one of the three forces it (an error event when it cannot run).
+     */
+    transport?: 'auto' | WatchTransport;
+    /** Delay between polls on the poll transport, at least 250. Default 2000. */
+    pollIntervalMs?: number;
+    /** After this long without `done`, an `error` of code `watcher_timeout` ends the watch; the job keeps running. Default: no limit. */
+    timeoutMs?: number;
+    /** The step cursor (a document's `cursor` / SSE `id`) to resume after: documents up to it are not delivered again. */
+    after?: string;
+    /** Aborting it closes the watcher, with no event. */
+    signal?: AbortSignal;
+    /** The WebSocket constructor to use; omitted takes the platform's (Node 22+, browsers); `null` tries no WebSocket. */
+    WebSocket?: WatcherWebSocketConstructor | null;
+}
+/** What the watcher needs from the SDK client: the server, its token and fetch, and the status and listing routes it polls. */
+export interface WatcherClient {
+    readonly baseUrl: string;
+    readonly token: string | undefined;
+    readonly fetch: typeof fetch;
+    headers(extra?: Record<string, string>): Record<string, string>;
+    getCrawl(id: string, request?: {
+        signal?: AbortSignal;
+    }): Promise<CrawlReport>;
+    getBatch(id: string, request?: {
+        signal?: AbortSignal;
+    }): Promise<BatchStatusResponse>;
+    getCrawlPages(id: string, options?: CrawlPageQuery, request?: {
+        signal?: AbortSignal;
+    }): Promise<CrawlPageList<CrawlPage>>;
+    getCrawlErrors(id: string, options?: CrawlPageQuery, request?: {
+        signal?: AbortSignal;
+    }): Promise<CrawlPageList<CrawlError>>;
+    getBatchItems(id: string, options?: CrawlPageQuery, request?: {
+        signal?: AbortSignal;
+    }): Promise<CrawlPageList<CrawlPage>>;
+}
+export declare const DEFAULT_WATCH_POLL_INTERVAL_MS = 2000;
+export declare const MIN_WATCH_POLL_INTERVAL_MS = 250;
+/**
+ * A page's step cursor as the API issues it (the `(createdAt, id)` position
+ * the listing routes take): what polling continues from after the last
+ * page of a listing, which carries no `nextCursor`.
+ */
+export declare function pageCursor(page: Pick<CrawlPage, 'createdAt' | 'id'>): string;
+/** One server-sent event block as a stream frame; null for a comment, an event W2L does not send or data that is not JSON. */
+export declare function parseSseBlock(block: string): JobStreamFrame | null;
+/**
+ * Watches one job. Events: `document` (CustomEvent<CrawlPage>), `snapshot`
+ * (CustomEvent<CrawlReport | BatchStatusResponse>, the report as the stream
+ * opens and after each page), `done` (the terminal report) and `error`
+ * (CustomEvent<WatcherError>); `for await (const event of watcher)` yields
+ * the same events from the start. `data` accumulates every document, each
+ * once; `status` is the last status seen; `transport` the one in use.
+ */
+export declare class JobWatcher extends EventTarget {
+    readonly jobId: string;
+    readonly kind: WatchKind;
+    /** Every document delivered so far, each step once, in arrival order. */
+    readonly data: CrawlPage[];
+    /** The job's status as last reported; null before the first report. */
+    status: TaskStatus | null;
+    /** The transport delivering events; null before one is open and after the watch ends. */
+    transport: WatchTransport | null;
+    private readonly client;
+    private readonly transportOption;
+    private readonly pollIntervalMs;
+    private readonly socketConstructor;
+    private readonly controller;
+    private readonly seen;
+    private readonly log;
+    private readonly waiters;
+    /** The cursor of the last document delivered: where the next transport, or a poll, continues from. */
+    private cursor;
+    private lastSnapshot;
+    private finished;
+    private closed;
+    private timer;
+    constructor(client: WatcherClient, jobId: string, options?: WatchOptions);
+    /** Stops watching: no further event, the job untouched. */
+    close(): void;
+    /** The events from the start, then as they arrive, until done, error or close. */
+    [Symbol.asyncIterator](): AsyncIterator<WatcherEvent>;
+    private get stopped();
+    private run;
+    /** The stream route of this job, with the cursor to resume after when one is known. */
+    private streamUrl;
+    /** The WebSocket route; the token, when there is one, as the `w2l.token.<token>` subprotocol, since the WebSocket API sets no header. */
+    private watchSocket;
+    /** The server-sent events route: 404 leaves it to polling, 401/403 ends the watch, a stream that ends before done hands over from the last cursor. */
+    private watchEvents;
+    /** The status and listing routes every pollIntervalMs: the documents since the last cursor, a snapshot, and done once the status is terminal and the last pages are read. */
+    private poll;
+    private handle;
+    private document;
+    private snapshot;
+    private done;
+    private fail;
+    private emit;
+    private stop;
+    private wake;
+}
+//# sourceMappingURL=watcher.d.ts.map