@hunt-labs/bytekit-sdk 0.1.0 → 0.2.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Hunt Labs, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,171 @@
+# @hunt-labs/bytekit-sdk
+
+Official TypeScript SDK for the [ByteKit](https://bytekit.com) API — web data
+infrastructure for AI agents and training pipelines: scraping, raw fetch,
+screenshots, sitemaps, page monitors, and web search.
+
+## Install
+
+```bash
+npm install @hunt-labs/bytekit-sdk
+```
+
+> **ESM-only.** This package ships as ESM (`"type": "module"`) and has no
+> CommonJS build. Use it from an ESM project (`"type": "module"` in your
+> `package.json`, or a `.mjs` entrypoint), or load it from CommonJS with a
+> dynamic `await import('@hunt-labs/bytekit-sdk')`. A top-level
+> `require('@hunt-labs/bytekit-sdk')` will not work.
+
+Requires Node.js 18+ (the client uses the global `fetch`).
+
+## Authentication
+
+Create an API key at [app.bytekit.com](https://app.bytekit.com) (keys are
+prefixed `sk_live_`) and pass it to the client constructor:
+
+```ts
+import { RapidCrawl } from '@hunt-labs/bytekit-sdk';
+
+const client = new RapidCrawl({ apiKey: process.env.BYTEKIT_API_KEY! }); // sk_live_...
+```
+
+The key is sent as `Authorization: Bearer <apiKey>` on every request. Keep it
+server-side — it is not safe to ship in a browser bundle.
+
+> The client class is named `RapidCrawl` for historical reasons (ByteKit's
+> previous product name). It is the SDK's only entrypoint class.
+
+## Base-URL override
+
+Requests go to `https://api.bytekit.com` by default. Point the client at
+staging, a local gateway, or a proxy with the `baseUrl` option:
+
+```ts
+const client = new RapidCrawl({
+  apiKey: process.env.BYTEKIT_API_KEY!,
+  baseUrl: 'https://api-stg.bytekit.com', // default: https://api.bytekit.com
+});
+```
+
+## Usage
+
+### Scrape a page
+
+```ts
+const result = await client.scrape.create({
+  url: 'https://example.com',
+  formats: ['markdown', 'html'],
+});
+```
+
+### Fetch an async scrape by id
+
+```ts
+const scrape = await client.scrape.get('sc_123abc');
+```
+
+### Raw fetch (unprocessed response body)
+
+```ts
+const raw = await client.fetch.create({ url: 'https://example.com' });
+```
+
+### Web search
+
+```ts
+const results = await client.search.query({ query: 'ai web scraping', limit: 10 });
+for (const hit of results.results) {
+  console.log(hit.position, hit.title, hit.url);
+}
+```
+
+### Screenshot a page
+
+```ts
+const shot = await client.screenshots.create({
+  url: 'https://example.com',
+  full_page: true,
+});
+const ready = await client.screenshots.get('ss_123abc');
+```
+
+### Bulk jobs
+
+`scrape` and `fetch` each expose a `bulk` sub-resource that takes a URL list
+and a webhook to notify on completion:
+
+```ts
+const job = await client.scrape.bulk.create({
+  urls: ['https://example.com', 'https://example.org'],
+  webhook_url: 'https://your.app/hooks/bytekit',
+});
+const status = await client.scrape.bulk.get('bulk_123abc');
+```
+
+`client.fetch.bulk.create()` / `client.fetch.bulk.get()` follow the same shape.
+
+Bulk **screenshots** are not a `screenshots.bulk` sub-resource — they go
+through the separate top-level `client.bulk` resource instead:
+
+```ts
+const shots = await client.bulk.create({
+  urls: ['https://example.com', 'https://example.org'],
+  webhook_url: 'https://your.app/hooks/bytekit',
+});
+const shotStatus = await client.bulk.get('bulk_123abc');
+await client.bulk.screenshots.list('bulk_123abc');
+```
+
+### Page monitors
+
+```ts
+const monitor = await client.monitors.create({
+  url: 'https://example.com/pricing',
+  interval_type: 'daily',
+  webhook_url: 'https://your.app/hooks/bytekit',
+});
+await client.monitors.list();
+await client.monitors.delete('mon_123abc');
+```
+
+### Sitemap crawl
+
+```ts
+const crawl = await client.sitemap.create({ url: 'https://example.com' });
+const sitemap = await client.sitemap.get('sm_123abc');
+```
+
+### Account, usage, and API keys
+
+```ts
+const account = await client.account.get();
+const keys = await client.account.apiKeys.list();
+```
+
+> `client.recordings.*` targets the `/v1/recordings` endpoints, which are
+> deactivated server-side and return HTTP 410.
+
+## Error handling
+
+Any 4xx/5xx response throws a `RapidCrawlError` carrying the HTTP status and
+the API's error code:
+
+```ts
+import { RapidCrawl, RapidCrawlError } from '@hunt-labs/bytekit-sdk';
+
+try {
+  await client.scrape.create({ url: 'https://example.com' });
+} catch (err) {
+  if (err instanceof RapidCrawlError) {
+    console.error(err.status, err.code, err.message);
+  }
+}
+```
+
+## Documentation
+
+Full API reference: [bytekit.com/docs](https://bytekit.com/docs).
+
+## License
+
+MIT — see [LICENSE](./LICENSE).

package/dist/allowlist.d.ts

@@ -10,4 +10,3 @@
  */
 export declare const V01_METHODS: readonly ["scrape.create", "scrape.get", "screenshots.create", "screenshots.get", "bulk.create", "bulk.get", "bulk.cancel", "bulk.screenshots.list", "scrape.bulk.create", "scrape.bulk.get", "fetch.create", "fetch.get", "fetch.bulk.create", "fetch.bulk.get", "monitors.create", "monitors.list", "monitors.get", "monitors.update", "monitors.delete", "monitors.captures.list", "sitemap.create", "sitemap.get", "search.query", "account.get"];
 export type V01Method = (typeof V01_METHODS)[number];
-//# sourceMappingURL=allowlist.d.ts.map

package/dist/allowlist.js

@@ -35,4 +35,3 @@ export const V01_METHODS = [
     'search.query',
     'account.get',
 ];
-//# sourceMappingURL=allowlist.js.map

package/dist/client.d.ts

@@ -10,16 +10,60 @@ import { AccountResource } from './resources/account.js';
 export interface RapidCrawlOptions {
     apiKey: string;
     baseUrl?: string;
+    /**
+     * Default CLIENT-SIDE request timeout in ms — the abort budget for the whole HTTP
+     * round-trip made by this SDK. `0` disables the SDK-side timeout. Default 120000.
+     *
+     * NOT the same thing as `ScrapeOpts.timeout_ms`, which is the SERVER-side budget the
+     * gateway applies to the scrape itself. Override per call with
+     * `RequestOptions.requestTimeoutMs`.
+     */
+    timeoutMs?: number;
+}
+/** Per-call overrides accepted by `request()` and by resource methods that opt in. */
+export interface RequestOptions {
+    /**
+     * Caller-owned abort signal. When supplied WITHOUT a per-call `requestTimeoutMs`, it
+     * fully replaces the constructor timeout — supply `requestTimeoutMs` alongside it to
+     * keep a bound (the request then aborts on whichever fires first).
+     */
+    signal?: AbortSignal;
+    /**
+     * Per-call CLIENT-SIDE abort budget in ms. Wins over the constructor's `timeoutMs`.
+     * `0` disables the SDK-side timeout for this call.
+     *
+     * Deliberately NOT named `timeoutMs`: it rides on the same call as `ScrapeOpts.timeout_ms`
+     * (the SERVER-side scrape budget), and the two mean different things —
+     * `scrape.create({ timeout_ms: 30_000 }, { requestTimeoutMs: 5_000 })` says "let the
+     * server spend up to 30 s scraping, but give up on the wire after 5 s".
+     */
+    requestTimeoutMs?: number;
 }
+/** Why a request never produced an HTTP response. */
+export type ConnectionErrorCode = 'timeout' | 'aborted' | 'connection';
 /** Thrown when the API responds with a 4xx or 5xx status code. */
 export declare class RapidCrawlError extends Error {
     readonly status: number;
     readonly code: string;
     constructor(status: number, code: string, message: string);
 }
+/**
+ * Thrown when the request never produced an HTTP response at all — DNS, socket, TLS,
+ * a client-side timeout, or a caller-initiated abort. Distinct from `RapidCrawlError`,
+ * which always carries a real HTTP status: catch both to cover every failure.
+ *
+ * The original failure is preserved on `.cause`.
+ */
+export declare class RapidCrawlConnectionError extends Error {
+    readonly code: ConnectionErrorCode;
+    constructor(code: ConnectionErrorCode, message: string, options?: {
+        cause?: unknown;
+    });
+}
 export declare class RapidCrawl {
     private readonly apiKey;
     private readonly baseUrl;
+    private readonly defaultTimeoutMs;
     readonly scrape: ScrapeResource;
     readonly screenshots: ScreenshotsResource;
     readonly recordings: RecordingsResource;
@@ -30,11 +74,26 @@ export declare class RapidCrawl {
     readonly search: SearchResource;
     readonly account: AccountResource;
     constructor(options: RapidCrawlOptions);
+    /**
+     * Resolve the abort budget for one request from the caller's options and the
+     * constructor default. Precedence:
+     *
+     * | caller `signal` | per-call `requestTimeoutMs` | signal attached to `fetch`            |
+     * |-----------------|----------------------------|---------------------------------------|
+     * | absent          | absent                     | timeout from the constructor default   |
+     * | absent          | present                    | timeout from the per-call value        |
+     * | present         | absent                     | the caller's signal, verbatim          |
+     * | present         | present                    | composition — aborts on either         |
+     *
+     * A timeout of `0` (either level) means "no SDK-side timeout".
+     */
+    private resolveSignal;
     /**
      * Make an authenticated JSON request to the RapidCrawl API.
      *
-     * Throws `RapidCrawlError` on 4xx/5xx responses.
+     * Throws `RapidCrawlError` on 4xx/5xx responses, and `RapidCrawlConnectionError`
+     * when no HTTP response was produced at all (DNS/socket/TLS failure, client-side
+     * timeout, or caller abort).
      */
-    request<T>(method: string, path: string, body?: unknown): Promise<T>;
+    request<T>(method: string, path: string, body?: unknown, options?: RequestOptions): Promise<T>;
 }
-//# sourceMappingURL=client.d.ts.map

package/dist/client.js

@@ -8,6 +8,14 @@ import { SitemapResource } from './resources/sitemap.js';
 import { SearchResource } from './resources/search.js';
 import { AccountResource } from './resources/account.js';
 const DEFAULT_BASE_URL = 'https://api.bytekit.com';
+/**
+ * Default client-side request timeout (2 minutes).
+ *
+ * Above the slowest expected synchronous scrape, and well below undici's ~5-minute
+ * headers timeout, so the SDK is always the tighter bound and a hung upstream can
+ * never hang the caller indefinitely.
+ */
+const DEFAULT_TIMEOUT_MS = 120_000;
 /** Thrown when the API responds with a 4xx or 5xx status code. */
 export class RapidCrawlError extends Error {
     status;
@@ -19,9 +27,134 @@ export class RapidCrawlError extends Error {
         this.code = code;
     }
 }
+/**
+ * Thrown when the request never produced an HTTP response at all — DNS, socket, TLS,
+ * a client-side timeout, or a caller-initiated abort. Distinct from `RapidCrawlError`,
+ * which always carries a real HTTP status: catch both to cover every failure.
+ *
+ * The original failure is preserved on `.cause`.
+ */
+export class RapidCrawlConnectionError extends Error {
+    code;
+    constructor(code, message, options) {
+        super(message, options);
+        this.name = 'RapidCrawlConnectionError';
+        this.code = code;
+    }
+}
+const NOOP = () => { };
+/**
+ * Validate a timeout option. `undefined` passes through; anything that is not a
+ * non-negative finite number throws a `TypeError` naming the option as the caller
+ * spelled it (`"timeoutMs"` at the constructor, `"requestTimeoutMs"` per call).
+ */
+function validateTimeout(value, optionName) {
+    if (value === undefined)
+        return undefined;
+    if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
+        throw new TypeError(`RapidCrawl: ${optionName} must be a non-negative finite number (got ${describe(value)})`);
+    }
+    return value;
+}
+/** Render a rejected option value for an error message without ever stringifying an object. */
+function describe(value) {
+    if (typeof value === 'number' || typeof value === 'boolean' || typeof value === 'bigint') {
+        return String(value);
+    }
+    if (typeof value === 'string')
+        return JSON.stringify(value);
+    if (value === null)
+        return 'null';
+    return typeof value;
+}
+/** The abort reason used by the `setTimeout` fallback; named so it maps to `code: 'timeout'`. */
+function timeoutReason(ms) {
+    const reason = new Error(`RapidCrawl: request timed out after ${ms}ms`);
+    reason.name = 'TimeoutError';
+    return reason;
+}
+/** `AbortSignal.timeout(ms)` where available, else an `AbortController` + `setTimeout`. */
+function createTimeoutSignal(ms) {
+    if (typeof AbortSignal.timeout === 'function') {
+        return { signal: AbortSignal.timeout(ms), cleanup: NOOP };
+    }
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(timeoutReason(ms)), ms);
+    timer.unref?.();
+    return {
+        signal: controller.signal,
+        cleanup: () => clearTimeout(timer),
+    };
+}
+/**
+ * Abort on whichever source fires first.
+ *
+ * Prefers `AbortSignal.any`, whose sources hold only WEAK references to the composed
+ * signal — so nothing has to be (and nothing must be) retained to keep it collectable.
+ * The manual fallback is the path that has to clean up after itself: it attaches a real
+ * listener to the CALLER's signal, which typically outlives the request (one long-lived
+ * `AbortController` reused across many calls), so `cleanup()` removes it again.
+ */
+function composeSignals(caller, timeout) {
+    const anyFn = AbortSignal.any?.bind(AbortSignal);
+    if (typeof anyFn === 'function') {
+        return { signal: anyFn([caller, timeout]), cleanup: NOOP };
+    }
+    const controller = new AbortController();
+    const sources = [caller, timeout];
+    const onAbort = (event) => {
+        controller.abort(event.target.reason);
+    };
+    let done = false;
+    const cleanup = () => {
+        if (done)
+            return;
+        done = true;
+        for (const source of sources)
+            source.removeEventListener('abort', onAbort);
+    };
+    for (const source of sources)
+        source.addEventListener('abort', onAbort);
+    for (const source of sources) {
+        if (source.aborted) {
+            controller.abort(source.reason);
+            break;
+        }
+    }
+    return { signal: controller.signal, cleanup };
+}
+/** Map a `fetch` rejection onto the connection-error taxonomy. Never leaks the API key. */
+function toConnectionError(err, method, url, timeoutSignal) {
+    // undici rejects an aborted fetch with the signal's own reason, so the thrown value's
+    // `name` is the primary discriminator (`TimeoutError` / `AbortError`).
+    const rawName = typeof err === 'object' && err !== null && 'name' in err ? err.name : undefined;
+    const name = typeof rawName === 'string' ? rawName : '';
+    let code;
+    if (name === 'TimeoutError') {
+        code = 'timeout';
+    }
+    else if (name === 'AbortError') {
+        code = 'aborted';
+    }
+    else if (timeoutSignal?.aborted === true) {
+        code = 'timeout';
+    }
+    else {
+        code = 'connection';
+    }
+    const detail = code === 'timeout'
+        ? 'request timed out'
+        : code === 'aborted'
+            ? 'request aborted by the caller'
+            : 'connection failed before a response was received';
+    return new RapidCrawlConnectionError(code, `RapidCrawl: ${method} ${url} — ${detail}`, {
+        cause: err,
+    });
+}
 export class RapidCrawl {
     apiKey;
     baseUrl;
+    defaultTimeoutMs;
     scrape;
     screenshots;
     recordings;
@@ -32,8 +165,12 @@ export class RapidCrawl {
     search;
     account;
     constructor(options) {
+        if (!options?.apiKey) {
+            throw new TypeError('RapidCrawl: `apiKey` is required — got an empty value. Pass options.apiKey (e.g. from process.env.BYTEKIT_API_KEY).');
+        }
         this.apiKey = options.apiKey;
         this.baseUrl = options.baseUrl ?? DEFAULT_BASE_URL;
+        this.defaultTimeoutMs = validateTimeout(options.timeoutMs, '"timeoutMs"') ?? DEFAULT_TIMEOUT_MS;
         this.scrape = new ScrapeResource(this);
         this.screenshots = new ScreenshotsResource(this);
         this.recordings = new RecordingsResource(this);
@@ -44,13 +181,52 @@ export class RapidCrawl {
         this.search = new SearchResource(this);
         this.account = new AccountResource(this);
     }
+    /**
+     * Resolve the abort budget for one request from the caller's options and the
+     * constructor default. Precedence:
+     *
+     * | caller `signal` | per-call `requestTimeoutMs` | signal attached to `fetch`            |
+     * |-----------------|----------------------------|---------------------------------------|
+     * | absent          | absent                     | timeout from the constructor default   |
+     * | absent          | present                    | timeout from the per-call value        |
+     * | present         | absent                     | the caller's signal, verbatim          |
+     * | present         | present                    | composition — aborts on either         |
+     *
+     * A timeout of `0` (either level) means "no SDK-side timeout".
+     */
+    resolveSignal(options) {
+        const callerSignal = options?.signal;
+        const perCallMs = validateTimeout(options?.requestTimeoutMs, '"requestTimeoutMs"');
+        // A caller-supplied signal suppresses the constructor default (it does not
+        // suppress an explicit per-call budget, which composes with it instead).
+        const effectiveMs = perCallMs ?? (callerSignal !== undefined ? 0 : this.defaultTimeoutMs);
+        if (effectiveMs === 0) {
+            return { signal: callerSignal, timeoutSignal: undefined, cleanup: NOOP };
+        }
+        const timeout = createTimeoutSignal(effectiveMs);
+        if (callerSignal === undefined) {
+            return { signal: timeout.signal, timeoutSignal: timeout.signal, cleanup: timeout.cleanup };
+        }
+        const composed = composeSignals(callerSignal, timeout.signal);
+        return {
+            signal: composed.signal,
+            timeoutSignal: timeout.signal,
+            cleanup: () => {
+                composed.cleanup();
+                timeout.cleanup();
+            },
+        };
+    }
     /**
      * Make an authenticated JSON request to the RapidCrawl API.
      *
-     * Throws `RapidCrawlError` on 4xx/5xx responses.
+     * Throws `RapidCrawlError` on 4xx/5xx responses, and `RapidCrawlConnectionError`
+     * when no HTTP response was produced at all (DNS/socket/TLS failure, client-side
+     * timeout, or caller abort).
      */
-    async request(method, path, body) {
+    async request(method, path, body, options) {
         const url = `${this.baseUrl}${path}`;
+        const { signal, timeoutSignal, cleanup } = this.resolveSignal(options);
         const init = {
             method,
             headers: {
@@ -61,7 +237,22 @@ export class RapidCrawl {
         if (body !== undefined) {
             init.body = JSON.stringify(body);
         }
-        const response = await fetch(url, init);
+        // `exactOptionalPropertyTypes` — assign only when we actually have a signal.
+        if (signal !== undefined) {
+            init.signal = signal;
+        }
+        let response;
+        try {
+            response = await fetch(url, init);
+        }
+        catch (err) {
+            throw toConnectionError(err, method, url, timeoutSignal);
+        }
+        finally {
+            // Runs on success, on HTTP error, and on abort alike — no dangling timer,
+            // no listener left behind on the caller's (long-lived) signal.
+            cleanup();
+        }
         if (!response.ok) {
             let code = 'internal_error';
             let message = response.statusText;
@@ -78,4 +269,3 @@ export class RapidCrawl {
         return response.json();
     }
 }
-//# sourceMappingURL=client.js.map