@spider-cloud/spider-client 0.0.40 → 0.0.44

package/README.md

@@ -59,6 +59,34 @@ app
   });
 ```
 
+A real world crawl example streaming the response.
+
+```javascript
+import { Spider } from "@spider-cloud/spider-client";
+
+// Initialize the SDK with your API key
+const app = new Spider({ apiKey: "YOUR_API_KEY" });
+
+// The target URL
+const url = "https://spider.cloud";
+
+// Crawl a website
+const crawlParams = {
+  limit: 5,
+  store_data: false,
+  metadata: true,
+  request: "http",
+};
+
+const stream = true;
+
+const streamCallback = (data) => {
+  console.log(data["url"]);
+};
+
+app.crawlUrl(url, crawlParams, stream, streamCallback);
+```
+
 ### Data Operations
 
 The Spider client can interact with specific data tables to create, retrieve, and delete data.
@@ -124,7 +152,6 @@ spider
 
 You can use [Supabase](https://supabase.com/docs/reference/javascript) to directly connect to instances and write your own logic. First, you need to install `@supabase/supabase-js` since this package does not include the dependency by default. This keeps the bundle size small and allows for lazy imports of the client.
 
-
 ```ts
 const spiderClient = new Spider({ apiKey: process.env.SPIDER_API_KEY });
 
@@ -149,8 +176,4 @@ Contributions are always welcome! Feel free to open an issue or submit a pull re
 
 ## License
 
-The Spider Cloud JavaScript SDK is open-source and released under the [MIT License](https://opensource.org/licenses/MIT).
-
-```
-
-```
+The Spider Cloud JavaScript SDK is open-source and released under the [MIT License](https://opensource.org/licenses/MIT).

package/dist/client.d.ts

@@ -1,4 +1,4 @@
-import { SpiderParams } from "./config";
+import { ChunkCallbackFunction, SpiderCoreResponse, SpiderParams } from "./config";
 /**
  * Generic params for core request.
  */
@@ -60,9 +60,10 @@ export declare class Spider {
      * @param {string} url - The URL to start crawling.
      * @param {GenericParams} [params={}] - Additional parameters for the crawl.
      * @param {boolean} [stream=false] - Whether to receive the response as a stream.
+     * @param {function} [callback=function] - The callback function when streaming per chunk. If this is set with stream you will not get a end response.
      * @returns {Promise<any | Response>} The result of the crawl, either structured data or a Response object if streaming.
      */
-    crawlUrl(url: string, params?: GenericParams, stream?: boolean): Promise<any>;
+    crawlUrl(url: string, params?: GenericParams, stream?: boolean, cb?: ChunkCallbackFunction): Promise<SpiderCoreResponse[] | void>;
     /**
      * Retrieves all links from the specified URL.
      * @param {string} url - The URL from which to gather links.
@@ -71,8 +72,8 @@ export declare class Spider {
      */
     links(url: string, params?: {}): Promise<any>;
     /**
-     * Takes a screenshot of the specified URL.
-     * @param {string} url - The URL to screenshot.
+     * Takes a screenshot of the website starting from this URL.
+     * @param {string} url - The URL to start the screenshot.
      * @param {GenericParams} [params={}] - Configuration parameters for the screenshot.
      * @returns {Promise<any>} The screenshot data.
      */
@@ -158,7 +159,16 @@ export declare class Spider {
      * Prepares common headers for each API request.
      * @returns {HeadersInit} A headers object for fetch requests.
      */
-    prepareHeaders(): {
+    get prepareHeaders(): {
+        "Content-Type": string;
+        Authorization: string;
+        "User-Agent": string;
+    };
+    /**
+     * Prepares common headers for each API request with JSONl content-type suitable for streaming.
+     * @returns {HeadersInit} A headers object for fetch requests.
+     */
+    get prepareHeadersJsonL(): {
         "Content-Type": string;
         Authorization: string;
         "User-Agent": string;

package/dist/client.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.Spider = void 0;
 const package_json_1 = require("../package.json");
 const supabase_1 = require("./supabase");
+const stream_reader_1 = require("./utils/stream-reader");
 /**
  * A class to interact with the Spider API.
  */
@@ -38,8 +39,8 @@ class Spider {
      * @param {boolean} [stream=false] - Whether to stream the response back without parsing.
      * @returns {Promise<Response | any>} The response in JSON if not streamed, or the Response object if streamed.
      */
-    async _apiPost(endpoint, data, stream = false) {
-        const headers = this.prepareHeaders();
+    async _apiPost(endpoint, data, stream, jsonl) {
+        const headers = jsonl ? this.prepareHeadersJsonL : this.prepareHeaders;
         const response = await fetch(`https://api.spider.cloud/v1/${endpoint}`, {
             method: "POST",
             headers: headers,
@@ -61,7 +62,7 @@ class Spider {
      * @returns {Promise<any>} The data returned from the endpoint in JSON format.
      */
     async _apiGet(endpoint) {
-        const headers = this.prepareHeaders();
+        const headers = this.prepareHeaders;
         const response = await fetch(`https://api.spider.cloud/v1/${endpoint}`, {
             method: "GET",
             headers: headers,
@@ -79,7 +80,7 @@ class Spider {
      * @returns {Promise<any>} The data returned from the endpoint in JSON format.
      */
     async _apiDelete(endpoint) {
-        const headers = this.prepareHeaders();
+        const headers = this.prepareHeaders;
         const response = await fetch(`https://api.spider.cloud/v1/${endpoint}`, {
             method: "DELETE",
             headers,
@@ -105,10 +106,16 @@ class Spider {
      * @param {string} url - The URL to start crawling.
      * @param {GenericParams} [params={}] - Additional parameters for the crawl.
      * @param {boolean} [stream=false] - Whether to receive the response as a stream.
+     * @param {function} [callback=function] - The callback function when streaming per chunk. If this is set with stream you will not get a end response.
      * @returns {Promise<any | Response>} The result of the crawl, either structured data or a Response object if streaming.
      */
-    async crawlUrl(url, params = {}, stream = false) {
-        return this._apiPost("crawl", { url: url, ...params }, stream);
+    async crawlUrl(url, params = {}, stream = false, cb) {
+        const jsonl = stream && cb;
+        const res = await this._apiPost("crawl", { url: url, ...params }, stream, !!jsonl);
+        if (jsonl) {
+            return await (0, stream_reader_1.streamReader)(res, cb);
+        }
+        return res;
     }
     /**
      * Retrieves all links from the specified URL.
@@ -120,8 +127,8 @@ class Spider {
         return this._apiPost("links", { url: url, ...params });
     }
     /**
-     * Takes a screenshot of the specified URL.
-     * @param {string} url - The URL to screenshot.
+     * Takes a screenshot of the website starting from this URL.
+     * @param {string} url - The URL to start the screenshot.
      * @param {GenericParams} [params={}] - Configuration parameters for the screenshot.
      * @returns {Promise<any>} The screenshot data.
      */
@@ -190,7 +197,7 @@ class Spider {
             ...(expiresIn && { expiresIn: expiresIn.toString() }),
         });
         const endpoint = `https://api.spider.cloud/v1/data/storage?${params.toString()}`;
-        const headers = this.prepareHeaders();
+        const headers = this.prepareHeaders;
         const response = await fetch(endpoint, {
             method: "GET",
             headers,
@@ -243,13 +250,23 @@ class Spider {
      * Prepares common headers for each API request.
      * @returns {HeadersInit} A headers object for fetch requests.
      */
-    prepareHeaders() {
+    get prepareHeaders() {
         return {
             "Content-Type": "application/json",
             Authorization: `Bearer ${this.apiKey}`,
             "User-Agent": `Spider-Client/${package_json_1.version}`,
         };
     }
+    /**
+     * Prepares common headers for each API request with JSONl content-type suitable for streaming.
+     * @returns {HeadersInit} A headers object for fetch requests.
+     */
+    get prepareHeadersJsonL() {
+        return {
+            ...this.prepareHeaders,
+            "Content-Type": "application/jsonl",
+        };
+    }
     /**
      * Handles errors from API requests.
      * @param {Response} response - The fetch response object.

package/dist/config.d.ts

@@ -169,3 +169,11 @@ export interface SpiderParams {
      */
     chunking_alg?: ChunkingAlg;
 }
+export type SpiderCoreResponse = {
+    data?: string;
+    message?: string;
+    error?: string;
+    status?: number;
+    url?: string;
+};
+export type ChunkCallbackFunction = (data: SpiderCoreResponse) => void;

package/dist/utils/process-chunk.d.ts

@@ -0,0 +1,2 @@
+import type { SpiderCoreResponse } from "../config";
+export declare const processChunk: (chunk: string, cb: (r: SpiderCoreResponse) => void) => boolean;

package/dist/utils/process-chunk.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.processChunk = void 0;
+const processChunk = (chunk, cb) => {
+    try {
+        cb(chunk ? JSON.parse(chunk.trimEnd()) : null);
+        return true;
+    }
+    catch (_error) {
+        return false;
+    }
+};
+exports.processChunk = processChunk;

package/dist/utils/stream-reader.d.ts

@@ -0,0 +1,2 @@
+import type { ChunkCallbackFunction } from "../config";
+export declare const streamReader: (res: Response, cb: ChunkCallbackFunction) => Promise<void>;

package/dist/utils/stream-reader.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.streamReader = void 0;
+const process_chunk_1 = require("./process-chunk");
+// stream the response via callbacks.
+const streamReader = async (res, cb) => {
+    var _a;
+    if (res.ok) {
+        const reader = (_a = res.body) === null || _a === void 0 ? void 0 : _a.getReader();
+        const decoder = new TextDecoder();
+        let content = "";
+        if (reader) {
+            while (true) {
+                const { done, value } = await reader.read();
+                if (done) {
+                    break;
+                }
+                content += decoder.decode(value, { stream: true });
+                if ((0, process_chunk_1.processChunk)(content, cb)) {
+                    content = "";
+                }
+            }
+            if (content.length > 0) {
+                (0, process_chunk_1.processChunk)(content, cb);
+            }
+        }
+    }
+};
+exports.streamReader = streamReader;

package/package.json

@@ -1,9 +1,9 @@
 {
   "name": "@spider-cloud/spider-client",
-  "version": "0.0.40",
+  "version": "0.0.44",
   "description": "A Javascript SDK for Spider Cloud services",
   "scripts": {
-    "test": "jest",
+    "test": "node --import tsx  --test __tests__/**/*test.ts",
     "build": "tsc",
     "prepublishOnly": "npm test && npm run build"
   },
@@ -23,40 +23,10 @@
   "author": "Jeff Mendez<jeff@a11ywatch.com>",
   "license": "MIT",
   "devDependencies": {
-    "@jest/globals": "^29.7.0",
-    "@types/jest": "^29.5.12",
+    "@supabase/supabase-js": "^2.44.2",
     "@types/node": "20.14.2",
     "dotenv": "^16.4.5",
-    "ts-jest": "^29.1.5",
-    "typescript": "5.4.5",
-    "@supabase/supabase-js": "^2.44.2"
-  },
-  "jest": {
-    "preset": "ts-jest",
-    "testEnvironment": "node",
-    "moduleFileExtensions": [
-      "ts",
-      "tsx",
-      "js",
-      "jsx"
-    ],
-    "roots": [
-      "<rootDir>/src",
-      "<rootDir>/__tests__"
-    ],
-    "transform": {
-      "^.+\\\\.tsx?$": "ts-jest"
-    },
-    "testRegex": "(/__tests__/.*|\\.(test|spec))\\.(ts|tsx)$",
-    "moduleDirectories": [
-      "node_modules",
-      "src"
-    ],
-    "collectCoverage": true,
-    "coverageDirectory": "coverage",
-    "coverageReporters": [
-      "text",
-      "lcov"
-    ]
+    "tsx": "^4.16.2",
+    "typescript": "5.4.5"
   }
 }