@spider-cloud/spider-client 0.1.24 → 0.1.26

package/dist/config.d.ts

@@ -297,9 +297,9 @@ export interface SpiderParams {
      */
     sitemap?: boolean;
     /**
-     * Get page insights to determine information like request duration, accessibility, and other web vitals. Requires the `metadata` parameter to be set to `true`.
+     * Specifies whether to only use the sitemap links.
      */
-    page_insights?: boolean;
+    sitemap_only?: boolean;
     /**
      * External domains to include the crawl.
      */

package/dist/utils/process-chunk.d.ts

@@ -1,2 +1,2 @@
 import type { SpiderCoreResponse } from "../config";
-export declare const processChunk: (chunk: string, cb: (r: SpiderCoreResponse) => void) => boolean;
+export declare const createJsonLineProcessor: (cb: (r: SpiderCoreResponse) => void) => (chunk: Buffer | string) => void;

package/dist/utils/process-chunk.js

@@ -1,13 +1,21 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.processChunk = void 0;
-const processChunk = (chunk, cb) => {
-    try {
-        cb(chunk ? JSON.parse(chunk.trim()) : null);
-        return true;
-    }
-    catch (_error) {
-        return false;
-    }
+exports.createJsonLineProcessor = void 0;
+const createJsonLineProcessor = (cb) => {
+    let buffer = "";
+    return (chunk) => {
+        buffer += chunk.toString();
+        let boundary;
+        while ((boundary = buffer.indexOf("\n")) !== -1) {
+            const line = buffer.slice(0, boundary);
+            buffer = buffer.slice(boundary + 1);
+            if (line.trim()) {
+                try {
+                    cb(JSON.parse(line));
+                }
+                catch (_error) { }
+            }
+        }
+    };
 };
-exports.processChunk = processChunk;
+exports.createJsonLineProcessor = createJsonLineProcessor;

package/dist/utils/stream-reader.js

@@ -2,12 +2,13 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.streamReader = void 0;
 const process_chunk_1 = require("./process-chunk");
-// stream the response via callbacks.
+// Stream the response via callbacks.
 const streamReader = async (res, cb) => {
     var _a;
     if (res.ok) {
         const reader = (_a = res.body) === null || _a === void 0 ? void 0 : _a.getReader();
         const decoder = new TextDecoder();
+        const processChunk = (0, process_chunk_1.createJsonLineProcessor)(cb);
         if (reader) {
             while (true) {
                 const { done, value } = await reader.read();
@@ -15,8 +16,9 @@ const streamReader = async (res, cb) => {
                     break;
                 }
                 const chunk = decoder.decode(value, { stream: true });
-                (0, process_chunk_1.processChunk)(chunk, cb);
+                processChunk(chunk);
             }
+            processChunk(decoder.decode(new Uint8Array(), { stream: false }));
         }
     }
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spider-cloud/spider-client",
-  "version": "0.1.24",
+  "version": "0.1.26",
   "description": "Isomorphic Javascript SDK for Spider Cloud services",
   "scripts": {
     "test": "node --import tsx  --test __tests__/*test.ts",
@@ -20,13 +20,13 @@
     "api",
     "llm scraping"
   ],
-  "author": "Jeff Mendez<jeff@a11ywatch.com>",
+  "author": "Jeff Mendez<jeff@spider.cloud>",
   "license": "MIT",
   "devDependencies": {
-    "@types/node": "22.10.1",
+    "@types/node": "22.10.7",
     "dotenv": "^16.4.7",
     "tsx": "^4.19.2",
-    "typescript": "5.7.2"
+    "typescript": "5.7.3"
   },
   "dependencies": {
     "exponential-backoff": "^3.1.1"