@spider-cloud/spider-client 0.0.20 → 0.0.22

package/README.md

@@ -95,8 +95,10 @@ spider
 
 - **`scrapeUrl(url, params)`**: Scrape data from a specified URL. Optional parameters can be passed to customize the scraping behavior.
 - **`crawlUrl(url, params, stream)`**: Begin crawling from a specific URL with optional parameters for customization and an optional streaming response.
+- **`search(q, params)`**: Perform a search and gather a list of websites to start crawling and collect resources.
 - **`links(url, params)`**: Retrieve all links from the specified URL with optional parameters.
 - **`screenshot(url, params)`**: Take a screenshot of the specified URL.
+- **`transform(data, params)`**: Perform a fast HTML transformation to markdown or text.
 - **`extractContacts(url, params)`**: Extract contact information from the specified URL.
 - **`label(url, params)`**: Apply labeling to data extracted from the specified URL.
 - **`getCrawlState(url, params)`**: Check the website crawl state.

package/dist/client.d.ts

@@ -9,7 +9,6 @@ export interface SpiderConfig {
  */
 export declare class Spider {
     private apiKey?;
-    private dataEndPoint;
     /**
      * Create an instance of Spider.
      * @param {string | null} apiKey - The API key used to authenticate to the Spider API. If null, attempts to source from environment variables.
@@ -30,6 +29,12 @@ export declare class Spider {
      * @returns {Promise<any>} The data returned from the endpoint in JSON format.
      */
     private _apiGet;
+    /**
+     * Internal method to handle DELETE requests.
+     * @param {string} endpoint - The API endpoint from which data should be retrieved.
+     * @returns {Promise<any>} The data returned from the endpoint in JSON format.
+     */
+    private _apiDelete;
     /**
      * Scrapes data from a specified URL.
      * @param {string} url - The URL to scrape.
@@ -59,6 +64,23 @@ export declare class Spider {
      * @returns {Promise<any>} The screenshot data.
      */
     screenshot(url: string, params?: {}): Promise<any>;
+    /**
+     *  Perform a search and gather a list of websites to start crawling and collect resources.
+     * @param {string} search - The search query.
+     * @param {object} [params={}] - Configuration parameters for the search.
+     * @returns {Promise<any>} The result of the crawl, either structured data or a Response object if streaming.
+     */
+    search(q: string, params?: {}): Promise<any>;
+    /**
+     *  Transform HTML to Markdown or text. You can send up to 10MB of data at once.
+     * @param {object} data - The data to trasnform, a list of objects with the key 'html' and optional 'url' key for readability.
+     * @param {object} [params={}] - Configuration parameters for the transformation.
+     * @returns {Promise<any>} The transformation result.
+     */
+    transform(data: {
+        html: string;
+        url?: string;
+    }[], params?: {}): Promise<any>;
     /**
      * Extracts contact information from the specified URL.
      * @param {string} url - The URL from which to extract contacts.
@@ -106,7 +128,6 @@ export declare class Spider {
      * @returns {Promise<any>} The response from the server.
      */
     deleteData(table: string, params: object): Promise<any>;
-    private _apiDataPost;
     /**
      * Prepares common headers for each API request.
      * @returns {HeadersInit} A headers object for fetch requests.

package/dist/client.js

@@ -12,7 +12,6 @@ class Spider {
      */
     constructor(props) {
         var _a;
-        this.dataEndPoint = "https://api.spider.cloud:3280";
         this.apiKey = (props === null || props === void 0 ? void 0 : props.apiKey) || ((_a = process === null || process === void 0 ? void 0 : process.env) === null || _a === void 0 ? void 0 : _a.SPIDER_API_KEY);
         if (!this.apiKey) {
             throw new Error("No API key provided");
@@ -60,6 +59,24 @@ class Spider {
             this.handleError(response, `get from ${endpoint}`);
         }
     }
+    /**
+     * Internal method to handle DELETE requests.
+     * @param {string} endpoint - The API endpoint from which data should be retrieved.
+     * @returns {Promise<any>} The data returned from the endpoint in JSON format.
+     */
+    async _apiDelete(endpoint) {
+        const headers = this.prepareHeaders();
+        const response = await fetch(`https://api.spider.cloud/v1/${endpoint}`, {
+            method: "DELETE",
+            headers,
+        });
+        if (response.ok) {
+            return response.json();
+        }
+        else {
+            this.handleError(response, `get from ${endpoint}`);
+        }
+    }
     /**
      * Scrapes data from a specified URL.
      * @param {string} url - The URL to scrape.
@@ -97,6 +114,24 @@ class Spider {
     async screenshot(url, params = {}) {
         return this._apiPost("screenshot", { url: url, ...params });
     }
+    /**
+     *  Perform a search and gather a list of websites to start crawling and collect resources.
+     * @param {string} search - The search query.
+     * @param {object} [params={}] - Configuration parameters for the search.
+     * @returns {Promise<any>} The result of the crawl, either structured data or a Response object if streaming.
+     */
+    async search(q, params = {}) {
+        return this._apiPost("search", { search: q, ...params });
+    }
+    /**
+     *  Transform HTML to Markdown or text. You can send up to 10MB of data at once.
+     * @param {object} data - The data to trasnform, a list of objects with the key 'html' and optional 'url' key for readability.
+     * @param {object} [params={}] - Configuration parameters for the transformation.
+     * @returns {Promise<any>} The transformation result.
+     */
+    async transform(data, params = {}) {
+        return this._apiPost("transform", { data, ...params });
+    }
     /**
      * Extracts contact information from the specified URL.
      * @param {string} url - The URL from which to extract contacts.
@@ -138,8 +173,7 @@ class Spider {
      * @returns {Promise<any>} The response from the server.
      */
     async postData(table, data) {
-        const endpoint = `/data/${table}`;
-        return this._apiDataPost(endpoint, data);
+        return this._apiPost(`data/${table}`, data);
     }
     /**
      * Send a GET request to retrieve data from a specified table.
@@ -148,18 +182,7 @@ class Spider {
      * @returns {Promise<any>} The response from the server.
      */
     async getData(table, params) {
-        const endpoint = `/data/${table}?${new URLSearchParams(params).toString()}`;
-        const headers = this.prepareHeaders();
-        const response = await fetch(`${this.dataEndPoint}${endpoint}`, {
-            method: "GET",
-            headers: headers,
-        });
-        if (response.ok) {
-            return response.json();
-        }
-        else {
-            this.handleError(response, `get data from ${table}`);
-        }
+        return this._apiGet(`data/${table}?${new URLSearchParams(params).toString()}`);
     }
     /**
      * Send a DELETE request to remove data from a specified table.
@@ -168,33 +191,7 @@ class Spider {
      * @returns {Promise<any>} The response from the server.
      */
     async deleteData(table, params) {
-        const endpoint = `/data/${table}?${new URLSearchParams(params).toString()}`;
-        const headers = this.prepareHeaders();
-        const response = await fetch(`${this.dataEndPoint}${endpoint}`, {
-            method: "DELETE",
-            headers,
-        });
-        if (response.ok) {
-            return response.json();
-        }
-        else {
-            this.handleError(response, `delete data from ${table}`);
-        }
-    }
-    // Create wrapper methods for external API access:
-    async _apiDataPost(endpoint, data) {
-        const headers = this.prepareHeaders();
-        const response = await fetch(`${this.dataEndPoint}${endpoint}`, {
-            method: "POST",
-            headers,
-            body: JSON.stringify(data),
-        });
-        if (response.ok) {
-            return response.json();
-        }
-        else {
-            this.handleError(response, `post to ${endpoint}`);
-        }
+        return this._apiDelete(`data/${table}?${new URLSearchParams(params).toString()}`);
     }
     /**
      * Prepares common headers for each API request.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spider-cloud/spider-client",
-  "version": "0.0.20",
+  "version": "0.0.22",
   "description": "A Javascript SDK for Spider Cloud services",
   "scripts": {
     "test": "jest",