@spider-cloud/spider-client 0.1.75 → 0.1.79

package/README.md

@@ -85,51 +85,6 @@ const streamCallback = (data) => {
 app.crawlUrl(url, crawlParams, stream, streamCallback);
 ```
 
-### Data Operations
-
-The Spider client can interact with specific data tables to create, retrieve, and delete data.
-
-#### Retrieve Data from a Table
-
-To fetch data from a specified table by applying query parameters, use the `getData` method. Provide the table name and an object containing query parameters:
-
-```javascript
-const tableName = "pages";
-const queryParams = { limit: 20 };
-spider
-  .getData(tableName, queryParams)
-  .then((response) => console.log(response))
-  .catch((error) => console.error(error));
-```
-
-This example retrieves data from the 'pages' table, limiting the results to 20 entries.
-
-#### Delete Data from a Table
-
-To delete data from a specified table based on certain conditions, use the `deleteData` method. Provide the table name and an object specifying the conditions for deletion:
-
-```javascript
-const tableName = "websites";
-const deleteParams = { domain: "www.example.com" };
-spider
-  .deleteData(tableName, deleteParams)
-  .then((response) => console.log(response))
-  .catch((error) => console.error(error));
-```
-
-#### Download storage data
-
-To download stored data like raw HTML or markdown use the `createSignedUrl` method. Provide the website name and an object containing query parameters:
-
-```javascript
-const websiteName = "spider.cloud";
-const queryParams = { limit: 20, page: 0 };
-spider
-  .createSignedUrl(websiteName, queryParams)
-  .then((response) => console.log(response))
-  .catch((error) => console.error(error));
-```
-
 ### Available Methods
 
 - **`scrapeUrl(url, params)`**: Scrape data from a specified URL. Optional parameters can be passed to customize the scraping behavior.
@@ -139,9 +94,6 @@ spider
 - **`screenshot(url, params)`**: Take a screenshot of the specified URL.
 - **`transform(data, params)`**: Perform a fast HTML transformation to markdown or text.
 - **`getCredits()`**: Retrieve account's remaining credits.
-- **`getData(table, params)`**: Retrieve data records from the DB.
-- **`deleteData(table, params)`**: Delete records from the DB.
-- **`createSignedUrl(domain, params)`**: Download the records from the DB.
 
 ## Error Handling
 

package/dist/client.d.ts

@@ -1,4 +1,4 @@
-import { ChunkCallbackFunction, Collection, QueryRequest, SpiderCoreResponse, SpiderParams, SearchRequestParams, RequestParamsTransform } from "./config";
+import { ChunkCallbackFunction, Collection, SpiderCoreResponse, SpiderParams, SearchRequestParams, RequestParamsTransform } from "./config";
 /**
  * Generic params for core request.
  */
@@ -34,12 +34,6 @@ export declare class Spider {
      * @returns {Promise<any>} The data returned from the endpoint in JSON format.
      */
     private _apiGet;
-    /**
-     * Internal method to handle DELETE requests.
-     * @param {string} endpoint - The API endpoint from which data should be retrieved.
-     * @returns {Promise<any>} The data returned from the endpoint in JSON format.
-     */
-    private _apiDelete;
     /**
      * Scrapes data from a specified URL.
      * @param {string} url - The URL to scrape.
@@ -89,21 +83,6 @@ export declare class Spider {
         html: string;
         url?: string;
     }[], params?: RequestParamsTransform): Promise<any>;
-    /**
-     * Create a signed url to download files from the storage.
-     * @param {string} [domain] - The domain for the user's storage. If not provided, downloads all files.
-     * @param {Object} [options] - The download options.
-     * @param {boolean} [raw] - Return the raw response.
-  
-     * @returns {Promise<Response>} The response containing the file stream.
-     */
-    createSignedUrl(url?: string, options?: {
-        page?: number;
-        limit?: number;
-        expiresIn?: number;
-        domain?: string;
-        pathname?: string;
-    }): Promise<any>;
     /**
      * Retrieves the number of credits available on the account.
      * @returns {Promise<any>} The current credit balance.
@@ -116,32 +95,6 @@ export declare class Spider {
      * @returns {Promise<any>} The response from the server.
      */
     postData(collection: Collection, data: GenericParams | Record<string, any>): Promise<any>;
-    /**
-     * Send a GET request to retrieve data from a specified table.
-     * @param {Collection} table - The table name in the database.
-     * @param {object} params - The query parameters for data retrieval.
-     * @returns {Promise<any>} The response from the server.
-     */
-    getData(collections: Collection, params: GenericParams | Record<string, any>): Promise<any>;
-    /**
-     * Download a record. The url is the path of the storage hash returned and not the exact website url.
-     * @param {QueryRequest} params - The query parameters for data retrieval.
-     * @returns {Promise<any>} The download response from the server.
-     */
-    download(query: QueryRequest, output?: "text" | "blob"): Promise<any>;
-    /**
-     * Perform a query to get a document.
-     * @param {QueryRequest} params - The query parameters for data retrieval.
-     * @returns {Promise<any>} The response from the server.
-     */
-    query(query: QueryRequest): Promise<any>;
-    /**
-     * Send a DELETE request to remove data from a specified table.
-     * @param {Collection} table - The table name in the database.
-     * @param {object} params - Parameters to identify records to delete.
-     * @returns {Promise<any>} The response from the server.
-     */
-    deleteData(collection: Collection, params: GenericParams | Record<string, any>): Promise<any>;
     /**
      * Prepares common headers for each API request.
      * @returns {HeadersInit} A headers object for fetch requests.

package/dist/client.js

@@ -67,27 +67,6 @@ class Spider {
             this.handleError(response, `get from ${endpoint}`);
         }
     }
-    /**
-     * Internal method to handle DELETE requests.
-     * @param {string} endpoint - The API endpoint from which data should be retrieved.
-     * @returns {Promise<any>} The data returned from the endpoint in JSON format.
-     */
-    async _apiDelete(endpoint) {
-        const headers = this.prepareHeaders;
-        const response = await (0, exponential_backoff_1.backOff)(() => fetch(`${config_1.APISchema["url"]}/${config_1.ApiVersion.V1}/${endpoint}`, {
-            method: "DELETE",
-            headers,
-            body: JSON.stringify({}),
-        }), {
-            numOfAttempts: 5,
-        });
-        if (response.ok) {
-            return response;
-        }
-        else {
-            return this.handleError(response, `delete from ${endpoint}`);
-        }
-    }
     /**
      * Scrapes data from a specified URL.
      * @param {string} url - The URL to scrape.
@@ -162,37 +141,6 @@ class Spider {
                 : data,
         });
     }
-    /**
-     * Create a signed url to download files from the storage.
-     * @param {string} [domain] - The domain for the user's storage. If not provided, downloads all files.
-     * @param {Object} [options] - The download options.
-     * @param {boolean} [raw] - Return the raw response.
-  
-     * @returns {Promise<Response>} The response containing the file stream.
-     */
-    async createSignedUrl(url, options) {
-        const { page, limit, expiresIn, domain, pathname } = options !== null && options !== void 0 ? options : {};
-        const params = new URLSearchParams({
-            ...(url && { url }),
-            ...(domain && { domain }),
-            ...(pathname && { pathname }),
-            ...(page && { page: page.toString() }),
-            ...(limit && { limit: limit.toString() }),
-            ...(expiresIn && { expiresIn: expiresIn.toString() }),
-        });
-        const endpoint = `${config_1.APISchema["url"]}/${config_1.APIRoutes.DataSignUrl}?${params.toString()}`;
-        const headers = this.prepareHeaders;
-        const response = await fetch(endpoint, {
-            method: "GET",
-            headers,
-        });
-        if (response.ok) {
-            return await response.json();
-        }
-        else {
-            this.handleError(response, `Failed to sign files`);
-        }
-    }
     /**
      * Retrieves the number of credits available on the account.
      * @returns {Promise<any>} The current credit balance.
@@ -209,54 +157,6 @@ class Spider {
     async postData(collection, data) {
         return this._apiPost(`${config_1.APIRoutes.Data}/${collection}`, data);
     }
-    /**
-     * Send a GET request to retrieve data from a specified table.
-     * @param {Collection} table - The table name in the database.
-     * @param {object} params - The query parameters for data retrieval.
-     * @returns {Promise<any>} The response from the server.
-     */
-    async getData(collections, params) {
-        return this._apiGet(`${config_1.APIRoutes.Data}/${collections}?${new URLSearchParams(params).toString()}`);
-    }
-    /**
-     * Download a record. The url is the path of the storage hash returned and not the exact website url.
-     * @param {QueryRequest} params - The query parameters for data retrieval.
-     * @returns {Promise<any>} The download response from the server.
-     */
-    async download(query, output) {
-        const headers = this.prepareHeaders;
-        const endpoint = `${config_1.APIRoutes.DataDownload}?${new URLSearchParams(query).toString()}`;
-        const response = await fetch(`${config_1.APISchema["url"]}/${config_1.ApiVersion.V1}/${endpoint}`, {
-            method: "GET",
-            headers,
-        });
-        if (response.ok) {
-            if (output === "text") {
-                return await response.text();
-            }
-            return await response.blob();
-        }
-        else {
-            this.handleError(response, `get from ${endpoint}`);
-        }
-    }
-    /**
-     * Perform a query to get a document.
-     * @param {QueryRequest} params - The query parameters for data retrieval.
-     * @returns {Promise<any>} The response from the server.
-     */
-    async query(query) {
-        return this._apiGet(`${config_1.APIRoutes.DataQuery}?${new URLSearchParams(query).toString()}`);
-    }
-    /**
-     * Send a DELETE request to remove data from a specified table.
-     * @param {Collection} table - The table name in the database.
-     * @param {object} params - Parameters to identify records to delete.
-     * @returns {Promise<any>} The response from the server.
-     */
-    async deleteData(collection, params) {
-        return this._apiDelete(`${config_1.APIRoutes.Data}/${collection}?${new URLSearchParams(params).toString()}`);
-    }
     /**
      * Prepares common headers for each API request.
      * @returns {HeadersInit} A headers object for fetch requests.

package/dist/config.d.ts

@@ -135,84 +135,72 @@ type CSSExtractionMap = {
     [path: string]: CSSSelector[];
 };
 export type Evaluate = {
-    type: "Evaluate";
     /** Rust: Evaluate(String) */
     code: string;
 };
 export type Click = {
-    type: "Click";
     /** Rust: Click(String) */
     selector: string;
 };
 export type ClickAll = {
-    type: "ClickAll";
     /** Rust: ClickAll(String) */
     selector: string;
 };
-export type ClickAllClickable = {
-    type: "ClickAllClickable";
+export type ClickAllClickable = {};
+export type ClickPoint = {
+    x: number;
+    y: number;
 };
 export type Wait = {
-    type: "Wait";
     /** Rust: u64 (milliseconds) */
     ms: number;
 };
-export type WaitForNavigation = {
-    type: "WaitForNavigation";
-};
+export type WaitForNavigation = {};
 export type WaitForDom = {
-    type: "WaitForDom";
     /** Rust: Option<String> */
     selector?: string | null;
     /** Rust: u32 (milliseconds) */
     timeout: number;
 };
 export type WaitFor = {
-    type: "WaitFor";
-    /** Rust: String */
     selector: string;
 };
 export type WaitForWithTimeout = {
-    type: "WaitForWithTimeout";
     selector: string;
     /** Rust: u64 (milliseconds) */
     timeout: number;
 };
 export type WaitForAndClick = {
-    type: "WaitForAndClick";
     selector: string;
 };
 export type ScrollX = {
-    type: "ScrollX";
     /** Rust: i32 (pixels) */
     dx: number;
 };
 export type ScrollY = {
-    type: "ScrollY";
     /** Rust: i32 (pixels) */
     dy: number;
 };
 export type Fill = {
-    type: "Fill";
     selector: string;
     value: string;
 };
+export type Type = {
+    modifier: number;
+    value: string;
+};
 export type InfiniteScroll = {
-    type: "InfiniteScroll";
     /** Rust: u32 (pixels/step or count—match your semantics) */
     step_px: number;
 };
 export type Screenshot = {
-    type: "Screenshot";
     /** Keep snake_case to match Rust JSON if interop is needed */
     full_page: boolean;
     omit_background: boolean;
     output: string;
 };
-export type ValidateChain = {
-    type: "ValidateChain";
-};
-export type WebAutomation = Evaluate | Click | ClickAll | ClickAllClickable | Wait | WaitForNavigation | WaitForDom | WaitFor | WaitForWithTimeout | WaitForAndClick | ScrollX | ScrollY | Fill | InfiniteScroll | Screenshot | ValidateChain;
+export type ValidateChain = {};
+export type WebAutomation = Evaluate | Click | ClickAll | ClickAllClickable | ClickPoint | Wait | WaitForNavigation | WaitForDom | WaitFor | WaitForWithTimeout | WaitForAndClick | ScrollX | ScrollY | Fill | Type | InfiniteScroll | Screenshot | ValidateChain;
 export type ReturnFormat = "markdown" | "commonmark" | "raw" | "screenshot" | "text" | "html2text" | "bytes" | "xml" | "empty";
 export type WebAutomationMap = Record<string, WebAutomation[]>;
 export type ExecutionScriptsMap = Record<string, string>;
@@ -299,10 +287,6 @@ export interface SpiderParams {
      * The headers to be used for the request.
      */
     headers?: Headers;
-    /**
-     * Specifies whether anti-bot measures should be used.
-     */
-    anti_bot?: boolean;
     /**
      * Specifies whether to include metadata in the response.
      */
@@ -554,10 +538,6 @@ export declare enum APIRoutes {
     Search = "search",
     Transform = "transform",
     Data = "data",
-    DataCrawlState = "data/crawl_state",
-    DataSignUrl = "data/sign-url",
-    DataDownload = "data/download",
-    DataQuery = "data/query",
     DataCredits = "data/credits"
 }
 export declare const APISchema: {

package/dist/config.js

@@ -42,14 +42,6 @@ var APIRoutes;
     APIRoutes["Transform"] = "transform";
     // Dynamic collection routes.
     APIRoutes["Data"] = "data";
-    // The last crawl state of a website.
-    APIRoutes["DataCrawlState"] = "data/crawl_state";
-    // Sign a file from storage based on the exact url path of the storage or domain - pathname.
-    APIRoutes["DataSignUrl"] = "data/sign-url";
-    // Download a file from storage based on the exact url path of the storage or domain - pathname.
-    APIRoutes["DataDownload"] = "data/download";
-    // Perform a query on the global database to grab content without crawling if available.
-    APIRoutes["DataQuery"] = "data/query";
     // Get the credits remaining for an account.
     APIRoutes["DataCredits"] = "data/credits";
 })(APIRoutes || (exports.APIRoutes = APIRoutes = {}));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spider-cloud/spider-client",
-  "version": "0.1.75",
+  "version": "0.1.79",
   "description": "Isomorphic Javascript SDK for Spider Cloud services",
   "scripts": {
     "test": "node --import tsx  --test __tests__/*test.ts",