@spider-cloud/spider-client 0.0.58 → 0.0.59

package/README.md

@@ -176,4 +176,4 @@ Contributions are always welcome! Feel free to open an issue or submit a pull re
 
 ## License
 
-The Spider Cloud JavaScript SDK is open-source and released under the [MIT License](https://opensource.org/licenses/MIT).
+The Spider Cloud JavaScript SDK is open-source and released under the [MIT License](https://opensource.org/licenses/MIT).

package/dist/client.d.ts

@@ -1,4 +1,5 @@
-import { ChunkCallbackFunction, QueryRequest, SpiderCoreResponse, SpiderParams } from "./config";
+import { ChunkCallbackFunction, Collection, QueryRequest, SpiderCoreResponse, SpiderParams } from "./config";
+export declare const BASE_API_URL = "https://api.spider.cloud";
 /**
  * Generic params for core request.
  */
@@ -124,11 +125,13 @@ export declare class Spider {
   
      * @returns {Promise<Response>} The response containing the file stream.
      */
-    createSignedUrl(domain?: string, options?: {
+    createSignedUrl(url?: string, options?: {
         page?: number;
         limit?: number;
         expiresIn?: number;
-    }, raw?: boolean): Promise<Response>;
+        domain?: string;
+        pathname?: string;
+    }): Promise<any>;
     /**
      * Retrieves the number of credits available on the account.
      * @returns {Promise<any>} The current credit balance.
@@ -143,11 +146,17 @@ export declare class Spider {
     postData(table: string, data: GenericParams | Record<string, any>): Promise<any>;
     /**
      * Send a GET request to retrieve data from a specified table.
-     * @param {string} table - The table name in the database.
+     * @param {Collection} table - The table name in the database.
      * @param {object} params - The query parameters for data retrieval.
      * @returns {Promise<any>} The response from the server.
      */
-    getData(table: string, params: GenericParams | Record<string, any>): Promise<any>;
+    getData(collections: Collection, params: GenericParams | Record<string, any>): Promise<any>;
+    /**
+     * Download a record. The url is the path of the storage hash returned and not the exact website url.
+     * @param {QueryRequest} params - The query parameters for data retrieval.
+     * @returns {Promise<any>} The download response from the server.
+     */
+    download(query: QueryRequest, output?: "text" | "blob"): Promise<any>;
     /**
      * Perform a query to get a document.
      * @param {QueryRequest} params - The query parameters for data retrieval.
@@ -156,11 +165,11 @@ export declare class Spider {
     query(query: QueryRequest): Promise<any>;
     /**
      * Send a DELETE request to remove data from a specified table.
-     * @param {string} table - The table name in the database.
+     * @param {Collection} table - The table name in the database.
      * @param {object} params - Parameters to identify records to delete.
      * @returns {Promise<any>} The response from the server.
      */
-    deleteData(table: string, params: GenericParams | Record<string, any>): Promise<any>;
+    deleteData(collection: Collection, params: GenericParams | Record<string, any>): Promise<any>;
     /**
      * Prepares common headers for each API request.
      * @returns {HeadersInit} A headers object for fetch requests.

package/dist/client.js

@@ -1,9 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Spider = void 0;
+exports.Spider = exports.BASE_API_URL = void 0;
 const package_json_1 = require("../package.json");
 const supabase_1 = require("./supabase");
 const stream_reader_1 = require("./utils/stream-reader");
+exports.BASE_API_URL = "https://api.spider.cloud";
 /**
  * A class to interact with the Spider API.
  */
@@ -41,7 +42,7 @@ class Spider {
      */
     async _apiPost(endpoint, data, stream, jsonl) {
         const headers = jsonl ? this.prepareHeadersJsonL : this.prepareHeaders;
-        const response = await fetch(`https://api.spider.cloud/v1/${endpoint}`, {
+        const response = await fetch(`${exports.BASE_API_URL}/v1/${endpoint}`, {
             method: "POST",
             headers: headers,
             body: JSON.stringify(data),
@@ -63,7 +64,7 @@ class Spider {
      */
     async _apiGet(endpoint) {
         const headers = this.prepareHeaders;
-        const response = await fetch(`https://api.spider.cloud/v1/${endpoint}`, {
+        const response = await fetch(`${exports.BASE_API_URL}/v1/${endpoint}`, {
             method: "GET",
             headers: headers,
         });
@@ -81,7 +82,7 @@ class Spider {
      */
     async _apiDelete(endpoint) {
         const headers = this.prepareHeaders;
-        const response = await fetch(`https://api.spider.cloud/v1/${endpoint}`, {
+        const response = await fetch(`${exports.BASE_API_URL}/v1/${endpoint}`, {
             method: "DELETE",
             headers,
         });
@@ -188,36 +189,35 @@ class Spider {
   
      * @returns {Promise<Response>} The response containing the file stream.
      */
-    async createSignedUrl(domain, options, raw) {
-        const { page, limit, expiresIn } = options !== null && options !== void 0 ? options : {};
+    async createSignedUrl(url, options) {
+        const { page, limit, expiresIn, domain, pathname } = options !== null && options !== void 0 ? options : {};
         const params = new URLSearchParams({
+            ...(url && { url }),
             ...(domain && { domain }),
+            ...(pathname && { pathname }),
             ...(page && { page: page.toString() }),
             ...(limit && { limit: limit.toString() }),
             ...(expiresIn && { expiresIn: expiresIn.toString() }),
         });
-        const endpoint = `https://api.spider.cloud/v1/data/storage?${params.toString()}`;
+        const endpoint = `${exports.BASE_API_URL}/data/sign-url?${params.toString()}`;
         const headers = this.prepareHeaders;
         const response = await fetch(endpoint, {
             method: "GET",
             headers,
         });
-        if (!raw) {
-            if (response.ok) {
-                return response.json();
-            }
-            else {
-                this.handleError(response, `Failed to download files`);
-            }
+        if (response.ok) {
+            return await response.json();
+        }
+        else {
+            this.handleError(response, `Failed to download files`);
         }
-        return response;
     }
     /**
      * Retrieves the number of credits available on the account.
      * @returns {Promise<any>} The current credit balance.
      */
     async getCredits() {
-        return this._apiGet("credits");
+        return this._apiGet("data/credits");
     }
     /**
      * Send a POST request to insert data into a specified table.
@@ -230,12 +230,34 @@ class Spider {
     }
     /**
      * Send a GET request to retrieve data from a specified table.
-     * @param {string} table - The table name in the database.
+     * @param {Collection} table - The table name in the database.
      * @param {object} params - The query parameters for data retrieval.
      * @returns {Promise<any>} The response from the server.
      */
-    async getData(table, params) {
-        return this._apiGet(`data/${table}?${new URLSearchParams(params).toString()}`);
+    async getData(collections, params) {
+        return this._apiGet(`data/${collections}?${new URLSearchParams(params).toString()}`);
+    }
+    /**
+     * Download a record. The url is the path of the storage hash returned and not the exact website url.
+     * @param {QueryRequest} params - The query parameters for data retrieval.
+     * @returns {Promise<any>} The download response from the server.
+     */
+    async download(query, output) {
+        const headers = this.prepareHeaders;
+        const endpoint = `data/download?${new URLSearchParams(query).toString()}`;
+        const response = await fetch(`${exports.BASE_API_URL}/v1/${endpoint}`, {
+            method: "GET",
+            headers,
+        });
+        if (response.ok) {
+            if (output === "text") {
+                return await response.text();
+            }
+            return await response.blob();
+        }
+        else {
+            this.handleError(response, `get from ${endpoint}`);
+        }
     }
     /**
      * Perform a query to get a document.
@@ -247,12 +269,12 @@ class Spider {
     }
     /**
      * Send a DELETE request to remove data from a specified table.
-     * @param {string} table - The table name in the database.
+     * @param {Collection} table - The table name in the database.
      * @param {object} params - Parameters to identify records to delete.
      * @returns {Promise<any>} The response from the server.
      */
-    async deleteData(table, params) {
-        return this._apiDelete(`data/${table}?${new URLSearchParams(params).toString()}`);
+    async deleteData(collection, params) {
+        return this._apiDelete(`data/${collection}?${new URLSearchParams(params).toString()}`);
     }
     /**
      * Prepares common headers for each API request.

package/dist/config.d.ts

@@ -260,4 +260,16 @@ export type SpiderCoreResponse = {
     url?: string;
 };
 export type ChunkCallbackFunction = (data: SpiderCoreResponse) => void;
+export declare enum Collection {
+    Websites = "websites",
+    Pages = "pages",
+    PagesMetadata = "pages_metadata",
+    Contacts = "contacts",
+    CrawlState = "crawl_state",
+    CrawlLogs = "crawl_logs",
+    Profiles = "profiles",
+    Credits = "credits",
+    Webhooks = "webhooks",
+    APIKeys = "api_keys"
+}
 export {};

package/dist/config.js

@@ -1,2 +1,18 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.Collection = void 0;
+// records that you can query
+var Collection;
+(function (Collection) {
+    Collection["Websites"] = "websites";
+    Collection["Pages"] = "pages";
+    Collection["PagesMetadata"] = "pages_metadata";
+    // Leads
+    Collection["Contacts"] = "contacts";
+    Collection["CrawlState"] = "crawl_state";
+    Collection["CrawlLogs"] = "crawl_logs";
+    Collection["Profiles"] = "profiles";
+    Collection["Credits"] = "credits";
+    Collection["Webhooks"] = "webhooks";
+    Collection["APIKeys"] = "api_keys";
+})(Collection || (exports.Collection = Collection = {}));

package/dist/index.d.ts

@@ -1,2 +1,3 @@
 export { Spider } from "./client";
 export type { SpiderParams, Budget, Viewport, QueryRequest } from "./config";
+export { Collection } from "./config";

package/dist/index.js

@@ -1,5 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Spider = void 0;
+exports.Collection = exports.Spider = void 0;
 var client_1 = require("./client");
 Object.defineProperty(exports, "Spider", { enumerable: true, get: function () { return client_1.Spider; } });
+var config_1 = require("./config");
+Object.defineProperty(exports, "Collection", { enumerable: true, get: function () { return config_1.Collection; } });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spider-cloud/spider-client",
-  "version": "0.0.58",
+  "version": "0.0.59",
   "description": "Isomorphic Javascript SDK for Spider Cloud services",
   "scripts": {
     "test": "node --import tsx  --test __tests__/*test.ts",