@spider-cloud/spider-client 0.0.58 → 0.0.60

package/README.md

@@ -176,4 +176,4 @@ Contributions are always welcome! Feel free to open an issue or submit a pull re
 
 ## License
 
-The Spider Cloud JavaScript SDK is open-source and released under the [MIT License](https://opensource.org/licenses/MIT).
+The Spider Cloud JavaScript SDK is open-source and released under the [MIT License](https://opensource.org/licenses/MIT).

package/dist/client.d.ts

@@ -1,4 +1,5 @@
-import { ChunkCallbackFunction, QueryRequest, SpiderCoreResponse, SpiderParams } from "./config";
+import { ChunkCallbackFunction, Collection, QueryRequest, SpiderCoreResponse, SpiderParams } from "./config";
+export declare const baseUrl: string;
 /**
  * Generic params for core request.
  */
@@ -96,7 +97,7 @@ export declare class Spider {
         url?: string;
     }[], params?: {}): Promise<any>;
     /**
-     * Extracts contact information from the specified URL.
+     * Extracts leads from a website.
      * @param {string} url - The URL from which to extract contacts.
      * @param {GenericParams} [params={}] - Configuration parameters for the extraction.
      * @returns {Promise<any>} The contact information extracted.
@@ -124,11 +125,13 @@ export declare class Spider {
   
      * @returns {Promise<Response>} The response containing the file stream.
      */
-    createSignedUrl(domain?: string, options?: {
+    createSignedUrl(url?: string, options?: {
         page?: number;
         limit?: number;
         expiresIn?: number;
-    }, raw?: boolean): Promise<Response>;
+        domain?: string;
+        pathname?: string;
+    }): Promise<any>;
     /**
      * Retrieves the number of credits available on the account.
      * @returns {Promise<any>} The current credit balance.
@@ -140,14 +143,20 @@ export declare class Spider {
      * @param {object} data - The data to be inserted.
      * @returns {Promise<any>} The response from the server.
      */
-    postData(table: string, data: GenericParams | Record<string, any>): Promise<any>;
+    postData(collection: Collection, data: GenericParams | Record<string, any>): Promise<any>;
     /**
      * Send a GET request to retrieve data from a specified table.
-     * @param {string} table - The table name in the database.
+     * @param {Collection} table - The table name in the database.
      * @param {object} params - The query parameters for data retrieval.
      * @returns {Promise<any>} The response from the server.
      */
-    getData(table: string, params: GenericParams | Record<string, any>): Promise<any>;
+    getData(collections: Collection, params: GenericParams | Record<string, any>): Promise<any>;
+    /**
+     * Download a record. The url is the path of the storage hash returned and not the exact website url.
+     * @param {QueryRequest} params - The query parameters for data retrieval.
+     * @returns {Promise<any>} The download response from the server.
+     */
+    download(query: QueryRequest, output?: "text" | "blob"): Promise<any>;
     /**
      * Perform a query to get a document.
      * @param {QueryRequest} params - The query parameters for data retrieval.
@@ -156,11 +165,11 @@ export declare class Spider {
     query(query: QueryRequest): Promise<any>;
     /**
      * Send a DELETE request to remove data from a specified table.
-     * @param {string} table - The table name in the database.
+     * @param {Collection} table - The table name in the database.
      * @param {object} params - Parameters to identify records to delete.
      * @returns {Promise<any>} The response from the server.
      */
-    deleteData(table: string, params: GenericParams | Record<string, any>): Promise<any>;
+    deleteData(collection: Collection, params: GenericParams | Record<string, any>): Promise<any>;
     /**
      * Prepares common headers for each API request.
      * @returns {HeadersInit} A headers object for fetch requests.

package/dist/client.js

@@ -1,9 +1,11 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Spider = void 0;
+exports.Spider = exports.baseUrl = void 0;
+const config_1 = require("./config");
 const package_json_1 = require("../package.json");
 const supabase_1 = require("./supabase");
 const stream_reader_1 = require("./utils/stream-reader");
+exports.baseUrl = config_1.APISchema["url"];
 /**
  * A class to interact with the Spider API.
  */
@@ -41,7 +43,7 @@ class Spider {
      */
     async _apiPost(endpoint, data, stream, jsonl) {
         const headers = jsonl ? this.prepareHeadersJsonL : this.prepareHeaders;
-        const response = await fetch(`https://api.spider.cloud/v1/${endpoint}`, {
+        const response = await fetch(`${exports.baseUrl}/v1/${endpoint}`, {
             method: "POST",
             headers: headers,
             body: JSON.stringify(data),
@@ -63,7 +65,7 @@ class Spider {
      */
     async _apiGet(endpoint) {
         const headers = this.prepareHeaders;
-        const response = await fetch(`https://api.spider.cloud/v1/${endpoint}`, {
+        const response = await fetch(`${exports.baseUrl}/v1/${endpoint}`, {
             method: "GET",
             headers: headers,
         });
@@ -81,7 +83,7 @@ class Spider {
      */
     async _apiDelete(endpoint) {
         const headers = this.prepareHeaders;
-        const response = await fetch(`https://api.spider.cloud/v1/${endpoint}`, {
+        const response = await fetch(`${exports.baseUrl}/v1/${endpoint}`, {
             method: "DELETE",
             headers,
         });
@@ -99,7 +101,7 @@ class Spider {
      * @returns {Promise<any>} The scraped data from the URL.
      */
     async scrapeUrl(url, params = {}) {
-        return this._apiPost("crawl", { url: url, limit: 1, ...params });
+        return this._apiPost(config_1.APIRoutes.Crawl, { url: url, limit: 1, ...params });
     }
     /**
      * Initiates a crawling job starting from the specified URL.
@@ -111,7 +113,7 @@ class Spider {
      */
     async crawlUrl(url, params = {}, stream = false, cb) {
         const jsonl = stream && cb;
-        const res = await this._apiPost("crawl", { url: url, ...params }, stream, !!jsonl);
+        const res = await this._apiPost(config_1.APIRoutes.Crawl, { url: url, ...params }, stream, !!jsonl);
         if (jsonl) {
             return await (0, stream_reader_1.streamReader)(res, cb);
         }
@@ -124,7 +126,7 @@ class Spider {
      * @returns {Promise<any>} A list of links extracted from the URL.
      */
     async links(url, params = {}) {
-        return this._apiPost("links", { url: url, ...params });
+        return this._apiPost(config_1.APIRoutes.Links, { url: url, ...params });
     }
     /**
      * Takes a screenshot of the website starting from this URL.
@@ -133,7 +135,7 @@ class Spider {
      * @returns {Promise<any>} The screenshot data.
      */
     async screenshot(url, params = {}) {
-        return this._apiPost("screenshot", { url: url, ...params });
+        return this._apiPost(config_1.APIRoutes.Screenshot, { url: url, ...params });
     }
     /**
      *  Perform a search and gather a list of websites to start crawling and collect resources.
@@ -142,7 +144,7 @@ class Spider {
      * @returns {Promise<any>} The result of the crawl, either structured data or a Response object if streaming.
      */
     async search(q, params = {}) {
-        return this._apiPost("search", { search: q, ...params });
+        return this._apiPost(config_1.APIRoutes.Search, { search: q, ...params });
     }
     /**
      *  Transform HTML to Markdown or text. You can send up to 10MB of data at once.
@@ -151,16 +153,19 @@ class Spider {
      * @returns {Promise<any>} The transformation result.
      */
     async transform(data, params = {}) {
-        return this._apiPost("transform", { data, ...params });
+        return this._apiPost(config_1.APIRoutes.Transform, { data, ...params });
     }
     /**
-     * Extracts contact information from the specified URL.
+     * Extracts leads from a website.
      * @param {string} url - The URL from which to extract contacts.
      * @param {GenericParams} [params={}] - Configuration parameters for the extraction.
      * @returns {Promise<any>} The contact information extracted.
      */
     async extractContacts(url, params = {}) {
-        return this._apiPost("pipeline/extract-contacts", { url: url, ...params });
+        return this._apiPost(config_1.APIRoutes.PiplineExtractLeads, {
+            url: url,
+            ...params,
+        });
     }
     /**
      * Applies labeling to data extracted from a specified URL.
@@ -169,7 +174,7 @@ class Spider {
      * @returns {Promise<any>} The labeled data.
      */
     async label(url, params = {}) {
-        return this._apiPost("pipeline/label", { url: url, ...params });
+        return this._apiPost(config_1.APIRoutes.PiplineLabel, { url: url, ...params });
     }
     /**
      * Check the crawl state of the website.
@@ -178,7 +183,7 @@ class Spider {
      * @returns {Promise<any>} The crawl state data.
      */
     async getCrawlState(url, params = {}) {
-        return this._apiPost("data/crawl_state", { url: url, ...params });
+        return this._apiPost(config_1.APIRoutes.DataCrawlState, { url: url, ...params });
     }
     /**
      * Create a signed url to download files from the storage.
@@ -188,36 +193,35 @@ class Spider {
   
      * @returns {Promise<Response>} The response containing the file stream.
      */
-    async createSignedUrl(domain, options, raw) {
-        const { page, limit, expiresIn } = options !== null && options !== void 0 ? options : {};
+    async createSignedUrl(url, options) {
+        const { page, limit, expiresIn, domain, pathname } = options !== null && options !== void 0 ? options : {};
         const params = new URLSearchParams({
+            ...(url && { url }),
             ...(domain && { domain }),
+            ...(pathname && { pathname }),
             ...(page && { page: page.toString() }),
             ...(limit && { limit: limit.toString() }),
             ...(expiresIn && { expiresIn: expiresIn.toString() }),
         });
-        const endpoint = `https://api.spider.cloud/v1/data/storage?${params.toString()}`;
+        const endpoint = `${exports.baseUrl}/${config_1.APIRoutes.DataSignUrl}?${params.toString()}`;
         const headers = this.prepareHeaders;
         const response = await fetch(endpoint, {
             method: "GET",
             headers,
         });
-        if (!raw) {
-            if (response.ok) {
-                return response.json();
-            }
-            else {
-                this.handleError(response, `Failed to download files`);
-            }
+        if (response.ok) {
+            return await response.json();
+        }
+        else {
+            this.handleError(response, `Failed to sign files`);
         }
-        return response;
     }
     /**
      * Retrieves the number of credits available on the account.
      * @returns {Promise<any>} The current credit balance.
      */
     async getCredits() {
-        return this._apiGet("credits");
+        return this._apiGet("data/credits");
     }
     /**
      * Send a POST request to insert data into a specified table.
@@ -225,17 +229,39 @@ class Spider {
      * @param {object} data - The data to be inserted.
      * @returns {Promise<any>} The response from the server.
      */
-    async postData(table, data) {
-        return this._apiPost(`data/${table}`, data);
+    async postData(collection, data) {
+        return this._apiPost(`${config_1.APIRoutes.Data}/${collection}`, data);
     }
     /**
      * Send a GET request to retrieve data from a specified table.
-     * @param {string} table - The table name in the database.
+     * @param {Collection} table - The table name in the database.
      * @param {object} params - The query parameters for data retrieval.
      * @returns {Promise<any>} The response from the server.
      */
-    async getData(table, params) {
-        return this._apiGet(`data/${table}?${new URLSearchParams(params).toString()}`);
+    async getData(collections, params) {
+        return this._apiGet(`${config_1.APIRoutes.Data}/${collections}?${new URLSearchParams(params).toString()}`);
+    }
+    /**
+     * Download a record. The url is the path of the storage hash returned and not the exact website url.
+     * @param {QueryRequest} params - The query parameters for data retrieval.
+     * @returns {Promise<any>} The download response from the server.
+     */
+    async download(query, output) {
+        const headers = this.prepareHeaders;
+        const endpoint = `${config_1.APIRoutes.DataDownload}?${new URLSearchParams(query).toString()}`;
+        const response = await fetch(`${exports.baseUrl}/v1/${endpoint}`, {
+            method: "GET",
+            headers,
+        });
+        if (response.ok) {
+            if (output === "text") {
+                return await response.text();
+            }
+            return await response.blob();
+        }
+        else {
+            this.handleError(response, `get from ${endpoint}`);
+        }
     }
     /**
      * Perform a query to get a document.
@@ -243,16 +269,16 @@ class Spider {
      * @returns {Promise<any>} The response from the server.
      */
     async query(query) {
-        return this._apiGet(`data/query?${new URLSearchParams(query).toString()}`);
+        return this._apiGet(`${config_1.APIRoutes.DataQuery}?${new URLSearchParams(query).toString()}`);
     }
     /**
      * Send a DELETE request to remove data from a specified table.
-     * @param {string} table - The table name in the database.
+     * @param {Collection} table - The table name in the database.
      * @param {object} params - Parameters to identify records to delete.
      * @returns {Promise<any>} The response from the server.
      */
-    async deleteData(table, params) {
-        return this._apiDelete(`data/${table}?${new URLSearchParams(params).toString()}`);
+    async deleteData(collection, params) {
+        return this._apiDelete(`${config_1.APIRoutes.Data}/${collection}?${new URLSearchParams(params).toString()}`);
     }
     /**
      * Prepares common headers for each API request.

package/dist/config.d.ts

@@ -260,4 +260,48 @@ export type SpiderCoreResponse = {
     url?: string;
 };
 export type ChunkCallbackFunction = (data: SpiderCoreResponse) => void;
+export declare enum Collection {
+    Websites = "websites",
+    Pages = "pages",
+    PagesMetadata = "pages_metadata",
+    Contacts = "contacts",
+    CrawlState = "crawl_state",
+    CrawlLogs = "crawl_logs",
+    Profiles = "profiles",
+    Credits = "credits",
+    Webhooks = "webhooks",
+    APIKeys = "api_keys"
+}
+declare enum ApiVersion {
+    V1 = "v1"
+}
+export declare enum APIRoutes {
+    Crawl = "crawl",
+    Links = "links",
+    Screenshot = "screenshot",
+    Search = "search",
+    Transform = "transform",
+    PiplineExtractLeads = "pipeline/extract-contacts",
+    PiplineLabel = "pipeline/label",
+    Data = "data",
+    DataCrawlState = "data/crawl_state",
+    DataSignUrl = "data/sign-url",
+    DataDownload = "data/download",
+    DataQuery = "data/query"
+}
+export declare const APISchema: {
+    url: string;
+    versions: {
+        current: ApiVersion;
+        v1: {
+            routes: typeof APIRoutes;
+            end_date: string;
+        };
+        latest: {
+            routes: typeof APIRoutes;
+            end_date: string;
+        };
+    };
+};
+export declare const setBaseUrl: (url: string) => void;
 export {};

package/dist/config.js

@@ -1,2 +1,73 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.setBaseUrl = exports.APISchema = exports.APIRoutes = exports.Collection = void 0;
+// records that you can query
+var Collection;
+(function (Collection) {
+    Collection["Websites"] = "websites";
+    Collection["Pages"] = "pages";
+    Collection["PagesMetadata"] = "pages_metadata";
+    // Leads
+    Collection["Contacts"] = "contacts";
+    Collection["CrawlState"] = "crawl_state";
+    Collection["CrawlLogs"] = "crawl_logs";
+    Collection["Profiles"] = "profiles";
+    Collection["Credits"] = "credits";
+    Collection["Webhooks"] = "webhooks";
+    Collection["APIKeys"] = "api_keys";
+})(Collection || (exports.Collection = Collection = {}));
+// The API version for Spider
+var ApiVersion;
+(function (ApiVersion) {
+    ApiVersion["V1"] = "v1";
+})(ApiVersion || (ApiVersion = {}));
+// The API routes paths.
+var APIRoutes;
+(function (APIRoutes) {
+    // Crawl a website to collect the contents. Can be one page or many.
+    APIRoutes["Crawl"] = "crawl";
+    // Crawl a website to collect the links. Can be one page or many.
+    APIRoutes["Links"] = "links";
+    // Crawl a website to collect screenshots. Can be one page or many.
+    APIRoutes["Screenshot"] = "screenshot";
+    // Search for something and optionally crawl the pages or get the results of the search.
+    APIRoutes["Search"] = "search";
+    // Transform HTML to markdown or text.
+    APIRoutes["Transform"] = "transform";
+    // Pipeline extract leads for a website - emails, phones, etc.
+    APIRoutes["PiplineExtractLeads"] = "pipeline/extract-contacts";
+    // Pipeline label a website by category using AI and metadata.
+    APIRoutes["PiplineLabel"] = "pipeline/label";
+    // Dynamic collection routes.
+    APIRoutes["Data"] = "data";
+    // The last crawl state of a website.
+    APIRoutes["DataCrawlState"] = "data/crawl_state";
+    // Sign a file from storage based on the exact url path of the storage or domain - pathname.
+    APIRoutes["DataSignUrl"] = "data/sign-url";
+    // Download a file from storage based on the exact url path of the storage or domain - pathname.
+    APIRoutes["DataDownload"] = "data/download";
+    // Perform a query on the global database to grab content without crawling if available.
+    APIRoutes["DataQuery"] = "data/query";
+})(APIRoutes || (exports.APIRoutes = APIRoutes = {}));
+// The base API target info for Spider Cloud.
+exports.APISchema = {
+    url: "https://api.spider.cloud",
+    versions: {
+        current: ApiVersion.V1,
+        v1: {
+            routes: APIRoutes,
+            end_date: "",
+        },
+        latest: {
+            routes: APIRoutes,
+            end_date: "",
+        },
+    },
+};
+// Adjust the Spider Cloud endpoint.
+const setBaseUrl = (url) => {
+    if (url) {
+        exports.APISchema["url"] = url;
+    }
+};
+exports.setBaseUrl = setBaseUrl;

package/dist/index.d.ts

@@ -1,2 +1,3 @@
 export { Spider } from "./client";
+export { Collection, setBaseUrl, APISchema } from "./config";
 export type { SpiderParams, Budget, Viewport, QueryRequest } from "./config";

package/dist/index.js

@@ -1,5 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Spider = void 0;
+exports.APISchema = exports.setBaseUrl = exports.Collection = exports.Spider = void 0;
 var client_1 = require("./client");
 Object.defineProperty(exports, "Spider", { enumerable: true, get: function () { return client_1.Spider; } });
+var config_1 = require("./config");
+Object.defineProperty(exports, "Collection", { enumerable: true, get: function () { return config_1.Collection; } });
+Object.defineProperty(exports, "setBaseUrl", { enumerable: true, get: function () { return config_1.setBaseUrl; } });
+Object.defineProperty(exports, "APISchema", { enumerable: true, get: function () { return config_1.APISchema; } });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spider-cloud/spider-client",
-  "version": "0.0.58",
+  "version": "0.0.60",
   "description": "Isomorphic Javascript SDK for Spider Cloud services",
   "scripts": {
     "test": "node --import tsx  --test __tests__/*test.ts",