@spider-cloud/spider-client 0.0.59 → 0.0.61

package/dist/client.d.ts

@@ -1,5 +1,4 @@
 import { ChunkCallbackFunction, Collection, QueryRequest, SpiderCoreResponse, SpiderParams } from "./config";
-export declare const BASE_API_URL = "https://api.spider.cloud";
 /**
  * Generic params for core request.
  */
@@ -97,7 +96,7 @@ export declare class Spider {
         url?: string;
     }[], params?: {}): Promise<any>;
     /**
-     * Extracts contact information from the specified URL.
+     * Extracts leads from a website.
      * @param {string} url - The URL from which to extract contacts.
      * @param {GenericParams} [params={}] - Configuration parameters for the extraction.
      * @returns {Promise<any>} The contact information extracted.
@@ -143,7 +142,7 @@ export declare class Spider {
      * @param {object} data - The data to be inserted.
      * @returns {Promise<any>} The response from the server.
      */
-    postData(table: string, data: GenericParams | Record<string, any>): Promise<any>;
+    postData(collection: Collection, data: GenericParams | Record<string, any>): Promise<any>;
     /**
      * Send a GET request to retrieve data from a specified table.
      * @param {Collection} table - The table name in the database.

package/dist/client.js

@@ -1,10 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Spider = exports.BASE_API_URL = void 0;
+exports.Spider = void 0;
+const config_1 = require("./config");
 const package_json_1 = require("../package.json");
 const supabase_1 = require("./supabase");
 const stream_reader_1 = require("./utils/stream-reader");
-exports.BASE_API_URL = "https://api.spider.cloud";
 /**
  * A class to interact with the Spider API.
  */
@@ -42,7 +42,7 @@ class Spider {
      */
     async _apiPost(endpoint, data, stream, jsonl) {
         const headers = jsonl ? this.prepareHeadersJsonL : this.prepareHeaders;
-        const response = await fetch(`${exports.BASE_API_URL}/v1/${endpoint}`, {
+        const response = await fetch(`${config_1.APISchema["url"]}/${config_1.ApiVersion.V1}/${endpoint}`, {
             method: "POST",
             headers: headers,
             body: JSON.stringify(data),
@@ -64,7 +64,7 @@ class Spider {
      */
     async _apiGet(endpoint) {
         const headers = this.prepareHeaders;
-        const response = await fetch(`${exports.BASE_API_URL}/v1/${endpoint}`, {
+        const response = await fetch(`${config_1.APISchema["url"]}/${config_1.ApiVersion.V1}/${endpoint}`, {
             method: "GET",
             headers: headers,
         });
@@ -82,7 +82,7 @@ class Spider {
      */
     async _apiDelete(endpoint) {
         const headers = this.prepareHeaders;
-        const response = await fetch(`${exports.BASE_API_URL}/v1/${endpoint}`, {
+        const response = await fetch(`${config_1.APISchema["url"]}/${config_1.ApiVersion.V1}/${endpoint}`, {
             method: "DELETE",
             headers,
         });
@@ -100,7 +100,7 @@ class Spider {
      * @returns {Promise<any>} The scraped data from the URL.
      */
     async scrapeUrl(url, params = {}) {
-        return this._apiPost("crawl", { url: url, limit: 1, ...params });
+        return this._apiPost(config_1.APIRoutes.Crawl, { url: url, limit: 1, ...params });
     }
     /**
      * Initiates a crawling job starting from the specified URL.
@@ -112,7 +112,7 @@ class Spider {
      */
     async crawlUrl(url, params = {}, stream = false, cb) {
         const jsonl = stream && cb;
-        const res = await this._apiPost("crawl", { url: url, ...params }, stream, !!jsonl);
+        const res = await this._apiPost(config_1.APIRoutes.Crawl, { url: url, ...params }, stream, !!jsonl);
         if (jsonl) {
             return await (0, stream_reader_1.streamReader)(res, cb);
         }
@@ -125,7 +125,7 @@ class Spider {
      * @returns {Promise<any>} A list of links extracted from the URL.
      */
     async links(url, params = {}) {
-        return this._apiPost("links", { url: url, ...params });
+        return this._apiPost(config_1.APIRoutes.Links, { url: url, ...params });
     }
     /**
      * Takes a screenshot of the website starting from this URL.
@@ -134,7 +134,7 @@ class Spider {
      * @returns {Promise<any>} The screenshot data.
      */
     async screenshot(url, params = {}) {
-        return this._apiPost("screenshot", { url: url, ...params });
+        return this._apiPost(config_1.APIRoutes.Screenshot, { url: url, ...params });
     }
     /**
      *  Perform a search and gather a list of websites to start crawling and collect resources.
@@ -143,7 +143,7 @@ class Spider {
      * @returns {Promise<any>} The result of the crawl, either structured data or a Response object if streaming.
      */
     async search(q, params = {}) {
-        return this._apiPost("search", { search: q, ...params });
+        return this._apiPost(config_1.APIRoutes.Search, { search: q, ...params });
     }
     /**
      *  Transform HTML to Markdown or text. You can send up to 10MB of data at once.
@@ -152,16 +152,19 @@ class Spider {
      * @returns {Promise<any>} The transformation result.
      */
     async transform(data, params = {}) {
-        return this._apiPost("transform", { data, ...params });
+        return this._apiPost(config_1.APIRoutes.Transform, { data, ...params });
     }
     /**
-     * Extracts contact information from the specified URL.
+     * Extracts leads from a website.
      * @param {string} url - The URL from which to extract contacts.
      * @param {GenericParams} [params={}] - Configuration parameters for the extraction.
      * @returns {Promise<any>} The contact information extracted.
      */
     async extractContacts(url, params = {}) {
-        return this._apiPost("pipeline/extract-contacts", { url: url, ...params });
+        return this._apiPost(config_1.APIRoutes.PiplineExtractLeads, {
+            url: url,
+            ...params,
+        });
     }
     /**
      * Applies labeling to data extracted from a specified URL.
@@ -170,7 +173,7 @@ class Spider {
      * @returns {Promise<any>} The labeled data.
      */
     async label(url, params = {}) {
-        return this._apiPost("pipeline/label", { url: url, ...params });
+        return this._apiPost(config_1.APIRoutes.PiplineLabel, { url: url, ...params });
     }
     /**
      * Check the crawl state of the website.
@@ -179,7 +182,7 @@ class Spider {
      * @returns {Promise<any>} The crawl state data.
      */
     async getCrawlState(url, params = {}) {
-        return this._apiPost("data/crawl_state", { url: url, ...params });
+        return this._apiPost(config_1.APIRoutes.DataCrawlState, { url: url, ...params });
     }
     /**
      * Create a signed url to download files from the storage.
@@ -199,7 +202,7 @@ class Spider {
             ...(limit && { limit: limit.toString() }),
             ...(expiresIn && { expiresIn: expiresIn.toString() }),
         });
-        const endpoint = `${exports.BASE_API_URL}/data/sign-url?${params.toString()}`;
+        const endpoint = `${config_1.APISchema["url"]}/${config_1.APIRoutes.DataSignUrl}?${params.toString()}`;
         const headers = this.prepareHeaders;
         const response = await fetch(endpoint, {
             method: "GET",
@@ -209,7 +212,7 @@ class Spider {
             return await response.json();
         }
         else {
-            this.handleError(response, `Failed to download files`);
+            this.handleError(response, `Failed to sign files`);
         }
     }
     /**
@@ -225,8 +228,8 @@ class Spider {
      * @param {object} data - The data to be inserted.
      * @returns {Promise<any>} The response from the server.
      */
-    async postData(table, data) {
-        return this._apiPost(`data/${table}`, data);
+    async postData(collection, data) {
+        return this._apiPost(`${config_1.APIRoutes.Data}/${collection}`, data);
     }
     /**
      * Send a GET request to retrieve data from a specified table.
@@ -235,7 +238,7 @@ class Spider {
      * @returns {Promise<any>} The response from the server.
      */
     async getData(collections, params) {
-        return this._apiGet(`data/${collections}?${new URLSearchParams(params).toString()}`);
+        return this._apiGet(`${config_1.APIRoutes.Data}/${collections}?${new URLSearchParams(params).toString()}`);
     }
     /**
      * Download a record. The url is the path of the storage hash returned and not the exact website url.
@@ -244,8 +247,8 @@ class Spider {
      */
     async download(query, output) {
         const headers = this.prepareHeaders;
-        const endpoint = `data/download?${new URLSearchParams(query).toString()}`;
-        const response = await fetch(`${exports.BASE_API_URL}/v1/${endpoint}`, {
+        const endpoint = `${config_1.APIRoutes.DataDownload}?${new URLSearchParams(query).toString()}`;
+        const response = await fetch(`${config_1.APISchema["url"]}/${config_1.ApiVersion.V1}/${endpoint}`, {
             method: "GET",
             headers,
         });
@@ -265,7 +268,7 @@ class Spider {
      * @returns {Promise<any>} The response from the server.
      */
     async query(query) {
-        return this._apiGet(`data/query?${new URLSearchParams(query).toString()}`);
+        return this._apiGet(`${config_1.APIRoutes.DataQuery}?${new URLSearchParams(query).toString()}`);
     }
     /**
      * Send a DELETE request to remove data from a specified table.
@@ -274,7 +277,7 @@ class Spider {
      * @returns {Promise<any>} The response from the server.
      */
     async deleteData(collection, params) {
-        return this._apiDelete(`data/${collection}?${new URLSearchParams(params).toString()}`);
+        return this._apiDelete(`${config_1.APIRoutes.Data}/${collection}?${new URLSearchParams(params).toString()}`);
     }
     /**
      * Prepares common headers for each API request.

package/dist/config.d.ts

@@ -272,4 +272,36 @@ export declare enum Collection {
     Webhooks = "webhooks",
     APIKeys = "api_keys"
 }
+export declare enum ApiVersion {
+    V1 = "v1"
+}
+export declare enum APIRoutes {
+    Crawl = "crawl",
+    Links = "links",
+    Screenshot = "screenshot",
+    Search = "search",
+    Transform = "transform",
+    PiplineExtractLeads = "pipeline/extract-contacts",
+    PiplineLabel = "pipeline/label",
+    Data = "data",
+    DataCrawlState = "data/crawl_state",
+    DataSignUrl = "data/sign-url",
+    DataDownload = "data/download",
+    DataQuery = "data/query"
+}
+export declare const APISchema: {
+    url: string;
+    versions: {
+        current: ApiVersion;
+        v1: {
+            routes: typeof APIRoutes;
+            end_date: string;
+        };
+        latest: {
+            routes: typeof APIRoutes;
+            end_date: string;
+        };
+    };
+};
+export declare const setBaseUrl: (url: string) => void;
 export {};

package/dist/config.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Collection = void 0;
+exports.setBaseUrl = exports.APISchema = exports.APIRoutes = exports.ApiVersion = exports.Collection = void 0;
 // records that you can query
 var Collection;
 (function (Collection) {
@@ -16,3 +16,58 @@ var Collection;
     Collection["Webhooks"] = "webhooks";
     Collection["APIKeys"] = "api_keys";
 })(Collection || (exports.Collection = Collection = {}));
+// The API version for Spider
+var ApiVersion;
+(function (ApiVersion) {
+    ApiVersion["V1"] = "v1";
+})(ApiVersion || (exports.ApiVersion = ApiVersion = {}));
+// The API routes paths.
+var APIRoutes;
+(function (APIRoutes) {
+    // Crawl a website to collect the contents. Can be one page or many.
+    APIRoutes["Crawl"] = "crawl";
+    // Crawl a website to collect the links. Can be one page or many.
+    APIRoutes["Links"] = "links";
+    // Crawl a website to collect screenshots. Can be one page or many.
+    APIRoutes["Screenshot"] = "screenshot";
+    // Search for something and optionally crawl the pages or get the results of the search.
+    APIRoutes["Search"] = "search";
+    // Transform HTML to markdown or text.
+    APIRoutes["Transform"] = "transform";
+    // Pipeline extract leads for a website - emails, phones, etc.
+    APIRoutes["PiplineExtractLeads"] = "pipeline/extract-contacts";
+    // Pipeline label a website by category using AI and metadata.
+    APIRoutes["PiplineLabel"] = "pipeline/label";
+    // Dynamic collection routes.
+    APIRoutes["Data"] = "data";
+    // The last crawl state of a website.
+    APIRoutes["DataCrawlState"] = "data/crawl_state";
+    // Sign a file from storage based on the exact url path of the storage or domain - pathname.
+    APIRoutes["DataSignUrl"] = "data/sign-url";
+    // Download a file from storage based on the exact url path of the storage or domain - pathname.
+    APIRoutes["DataDownload"] = "data/download";
+    // Perform a query on the global database to grab content without crawling if available.
+    APIRoutes["DataQuery"] = "data/query";
+})(APIRoutes || (exports.APIRoutes = APIRoutes = {}));
+// The base API target info for Spider Cloud.
+exports.APISchema = {
+    url: "https://api.spider.cloud",
+    versions: {
+        current: ApiVersion.V1,
+        v1: {
+            routes: APIRoutes,
+            end_date: "",
+        },
+        latest: {
+            routes: APIRoutes,
+            end_date: "",
+        },
+    },
+};
+// Adjust the Spider Cloud endpoint.
+const setBaseUrl = (url) => {
+    if (url) {
+        exports.APISchema["url"] = url;
+    }
+};
+exports.setBaseUrl = setBaseUrl;

package/dist/index.d.ts

@@ -1,3 +1,3 @@
 export { Spider } from "./client";
+export { Collection, setBaseUrl, APISchema } from "./config";
 export type { SpiderParams, Budget, Viewport, QueryRequest } from "./config";
-export { Collection } from "./config";

package/dist/index.js

@@ -1,7 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Collection = exports.Spider = void 0;
+exports.APISchema = exports.setBaseUrl = exports.Collection = exports.Spider = void 0;
 var client_1 = require("./client");
 Object.defineProperty(exports, "Spider", { enumerable: true, get: function () { return client_1.Spider; } });
 var config_1 = require("./config");
 Object.defineProperty(exports, "Collection", { enumerable: true, get: function () { return config_1.Collection; } });
+Object.defineProperty(exports, "setBaseUrl", { enumerable: true, get: function () { return config_1.setBaseUrl; } });
+Object.defineProperty(exports, "APISchema", { enumerable: true, get: function () { return config_1.APISchema; } });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spider-cloud/spider-client",
-  "version": "0.0.59",
+  "version": "0.0.61",
   "description": "Isomorphic Javascript SDK for Spider Cloud services",
   "scripts": {
     "test": "node --import tsx  --test __tests__/*test.ts",