@spider-cloud/spider-client 0.0.1

package/LICENSE

@@ -0,0 +1 @@
+../LICENSE

package/README.md

@@ -0,0 +1,82 @@
+# Spider Cloud JavaScript SDK
+
+The Spider Cloud JavaScript SDK offers a streamlined set of tools for web scraping and crawling, with capabilities that allow for comprehensive data extraction suitable for interfacing with AI language models. This SDK makes it easy to interact programmatically with the Spider Cloud API from any JavaScript or Node.js application.
+
+## Installation
+
+You can install the Spider Cloud JavaScript SDK via npm:
+
+```bash
+npm install @spider-cloud/spider-client
+```
+
+Or with yarn:
+
+```bash
+yarn add @spider-cloud/spider-client
+```
+
+## Configuration
+
+Before using the SDK, you will need to provide it with your API key. Obtain an API key from [spider.cloud](https://spider.cloud) and either pass it directly to the constructor or set it as an environment variable `SPIDER_API_KEY`.
+
+## Usage
+
+Here's a basic example to demonstrate how to use the SDK:
+
+```javascript
+import Spider from "spider-client";
+
+// Initialize the SDK with your API key
+const app = new Spider("your_api_key");
+
+// Scrape a URL
+const url = "https://spiderwebai.xyz";
+app
+  .scrapeUrl(url)
+  .then((data) => {
+    console.log("Scraped Data:", data);
+  })
+  .catch((error) => {
+    console.error("Scrape Error:", error);
+  });
+
+// Crawl a website
+const crawlParams = {
+  limit: 5,
+  proxy_enabled: true,
+  store_data: false,
+  metadata: false,
+  request: "http",
+};
+app
+  .crawlUrl(url, crawlParams)
+  .then((result) => {
+    console.log("Crawl Result:", result);
+  })
+  .catch((error) => {
+    console.error("Crawl Error:", error);
+  });
+```
+
+### Available Methods
+
+- **`scrapeUrl(url, params)`**: Scrape data from a specified URL. Optional parameters can be passed to customize the scraping behavior.
+- **`crawlUrl(url, params, stream)`**: Begin crawling from a specific URL with optional parameters for customization and an optional streaming response.
+- **`links(url, params)`**: Retrieve all links from the specified URL with optional parameters.
+- **`screenshot(url, params)`**: Take a screenshot of the specified URL.
+- **`extractContacts(url, params)`**: Extract contact information from the specified URL.
+- **`label(url, params)`**: Apply labeling to data extracted from the specified URL.
+- **`getCredits()`**: Retrieve account's remaining credits.
+
+## Error Handling
+
+The SDK provides robust error handling and will throw exceptions when it encounters critical issues. Always use `.catch()` on promises to handle these errors gracefully.
+
+## Contributing
+
+Contributions are always welcome! Feel free to open an issue or submit a pull request on our GitHub repository.
+
+## License
+
+The Spider Cloud JavaScript SDK is open-source and released under the [MIT License](https://opensource.org/licenses/MIT).

package/dist/spiderwebai.d.ts

@@ -0,0 +1,90 @@
+/// <reference types="node" />
+/**
+ * A class to interact with the SpiderWeb AI API.
+ */
+export default class Spider {
+    private apiKey?;
+    /**
+     * Create an instance of Spider.
+     * @param {string | null} apiKey - The API key used to authenticate to the SpiderWeb AI API. If null, attempts to source from environment variables.
+     * @throws Will throw an error if the API key is not provided.
+     */
+    constructor(apiKey?: string);
+    /**
+     * Internal method to handle POST requests.
+     * @param {string} endpoint - The API endpoint to which the POST request should be sent.
+     * @param {Record<string, any>} data - The JSON data to be sent in the request body.
+     * @param {boolean} [stream=false] - Whether to stream the response back without parsing.
+     * @returns {Promise<Response | any>} The response in JSON if not streamed, or the Response object if streamed.
+     */
+    private _apiPost;
+    /**
+     * Internal method to handle GET requests.
+     * @param {string} endpoint - The API endpoint from which data should be retrieved.
+     * @returns {Promise<any>} The data returned from the endpoint in JSON format.
+     */
+    private _apiGet;
+    /**
+     * Scrapes data from a specified URL.
+     * @param {string} url - The URL to scrape.
+     * @param {object} [params={}] - Additional parameters for the scraping request.
+     * @returns {Promise<any>} The scraped data from the URL.
+     */
+    scrapeUrl(url: string, params?: {}): Promise<unknown>;
+    /**
+     * Initiates a crawling job starting from the specified URL.
+     * @param {string} url - The URL to start crawling.
+     * @param {object} [params={}] - Additional parameters for the crawl.
+     * @param {boolean} [stream=false] - Whether to receive the response as a stream.
+     * @returns {Promise<any | Response>} The result of the crawl, either structured data or a Response object if streaming.
+     */
+    crawlUrl(url: string, params?: {}, stream?: boolean): Promise<unknown>;
+    /**
+     * Retrieves all links from the specified URL.
+     * @param {string} url - The URL from which to gather links.
+     * @param {object} [params={}] - Additional parameters for the request.
+     * @returns {Promise<any>} A list of links extracted from the URL.
+     */
+    links(url: string, params?: {}): Promise<unknown>;
+    /**
+     * Takes a screenshot of the specified URL.
+     * @param {string} url - The URL to screenshot.
+     * @param {object} [params={}] - Configuration parameters for the screenshot.
+     * @returns {Promise<any>} The screenshot data.
+     */
+    screenshot(url: string, params?: {}): Promise<unknown>;
+    /**
+     * Extracts contact information from the specified URL.
+     * @param {string} url - The URL from which to extract contacts.
+     * @param {object} [params={}] - Configuration parameters for the extraction.
+     * @returns {Promise<any>} The contact information extracted.
+     */
+    extractContacts(url: string, params?: {}): Promise<unknown>;
+    /**
+     * Applies labeling to data extracted from a specified URL.
+     * @param {string} url - The URL to label.
+     * @param {object} [params={}] - Configuration parameters for labeling.
+     * @returns {Promise<any>} The labeled data.
+     */
+    label(url: string, params?: {}): Promise<unknown>;
+    /**
+     * Retrieves the number of credits available on the account.
+     * @returns {Promise<any>} The current credit balance.
+     */
+    getCredits(): Promise<unknown>;
+    /**
+     * Prepares common headers for each API request.
+     * @returns {HeadersInit} A headers object for fetch requests.
+     */
+    prepareHeaders(): {
+        "Content-Type": string;
+        Authorization: string;
+    };
+    /**
+     * Handles errors from API requests.
+     * @param {Response} response - The fetch response object.
+     * @param {string} action - Description of the attempted action.
+     * @throws Will throw an error with detailed status information.
+     */
+    handleError(response: Response, action: string): void;
+}

package/dist/spiderwebai.js

@@ -0,0 +1,142 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+/**
+ * A class to interact with the SpiderWeb AI API.
+ */
+class Spider {
+    /**
+     * Create an instance of Spider.
+     * @param {string | null} apiKey - The API key used to authenticate to the SpiderWeb AI API. If null, attempts to source from environment variables.
+     * @throws Will throw an error if the API key is not provided.
+     */
+    constructor(apiKey) {
+        this.apiKey = apiKey || process?.env?.SPIDER_API_KEY;
+        if (!this.apiKey) {
+            throw new Error("No API key provided");
+        }
+    }
+    /**
+     * Internal method to handle POST requests.
+     * @param {string} endpoint - The API endpoint to which the POST request should be sent.
+     * @param {Record<string, any>} data - The JSON data to be sent in the request body.
+     * @param {boolean} [stream=false] - Whether to stream the response back without parsing.
+     * @returns {Promise<Response | any>} The response in JSON if not streamed, or the Response object if streamed.
+     */
+    async _apiPost(endpoint, data, stream = false) {
+        const headers = this.prepareHeaders();
+        const response = await fetch(`https://spider.a11ywatch.com/v1/${endpoint}`, {
+            method: "POST",
+            headers: headers,
+            body: JSON.stringify(data),
+        });
+        if (!stream) {
+            if (response.ok) {
+                return response.json();
+            }
+            else {
+                this.handleError(response, `post to ${endpoint}`);
+            }
+        }
+        return response;
+    }
+    /**
+     * Internal method to handle GET requests.
+     * @param {string} endpoint - The API endpoint from which data should be retrieved.
+     * @returns {Promise<any>} The data returned from the endpoint in JSON format.
+     */
+    async _apiGet(endpoint) {
+        const headers = this.prepareHeaders();
+        const response = await fetch(`https://spider.a11ywatch.com/v1/${endpoint}`, {
+            method: "GET",
+            headers: headers,
+        });
+        if (response.ok) {
+            return response.json();
+        }
+        else {
+            this.handleError(response, `get from ${endpoint}`);
+        }
+    }
+    /**
+     * Scrapes data from a specified URL.
+     * @param {string} url - The URL to scrape.
+     * @param {object} [params={}] - Additional parameters for the scraping request.
+     * @returns {Promise<any>} The scraped data from the URL.
+     */
+    async scrapeUrl(url, params = {}) {
+        return this._apiPost("crawl", { url: url, budget: '{"*":1}', ...params });
+    }
+    /**
+     * Initiates a crawling job starting from the specified URL.
+     * @param {string} url - The URL to start crawling.
+     * @param {object} [params={}] - Additional parameters for the crawl.
+     * @param {boolean} [stream=false] - Whether to receive the response as a stream.
+     * @returns {Promise<any | Response>} The result of the crawl, either structured data or a Response object if streaming.
+     */
+    async crawlUrl(url, params = {}, stream = false) {
+        return this._apiPost("crawl", { url: url, ...params }, stream);
+    }
+    /**
+     * Retrieves all links from the specified URL.
+     * @param {string} url - The URL from which to gather links.
+     * @param {object} [params={}] - Additional parameters for the request.
+     * @returns {Promise<any>} A list of links extracted from the URL.
+     */
+    async links(url, params = {}) {
+        return this._apiPost("links", { url: url, ...params });
+    }
+    /**
+     * Takes a screenshot of the specified URL.
+     * @param {string} url - The URL to screenshot.
+     * @param {object} [params={}] - Configuration parameters for the screenshot.
+     * @returns {Promise<any>} The screenshot data.
+     */
+    async screenshot(url, params = {}) {
+        return this._apiPost("screenshot", { url: url, ...params });
+    }
+    /**
+     * Extracts contact information from the specified URL.
+     * @param {string} url - The URL from which to extract contacts.
+     * @param {object} [params={}] - Configuration parameters for the extraction.
+     * @returns {Promise<any>} The contact information extracted.
+     */
+    async extractContacts(url, params = {}) {
+        return this._apiPost("pipeline/extract-contacts", { url: url, ...params });
+    }
+    /**
+     * Applies labeling to data extracted from a specified URL.
+     * @param {string} url - The URL to label.
+     * @param {object} [params={}] - Configuration parameters for labeling.
+     * @returns {Promise<any>} The labeled data.
+     */
+    async label(url, params = {}) {
+        return this._apiPost("pipeline/label", { url: url, ...params });
+    }
+    /**
+     * Retrieves the number of credits available on the account.
+     * @returns {Promise<any>} The current credit balance.
+     */
+    async getCredits() {
+        return this._apiGet("credits");
+    }
+    /**
+     * Prepares common headers for each API request.
+     * @returns {HeadersInit} A headers object for fetch requests.
+     */
+    prepareHeaders() {
+        return {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${this.apiKey}`,
+        };
+    }
+    /**
+     * Handles errors from API requests.
+     * @param {Response} response - The fetch response object.
+     * @param {string} action - Description of the attempted action.
+     * @throws Will throw an error with detailed status information.
+     */
+    handleError(response, action) {
+        throw new Error(`Failed to ${action}. Status code: ${response.status}.`);
+    }
+}
+exports.default = Spider;

package/package.json

@@ -0,0 +1,59 @@
+{
+  "name": "@spider-cloud/spider-client",
+  "version": "0.0.1",
+  "description": "A Javascript SDK for Spider Cloud services",
+  "private": false,
+  "scripts": {
+    "test": "jest",
+    "build": "tsc",
+    "prepublishOnly": "npm test && npm run build"
+  },
+  "main": "dist/spiderwebai.js",
+  "types": "dist/spiderwebai.d.ts",
+  "files": [
+    "dist/**/*"
+  ],
+  "keywords": [
+    "spiderwebai",
+    "sdk",
+    "web scraping",
+    "api"
+  ],
+  "author": "Jeff Mendez<jeff@a11ywatch.com>",
+  "license": "MIT",
+  "devDependencies": {
+    "@jest/globals": "^29.7.0",
+    "@types/jest": "^29.5.12",
+    "@types/node": "20.12.7",
+    "ts-jest": "^29.1.2",
+    "typescript": "5.4.5"
+  },
+  "jest": {
+    "preset": "ts-jest",
+    "testEnvironment": "node",
+    "moduleFileExtensions": [
+      "ts",
+      "tsx",
+      "js",
+      "jsx"
+    ],
+    "roots": [
+      "<rootDir>/src",
+      "<rootDir>/__tests__"
+    ],
+    "transform": {
+      "^.+\\\\.tsx?$": "ts-jest"
+    },
+    "testRegex": "(/__tests__/.*|\\.(test|spec))\\.(ts|tsx)$",
+    "moduleDirectories": [
+      "node_modules",
+      "src"
+    ],
+    "collectCoverage": true,
+    "coverageDirectory": "coverage",
+    "coverageReporters": [
+      "text",
+      "lcov"
+    ]
+  }
+}