@spider-cloud/spider-client 0.0.33 → 0.0.35

package/README.md

@@ -91,6 +91,19 @@ spider
   .catch((error) => console.error(error));
 ```
 
+#### Download storage data
+
+To download stored data like raw HTML or markdown  use the `downloadFiles` method. Provide the website name and an object containing query parameters:
+
+```javascript
+const websiteName = "spider.cloud";
+const queryParams = { limit: 20, page: 0 };
+spider
+  .downloadFiles(websiteName, queryParams)
+  .then((response) => console.log(response))
+  .catch((error) => console.error(error));
+```
+
 ### Available Methods
 
 - **`scrapeUrl(url, params)`**: Scrape data from a specified URL. Optional parameters can be passed to customize the scraping behavior.
@@ -105,6 +118,7 @@ spider
 - **`getCredits()`**: Retrieve account's remaining credits.
 - **`getData(table, params)`**: Retrieve data records from the DB.
 - **`deleteData(table, params)`**: Delete records from the DB.
+- **`downloadFiles(domain, params)`**: Download the records from the DB.
 
 ## Error Handling
 

package/dist/client.d.ts

@@ -108,16 +108,18 @@ export declare class Spider {
      */
     getCrawlState(url: string, params?: GenericParams): Promise<any>;
     /**
-     * Downloads files from the specified user's storage.
+     * Downloads files from the specified user's storage. This will create signed URLS to use.
      * @param {string} [domain] - The domain for the user's storage. If not provided, downloads all files.
-     * @param {number} [page] - The page number for pagination.
-     * @param {number} [limit] - The number of files to return per page.
+     * @param {Object} [options] - The download options.
+     * @param {boolean} [raw] - Return the raw response.
+  
      * @returns {Promise<Response>} The response containing the file stream.
      */
     downloadFiles(domain?: string, options?: {
         page?: number;
         limit?: number;
-    }): Promise<Response>;
+        expiresIn?: number;
+    }, raw?: boolean): Promise<Response>;
     /**
      * Retrieves the number of credits available on the account.
      * @returns {Promise<any>} The current credit balance.

package/dist/client.js

@@ -161,18 +161,20 @@ class Spider {
         return this._apiPost("data/crawl_state", { url: url, ...params });
     }
     /**
-     * Downloads files from the specified user's storage.
+     * Downloads files from the specified user's storage. This will create signed URLS to use.
      * @param {string} [domain] - The domain for the user's storage. If not provided, downloads all files.
-     * @param {number} [page] - The page number for pagination.
-     * @param {number} [limit] - The number of files to return per page.
+     * @param {Object} [options] - The download options.
+     * @param {boolean} [raw] - Return the raw response.
+  
      * @returns {Promise<Response>} The response containing the file stream.
      */
-    async downloadFiles(domain, options) {
-        const { page, limit } = options !== null && options !== void 0 ? options : {};
+    async downloadFiles(domain, options, raw) {
+        const { page, limit, expiresIn } = options !== null && options !== void 0 ? options : {};
         const params = new URLSearchParams({
             ...(domain && { domain }),
             ...(page && { page: page.toString() }),
             ...(limit && { limit: limit.toString() }),
+            ...(expiresIn && { expiresIn: expiresIn.toString() }),
         });
         const endpoint = `https://api.spider.cloud/v1/data/storage?${params.toString()}`;
         const headers = this.prepareHeaders();
@@ -180,8 +182,13 @@ class Spider {
             method: "GET",
             headers,
         });
-        if (!response.ok) {
-            this.handleError(response, `Cannot download files`);
+        if (!raw) {
+            if (response.ok) {
+                return response.json();
+            }
+            else {
+                this.handleError(response, `Failed to download files`);
+            }
         }
         return response;
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spider-cloud/spider-client",
-  "version": "0.0.33",
+  "version": "0.0.35",
   "description": "A Javascript SDK for Spider Cloud services",
   "scripts": {
     "test": "jest",