@spider-cloud/spider-client 0.0.32 → 0.0.34

package/README.md

@@ -91,6 +91,19 @@ spider
   .catch((error) => console.error(error));
 ```
 
+#### Download storage data
+
+To download stored data like raw HTML or markdown  use the `downloadFiles` method. Provide the website name and an object containing query parameters:
+
+```javascript
+const websiteName = "spider.cloud";
+const queryParams = { limit: 20, page: 0 };
+spider
+  .downloadFiles(websiteName, queryParams)
+  .then((response) => console.log(response))
+  .catch((error) => console.error(error));
+```
+
 ### Available Methods
 
 - **`scrapeUrl(url, params)`**: Scrape data from a specified URL. Optional parameters can be passed to customize the scraping behavior.
@@ -105,6 +118,7 @@ spider
 - **`getCredits()`**: Retrieve account's remaining credits.
 - **`getData(table, params)`**: Retrieve data records from the DB.
 - **`deleteData(table, params)`**: Delete records from the DB.
+- **`downloadFiles(domain, params)`**: Download the records from the DB.
 
 ## Error Handling
 

package/dist/client.d.ts

@@ -108,14 +108,16 @@ export declare class Spider {
      */
     getCrawlState(url: string, params?: GenericParams): Promise<any>;
     /**
-     * Downloads files from the specified user's storage.
-     * @param {string} userId - The user's ID whose files you want to download.
+     * Downloads files from the specified user's storage. This will create signed URLS to use.
      * @param {string} [domain] - The domain for the user's storage. If not provided, downloads all files.
-     * @param {number} [page] - The page number for pagination.
-     * @param {number} [limit] - The number of files to return per page.
+     * @param {Object} [options] - The download options
      * @returns {Promise<Response>} The response containing the file stream.
      */
-    downloadFiles(userId: string, domain?: string, page?: number, limit?: number): Promise<Response>;
+    downloadFiles(domain?: string, options?: {
+        page?: number;
+        limit?: number;
+        expiresIn?: number;
+    }): Promise<Response>;
     /**
      * Retrieves the number of credits available on the account.
      * @returns {Promise<any>} The current credit balance.

package/dist/client.js

@@ -161,18 +161,18 @@ class Spider {
         return this._apiPost("data/crawl_state", { url: url, ...params });
     }
     /**
-     * Downloads files from the specified user's storage.
-     * @param {string} userId - The user's ID whose files you want to download.
+     * Downloads files from the specified user's storage. This will create signed URLS to use.
      * @param {string} [domain] - The domain for the user's storage. If not provided, downloads all files.
-     * @param {number} [page] - The page number for pagination.
-     * @param {number} [limit] - The number of files to return per page.
+     * @param {Object} [options] - The download options
      * @returns {Promise<Response>} The response containing the file stream.
      */
-    async downloadFiles(userId, domain, page, limit) {
+    async downloadFiles(domain, options) {
+        const { page, limit, expiresIn } = options !== null && options !== void 0 ? options : {};
         const params = new URLSearchParams({
             ...(domain && { domain }),
             ...(page && { page: page.toString() }),
             ...(limit && { limit: limit.toString() }),
+            ...(expiresIn && { expiresIn: expiresIn.toString() }),
         });
         const endpoint = `https://api.spider.cloud/v1/data/storage?${params.toString()}`;
         const headers = this.prepareHeaders();
@@ -181,7 +181,7 @@ class Spider {
             headers,
         });
         if (!response.ok) {
-            this.handleError(response, `download files for user: ${userId}`);
+            this.handleError(response, `Failed to download files`);
         }
         return response;
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spider-cloud/spider-client",
-  "version": "0.0.32",
+  "version": "0.0.34",
   "description": "A Javascript SDK for Spider Cloud services",
   "scripts": {
     "test": "jest",