@taskmagic/apps-scrapeless 0.0.1

package/src/lib/common/client.js

@@ -0,0 +1,121 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.scrapelessCommon = exports.SCRAPELESS_BASE_URL = void 0;
+const tslib_1 = require("tslib");
+const pieces_common_1 = require("@taskmagic/pieces-common");
+exports.SCRAPELESS_BASE_URL = 'https://api.scrapeless.com';
+function sendRequest(apiKey, method, path, body, queryParams) {
+    return tslib_1.__awaiter(this, void 0, void 0, function* () {
+        const request = Object.assign(Object.assign({ method, url: path.startsWith('http') ? path : `${exports.SCRAPELESS_BASE_URL}${path}`, headers: {
+                'Content-Type': 'application/json',
+                'x-api-token': apiKey,
+            } }, (body !== undefined ? { body } : {})), (queryParams !== undefined ? { queryParams } : {}));
+        const response = yield pieces_common_1.httpClient.sendRequest(request);
+        return { status: response.status, body: response.body };
+    });
+}
+function sleep(ms) {
+    return tslib_1.__awaiter(this, void 0, void 0, function* () {
+        return new Promise((resolve) => setTimeout(resolve, ms));
+    });
+}
+function createScraperTask(apiKey, actor, input) {
+    return tslib_1.__awaiter(this, void 0, void 0, function* () {
+        return sendRequest(apiKey, pieces_common_1.HttpMethod.POST, '/api/v1/scraper/request', {
+            actor,
+            input,
+            async: true,
+        });
+    });
+}
+function getScraperTaskResult(apiKey, taskId) {
+    return tslib_1.__awaiter(this, void 0, void 0, function* () {
+        return sendRequest(apiKey, pieces_common_1.HttpMethod.GET, `/api/v1/scraper/result/${taskId}`);
+    });
+}
+function runScraperTask(apiKey, actor, input) {
+    return tslib_1.__awaiter(this, void 0, void 0, function* () {
+        var _a, _b, _c;
+        const response = yield createScraperTask(apiKey, actor, input);
+        if (response.status === 200) {
+            return (_a = response.body) !== null && _a !== void 0 ? _a : null;
+        }
+        const taskId = (_b = response.body) === null || _b === void 0 ? void 0 : _b.taskId;
+        // eslint-disable-next-line no-constant-condition
+        while (true) {
+            yield sleep(1000);
+            const result = yield getScraperTaskResult(apiKey, taskId);
+            if (result.status === 200) {
+                return (_c = result.body) !== null && _c !== void 0 ? _c : null;
+            }
+        }
+    });
+}
+function universalScrape(apiKey, request) {
+    return tslib_1.__awaiter(this, void 0, void 0, function* () {
+        var _a;
+        const response = yield sendRequest(apiKey, pieces_common_1.HttpMethod.POST, '/api/v2/unlocker/request', request);
+        return (_a = response.body) !== null && _a !== void 0 ? _a : null;
+    });
+}
+function crawlScrapeUrl(apiKey, url, params) {
+    return tslib_1.__awaiter(this, void 0, void 0, function* () {
+        var _a, _b;
+        const start = yield sendRequest(apiKey, pieces_common_1.HttpMethod.POST, '/api/v1/crawler/scrape', Object.assign({ url }, params));
+        const id = (_a = start.body) === null || _a === void 0 ? void 0 : _a.id;
+        if (!id) {
+            throw new Error('Failed to start a scrape job');
+        }
+        // eslint-disable-next-line no-constant-condition
+        while (true) {
+            const status = yield sendRequest(apiKey, pieces_common_1.HttpMethod.GET, `/api/v1/crawler/scrape/${id}`);
+            if (((_b = status.body) === null || _b === void 0 ? void 0 : _b.status) !== 'scraping') {
+                return status.body;
+            }
+            yield sleep(2000);
+        }
+    });
+}
+function crawlCrawlUrl(apiKey, url, params) {
+    return tslib_1.__awaiter(this, void 0, void 0, function* () {
+        var _a, _b, _c, _d, _e, _f;
+        const start = yield sendRequest(apiKey, pieces_common_1.HttpMethod.POST, '/api/v1/crawler/crawl', Object.assign({ url }, params));
+        const id = (_a = start.body) === null || _a === void 0 ? void 0 : _a.id;
+        if (!id) {
+            throw new Error('Failed to start a crawl job');
+        }
+        let pollInterval = 2000;
+        // eslint-disable-next-line no-constant-condition
+        while (true) {
+            let status = yield sendRequest(apiKey, pieces_common_1.HttpMethod.GET, `/api/v1/crawler/crawl/${id}`);
+            const state = (_b = status.body) === null || _b === void 0 ? void 0 : _b.status;
+            if (state === 'completed') {
+                let data = (_d = (_c = status.body) === null || _c === void 0 ? void 0 : _c.data) !== null && _d !== void 0 ? _d : [];
+                while (typeof status.body === 'object' &&
+                    status.body !== null &&
+                    'next' in status.body &&
+                    status.body.next) {
+                    if (!Array.isArray(data) || data.length === 0)
+                        break;
+                    status = yield sendRequest(apiKey, pieces_common_1.HttpMethod.GET, status.body.next);
+                    data = data.concat((_f = (_e = status.body) === null || _e === void 0 ? void 0 : _e.data) !== null && _f !== void 0 ? _f : []);
+                }
+                return Object.assign(Object.assign({}, status.body), { status: 'completed', data });
+            }
+            else if (['active', 'paused', 'pending', 'queued', 'waiting', 'scraping'].includes(state)) {
+                yield sleep(pollInterval);
+                pollInterval = Math.max(pollInterval, 2000);
+            }
+            else {
+                throw new Error(`Crawl job failed or was stopped. Status: ${state}`);
+            }
+        }
+    });
+}
+exports.scrapelessCommon = {
+    runScraperTask,
+    universalScrape,
+    crawlScrapeUrl,
+    crawlCrawlUrl,
+};
+//# sourceMappingURL=client.js.map

package/src/lib/common/client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.js","sourceRoot":"","sources":["../../../../../../../../packages/pieces/community/scrapeless/src/lib/common/client.ts"],"names":[],"mappings":";;;;AAAA,4DAKkC;AAErB,QAAA,mBAAmB,GAAG,4BAA4B,CAAC;AAEhE,SAAe,WAAW,CACxB,MAAc,EACd,MAAkB,EAClB,IAAY,EACZ,IAAc,EACd,WAAyB;;QAEzB,MAAM,OAAO,iCACX,MAAM,EACN,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,2BAAmB,GAAG,IAAI,EAAE,EACrE,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,MAAM;aACtB,IACE,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,GACpC,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CACtD,CAAC;QACF,MAAM,QAAQ,GAAG,MAAM,0BAAU,CAAC,WAAW,CAAI,OAAO,CAAC,CAAC;QAC1D,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC;IAC1D,CAAC;CAAA;AAED,SAAe,KAAK,CAAC,EAAU;;QAC7B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IAC3D,CAAC;CAAA;AAED,SAAe,iBAAiB,CAC9B,MAAc,EACd,KAAa,EACb,KAA8B;;QAE9B,OAAO,WAAW,CAAM,MAAM,EAAE,0BAAU,CAAC,IAAI,EAAE,yBAAyB,EAAE;YAC1E,KAAK;YACL,KAAK;YACL,KAAK,EAAE,IAAI;SACZ,CAAC,CAAC;IACL,CAAC;CAAA;AAED,SAAe,oBAAoB,CACjC,MAAc,EACd,MAAc;;QAEd,OAAO,WAAW,CAChB,MAAM,EACN,0BAAU,CAAC,GAAG,EACd,0BAA0B,MAAM,EAAE,CACnC,CAAC;IACJ,CAAC;CAAA;AAED,SAAe,cAAc,CAC3B,MAAc,EACd,KAAa,EACb,KAA8B;;;QAE9B,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;QAC/D,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC5B,OAAO,MAAA,QAAQ,CAAC,IAAI,mCAAI,IAAI,CAAC;QAC/B,CAAC;QACD,MAAM,MAAM,GAAG,MAAA,QAAQ,CAAC,IAAI,0CAAE,MAAM,CAAC;QACrC,iDAAiD;QACjD,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;YAClB,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YAC1D,IAAI,MAAM,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC1B,OAAO,MAAA,MAAM,CAAC,IAAI,mCAAI,IAAI,CAAC;YAC7B,CAAC;QACH,CAAC;IACH,CAAC;CAAA;AAED,SAAe,eAAe,CAC5B,MAAc,EACd,OAAgC;;;QAEhC,MAAM,QAAQ,GAAG,MAAM,WAAW,CAChC,MAAM,EACN,0BAAU,CAAC,IAAI,EACf,0BAA0B,EAC1B,OAAO,CACR,CAAC;QACF,OAAO,MAAA,QAAQ,CAAC,IAAI,mCAAI,IAAI,CAAC;IAC/B,CAAC;CAAA;AAED,SAAe,cAAc,CAC3B,MAAc,EACd,GAAW,EACX,MAA+B;;;QAE/B,MAAM,KAAK,GAAG,MAAM,WAAW,CAC7B,MAAM,EACN,0BAAU,CAAC,IAAI,EACf,wBAAwB,kBACtB,GAAG,IAAK,MAAM,EACjB,CAAC;QACF,MAAM,EAAE,GAAG,MAAA,KAAK,CAAC,IAAI,0CAAE,EAAE,CAAC;QAC1B,IAAI,CAAC,EAAE,EAAE,CAAC;YACR,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAClD,CAAC;QACD,iDAAiD;QACjD,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,MAAM,WAAW,CAC9B,MAAM,EACN,0BAAU,CAAC,GAAG,EACd,0BAA0B,EAAE,EAAE,CAC/B,CAAC;YACF,IAAI,CAAA,MAAA,MAAM,CAAC,IAAI,0CAAE,MAAM,MAAK,UAAU,EAAE,CAAC;gBACvC,OAAO,MAAM,CAAC,IAAI,CAAC;YACrB,CAAC;YACD,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;CAAA;AAED,SAAe,aAAa,CAC1B,MAAc,EACd,GAAW,EACX,MAA+B;;;QAE/B,MAAM,KAAK,GAAG,MAAM,WAAW,CAC7B,MAAM,EACN,0BAAU,CAAC,IAAI,EACf,uBAAuB,kBACrB,GAAG,IAAK,MAAM,EACjB,CAAC;QACF,MAAM,EAAE,GAAG,MAAA,KAAK,CAAC,IAAI,0CAAE,EAAE,CAAC;QAC1B,IAAI,CAAC,EAAE,EAAE,CAAC;YACR,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACjD,CAAC;QACD,IAAI,YAAY,GAAG,IAAI,CAAC;QACxB,iDAAiD;QACjD,OAAO,IAAI,EAAE,CAAC;YACZ,IAAI,MAAM,GAAG,MAAM,WAAW,CAC5B,MAAM,EACN,0BAAU,CAAC,GAAG,EACd,yBAAyB,EAAE,EAAE,CAC9B,CAAC;YACF,MAAM,KAAK,GAAG,MAAA,MAAM,CAAC,IAAI,0CAAE,MAAM,CAAC;YAClC,IAAI,KAAK,KAAK,WAAW,EAAE,CAAC;gBAC1B,IAAI,IAAI,GAAG,MAAA,MAAA,MAAM,CAAC,IAAI,0CAAE,IAAI,mCAAI,EAAE,CAAC;gBACnC,OACE,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ;oBAC/B,MAAM,CAAC,IAAI,KAAK,IAAI;oBACpB,MAAM,IAAI,MAAM,CAAC,IAAI;oBACrB,MAAM,CAAC,IAAI,CAAC,IAAI,EAChB,CAAC;oBACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;wBAAE,MAAM;oBACrD,MAAM,GAAG,MAAM,WAAW,CAAM,MAAM,EAAE,0BAAU,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAC1E,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,MAAA,MAAA,MAAM,CAAC,IAAI,0CAAE,IAAI,mCAAI,EAAE,CAAC,CAAC;gBAC9C,CAAC;gBACD,uCAAY,MAAM,CAAC,IAAI,KAAE,MAAM,EAAE,WAAW,EAAE,IAAI,IAAG;YACvD,CAAC;iBAAM,IACL,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC,QAAQ,CACvE,KAAK,CACN,EACD,CAAC;gBACD,MAAM,KAAK,CAAC,YAAY,CAAC,CAAC;gBAC1B,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;YAC9C,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,KAAK,CAAC,4CAA4C,KAAK,EAAE,CAAC,CAAC;YACvE,CAAC;QACH,CAAC;IACH,CAAC;CAAA;AAEY,QAAA,gBAAgB,GAAG;IAC9B,cAAc;IACd,eAAe;IACf,cAAc;IACd,aAAa;CACd,CAAC"}

package/src/lib/constants/index.d.ts

@@ -0,0 +1,8 @@
+export declare const googleTrendsDataTypeOptions: {
+    label: string;
+    value: string;
+}[];
+export declare const proxyCountryOptions: {
+    label: string;
+    value: string;
+}[];