@activepieces/piece-scrapegrapghai 0.0.1

package/README.md

@@ -0,0 +1,33 @@
+# ScrapeGraphAI
+
+## Description
+ScrapeGraphAI is a powerful web scraping and content extraction API. This piece enables integration with ScrapeGraphAI's API to perform smart scraping, local scraping, and markdown conversion.
+
+## Actions
+1. **Smart Scraper**: Advanced web scraping with AI-powered content extraction
+   - Automatic content detection
+   - Clean HTML output
+   - Support for dynamic websites
+   - Configurable options for content types
+
+2. **Local Scraper**: Fast and lightweight web scraping
+   - Basic content extraction
+   - Static website support
+   - Configurable selectors
+   - Resource-efficient
+
+3. **Markdownify**: Convert web content to clean Markdown
+   - Clean and formatted markdown output
+   - Preserves content structure
+   - Handles various content types
+   - Customizable output options
+
+## Authentication
+This piece requires an API key from ScrapeGraphAI. To obtain your API key:
+1. Visit https://scrapegraphai.com
+2. Sign up for an account
+3. Navigate to your dashboard
+4. Copy your API key
+
+## Requirements
+- ScrapeGraphAI API Key

package/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "@activepieces/piece-scrapegrapghai",
+  "version": "0.0.1",
+  "dependencies": {
+    "@anthropic-ai/sdk": "0.33.1",
+    "@sinclair/typebox": "0.34.11",
+    "axios": "1.7.8",
+    "axios-retry": "4.4.1",
+    "deepmerge-ts": "7.1.0",
+    "mime-types": "2.1.35",
+    "nanoid": "3.3.8",
+    "openai": "4.67.1",
+    "replicate": "0.34.1",
+    "semver": "7.6.0",
+    "zod": "3.23.8",
+    "@activepieces/pieces-common": "0.3.0",
+    "@activepieces/pieces-framework": "0.7.42",
+    "@activepieces/shared": "0.10.140",
+    "tslib": "1.14.1"
+  },
+  "overrides": {
+    "cross-spawn": "7.0.6",
+    "@tryfabric/martian": {
+      "@notionhq/client": "$@notionhq/client"
+    },
+    "vite": {
+      "rollup": "npm:@rollup/wasm-node"
+    }
+  },
+  "resolutions": {
+    "rollup": "npm:@rollup/wasm-node"
+  },
+  "main": "./src/index.js",
+  "type": "commonjs",
+  "types": "./src/index.d.ts"
+}

package/src/index.d.ts

@@ -0,0 +1,2 @@
+export declare const scrapegraphaiAuth: import("@activepieces/pieces-framework").SecretTextProperty<true>;
+export declare const scrapegraphai: import("@activepieces/pieces-framework").Piece<import("@activepieces/pieces-framework").SecretTextProperty<true>>;

package/src/index.js

@@ -0,0 +1,72 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.scrapegraphai = exports.scrapegraphaiAuth = void 0;
+const tslib_1 = require("tslib");
+const pieces_common_1 = require("@activepieces/pieces-common");
+const pieces_framework_1 = require("@activepieces/pieces-framework");
+const shared_1 = require("@activepieces/shared");
+const smart_scraper_1 = require("./lib/actions/smart-scraper");
+const local_scraper_1 = require("./lib/actions/local-scraper");
+const markdownify_1 = require("./lib/actions/markdownify");
+const markdownDescription = `
+Follow these steps to obtain your ScrapeGraphAI API Key:
+
+1. Visit [ScrapeGraphAI](https://scrapegraphai.com) and create an account.
+2. Log in and navigate to your dashboard.
+3. Locate and copy your API key from the dashboard.
+`;
+exports.scrapegraphaiAuth = pieces_framework_1.PieceAuth.SecretText({
+    description: markdownDescription,
+    displayName: 'API Key',
+    required: true,
+    validate: (_a) => tslib_1.__awaiter(void 0, [_a], void 0, function* ({ auth }) {
+        try {
+            yield pieces_common_1.httpClient.sendRequest({
+                method: pieces_common_1.HttpMethod.POST,
+                url: 'https://api.scrapegraphai.com/v1/smartscraper',
+                headers: {
+                    'Content-Type': 'application/json',
+                    'SGAI-APIKEY': auth,
+                },
+                body: {
+                    user_prompt: 'test',
+                    website_url: 'https://www.example.com',
+                },
+            });
+            return {
+                valid: true,
+            };
+        }
+        catch (e) {
+            return {
+                valid: false,
+                error: 'Invalid API Key',
+            };
+        }
+    }),
+});
+exports.scrapegraphai = (0, pieces_framework_1.createPiece)({
+    displayName: 'ScrapeGraphAI',
+    description: 'AI-powered web scraping and content extraction.',
+    minimumSupportedRelease: '0.30.0',
+    logoUrl: 'https://cdn.activepieces.com/pieces/scrapegraphai.jpg',
+    categories: [shared_1.PieceCategory.ARTIFICIAL_INTELLIGENCE],
+    authors: ["OsamaHaikal"],
+    auth: exports.scrapegraphaiAuth,
+    actions: [
+        smart_scraper_1.smartScraper,
+        local_scraper_1.localScraper,
+        markdownify_1.markdownify,
+        (0, pieces_common_1.createCustomApiCallAction)({
+            baseUrl: () => 'https://api.scrapegraphai.com/v1',
+            auth: exports.scrapegraphaiAuth,
+            authMapping: (auth) => tslib_1.__awaiter(void 0, void 0, void 0, function* () {
+                return ({
+                    'SGAI-APIKEY': `${auth}`,
+                });
+            }),
+        }),
+    ],
+    triggers: [],
+});
+//# sourceMappingURL=index.js.map

package/src/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../../packages/pieces/community/scrapegrapghai/src/index.ts"],"names":[],"mappings":";;;;AAAA,+DAAgG;AAChG,qEAAwE;AACxE,iDAAqD;AACrD,+DAA2D;AAC3D,+DAA2D;AAC3D,2DAAwD;AAExD,MAAM,mBAAmB,GAAG;;;;;;CAM3B,CAAC;AAEW,QAAA,iBAAiB,GAAG,4BAAS,CAAC,UAAU,CAAC;IACpD,WAAW,EAAE,mBAAmB;IAChC,WAAW,EAAE,SAAS;IACtB,QAAQ,EAAE,IAAI;IACd,QAAQ,EAAE,KAAiB,EAAE,oDAAZ,EAAE,IAAI,EAAE;QACvB,IAAI,CAAC;YACH,MAAM,0BAAU,CAAC,WAAW,CAAC;gBAC3B,MAAM,EAAE,0BAAU,CAAC,IAAI;gBACvB,GAAG,EAAE,+CAA+C;gBACpD,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,IAAI;iBACpB;gBACD,IAAI,EAAE;oBACJ,WAAW,EAAE,MAAM;oBACnB,WAAW,EAAE,yBAAyB;iBACvC;aACF,CAAC,CAAC;YACH,OAAO;gBACL,KAAK,EAAE,IAAI;aACZ,CAAC;QACJ,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO;gBACL,KAAK,EAAE,KAAK;gBACZ,KAAK,EAAE,iBAAiB;aACzB,CAAC;QACJ,CAAC;IACH,CAAC,CAAA;CACF,CAAC,CAAC;AAEU,QAAA,aAAa,GAAG,IAAA,8BAAW,EAAC;IACvC,WAAW,EAAE,eAAe;IAC5B,WAAW,EAAE,iDAAiD;IAC9D,uBAAuB,EAAE,QAAQ;IACjC,OAAO,EAAE,uDAAuD;IAChE,UAAU,EAAE,CAAC,sBAAa,CAAC,uBAAuB,CAAC;IACnD,OAAO,EAAE,CAAC,aAAa,CAAC;IACxB,IAAI,EAAE,yBAAiB;IACvB,OAAO,EAAE;QACP,4BAAY;QACZ,4BAAY;QACZ,yBAAW;QACX,IAAA,yCAAyB,EAAC;YACxB,OAAO,EAAE,GAAG,EAAE,CAAC,kCAAkC;YACjD,IAAI,EAAE,yBAAiB;YACvB,WAAW,EAAE,CAAO,IAAI,EAAE,EAAE;gBAAC,OAAA,CAAC;oBAC5B,aAAa,EAAE,GAAG,IAAI,EAAE;iBACzB,CAAC,CAAA;cAAA;SACH,CAAC;KACH;IACD,QAAQ,EAAE,EAAE;CACb,CAAC,CAAC"}

package/src/lib/actions/local-scraper.d.ts

@@ -0,0 +1,5 @@
+export declare const localScraper: import("@activepieces/pieces-framework").IAction<import("@activepieces/pieces-framework").SecretTextProperty<true>, {
+    website_html: import("@activepieces/pieces-framework").LongTextProperty<true>;
+    user_prompt: import("@activepieces/pieces-framework").LongTextProperty<true>;
+    output_schema: import("@activepieces/pieces-framework").JsonProperty<false>;
+}>;

package/src/lib/actions/local-scraper.js

@@ -0,0 +1,49 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.localScraper = void 0;
+const tslib_1 = require("tslib");
+const pieces_framework_1 = require("@activepieces/pieces-framework");
+const pieces_common_1 = require("@activepieces/pieces-common");
+const index_1 = require("../../index");
+exports.localScraper = (0, pieces_framework_1.createAction)({
+    name: 'local_scraper',
+    displayName: 'Local Scraper',
+    description: 'Extract content from HTML content using AI by providing a natural language prompt.',
+    auth: index_1.scrapegraphaiAuth,
+    props: {
+        website_html: pieces_framework_1.Property.LongText({
+            displayName: 'HTML Content',
+            description: 'The HTML content to process (max 2MB).',
+            required: true,
+        }),
+        user_prompt: pieces_framework_1.Property.LongText({
+            displayName: 'Extraction Prompt',
+            description: 'Describe what information you want to extract in natural language.',
+            required: true,
+        }),
+        output_schema: pieces_framework_1.Property.Json({
+            displayName: 'Output Schema',
+            description: 'Optional schema to structure the output data.',
+            required: false,
+        }),
+    },
+    run(_a) {
+        return tslib_1.__awaiter(this, arguments, void 0, function* ({ auth, propsValue }) {
+            const response = yield pieces_common_1.httpClient.sendRequest({
+                method: pieces_common_1.HttpMethod.POST,
+                url: 'https://api.scrapegraphai.com/v1/localscraper',
+                headers: {
+                    'Content-Type': 'application/json',
+                    'SGAI-APIKEY': auth,
+                },
+                body: {
+                    website_html: propsValue.website_html,
+                    user_prompt: propsValue.user_prompt,
+                    output_schema: propsValue.output_schema,
+                },
+            });
+            return response.body;
+        });
+    },
+});
+//# sourceMappingURL=local-scraper.js.map

package/src/lib/actions/local-scraper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"local-scraper.js","sourceRoot":"","sources":["../../../../../../../../packages/pieces/community/scrapegrapghai/src/lib/actions/local-scraper.ts"],"names":[],"mappings":";;;;AAAA,qEAAwE;AACxE,+DAAqE;AACrE,uCAAgD;AAEnC,QAAA,YAAY,GAAG,IAAA,+BAAY,EAAC;IACvC,IAAI,EAAE,eAAe;IACrB,WAAW,EAAE,eAAe;IAC5B,WAAW,EAAE,oFAAoF;IACjG,IAAI,EAAE,yBAAiB;IACvB,KAAK,EAAE;QACL,YAAY,EAAE,2BAAQ,CAAC,QAAQ,CAAC;YAC9B,WAAW,EAAE,cAAc;YAC3B,WAAW,EAAE,wCAAwC;YACrD,QAAQ,EAAE,IAAI;SACf,CAAC;QACF,WAAW,EAAE,2BAAQ,CAAC,QAAQ,CAAC;YAC7B,WAAW,EAAE,mBAAmB;YAChC,WAAW,EAAE,oEAAoE;YACjF,QAAQ,EAAE,IAAI;SACf,CAAC;QACF,aAAa,EAAE,2BAAQ,CAAC,IAAI,CAAC;YAC3B,WAAW,EAAE,eAAe;YAC5B,WAAW,EAAE,+CAA+C;YAC5D,QAAQ,EAAE,KAAK;SAChB,CAAC;KACH;IACK,GAAG;qEAAC,EAAE,IAAI,EAAE,UAAU,EAAE;YAC5B,MAAM,QAAQ,GAAG,MAAM,0BAAU,CAAC,WAAW,CAAC;gBAC5C,MAAM,EAAE,0BAAU,CAAC,IAAI;gBACvB,GAAG,EAAE,+CAA+C;gBACpD,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,IAAI;iBACpB;gBACD,IAAI,EAAE;oBACJ,YAAY,EAAE,UAAU,CAAC,YAAY;oBACrC,WAAW,EAAE,UAAU,CAAC,WAAW;oBACnC,aAAa,EAAE,UAAU,CAAC,aAAa;iBACxC;aACF,CAAC,CAAC;YAEH,OAAO,QAAQ,CAAC,IAAI,CAAC;QACvB,CAAC;KAAA;CACF,CAAC,CAAC"}

package/src/lib/actions/markdownify.d.ts

@@ -0,0 +1,3 @@
+export declare const markdownify: import("@activepieces/pieces-framework").IAction<import("@activepieces/pieces-framework").SecretTextProperty<true>, {
+    website_url: import("@activepieces/pieces-framework").ShortTextProperty<true>;
+}>;

package/src/lib/actions/markdownify.js

@@ -0,0 +1,37 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.markdownify = void 0;
+const tslib_1 = require("tslib");
+const pieces_framework_1 = require("@activepieces/pieces-framework");
+const pieces_common_1 = require("@activepieces/pieces-common");
+const index_1 = require("../../index");
+exports.markdownify = (0, pieces_framework_1.createAction)({
+    name: 'markdownify',
+    displayName: 'Convert to Markdown',
+    description: 'Convert any webpage into clean, readable Markdown format.',
+    auth: index_1.scrapegraphaiAuth,
+    props: {
+        website_url: pieces_framework_1.Property.ShortText({
+            displayName: 'Website URL',
+            description: 'The webpage URL to convert to Markdown',
+            required: true,
+        }),
+    },
+    run(_a) {
+        return tslib_1.__awaiter(this, arguments, void 0, function* ({ auth, propsValue }) {
+            const response = yield pieces_common_1.httpClient.sendRequest({
+                method: pieces_common_1.HttpMethod.POST,
+                url: 'https://api.scrapegraphai.com/v1/markdownify',
+                headers: {
+                    'Content-Type': 'application/json',
+                    'SGAI-APIKEY': auth,
+                },
+                body: {
+                    website_url: propsValue.website_url,
+                },
+            });
+            return response.body;
+        });
+    },
+});
+//# sourceMappingURL=markdownify.js.map

package/src/lib/actions/markdownify.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"markdownify.js","sourceRoot":"","sources":["../../../../../../../../packages/pieces/community/scrapegrapghai/src/lib/actions/markdownify.ts"],"names":[],"mappings":";;;;AAAA,qEAAwE;AACxE,+DAAqE;AACrE,uCAAgD;AAEnC,QAAA,WAAW,GAAG,IAAA,+BAAY,EAAC;IACtC,IAAI,EAAE,aAAa;IACnB,WAAW,EAAE,qBAAqB;IAClC,WAAW,EAAE,2DAA2D;IACxE,IAAI,EAAE,yBAAiB;IACvB,KAAK,EAAE;QACL,WAAW,EAAE,2BAAQ,CAAC,SAAS,CAAC;YAC9B,WAAW,EAAE,aAAa;YAC1B,WAAW,EAAE,wCAAwC;YACrD,QAAQ,EAAE,IAAI;SACf,CAAC;KACH;IACK,GAAG;qEAAC,EAAE,IAAI,EAAE,UAAU,EAAE;YAC5B,MAAM,QAAQ,GAAG,MAAM,0BAAU,CAAC,WAAW,CAAC;gBAC5C,MAAM,EAAE,0BAAU,CAAC,IAAI;gBACvB,GAAG,EAAE,8CAA8C;gBACnD,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,IAAI;iBACpB;gBACD,IAAI,EAAE;oBACJ,WAAW,EAAE,UAAU,CAAC,WAAW;iBACpC;aACF,CAAC,CAAC;YAEH,OAAO,QAAQ,CAAC,IAAI,CAAC;QACvB,CAAC;KAAA;CACF,CAAC,CAAC"}

package/src/lib/actions/smart-scraper.d.ts

@@ -0,0 +1,5 @@
+export declare const smartScraper: import("@activepieces/pieces-framework").IAction<import("@activepieces/pieces-framework").SecretTextProperty<true>, {
+    website_url: import("@activepieces/pieces-framework").ShortTextProperty<true>;
+    user_prompt: import("@activepieces/pieces-framework").LongTextProperty<true>;
+    output_schema: import("@activepieces/pieces-framework").JsonProperty<false>;
+}>;

package/src/lib/actions/smart-scraper.js

@@ -0,0 +1,49 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.smartScraper = void 0;
+const tslib_1 = require("tslib");
+const pieces_framework_1 = require("@activepieces/pieces-framework");
+const pieces_common_1 = require("@activepieces/pieces-common");
+const index_1 = require("../../index");
+exports.smartScraper = (0, pieces_framework_1.createAction)({
+    name: 'smart_scraper',
+    displayName: 'Smart Scraper',
+    description: 'Extract content from a webpage using AI by providing a natural language prompt.',
+    auth: index_1.scrapegraphaiAuth,
+    props: {
+        website_url: pieces_framework_1.Property.ShortText({
+            displayName: 'Website URL',
+            description: 'The webpage URL to scrape.',
+            required: true,
+        }),
+        user_prompt: pieces_framework_1.Property.LongText({
+            displayName: 'Extraction Prompt',
+            description: 'Describe what information you want to extract in natural language.',
+            required: true,
+        }),
+        output_schema: pieces_framework_1.Property.Json({
+            displayName: 'Output Schema',
+            description: 'Optional schema to structure the output data.',
+            required: false,
+        }),
+    },
+    run(_a) {
+        return tslib_1.__awaiter(this, arguments, void 0, function* ({ auth, propsValue }) {
+            const response = yield pieces_common_1.httpClient.sendRequest({
+                method: pieces_common_1.HttpMethod.POST,
+                url: 'https://api.scrapegraphai.com/v1/smartscraper',
+                headers: {
+                    'Content-Type': 'application/json',
+                    'SGAI-APIKEY': auth,
+                },
+                body: {
+                    website_url: propsValue.website_url,
+                    user_prompt: propsValue.user_prompt,
+                    output_schema: propsValue.output_schema,
+                },
+            });
+            return response.body;
+        });
+    },
+});
+//# sourceMappingURL=smart-scraper.js.map

package/src/lib/actions/smart-scraper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"smart-scraper.js","sourceRoot":"","sources":["../../../../../../../../packages/pieces/community/scrapegrapghai/src/lib/actions/smart-scraper.ts"],"names":[],"mappings":";;;;AAAA,qEAAwE;AACxE,+DAAqE;AACrE,uCAAgD;AAEnC,QAAA,YAAY,GAAG,IAAA,+BAAY,EAAC;IACvC,IAAI,EAAE,eAAe;IACrB,WAAW,EAAE,eAAe;IAC5B,WAAW,EAAE,iFAAiF;IAC9F,IAAI,EAAE,yBAAiB;IACvB,KAAK,EAAE;QACL,WAAW,EAAE,2BAAQ,CAAC,SAAS,CAAC;YAC9B,WAAW,EAAE,aAAa;YAC1B,WAAW,EAAE,4BAA4B;YACzC,QAAQ,EAAE,IAAI;SACf,CAAC;QACF,WAAW,EAAE,2BAAQ,CAAC,QAAQ,CAAC;YAC7B,WAAW,EAAE,mBAAmB;YAChC,WAAW,EAAE,oEAAoE;YACjF,QAAQ,EAAE,IAAI;SACf,CAAC;QACF,aAAa,EAAE,2BAAQ,CAAC,IAAI,CAAC;YAC3B,WAAW,EAAE,eAAe;YAC5B,WAAW,EAAE,+CAA+C;YAC5D,QAAQ,EAAE,KAAK;SAChB,CAAC;KACH;IACK,GAAG;qEAAC,EAAE,IAAI,EAAE,UAAU,EAAE;YAC5B,MAAM,QAAQ,GAAG,MAAM,0BAAU,CAAC,WAAW,CAAC;gBAC5C,MAAM,EAAE,0BAAU,CAAC,IAAI;gBACvB,GAAG,EAAE,+CAA+C;gBACpD,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,IAAI;iBACpB;gBACD,IAAI,EAAE;oBACJ,WAAW,EAAE,UAAU,CAAC,WAAW;oBACnC,WAAW,EAAE,UAAU,CAAC,WAAW;oBACnC,aAAa,EAAE,UAAU,CAAC,aAAa;iBACxC;aACF,CAAC,CAAC;YAEH,OAAO,QAAQ,CAAC,IAAI,CAAC;QACvB,CAAC;KAAA;CACF,CAAC,CAAC"}