@ever-works/firecrawl-plugin 1.0.0 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +282 -0
- package/dist/index.d.cts +28 -0
- package/dist/index.d.ts +28 -0
- package/dist/index.js +273 -0
- package/package.json +90 -77
- package/LICENSE +0 -661
- package/src/__tests__/firecrawl.plugin.spec.ts +0 -291
- package/src/firecrawl.plugin.ts +0 -336
- package/src/index.ts +0 -2
- package/tsconfig.json +0 -22
- package/tsup.config.ts +0 -12
package/dist/index.cjs
ADDED
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
Object.defineProperty(exports, '__esModule', { value: true });
|
|
4
|
+
|
|
5
|
+
var FirecrawlApp = require('@mendable/firecrawl-js');
|
|
6
|
+
|
|
7
|
+
function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
|
|
8
|
+
|
|
9
|
+
var FirecrawlApp__default = /*#__PURE__*/_interopDefault(FirecrawlApp);
|
|
10
|
+
|
|
11
|
+
// src/firecrawl.plugin.ts
|
|
12
|
+
var API_KEY_ERROR = "Firecrawl API key not configured. Set it in plugin settings or via PLUGIN_FIRECRAWL_API_KEY environment variable.";
|
|
13
|
+
var FirecrawlPlugin = class {
|
|
14
|
+
id = "firecrawl";
|
|
15
|
+
name = "Firecrawl";
|
|
16
|
+
version = "1.0.0";
|
|
17
|
+
category = "search";
|
|
18
|
+
capabilities = ["search", "content-extractor"];
|
|
19
|
+
providerName = "Firecrawl";
|
|
20
|
+
settingsSchema = {
|
|
21
|
+
type: "object",
|
|
22
|
+
properties: {
|
|
23
|
+
apiKey: {
|
|
24
|
+
type: "string",
|
|
25
|
+
title: "API Key",
|
|
26
|
+
description: "Your Firecrawl API key. Get one at https://firecrawl.dev",
|
|
27
|
+
"x-secret": true,
|
|
28
|
+
"x-envVar": "PLUGIN_FIRECRAWL_API_KEY",
|
|
29
|
+
"x-scope": "user"
|
|
30
|
+
}
|
|
31
|
+
},
|
|
32
|
+
required: ["apiKey"]
|
|
33
|
+
};
|
|
34
|
+
configurationMode = "hybrid";
|
|
35
|
+
context;
|
|
36
|
+
// ============================================================================
|
|
37
|
+
// ISearchPlugin Interface
|
|
38
|
+
// ============================================================================
|
|
39
|
+
async search(options) {
|
|
40
|
+
const client = this.getClient(options.settings);
|
|
41
|
+
const startTime = Date.now();
|
|
42
|
+
try {
|
|
43
|
+
const response = await client.search(options.query, {
|
|
44
|
+
limit: options.limit
|
|
45
|
+
});
|
|
46
|
+
const items = response.web || [];
|
|
47
|
+
const results = items.map((item, index) => {
|
|
48
|
+
const url = ("url" in item ? item.url : "") || "";
|
|
49
|
+
const title = ("title" in item ? item.title : "") || "";
|
|
50
|
+
const snippet = ("description" in item ? item.description : "") || ("markdown" in item ? item.markdown : "") || "";
|
|
51
|
+
let hostname;
|
|
52
|
+
try {
|
|
53
|
+
hostname = url ? new URL(url).hostname : void 0;
|
|
54
|
+
} catch {
|
|
55
|
+
hostname = void 0;
|
|
56
|
+
}
|
|
57
|
+
return {
|
|
58
|
+
title,
|
|
59
|
+
url,
|
|
60
|
+
snippet,
|
|
61
|
+
position: index + 1,
|
|
62
|
+
source: hostname
|
|
63
|
+
};
|
|
64
|
+
});
|
|
65
|
+
return {
|
|
66
|
+
results,
|
|
67
|
+
query: options.query,
|
|
68
|
+
totalResults: results.length,
|
|
69
|
+
hasMore: false,
|
|
70
|
+
duration: Date.now() - startTime
|
|
71
|
+
};
|
|
72
|
+
} catch (error) {
|
|
73
|
+
this.context?.logger.error(
|
|
74
|
+
`Firecrawl search failed: ${error instanceof Error ? error.message : String(error)}`
|
|
75
|
+
);
|
|
76
|
+
throw error;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
async getRateLimitInfo() {
|
|
80
|
+
return {
|
|
81
|
+
remaining: -1,
|
|
82
|
+
limit: -1,
|
|
83
|
+
period: "month"
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
// ============================================================================
|
|
87
|
+
// IContentExtractorPlugin Interface
|
|
88
|
+
// ============================================================================
|
|
89
|
+
async extract(options) {
|
|
90
|
+
const startTime = Date.now();
|
|
91
|
+
try {
|
|
92
|
+
const client = this.getClient(options.settings);
|
|
93
|
+
const doc = await client.scrape(options.url, { formats: ["markdown"] });
|
|
94
|
+
const markdown = doc.markdown || "";
|
|
95
|
+
const title = doc.metadata?.title || void 0;
|
|
96
|
+
const finalUrl = doc.metadata?.url;
|
|
97
|
+
const wordCount = markdown ? markdown.split(/\s+/).filter((w) => w.length > 0).length : 0;
|
|
98
|
+
if (!markdown) {
|
|
99
|
+
return {
|
|
100
|
+
success: false,
|
|
101
|
+
url: options.url,
|
|
102
|
+
error: "No content extracted from page",
|
|
103
|
+
duration: Date.now() - startTime
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
return {
|
|
107
|
+
success: true,
|
|
108
|
+
url: options.url,
|
|
109
|
+
finalUrl: finalUrl && finalUrl !== options.url ? finalUrl : void 0,
|
|
110
|
+
title,
|
|
111
|
+
content: markdown,
|
|
112
|
+
markdown,
|
|
113
|
+
duration: Date.now() - startTime,
|
|
114
|
+
wordCount,
|
|
115
|
+
readingTime: Math.ceil(wordCount / 200)
|
|
116
|
+
};
|
|
117
|
+
} catch (error) {
|
|
118
|
+
return {
|
|
119
|
+
success: false,
|
|
120
|
+
url: options.url,
|
|
121
|
+
error: error instanceof Error ? error.message : String(error),
|
|
122
|
+
duration: Date.now() - startTime
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
async extractBatch(urls, options) {
|
|
127
|
+
const client = this.getClient(options?.settings);
|
|
128
|
+
const startTime = Date.now();
|
|
129
|
+
try {
|
|
130
|
+
const job = await client.batchScrape([...urls], { options: { formats: ["markdown"] } });
|
|
131
|
+
if (job.data && job.data.length > 0) {
|
|
132
|
+
return job.data.map((doc, index) => {
|
|
133
|
+
const markdown = doc.markdown || "";
|
|
134
|
+
const wordCount = markdown ? markdown.split(/\s+/).filter((w) => w.length > 0).length : 0;
|
|
135
|
+
const requestedUrl = urls[index] || doc.metadata?.url || "";
|
|
136
|
+
if (!markdown) {
|
|
137
|
+
return {
|
|
138
|
+
success: false,
|
|
139
|
+
url: requestedUrl,
|
|
140
|
+
error: "No content extracted from page",
|
|
141
|
+
duration: Date.now() - startTime
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
return {
|
|
145
|
+
success: true,
|
|
146
|
+
url: requestedUrl,
|
|
147
|
+
finalUrl: doc.metadata?.url && doc.metadata.url !== requestedUrl ? doc.metadata.url : void 0,
|
|
148
|
+
title: doc.metadata?.title || void 0,
|
|
149
|
+
content: markdown,
|
|
150
|
+
markdown,
|
|
151
|
+
duration: Date.now() - startTime,
|
|
152
|
+
wordCount,
|
|
153
|
+
readingTime: Math.ceil(wordCount / 200)
|
|
154
|
+
};
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
} catch {
|
|
158
|
+
}
|
|
159
|
+
const results = await Promise.allSettled(
|
|
160
|
+
urls.map(
|
|
161
|
+
(url) => this.extract({
|
|
162
|
+
url,
|
|
163
|
+
...options
|
|
164
|
+
})
|
|
165
|
+
)
|
|
166
|
+
);
|
|
167
|
+
return results.map((result, index) => {
|
|
168
|
+
if (result.status === "fulfilled") {
|
|
169
|
+
return result.value;
|
|
170
|
+
}
|
|
171
|
+
return {
|
|
172
|
+
success: false,
|
|
173
|
+
url: urls[index],
|
|
174
|
+
error: result.reason instanceof Error ? result.reason.message : String(result.reason),
|
|
175
|
+
duration: Date.now() - startTime
|
|
176
|
+
};
|
|
177
|
+
});
|
|
178
|
+
}
|
|
179
|
+
async isAvailable() {
|
|
180
|
+
if (!this.context) return false;
|
|
181
|
+
const settings = await this.context.getSettings();
|
|
182
|
+
return Boolean(settings?.apiKey);
|
|
183
|
+
}
|
|
184
|
+
async validateConnection(settings) {
|
|
185
|
+
const apiKey = settings.apiKey;
|
|
186
|
+
if (!apiKey) {
|
|
187
|
+
return { success: false, message: "Firecrawl API key is not configured." };
|
|
188
|
+
}
|
|
189
|
+
try {
|
|
190
|
+
const client = this.getClient(settings);
|
|
191
|
+
await client.search("test", { limit: 1 });
|
|
192
|
+
return { success: true, message: "Firecrawl connection verified." };
|
|
193
|
+
} catch (error) {
|
|
194
|
+
return {
|
|
195
|
+
success: false,
|
|
196
|
+
message: `Firecrawl connection failed: ${error instanceof Error ? error.message : String(error)}`
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
async canExtract(url) {
|
|
201
|
+
try {
|
|
202
|
+
const parsed = new URL(url);
|
|
203
|
+
return parsed.protocol === "http:" || parsed.protocol === "https:";
|
|
204
|
+
} catch {
|
|
205
|
+
return false;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
getSupportedFormats() {
|
|
209
|
+
return ["markdown"];
|
|
210
|
+
}
|
|
211
|
+
// ============================================================================
|
|
212
|
+
// IPlugin Lifecycle
|
|
213
|
+
// ============================================================================
|
|
214
|
+
getClient(settings) {
|
|
215
|
+
const apiKey = settings?.apiKey;
|
|
216
|
+
if (!apiKey) {
|
|
217
|
+
throw new Error(API_KEY_ERROR);
|
|
218
|
+
}
|
|
219
|
+
return new FirecrawlApp__default.default({ apiKey });
|
|
220
|
+
}
|
|
221
|
+
async onLoad(context) {
|
|
222
|
+
this.context = context;
|
|
223
|
+
context.logger.log("Firecrawl Plugin loaded");
|
|
224
|
+
}
|
|
225
|
+
async onUnload() {
|
|
226
|
+
this.context = void 0;
|
|
227
|
+
}
|
|
228
|
+
async healthCheck() {
|
|
229
|
+
return {
|
|
230
|
+
status: "healthy",
|
|
231
|
+
message: "Firecrawl plugin is ready (API key required for operations)",
|
|
232
|
+
checkedAt: Date.now()
|
|
233
|
+
};
|
|
234
|
+
}
|
|
235
|
+
getManifest() {
|
|
236
|
+
return {
|
|
237
|
+
id: this.id,
|
|
238
|
+
name: this.name,
|
|
239
|
+
version: this.version,
|
|
240
|
+
description: "Search the web and extract content from websites using the Firecrawl API",
|
|
241
|
+
category: this.category,
|
|
242
|
+
capabilities: [...this.capabilities],
|
|
243
|
+
author: { name: "Ever Works Team" },
|
|
244
|
+
license: "AGPL-3.0",
|
|
245
|
+
builtIn: true,
|
|
246
|
+
systemPlugin: false,
|
|
247
|
+
autoEnable: false,
|
|
248
|
+
readme: [
|
|
249
|
+
"## What does Firecrawl do?",
|
|
250
|
+
"",
|
|
251
|
+
"Firecrawl is a web scraping and search API that can search the web for relevant results and extract clean, well-formatted markdown content from any web page. It handles JavaScript rendering, anti-bot bypasses, and content cleaning automatically.",
|
|
252
|
+
"",
|
|
253
|
+
"## Why use it?",
|
|
254
|
+
"",
|
|
255
|
+
"- **Web search** \u2014 search the web and get structured results with content snippets",
|
|
256
|
+
"- **Clean markdown output** \u2014 returns well-structured markdown from any web page",
|
|
257
|
+
"- **JavaScript rendering** \u2014 handles dynamic/SPA pages that simple HTTP fetches miss",
|
|
258
|
+
"- **Anti-bot handling** \u2014 bypasses common protections automatically",
|
|
259
|
+
"- **Metadata extraction** \u2014 captures title, description, and other page metadata",
|
|
260
|
+
"",
|
|
261
|
+
"## How it works in Ever Works",
|
|
262
|
+
"",
|
|
263
|
+
"When enabled, Firecrawl can be used as both a search provider and content extractor during work generation. It searches the web for relevant information and extracts high-quality content from web pages, including JavaScript-heavy sites.",
|
|
264
|
+
"",
|
|
265
|
+
"## Getting started",
|
|
266
|
+
"",
|
|
267
|
+
"1. Create an account at [firecrawl.dev](https://firecrawl.dev)",
|
|
268
|
+
"2. Copy your API key from the Firecrawl dashboard",
|
|
269
|
+
"3. Enter the key in the **API Key** field below",
|
|
270
|
+
"4. Enable this plugin to use it for search and content extraction"
|
|
271
|
+
].join("\n"),
|
|
272
|
+
homepage: "https://firecrawl.dev",
|
|
273
|
+
icon: {
|
|
274
|
+
type: "svg",
|
|
275
|
+
value: `<svg fill=none height=20 viewBox="0 0 20 20"width=20 xmlns=http://www.w3.org/2000/svg><path d="M13.7605 6.61389C13.138 6.79867 12.6687 7.21667 12.3251 7.67073C12.2513 7.76819 12.0975 7.69495 12.1268 7.57552C12.7848 4.86978 11.9155 2.6209 9.20582 1.51393C9.06836 1.4576 8.92527 1.58097 8.96132 1.72519C10.1939 6.67417 5.00941 6.25673 5.66459 11.8671C5.67585 11.9634 5.56769 12.0293 5.48882 11.973C5.2432 11.7967 4.96885 11.4288 4.78069 11.1702C4.72548 11.0942 4.60605 11.1156 4.5807 11.2063C4.43085 11.7482 4.35986 12.2586 4.35986 12.7656C4.35986 14.7373 5.37333 16.473 6.90734 17.4791C6.99522 17.5366 7.10789 17.4543 7.07804 17.3535C6.99917 17.0887 6.95466 16.8093 6.95128 16.5203C6.95128 16.3429 6.96255 16.1615 6.99015 15.9925C7.05438 15.5677 7.20197 15.1632 7.44985 14.7948C8.29995 13.5188 10.0041 12.2862 9.73199 10.6125C9.71453 10.5066 9.83959 10.4368 9.91846 10.5094C11.119 11.6063 11.3567 13.0817 11.1595 14.405C11.1426 14.5199 11.2868 14.5813 11.3595 14.4912C11.5432 14.2613 11.7674 14.0596 12.0113 13.9081C12.0722 13.8703 12.1533 13.8991 12.1764 13.9667C12.3121 14.3616 12.5138 14.7323 12.7042 15.1029C12.9318 15.5485 13.0529 16.0573 13.0338 16.5958C13.0242 16.8578 12.9808 17.1113 12.9082 17.3524C12.8772 17.4543 12.9887 17.5394 13.0783 17.4808C14.6134 16.4747 15.6275 14.739 15.6275 12.7662C15.6275 12.0806 15.5075 11.4085 15.2804 10.7787C14.8044 9.45766 13.5966 8.46561 13.9019 6.74403C13.9166 6.66178 13.8405 6.59023 13.7605 6.61389Z"fill=#ff4d00 /></svg>`
|
|
276
|
+
}
|
|
277
|
+
};
|
|
278
|
+
}
|
|
279
|
+
};
|
|
280
|
+
|
|
281
|
+
exports.FirecrawlPlugin = FirecrawlPlugin;
|
|
282
|
+
exports.default = FirecrawlPlugin;
|
package/dist/index.d.cts
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { IPlugin, ISearchPlugin, IContentExtractorPlugin, PluginCategory, JsonSchema, SearchOptions, SearchResponse, RateLimitInfo, ContentExtractionOptions, ContentExtractionResult, ConnectionValidationResult, PluginContext, PluginHealthCheck, PluginManifest } from '@ever-works/plugin';
|
|
2
|
+
|
|
3
|
+
declare class FirecrawlPlugin implements IPlugin, ISearchPlugin, IContentExtractorPlugin {
|
|
4
|
+
readonly id = "firecrawl";
|
|
5
|
+
readonly name = "Firecrawl";
|
|
6
|
+
readonly version = "1.0.0";
|
|
7
|
+
readonly category: PluginCategory;
|
|
8
|
+
readonly capabilities: readonly string[];
|
|
9
|
+
readonly providerName = "Firecrawl";
|
|
10
|
+
readonly settingsSchema: JsonSchema;
|
|
11
|
+
readonly configurationMode: 'admin-only' | 'user-required' | 'hybrid';
|
|
12
|
+
private context?;
|
|
13
|
+
search(options: SearchOptions): Promise<SearchResponse>;
|
|
14
|
+
getRateLimitInfo(): Promise<RateLimitInfo>;
|
|
15
|
+
extract(options: ContentExtractionOptions): Promise<ContentExtractionResult>;
|
|
16
|
+
extractBatch(urls: readonly string[], options?: Partial<ContentExtractionOptions>): Promise<readonly ContentExtractionResult[]>;
|
|
17
|
+
isAvailable(): Promise<boolean>;
|
|
18
|
+
validateConnection(settings: Record<string, unknown>): Promise<ConnectionValidationResult>;
|
|
19
|
+
canExtract(url: string): Promise<boolean>;
|
|
20
|
+
getSupportedFormats(): readonly ('text' | 'html' | 'markdown')[];
|
|
21
|
+
private getClient;
|
|
22
|
+
onLoad(context: PluginContext): Promise<void>;
|
|
23
|
+
onUnload(): Promise<void>;
|
|
24
|
+
healthCheck(): Promise<PluginHealthCheck>;
|
|
25
|
+
getManifest(): PluginManifest;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export { FirecrawlPlugin, FirecrawlPlugin as default };
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { IPlugin, ISearchPlugin, IContentExtractorPlugin, PluginCategory, JsonSchema, SearchOptions, SearchResponse, RateLimitInfo, ContentExtractionOptions, ContentExtractionResult, ConnectionValidationResult, PluginContext, PluginHealthCheck, PluginManifest } from '@ever-works/plugin';
|
|
2
|
+
|
|
3
|
+
declare class FirecrawlPlugin implements IPlugin, ISearchPlugin, IContentExtractorPlugin {
|
|
4
|
+
readonly id = "firecrawl";
|
|
5
|
+
readonly name = "Firecrawl";
|
|
6
|
+
readonly version = "1.0.0";
|
|
7
|
+
readonly category: PluginCategory;
|
|
8
|
+
readonly capabilities: readonly string[];
|
|
9
|
+
readonly providerName = "Firecrawl";
|
|
10
|
+
readonly settingsSchema: JsonSchema;
|
|
11
|
+
readonly configurationMode: 'admin-only' | 'user-required' | 'hybrid';
|
|
12
|
+
private context?;
|
|
13
|
+
search(options: SearchOptions): Promise<SearchResponse>;
|
|
14
|
+
getRateLimitInfo(): Promise<RateLimitInfo>;
|
|
15
|
+
extract(options: ContentExtractionOptions): Promise<ContentExtractionResult>;
|
|
16
|
+
extractBatch(urls: readonly string[], options?: Partial<ContentExtractionOptions>): Promise<readonly ContentExtractionResult[]>;
|
|
17
|
+
isAvailable(): Promise<boolean>;
|
|
18
|
+
validateConnection(settings: Record<string, unknown>): Promise<ConnectionValidationResult>;
|
|
19
|
+
canExtract(url: string): Promise<boolean>;
|
|
20
|
+
getSupportedFormats(): readonly ('text' | 'html' | 'markdown')[];
|
|
21
|
+
private getClient;
|
|
22
|
+
onLoad(context: PluginContext): Promise<void>;
|
|
23
|
+
onUnload(): Promise<void>;
|
|
24
|
+
healthCheck(): Promise<PluginHealthCheck>;
|
|
25
|
+
getManifest(): PluginManifest;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export { FirecrawlPlugin, FirecrawlPlugin as default };
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
import FirecrawlApp from '@mendable/firecrawl-js';
|
|
2
|
+
|
|
3
|
+
// src/firecrawl.plugin.ts
|
|
4
|
+
var API_KEY_ERROR = "Firecrawl API key not configured. Set it in plugin settings or via PLUGIN_FIRECRAWL_API_KEY environment variable.";
|
|
5
|
+
var FirecrawlPlugin = class {
|
|
6
|
+
id = "firecrawl";
|
|
7
|
+
name = "Firecrawl";
|
|
8
|
+
version = "1.0.0";
|
|
9
|
+
category = "search";
|
|
10
|
+
capabilities = ["search", "content-extractor"];
|
|
11
|
+
providerName = "Firecrawl";
|
|
12
|
+
settingsSchema = {
|
|
13
|
+
type: "object",
|
|
14
|
+
properties: {
|
|
15
|
+
apiKey: {
|
|
16
|
+
type: "string",
|
|
17
|
+
title: "API Key",
|
|
18
|
+
description: "Your Firecrawl API key. Get one at https://firecrawl.dev",
|
|
19
|
+
"x-secret": true,
|
|
20
|
+
"x-envVar": "PLUGIN_FIRECRAWL_API_KEY",
|
|
21
|
+
"x-scope": "user"
|
|
22
|
+
}
|
|
23
|
+
},
|
|
24
|
+
required: ["apiKey"]
|
|
25
|
+
};
|
|
26
|
+
configurationMode = "hybrid";
|
|
27
|
+
context;
|
|
28
|
+
// ============================================================================
|
|
29
|
+
// ISearchPlugin Interface
|
|
30
|
+
// ============================================================================
|
|
31
|
+
async search(options) {
|
|
32
|
+
const client = this.getClient(options.settings);
|
|
33
|
+
const startTime = Date.now();
|
|
34
|
+
try {
|
|
35
|
+
const response = await client.search(options.query, {
|
|
36
|
+
limit: options.limit
|
|
37
|
+
});
|
|
38
|
+
const items = response.web || [];
|
|
39
|
+
const results = items.map((item, index) => {
|
|
40
|
+
const url = ("url" in item ? item.url : "") || "";
|
|
41
|
+
const title = ("title" in item ? item.title : "") || "";
|
|
42
|
+
const snippet = ("description" in item ? item.description : "") || ("markdown" in item ? item.markdown : "") || "";
|
|
43
|
+
let hostname;
|
|
44
|
+
try {
|
|
45
|
+
hostname = url ? new URL(url).hostname : void 0;
|
|
46
|
+
} catch {
|
|
47
|
+
hostname = void 0;
|
|
48
|
+
}
|
|
49
|
+
return {
|
|
50
|
+
title,
|
|
51
|
+
url,
|
|
52
|
+
snippet,
|
|
53
|
+
position: index + 1,
|
|
54
|
+
source: hostname
|
|
55
|
+
};
|
|
56
|
+
});
|
|
57
|
+
return {
|
|
58
|
+
results,
|
|
59
|
+
query: options.query,
|
|
60
|
+
totalResults: results.length,
|
|
61
|
+
hasMore: false,
|
|
62
|
+
duration: Date.now() - startTime
|
|
63
|
+
};
|
|
64
|
+
} catch (error) {
|
|
65
|
+
this.context?.logger.error(
|
|
66
|
+
`Firecrawl search failed: ${error instanceof Error ? error.message : String(error)}`
|
|
67
|
+
);
|
|
68
|
+
throw error;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
async getRateLimitInfo() {
|
|
72
|
+
return {
|
|
73
|
+
remaining: -1,
|
|
74
|
+
limit: -1,
|
|
75
|
+
period: "month"
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
// ============================================================================
|
|
79
|
+
// IContentExtractorPlugin Interface
|
|
80
|
+
// ============================================================================
|
|
81
|
+
async extract(options) {
|
|
82
|
+
const startTime = Date.now();
|
|
83
|
+
try {
|
|
84
|
+
const client = this.getClient(options.settings);
|
|
85
|
+
const doc = await client.scrape(options.url, { formats: ["markdown"] });
|
|
86
|
+
const markdown = doc.markdown || "";
|
|
87
|
+
const title = doc.metadata?.title || void 0;
|
|
88
|
+
const finalUrl = doc.metadata?.url;
|
|
89
|
+
const wordCount = markdown ? markdown.split(/\s+/).filter((w) => w.length > 0).length : 0;
|
|
90
|
+
if (!markdown) {
|
|
91
|
+
return {
|
|
92
|
+
success: false,
|
|
93
|
+
url: options.url,
|
|
94
|
+
error: "No content extracted from page",
|
|
95
|
+
duration: Date.now() - startTime
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
return {
|
|
99
|
+
success: true,
|
|
100
|
+
url: options.url,
|
|
101
|
+
finalUrl: finalUrl && finalUrl !== options.url ? finalUrl : void 0,
|
|
102
|
+
title,
|
|
103
|
+
content: markdown,
|
|
104
|
+
markdown,
|
|
105
|
+
duration: Date.now() - startTime,
|
|
106
|
+
wordCount,
|
|
107
|
+
readingTime: Math.ceil(wordCount / 200)
|
|
108
|
+
};
|
|
109
|
+
} catch (error) {
|
|
110
|
+
return {
|
|
111
|
+
success: false,
|
|
112
|
+
url: options.url,
|
|
113
|
+
error: error instanceof Error ? error.message : String(error),
|
|
114
|
+
duration: Date.now() - startTime
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
async extractBatch(urls, options) {
|
|
119
|
+
const client = this.getClient(options?.settings);
|
|
120
|
+
const startTime = Date.now();
|
|
121
|
+
try {
|
|
122
|
+
const job = await client.batchScrape([...urls], { options: { formats: ["markdown"] } });
|
|
123
|
+
if (job.data && job.data.length > 0) {
|
|
124
|
+
return job.data.map((doc, index) => {
|
|
125
|
+
const markdown = doc.markdown || "";
|
|
126
|
+
const wordCount = markdown ? markdown.split(/\s+/).filter((w) => w.length > 0).length : 0;
|
|
127
|
+
const requestedUrl = urls[index] || doc.metadata?.url || "";
|
|
128
|
+
if (!markdown) {
|
|
129
|
+
return {
|
|
130
|
+
success: false,
|
|
131
|
+
url: requestedUrl,
|
|
132
|
+
error: "No content extracted from page",
|
|
133
|
+
duration: Date.now() - startTime
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
return {
|
|
137
|
+
success: true,
|
|
138
|
+
url: requestedUrl,
|
|
139
|
+
finalUrl: doc.metadata?.url && doc.metadata.url !== requestedUrl ? doc.metadata.url : void 0,
|
|
140
|
+
title: doc.metadata?.title || void 0,
|
|
141
|
+
content: markdown,
|
|
142
|
+
markdown,
|
|
143
|
+
duration: Date.now() - startTime,
|
|
144
|
+
wordCount,
|
|
145
|
+
readingTime: Math.ceil(wordCount / 200)
|
|
146
|
+
};
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
} catch {
|
|
150
|
+
}
|
|
151
|
+
const results = await Promise.allSettled(
|
|
152
|
+
urls.map(
|
|
153
|
+
(url) => this.extract({
|
|
154
|
+
url,
|
|
155
|
+
...options
|
|
156
|
+
})
|
|
157
|
+
)
|
|
158
|
+
);
|
|
159
|
+
return results.map((result, index) => {
|
|
160
|
+
if (result.status === "fulfilled") {
|
|
161
|
+
return result.value;
|
|
162
|
+
}
|
|
163
|
+
return {
|
|
164
|
+
success: false,
|
|
165
|
+
url: urls[index],
|
|
166
|
+
error: result.reason instanceof Error ? result.reason.message : String(result.reason),
|
|
167
|
+
duration: Date.now() - startTime
|
|
168
|
+
};
|
|
169
|
+
});
|
|
170
|
+
}
|
|
171
|
+
async isAvailable() {
|
|
172
|
+
if (!this.context) return false;
|
|
173
|
+
const settings = await this.context.getSettings();
|
|
174
|
+
return Boolean(settings?.apiKey);
|
|
175
|
+
}
|
|
176
|
+
async validateConnection(settings) {
|
|
177
|
+
const apiKey = settings.apiKey;
|
|
178
|
+
if (!apiKey) {
|
|
179
|
+
return { success: false, message: "Firecrawl API key is not configured." };
|
|
180
|
+
}
|
|
181
|
+
try {
|
|
182
|
+
const client = this.getClient(settings);
|
|
183
|
+
await client.search("test", { limit: 1 });
|
|
184
|
+
return { success: true, message: "Firecrawl connection verified." };
|
|
185
|
+
} catch (error) {
|
|
186
|
+
return {
|
|
187
|
+
success: false,
|
|
188
|
+
message: `Firecrawl connection failed: ${error instanceof Error ? error.message : String(error)}`
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
async canExtract(url) {
|
|
193
|
+
try {
|
|
194
|
+
const parsed = new URL(url);
|
|
195
|
+
return parsed.protocol === "http:" || parsed.protocol === "https:";
|
|
196
|
+
} catch {
|
|
197
|
+
return false;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
getSupportedFormats() {
|
|
201
|
+
return ["markdown"];
|
|
202
|
+
}
|
|
203
|
+
// ============================================================================
|
|
204
|
+
// IPlugin Lifecycle
|
|
205
|
+
// ============================================================================
|
|
206
|
+
getClient(settings) {
|
|
207
|
+
const apiKey = settings?.apiKey;
|
|
208
|
+
if (!apiKey) {
|
|
209
|
+
throw new Error(API_KEY_ERROR);
|
|
210
|
+
}
|
|
211
|
+
return new FirecrawlApp({ apiKey });
|
|
212
|
+
}
|
|
213
|
+
async onLoad(context) {
|
|
214
|
+
this.context = context;
|
|
215
|
+
context.logger.log("Firecrawl Plugin loaded");
|
|
216
|
+
}
|
|
217
|
+
async onUnload() {
|
|
218
|
+
this.context = void 0;
|
|
219
|
+
}
|
|
220
|
+
async healthCheck() {
|
|
221
|
+
return {
|
|
222
|
+
status: "healthy",
|
|
223
|
+
message: "Firecrawl plugin is ready (API key required for operations)",
|
|
224
|
+
checkedAt: Date.now()
|
|
225
|
+
};
|
|
226
|
+
}
|
|
227
|
+
getManifest() {
|
|
228
|
+
return {
|
|
229
|
+
id: this.id,
|
|
230
|
+
name: this.name,
|
|
231
|
+
version: this.version,
|
|
232
|
+
description: "Search the web and extract content from websites using the Firecrawl API",
|
|
233
|
+
category: this.category,
|
|
234
|
+
capabilities: [...this.capabilities],
|
|
235
|
+
author: { name: "Ever Works Team" },
|
|
236
|
+
license: "AGPL-3.0",
|
|
237
|
+
builtIn: true,
|
|
238
|
+
systemPlugin: false,
|
|
239
|
+
autoEnable: false,
|
|
240
|
+
readme: [
|
|
241
|
+
"## What does Firecrawl do?",
|
|
242
|
+
"",
|
|
243
|
+
"Firecrawl is a web scraping and search API that can search the web for relevant results and extract clean, well-formatted markdown content from any web page. It handles JavaScript rendering, anti-bot bypasses, and content cleaning automatically.",
|
|
244
|
+
"",
|
|
245
|
+
"## Why use it?",
|
|
246
|
+
"",
|
|
247
|
+
"- **Web search** \u2014 search the web and get structured results with content snippets",
|
|
248
|
+
"- **Clean markdown output** \u2014 returns well-structured markdown from any web page",
|
|
249
|
+
"- **JavaScript rendering** \u2014 handles dynamic/SPA pages that simple HTTP fetches miss",
|
|
250
|
+
"- **Anti-bot handling** \u2014 bypasses common protections automatically",
|
|
251
|
+
"- **Metadata extraction** \u2014 captures title, description, and other page metadata",
|
|
252
|
+
"",
|
|
253
|
+
"## How it works in Ever Works",
|
|
254
|
+
"",
|
|
255
|
+
"When enabled, Firecrawl can be used as both a search provider and content extractor during work generation. It searches the web for relevant information and extracts high-quality content from web pages, including JavaScript-heavy sites.",
|
|
256
|
+
"",
|
|
257
|
+
"## Getting started",
|
|
258
|
+
"",
|
|
259
|
+
"1. Create an account at [firecrawl.dev](https://firecrawl.dev)",
|
|
260
|
+
"2. Copy your API key from the Firecrawl dashboard",
|
|
261
|
+
"3. Enter the key in the **API Key** field below",
|
|
262
|
+
"4. Enable this plugin to use it for search and content extraction"
|
|
263
|
+
].join("\n"),
|
|
264
|
+
homepage: "https://firecrawl.dev",
|
|
265
|
+
icon: {
|
|
266
|
+
type: "svg",
|
|
267
|
+
value: `<svg fill=none height=20 viewBox="0 0 20 20"width=20 xmlns=http://www.w3.org/2000/svg><path d="M13.7605 6.61389C13.138 6.79867 12.6687 7.21667 12.3251 7.67073C12.2513 7.76819 12.0975 7.69495 12.1268 7.57552C12.7848 4.86978 11.9155 2.6209 9.20582 1.51393C9.06836 1.4576 8.92527 1.58097 8.96132 1.72519C10.1939 6.67417 5.00941 6.25673 5.66459 11.8671C5.67585 11.9634 5.56769 12.0293 5.48882 11.973C5.2432 11.7967 4.96885 11.4288 4.78069 11.1702C4.72548 11.0942 4.60605 11.1156 4.5807 11.2063C4.43085 11.7482 4.35986 12.2586 4.35986 12.7656C4.35986 14.7373 5.37333 16.473 6.90734 17.4791C6.99522 17.5366 7.10789 17.4543 7.07804 17.3535C6.99917 17.0887 6.95466 16.8093 6.95128 16.5203C6.95128 16.3429 6.96255 16.1615 6.99015 15.9925C7.05438 15.5677 7.20197 15.1632 7.44985 14.7948C8.29995 13.5188 10.0041 12.2862 9.73199 10.6125C9.71453 10.5066 9.83959 10.4368 9.91846 10.5094C11.119 11.6063 11.3567 13.0817 11.1595 14.405C11.1426 14.5199 11.2868 14.5813 11.3595 14.4912C11.5432 14.2613 11.7674 14.0596 12.0113 13.9081C12.0722 13.8703 12.1533 13.8991 12.1764 13.9667C12.3121 14.3616 12.5138 14.7323 12.7042 15.1029C12.9318 15.5485 13.0529 16.0573 13.0338 16.5958C13.0242 16.8578 12.9808 17.1113 12.9082 17.3524C12.8772 17.4543 12.9887 17.5394 13.0783 17.4808C14.6134 16.4747 15.6275 14.739 15.6275 12.7662C15.6275 12.0806 15.5075 11.4085 15.2804 10.7787C14.8044 9.45766 13.5966 8.46561 13.9019 6.74403C13.9166 6.66178 13.8405 6.59023 13.7605 6.61389Z"fill=#ff4d00 /></svg>`
|
|
268
|
+
}
|
|
269
|
+
};
|
|
270
|
+
}
|
|
271
|
+
};
|
|
272
|
+
|
|
273
|
+
export { FirecrawlPlugin, FirecrawlPlugin as default };
|