firecrawl-sdk 1.10.0 → 1.17.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +40 -1
- data/lib/firecrawl/client.rb +74 -0
- data/lib/firecrawl/models/agent_exchange_options.rb +68 -0
- data/lib/firecrawl/models/agent_exchange_summary.rb +72 -0
- data/lib/firecrawl/models/agent_list_response.rb +21 -0
- data/lib/firecrawl/models/agent_options.rb +16 -1
- data/lib/firecrawl/models/agent_pending_approval.rb +64 -0
- data/lib/firecrawl/models/agent_response.rb +4 -1
- data/lib/firecrawl/models/agent_snapshot_response.rb +22 -0
- data/lib/firecrawl/models/agent_status_response.rb +12 -1
- data/lib/firecrawl/models/agent_trace_response.rb +28 -0
- data/lib/firecrawl/models/audit_metadata.rb +20 -0
- data/lib/firecrawl/models/developer_search_options.rb +37 -0
- data/lib/firecrawl/models/developer_search_response.rb +76 -0
- data/lib/firecrawl/models/document.rb +4 -1
- data/lib/firecrawl/models/map_options.rb +5 -1
- data/lib/firecrawl/models/parse_format.rb +36 -0
- data/lib/firecrawl/models/parse_options.rb +13 -2
- data/lib/firecrawl/models/pdf_parser.rb +33 -0
- data/lib/firecrawl/models/query_format.rb +27 -0
- data/lib/firecrawl/models/scrape_options.rb +13 -2
- data/lib/firecrawl/models/search_options.rb +4 -2
- data/lib/firecrawl/version.rb +1 -1
- data/lib/firecrawl.rb +11 -0
- metadata +17 -5
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 22543b5053874d9d1a815949df7212abbcca9a72738a23ecfc696194696b6ad6
|
|
4
|
+
data.tar.gz: a53387ffdbb11b21ce357368f617dd83fdf341619a022db5997b8a4e3bfab6b1
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 7345b8e668fd1ece22c0de23408a7f99e6e5da42f2a573973cb32b4db18cd5bb2d8f61f505e0de5df4e0a0c96fd11af8c35531cd98bcdaa2aab470227678a9ac
|
|
7
|
+
data.tar.gz: 7a4ec95b23cd3e767e93742600a01c8a51123d9c9589823bf62f6a2fb2561ad77fb05c95a45324e44a0ef7322aea4fcd94a46ce6e3b705f61ec5831901ff27ec
|
data/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Firecrawl Ruby SDK
|
|
2
2
|
|
|
3
|
-
Ruby SDK for
|
|
3
|
+
Ruby SDK for [Firecrawl](https://firecrawl.dev), the context API for AI agents. Search, scrape, parse, crawl, map, and interact with the web, and run Agent and Monitor jobs on the v2 API.
|
|
4
4
|
|
|
5
5
|
## Prerequisites
|
|
6
6
|
|
|
@@ -126,6 +126,15 @@ doc = client.parse(file,
|
|
|
126
126
|
puts doc.markdown
|
|
127
127
|
```
|
|
128
128
|
|
|
129
|
+
List the file formats the parse endpoint accepts. `available` is `false` for
|
|
130
|
+
formats the deployment can't currently handle (e.g. images when OCR is off).
|
|
131
|
+
|
|
132
|
+
```ruby
|
|
133
|
+
client.get_parse_formats.each do |f|
|
|
134
|
+
puts "#{f.format} (#{f.kind}) #{f.extensions.join(", ")} #{f.mime_types.join(", ")} available=#{f.available}"
|
|
135
|
+
end
|
|
136
|
+
```
|
|
137
|
+
|
|
129
138
|
### Crawl
|
|
130
139
|
|
|
131
140
|
```ruby
|
|
@@ -184,6 +193,25 @@ results = client.search("latest news",
|
|
|
184
193
|
Firecrawl::Models::SearchOptions.new(limit: 5, location: "US"))
|
|
185
194
|
```
|
|
186
195
|
|
|
196
|
+
### Developer Search
|
|
197
|
+
|
|
198
|
+
Search public repositories, GitHub issues, merged pull requests, READMEs and code documentation. `developer_search` replaces the deprecated `search_github`.
|
|
199
|
+
|
|
200
|
+
```ruby
|
|
201
|
+
response = client.developer_search(
|
|
202
|
+
"retry an http request with exponential backoff",
|
|
203
|
+
Firecrawl::Models::DeveloperSearchOptions.new(
|
|
204
|
+
k: 5,
|
|
205
|
+
types: ["issue", "pull_request"],
|
|
206
|
+
repos: ["square/okhttp"]
|
|
207
|
+
)
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
response.results.each do |result|
|
|
211
|
+
puts "#{result.title} #{result.url}"
|
|
212
|
+
end
|
|
213
|
+
```
|
|
214
|
+
|
|
187
215
|
### Agent
|
|
188
216
|
|
|
189
217
|
```ruby
|
|
@@ -194,6 +222,17 @@ status = client.agent(
|
|
|
194
222
|
urls: ["https://example.com"]
|
|
195
223
|
))
|
|
196
224
|
puts status.data
|
|
225
|
+
|
|
226
|
+
# Use Alexandria data providers connected in the dashboard
|
|
227
|
+
status = client.agent(
|
|
228
|
+
Firecrawl::Models::AgentOptions.new(
|
|
229
|
+
prompt: "Find the work email of the CEO of example.com",
|
|
230
|
+
exchange: Firecrawl::Models::AgentExchangeOptions.new(
|
|
231
|
+
enabled: true,
|
|
232
|
+
toolkits: ["apollo"]
|
|
233
|
+
)
|
|
234
|
+
))
|
|
235
|
+
puts status.exchange&.paid_calls
|
|
197
236
|
```
|
|
198
237
|
|
|
199
238
|
### Usage & Metrics
|
data/lib/firecrawl/client.rb
CHANGED
|
@@ -81,6 +81,10 @@ module Firecrawl
|
|
|
81
81
|
body.merge!(options.to_h) if options
|
|
82
82
|
body["origin"] ||= "ruby-sdk@#{Firecrawl::VERSION}"
|
|
83
83
|
raw = @http.post("/v2/scrape", body)
|
|
84
|
+
# Some scrape failures (e.g. SCRAPE_DNS_RESOLUTION_ERROR) arrive as HTTP 200 with success: false.
|
|
85
|
+
if raw["success"] == false
|
|
86
|
+
raise FirecrawlError.new(raw["error"] || "Scrape failed", status_code: 200, error_code: raw["code"], details: raw["details"])
|
|
87
|
+
end
|
|
84
88
|
data = raw["data"] || raw
|
|
85
89
|
Models::Document.new(data)
|
|
86
90
|
end
|
|
@@ -129,6 +133,9 @@ module Firecrawl
|
|
|
129
133
|
|
|
130
134
|
# Search GitHub research content.
|
|
131
135
|
#
|
|
136
|
+
# @deprecated Stops responding after 2026-11-03. Use {#developer_search},
|
|
137
|
+
# which does not carry over the score breakdown or the web fallback
|
|
138
|
+
# results.
|
|
132
139
|
# @param query_text [String] GitHub query
|
|
133
140
|
# @param options [Hash] optional query parameters
|
|
134
141
|
# @return [Hash]
|
|
@@ -136,6 +143,22 @@ module Firecrawl
|
|
|
136
143
|
@http.get("/v2/search/research/github#{query(options.merge("query" => query_text, "origin" => "ruby-sdk@#{Firecrawl::VERSION}"))}")
|
|
137
144
|
end
|
|
138
145
|
|
|
146
|
+
# Searches the developer index: public repositories, GitHub issues, merged
|
|
147
|
+
# pull requests, READMEs and code documentation.
|
|
148
|
+
#
|
|
149
|
+
# @param query [String] the search query
|
|
150
|
+
# @param options [Models::DeveloperSearchOptions, nil] developer search filters
|
|
151
|
+
# @return [Models::DeveloperSearchResponse]
|
|
152
|
+
# @raise [ArgumentError] if the query is blank
|
|
153
|
+
def developer_search(query, options = nil)
|
|
154
|
+
raise ArgumentError, "Query is required" if query.nil? || query.to_s.strip.empty?
|
|
155
|
+
|
|
156
|
+
body = { "query" => query }
|
|
157
|
+
body.merge!(options.to_h) if options
|
|
158
|
+
body["origin"] ||= "ruby-sdk@#{Firecrawl::VERSION}"
|
|
159
|
+
Models::DeveloperSearchResponse.new(@http.post("/v2/search/developer", body))
|
|
160
|
+
end
|
|
161
|
+
|
|
139
162
|
# Interacts with the scrape-bound browser session for a scrape job.
|
|
140
163
|
#
|
|
141
164
|
# @param job_id [String] the scrape job ID
|
|
@@ -191,6 +214,15 @@ module Firecrawl
|
|
|
191
214
|
Models::Document.new(data)
|
|
192
215
|
end
|
|
193
216
|
|
|
217
|
+
# Lists the file formats accepted by {#parse} on this deployment.
|
|
218
|
+
#
|
|
219
|
+
# @return [Array<Models::ParseFormat>]
|
|
220
|
+
def get_parse_formats
|
|
221
|
+
raw = @http.get("/v2/parse/formats")
|
|
222
|
+
formats = (raw["data"] || {})["formats"] || []
|
|
223
|
+
formats.map { |f| Models::ParseFormat.new(f) }
|
|
224
|
+
end
|
|
225
|
+
|
|
194
226
|
# ================================================================
|
|
195
227
|
# CRAWL
|
|
196
228
|
# ================================================================
|
|
@@ -205,6 +237,7 @@ module Firecrawl
|
|
|
205
237
|
|
|
206
238
|
body = { "url" => url }
|
|
207
239
|
body.merge!(options.to_h) if options
|
|
240
|
+
body["origin"] ||= "ruby-sdk@#{Firecrawl::VERSION}"
|
|
208
241
|
raw = @http.post("/v2/crawl", body)
|
|
209
242
|
Models::CrawlResponse.new(raw)
|
|
210
243
|
end
|
|
@@ -279,6 +312,7 @@ module Firecrawl
|
|
|
279
312
|
body.merge!(opts_hash)
|
|
280
313
|
body.merge!(nested) if nested
|
|
281
314
|
end
|
|
315
|
+
body["origin"] ||= "ruby-sdk@#{Firecrawl::VERSION}"
|
|
282
316
|
raw = @http.post("/v2/batch/scrape", body, extra_headers: extra_headers)
|
|
283
317
|
Models::BatchScrapeResponse.new(raw)
|
|
284
318
|
end
|
|
@@ -330,6 +364,7 @@ module Firecrawl
|
|
|
330
364
|
|
|
331
365
|
body = { "url" => url }
|
|
332
366
|
body.merge!(options.to_h) if options
|
|
367
|
+
body["origin"] ||= "ruby-sdk@#{Firecrawl::VERSION}"
|
|
333
368
|
raw = @http.post("/v2/map", body)
|
|
334
369
|
data = raw["data"] || raw
|
|
335
370
|
Models::MapData.new(data)
|
|
@@ -463,6 +498,20 @@ module Firecrawl
|
|
|
463
498
|
Models::AgentStatusResponse.new(raw)
|
|
464
499
|
end
|
|
465
500
|
|
|
501
|
+
# Lists agent tasks, most recent first.
|
|
502
|
+
#
|
|
503
|
+
# Pages are fixed at 20 runs. To fetch the next page, pass the before
|
|
504
|
+
# value from the previous page's next URL. This method does not
|
|
505
|
+
# auto-paginate.
|
|
506
|
+
#
|
|
507
|
+
# @param before [Integer, nil] only return agent runs created before this
|
|
508
|
+
# unix millisecond timestamp
|
|
509
|
+
# @return [Models::AgentListResponse]
|
|
510
|
+
def list_agents(before: nil)
|
|
511
|
+
raw = @http.get("/v2/agent#{query(before: before)}")
|
|
512
|
+
Models::AgentListResponse.new(raw)
|
|
513
|
+
end
|
|
514
|
+
|
|
466
515
|
# Runs an agent task and waits for completion (auto-polling).
|
|
467
516
|
#
|
|
468
517
|
# @param options [Models::AgentOptions] agent configuration
|
|
@@ -493,6 +542,31 @@ module Firecrawl
|
|
|
493
542
|
@http.delete("/v2/agent/#{job_id}")
|
|
494
543
|
end
|
|
495
544
|
|
|
545
|
+
# Gets the trace of an agent task.
|
|
546
|
+
#
|
|
547
|
+
# @param job_id [String] the agent job ID
|
|
548
|
+
# @param live_view [Boolean] include live view URLs for active browser sessions
|
|
549
|
+
# @return [Models::AgentTraceResponse]
|
|
550
|
+
def get_agent_trace(job_id, live_view: false)
|
|
551
|
+
raise ArgumentError, "Job ID is required" if job_id.nil?
|
|
552
|
+
|
|
553
|
+
raw = @http.get("/v2/agent/#{job_id}/trace#{query(liveView: live_view ? true : nil)}")
|
|
554
|
+
Models::AgentTraceResponse.new(raw)
|
|
555
|
+
end
|
|
556
|
+
|
|
557
|
+
# Gets a snapshot of an agent task.
|
|
558
|
+
#
|
|
559
|
+
# @param job_id [String] the agent job ID
|
|
560
|
+
# @param snapshot_id [String] the snapshot ID
|
|
561
|
+
# @return [Models::AgentSnapshotResponse]
|
|
562
|
+
def get_agent_snapshot(job_id, snapshot_id)
|
|
563
|
+
raise ArgumentError, "Job ID is required" if job_id.nil?
|
|
564
|
+
raise ArgumentError, "Snapshot ID is required" if snapshot_id.nil?
|
|
565
|
+
|
|
566
|
+
raw = @http.get("/v2/agent/#{job_id}/snapshots/#{snapshot_id}")
|
|
567
|
+
Models::AgentSnapshotResponse.new(raw)
|
|
568
|
+
end
|
|
569
|
+
|
|
496
570
|
# ================================================================
|
|
497
571
|
# USAGE & METRICS
|
|
498
572
|
# ================================================================
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Firecrawl
|
|
4
|
+
module Models
|
|
5
|
+
# Alexandria (Exchange) options for an agent run, forwarded verbatim; the
|
|
6
|
+
# server owns every default and limit. On a follow-up turn, omitting the
|
|
7
|
+
# exchange options inherits the previous turn's settings.
|
|
8
|
+
#
|
|
9
|
+
# on_terms_required: "skip" or "ask".
|
|
10
|
+
class AgentExchangeOptions
|
|
11
|
+
# Answers the previous turn's pending approval. A "terms" approval is
|
|
12
|
+
# accepted as a whole, so call_ids and always are ignored on it.
|
|
13
|
+
class Approve
|
|
14
|
+
attr_reader :approval_id, :call_ids, :always
|
|
15
|
+
|
|
16
|
+
def initialize(approval_id:, call_ids: nil, always: nil)
|
|
17
|
+
@approval_id = approval_id
|
|
18
|
+
@call_ids = call_ids
|
|
19
|
+
@always = always
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def to_h
|
|
23
|
+
{
|
|
24
|
+
"approvalId" => approval_id,
|
|
25
|
+
"callIds" => call_ids,
|
|
26
|
+
"always" => always,
|
|
27
|
+
}.compact
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# Refuses the previous turn's pending approval.
|
|
32
|
+
class Decline
|
|
33
|
+
attr_reader :approval_id
|
|
34
|
+
|
|
35
|
+
def initialize(approval_id:)
|
|
36
|
+
@approval_id = approval_id
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def to_h
|
|
40
|
+
{ "approvalId" => approval_id }
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
FIELDS = %i[
|
|
45
|
+
enabled toolkits max_calls require_approval approve decline
|
|
46
|
+
on_terms_required
|
|
47
|
+
].freeze
|
|
48
|
+
|
|
49
|
+
attr_reader(*FIELDS)
|
|
50
|
+
|
|
51
|
+
def initialize(**kwargs)
|
|
52
|
+
FIELDS.each { |f| instance_variable_set(:"@#{f}", kwargs[f]) }
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def to_h
|
|
56
|
+
{
|
|
57
|
+
"enabled" => enabled,
|
|
58
|
+
"toolkits" => toolkits,
|
|
59
|
+
"maxCalls" => max_calls,
|
|
60
|
+
"requireApproval" => require_approval,
|
|
61
|
+
"approve" => approve&.to_h,
|
|
62
|
+
"decline" => decline&.to_h,
|
|
63
|
+
"onTermsRequired" => on_terms_required,
|
|
64
|
+
}.compact
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Firecrawl
|
|
4
|
+
module Models
|
|
5
|
+
# What an agent run did with Alexandria (Exchange). toolkits and
|
|
6
|
+
# require_approval are what the run resolved to after thread inheritance.
|
|
7
|
+
class AgentExchangeSummary
|
|
8
|
+
# A provider that needs data terms the team has not accepted, so the run
|
|
9
|
+
# did not use it.
|
|
10
|
+
class SkippedProvider
|
|
11
|
+
attr_reader :provider, :name, :capability, :adds, :reason, :version,
|
|
12
|
+
:terms_url
|
|
13
|
+
|
|
14
|
+
def initialize(data)
|
|
15
|
+
@provider = data["provider"]
|
|
16
|
+
@name = data["name"]
|
|
17
|
+
@capability = data["capability"]
|
|
18
|
+
@adds = data["adds"]
|
|
19
|
+
@reason = data["reason"]
|
|
20
|
+
@version = data["version"]
|
|
21
|
+
@terms_url = data["termsUrl"]
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# The Exchange calls to view and accept providers' terms ("ask" mode).
|
|
26
|
+
# Nothing here runs for you; only call accept after your user agrees.
|
|
27
|
+
class TermsRequiredAction
|
|
28
|
+
# show and accept are the raw Exchange calls ("provider",
|
|
29
|
+
# "capability", "options").
|
|
30
|
+
class Provider
|
|
31
|
+
attr_reader :provider, :name, :capability, :adds, :version, :digest,
|
|
32
|
+
:url, :show, :accept
|
|
33
|
+
|
|
34
|
+
def initialize(data)
|
|
35
|
+
@provider = data["provider"]
|
|
36
|
+
@name = data["name"]
|
|
37
|
+
@capability = data["capability"]
|
|
38
|
+
@adds = data["adds"]
|
|
39
|
+
@version = data["version"]
|
|
40
|
+
@digest = data["digest"]
|
|
41
|
+
@url = data["url"]
|
|
42
|
+
@show = data["show"]
|
|
43
|
+
@accept = data["accept"]
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
attr_reader :type, :approval_id, :providers
|
|
48
|
+
|
|
49
|
+
def initialize(data)
|
|
50
|
+
@type = data["type"]
|
|
51
|
+
@approval_id = data["approvalId"]
|
|
52
|
+
@providers = data["providers"]&.map { |p| Provider.new(p) }
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
attr_reader :enabled, :toolkits, :require_approval, :on_terms_required,
|
|
57
|
+
:paid_calls, :credits_used, :skipped_providers,
|
|
58
|
+
:requires_action
|
|
59
|
+
|
|
60
|
+
def initialize(data)
|
|
61
|
+
@enabled = data["enabled"]
|
|
62
|
+
@toolkits = data["toolkits"]
|
|
63
|
+
@require_approval = data["requireApproval"]
|
|
64
|
+
@on_terms_required = data["onTermsRequired"]
|
|
65
|
+
@paid_calls = data["paidCalls"]
|
|
66
|
+
@credits_used = data["creditsUsed"]
|
|
67
|
+
@skipped_providers = data["skippedProviders"]&.map { |p| SkippedProvider.new(p) }
|
|
68
|
+
@requires_action = data["requiresAction"] && TermsRequiredAction.new(data["requiresAction"])
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Firecrawl
|
|
4
|
+
module Models
|
|
5
|
+
# Response from listing agent tasks.
|
|
6
|
+
class AgentListResponse
|
|
7
|
+
attr_reader :success, :agents, :next, :error
|
|
8
|
+
|
|
9
|
+
def initialize(raw)
|
|
10
|
+
@success = raw["success"]
|
|
11
|
+
@agents = raw["agents"]
|
|
12
|
+
@next = raw["next"]
|
|
13
|
+
@error = raw["error"]
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def to_s
|
|
17
|
+
"AgentListResponse{success=#{success}, agents=#{agents&.size}}"
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -3,16 +3,26 @@
|
|
|
3
3
|
module Firecrawl
|
|
4
4
|
module Models
|
|
5
5
|
# Options for starting an agent task.
|
|
6
|
+
#
|
|
7
|
+
# effort: valid values are "low", "medium", and "high"; sets the reasoning
|
|
8
|
+
# budget for the run (every level runs spark-2).
|
|
9
|
+
#
|
|
10
|
+
# thread_id continues an existing thread as its next turn; omitted starts a
|
|
11
|
+
# new thread. mode: "extract" or "chat".
|
|
6
12
|
class AgentOptions
|
|
7
13
|
FIELDS = %i[
|
|
8
14
|
urls prompt schema integration max_credits
|
|
9
|
-
strict_constrain_to_urls model webhook
|
|
15
|
+
strict_constrain_to_urls model effort webhook audit_metadata
|
|
16
|
+
thread_id mode exchange
|
|
10
17
|
].freeze
|
|
11
18
|
|
|
12
19
|
attr_reader(*FIELDS)
|
|
13
20
|
|
|
14
21
|
def initialize(**kwargs)
|
|
15
22
|
FIELDS.each { |f| instance_variable_set(:"@#{f}", kwargs[f]) }
|
|
23
|
+
if audit_metadata && !audit_metadata.is_a?(AuditMetadata)
|
|
24
|
+
raise ArgumentError, "audit_metadata must be an AuditMetadata"
|
|
25
|
+
end
|
|
16
26
|
raise ArgumentError, "Agent prompt is required" if prompt.nil? || prompt.empty?
|
|
17
27
|
end
|
|
18
28
|
|
|
@@ -25,7 +35,12 @@ module Firecrawl
|
|
|
25
35
|
"maxCredits" => max_credits,
|
|
26
36
|
"strictConstrainToURLs" => strict_constrain_to_urls,
|
|
27
37
|
"model" => model,
|
|
38
|
+
"effort" => effort,
|
|
28
39
|
"webhook" => webhook.is_a?(Hash) ? webhook : webhook&.to_h,
|
|
40
|
+
"auditMetadata" => audit_metadata&.to_h,
|
|
41
|
+
"threadId" => thread_id,
|
|
42
|
+
"mode" => mode,
|
|
43
|
+
"exchange" => exchange&.to_h,
|
|
29
44
|
}.compact
|
|
30
45
|
end
|
|
31
46
|
end
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Firecrawl
|
|
4
|
+
module Models
|
|
5
|
+
# A turn that ended waiting for the caller. kind "calls" (or nil on older
|
|
6
|
+
# runs) holds paid calls to allow or refuse; kind "terms" lists providers
|
|
7
|
+
# whose data terms need accepting, with calls empty. Answer it on the next
|
|
8
|
+
# turn of the thread with AgentExchangeOptions approve or decline.
|
|
9
|
+
class AgentPendingApproval
|
|
10
|
+
# input and more are the raw call arguments.
|
|
11
|
+
class Call
|
|
12
|
+
attr_reader :id, :provider, :capability, :input, :more,
|
|
13
|
+
:credits_estimate
|
|
14
|
+
|
|
15
|
+
def initialize(data)
|
|
16
|
+
@id = data["id"]
|
|
17
|
+
@provider = data["provider"]
|
|
18
|
+
@capability = data["capability"]
|
|
19
|
+
@input = data["input"]
|
|
20
|
+
@more = data["more"]
|
|
21
|
+
@credits_estimate = data["creditsEstimate"]
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
class TermsGate
|
|
26
|
+
attr_reader :provider, :name, :logo, :capability, :adds, :version,
|
|
27
|
+
:digest, :url
|
|
28
|
+
|
|
29
|
+
def initialize(data)
|
|
30
|
+
@provider = data["provider"]
|
|
31
|
+
@name = data["name"]
|
|
32
|
+
@logo = data["logo"]
|
|
33
|
+
@capability = data["capability"]
|
|
34
|
+
@adds = data["adds"]
|
|
35
|
+
@version = data["version"]
|
|
36
|
+
@digest = data["digest"]
|
|
37
|
+
@url = data["url"]
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
class Resolution
|
|
42
|
+
attr_reader :approved, :call_ids, :always, :by_run_id
|
|
43
|
+
|
|
44
|
+
def initialize(data)
|
|
45
|
+
@approved = data["approved"]
|
|
46
|
+
@call_ids = data["callIds"]
|
|
47
|
+
@always = data["always"]
|
|
48
|
+
@by_run_id = data["byRunId"]
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
attr_reader :id, :kind, :reason, :calls, :terms, :resolution
|
|
53
|
+
|
|
54
|
+
def initialize(data)
|
|
55
|
+
@id = data["id"]
|
|
56
|
+
@kind = data["kind"]
|
|
57
|
+
@reason = data["reason"]
|
|
58
|
+
@calls = data["calls"]&.map { |c| Call.new(c) }
|
|
59
|
+
@terms = data["terms"]&.map { |t| TermsGate.new(t) }
|
|
60
|
+
@resolution = data["resolution"] && Resolution.new(data["resolution"])
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
@@ -4,12 +4,15 @@ module Firecrawl
|
|
|
4
4
|
module Models
|
|
5
5
|
# Response from starting an agent task.
|
|
6
6
|
class AgentResponse
|
|
7
|
-
attr_reader :success, :id, :error
|
|
7
|
+
attr_reader :success, :id, :error, :thread_id, :thread_turn
|
|
8
8
|
|
|
9
9
|
def initialize(data)
|
|
10
10
|
@success = data["success"]
|
|
11
11
|
@id = data["id"]
|
|
12
12
|
@error = data["error"]
|
|
13
|
+
# Pass thread_id back in AgentOptions to continue the conversation.
|
|
14
|
+
@thread_id = data["threadId"]
|
|
15
|
+
@thread_turn = data["threadTurn"]
|
|
13
16
|
end
|
|
14
17
|
|
|
15
18
|
def to_s
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Firecrawl
|
|
4
|
+
module Models
|
|
5
|
+
# Snapshot response for an agent task.
|
|
6
|
+
class AgentSnapshotResponse
|
|
7
|
+
attr_reader :success, :id, :snapshot_id, :snapshot, :error
|
|
8
|
+
|
|
9
|
+
def initialize(raw)
|
|
10
|
+
@success = raw["success"]
|
|
11
|
+
@id = raw["id"]
|
|
12
|
+
@snapshot_id = raw["snapshotId"]
|
|
13
|
+
@snapshot = raw["snapshot"]
|
|
14
|
+
@error = raw["error"]
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def to_s
|
|
18
|
+
"AgentSnapshotResponse{id=#{id}, snapshot_id=#{snapshot_id}, success=#{success}}"
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
@@ -4,13 +4,24 @@ module Firecrawl
|
|
|
4
4
|
module Models
|
|
5
5
|
# Status response for monitoring agent tasks.
|
|
6
6
|
class AgentStatusResponse
|
|
7
|
-
attr_reader :status, :data, :credits_used, :expires_at
|
|
7
|
+
attr_reader :status, :data, :credits_used, :expires_at, :effort,
|
|
8
|
+
:thread_id, :thread_turn, :mode, :message, :pending_approval,
|
|
9
|
+
:exchange
|
|
8
10
|
|
|
9
11
|
def initialize(raw)
|
|
10
12
|
@status = raw["status"]
|
|
11
13
|
@data = raw["data"]
|
|
12
14
|
@credits_used = raw["creditsUsed"]
|
|
13
15
|
@expires_at = raw["expiresAt"]
|
|
16
|
+
# The effort the job ran with; only present for runs that specified it.
|
|
17
|
+
@effort = raw["effort"]
|
|
18
|
+
@thread_id = raw["threadId"]
|
|
19
|
+
@thread_turn = raw["threadTurn"]
|
|
20
|
+
@mode = raw["mode"]
|
|
21
|
+
# Chat-mode runs answer here instead of in data.
|
|
22
|
+
@message = raw["message"]
|
|
23
|
+
@pending_approval = raw["pendingApproval"] && AgentPendingApproval.new(raw["pendingApproval"])
|
|
24
|
+
@exchange = raw["exchange"] && AgentExchangeSummary.new(raw["exchange"])
|
|
14
25
|
end
|
|
15
26
|
|
|
16
27
|
def done?
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Firecrawl
|
|
4
|
+
module Models
|
|
5
|
+
# Trace response for an agent task.
|
|
6
|
+
#
|
|
7
|
+
# Events are raw hashes: each carries a "type" (e.g. "run.started",
|
|
8
|
+
# "agent.finished", "tool_call.started") plus base fields like
|
|
9
|
+
# "schemaVersion", "eventId", "runId", "occurredAt", "producerSequence",
|
|
10
|
+
# and "agent".
|
|
11
|
+
class AgentTraceResponse
|
|
12
|
+
attr_reader :success, :id, :events, :active_browser_sessions, :credits_used, :error
|
|
13
|
+
|
|
14
|
+
def initialize(raw)
|
|
15
|
+
@success = raw["success"]
|
|
16
|
+
@id = raw["id"]
|
|
17
|
+
@events = raw["events"]
|
|
18
|
+
@active_browser_sessions = raw["activeBrowserSessions"]
|
|
19
|
+
@credits_used = raw["creditsUsed"]
|
|
20
|
+
@error = raw["error"]
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def to_s
|
|
24
|
+
"AgentTraceResponse{id=#{id}, success=#{success}}"
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Firecrawl
|
|
4
|
+
module Models
|
|
5
|
+
# User attribution included with SIEM logging events.
|
|
6
|
+
class AuditMetadata
|
|
7
|
+
attr_reader :username
|
|
8
|
+
|
|
9
|
+
def initialize(username:)
|
|
10
|
+
raise ArgumentError, "username must be a string" unless username.is_a?(String)
|
|
11
|
+
|
|
12
|
+
@username = username
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def to_h
|
|
16
|
+
{ "username" => username }
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Firecrawl
|
|
4
|
+
module Models
|
|
5
|
+
# Options for a developer search request. Unset fields are omitted.
|
|
6
|
+
#
|
|
7
|
+
# - k: total ranked results, 1 to 100 (default 10)
|
|
8
|
+
# - passages: passages per result, 1 to 5 (default 1)
|
|
9
|
+
# - types: result kinds, any of "doc", "issue", "pull_request", "readme"
|
|
10
|
+
# - repos: GitHub owner/name filters, at most 20
|
|
11
|
+
# - sources: developer documentation source IDs, at most 20
|
|
12
|
+
# - language: one case-insensitive GitHub Linguist primary language
|
|
13
|
+
# - topic: repository topics that must all match, at most 8
|
|
14
|
+
# - license: one case-insensitive SPDX repository license identifier
|
|
15
|
+
# - min_stars, max_stars, archived, fork: repository filters
|
|
16
|
+
# - skills: "only" returns packaged agent-skill evidence only
|
|
17
|
+
class DeveloperSearchOptions
|
|
18
|
+
FIELDS = %i[
|
|
19
|
+
k passages types repos sources language topic license
|
|
20
|
+
min_stars max_stars archived fork skills
|
|
21
|
+
].freeze
|
|
22
|
+
|
|
23
|
+
attr_reader(*FIELDS)
|
|
24
|
+
|
|
25
|
+
def initialize(**kwargs)
|
|
26
|
+
unknown = kwargs.keys - FIELDS
|
|
27
|
+
raise ArgumentError, "unknown developer search option(s): #{unknown.join(", ")}" unless unknown.empty?
|
|
28
|
+
|
|
29
|
+
FIELDS.each { |f| instance_variable_set(:"@#{f}", kwargs[f]) }
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def to_h
|
|
33
|
+
FIELDS.to_h { |f| [f.to_s, public_send(f)] }.compact
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Firecrawl
|
|
4
|
+
module Models
|
|
5
|
+
# One matched passage of a developer search result.
|
|
6
|
+
class DeveloperSearchPassage
|
|
7
|
+
attr_reader :text, :citation_url
|
|
8
|
+
|
|
9
|
+
def initialize(data)
|
|
10
|
+
@text = data["text"]
|
|
11
|
+
@citation_url = data["citation_url"]
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# One ranked developer search result. `license` is either an SPDX string or
|
|
16
|
+
# a disclosure object (Hash), so it is kept as returned.
|
|
17
|
+
class DeveloperSearchResult
|
|
18
|
+
attr_reader :id, :url, :title, :passages, :license
|
|
19
|
+
|
|
20
|
+
def initialize(data)
|
|
21
|
+
@id = data["id"]
|
|
22
|
+
@url = data["url"]
|
|
23
|
+
@title = data["title"]
|
|
24
|
+
@passages = (data["passages"] || []).map { |p| DeveloperSearchPassage.new(p) }
|
|
25
|
+
@license = data["license"]
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Whether a requested repository is indexed, and which result types it has.
|
|
30
|
+
class DeveloperSearchRepoStatus
|
|
31
|
+
attr_reader :repo, :indexed, :types
|
|
32
|
+
|
|
33
|
+
def initialize(data)
|
|
34
|
+
@repo = data["repo"]
|
|
35
|
+
@indexed = data["indexed"] == true
|
|
36
|
+
@types = data["types"] || {}
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def indexed?
|
|
40
|
+
indexed
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Whether a requested documentation source is indexed.
|
|
45
|
+
class DeveloperSearchSourceStatus
|
|
46
|
+
attr_reader :source, :indexed
|
|
47
|
+
|
|
48
|
+
def initialize(data)
|
|
49
|
+
@source = data["source"]
|
|
50
|
+
@indexed = data["indexed"] == true
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def indexed?
|
|
54
|
+
indexed
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Response from a developer search. `repos` and `sources` are present only
|
|
59
|
+
# when those filters were requested.
|
|
60
|
+
class DeveloperSearchResponse
|
|
61
|
+
attr_reader :success, :partial, :results, :repos, :sources
|
|
62
|
+
|
|
63
|
+
def initialize(data)
|
|
64
|
+
@success = data["success"] == true
|
|
65
|
+
@partial = data["partial"] == true
|
|
66
|
+
@results = (data["results"] || []).map { |r| DeveloperSearchResult.new(r) }
|
|
67
|
+
@repos = data["repos"]&.map { |r| DeveloperSearchRepoStatus.new(r) }
|
|
68
|
+
@sources = data["sources"]&.map { |s| DeveloperSearchSourceStatus.new(s) }
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def to_s
|
|
72
|
+
"DeveloperSearchResponse{results=#{results.size}}"
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
@@ -7,7 +7,8 @@ module Firecrawl
|
|
|
7
7
|
attr_reader :markdown, :html, :raw_html, :json, :summary,
|
|
8
8
|
:metadata, :links, :images, :screenshot, :audio,
|
|
9
9
|
:video, :attributes, :actions, :answer, :highlights,
|
|
10
|
-
:warning, :change_tracking, :branding, :product, :menu
|
|
10
|
+
:warning, :change_tracking, :branding, :product, :menu,
|
|
11
|
+
:pages, :blocks
|
|
11
12
|
|
|
12
13
|
def initialize(data)
|
|
13
14
|
@markdown = data["markdown"]
|
|
@@ -30,6 +31,8 @@ module Firecrawl
|
|
|
30
31
|
@branding = data["branding"]
|
|
31
32
|
@product = data["product"] && ProductProfile.new(data["product"])
|
|
32
33
|
@menu = data["menu"] && MenuProfile.new(data["menu"])
|
|
34
|
+
@pages = data["pages"]
|
|
35
|
+
@blocks = data["blocks"]
|
|
33
36
|
end
|
|
34
37
|
|
|
35
38
|
def to_s
|
|
@@ -6,13 +6,16 @@ module Firecrawl
|
|
|
6
6
|
class MapOptions
|
|
7
7
|
FIELDS = %i[
|
|
8
8
|
search sitemap include_subdomains ignore_query_parameters
|
|
9
|
-
limit timeout integration location
|
|
9
|
+
limit timeout integration location audit_metadata
|
|
10
10
|
].freeze
|
|
11
11
|
|
|
12
12
|
attr_reader(*FIELDS)
|
|
13
13
|
|
|
14
14
|
def initialize(**kwargs)
|
|
15
15
|
FIELDS.each { |f| instance_variable_set(:"@#{f}", kwargs[f]) }
|
|
16
|
+
if audit_metadata && !audit_metadata.is_a?(AuditMetadata)
|
|
17
|
+
raise ArgumentError, "audit_metadata must be an AuditMetadata"
|
|
18
|
+
end
|
|
16
19
|
end
|
|
17
20
|
|
|
18
21
|
def to_h
|
|
@@ -25,6 +28,7 @@ module Firecrawl
|
|
|
25
28
|
"timeout" => timeout,
|
|
26
29
|
"integration" => integration,
|
|
27
30
|
"location" => location.is_a?(Hash) ? location : location&.to_h,
|
|
31
|
+
"auditMetadata" => audit_metadata&.to_h,
|
|
28
32
|
}.compact
|
|
29
33
|
end
|
|
30
34
|
end
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Firecrawl
|
|
4
|
+
module Models
|
|
5
|
+
# A file format accepted by the `/v2/parse` endpoint.
|
|
6
|
+
#
|
|
7
|
+
# `kind` is usually one of {KIND_DOCUMENT} or {KIND_IMAGE}, but newer API
|
|
8
|
+
# versions may return other values, which are kept as-is.
|
|
9
|
+
class ParseFormat
|
|
10
|
+
KIND_DOCUMENT = "document"
|
|
11
|
+
KIND_IMAGE = "image"
|
|
12
|
+
|
|
13
|
+
attr_reader :format, :kind, :extensions, :mime_types, :available
|
|
14
|
+
|
|
15
|
+
def initialize(data)
|
|
16
|
+
@format = data["format"]
|
|
17
|
+
@kind = data["kind"]
|
|
18
|
+
@extensions = data["extensions"] || []
|
|
19
|
+
@mime_types = data["mimeTypes"] || []
|
|
20
|
+
@available = data["available"]
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def document?
|
|
24
|
+
kind == KIND_DOCUMENT
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def image?
|
|
28
|
+
kind == KIND_IMAGE
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def to_s
|
|
32
|
+
"ParseFormat{format=#{format}, kind=#{kind}, available=#{available}}"
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
@@ -7,19 +7,28 @@ module Firecrawl
|
|
|
7
7
|
# Parse does not support browser-rendering features (actions, waitFor,
|
|
8
8
|
# location, mobile) nor screenshot/branding/product/menu/audio/video/changeTracking formats. The
|
|
9
9
|
# proxy field only accepts "auto" or "basic".
|
|
10
|
+
#
|
|
11
|
+
# check_prompt_injection scans the parsed content for prompt injection
|
|
12
|
+
# before LLM-backed formats run. A detection fails the parse with
|
|
13
|
+
# SCRAPE_PROMPT_INJECTION_DETECTED. Adds 4 credits when the check scans
|
|
14
|
+
# the whole content.
|
|
10
15
|
class ParseOptions
|
|
11
16
|
UNSUPPORTED_FORMATS = %w[changeTracking screenshot screenshot@fullPage branding product menu audio video].freeze
|
|
12
17
|
|
|
13
18
|
FIELDS = %i[
|
|
14
19
|
formats headers include_tags exclude_tags only_main_content
|
|
15
20
|
timeout parsers skip_tls_verification remove_base64_images
|
|
16
|
-
block_ads proxy integration redact_pii json_options
|
|
21
|
+
block_ads proxy integration redact_pii json_options audit_metadata
|
|
22
|
+
check_prompt_injection
|
|
17
23
|
].freeze
|
|
18
24
|
|
|
19
25
|
attr_reader(*FIELDS)
|
|
20
26
|
|
|
21
27
|
def initialize(**kwargs)
|
|
22
28
|
FIELDS.each { |f| instance_variable_set(:"@#{f}", kwargs[f]) }
|
|
29
|
+
if audit_metadata && !audit_metadata.is_a?(AuditMetadata)
|
|
30
|
+
raise ArgumentError, "audit_metadata must be an AuditMetadata"
|
|
31
|
+
end
|
|
23
32
|
|
|
24
33
|
validate!
|
|
25
34
|
end
|
|
@@ -32,14 +41,16 @@ module Firecrawl
|
|
|
32
41
|
"excludeTags" => exclude_tags,
|
|
33
42
|
"onlyMainContent" => only_main_content,
|
|
34
43
|
"timeout" => timeout,
|
|
35
|
-
"parsers" => parsers,
|
|
44
|
+
"parsers" => parsers&.map { |parser| parser.respond_to?(:to_h) ? parser.to_h : parser },
|
|
36
45
|
"skipTlsVerification" => skip_tls_verification,
|
|
37
46
|
"removeBase64Images" => remove_base64_images,
|
|
38
47
|
"blockAds" => block_ads,
|
|
39
48
|
"proxy" => proxy,
|
|
40
49
|
"integration" => integration,
|
|
41
50
|
"redactPII" => redact_pii,
|
|
51
|
+
"checkPromptInjection" => check_prompt_injection,
|
|
42
52
|
"jsonOptions" => json_options.is_a?(Hash) ? json_options : json_options&.to_h,
|
|
53
|
+
"auditMetadata" => audit_metadata&.to_h,
|
|
43
54
|
}.compact
|
|
44
55
|
end
|
|
45
56
|
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Firecrawl
|
|
4
|
+
module Models
|
|
5
|
+
# PDF parser configuration for use in ScrapeOptions / ParseOptions parsers.
|
|
6
|
+
class PDFParser
|
|
7
|
+
attr_reader :mode, :max_pages, :pages, :blocks, :page_markers
|
|
8
|
+
|
|
9
|
+
def initialize(mode: nil, max_pages: nil, pages: nil, blocks: nil, page_markers: nil)
|
|
10
|
+
@mode = mode
|
|
11
|
+
@max_pages = max_pages
|
|
12
|
+
@pages = pages
|
|
13
|
+
@blocks = blocks
|
|
14
|
+
@page_markers = page_markers
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def to_h
|
|
18
|
+
{
|
|
19
|
+
"type" => "pdf",
|
|
20
|
+
"mode" => mode,
|
|
21
|
+
"maxPages" => max_pages,
|
|
22
|
+
"pages" => pages,
|
|
23
|
+
"blocks" => blocks,
|
|
24
|
+
"pageMarkers" => page_markers,
|
|
25
|
+
}.compact
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def type
|
|
29
|
+
"pdf"
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
@@ -42,6 +42,33 @@ module Firecrawl
|
|
|
42
42
|
end
|
|
43
43
|
end
|
|
44
44
|
|
|
45
|
+
# JSON format for extracting structured data from page content using a schema.
|
|
46
|
+
#
|
|
47
|
+
# check_prompt_injection is deprecated: use ScrapeOptions#check_prompt_injection
|
|
48
|
+
# or ParseOptions#check_prompt_injection.
|
|
49
|
+
class JsonFormat
|
|
50
|
+
attr_reader :schema, :prompt, :check_prompt_injection
|
|
51
|
+
|
|
52
|
+
def initialize(schema: nil, prompt: nil, check_prompt_injection: nil)
|
|
53
|
+
@schema = schema
|
|
54
|
+
@prompt = prompt
|
|
55
|
+
@check_prompt_injection = check_prompt_injection
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def to_h
|
|
59
|
+
{
|
|
60
|
+
"type" => "json",
|
|
61
|
+
"schema" => schema,
|
|
62
|
+
"prompt" => prompt,
|
|
63
|
+
"checkPromptInjection" => check_prompt_injection,
|
|
64
|
+
}.compact
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def type
|
|
68
|
+
"json"
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
|
|
45
72
|
# Deprecated query format for asking a question about page content.
|
|
46
73
|
class QueryFormat
|
|
47
74
|
MODE_FREEFORM = "freeform"
|
|
@@ -3,18 +3,27 @@
|
|
|
3
3
|
module Firecrawl
|
|
4
4
|
module Models
|
|
5
5
|
# Options for scraping a single URL.
|
|
6
|
+
#
|
|
7
|
+
# check_prompt_injection scans the page content for prompt injection with
|
|
8
|
+
# any format except rawBase64, before LLM-backed formats run. A detection
|
|
9
|
+
# fails the scrape with SCRAPE_PROMPT_INJECTION_DETECTED. Adds 4 credits
|
|
10
|
+
# when the check scans the whole page.
|
|
6
11
|
class ScrapeOptions
|
|
7
12
|
FIELDS = %i[
|
|
8
13
|
formats headers include_tags exclude_tags only_main_content
|
|
9
14
|
timeout wait_for mobile parsers actions location
|
|
10
15
|
skip_tls_verification remove_base64_images block_ads proxy
|
|
11
|
-
max_age store_in_cache lockdown redact_pii
|
|
16
|
+
max_age store_in_cache lockdown check_prompt_injection redact_pii
|
|
17
|
+
integration audit_metadata
|
|
12
18
|
].freeze
|
|
13
19
|
|
|
14
20
|
attr_reader(*FIELDS)
|
|
15
21
|
|
|
16
22
|
def initialize(**kwargs)
|
|
17
23
|
FIELDS.each { |f| instance_variable_set(:"@#{f}", kwargs[f]) }
|
|
24
|
+
if audit_metadata && !audit_metadata.is_a?(AuditMetadata)
|
|
25
|
+
raise ArgumentError, "audit_metadata must be an AuditMetadata"
|
|
26
|
+
end
|
|
18
27
|
@skip_tls_verification = false if @skip_tls_verification.nil?
|
|
19
28
|
end
|
|
20
29
|
|
|
@@ -28,7 +37,7 @@ module Firecrawl
|
|
|
28
37
|
"timeout" => timeout,
|
|
29
38
|
"waitFor" => wait_for,
|
|
30
39
|
"mobile" => mobile,
|
|
31
|
-
"parsers" => parsers,
|
|
40
|
+
"parsers" => parsers&.map { |parser| parser.respond_to?(:to_h) ? parser.to_h : parser },
|
|
32
41
|
"actions" => actions,
|
|
33
42
|
"location" => location.is_a?(Hash) ? location : location&.to_h,
|
|
34
43
|
"skipTlsVerification" => skip_tls_verification,
|
|
@@ -38,8 +47,10 @@ module Firecrawl
|
|
|
38
47
|
"maxAge" => max_age,
|
|
39
48
|
"storeInCache" => store_in_cache,
|
|
40
49
|
"lockdown" => lockdown,
|
|
50
|
+
"checkPromptInjection" => check_prompt_injection,
|
|
41
51
|
"redactPII" => redact_pii,
|
|
42
52
|
"integration" => integration,
|
|
53
|
+
"auditMetadata" => audit_metadata&.to_h,
|
|
43
54
|
}.compact
|
|
44
55
|
end
|
|
45
56
|
|
|
@@ -5,8 +5,8 @@ module Firecrawl
|
|
|
5
5
|
# Options for a web search request.
|
|
6
6
|
class SearchOptions
|
|
7
7
|
FIELDS = %i[
|
|
8
|
-
sources categories include_domains exclude_domains limit tbs location
|
|
9
|
-
ignore_invalid_urls timeout scrape_options integration enterprise
|
|
8
|
+
sources categories include_domains exclude_domains limit tbs location country
|
|
9
|
+
ignore_invalid_urls timeout highlights scrape_options integration enterprise
|
|
10
10
|
].freeze
|
|
11
11
|
|
|
12
12
|
attr_reader(*FIELDS)
|
|
@@ -24,8 +24,10 @@ module Firecrawl
|
|
|
24
24
|
"limit" => limit,
|
|
25
25
|
"tbs" => tbs,
|
|
26
26
|
"location" => location,
|
|
27
|
+
"country" => country,
|
|
27
28
|
"ignoreInvalidURLs" => ignore_invalid_urls,
|
|
28
29
|
"timeout" => timeout,
|
|
30
|
+
"highlights" => highlights,
|
|
29
31
|
"scrapeOptions" => scrape_options&.to_h,
|
|
30
32
|
"integration" => integration,
|
|
31
33
|
# Enterprise search options. Use ["zdr"] for end-to-end Zero Data
|
data/lib/firecrawl/version.rb
CHANGED
data/lib/firecrawl.rb
CHANGED
|
@@ -4,9 +4,11 @@ require_relative "firecrawl/version"
|
|
|
4
4
|
require_relative "firecrawl/errors"
|
|
5
5
|
require_relative "firecrawl/http_client"
|
|
6
6
|
require_relative "firecrawl/models/query_format"
|
|
7
|
+
require_relative "firecrawl/models/pdf_parser"
|
|
7
8
|
require_relative "firecrawl/models/product_profile"
|
|
8
9
|
require_relative "firecrawl/models/menu_profile"
|
|
9
10
|
require_relative "firecrawl/models/document"
|
|
11
|
+
require_relative "firecrawl/models/audit_metadata"
|
|
10
12
|
require_relative "firecrawl/models/scrape_options"
|
|
11
13
|
require_relative "firecrawl/models/crawl_options"
|
|
12
14
|
require_relative "firecrawl/models/crawl_response"
|
|
@@ -18,11 +20,20 @@ require_relative "firecrawl/models/map_options"
|
|
|
18
20
|
require_relative "firecrawl/models/map_data"
|
|
19
21
|
require_relative "firecrawl/models/search_options"
|
|
20
22
|
require_relative "firecrawl/models/search_data"
|
|
23
|
+
require_relative "firecrawl/models/developer_search_options"
|
|
24
|
+
require_relative "firecrawl/models/developer_search_response"
|
|
21
25
|
require_relative "firecrawl/models/parse_file"
|
|
22
26
|
require_relative "firecrawl/models/parse_options"
|
|
27
|
+
require_relative "firecrawl/models/parse_format"
|
|
28
|
+
require_relative "firecrawl/models/agent_exchange_options"
|
|
23
29
|
require_relative "firecrawl/models/agent_options"
|
|
24
30
|
require_relative "firecrawl/models/agent_response"
|
|
31
|
+
require_relative "firecrawl/models/agent_list_response"
|
|
32
|
+
require_relative "firecrawl/models/agent_exchange_summary"
|
|
33
|
+
require_relative "firecrawl/models/agent_pending_approval"
|
|
25
34
|
require_relative "firecrawl/models/agent_status_response"
|
|
35
|
+
require_relative "firecrawl/models/agent_trace_response"
|
|
36
|
+
require_relative "firecrawl/models/agent_snapshot_response"
|
|
26
37
|
require_relative "firecrawl/models/concurrency_check"
|
|
27
38
|
require_relative "firecrawl/models/credit_usage"
|
|
28
39
|
require_relative "firecrawl/models/monitor"
|
metadata
CHANGED
|
@@ -1,17 +1,18 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: firecrawl-sdk
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.17.7
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Firecrawl
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-
|
|
11
|
+
date: 2026-10-20 00:00:00.000000000 Z
|
|
12
12
|
dependencies: []
|
|
13
|
-
description: A type-safe Ruby client for the Firecrawl v2 API
|
|
14
|
-
|
|
13
|
+
description: 'A type-safe Ruby client for the Firecrawl v2 API: search, scrape, parse,
|
|
14
|
+
crawl, map, and interact with the web, run batch scrapes, and run Agent and Monitor
|
|
15
|
+
jobs.'
|
|
15
16
|
email:
|
|
16
17
|
- hello@firecrawl.dev
|
|
17
18
|
executables: []
|
|
@@ -24,9 +25,16 @@ files:
|
|
|
24
25
|
- lib/firecrawl/client.rb
|
|
25
26
|
- lib/firecrawl/errors.rb
|
|
26
27
|
- lib/firecrawl/http_client.rb
|
|
28
|
+
- lib/firecrawl/models/agent_exchange_options.rb
|
|
29
|
+
- lib/firecrawl/models/agent_exchange_summary.rb
|
|
30
|
+
- lib/firecrawl/models/agent_list_response.rb
|
|
27
31
|
- lib/firecrawl/models/agent_options.rb
|
|
32
|
+
- lib/firecrawl/models/agent_pending_approval.rb
|
|
28
33
|
- lib/firecrawl/models/agent_response.rb
|
|
34
|
+
- lib/firecrawl/models/agent_snapshot_response.rb
|
|
29
35
|
- lib/firecrawl/models/agent_status_response.rb
|
|
36
|
+
- lib/firecrawl/models/agent_trace_response.rb
|
|
37
|
+
- lib/firecrawl/models/audit_metadata.rb
|
|
30
38
|
- lib/firecrawl/models/batch_scrape_job.rb
|
|
31
39
|
- lib/firecrawl/models/batch_scrape_options.rb
|
|
32
40
|
- lib/firecrawl/models/batch_scrape_response.rb
|
|
@@ -35,13 +43,17 @@ files:
|
|
|
35
43
|
- lib/firecrawl/models/crawl_options.rb
|
|
36
44
|
- lib/firecrawl/models/crawl_response.rb
|
|
37
45
|
- lib/firecrawl/models/credit_usage.rb
|
|
46
|
+
- lib/firecrawl/models/developer_search_options.rb
|
|
47
|
+
- lib/firecrawl/models/developer_search_response.rb
|
|
38
48
|
- lib/firecrawl/models/document.rb
|
|
39
49
|
- lib/firecrawl/models/map_data.rb
|
|
40
50
|
- lib/firecrawl/models/map_options.rb
|
|
41
51
|
- lib/firecrawl/models/menu_profile.rb
|
|
42
52
|
- lib/firecrawl/models/monitor.rb
|
|
43
53
|
- lib/firecrawl/models/parse_file.rb
|
|
54
|
+
- lib/firecrawl/models/parse_format.rb
|
|
44
55
|
- lib/firecrawl/models/parse_options.rb
|
|
56
|
+
- lib/firecrawl/models/pdf_parser.rb
|
|
45
57
|
- lib/firecrawl/models/product_profile.rb
|
|
46
58
|
- lib/firecrawl/models/query_format.rb
|
|
47
59
|
- lib/firecrawl/models/scrape_options.rb
|
|
@@ -74,5 +86,5 @@ requirements: []
|
|
|
74
86
|
rubygems_version: 3.5.22
|
|
75
87
|
signing_key:
|
|
76
88
|
specification_version: 4
|
|
77
|
-
summary: Ruby SDK for the
|
|
89
|
+
summary: Ruby SDK for Firecrawl, the context API for AI agents
|
|
78
90
|
test_files: []
|