firecrawl-sdk 1.7.0 → 1.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 1c5ff8541498b47b08ff9b54d1c14969c734fba849e09c3fcc1d48143be53f1e
|
|
4
|
+
data.tar.gz: 71fb5c61c9c20ec229d077b7db82eb2ba60c48581a0b96969f7d515fa91778eb
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 2a44065aa34a8bd32f691ebd249c4d11b48998582dd6ba9f5da6bca80d95370341be3b0dd0482076918b8ccad268b928034b9bd477daddd831bc042da59391a2
|
|
7
|
+
data.tar.gz: 347d45a6810b3b66afb4a6bbb6b977f13dbf770c2c57eaedcb8bc9d7c4b071bf3be25f299c606adc74cc74819a7f6c23ae9009dbe183d789a7c36dc76a43ec8e
|
data/lib/firecrawl/client.rb
CHANGED
|
@@ -39,9 +39,10 @@ module Firecrawl
|
|
|
39
39
|
backoff_factor: DEFAULT_BACKOFF_FACTOR
|
|
40
40
|
)
|
|
41
41
|
resolved_key = api_key || ENV["FIRECRAWL_API_KEY"]
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
42
|
+
# A nil/empty key is allowed: scrape, search, and interact fall back to the
|
|
43
|
+
# keyless free tier (rate-limited per IP). Other methods return 401 from the
|
|
44
|
+
# API until a key is provided.
|
|
45
|
+
resolved_key = nil if resolved_key.nil? || resolved_key.strip.empty?
|
|
45
46
|
|
|
46
47
|
resolved_url = api_url || ENV["FIRECRAWL_API_URL"] || DEFAULT_API_URL
|
|
47
48
|
unless resolved_url.match?(%r{\Ahttps?://}i)
|
|
@@ -78,6 +79,7 @@ module Firecrawl
|
|
|
78
79
|
|
|
79
80
|
body = { "url" => url }
|
|
80
81
|
body.merge!(options.to_h) if options
|
|
82
|
+
body["origin"] ||= "ruby-sdk@#{Firecrawl::VERSION}"
|
|
81
83
|
raw = @http.post("/v2/scrape", body)
|
|
82
84
|
data = raw["data"] || raw
|
|
83
85
|
Models::Document.new(data)
|
|
@@ -96,6 +98,7 @@ module Firecrawl
|
|
|
96
98
|
|
|
97
99
|
body = { "code" => code, "language" => language }
|
|
98
100
|
body["timeout"] = timeout if timeout
|
|
101
|
+
body["origin"] ||= "ruby-sdk@#{Firecrawl::VERSION}"
|
|
99
102
|
@http.post("/v2/scrape/#{job_id}/interact", body)
|
|
100
103
|
end
|
|
101
104
|
|
|
@@ -377,6 +380,7 @@ module Firecrawl
|
|
|
377
380
|
|
|
378
381
|
body = { "query" => query }
|
|
379
382
|
body.merge!(options.to_h) if options
|
|
383
|
+
body["origin"] ||= "ruby-sdk@#{Firecrawl::VERSION}"
|
|
380
384
|
raw = @http.post("/v2/search", body)
|
|
381
385
|
data = raw["data"] || raw
|
|
382
386
|
Models::SearchData.new(data)
|
|
@@ -23,7 +23,7 @@ module Firecrawl
|
|
|
23
23
|
def post(path, body, extra_headers: {})
|
|
24
24
|
uri = URI("#{@base_url}#{path}")
|
|
25
25
|
request = Net::HTTP::Post.new(uri)
|
|
26
|
-
request["Authorization"] = "Bearer #{@api_key}"
|
|
26
|
+
request["Authorization"] = "Bearer #{@api_key}" if @api_key
|
|
27
27
|
request["Content-Type"] = "application/json"
|
|
28
28
|
extra_headers.each { |k, v| request[k] = v }
|
|
29
29
|
request.body = JSON.generate(body)
|
|
@@ -34,7 +34,7 @@ module Firecrawl
|
|
|
34
34
|
def get(path)
|
|
35
35
|
uri = URI("#{@base_url}#{path}")
|
|
36
36
|
request = Net::HTTP::Get.new(uri)
|
|
37
|
-
request["Authorization"] = "Bearer #{@api_key}"
|
|
37
|
+
request["Authorization"] = "Bearer #{@api_key}" if @api_key
|
|
38
38
|
execute_with_retry(uri, request)
|
|
39
39
|
end
|
|
40
40
|
|
|
@@ -47,7 +47,7 @@ module Firecrawl
|
|
|
47
47
|
raise FirecrawlError, "Absolute URL origin (#{uri.scheme}://#{uri.host}:#{uri.port}) does not match API base URL origin (#{base_uri.scheme}://#{base_uri.host}:#{base_uri.port}). Refusing to send credentials."
|
|
48
48
|
end
|
|
49
49
|
request = Net::HTTP::Get.new(uri)
|
|
50
|
-
request["Authorization"] = "Bearer #{@api_key}"
|
|
50
|
+
request["Authorization"] = "Bearer #{@api_key}" if @api_key
|
|
51
51
|
execute_with_retry(uri, request)
|
|
52
52
|
end
|
|
53
53
|
|
|
@@ -55,7 +55,7 @@ module Firecrawl
|
|
|
55
55
|
def delete(path)
|
|
56
56
|
uri = URI("#{@base_url}#{path}")
|
|
57
57
|
request = Net::HTTP::Delete.new(uri)
|
|
58
|
-
request["Authorization"] = "Bearer #{@api_key}"
|
|
58
|
+
request["Authorization"] = "Bearer #{@api_key}" if @api_key
|
|
59
59
|
execute_with_retry(uri, request)
|
|
60
60
|
end
|
|
61
61
|
|
|
@@ -63,7 +63,7 @@ module Firecrawl
|
|
|
63
63
|
def patch(path, body)
|
|
64
64
|
uri = URI("#{@base_url}#{path}")
|
|
65
65
|
request = Net::HTTP::Patch.new(uri)
|
|
66
|
-
request["Authorization"] = "Bearer #{@api_key}"
|
|
66
|
+
request["Authorization"] = "Bearer #{@api_key}" if @api_key
|
|
67
67
|
request["Content-Type"] = "application/json"
|
|
68
68
|
request.body = JSON.generate(body)
|
|
69
69
|
execute_with_retry(uri, request)
|
|
@@ -84,7 +84,7 @@ module Firecrawl
|
|
|
84
84
|
|
|
85
85
|
builder = lambda do
|
|
86
86
|
request = Net::HTTP::Post.new(uri)
|
|
87
|
-
request["Authorization"] = "Bearer #{@api_key}"
|
|
87
|
+
request["Authorization"] = "Bearer #{@api_key}" if @api_key
|
|
88
88
|
request["Content-Type"] = "multipart/form-data; boundary=#{boundary}"
|
|
89
89
|
request.body = body
|
|
90
90
|
request
|
|
@@ -13,7 +13,7 @@ module Firecrawl
|
|
|
13
13
|
FIELDS = %i[
|
|
14
14
|
formats headers include_tags exclude_tags only_main_content
|
|
15
15
|
timeout parsers skip_tls_verification remove_base64_images
|
|
16
|
-
block_ads proxy integration json_options
|
|
16
|
+
block_ads proxy integration redact_pii json_options
|
|
17
17
|
].freeze
|
|
18
18
|
|
|
19
19
|
attr_reader(*FIELDS)
|
|
@@ -38,6 +38,7 @@ module Firecrawl
|
|
|
38
38
|
"blockAds" => block_ads,
|
|
39
39
|
"proxy" => proxy,
|
|
40
40
|
"integration" => integration,
|
|
41
|
+
"redactPII" => redact_pii,
|
|
41
42
|
"jsonOptions" => json_options.is_a?(Hash) ? json_options : json_options&.to_h,
|
|
42
43
|
}.compact
|
|
43
44
|
end
|
|
@@ -8,7 +8,7 @@ module Firecrawl
|
|
|
8
8
|
formats headers include_tags exclude_tags only_main_content
|
|
9
9
|
timeout wait_for mobile parsers actions location
|
|
10
10
|
skip_tls_verification remove_base64_images block_ads proxy
|
|
11
|
-
max_age store_in_cache lockdown integration
|
|
11
|
+
max_age store_in_cache lockdown redact_pii integration
|
|
12
12
|
].freeze
|
|
13
13
|
|
|
14
14
|
attr_reader(*FIELDS)
|
|
@@ -38,6 +38,7 @@ module Firecrawl
|
|
|
38
38
|
"maxAge" => max_age,
|
|
39
39
|
"storeInCache" => store_in_cache,
|
|
40
40
|
"lockdown" => lockdown,
|
|
41
|
+
"redactPII" => redact_pii,
|
|
41
42
|
"integration" => integration,
|
|
42
43
|
}.compact
|
|
43
44
|
end
|
data/lib/firecrawl/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: firecrawl-sdk
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.8.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Firecrawl
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-
|
|
11
|
+
date: 2026-06-15 00:00:00.000000000 Z
|
|
12
12
|
dependencies: []
|
|
13
13
|
description: A type-safe Ruby client for the Firecrawl v2 API. Supports scraping,
|
|
14
14
|
crawling, batch scraping, URL mapping, web search, and AI agent operations.
|