crawlfox 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bee7b5227e2463cfd1bd2f1bc30eebe985d9dc40672af60446174027a80c2e61
-  data.tar.gz: 64cbd128cf1ed931049d053f9ffd2e6baa93a40fa15f2621dda7fa99edcb441d
+  metadata.gz: 3f00152d3e38ba69cf8c03b41ffafd218d383f00da3229716440bd4c931c4c38
+  data.tar.gz: 3094b245a4b034b50f05a38bd93b6593e9796c5ca7184f01beaafa887ac3bd6f
 SHA512:
-  metadata.gz: 957829da07e3df0c495235ad1df257be32a699db15893681034a7e466278c20bba86b24047737e22a238bd9796a42ef533f314eb19bffc67adb20a8e6cd91f46
-  data.tar.gz: 422d16cc631d9bd98c83e0f1606d8b43b5f6dbdac321f93bc00800fe1d4799058530e1f31960e700bedfe3fdf14231c587965faae5ca81e7370073df277fca1a
+  metadata.gz: 55be4fd634749a72da737def709b7c772e76317c19b1684acbfcb7b8d571ceb64ac1489c908b6dc5f92ff9effab624c96ee798a060c8d18895bc12ef243fe524
+  data.tar.gz: 984f31a8c6149f885fb6b62a9f976c59745e562c3e43401c039e5bcd5d563c9d6a6c0b589780045181b7db1ce0c789005148a8fe671d65a2d6c65277188a1e7c

data/README.md

@@ -1,6 +1,6 @@
 # crawlfox (Ruby)
 
-Official Ruby SDK for the [CrawlFox](https://crawlfox.io) API: scrape, search (Google, Bing, or DuckDuckGo), batch scrape, and logs.
+Official Ruby SDK for the [CrawlFox](https://crawlfox.io) API: scrape, search (Google, Bing, or DuckDuckGo), batch scrape, map site links, crawl multi-page jobs, and logs.
 
 Get a key from the [dashboard](https://crawlfox.io). Set `CRAWLFOX_API_KEY`.
 
@@ -30,7 +30,28 @@ batch = app.batch(
 )
 ```
 
-Formats: `markdown`, `html`, `rawHtml`, `json`, `links`, `images`, `emails`. Use `json_options` / CSS selectors for `json`. Credits: 1 per page, 1 per 10 requested search results.
+Formats: `markdown`, `html`, `rawHtml`, `json`, `links`, `images`, `emails`. Use `json_options` / CSS selectors for `json`.
+
+## Map
+
+`POST /v1/map`. One credit per call.
+
+```ruby
+mapped = app.map("https://example.com/", limit: 10)
+mapped["links"].each { |link| puts link["url"] }
+```
+
+## Crawl
+
+```ruby
+job = app.crawl("https://example.com/", limit: 3, scrapeOptions: { formats: ["markdown"] })
+status = app.get_crawl(job["id"])
+done = app.wait_for_crawl(job["id"])
+app.cancel_crawl(job["id"])
+errors = app.get_crawl_errors(job["id"])
+```
+
+Credits: scrape 1 per page, search 1 per 10 requested results, map 1 per call, crawl 1 per page per format.
 
 ```bash
 ruby -Ilib:test test/client_test.rb

data/lib/crawlfox.rb

@@ -5,7 +5,7 @@ require "net/http"
 require "uri"
 
 module Crawlfox
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
   DEFAULT_API_URL = "https://api.crawlfox.io"
 
   class Error < StandardError
@@ -74,6 +74,78 @@ module Crawlfox
       request("GET", "/v1/logs/#{URI.encode_www_form_component(id)}/result")
     end
 
+    def map(url, **options)
+      env = request("POST", "/v1/map", { url: url }.merge(options))
+      {
+        "success" => env.fetch("success", true),
+        "id" => env["id"],
+        "links" => env["links"] || []
+      }
+    end
+
+    def crawl(url, **options)
+      env = request("POST", "/v1/crawl", { url: url }.merge(options))
+      raise Error.new("crawl response missing id", status: 500, retryable: false) if env["id"].to_s.empty?
+      env
+    end
+
+    def get_crawl(id, skip: nil)
+      path = "/v1/crawl/#{URI.encode_www_form_component(id)}"
+      path += "?skip=#{skip}" if skip && skip.to_i > 0
+      request("GET", path)
+    end
+
+    def cancel_crawl(id)
+      request("DELETE", "/v1/crawl/#{URI.encode_www_form_component(id)}")
+      nil
+    end
+
+    def get_crawl_errors(id)
+      request("GET", "/v1/crawl/#{URI.encode_www_form_component(id)}/errors")
+    end
+
+    def wait_for_crawl(id, poll_interval_ms: 1500, timeout_ms: 180_000)
+      started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      skip = 0
+      merged = nil
+
+      loop do
+        page = get_crawl(id, skip: skip > 0 ? skip : nil)
+        if merged.nil?
+          merged = page.dup
+          merged["data"] = Array(page["data"]).dup
+        else
+          merged["status"] = page["status"]
+          merged["total"] = page["total"]
+          merged["completed"] = page["completed"]
+          merged["creditsUsed"] = page["creditsUsed"]
+          merged["next"] = page["next"]
+          merged["data"].concat(Array(page["data"]))
+        end
+
+        status = page["status"]
+        nxt = page["next"]
+        if %w[completed failed cancelled].include?(status) && nxt.to_s.empty?
+          return merged
+        end
+        if nxt && !nxt.empty?
+          begin
+            u = URI.parse(nxt)
+            s = u.query ? URI.decode_www_form(u.query).to_h["skip"] : nil
+            skip = s ? s.to_i : skip + Array(page["data"]).length
+          rescue StandardError
+            skip += Array(page["data"]).length
+          end
+        elsif %w[completed failed cancelled].include?(status)
+          return merged
+        end
+        if (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000 > timeout_ms
+          raise Error.new("crawl poll timed out", status: 408, retryable: true)
+        end
+        sleep(poll_interval_ms / 1000.0)
+      end
+    end
+
     private
 
     def document(envelope)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: crawlfox
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Automote LLC
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-09-10 00:00:00.000000000 Z
+date: 2026-09-18 00:00:00.000000000 Z
 dependencies: []
 description:
 email:
@@ -45,5 +45,6 @@ requirements: []
 rubygems_version: 3.4.20
 signing_key:
 specification_version: 4
-summary: Official Ruby SDK for the CrawlFox scrape and search API
+summary: 'Official Ruby SDK for CrawlFox: scrape, batch, search, map site URLs, and
+  crawl sites with polling helpers.'
 test_files: []