crawlfox 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: bee7b5227e2463cfd1bd2f1bc30eebe985d9dc40672af60446174027a80c2e61
+  data.tar.gz: 64cbd128cf1ed931049d053f9ffd2e6baa93a40fa15f2621dda7fa99edcb441d
+SHA512:
+  metadata.gz: 957829da07e3df0c495235ad1df257be32a699db15893681034a7e466278c20bba86b24047737e22a238bd9796a42ef533f314eb19bffc67adb20a8e6cd91f46
+  data.tar.gz: 422d16cc631d9bd98c83e0f1606d8b43b5f6dbdac321f93bc00800fe1d4799058530e1f31960e700bedfe3fdf14231c587965faae5ca81e7370073df277fca1a

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Automote LLC
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,39 @@
+# crawlfox (Ruby)
+
+Official Ruby SDK for the [CrawlFox](https://crawlfox.io) API: scrape, search (Google, Bing, or DuckDuckGo), batch scrape, and logs.
+
+Get a key from the [dashboard](https://crawlfox.io). Set `CRAWLFOX_API_KEY`.
+
+```bash
+gem install crawlfox
+```
+
+Until RubyGems lists it:
+
+```ruby
+gem "crawlfox", github: "Automote-LLC/crawlfox-integrations", glob: "packages/ruby/*.gemspec"
+```
+
+```ruby
+require "crawlfox"
+
+app = Crawlfox::Client.new
+
+doc = app.scrape("https://example.com", formats: ["markdown", "links"])
+puts doc["markdown"]
+
+hits = app.search("rust async tutorial", engine: "google", num: 10)
+
+batch = app.batch(
+  ["https://example.com/", "https://example.org/"],
+  formats: ["markdown"]
+)
+```
+
+Formats: `markdown`, `html`, `rawHtml`, `json`, `links`, `images`, `emails`. Use `json_options` / CSS selectors for `json`. Credits: 1 per page, 1 per 10 requested search results.
+
+```bash
+ruby -Ilib:test test/client_test.rb
+```
+
+Docs: https://docs.crawlfox.io

data/lib/crawlfox.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+require "uri"
+
+module Crawlfox
+  VERSION = "0.1.0"
+  DEFAULT_API_URL = "https://api.crawlfox.io"
+
+  class Error < StandardError
+    attr_reader :status, :code, :retryable, :body
+
+    def initialize(message, status:, code: nil, retryable: nil, body: {})
+      super(message)
+      @status = status
+      @code = code
+      @retryable = retryable
+      @body = body
+    end
+
+    def retryable_effective?
+      return false if retryable == false
+      return true if retryable == true
+
+      [502, 503, 504].include?(status)
+    end
+  end
+
+  class Client
+    def initialize(api_key: nil, api_url: nil, timeout: 120, max_retries: 2, retry_backoff_ms: 200, transport: nil)
+      @api_key = api_key || ENV["CRAWLFOX_API_KEY"]
+      raise ArgumentError, "CrawlFox API key required. Pass api_key or set CRAWLFOX_API_KEY." if @api_key.to_s.empty?
+
+      @api_url = (api_url || ENV["CRAWLFOX_API_URL"] || DEFAULT_API_URL).to_s.sub(%r{/$}, "")
+      @timeout = timeout
+      @max_retries = max_retries
+      @retry_backoff_ms = retry_backoff_ms
+      @transport = transport || method(:net_http_transport)
+    end
+
+    def scrape(url, **options)
+      document(request("POST", "/v1/scrape", { url: url }.merge(options)))
+    end
+
+    def scrape_get(url)
+      document(request("GET", "/v1/scrape/#{URI.encode_www_form_component(url)}"))
+    end
+
+    def batch(urls, **options)
+      env = request("POST", "/v1/batch", { urls: urls }.merge(options))
+      {
+        "success" => env.fetch("success", true),
+        "count" => env["count"] || Array(env["results"]).length,
+        "data" => Array(env["results"]).map { |item| document(item) }
+      }
+    end
+
+    def search(q, **options)
+      env = request("POST", "/v1/search", { q: q }.merge(options))
+      {
+        "success" => env.fetch("success", true),
+        "web" => env.dig("data", "web") || [],
+        "creditsUsed" => env["creditsUsed"],
+        "id" => env["id"]
+      }
+    end
+
+    def get_log(id)
+      request("GET", "/v1/logs/#{URI.encode_www_form_component(id)}")
+    end
+
+    def get_log_result(id)
+      request("GET", "/v1/logs/#{URI.encode_www_form_component(id)}/result")
+    end
+
+    private
+
+    def document(envelope)
+      data = envelope["data"].is_a?(Hash) ? envelope["data"].dup : envelope.dup
+      data["success"] = envelope.fetch("success", true)
+      data
+    end
+
+    def request(method, path, body = nil)
+      payload = body && JSON.generate(body)
+      headers = {
+        "Authorization" => "Bearer #{@api_key}",
+        "Content-Type" => "application/json",
+        "User-Agent" => "crawlfox-ruby/#{VERSION}"
+      }
+      url = @api_url + path
+      attempts = @max_retries + 1
+      last = nil
+      attempts.times do |i|
+        status, raw = @transport.call(method, url, headers, payload)
+        json = JSON.parse(raw.to_s.empty? ? "{}" : raw)
+        return json if status >= 200 && status < 300
+
+        err = Error.new(json["message"] || json["title"] || "CrawlFox request failed (#{status})",
+                        status: status, code: json["code"], retryable: json["retryable"], body: json)
+        raise err unless i < attempts - 1 && err.retryable_effective?
+
+        last = err
+        sleep(@retry_backoff_ms / 1000.0 * (2**i))
+      end
+      raise last || Error.new("Request failed", status: 0)
+    end
+
+    def net_http_transport(method, url, headers, body)
+      uri = URI.parse(url)
+      req = Net::HTTPGenericRequest.new(method, !body.nil?, true, uri)
+      headers.each { |k, v| req[k] = v }
+      req.body = body if body
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = uri.scheme == "https"
+      http.read_timeout = @timeout
+      res = http.request(req)
+      [res.code.to_i, res.body.to_s]
+    end
+  end
+end

metadata

@@ -0,0 +1,49 @@
+--- !ruby/object:Gem::Specification
+name: crawlfox
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Automote LLC
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2026-09-10 00:00:00.000000000 Z
+dependencies: []
+description:
+email:
+- pratik@automote.io
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- lib/crawlfox.rb
+homepage: https://crawlfox.io
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://crawlfox.io
+  source_code_uri: https://github.com/Automote-LLC/crawlfox-integrations
+  documentation_uri: https://docs.crawlfox.io
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.20
+signing_key:
+specification_version: 4
+summary: Official Ruby SDK for the CrawlFox scrape and search API
+test_files: []