webcrawlerapi 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 3acecf5769685f3baf072a393c7b9a997c91a896413729a12777fab44500ff6a
+  data.tar.gz: 6fefd3be1bdc588b9a0cbb5d124619576b4207f31d707fbfa4d77d42f5387a5e
+SHA512:
+  metadata.gz: ba2c0ce429435fbfb2af6e45f7499f5d61c05716918ddd2ada0811e6937650da2bfc6aea55d85b10d3742bcd87364d2bf91a58b3dc9bf0a3001389a90860f38e
+  data.tar.gz: 7ff338ddc9e2d709843a19b863ec1ddd377a872217e2784af9a59cf28360dcf335a41997cc18d3e2031e2aaf5c22be9790f17ae3bf8adb262406be521e81f88b

data/CHANGELOG.md

@@ -0,0 +1,10 @@
+# Changelog
+
+## Unreleased
+
+- Batch scrape: `scrape(url: [...])`, `scrape_batch`, `scrape_batch_async`, `get_batch`, `get_batch_errors`, `cancel_batch`.
+
+## 0.1.0
+
+- Initial release. Port of the JS SDK: `scrape`, `scrape_async`, `get_scrape`, `crawl`, `crawl_async`, `get_job`,
+  `get_job_markdown`, `get_job_markdown_content`, `markdown`, `run_agent`, `run_agent_async`, `get_agent_job`.

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 WebcrawlerAPI
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,93 @@
+# WebcrawlerAPI Ruby SDK
+
+Ruby client for [WebcrawlerAPI](https://webcrawlerapi.com). No runtime dependencies, Ruby 3.0+.
+
+## Install
+
+```ruby
+gem "webcrawlerapi"
+```
+
+or `gem install webcrawlerapi`.
+
+## Usage
+
+```ruby
+require "webcrawlerapi"
+
+client = WebCrawlerAPI::Client.new("YOUR_API_KEY")
+
+# Scrape one page (polls until done)
+res = client.scrape(url: "https://example.com", output_formats: ["markdown"])
+puts res.markdown
+
+# Scrape many URLs: pass an array, polls until the batch finishes.
+# Returns the first page of items (`data`) and the `next` cursor.
+batch = client.scrape(url: ["https://example.com", "https://example.com/pricing"],
+                      output_formats: ["markdown"], max_concurrency: 5)
+batch.data.each { |item| puts "#{item['url']} #{item['status']} #{item['markdown_url']}" }
+more = client.get_batch(batch.id, cursor: batch.next, limit: 100) unless batch.next.to_s.empty?
+
+# Crawl a site (polls until the job finishes)
+job = client.crawl(url: "https://example.com", items_limit: 10)
+job.job_items.each do |item|
+  puts item.original_url
+  puts item.get_content   # markdown by default
+end
+
+# Markdown of a single article
+puts client.markdown(url: "https://example.com/post").markdown
+
+# Agent
+run = client.run_agent(prompt: "Find pricing of example.com", urls: ["https://example.com"], max_spend_usd: 1)
+puts run.data
+```
+
+## Methods (same as the JS SDK, snake_case)
+
+| JS | Ruby |
+|----|------|
+| `scrape` / `scrapeAsync` / `getScrape` | `scrape` / `scrape_async` / `get_scrape` |
+| `crawl` / `crawlAsync` / `getJob` | `crawl` / `crawl_async` / `get_job` |
+| `getJobMarkdown` / `getJobMarkdownContent` | `get_job_markdown` / `get_job_markdown_content` |
+| `markdown` | `markdown` |
+| `runAgent` / `runAgentAsync` / `getAgentJob` | `run_agent` / `run_agent_async` / `get_agent_job` |
+| `item.getContent/getMarkdown/getCleaned/getHTML` | `item.get_content/get_markdown/get_cleaned/get_html` |
+
+Batch scrape: `scrape(url: [...])` (or `scrape_batch`) waits for the batch; `scrape_batch_async`, `get_batch(id, limit:, cursor:)`,
+`get_batch_errors(id)` and `cancel_batch(id)` cover the non-waiting flow. Batch-only options: `max_concurrency`,
+`ignore_invalid_urls`, `webhook_url`.
+
+Full batch scrape documentation: [docs/batch-scrape.md](docs/batch-scrape.md).
+
+`scrape`, `crawl` and `run_agent` take an optional second argument `max_polls`.
+Responses are `WebCrawlerAPI::Model` objects: `res.markdown`, `res[:markdown]` or `res.to_h`.
+
+## Errors
+
+Every failure raises `WebCrawlerAPI::Error` with `error_code`, `error_message`, `status_code`
+(`network_error`, `timeout`, `invalid_response` are raised client-side).
+
+```ruby
+begin
+  client.crawl(url: "https://example.com")
+rescue WebCrawlerAPI::Error => e
+  warn "#{e.error_code} (#{e.status_code}): #{e.error_message}"
+end
+```
+
+## Options
+
+```ruby
+WebCrawlerAPI::Client.new("KEY", base_path: "https://api.webcrawlerapi.com", api_version: "v1",
+                                 open_timeout: 10, read_timeout: 60)
+```
+
+## Development
+
+```sh
+bundle install
+bundle exec rake test
+```
+
+MIT license.

data/lib/webcrawlerapi/client.rb

@@ -0,0 +1,268 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+require "uri"
+
+module WebCrawlerAPI
+  class Client
+    NO_CACHE_HEADERS = {
+      "Cache-Control" => "no-cache, no-store, must-revalidate",
+      "Pragma" => "no-cache",
+      "Expires" => "0"
+    }.freeze
+
+    # @param api_key [String]
+    # @param base_path [String]
+    # @param api_version [String]
+    # @param open_timeout [Numeric] seconds
+    # @param read_timeout [Numeric] seconds
+    # @param sleeper [#call] called with seconds to wait between polls (override in tests)
+    def initialize(api_key, base_path: BASE_PATH, api_version: "v1", open_timeout: 10, read_timeout: 60,
+                   sleeper: Kernel.method(:sleep))
+      raise ArgumentError, "api_key is required" if api_key.nil? || api_key.to_s.strip.empty?
+
+      @api_key = api_key
+      @base_path = base_path.to_s.chomp("/")
+      @api_version = api_version
+      @open_timeout = open_timeout
+      @read_timeout = read_timeout
+      @sleeper = sleeper
+    end
+
+    # ---- Scrape -----------------------------------------------------------
+
+    # Starts a scrape and returns its id.
+    # @return [ScrapeId]
+    def scrape_async(request)
+      data = send_request(:post, "#{@base_path}/#{SCRAPE_VERSION}/scrape?async=true", body: request)
+      ScrapeId.new("id" => data["id"])
+    end
+
+    # Returns the scrape result when done/error, otherwise an in-progress placeholder.
+    # @return [ScrapeResponse]
+    def get_scrape(scrape_id)
+      data = send_request(:get, "#{@base_path}/#{SCRAPE_VERSION}/scrape/#{scrape_id}", no_cache: true)
+      status = data["status"]
+      return ScrapeResponse.new(data) if %w[done error].include?(status)
+
+      ScrapeResponse.new(
+        "success" => false,
+        "status" => status,
+        "page_status_code" => 0,
+        "recommended_pull_delay_ms" => data["recommended_pull_delay_ms"]
+      )
+    end
+
+    # Starts a scrape and polls until done, error or timeout. Returns the last known state on timeout.
+    #
+    # If +request[:url]+ is an Array of URLs this creates a batch scrape instead (see #scrape_batch_async),
+    # polls until the batch leaves +scraping+ and returns the first page of items plus the +next+ cursor
+    # (use #get_batch with +cursor:+ for more). Raises a +timeout+ Error if the batch is still running
+    # after +max_polls+ polls. Batch-only options (+max_concurrency+, +webhook_url+, +ignore_invalid_urls+)
+    # go in the same request hash.
+    # @return [ScrapeResponse, BatchScrapeResponse]
+    def scrape(request, max_polls = SCRAPE_MAX_POLLS)
+      return scrape_batch(request, max_polls) if request[:url].is_a?(Array) || request["url"].is_a?(Array)
+
+      scrape_id = scrape_async(request)[:id]
+      started = monotonic_now
+      polls = 0
+      result = nil
+
+      while polls < max_polls && monotonic_now - started < SCRAPE_MAX_WAIT_SECONDS
+        result = get_scrape(scrape_id)
+        return result if %w[done error].include?(result[:status]) || result.key?(:error_code)
+
+        recommended = result[:recommended_pull_delay_ms]
+        delay = recommended.to_f.positive? ? recommended / 1000.0 : DEFAULT_POLL_DELAY_SECONDS
+        remaining = SCRAPE_MAX_WAIT_SECONDS - (monotonic_now - started)
+        break if remaining <= 0
+
+        @sleeper.call([delay, remaining].min)
+        polls += 1
+      end
+
+      result
+    end
+
+    # ---- Batch scrape -----------------------------------------------------
+
+    # Creates a batch scrape (POST /v2/batch/scrape) and returns +id+, +status_url+ and +invalid_urls+.
+    # The URL list is taken from +url+ (Array) or +urls+.
+    # @return [BatchScrapeId]
+    def scrape_batch_async(request)
+      body = stringify_keys(request)
+      urls = body.delete("url")
+      body["urls"] = urls if urls.is_a?(Array)
+      BatchScrapeId.new(send_request(:post, "#{@base_path}/#{SCRAPE_VERSION}/batch/scrape", body: body))
+    end
+
+    # Returns batch progress and one page of items. +next+ is the cursor for the following page ("" on the last).
+    # @param limit [Integer, nil] items per page, 1 to 500
+    # @param cursor [String, nil] +next+ value from the previous page
+    # @return [BatchScrapeResponse]
+    def get_batch(batch_id, limit: nil, cursor: nil)
+      query = URI.encode_www_form({ "limit" => limit, "cursor" => cursor }.compact)
+      url = "#{@base_path}/#{SCRAPE_VERSION}/batch/scrape/#{batch_id}"
+      url += "?#{query}" unless query.empty?
+      BatchScrapeResponse.new(send_request(:get, url, no_cache: true))
+    end
+
+    # Lists failed URLs (+errors+) and URLs blocked by robots.txt (+robots_blocked+).
+    # @return [BatchScrapeErrors]
+    def get_batch_errors(batch_id)
+      BatchScrapeErrors.new(send_request(:get, "#{@base_path}/#{SCRAPE_VERSION}/batch/scrape/#{batch_id}/errors",
+                                         no_cache: true))
+    end
+
+    # Cancels a running batch. Returns +{"status" => "canceled"}+.
+    # @return [Model]
+    def cancel_batch(batch_id)
+      Model.new(send_request(:delete, "#{@base_path}/#{SCRAPE_VERSION}/batch/scrape/#{batch_id}"))
+    end
+
+    # Creates a batch and polls until it is no longer +scraping+. Same as +scrape(url: [...])+.
+    # @return [BatchScrapeResponse]
+    def scrape_batch(request, max_polls = SCRAPE_MAX_POLLS)
+      batch_id = scrape_batch_async(request)[:id]
+      raise Error.new("invalid_response", "Failed to create batch", 0) if batch_id.nil? || batch_id.empty?
+
+      max_polls.times do
+        batch = get_batch(batch_id)
+        return batch unless batch[:status] == BATCH_SCRAPING_STATUS
+
+        @sleeper.call(DEFAULT_POLL_DELAY_SECONDS)
+      end
+
+      raise Error.new("timeout", "Batch scrape took too long, please retry or increase the number of polling retries", 0)
+    end
+
+    # ---- Crawl ------------------------------------------------------------
+
+    # Starts a crawl and polls until the job leaves new/in_progress.
+    # @return [Job]
+    def crawl(request, max_polls = MAX_PULL_RETRIES)
+      job_id = crawl_async(request)[:id]
+      raise Error.new("invalid_response", "Failed to fetch job status", 0) if job_id.nil? || job_id.empty?
+
+      delay = INITIAL_PULL_DELAY_SECONDS
+      max_polls.times do
+        @sleeper.call(delay)
+        job = get_job(job_id)
+        return job unless [JobStatus::IN_PROGRESS, JobStatus::NEW].include?(job[:status])
+
+        delay = job[:recommended_pull_delay_ms] / 1000.0 if job[:recommended_pull_delay_ms].to_i.positive?
+      end
+
+      raise Error.new("timeout", "Crawling took too long, please retry or increase the number of polling retries", 0)
+    end
+
+    # @return [JobId]
+    def crawl_async(request)
+      body = { "output_formats" => ["markdown"] }.merge(stringify_keys(request))
+      JobId.new(send_request(:post, "#{@base_path}/#{@api_version}/crawl", body: body))
+    end
+
+    # @return [Job]
+    def get_job(job_id)
+      Job.new(send_request(:get, "#{@base_path}/#{@api_version}/job/#{job_id}", no_cache: true))
+    end
+
+    # @return [JobMarkdownResponse]
+    def get_job_markdown(job_id)
+      JobMarkdownResponse.new(send_request(:get, "#{@base_path}/#{@api_version}/job/#{job_id}/markdown", no_cache: true))
+    end
+
+    # @return [String]
+    def get_job_markdown_content(job_id)
+      send_request(:get, "#{@base_path}/#{@api_version}/job/#{job_id}/markdown/content", no_cache: true, raw: true)
+    end
+
+    # ---- Markdown ---------------------------------------------------------
+
+    # @return [MarkdownResponse]
+    def markdown(request)
+      MarkdownResponse.new(send_request(:post, "#{@base_path}/markdown", body: request))
+    end
+
+    # ---- Agent ------------------------------------------------------------
+
+    # @return [AgentRun]
+    def run_agent_async(request)
+      AgentRun.new(send_request(:post, "#{@base_path}/#{@api_version}/agent", body: request))
+    end
+
+    # @return [AgentRun]
+    def get_agent_job(job_id)
+      AgentRun.new(send_request(:get, "#{@base_path}/#{@api_version}/agent/job/#{job_id}", no_cache: true))
+    end
+
+    # Runs an agent and polls until done, error or canceled.
+    # @return [AgentRun]
+    def run_agent(request, max_polls = MAX_PULL_RETRIES)
+      run = run_agent_async(request)
+      raise Error.new("invalid_response", "Failed to fetch agent job status", 0) if run[:id].nil? || run[:id].empty?
+      return run if AGENT_TERMINAL_STATUSES.include?(run[:status])
+
+      max_polls.times do
+        @sleeper.call(DEFAULT_POLL_DELAY_SECONDS)
+        agent_job = get_agent_job("#{run[:id]}?t=#{(Time.now.to_f * 1000).to_i}")
+        return agent_job if AGENT_TERMINAL_STATUSES.include?(agent_job[:status])
+      end
+
+      raise Error.new("timeout", "Agent run took too long, please retry or increase the number of polling retries", 0)
+    end
+
+    private
+
+    def monotonic_now
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+
+    def stringify_keys(hash)
+      JSON.parse(JSON.generate(hash))
+    end
+
+    def headers(no_cache)
+      base = {
+        "Content-Type" => "application/json",
+        "Authorization" => "Bearer #{@api_key}",
+        "User-Agent" => "WebcrawlerAPI-Ruby-Client/#{VERSION}"
+      }
+      no_cache ? base.merge(NO_CACHE_HEADERS) : base
+    end
+
+    def send_request(method, url, body: nil, no_cache: false, raw: false)
+      uri = URI.parse(url)
+      klass = { post: Net::HTTP::Post, delete: Net::HTTP::Delete }.fetch(method, Net::HTTP::Get)
+      req = klass.new(uri, headers(no_cache))
+      req.body = JSON.generate(body) if body
+
+      response = begin
+        Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https",
+                                            open_timeout: @open_timeout, read_timeout: @read_timeout) do |http|
+          http.request(req)
+        end
+      rescue SystemCallError, SocketError, Timeout::Error, IOError, OpenSSL::SSL::SSLError => e
+        raise Error.new("network_error", "Failed to send request: #{e.message}", 0)
+      end
+
+      raise error_from(response) unless response.is_a?(Net::HTTPSuccess)
+
+      text = response.body.to_s.dup.force_encoding(Encoding::UTF_8)
+      return text if raw
+
+      JSON.parse(text)
+    rescue JSON::ParserError
+      raise Error.new("invalid_response", "Response is not valid JSON", response.code.to_i)
+    end
+
+    def error_from(response)
+      data = JSON.parse(response.body.to_s)
+      Error.from_response(response.code.to_i, data)
+    rescue JSON::ParserError
+      Error.new("unknown_error", "Request failed with status #{response.code} #{response.message}", response.code.to_i)
+    end
+  end
+end

data/lib/webcrawlerapi/constants.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module WebCrawlerAPI
+  module JobStatus
+    IN_PROGRESS = "in_progress"
+    NEW = "new"
+    DONE = "done"
+  end
+
+  module ErrorCode
+    BLOCKED_BY_ROBOTS_TXT = "blocked_by_robots_txt"
+  end
+
+  BASE_PATH = "https://api.webcrawlerapi.com"
+  SCRAPE_VERSION = "v2"
+  INITIAL_PULL_DELAY_SECONDS = 2.0
+  MAX_PULL_RETRIES = 100
+  DEFAULT_POLL_DELAY_SECONDS = 2.0
+  SCRAPE_MAX_WAIT_SECONDS = 400.0
+  # Effectively unbounded by count at the 2s default; elapsed time is the real limit.
+  SCRAPE_MAX_POLLS = 1000
+  BATCH_SCRAPING_STATUS = "scraping"
+  AGENT_TERMINAL_STATUSES = %w[done error canceled].freeze
+end

data/lib/webcrawlerapi/errors.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module WebCrawlerAPI
+  # Raised for every API, network and polling failure.
+  class Error < StandardError
+    attr_reader :error_code, :error_message, :status_code
+
+    def initialize(error_code, error_message, status_code = nil)
+      super(error_message)
+      @error_code = error_code
+      @error_message = error_message
+      @status_code = status_code
+    end
+
+    # Builds an error from a parsed API error body (Hash with string keys).
+    def self.from_response(status_code, data)
+      data = {} unless data.is_a?(Hash)
+      new(
+        data["error_code"] || "unknown_error",
+        data["error_message"] || data["error"] || "Unknown error",
+        status_code
+      )
+    end
+  end
+end

data/lib/webcrawlerapi/job.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+
+module WebCrawlerAPI
+  class JobItem < Model
+    def initialize(attributes, job)
+      super(attributes)
+      @job = job
+    end
+
+    # Content in the job's preferred format (markdown > cleaned > html). nil unless job and item are done.
+    def get_content
+      return nil unless @job[:status] == JobStatus::DONE && self[:status] == JobStatus::DONE
+
+      ContentFetcher.fetch(@job.resolve_content_url(self))
+    end
+
+    def get_markdown
+      ContentFetcher.fetch(self[:markdown_content_url])
+    end
+
+    def get_cleaned
+      ContentFetcher.fetch(self[:cleaned_content_url])
+    end
+
+    def get_html
+      ContentFetcher.fetch(self[:raw_content_url])
+    end
+  end
+
+  class Job < Model
+    CONTENT_PRIORITY = [
+      ["markdown", :markdown_content_url],
+      ["cleaned", :cleaned_content_url],
+      ["html", :raw_content_url]
+    ].freeze
+
+    SCRAPE_TYPE_URLS = {
+      "html" => :raw_content_url,
+      "cleaned" => :cleaned_content_url,
+      "markdown" => :markdown_content_url
+    }.freeze
+
+    def job_items
+      @job_items ||= Array(self["job_items"]).map { |item| JobItem.new(item, self) }
+    end
+
+    # @api private
+    def resolve_content_url(item)
+      formats = self[:output_formats]
+      if formats.is_a?(Array) && !formats.empty?
+        fmt, reader = CONTENT_PRIORITY.find { |name, _| formats.include?(name) }
+        return fmt ? item[reader] : nil
+      end
+
+      reader = SCRAPE_TYPE_URLS[self["scrape_type"]]
+      reader && item[reader]
+    end
+  end
+
+  # @api private
+  module ContentFetcher
+    def self.fetch(url)
+      return nil if url.nil? || url.to_s.empty?
+
+      uri = URI.parse(url)
+      response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http|
+        http.request(Net::HTTP::Get.new(uri, "Accept" => "*/*"))
+      end
+      unless response.is_a?(Net::HTTPSuccess)
+        raise Error.new("fetch_error", "Failed to fetch content: #{response.message}", response.code.to_i)
+      end
+
+      response.body.to_s.dup.force_encoding(Encoding::UTF_8)
+    rescue SystemCallError, SocketError, Timeout::Error, IOError, OpenSSL::SSL::SSLError => e
+      raise Error.new("network_error", "Failed to fetch content: #{e.message}", 0)
+    end
+  end
+end

data/lib/webcrawlerapi/model.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module WebCrawlerAPI
+  # Thin read-only wrapper over an API JSON object. Fields are available as
+  # methods (+job.status+), by key (+job["status"]+ / +job[:status]+) or via +to_h+.
+  class Model
+    # Declares documented optional fields: readers return nil when the API omits them.
+    def self.optional_attributes(*names)
+      names.each { |name| define_method(name) { @attributes[name.to_s] } }
+    end
+
+    def initialize(attributes)
+      @attributes = attributes.is_a?(Hash) ? attributes : {}
+    end
+
+    def [](key)
+      @attributes[key.to_s]
+    end
+
+    def to_h
+      @attributes
+    end
+
+    def key?(key)
+      @attributes.key?(key.to_s)
+    end
+
+    def ==(other)
+      other.is_a?(self.class) && other.to_h == to_h
+    end
+
+    def method_missing(name, *args)
+      return super unless args.empty? && @attributes.key?(name.to_s)
+
+      @attributes[name.to_s]
+    end
+
+    def respond_to_missing?(name, include_private = false)
+      @attributes.key?(name.to_s) || super
+    end
+
+    def inspect
+      "#<#{self.class.name} #{@attributes.inspect}>"
+    end
+  end
+
+  class ScrapeResponse < Model
+    optional_attributes :success, :status, :markdown, :cleaned_content, :raw_content, :page_status_code,
+                        :page_title, :structured_data, :links, :recommended_pull_delay_ms,
+                        :error_code, :error_message
+  end
+  class AgentRun < Model
+    optional_attributes :data, :error_reason, :max_spend_usd, :urls
+  end
+  class MarkdownResponse < Model
+    optional_attributes :markdown
+  end
+  class JobMarkdownResponse < Model; end
+  class ScrapeId < Model; end
+  class JobId < Model; end
+  class BatchScrapeId < Model; end
+  class BatchScrapeResponse < Model
+    # +next+ is absent on the last page.
+    optional_attributes :next, :data
+  end
+  class BatchScrapeErrors < Model; end
+end

data/lib/webcrawlerapi/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module WebCrawlerAPI
+  VERSION = "0.1.0"
+end

data/lib/webcrawlerapi.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require_relative "webcrawlerapi/version"
+require_relative "webcrawlerapi/constants"
+require_relative "webcrawlerapi/errors"
+require_relative "webcrawlerapi/model"
+require_relative "webcrawlerapi/job"
+require_relative "webcrawlerapi/client"
+
+# Ruby client for WebcrawlerAPI. See WebCrawlerAPI::Client.
+module WebCrawlerAPI
+end

metadata

@@ -0,0 +1,98 @@
+--- !ruby/object:Gem::Specification
+name: webcrawlerapi
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- WebcrawlerAPI
+bindir: bin
+cert_chain: []
+date: 2026-11-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.20'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.20'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.19'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.19'
+description: Crawl and scrape websites, convert pages to markdown and run AI agents
+  with WebcrawlerAPI.
+email:
+- sdk@webcrawlerapi.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE
+- README.md
+- lib/webcrawlerapi.rb
+- lib/webcrawlerapi/client.rb
+- lib/webcrawlerapi/constants.rb
+- lib/webcrawlerapi/errors.rb
+- lib/webcrawlerapi/job.rb
+- lib/webcrawlerapi/model.rb
+- lib/webcrawlerapi/version.rb
+homepage: https://webcrawlerapi.com
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://webcrawlerapi.com
+  source_code_uri: https://github.com/WebCrawlerAPI/webcrawlerapi-ruby-sdk
+  changelog_uri: https://github.com/WebCrawlerAPI/webcrawlerapi-ruby-sdk/blob/main/CHANGELOG.md
+  documentation_uri: https://webcrawlerapi.com/docs
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.2
+specification_version: 4
+summary: Ruby client for WebcrawlerAPI
+test_files: []