webpipe-sdk 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: b49c39a2dc09f4401f0cee5341c3aff38edf3faea72ca270e0ad3568918931a5
+  data.tar.gz: 7faf803b7c1193549d4c611b5b0cf42b51f0a2511c377d3379737a99a1ca0c90
+SHA512:
+  metadata.gz: dca50f00ecced076a680314d2131122a52e9dd2e5a19982b8549d6486107a74d40e2c08db1deeb42c1743978ce415e596209a912f1a251b7536fd4a90d97c0c9
+  data.tar.gz: de0802c5500aa2a080ed86a97f17471b8187bfa3d5b4f188d19d77ab0b0ae017c54d822cff3e1685ca4b4707ebc7504a543c6a2135f797a6c30aae1ac01c6a41

data/README.md

@@ -0,0 +1,95 @@
+# webpipe-sdk (Ruby)
+
+Official Ruby SDK for [WebPipe.ai](https://webpipe.ai/) — turn any webpage into clean, structured data.
+
+## Install
+
+```bash
+gem install webpipe-sdk
+```
+
+Requires Ruby ≥ 3.0. **Zero runtime dependencies** — stdlib `net/http` only.
+
+## Quick Start
+
+```ruby
+require "webpipe"
+
+client = Webpipe::Client.new # reads WEBPIPE_API_KEY, or Webpipe::Client.new(api_key: "wc-...")
+
+# Scrape a page
+doc = client.scrape("https://example.com", formats: ["markdown", "metadata"])
+puts doc.markdown
+puts doc.metadata["title"]
+```
+
+### Crawl a whole site (polls automatically)
+
+```ruby
+job = client.crawl("https://docs.example.com", limit: 50,
+                   scrape_options: { formats: ["markdown"] }, timeout: 600)
+job.data.each do |page|
+  puts "#{page.metadata["sourceURL"]} #{page.metadata["title"]}"
+end
+```
+
+Need control? `job = client.start_crawl(url, limit: 50)` returns immediately; poll with `client.get_crawl_status(job.id)`.
+
+### Browser session (JS-rendered / logged-in pages)
+
+```ruby
+session = client.browser.create(url: "https://example.com/login")
+begin
+  session.wait_until_running
+  session.fill("input[name=email]", "you@example.com")
+  session.fill("input[name=password]", "secret")
+  session.press("Enter", wait_after: 3000)
+
+  doc = session.navigate("https://example.com/dashboard", formats: ["markdown"])
+  puts doc.markdown
+  session.save_session # reuse login state next time via session_name
+ensure
+  session.close
+end
+```
+
+### Map (discover all URLs)
+
+```ruby
+client.map("https://example.com", limit: 100, search: "blog").each do |link|
+  puts "#{link.url} #{link.title}"
+end
+```
+
+## Errors
+
+```ruby
+begin
+  client.scrape("https://example.com")
+rescue Webpipe::AuthenticationError
+  # bad API key
+rescue Webpipe::RateLimitError => e
+  # e.status_code == 429; e.message keeps the server's error text
+rescue Webpipe::Error
+  # anything else
+end
+```
+
+`429` and `5xx` responses are retried automatically with exponential backoff (`max_retries: 2` by default).
+
+## Configuration
+
+| Param | Env var | Default |
+|---|---|---|
+| `api_key` | `WEBPIPE_API_KEY` | — (required) |
+| `base_url` | `WEBPIPE_API_URL` | `https://api.web2json.ai` |
+| `timeout` | — | 60s |
+| `max_retries` | — | 2 |
+
+## Development
+
+```bash
+bundle install
+ruby -Ilib -Itest test/client_test.rb
+ruby -Ilib -Itest test/browser_test.rb
+```

data/lib/webpipe/browser.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+module Webpipe
+  # Factory for browser sessions, exposed as +client.browser+.
+  # @api private (constructed via Client)
+  class BrowserResource
+    def initialize(http)
+      @http = http
+    end
+
+    # Create a session. It starts asynchronously — call +wait_until_running+
+    # before sending commands for a controlled experience.
+    def create(url: nil, cookie: nil, proxy: nil, session_name: nil)
+      body = drop_nil(url: url, cookie: cookie, proxy: proxy, session_name: session_name)
+      info = @http.post("/api/v1/browser", body)
+      BrowserSession.new(@http, id: info["id"], status: info["status"], url: info["url"],
+                                session_name: info["session_name"], expires_at: info["expires_at"])
+    end
+
+    private
+
+    def drop_nil(hash) = hash.reject { |_, v| v.nil? }
+  end
+
+  # Handle to a persistent headless Chromium session on the server.
+  #
+  # Server-side facts: sessions expire after 30 minutes idle; at most 2
+  # concurrent running sessions per user. Always +close+ when done.
+  class BrowserSession
+    attr_reader :id, :status, :url, :session_name, :expires_at
+
+    TERMINAL_STATUSES = %w[stopped error expired].freeze
+
+    def initialize(http, id:, status:, url: nil, session_name: nil, expires_at: nil)
+      @http = http
+      @id = id
+      @status = status
+      @url = url
+      @session_name = session_name
+      @expires_at = expires_at
+    end
+
+    # Fetch the latest session state from the server.
+    def refresh
+      info = @http.get(base)
+      @status = info["status"]
+      @url = info["url"]
+      @session_name = info["session_name"]
+      @expires_at = info["expires_at"]
+      self
+    end
+
+    # Block until the session is ready to accept commands.
+    def wait_until_running(timeout: 60, poll_interval: 1)
+      deadline = Time.now + timeout
+      loop do
+        refresh
+        return self if @status == "running"
+        raise Error, "Browser session #{@id} entered terminal status '#{@status}'" if TERMINAL_STATUSES.include?(@status)
+        raise PollTimeoutError, "Browser session #{@id} not running after #{timeout}s" if Time.now >= deadline
+
+        sleep poll_interval
+      end
+    end
+
+    # Stop the session and release server resources.
+    def close
+      @http.delete(base)
+      @status = "stopped"
+      nil
+    end
+
+    # Navigate to a new URL (with anti-bot handling) and scrape it.
+    def navigate(url, formats: nil, scroll_to_bottom: nil, max_scrolls: nil, scroll_wait: nil, scroll_step: nil)
+      body = { url: url }.merge(scrape_body(formats:, scroll_to_bottom:, max_scrolls:, scroll_wait:, scroll_step:))
+      Document.from_hash(@http.post("#{base}/navigate", body)["data"])
+    end
+
+    # Scrape the current page without navigating (faster than +navigate+).
+    def scrape(formats: nil, scroll_to_bottom: nil, max_scrolls: nil, scroll_wait: nil, scroll_step: nil)
+      body = scrape_body(formats:, scroll_to_bottom:, max_scrolls:, scroll_wait:, scroll_step:)
+      Document.from_hash(@http.post("#{base}/scrape", body)["data"])
+    end
+
+    # Run a raw browser action. Returns the raw API payload Hash.
+    def action(type, **params)
+      @http.post("#{base}/action", { type: type }.merge(params))
+    end
+
+    def click(selector, wait_after: nil) = action("click", **drop_nil(selector:, wait_after:))
+    def fill(selector, value) = action("fill", selector:, value:)
+    def select(selector, value) = action("select", selector:, value:)
+
+    def press(key, selector: nil, wait_after: nil)
+      action("press", **drop_nil(key:, selector:, wait_after:))
+    end
+
+    def scroll(direction: "down", amount: nil) = action("scroll", **drop_nil(direction:, amount:))
+    def wait(ms = 1000) = action("wait", ms:)
+    def wait_for(selector, timeout: nil) = action("wait_for", **drop_nil(selector:, timeout:))
+
+    # Take a screenshot. The API returns a base64 PNG inside the payload.
+    def screenshot(full_page: false) = action("screenshot", full_page:)
+
+    # Evaluate a JavaScript expression in the page (value in payload's "result").
+    def evaluate(expression) = action("evaluate", expression:)
+    def hover(selector) = action("hover", selector:)
+    def check(selector) = action("check", selector:)
+
+    def get_cookies
+      data = @http.get("#{base}/cookies")
+      data["cookies"] || data["data"] || []
+    end
+
+    def set_cookies(cookies) = @http.post("#{base}/cookies", { op: "set", cookies: cookies })
+    def clear_cookies = @http.post("#{base}/cookies", { op: "clear" })
+
+    # Persist cookies/storage under this session's +session_name+.
+    def save_session = @http.post("#{base}/save_session")
+
+    private
+
+    def base = "/api/v1/browser/#{@id}"
+
+    def scrape_body(formats:, scroll_to_bottom:, max_scrolls:, scroll_wait:, scroll_step:)
+      drop_nil(
+        formats: formats, scroll_to_bottom: scroll_to_bottom,
+        max_scrolls: max_scrolls, scroll_wait: scroll_wait, scroll_step: scroll_step
+      )
+    end
+
+    def drop_nil(hash) = hash.reject { |_, v| v.nil? }
+  end
+end

data/lib/webpipe/client.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Webpipe
+  # The WebPipe API client.
+  #
+  #   client = Webpipe::Client.new # reads WEBPIPE_API_KEY
+  #   doc = client.scrape("https://example.com", formats: ["markdown"])
+  #   puts doc.markdown
+  class Client
+    DEFAULT_BASE_URL = "https://api.web2json.ai"
+
+    attr_reader :browser
+
+    def initialize(api_key: nil, base_url: nil, timeout: 60, max_retries: 2)
+      api_key ||= ENV["WEBPIPE_API_KEY"]
+      if api_key.nil? || api_key.empty?
+        raise ArgumentError,
+              "Missing API key. Pass api_key: or set the WEBPIPE_API_KEY environment " \
+              "variable. Create one at https://webpipe.ai/api-keys.html"
+      end
+
+      base_url ||= ENV["WEBPIPE_API_URL"] || DEFAULT_BASE_URL
+      @http = HttpClient.new(
+        base_url: base_url.sub(%r{/+$}, ""),
+        api_key: api_key,
+        timeout: timeout,
+        max_retries: max_retries,
+        user_agent: "webpipe-sdk-ruby/#{Webpipe::VERSION}"
+      )
+      @browser = BrowserResource.new(@http)
+    end
+
+    # Scrape a single page into markdown/html/links/metadata/json.
+    def scrape(url, formats: nil, json_schema: nil, use_proxy: nil, proxy: nil, cookie: nil)
+      body = drop_nil(url:, formats:, json_schema:, use_proxy:, proxy:, cookie:)
+      Document.from_hash(@http.post("/api/v1/scrape", body)["data"])
+    end
+
+    # Discover URLs of a website (robots.txt → sitemap → in-page links).
+    def map(url, limit: nil, search: nil, sitemap: nil, same_domain: nil,
+            use_proxy: nil, proxy: nil, cookie: nil)
+      body = drop_nil(url:, limit:, search:, sitemap:, same_domain:, use_proxy:, proxy:, cookie:)
+      (@http.post("/api/v1/map", body)["links"] || []).map { |l| MapLink.from_hash(l) }
+    end
+
+    # Submit an async crawl job and return its handle immediately.
+    def start_crawl(url, limit: nil, max_discovery_depth: nil, include_paths: nil,
+                    exclude_paths: nil, allow_subdomains: nil, allow_external_links: nil,
+                    crawl_entire_domain: nil, ignore_query_params: nil, sitemap: nil,
+                    delay: nil, max_concurrency: nil, scrape_options: nil,
+                    use_proxy: nil, proxy: nil, cookie: nil)
+      body = drop_nil(
+        url:, limit:, max_discovery_depth:, include_paths:, exclude_paths:,
+        allow_subdomains:, allow_external_links:, crawl_entire_domain:,
+        ignore_query_params:, sitemap:, delay:, max_concurrency:,
+        scrape_options:, use_proxy:, proxy:, cookie:
+      )
+      res = @http.post("/api/v1/crawl", body)
+      CrawlJobStart.new(res.fetch("id"), res["url"])
+    end
+
+    # Fetch the current status and partial results of a crawl job.
+    def get_crawl_status(job_id)
+      CrawlJob.from_hash(@http.get("/api/v1/crawl/#{job_id}"))
+    end
+
+    # Submit a crawl job and poll until it completes.
+    #
+    # Yields the current CrawlJob after each poll when a block is given —
+    # the idiomatic Ruby way to observe a long crawl (progress bars, logging).
+    #
+    # Raises CrawlError if the job fails, PollTimeoutError if +timeout+ is
+    # exceeded. Other keywords are the same as +start_crawl+.
+    def crawl(url, poll_interval: 2, timeout: nil, **options)
+      job = start_crawl(url, **options)
+      deadline = timeout && Time.now + timeout
+      loop do
+        status = get_crawl_status(job.id)
+        return status if status.status == CrawlJob::STATUS_COMPLETED
+        raise CrawlError, "Crawl job #{job.id} failed: #{status.errors}" if status.status == CrawlJob::STATUS_FAILED
+        yield status if block_given?
+        if deadline && Time.now >= deadline
+          raise PollTimeoutError, "Crawl job #{job.id} did not complete within #{timeout}s"
+        end
+
+        sleep poll_interval
+      end
+    end
+
+    private
+
+    def drop_nil(hash) = hash.reject { |_, v| v.nil? }
+  end
+end

data/lib/webpipe/errors.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Webpipe
+  # Base class for all webpipe-sdk errors.
+  class Error < StandardError; end
+
+  # An error response returned by the WebPipe API. Always carries the server's
+  # original error message and the HTTP status code.
+  class ApiError < Error
+    attr_reader :status_code, :body
+
+    def initialize(message, status_code:, body: nil)
+      super("[#{status_code}] #{message}")
+      @status_code = status_code
+      @body = body
+    end
+  end
+
+  # 400 — invalid parameters (missing url, bad format, ...).
+  class BadRequestError < ApiError; end
+  # 401 — API key missing, invalid or deleted.
+  class AuthenticationError < ApiError; end
+  # 403 — accessing a resource owned by another user.
+  class ForbiddenError < ApiError; end
+  # 404 — job/session does not exist or has expired.
+  class NotFoundError < ApiError; end
+  # 429 — rate limit exceeded, or too many concurrent browser sessions.
+  class RateLimitError < ApiError; end
+  # 5xx — scrape timeout, target site unreachable, ...
+  class ServerError < ApiError; end
+
+  # A crawl job reached status "failed".
+  class CrawlError < Error; end
+
+  # Polling did not finish within the allotted timeout.
+  class PollTimeoutError < Error; end
+end

data/lib/webpipe/http_client.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "json"
+require "uri"
+
+module Webpipe
+  # Thin net/http wrapper: auth headers, retries with backoff, error mapping.
+  # @api private
+  class HttpClient
+    RETRYABLE_STATUSES = [429, 500, 502, 503, 504].freeze
+    RETRYABLE_ERRORS = [
+      Net::OpenTimeout, Net::ReadTimeout, SocketError, SystemCallError
+    ].freeze
+
+    STATUS_TO_ERROR = {
+      400 => BadRequestError,
+      401 => AuthenticationError,
+      403 => ForbiddenError,
+      404 => NotFoundError,
+      429 => RateLimitError
+    }.freeze
+
+    def initialize(base_url:, api_key:, timeout:, max_retries:, user_agent:)
+      @base_uri = URI(base_url)
+      @api_key = api_key
+      @timeout = timeout
+      @max_retries = max_retries
+      @user_agent = user_agent
+    end
+
+    def get(path) = request("GET", path)
+    def post(path, body = nil) = request("POST", path, body)
+    def delete(path) = request("DELETE", path)
+
+    def request(method, path, body = nil)
+      attempt = 0
+      loop do
+        response = perform(method, path, body)
+        if RETRYABLE_STATUSES.include?(response.code.to_i) && attempt < @max_retries
+          attempt += 1
+          sleep backoff(response, attempt)
+          next
+        end
+        return handle(response)
+      rescue *RETRYABLE_ERRORS
+        attempt += 1
+        raise if attempt > @max_retries
+
+        sleep(0.5 * (2**attempt))
+        retry
+      end
+    end
+
+    private
+
+    def perform(method, path, body)
+      http = Net::HTTP.new(@base_uri.host, @base_uri.port)
+      http.use_ssl = @base_uri.scheme == "https"
+      http.open_timeout = @timeout
+      http.read_timeout = @timeout
+
+      request_class = Net::HTTP.const_get(method.capitalize)
+      request = request_class.new(path)
+      request["Authorization"] = "Bearer #{@api_key}"
+      request["Content-Type"] = "application/json"
+      request["User-Agent"] = @user_agent
+      request.body = JSON.generate(body) if body
+
+      http.request(request)
+    end
+
+    def handle(response)
+      status = response.code.to_i
+      payload = begin
+        body = response.body.to_s
+        body.empty? ? {} : JSON.parse(body)
+      rescue JSON::ParserError
+        {}
+      end
+
+      return payload if status < 400
+
+      message = payload.is_a?(Hash) && payload["error"] ? payload["error"] : response.message
+      error_class = status >= 500 ? ServerError : (STATUS_TO_ERROR[status] || ApiError)
+      raise error_class.new(message, status_code: status, body: payload)
+    end
+
+    def backoff(response, attempt)
+      retry_after = response["Retry-After"]
+      return Float(retry_after) if retry_after
+
+      0.5 * (2**attempt)
+    rescue ArgumentError, TypeError
+      0.5 * (2**attempt)
+    end
+  end
+end

data/lib/webpipe/types.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Webpipe
+  # A link extracted from a page.
+  Link = Struct.new(:text, :href) do
+    def self.from_hash(hash)
+      new(hash["text"], hash["href"])
+    end
+  end
+
+  # Result of scraping a single page (scrape / browser / crawl data item).
+  # `metadata` is a plain Hash — the API adds extra keys (og:*, sourceURL, ...).
+  class Document
+    attr_reader :markdown, :html, :raw_html, :links, :metadata, :json
+
+    def initialize(markdown: nil, html: nil, raw_html: nil, links: nil, metadata: nil, json: nil)
+      @markdown = markdown
+      @html = html
+      @raw_html = raw_html
+      @links = links
+      @metadata = metadata
+      @json = json
+    end
+
+    def self.from_hash(hash)
+      hash ||= {}
+      new(
+        markdown: hash["markdown"],
+        html: hash["html"],
+        raw_html: hash["rawHtml"],
+        links: hash["links"]&.map { |l| Link.from_hash(l) },
+        metadata: hash["metadata"],
+        json: hash["json"]
+      )
+    end
+  end
+
+  # A URL discovered by the Map endpoint.
+  MapLink = Struct.new(:url, :title, :description) do
+    def self.from_hash(hash)
+      new(hash["url"], hash["title"], hash["description"])
+    end
+  end
+
+  # Handle returned when a crawl job is submitted.
+  CrawlJobStart = Struct.new(:id, :url)
+
+  # Status and (partial or final) results of a crawl job.
+  # Results expire 24h after completion (see +expires_at+).
+  class CrawlJob
+    STATUS_SCRAPING = "scraping"
+    STATUS_COMPLETED = "completed"
+    STATUS_FAILED = "failed"
+
+    attr_reader :status, :total, :completed, :data, :errors, :expires_at
+
+    def initialize(status:, total: 0, completed: 0, data: [], errors: [], expires_at: nil)
+      @status = status
+      @total = total
+      @completed = completed
+      @data = data
+      @errors = errors
+      @expires_at = expires_at
+    end
+
+    def self.from_hash(hash)
+      new(
+        status: hash["status"],
+        total: hash["total"] || 0,
+        completed: hash["completed"] || 0,
+        data: (hash["data"] || []).map { |d| Document.from_hash(d) },
+        errors: hash["errors"] || [],
+        expires_at: hash["expires_at"]
+      )
+    end
+  end
+end

data/lib/webpipe/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Webpipe
+  VERSION = "0.1.0"
+end

data/lib/webpipe.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+# webpipe-sdk — the official Ruby SDK for WebPipe.ai.
+#
+#   require "webpipe"
+#   client = Webpipe::Client.new(api_key: "wc-...")
+#   doc = client.scrape("https://example.com", formats: ["markdown"])
+#   puts doc.markdown
+
+require_relative "webpipe/version"
+require_relative "webpipe/errors"
+require_relative "webpipe/types"
+require_relative "webpipe/http_client"
+require_relative "webpipe/browser"
+require_relative "webpipe/client"

metadata

@@ -0,0 +1,48 @@
+--- !ruby/object:Gem::Specification
+name: webpipe-sdk
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- WebPipe.ai
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies: []
+description: Scrape, Map, Crawl and persistent Browser Sessions via the WebPipe.ai
+  API. Zero runtime dependencies (stdlib net/http).
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/webpipe.rb
+- lib/webpipe/browser.rb
+- lib/webpipe/client.rb
+- lib/webpipe/errors.rb
+- lib/webpipe/http_client.rb
+- lib/webpipe/types.rb
+- lib/webpipe/version.rb
+homepage: https://webpipe.ai
+licenses:
+- MIT
+metadata: {}
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.16
+specification_version: 4
+summary: Official Ruby SDK for WebPipe.ai — turn any webpage into clean, structured
+  data
+test_files: []