crawlbase 1.0.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 50c42144b472e240907828a2656215a7f1e5004f07a3288d3eafca184b4dc16c
-  data.tar.gz: fdbbd3ebe2a64ecde61e34b94bd5716574d04d5f1bffb5e1a9225f6500b0a793
+  metadata.gz: 42efa911e98c21bc83337c2fd34524b205a04520ae7f13d8c90efd8832cadbee
+  data.tar.gz: a031d03c6ed8994135a50ae81279a866918e7feba128e630e22f7908d62c6dee
 SHA512:
-  metadata.gz: c29927980f6cf82b431c7385e78429802916ebe188a40e789ca17e1c5a63d7183f4f8b5b30d8b9b7bd42c84fa6f1223eda48703cf317395e359a422a88cddfee
-  data.tar.gz: db7cf49a5dc174920a76bb35fecb92097b61025e3b2f7fb372a7143282fba93edd748763020001b0cdd928da4500cfe1a948e8123a2232a614cbfb399f3a4843
+  metadata.gz: ffcd59a906f655aff0010147ab1db09544137344aa987617c544a058d14a3771694a0cf9e641c46cadc36fc42976156a2e1611af9742acc11119a362e4f694b8
+  data.tar.gz: 48cddc484f162578332f974b34b1921e7f5429b909bf71389134ec96cf6b0bf54deccd6301371f5680af20394a4b89192194f35986ab70543532d8351dec511c

data/CHANGELOG.md

@@ -0,0 +1,27 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [1.2.0] - 2026-07-15
+
+### Added
+
+- `cb_status` as the preferred public attribute for Crawlbase status on `Crawlbase::API` (and subclasses) and `Crawlbase::StorageAPI`.
+- Status resolution prefers the `cb_status` response header or JSON field, then falls back to `pc_status` when `cb_status` is absent.
+
+### Deprecated
+
+- `pc_status` is deprecated but still supported and returns the same resolved value as `cb_status`. A one-time deprecation warning is emitted when `pc_status` is accessed. Prefer `cb_status`. Removal is planned for a future major release.
+
+### Migration
+
+```ruby
+# Before (deprecated)
+response.pc_status
+
+# After (preferred)
+response.cb_status
+```

data/README.md

@@ -34,6 +34,12 @@ You can get a token for free by [creating a Crawlbase account](https://crawlbase
 api = Crawlbase::API.new(token: 'YOUR_TOKEN')
 ```
 
+By default, the timeout for API requests is set to 90 seconds. You can configure a custom timeout by passing a `timeout` option during initialization.
+
+```ruby
+api = Crawlbase::API.new(token: 'YOUR_TOKEN', timeout: 120)
+```
+
 ### GET requests
 
 Pass the url that you want to scrape plus any options from the ones available in the [API documentation](https://crawlbase.com/dashboard/docs).
@@ -50,7 +56,7 @@ begin
   response = api.get('https://www.facebook.com/britneyspears')
   puts response.status_code
   puts response.original_status
-  puts response.pc_status
+  puts response.cb_status
   puts response.body
 rescue => exception
   puts exception.backtrace
@@ -103,7 +109,7 @@ puts response.body
 If you need to scrape any website built with Javascript like React, Angular, Vue, etc. You just need to pass your javascript token and use the same calls. Note that only `.get` is available for javascript and not `.post`.
 
 ```ruby
-api = Crawlbase::API.new(token: 'YOUR_JAVASCRIPT_TOKEN' })
+api = Crawlbase::API.new(token: 'YOUR_JAVASCRIPT_TOKEN')
 ```
 
 ```ruby
@@ -119,17 +125,32 @@ response = api.get('https://www.freelancer.com', options: { page_wait: 5000 })
 puts response.status_code
 ```
 
-## Original status
+## Original status and Crawlbase status
+
+You can always get the original status and Crawlbase status from the response. Read the [Crawlbase documentation](https://crawlbase.com/dashboard/docs) to learn more about those statuses.
 
-You can always get the original status and crawlbase status from the response. Read the [Crawlbase documentation](https://crawlbase.com/dashboard/docs) to learn more about those status.
+Prefer `cb_status` for the Crawlbase status. `pc_status` is deprecated but still supported temporarily and returns the same resolved value.
 
 ```ruby
 response = api.get('https://sfbay.craigslist.org/')
 
 puts response.original_status
-puts response.pc_status
+puts response.cb_status # preferred
+# puts response.pc_status # deprecated; same value as cb_status
 ```
 
+### Migrating from `pc_status` to `cb_status`
+
+```ruby
+# Before (deprecated)
+response.pc_status
+
+# After (preferred)
+response.cb_status
+```
+
+The library reads `cb_status` from the API response first, then falls back to `pc_status` if `cb_status` is not present.
+
 ## Scraper API usage
 
 Initialize the Scraper API using your normal token and call the `get` method.
@@ -248,7 +269,7 @@ Pass the [url](https://crawlbase.com/docs/storage-api/parameters/#url) that you
 begin
   response = storage_api.get('https://www.apple.com')
   puts response.original_status
-  puts response.pc_status
+  puts response.cb_status
   puts response.url
   puts response.status_code
   puts response.rid
@@ -265,7 +286,7 @@ or you can use the [RID](https://crawlbase.com/docs/storage-api/parameters/#rid)
 begin
   response = storage_api.get(RID)
   puts response.original_status
-  puts response.pc_status
+  puts response.cb_status
   puts response.url
   puts response.status_code
   puts response.rid
@@ -298,7 +319,7 @@ To do a bulk request with a list of RIDs, please send the list of rids as an arr
 begin
   response = storage_api.bulk([RID1, RID2, RID3, ...])
   puts response.original_status
-  puts response.pc_status
+  puts response.cb_status
   puts response.url
   puts response.status_code
   puts response.rid
@@ -357,8 +378,8 @@ The gem is available as open source under the terms of the [MIT License](http://
 
 ## Code of Conduct
 
-Everyone interacting in the Crawlbase project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/crawlbase-source/crawlbase-ruby/blob/master/CODE_OF_CONDUCT.md).
+Everyone interacting in the Crawlbase project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/crawlbase-source/crawlbase-ruby/blob/master/CODE_OF_CONDUCT.md).
 
 ---
 
-Copyright 2023 Crawlbase
+Copyright 2026 Crawlbase

data/crawlbase.gemspec

@@ -28,4 +28,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "webmock", "~> 3.4"
   spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "rake", "~> 12.3.3"
+  spec.add_development_dependency "rexml", "~> 3.2"
 end

data/lib/crawlbase/api.rb

@@ -6,23 +6,36 @@ require 'uri'
 
 module Crawlbase
   class API
-    attr_reader :token, :body, :status_code, :original_status, :pc_status, :url, :storage_url
+    # Crawlbase HTTP response code for the API call itself.
+    attr_reader :token, :body, :status_code, :original_status, :url, :storage_url, :timeout
+
+    # Crawlbase status from the response (preferred name for former +pc_status+).
+    attr_reader :cb_status
 
     INVALID_TOKEN = 'Token is required'
     INVALID_URL = 'URL is required'
+    DEFAULT_TIMEOUT = 90
 
     def initialize(options = {})
       raise INVALID_TOKEN if options[:token].nil?
 
       @token = options[:token]
+      @timeout = options.fetch(:timeout, DEFAULT_TIMEOUT)
+    end
+
+    # Deprecated: use {#cb_status}. Returns the same resolved Crawlbase status.
+    def pc_status
+      StatusResolution.warn_pc_status_deprecated
+      @pc_status
     end
 
     def get(url, options = {})
       raise INVALID_URL if url.empty?
 
       uri = prepare_uri(url, options)
-
-      response = Net::HTTP.get_response(uri)
+      http = build_http(uri)
+      request = Net::HTTP::Get.new(uri.request_uri)
+      response = http.request(request)
 
       prepare_response(response, options[:format])
 
@@ -33,10 +46,7 @@ module Crawlbase
       raise INVALID_URL if url.empty?
 
       uri = prepare_uri(url, options)
-
-      http = Net::HTTP.new(uri.host, uri.port)
-
-      http.use_ssl = true
+      http = build_http(uri)
 
       content_type = options[:post_content_type].to_s.include?('json') ? { 'Content-Type': 'text/json' } : nil
 
@@ -57,6 +67,14 @@ module Crawlbase
 
     private
 
+    def build_http(uri)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = true
+      http.open_timeout = @timeout
+      http.read_timeout = @timeout
+      http
+    end
+
     def base_url
       'https://api.crawlbase.com'
     end
@@ -72,7 +90,8 @@ module Crawlbase
       res = format == 'json' || base_url.include?('/scraper') ? JSON.parse(response.body) : response
 
       @original_status = res['original_status'].to_i
-      @pc_status = res['pc_status'].to_i
+      @cb_status = StatusResolution.resolve_cb_status(res)
+      @pc_status = @cb_status
       @url = res['url']
       @storage_url = res['storage_url']
       @status_code = response.code.to_i

data/lib/crawlbase/status_resolution.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Crawlbase
+  # Resolves Crawlbase status from response headers or JSON bodies.
+  # Prefers +cb_status+, falling back to deprecated +pc_status+.
+  module StatusResolution
+    PC_STATUS_DEPRECATION =
+      '[Crawlbase] `pc_status` is deprecated and will be removed in a future major release. Use `cb_status` instead.'
+
+    module_function
+
+    def resolve_cb_status(source)
+      raw = source['cb_status']
+      raw = source['pc_status'] if raw.nil? || raw.to_s.empty?
+      raw.to_i
+    end
+
+    def warn_pc_status_deprecated
+      return if Crawlbase.instance_variable_get(:@pc_status_deprecation_warned)
+
+      Crawlbase.instance_variable_set(:@pc_status_deprecation_warned, true)
+      warn PC_STATUS_DEPRECATION, uplevel: 1
+    end
+  end
+end

data/lib/crawlbase/storage_api.rb

@@ -6,7 +6,10 @@ require 'uri'
 
 module Crawlbase
   class StorageAPI
-    attr_reader :token, :original_status, :pc_status, :url, :status_code, :rid, :body, :stored_at
+    attr_reader :token, :original_status, :url, :status_code, :rid, :body, :stored_at
+
+    # Crawlbase status from the response (preferred name for former +pc_status+).
+    attr_reader :cb_status
 
     INVALID_TOKEN = 'Token is required'
     INVALID_RID = 'RID is required'
@@ -20,6 +23,12 @@ module Crawlbase
       @token = options[:token]
     end
 
+    # Deprecated: use {#cb_status}. Returns the same resolved Crawlbase status.
+    def pc_status
+      StatusResolution.warn_pc_status_deprecated
+      @pc_status
+    end
+
     def get(url_or_rid, format = 'html')
       raise INVALID_URL_OR_RID if url_or_rid.nil? || url_or_rid.empty?
 
@@ -30,7 +39,8 @@ module Crawlbase
       res = format == 'json' ? JSON.parse(response.body) : response
 
       @original_status = res['original_status'].to_i
-      @pc_status = res['pc_status'].to_i
+      @cb_status = StatusResolution.resolve_cb_status(res)
+      @pc_status = @cb_status
       @url = res['url']
       @rid = res['rid']
       @stored_at = res['stored_at']
@@ -50,7 +60,7 @@ module Crawlbase
       request = Net::HTTP::Delete.new(uri.request_uri)
       response = http.request(request)
 
-      @url, @original_status, @pc_status, @stored_at = nil
+      @url, @original_status, @cb_status, @pc_status, @stored_at = nil
       @status_code = response.code.to_i
       @rid = rid
       @body = JSON.parse(response.body)
@@ -71,7 +81,8 @@ module Crawlbase
       @body = JSON.parse(response.body)
       @original_status = @body.map { |item| item['original_status'].to_i }
       @status_code = response.code.to_i
-      @pc_status = @body.map { |item| item['pc_status'].to_i }
+      @cb_status = @body.map { |item| StatusResolution.resolve_cb_status(item) }
+      @pc_status = @cb_status
       @url = @body.map { |item| item['url'] }
       @rid = @body.map { |item| item['rid'] }
       @stored_at = @body.map { |item| item['stored_at'] }
@@ -86,7 +97,7 @@ module Crawlbase
       uri.query = URI.encode_www_form(query_hash)
 
       response = Net::HTTP.get_response(uri)
-      @url, @original_status, @pc_status, @stored_at = nil
+      @url, @original_status, @cb_status, @pc_status, @stored_at = nil
       @status_code = response.code.to_i
       @body = JSON.parse(response.body)
       @rid = @body
@@ -99,7 +110,7 @@ module Crawlbase
       uri.query = URI.encode_www_form(token: token)
 
       response = Net::HTTP.get_response(uri)
-      @url, @original_status, @pc_status, @stored_at = nil
+      @url, @original_status, @cb_status, @pc_status, @stored_at = nil
       @status_code = response.code.to_i
       @rid = rid
       @body = JSON.parse(response.body)

data/lib/crawlbase/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Crawlbase
-  VERSION = '1.0.0'
+  VERSION = '1.2.0'
 end

data/lib/crawlbase.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'crawlbase/version'
+require 'crawlbase/status_resolution'
 require 'crawlbase/api'
 require 'crawlbase/scraper_api'
 require 'crawlbase/leads_api'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: crawlbase
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.2.0
 platform: ruby
 authors:
 - crawlbase
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-06-29 00:00:00.000000000 Z
+date: 2026-07-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 12.3.3
+- !ruby/object:Gem::Dependency
+  name: rexml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
 description: Ruby based client for the Crawlbase API that helps developers crawl or
   scrape thousands of web pages anonymously
 email:
@@ -75,6 +89,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
 - LICENSE.txt
@@ -88,13 +103,14 @@ files:
 - lib/crawlbase/leads_api.rb
 - lib/crawlbase/scraper_api.rb
 - lib/crawlbase/screenshots_api.rb
+- lib/crawlbase/status_resolution.rb
 - lib/crawlbase/storage_api.rb
 - lib/crawlbase/version.rb
 homepage: https://github.com/crawlbase-source/crawlbase-ruby
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -109,8 +125,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.5.11
+signing_key:
 specification_version: 4
 summary: Crawlbase API client for web scraping and crawling
 test_files: []