crawlbase 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 50c42144b472e240907828a2656215a7f1e5004f07a3288d3eafca184b4dc16c
-  data.tar.gz: fdbbd3ebe2a64ecde61e34b94bd5716574d04d5f1bffb5e1a9225f6500b0a793
+  metadata.gz: 6a1ccd286be2a6973c23cc8c9dc84a292af4d49910995054312a68e9beca6761
+  data.tar.gz: 0da3a67ba36a2151d6b982f7a9dc961adae4a1a58f0aac311d089e2dfd99cce9
 SHA512:
-  metadata.gz: c29927980f6cf82b431c7385e78429802916ebe188a40e789ca17e1c5a63d7183f4f8b5b30d8b9b7bd42c84fa6f1223eda48703cf317395e359a422a88cddfee
-  data.tar.gz: db7cf49a5dc174920a76bb35fecb92097b61025e3b2f7fb372a7143282fba93edd748763020001b0cdd928da4500cfe1a948e8123a2232a614cbfb399f3a4843
+  metadata.gz: 4d4b9ce783a5fa43cb0dab14550f01f2fcaed958b31bceeab91a9540456361a226f18fb41e11ae874f7d20862ba49d62088ab492b44c8a9b73c96c5812b8d4f3
+  data.tar.gz: 39ce7ce6996726bb7a3043277b1170fbc06b05f4cebf2903dc1db242bb4ae865de7a5a7a5eaa5af9f7861fcbccbfff92cce8227f5b2ed7df284c47d4c9f7cbe0

data/README.md

@@ -34,6 +34,12 @@ You can get a token for free by [creating a Crawlbase account](https://crawlbase
 api = Crawlbase::API.new(token: 'YOUR_TOKEN')
 ```
 
+By default, the timeout for API requests is set to 90 seconds. You can configure a custom timeout by passing a `timeout` option during initialization.
+
+```ruby
+api = Crawlbase::API.new(token: 'YOUR_TOKEN', timeout: 120)
+```
+
 ### GET requests
 
 Pass the url that you want to scrape plus any options from the ones available in the [API documentation](https://crawlbase.com/dashboard/docs).
@@ -103,7 +109,7 @@ puts response.body
 If you need to scrape any website built with Javascript like React, Angular, Vue, etc. You just need to pass your javascript token and use the same calls. Note that only `.get` is available for javascript and not `.post`.
 
 ```ruby
-api = Crawlbase::API.new(token: 'YOUR_JAVASCRIPT_TOKEN' })
+api = Crawlbase::API.new(token: 'YOUR_JAVASCRIPT_TOKEN')
 ```
 
 ```ruby
@@ -357,8 +363,8 @@ The gem is available as open source under the terms of the [MIT License](http://
 
 ## Code of Conduct
 
-Everyone interacting in the Crawlbase project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/crawlbase-source/crawlbase-ruby/blob/master/CODE_OF_CONDUCT.md).
+Everyone interacting in the Crawlbase project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/crawlbase-source/crawlbase-ruby/blob/master/CODE_OF_CONDUCT.md).
 
 ---
 
-Copyright 2023 Crawlbase
+Copyright 2025 Crawlbase

data/crawlbase.gemspec

@@ -28,4 +28,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "webmock", "~> 3.4"
   spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "rake", "~> 12.3.3"
+  spec.add_development_dependency "rexml", "~> 3.2"
 end

data/lib/crawlbase/api.rb

@@ -6,23 +6,26 @@ require 'uri'
 
 module Crawlbase
   class API
-    attr_reader :token, :body, :status_code, :original_status, :pc_status, :url, :storage_url
+    attr_reader :token, :body, :status_code, :original_status, :pc_status, :url, :storage_url, :timeout
 
     INVALID_TOKEN = 'Token is required'
     INVALID_URL = 'URL is required'
+    DEFAULT_TIMEOUT = 90
 
     def initialize(options = {})
       raise INVALID_TOKEN if options[:token].nil?
 
       @token = options[:token]
+      @timeout = options.fetch(:timeout, DEFAULT_TIMEOUT)
     end
 
     def get(url, options = {})
       raise INVALID_URL if url.empty?
 
       uri = prepare_uri(url, options)
-
-      response = Net::HTTP.get_response(uri)
+      http = build_http(uri)
+      request = Net::HTTP::Get.new(uri.request_uri)
+      response = http.request(request)
 
       prepare_response(response, options[:format])
 
@@ -33,10 +36,7 @@ module Crawlbase
       raise INVALID_URL if url.empty?
 
       uri = prepare_uri(url, options)
-
-      http = Net::HTTP.new(uri.host, uri.port)
-
-      http.use_ssl = true
+      http = build_http(uri)
 
       content_type = options[:post_content_type].to_s.include?('json') ? { 'Content-Type': 'text/json' } : nil
 
@@ -57,6 +57,14 @@ module Crawlbase
 
     private
 
+    def build_http(uri)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = true
+      http.open_timeout = @timeout
+      http.read_timeout = @timeout
+      http
+    end
+
     def base_url
       'https://api.crawlbase.com'
     end

data/lib/crawlbase/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Crawlbase
-  VERSION = '1.0.0'
+  VERSION = '1.1.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: crawlbase
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - crawlbase
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-06-29 00:00:00.000000000 Z
+date: 2025-06-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 12.3.3
+- !ruby/object:Gem::Dependency
+  name: rexml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
 description: Ruby based client for the Crawlbase API that helps developers crawl or
   scrape thousands of web pages anonymously
 email:
@@ -94,7 +108,7 @@ homepage: https://github.com/crawlbase-source/crawlbase-ruby
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -109,8 +123,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.6.0.dev
+signing_key:
 specification_version: 4
 summary: Crawlbase API client for web scraping and crawling
 test_files: []