crawlr 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f4a2b21633eead87fe3b879552db225aecc2975cad779fd86413f2531cd3f079
-  data.tar.gz: 9b20eb81f931b0f514609e9b699a85a8f25d2e64e0fb7f8f6845343d4872a893
+  metadata.gz: fe3c5b1d19db6a4fda1bd66a9e2c62a1b2bdb80c361fe06e84023a6bf3f024bb
+  data.tar.gz: 6f26c3350a3cbf7e967899d8f5490312d83caa8ad9223cefcb5ad8423bec1e97
 SHA512:
-  metadata.gz: 3e5d343dd502ed23343ad0e6bfbe9fbe6b8696954e171a181ae385c8c679d60cfeeeec4d65cdbb9e841731664ae27fa4034405c8265ab12f967470e91321208a
-  data.tar.gz: ecb186a9d6e9a5f34a4e429b1c3971a1eaf7f708537d698557fab3272010c0f4fb953362b64ed498556867a0938387c70cc24f0b7f41563026be7f1157f373a1
+  metadata.gz: 4c58780044aa20341737127823958728deb6b3574c781cb804db45e5c81971678058f779657b379bfa566c0608d273c72ec8331e2226885f71e3d476af1c0076
+  data.tar.gz: a094872a4ad346cae330a6daa894c6a49a72e7082f9279fa878dd14d09f7fdbccad5617433e683d15309eb4b1f14bcc05aa59cd47f2f7a9c460a5b2728530ad0

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 ## [Unreleased]
 
+## [0.2.1] - 2025-09-30
+
+- Fix paginated_visit to properly handle provided url queries (if present)
+- Update paginated_visit batch size parameter to respect max_depth (if max_depth set > 0)
+
 ## [0.2.0] - 2025-09-30
 
 - Tidied up documentation and inline comments

data/lib/crawlr/collector.rb

@@ -570,17 +570,19 @@ module Crawlr
     end
 
     def build_initial_pages(url, query, batch_size, start_page)
-      max_batch = [@config.max_depth, batch_size].min
+      uri = URI.parse(url)
+      max_batch = @config.max_depth.zero? ? batch_size : [@config.max_depth, batch_size].min
+
       if start_page == 1
-        [url] + (max_batch - 1).times.map { |i| "#{url}?#{query}=#{i + 2}" }
+        [url] + (max_batch - 1).times.map { |i| build_page_url(uri, query, i + 2) }
       else
-        max_batch.times.map { |i| "#{url}?#{query}=#{i + start_page}" }
+        max_batch.times.map { |i| build_page_url(uri, query, i + start_page) }
       end
     end
 
     def process_page_batches(pages, current_depth, batch_size, query)
       scheduled_depth = current_depth
-      max_batch = [@config.max_depth, batch_size].min
+      max_batch = @config.max_depth.zero? ? batch_size : [@config.max_depth, batch_size].min
 
       loop do
         break if reached_max_depth?(scheduled_depth)
@@ -625,7 +627,17 @@ module Crawlr
     end
 
     def generate_next_pages(batch, scheduled_depth, max_batch, query)
-      max_batch.times.map { |i| "#{batch.first}?#{query}=#{i + scheduled_depth + 1}" }
+      uri = URI.parse(batch.first)
+      (0...max_batch).map { |i| build_page_url(uri, query, i + scheduled_depth + 1) }
+    end
+
+    def build_page_url(uri, query, value)
+      new_uri = uri.dup
+      params = URI.decode_www_form(new_uri.query || "")
+      params.reject! { |k, _| k == query }
+      params << [query, value]
+      new_uri.query = URI.encode_www_form(params)
+      new_uri.to_s
     end
   end
 end

data/lib/crawlr/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Crawlr
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: crawlr
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Aristotelis Rapai
 bindir: exe
 cert_chain: []
-date: 2025-09-29 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: async
@@ -181,7 +181,6 @@ files:
 - lib/crawlr/robots.rb
 - lib/crawlr/version.rb
 - lib/crawlr/visits.rb
-- rubygems.rb
 homepage: https://github.com/aristorap/crawlr
 licenses:
 - MIT
@@ -206,7 +205,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.3
+rubygems_version: 3.7.2
 specification_version: 4
 summary: A powerful, async Ruby web scraping framework
 test_files: []

data/rubygems.rb

@@ -1,18 +0,0 @@
-require "lib/crawlr"
-
-clct = Crawlr::Collector.new
-gems = []
-
-clct.visit("https://rubygems.org/releases/popular") do |collector|
-  collector.on_html(:css, ".main--interior a.gems__gem") do |node, ctx|
-    link = node["href"]
-    full_link = ctx.resolve_url(link) if link
-    gems << full_link
-  end
-end
-
-puts "Found #{gems.size} gems"
-
-gems.each do |gem|
-  puts gem
-end