page_by_page 0.1.13 → 0.1.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a00d32af004cc151bf366b9498e18f2e7a2da73629a5db0373b7aad7992f995d
-  data.tar.gz: 5301c76cc3968c6f187e3349d1de5bf2f4a19a48440f1484a29e25ea35af0342
+  metadata.gz: 51cda3ebfc8bc9a353a51f1386a91cf6ea3d2fbbc219334f278844765b35ad98
+  data.tar.gz: 8329a8af72dd945284617f053cbb1ae686cd5eb9151dd1cf9be0c99b7cdd12ac
 SHA512:
-  metadata.gz: d1da5e785a40849da75b27c9067098e700444be462791654de47914e45b7f45adb3bd5e94945d7908384cd22668ed30cfbbac6a39a73ef6386d31f378ac6dad4
-  data.tar.gz: 751d3f450cecbf45b2fe5cede0af225adee8948cc5352c936e279c88f765e589fe8c8cad201c320bab03dacb7bff66be70fc5b9325fbebd3e9a017474366d0ca
+  metadata.gz: 6761cb5c91ed8736be1193237caa2725110070b9de332af5bd6175371064b3b3d939ba997634c8e3711f8f1d81c7dc3988436d807858951c6fd398e39249b8c7
+  data.tar.gz: 1cee71ddc3dab888aaf75a8cf7139bfd630ef63c29dc954d00c5cdac7c4a251e78a19dedc56ea3e6b8713289b514cabe2519322efc5accd977fb6084a1768088

data/README.md

@@ -20,7 +20,9 @@ Or install it yourself as:
 
 ## Usage
 
-If you know page number pattern, use fetch:
+### number pattern
+
+If you know page number pattern, use `fetch`:
 
 ```ruby
 nodes = PageByPage.fetch do
@@ -36,7 +38,21 @@ nodes = PageByPage.fetch do
 end
 ```
 
-If you don't know the pattern, but you see link to next page, use jump:
+### other pattern
+
+If the pattern is not simple numbers, use `enumerator` in `fetch`:
+
+```ruby
+nodes = PageByiPage.fetch do
+  url 'http://mysql.taobao.org/monthly/<%= n %>'
+  selector 'h3'
+  enumerator ['2020/09/', '2020/08/'].to_enum
+end
+```
+
+### unknown pattern
+
+If you don't know the pattern, but you see link to next page, use `jump`:
 
 ```ruby
 nodes = PageByPage.jump do
@@ -50,6 +66,8 @@ nodes = PageByPage.jump do
 end
 ```
 
+### parameters instead of block
+
 You may just pass parameters instead of block:
 
 ```ruby
@@ -66,6 +84,8 @@ nodes = PageByPage.fetch(
 )
 ```
 
+### lazy
+
 Also note that, instead of Array, `lazy_fetch` returns an Enumerator, which is native lazy-loading:
 
 ```ruby

data/lib/page_by_page/enum.rb

@@ -1,12 +1,12 @@
 module PageByPage
   class Enum
 
-    def initialize from: 1, step: 1
-      @enum = (from..Float::INFINITY).step(step).lazy.map(&:to_i).to_enum
+    def initialize from: 1, step: 1, limit: nil, enumerator: nil
+      @enum = enumerator || (from..limit).step(step).lazy.map(&:to_i).to_enum
     end
 
     def next
-      @enum.next
+      @enum.next rescue nil
     end
 
   end

data/lib/page_by_page/fetch.rb

@@ -11,7 +11,8 @@ module PageByPage
     def initialize(opt = {}, &block)
       @from, @step, @to = 1, 1, Float::INFINITY
       super
-      @enum = (defined?(@threads) ? MutexEnum : Enum).new(enum_options)
+      @enum = Enum.new(enum_options)
+      @enum = MutexEnum.new(@enum) if defined? @threads
     end
 
     def url tmpl
@@ -30,10 +31,17 @@ module PageByPage
       @threads = n
     end
 
+    def enumerator e
+      @enumerator = e
+    end
+
     def process
       nodes_2d = defined?(@threads) ? parallel_fetch : _fetch
       puts if @progress
-      nodes_2d.reject(&:nil?).flatten
+
+      nodes_2d.sort.each_with_object([]) do |key_items, res|
+        res.concat key_items[1] unless key_items[1].nil?
+      end
     end
 
     def iterator
@@ -49,7 +57,7 @@ module PageByPage
     protected
 
     def _fetch
-      pages = []
+      pages = {}
 
       items_enum.each do |page_num, items|
         pages[page_num] = items
@@ -64,7 +72,7 @@ module PageByPage
         catch :no_more do
           until items.empty?
             n = @enum.next
-            break if n > limit
+            break if n.nil?
 
             url = @tmpl.result binding
             doc = parse url
@@ -84,16 +92,14 @@ module PageByPage
           Thread.current[:sub] = _fetch
         end
       end
-      ts.each_with_object([]) do |t, pages|
+      ts.each_with_object({}) do |t, pages|
         t.join
-        t[:sub].each_with_index do |items, i|
-          pages[i] = items if items
-        end
+        pages.merge! t[:sub]
       end
     end
 
     def enum_options
-      {from: @from, step: @step}
+      {from: @from, step: @step, limit: limit, enumerator: @enumerator}
     end
 
   end

data/lib/page_by_page/jump.rb

@@ -28,7 +28,7 @@ module PageByPage
         break unless next_url
 
         path = next_url.attr('href')
-        url = concat_host path
+        url = path.start_with?('/') ? concat_host(path) : path
 
         sleep @interval if @interval
       end

data/lib/page_by_page/mutex_enum.rb

@@ -1,15 +1,15 @@
 require 'page_by_page/enum'
+require 'thread'
 
 module PageByPage
   class MutexEnum < Enum
 
-    def initialize from: 1, step: 1
-      super
+    def initialize enum
       @q = SizedQueue.new 10
+      @enum = enum
       Thread.new do
         loop do
           @q << @enum.next
-          sleep 0.1
         end
       end
     end

data/lib/page_by_page/version.rb

@@ -1,3 +1,3 @@
 module PageByPage
-  VERSION = "0.1.13"
+  VERSION = "0.1.14"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: page_by_page
 version: !ruby/object:Gem::Version
-  version: 0.1.13
+  version: 0.1.14
 platform: ruby
 authors:
 - ken
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-11-20 00:00:00.000000000 Z
+date: 2020-10-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler