page_by_page 0.1.7 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b503de846c6f14e44ebb68d6da30f0647e3a9e42
-  data.tar.gz: 0b74d6adf88048ed8a78441ec24fad1f686c8ac7
+  metadata.gz: e272639f27ba7c77e9c211f7189099dedade9d8f
+  data.tar.gz: b4571013bb77691327b37fe8a646cf162469d394
 SHA512:
-  metadata.gz: 032d1274977d459339c51d44648b56ba3eb066818137866ed8c80753f0b9009bebf40a59ef614601034e56e02d466a8636cbf5af86bb9b68309da20c01ba09bd
-  data.tar.gz: 70f97621564edf8ccbade3a23131ca53722f5e7357382d1803ffdf5e17cb1a359e397ac2d6d32c97d8411bca1391a2bc1c51aae6e9378ae1dd508cf50d254f40
+  metadata.gz: 12888cf1ddf115bfe0866339fc1c574176c757bfa13154a34d0c1c777c709c2ef083037018f08584a6550c14d9e22522a1cbed0e2dc7f981759af4c9bf28a9a9
+  data.tar.gz: 53de9357b1c06ce217e92ff3619b45455280fe351b281e37450ad17b7041737985a187cace0305dda93a62d284dc588393ab4de9fb1aa43e575dc41894a4ff8b

data/README.md

@@ -24,6 +24,11 @@ Or install it yourself as:
 nodes = PageByPage.fetch do
   url 'https://book.douban.com/subject/25846075/comments/hot?p=<%= n %>'
   selector '.comment-item'
+  # from 2
+  # step 2
+  # to 100
+  # threads 4
+  # no_progress
 end
 ```
 

data/lib/page_by_page.rb

@@ -15,6 +15,8 @@ class PageByPage
   end
 
   def initialize &block
+    @from, @step, @to = 1, 1, Float::INFINITY
+    @progress = {}
     instance_eval &block
   end
 
@@ -42,15 +44,20 @@ class PageByPage
     @threads = n
   end
 
+  def no_progress
+    @progress = nil
+  end
+
   def fetch
     nodes_2d =
-      unless @threads
+      unless defined? @threads
         @enum = Enum.new options
         _fetch
       else
         @enum = MutexEnum.new options
         parallel_fetch
       end
+    puts if @progress
     nodes_2d.reject(&:nil?).flatten
   end
 
@@ -66,6 +73,7 @@ class PageByPage
         doc = parse url
         items = doc.css @selector
         pages[n] = items
+        update_progress Thread.current, n if @progress
       end
     end
     pages
@@ -97,14 +105,16 @@ class PageByPage
   end
 
   def options
-    opt = {}
-    opt[:from] = @from || 1
-    opt[:step] = @step || 1
-    opt
+    {from: @from, step: @step}
   end
 
   def limit
-    @to || Float::INFINITY
+    @to ||= Float::INFINITY
+  end
+
+  def update_progress thread, page_num
+    @progress[thread] = page_num
+    printf "\r%s => %s", Time.now.strftime('%F %T'), @progress.values.sort
   end
 
 end

data/lib/page_by_page/version.rb

@@ -1,3 +1,3 @@
 class PageByPage
-  VERSION = "0.1.7"
+  VERSION = "0.1.8"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: page_by_page
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.1.8
 platform: ruby
 authors:
 - ken
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-01-04 00:00:00.000000000 Z
+date: 2017-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -108,7 +108,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.7
+rubygems_version: 2.6.8
 signing_key: 
 specification_version: 4
 summary: scrape page by page , according to url pattern