page_by_page 0.1.9 → 0.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: b7b7ac7eab723d81c77692420ccd3fd3e6a69d55
-  data.tar.gz: cfc4977c9d1e3f635e0bc0bded551eb29ee97195
+SHA256:
+  metadata.gz: 47ea21770030654ef4b0d4a7b5d3dec3c1c20d7d43b11e39c5ea7e68be86478f
+  data.tar.gz: 7341f8e5293250b308bb4c223a14f1ec8480513269f3b6c1ef06c5aa9f825e92
 SHA512:
-  metadata.gz: 1840cd52ed71ad5f21c9647a75a9a8630d5df17c6c731e16220d7b8c92361711fe2fdc8eefa2afa458dc106342c133f46c9fde0246e21d6cd0cb7fac279e7be9
-  data.tar.gz: c9625120e03c85610165c6f0b791a9f00ba9939c31c3097371bcd0a0756f62afab99001b976bfbb23beffec977eb3a3158afcffa3e3b383a2c7817c0a84748f1
+  metadata.gz: '0912271bc28adfd2e97313bbecdeed894976eb37814c84bbbb009ca8ec10dbf04c59b25ce4716dee1701b42cbc1015e7dd83441afeb9a8d7cf7678c477328beb'
+  data.tar.gz: 6e46ac599c44dff52b9527de7d526bac874c166191f10e212168034b1477fead6194d1dabfddad46ec74494676e4fc7260b1f25616e5d3c6a08d0bc6250e5fab

data/README.md

@@ -20,6 +20,8 @@ Or install it yourself as:
 
 ## Usage
 
+If you know page number pattern, use fetch:
+
 ```ruby
 nodes = PageByPage.fetch do
   url 'https://book.douban.com/subject/25846075/comments/hot?p=<%= n %>'
@@ -27,12 +29,28 @@ nodes = PageByPage.fetch do
   # from 2
   # step 2
   # to 100
+  # interval 3
   # threads 4
   # no_progress
+  # header Cookie: 'douban-fav-remind=1'
+end
+```
+
+If you don't know the pattern, but you see link to next page, use jump:
+
+```ruby
+nodes = PageByPage.jump do
+  start 'https://book.douban.com/subject/25846075/comments/hot'
+  iterate '.comment-paginator li:nth-child(3) a'
+  selector '.comment-item'
+  # to 100
+  # interval 3
+  # no_progress
+  # header Cookie: 'douban-fav-remind=1'
 end
 ```
 
-Or just passing parameters:
+You may just pass parameters instead of block:
 
 ```ruby
 nodes = PageByPage.fetch(
@@ -41,7 +59,9 @@ nodes = PageByPage.fetch(
   # from: 2,
   # step: 2,
   # to: 100,
+  # interval: 3
   # threads: 4,
   # no_progress: true
+  # header: {Cookie: 'douban-fav-remind=1'}
 )
 ```

data/lib/page_by_page.rb

@@ -1,16 +1,21 @@
 require 'page_by_page/version'
-require 'page_by_page/enum'
-require 'page_by_page/mutex_enum'
+require 'page_by_page/fetch'
+require 'page_by_page/jump'
 require 'nokogiri'
 require 'open-uri'
-require 'erb'
 
 class PageByPage
 
+  include Fetch
+  include Jump
+
   class << self
-    def fetch(opt ={}, &block)
-      pbp = self.new(opt, &block)
-      pbp.fetch
+    def fetch(*args, &block)
+      new(*args, &block).fetch
+    end
+
+    def jump(*args, &block)
+      new(*args, &block).jump
     end
   end
 
@@ -21,81 +26,26 @@ class PageByPage
     instance_eval &block if block
   end
 
-  def url tmpl
-    @tmpl = ERB.new tmpl
-  end
-
-  def selector sl
-    @selector = sl
-  end
-
-  def from n
-    @from = n
-  end
-
-  def step n
-    @step = n
-  end
-
   def to n
     @to = n
   end
 
-  def threads n
-    @threads = n
+  def selector sl
+    @selector = sl
   end
 
-  def no_progress *arg
-    @progress = nil
+  def header hash
+    @header = hash
   end
 
-  def fetch
-    nodes_2d =
-      unless defined? @threads
-        @enum = Enum.new options
-        _fetch
-      else
-        @enum = MutexEnum.new options
-        parallel_fetch
-      end
-    puts if @progress
-    nodes_2d.reject(&:nil?).flatten
+  def interval second
+    @interval = second
   end
 
   private
 
-  def _fetch
-    items, pages = [nil], []
-    catch :no_more do
-      until items.empty?
-        n = @enum.next
-        break if n > limit
-        url = @tmpl.result binding
-        doc = parse url
-        items = doc.css @selector
-        pages[n] = items
-        update_progress Thread.current, n if @progress
-      end
-    end
-    pages
-  end
-
-  def parallel_fetch
-    ts = @threads.times.map do |n|
-      Thread.new do
-        Thread.current[:sub] = _fetch
-      end
-    end
-    ts.each_with_object([]) do |t, pages|
-      t.join
-      t[:sub].each_with_index do |items, i|
-        pages[i] = items if items
-      end
-    end
-  end
-
   def parse url
-    page = open(url)
+    page = open(url, http_header)
     Nokogiri::HTML page.read
   rescue OpenURI::HTTPError => e
     if e.message == '404 Not Found'
@@ -105,8 +55,12 @@ class PageByPage
     end
   end
 
-  def options
-    {from: @from, step: @step}
+  def http_header
+    @http_header ||= (
+      h = {}
+      Hash(@header).each_pair{ |k, v| h[k.to_s] = v }
+      h
+    )
   end
 
   def limit

data/lib/page_by_page/fetch.rb

@@ -0,0 +1,81 @@
+require 'page_by_page/enum'
+require 'page_by_page/mutex_enum'
+require 'erb'
+
+class PageByPage
+  module Fetch
+
+    def url tmpl
+      @tmpl = ERB.new tmpl
+    end
+
+    def from n
+      @from = n
+    end
+
+    def step n
+      @step = n
+    end
+
+    def threads n
+      @threads = n
+    end
+
+    def no_progress *arg
+      @progress = nil
+    end
+
+    def fetch
+      nodes_2d =
+        unless defined? @threads
+          @enum = Enum.new enum_options
+          _fetch
+        else
+          @enum = MutexEnum.new enum_options
+          parallel_fetch
+        end
+      puts if @progress
+      nodes_2d.reject(&:nil?).flatten
+    end
+
+    protected
+
+    def _fetch
+      items, pages = [nil], []
+      catch :no_more do
+        until items.empty?
+          n = @enum.next
+          break if n > limit
+
+          url = @tmpl.result binding
+          doc = parse url
+          items = doc.css @selector
+          pages[n] = items
+
+          update_progress Thread.current, n if @progress
+          sleep @interval if @interval
+        end
+      end
+      pages
+    end
+
+    def parallel_fetch
+      ts = @threads.times.map do |n|
+        Thread.new do
+          Thread.current[:sub] = _fetch
+        end
+      end
+      ts.each_with_object([]) do |t, pages|
+        t.join
+        t[:sub].each_with_index do |items, i|
+          pages[i] = items if items
+        end
+      end
+    end
+
+    def enum_options
+      {from: @from, step: @step}
+    end
+
+  end
+end

data/lib/page_by_page/jump.rb

@@ -0,0 +1,45 @@
+class PageByPage
+  module Jump
+
+    def start url
+      @start = url
+    end
+
+    def iterate selector
+      @iterate = selector
+    end
+
+    def jump
+      url, items, page_count = @start, [], 0
+
+      while true do
+        doc = parse url
+        doc.css(@selector).each{ |item| items << item }
+
+        next_url = doc.at_css(@iterate)
+        break unless next_url
+
+        path = next_url.attr('href')
+        url = concat_host path
+
+        page_count += 1
+        update_progress Thread.current, page_count if @progress
+        break if page_count >= limit
+
+        sleep @interval if @interval
+      end
+
+      items
+    end
+
+    private
+
+    def concat_host path
+      @prefix = (
+        regex = path.start_with?('/') ? /([^:|\/])\/.*/ : /(.*[^:|\/])\/.*/
+        @start.gsub(regex, '\1')
+      )
+      File.join @prefix, path
+    end
+  end
+end

data/lib/page_by_page/version.rb

@@ -1,3 +1,3 @@
 class PageByPage
-  VERSION = "0.1.9"
+  VERSION = "0.1.10"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: page_by_page
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.1.10
 platform: ruby
 authors:
 - ken
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-10-08 00:00:00.000000000 Z
+date: 2019-02-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -84,6 +84,8 @@ files:
 - bin/setup
 - lib/page_by_page.rb
 - lib/page_by_page/enum.rb
+- lib/page_by_page/fetch.rb
+- lib/page_by_page/jump.rb
 - lib/page_by_page/mutex_enum.rb
 - lib/page_by_page/version.rb
 - page_by_page.gemspec
@@ -108,7 +110,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: scrape page by page , according to url pattern