page_by_page 0.1.9 → 0.1.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: b7b7ac7eab723d81c77692420ccd3fd3e6a69d55
-  data.tar.gz: cfc4977c9d1e3f635e0bc0bded551eb29ee97195
+SHA256:
+  metadata.gz: 51cda3ebfc8bc9a353a51f1386a91cf6ea3d2fbbc219334f278844765b35ad98
+  data.tar.gz: 8329a8af72dd945284617f053cbb1ae686cd5eb9151dd1cf9be0c99b7cdd12ac
 SHA512:
-  metadata.gz: 1840cd52ed71ad5f21c9647a75a9a8630d5df17c6c731e16220d7b8c92361711fe2fdc8eefa2afa458dc106342c133f46c9fde0246e21d6cd0cb7fac279e7be9
-  data.tar.gz: c9625120e03c85610165c6f0b791a9f00ba9939c31c3097371bcd0a0756f62afab99001b976bfbb23beffec977eb3a3158afcffa3e3b383a2c7817c0a84748f1
+  metadata.gz: 6761cb5c91ed8736be1193237caa2725110070b9de332af5bd6175371064b3b3d939ba997634c8e3711f8f1d81c7dc3988436d807858951c6fd398e39249b8c7
+  data.tar.gz: 1cee71ddc3dab888aaf75a8cf7139bfd630ef63c29dc954d00c5cdac7c4a251e78a19dedc56ea3e6b8713289b514cabe2519322efc5accd977fb6084a1768088

data/README.md

@@ -20,6 +20,10 @@ Or install it yourself as:
 
 ## Usage
 
+### number pattern
+
+If you know page number pattern, use `fetch`:
+
 ```ruby
 nodes = PageByPage.fetch do
   url 'https://book.douban.com/subject/25846075/comments/hot?p=<%= n %>'
@@ -27,12 +31,44 @@ nodes = PageByPage.fetch do
   # from 2
   # step 2
   # to 100
+  # interval 3
   # threads 4
   # no_progress
+  # header Cookie: 'douban-fav-remind=1'
+end
+```
+
+### other pattern
+
+If the pattern is not simple numbers, use `enumerator` in `fetch`:
+
+```ruby
+nodes = PageByiPage.fetch do
+  url 'http://mysql.taobao.org/monthly/<%= n %>'
+  selector 'h3'
+  enumerator ['2020/09/', '2020/08/'].to_enum
+end
+```
+
+### unknown pattern
+
+If you don't know the pattern, but you see link to next page, use `jump`:
+
+```ruby
+nodes = PageByPage.jump do
+  start 'https://book.douban.com/subject/25846075/comments/hot'
+  iterate '.comment-paginator li:nth-child(3) a'
+  selector '.comment-item'
+  # to 100
+  # interval 3
+  # no_progress
+  # header Cookie: 'douban-fav-remind=1'
 end
 ```
 
-Or just passing parameters:
+### parameters instead of block
+
+You may just pass parameters instead of block:
 
 ```ruby
 nodes = PageByPage.fetch(
@@ -41,7 +77,19 @@ nodes = PageByPage.fetch(
   # from: 2,
   # step: 2,
   # to: 100,
+  # interval: 3
   # threads: 4,
   # no_progress: true
+  # header: {Cookie: 'douban-fav-remind=1'}
+)
+```
+
+### lazy
+
+Also note that, instead of Array, `lazy_fetch` returns an Enumerator, which is native lazy-loading:
+
+```ruby
+nodes = PageByPage.lazy_fetch(
+  #...
 )
 ```

data/bin/console

@@ -10,5 +10,5 @@ require "page_by_page"
 # require "pry"
 # Pry.start
 
-require "irb"
-IRB.start
+require "pry"
+pry

data/lib/page_by_page.rb

@@ -1,121 +1,21 @@
 require 'page_by_page/version'
-require 'page_by_page/enum'
-require 'page_by_page/mutex_enum'
-require 'nokogiri'
-require 'open-uri'
-require 'erb'
+require 'page_by_page/fetch'
+require 'page_by_page/jump'
 
-class PageByPage
+module PageByPage
 
   class << self
-    def fetch(opt ={}, &block)
-      pbp = self.new(opt, &block)
-      pbp.fetch
+    def fetch(*args, &block)
+      Fetch.new(*args, &block).process
     end
-  end
-
-  def initialize(opt = {}, &block)
-    @from, @step, @to = 1, 1, Float::INFINITY
-    @progress = {}
-    opt.each{ |name, value| send name, value }
-    instance_eval &block if block
-  end
-
-  def url tmpl
-    @tmpl = ERB.new tmpl
-  end
-
-  def selector sl
-    @selector = sl
-  end
-
-  def from n
-    @from = n
-  end
-
-  def step n
-    @step = n
-  end
-
-  def to n
-    @to = n
-  end
-
-  def threads n
-    @threads = n
-  end
 
-  def no_progress *arg
-    @progress = nil
-  end
-
-  def fetch
-    nodes_2d =
-      unless defined? @threads
-        @enum = Enum.new options
-        _fetch
-      else
-        @enum = MutexEnum.new options
-        parallel_fetch
-      end
-    puts if @progress
-    nodes_2d.reject(&:nil?).flatten
-  end
-
-  private
-
-  def _fetch
-    items, pages = [nil], []
-    catch :no_more do
-      until items.empty?
-        n = @enum.next
-        break if n > limit
-        url = @tmpl.result binding
-        doc = parse url
-        items = doc.css @selector
-        pages[n] = items
-        update_progress Thread.current, n if @progress
-      end
+    def lazy_fetch(*args, &block)
+      Fetch.new(*args, &block).iterator
     end
-    pages
-  end
 
-  def parallel_fetch
-    ts = @threads.times.map do |n|
-      Thread.new do
-        Thread.current[:sub] = _fetch
-      end
-    end
-    ts.each_with_object([]) do |t, pages|
-      t.join
-      t[:sub].each_with_index do |items, i|
-        pages[i] = items if items
-      end
+    def jump(*args, &block)
+      Jump.new(*args, &block).process
     end
   end
 
-  def parse url
-    page = open(url)
-    Nokogiri::HTML page.read
-  rescue OpenURI::HTTPError => e
-    if e.message == '404 Not Found'
-      throw :no_more
-    else
-      raise e
-    end
-  end
-
-  def options
-    {from: @from, step: @step}
-  end
-
-  def limit
-    @to ||= Float::INFINITY
-  end
-
-  def update_progress thread, page_num
-    @progress[thread] = page_num
-    printf "\r%s => %s", Time.now.strftime('%F %T'), @progress.values.sort
-  end
-
 end

data/lib/page_by_page/common.rb

@@ -0,0 +1,64 @@
+require 'nokogiri'
+require 'open-uri'
+
+module PageByPage
+  module Common
+    def initialize(opt = {}, &block)
+      @progress = {}
+      opt.each{ |name, value| send name, value }
+      instance_eval &block if block
+    end
+
+    def to n
+      @to = n
+    end
+
+    def selector sl
+      @selector = sl
+    end
+
+    def header hash
+      @header = hash
+    end
+
+    def interval second
+      @interval = second
+    end
+
+    def no_progress *arg
+      @progress = nil
+    end
+
+    protected
+
+    def parse url
+      url = URI::encode url
+      page = open(url, http_header)
+      Nokogiri::HTML page.read
+    rescue OpenURI::HTTPError => e
+      if e.message == '404 Not Found'
+        throw :no_more
+      else
+        raise e
+      end
+    end
+
+    def http_header
+      @http_header ||= (
+        h = {}
+        Hash(@header).each_pair{ |k, v| h[k.to_s] = v }
+        h
+      )
+    end
+
+    def limit
+      @to ||= Float::INFINITY
+    end
+
+    def update_progress thread, page_num
+      @progress[thread] = page_num
+      printf "\r%s => %s", Time.now.strftime('%F %T'), @progress.values.sort
+    end
+
+  end
+end

data/lib/page_by_page/enum.rb

@@ -1,12 +1,12 @@
-class PageByPage
+module PageByPage
   class Enum
 
-    def initialize from: 1, step: 1
-      @enum = (from..Float::INFINITY).step(step).lazy.map(&:to_i).to_enum
+    def initialize from: 1, step: 1, limit: nil, enumerator: nil
+      @enum = enumerator || (from..limit).step(step).lazy.map(&:to_i).to_enum
     end
 
     def next
-      @enum.next
+      @enum.next rescue nil
     end
 
   end

data/lib/page_by_page/fetch.rb

@@ -0,0 +1,106 @@
+require 'page_by_page/enum'
+require 'page_by_page/mutex_enum'
+require 'page_by_page/common'
+require 'erb'
+
+module PageByPage
+  class Fetch
+
+    include Common
+
+    def initialize(opt = {}, &block)
+      @from, @step, @to = 1, 1, Float::INFINITY
+      super
+      @enum = Enum.new(enum_options)
+      @enum = MutexEnum.new(@enum) if defined? @threads
+    end
+
+    def url tmpl
+      @tmpl = ERB.new tmpl
+    end
+
+    def from n
+      @from = n
+    end
+
+    def step n
+      @step = n
+    end
+
+    def threads n
+      @threads = n
+    end
+
+    def enumerator e
+      @enumerator = e
+    end
+
+    def process
+      nodes_2d = defined?(@threads) ? parallel_fetch : _fetch
+      puts if @progress
+
+      nodes_2d.sort.each_with_object([]) do |key_items, res|
+        res.concat key_items[1] unless key_items[1].nil?
+      end
+    end
+
+    def iterator
+      Enumerator.new do |yielder|
+        items_enum.each do |_, items|
+          items.each do |i|
+            yielder.yield(i)
+          end
+        end
+      end
+    end
+
+    protected
+
+    def _fetch
+      pages = {}
+
+      items_enum.each do |page_num, items|
+        pages[page_num] = items
+      end
+
+      pages
+    end
+
+    def items_enum
+      Enumerator.new do |yielder|
+        items = [nil]
+        catch :no_more do
+          until items.empty?
+            n = @enum.next
+            break if n.nil?
+
+            url = @tmpl.result binding
+            doc = parse url
+            items = doc.css @selector
+            yielder.yield(n, items)
+
+            update_progress Thread.current, n if @progress
+            sleep @interval if @interval
+          end
+        end
+      end
+    end
+
+    def parallel_fetch
+      ts = @threads.times.map do |n|
+        Thread.new do
+          Thread.current[:sub] = _fetch
+        end
+      end
+      ts.each_with_object({}) do |t, pages|
+        t.join
+        pages.merge! t[:sub]
+      end
+    end
+
+    def enum_options
+      {from: @from, step: @step, limit: limit, enumerator: @enumerator}
+    end
+
+  end
+end

data/lib/page_by_page/jump.rb

@@ -0,0 +1,50 @@
+require 'page_by_page/common'
+
+module PageByPage
+  class Jump
+
+    include Common
+
+    def start url
+      @start = url
+    end
+
+    def iterate selector
+      @iterate = selector
+    end
+
+    def process
+      url, items, page_count = @start, [], 0
+
+      while true do
+        doc = parse url
+        doc.css(@selector).each{ |item| items << item }
+
+        page_count += 1
+        update_progress Thread.current, page_count if @progress
+        break if page_count >= limit
+
+        next_url = doc.at_css(@iterate)
+        break unless next_url
+
+        path = next_url.attr('href')
+        url = path.start_with?('/') ? concat_host(path) : path
+
+        sleep @interval if @interval
+      end
+
+      puts if @progress
+      items
+    end
+
+    private
+
+    def concat_host path
+      @prefix = (
+        regex = path.start_with?('/') ? /([^:|\/])\/.*/ : /(.*[^:|\/])\/.*/
+        @start.gsub(regex, '\1')
+      )
+      File.join @prefix, path
+    end
+  end
+end

data/lib/page_by_page/mutex_enum.rb

@@ -1,15 +1,15 @@
 require 'page_by_page/enum'
+require 'thread'
 
-class PageByPage
+module PageByPage
   class MutexEnum < Enum
 
-    def initialize from: 1, step: 1
-      super
+    def initialize enum
       @q = SizedQueue.new 10
+      @enum = enum
       Thread.new do
         loop do
           @q << @enum.next
-          sleep 0.1
         end
       end
     end

data/lib/page_by_page/version.rb

@@ -1,3 +1,3 @@
-class PageByPage
-  VERSION = "0.1.9"
+module PageByPage
+  VERSION = "0.1.14"
 end

data/page_by_page.gemspec

@@ -32,5 +32,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.13"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "minitest", "~> 5.0"
+  spec.add_development_dependency "pry"
   spec.add_dependency 'nokogiri', '~> 1.6'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: page_by_page
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.1.14
 platform: ruby
 authors:
 - ken
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-10-08 00:00:00.000000000 Z
+date: 2020-10-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -83,7 +97,10 @@ files:
 - bin/console
 - bin/setup
 - lib/page_by_page.rb
+- lib/page_by_page/common.rb
 - lib/page_by_page/enum.rb
+- lib/page_by_page/fetch.rb
+- lib/page_by_page/jump.rb
 - lib/page_by_page/mutex_enum.rb
 - lib/page_by_page/version.rb
 - page_by_page.gemspec
@@ -107,8 +124,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: scrape page by page , according to url pattern