page_by_page 0.1.10 → 0.1.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 47ea21770030654ef4b0d4a7b5d3dec3c1c20d7d43b11e39c5ea7e68be86478f
-  data.tar.gz: 7341f8e5293250b308bb4c223a14f1ec8480513269f3b6c1ef06c5aa9f825e92
+  metadata.gz: 46595cb8e0d8590c5b614eac4301b0423824a12f43b94e83294e325fd1c7f5c7
+  data.tar.gz: 11ac56d8a0061a00d2807307f285332c5548b06b915bfcd94675d5d57d7b8454
 SHA512:
-  metadata.gz: '0912271bc28adfd2e97313bbecdeed894976eb37814c84bbbb009ca8ec10dbf04c59b25ce4716dee1701b42cbc1015e7dd83441afeb9a8d7cf7678c477328beb'
-  data.tar.gz: 6e46ac599c44dff52b9527de7d526bac874c166191f10e212168034b1477fead6194d1dabfddad46ec74494676e4fc7260b1f25616e5d3c6a08d0bc6250e5fab
+  metadata.gz: fa81661e065393f115f355e666f43b5bf636019f50c4c40a33faf8bfe44e1134cce5f7eb3255f68a427d178b3e90a986a69b059551cfecca51ff779b3bd3c25d
+  data.tar.gz: 3f92a09ec86789cd0517597a5f15850bcb48303c4e1c3980efb3d57c6ba2d34c4c46fc287df060fb9328ca6efc5543f3c5abe5b2f717974b618b2a171fd0116f

data/lib/page_by_page/common.rb

@@ -0,0 +1,63 @@
+require 'nokogiri'
+require 'open-uri'
+
+module PageByPage
+  module Common
+    def initialize(opt = {}, &block)
+      @progress = {}
+      opt.each{ |name, value| send name, value }
+      instance_eval &block if block
+    end
+
+    def to n
+      @to = n
+    end
+
+    def selector sl
+      @selector = sl
+    end
+
+    def header hash
+      @header = hash
+    end
+
+    def interval second
+      @interval = second
+    end
+
+    def no_progress *arg
+      @progress = nil
+    end
+
+    protected
+
+    def parse url
+      page = open(url, http_header)
+      Nokogiri::HTML page.read
+    rescue OpenURI::HTTPError => e
+      if e.message == '404 Not Found'
+        throw :no_more
+      else
+        raise e
+      end
+    end
+
+    def http_header
+      @http_header ||= (
+        h = {}
+        Hash(@header).each_pair{ |k, v| h[k.to_s] = v }
+        h
+      )
+    end
+
+    def limit
+      @to ||= Float::INFINITY
+    end
+
+    def update_progress thread, page_num
+      @progress[thread] = page_num
+      printf "\r%s => %s", Time.now.strftime('%F %T'), @progress.values.sort
+    end
+
+  end
+end

data/lib/page_by_page/enum.rb

@@ -1,4 +1,4 @@
-class PageByPage
+module PageByPage
   class Enum
 
     def initialize from: 1, step: 1

data/lib/page_by_page/fetch.rb

@@ -1,9 +1,17 @@
 require 'page_by_page/enum'
 require 'page_by_page/mutex_enum'
+require 'page_by_page/common'
 require 'erb'
 
-class PageByPage
-  module Fetch
+module PageByPage
+  class Fetch
+
+    include Common
+
+    def initialize(opt = {}, &block)
+      @from, @step, @to = 1, 1, Float::INFINITY
+      super
+    end
 
     def url tmpl
       @tmpl = ERB.new tmpl
@@ -21,11 +29,7 @@ class PageByPage
       @threads = n
     end
 
-    def no_progress *arg
-      @progress = nil
-    end
-
-    def fetch
+    def process
       nodes_2d =
         unless defined? @threads
           @enum = Enum.new enum_options

data/lib/page_by_page/jump.rb

@@ -1,5 +1,9 @@
-class PageByPage
-  module Jump
+require 'page_by_page/common'
+
+module PageByPage
+  class Jump
+
+    include Common
 
     def start url
       @start = url
@@ -9,26 +13,27 @@ class PageByPage
       @iterate = selector
     end
 
-    def jump
+    def process
       url, items, page_count = @start, [], 0
 
       while true do
         doc = parse url
         doc.css(@selector).each{ |item| items << item }
 
+        page_count += 1
+        update_progress Thread.current, page_count if @progress
+        break if page_count >= limit
+
         next_url = doc.at_css(@iterate)
         break unless next_url
 
         path = next_url.attr('href')
         url = concat_host path
 
-        page_count += 1
-        update_progress Thread.current, page_count if @progress
-        break if page_count >= limit
-
         sleep @interval if @interval
       end
 
+      puts if @progress
       items
     end
 

data/lib/page_by_page/mutex_enum.rb

@@ -1,6 +1,6 @@
 require 'page_by_page/enum'
 
-class PageByPage
+module PageByPage
   class MutexEnum < Enum
 
     def initialize from: 1, step: 1

data/lib/page_by_page/version.rb

@@ -1,3 +1,3 @@
-class PageByPage
-  VERSION = "0.1.10"
+module PageByPage
+  VERSION = "0.1.11"
 end

data/lib/page_by_page.rb

@@ -1,75 +1,17 @@
 require 'page_by_page/version'
 require 'page_by_page/fetch'
 require 'page_by_page/jump'
-require 'nokogiri'
-require 'open-uri'
 
-class PageByPage
-
-  include Fetch
-  include Jump
+module PageByPage
 
   class << self
     def fetch(*args, &block)
-      new(*args, &block).fetch
+      Fetch.new(*args, &block).process
     end
 
     def jump(*args, &block)
-      new(*args, &block).jump
+      Jump.new(*args, &block).process
     end
   end
 
-  def initialize(opt = {}, &block)
-    @from, @step, @to = 1, 1, Float::INFINITY
-    @progress = {}
-    opt.each{ |name, value| send name, value }
-    instance_eval &block if block
-  end
-
-  def to n
-    @to = n
-  end
-
-  def selector sl
-    @selector = sl
-  end
-
-  def header hash
-    @header = hash
-  end
-
-  def interval second
-    @interval = second
-  end
-
-  private
-
-  def parse url
-    page = open(url, http_header)
-    Nokogiri::HTML page.read
-  rescue OpenURI::HTTPError => e
-    if e.message == '404 Not Found'
-      throw :no_more
-    else
-      raise e
-    end
-  end
-
-  def http_header
-    @http_header ||= (
-      h = {}
-      Hash(@header).each_pair{ |k, v| h[k.to_s] = v }
-      h
-    )
-  end
-
-  def limit
-    @to ||= Float::INFINITY
-  end
-
-  def update_progress thread, page_num
-    @progress[thread] = page_num
-    printf "\r%s => %s", Time.now.strftime('%F %T'), @progress.values.sort
-  end
-
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: page_by_page
 version: !ruby/object:Gem::Version
-  version: 0.1.10
+  version: 0.1.11
 platform: ruby
 authors:
 - ken
@@ -83,6 +83,7 @@ files:
 - bin/console
 - bin/setup
 - lib/page_by_page.rb
+- lib/page_by_page/common.rb
 - lib/page_by_page/enum.rb
 - lib/page_by_page/fetch.rb
 - lib/page_by_page/jump.rb