page_by_page 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b0f56c80bdb70ae4aef3c770e9c0fb0554a2bad2
-  data.tar.gz: 2bdb85e741a4de3c5d109fc6539f548e2d8f48d3
+  metadata.gz: ac89d6a7b6b45addd50b7738d2a8b90c6609b1f9
+  data.tar.gz: 3ea2de6bddb75c604dea419092b50e1f353fc4ec
 SHA512:
-  metadata.gz: 510c3c2a34d8de3940659c798aa15758c62fd3bd7311fd14c3a09f77f0f9c9c4e4fd6fe19dfb992947e1ba07ecffc70605a82de747d8e95aa6a7ed74f76fb839
-  data.tar.gz: 8c6ba45664bcdfe832ccd41017a27710cef380cf86c3dd77440cae0c27ecf57f38340af0df819a8fe7569fa60f308cdda6f9c00a4aa4e756bcef55ca2fa39a88
+  metadata.gz: 7a826438ea81f09e717674bbbc955af03676303056ad576c28905bfb74eb45a935f34e3127e85c19c0e5c31d3329a9f91b8e0fbdd87fc76a65b76ffa19dd41cd
+  data.tar.gz: 2706c0b89de6f87b454061e9372db5f5cb1885d5cd735df4d587421a6aa464576b64e948cbc1f5d22e91f3814a3d9b3935410a96d1977d6fab1becf0652aa54c

data/lib/page_by_page.rb

@@ -1,7 +1,9 @@
 require 'page_by_page/version'
 require 'page_by_page/enum'
+require 'page_by_page/mutex_enum'
 require 'nokogiri'
 require 'open-uri'
+require 'erb'
 
 class PageByPage
 
@@ -36,23 +38,52 @@ class PageByPage
     @to = n
   end
 
+  def threads n
+    @threads = n
+  end
+
   def fetch
-    enum = Enum.new options
+    nodes_2d =
+      unless @threads
+        @enum = Enum.new options
+        _fetch
+      else
+        @enum = MutexEnum.new options
+        parallel_fetch
+      end
+    nodes_2d.flatten
+  end
+
+  private
+
+  def _fetch
     items, all_items = [nil], []
     catch :no_more do
       until items.empty?
-        n = enum.next
+        n = @enum.next
         break if n > limit
         url = @tmpl.result binding
         doc = parse url
         items = doc.css @selector
-        all_items << items
+        all_items[n] = items
       end
     end
-    all_items.flatten
+    all_items
   end
 
-  private
+  def parallel_fetch
+    ts = @threads.times.map do |n|
+      Thread.new do
+        Thread.current[:sub] = _fetch
+      end
+    end
+    ts.each_with_object([]) do |t, pages|
+      t.join
+      t[:sub].each_with_index do |items, i|
+        pages[i] = items if items
+      end
+    end
+  end
 
   def parse url
     page = open(url)

data/lib/page_by_page/enum.rb

@@ -1,20 +1,12 @@
-require 'forwardable'
-require 'erb'
-
 class PageByPage
   class Enum
-    extend Forwardable
-
-    def_delegator :@enum, :next
 
     def initialize from: 1, step: 1
-      @enum = Enumerator.new do |yielder|
-        n = from
-        loop do
-          yielder.yield n
-          n = n + step
-        end
-      end
+      @enum = (from..Float::INFINITY).step(step).lazy.map(&:to_i).to_enum
+    end
+
+    def next
+      @enum.next
     end
 
   end

data/lib/page_by_page/mutex_enum.rb

@@ -0,0 +1,22 @@
+require 'page_by_page/enum'
+
+class PageByPage
+  class MutexEnum < Enum
+
+    def initialize from: 1, step: 1
+      super
+      @q = SizedQueue.new 10
+      Thread.new do
+        loop do
+          @q << @enum.next
+          sleep 0.1
+        end
+      end
+    end
+
+    def next
+      @q.deq
+    end
+
+  end
+end

data/lib/page_by_page/version.rb

@@ -1,3 +1,3 @@
 class PageByPage
-  VERSION = "0.1.5"
+  VERSION = "0.1.6"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: page_by_page
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - ken
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-01-01 00:00:00.000000000 Z
+date: 2017-01-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -84,6 +84,7 @@ files:
 - bin/setup
 - lib/page_by_page.rb
 - lib/page_by_page/enum.rb
+- lib/page_by_page/mutex_enum.rb
 - lib/page_by_page/version.rb
 - page_by_page.gemspec
 homepage: https://github.com/turnon/page_by_page