page_by_page 0.1.12 → 0.1.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 46a381dd7f98d7e663cc854796d1db3394b7a7b65f9a415669106f1c6be8f61d
-  data.tar.gz: d8db8c6100555d3c7a015ec010e12fb291a67cd1fda762c04dc046419bd8ff6d
+  metadata.gz: a00d32af004cc151bf366b9498e18f2e7a2da73629a5db0373b7aad7992f995d
+  data.tar.gz: 5301c76cc3968c6f187e3349d1de5bf2f4a19a48440f1484a29e25ea35af0342
 SHA512:
-  metadata.gz: 61a81b079f177ac91b9d4dbf17bbea9ec4f78b2f6ee17a4da240a6ed012309c2d7dd75bc6996af3f2b7ad8be947baa0c6531978317aac02f2dc5791ebeb8f2e3
-  data.tar.gz: 85032f0d49c01bb26ec4c714aa499cdb152a67a75752cf57c60e71a65e77a0a0b83ef81d11f70a60374cdcc18915bf158f51d9646ddbe8c05bd1dfc564450e53
+  metadata.gz: d1da5e785a40849da75b27c9067098e700444be462791654de47914e45b7f45adb3bd5e94945d7908384cd22668ed30cfbbac6a39a73ef6386d31f378ac6dad4
+  data.tar.gz: 751d3f450cecbf45b2fe5cede0af225adee8948cc5352c936e279c88f765e589fe8c8cad201c320bab03dacb7bff66be70fc5b9325fbebd3e9a017474366d0ca

data/README.md

@@ -65,3 +65,11 @@ nodes = PageByPage.fetch(
   # header: {Cookie: 'douban-fav-remind=1'}
 )
 ```
+
+Also note that, instead of Array, `lazy_fetch` returns an Enumerator, which is native lazy-loading:
+
+```ruby
+nodes = PageByPage.lazy_fetch(
+  #...
+)
+```

data/bin/console

@@ -10,5 +10,5 @@ require "page_by_page"
 # require "pry"
 # Pry.start
 
-require "irb"
-IRB.start
+require "pry"
+pry

data/lib/page_by_page.rb

@@ -9,6 +9,10 @@ module PageByPage
       Fetch.new(*args, &block).process
     end
 
+    def lazy_fetch(*args, &block)
+      Fetch.new(*args, &block).iterator
+    end
+
     def jump(*args, &block)
       Jump.new(*args, &block).process
     end

data/lib/page_by_page/fetch.rb

@@ -11,6 +11,7 @@ module PageByPage
     def initialize(opt = {}, &block)
       @from, @step, @to = 1, 1, Float::INFINITY
       super
+      @enum = (defined?(@threads) ? MutexEnum : Enum).new(enum_options)
     end
 
     def url tmpl
@@ -30,39 +31,53 @@ module PageByPage
     end
 
     def process
-      nodes_2d =
-        unless defined? @threads
-          @enum = Enum.new enum_options
-          _fetch
-        else
-          @enum = MutexEnum.new enum_options
-          parallel_fetch
-        end
+      nodes_2d = defined?(@threads) ? parallel_fetch : _fetch
       puts if @progress
       nodes_2d.reject(&:nil?).flatten
     end
 
+    def iterator
+      Enumerator.new do |yielder|
+        items_enum.each do |_, items|
+          items.each do |i|
+            yielder.yield(i)
+          end
+        end
+      end
+    end
+
     protected
 
     def _fetch
-      items, pages = [nil], []
-      catch :no_more do
-        until items.empty?
-          n = @enum.next
-          break if n > limit
-
-          url = @tmpl.result binding
-          doc = parse url
-          items = doc.css @selector
-          pages[n] = items
-
-          update_progress Thread.current, n if @progress
-          sleep @interval if @interval
-        end
+      pages = []
+
+      items_enum.each do |page_num, items|
+        pages[page_num] = items
       end
+
       pages
     end
 
+    def items_enum
+      Enumerator.new do |yielder|
+        items = [nil]
+        catch :no_more do
+          until items.empty?
+            n = @enum.next
+            break if n > limit
+
+            url = @tmpl.result binding
+            doc = parse url
+            items = doc.css @selector
+            yielder.yield(n, items)
+
+            update_progress Thread.current, n if @progress
+            sleep @interval if @interval
+          end
+        end
+      end
+    end
+
     def parallel_fetch
       ts = @threads.times.map do |n|
         Thread.new do

data/lib/page_by_page/version.rb

@@ -1,3 +1,3 @@
 module PageByPage
-  VERSION = "0.1.12"
+  VERSION = "0.1.13"
 end

data/page_by_page.gemspec

@@ -32,5 +32,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.13"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "minitest", "~> 5.0"
+  spec.add_development_dependency "pry"
   spec.add_dependency 'nokogiri', '~> 1.6'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: page_by_page
 version: !ruby/object:Gem::Version
-  version: 0.1.12
+  version: 0.1.13
 platform: ruby
 authors:
 - ken
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-03-02 00:00:00.000000000 Z
+date: 2019-11-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -110,8 +124,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: scrape page by page , according to url pattern