digger 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 294316d0968a1ae668be091b960574af4553dd61
-  data.tar.gz: 8fa02db80d4d6bf818f64381f60fd5010ae8c4a3
+  metadata.gz: 204be25e4d5bb59529fc7f5a392b86bf6f7ed631
+  data.tar.gz: 21119d3c972aa60d2f6c95dddfd66c4b1ab4cfa0
 SHA512:
-  metadata.gz: 8c322b4404485b79b0459e518191504a53aec1ed1bc2497f0aa746ada619eed474301e04d56aef225791188c84a6664049c63254e860ecfddcb6a8d436dd43ae
-  data.tar.gz: 2d7a631ee5080790fd365e6b25beb5f8c6ac08a9a073868ebcaba3cc96327629a2e05d617ae735176e91e6f91bf462b6425c9006c519323054c56b279b79956b
+  metadata.gz: 3f1dcd8583e6038c4237308b8a7488336f8d3ecb3b8a87c745622ee95719f964e832fe6cfba5a515c1cfad61ae02382875e3c93b19156e2e391b8eee15832c9a
+  data.tar.gz: ab18eff67be45f8d9ed3b43fcefe14a92a8f7089cd30c366b6e1f3ab8c1cd78f7727b63bfa318ea828e2ae0d9a8386336e761bb3417815cf24492f8609d2e328

data/lib/digger/index.rb

@@ -0,0 +1,40 @@
+module Digger
+  class Index < Struct.new(:pattern, :args)
+    class NoBlockError < ArgumentError; end
+
+    def process(cocurrence = 1, &block)
+      Index.batch(urls, cocurrence, block)
+    end
+
+    def urls
+      @urls ||= begin
+        args = self.args.map{|a| (a.respond_to? :each) ? a.to_a : [a]}
+        args.shift.product(*args).map{|arg| pattern_applied_url(arg)}
+      end
+    end
+
+    def pattern_applied_url(arg)
+      pattern.gsub('*').each_with_index{|_, i| arg[i]}
+    end
+
+    def self.batch(entities, cocurrence = 1, &block)
+      raise NoBlockError, "No block given" unless block
+
+      if cocurrence > 1
+        results = {}
+        entities.each_slice(cocurrence) do |group|
+          threads = []
+          group.each do |entity|
+            threads << Thread.new(entity) do |ent|
+              results[ent] = block.call(ent)
+            end
+          end
+          threads.each{|thread| thread.join}
+        end
+        entities.map{|ent| results[ent]}
+      else
+        entities.map{|ent| block.call(ent) }
+      end
+    end
+  end
+end

data/lib/digger/model.rb

@@ -2,11 +2,12 @@ require 'digger/pattern'
 
 module Digger
   class Model
-    @@patterns = {}
+    @@digger_config = {'pattern'=>{}, 'index'=>{}}
 
     class << self
+      # patterns
       def pattern_config
-        @@patterns[self.name] ||= {}
+        @@digger_config['pattern'][self.name] ||= {}
       end
 
       Pattern::TYPES.each do |method|
@@ -17,10 +18,17 @@ module Digger
         }
       end
 
-      def index_page
+      # index page
+      def index_config
+        @@digger_config['index'][self.name]
       end
 
-      def one_page
+      def index_page(pattern, *args)
+         @@digger_config['index'][self.name] = Index.new(pattern, args)
+      end
+
+      def index_page?
+        !index_config.nil?
       end
     end
 
@@ -32,10 +40,20 @@ module Digger
       result
     end
 
-    def dig(url)
+    def dig_url(url)
       client = Digger::HTTP.new
       page = client.fetch_page(url)
       match_page(page)
     end
+
+    def dig(urls = [], cocurrence = 1)
+      if urls.empty?
+        if self.class.index_page?
+          self.class.index_config.process(cocurrence){|url| dig_url(url) }
+        end
+      else
+        Index.batch(urls, cocurrence){|url| dig_url(url) }
+      end
+    end
   end
 end

data/lib/digger/version.rb

@@ -1,3 +1,3 @@
 module Digger
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: digger
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - binz
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-27 00:00:00.000000000 Z
+date: 2015-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -80,8 +80,8 @@ files:
 - Rakefile
 - digger.gemspec
 - lib/digger.rb
-- lib/digger/batch.rb
 - lib/digger/http.rb
+- lib/digger/index.rb
 - lib/digger/model.rb
 - lib/digger/page.rb
 - lib/digger/pattern.rb

data/lib/digger/batch.rb

@@ -1,17 +0,0 @@
-module Digger
-  module Batch
-    class NoBlockError < ArgumentError; end
-    
-    def self.do(cocurrence, entities, &block)
-      raise NoBlockError, "No block given" unless block
-      entities.each_slice(cocurrence) do |group|
-        threads = []
-        group.each do |entity|
-          threads << Thread.new(entity) do |ent|
-            block.call(ent)
-          end
-        end
-        threads.each{|thread| thread.join}
-      end
-  end
-end