digger 0.0.2 → 0.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 294316d0968a1ae668be091b960574af4553dd61
4
- data.tar.gz: 8fa02db80d4d6bf818f64381f60fd5010ae8c4a3
3
+ metadata.gz: 204be25e4d5bb59529fc7f5a392b86bf6f7ed631
4
+ data.tar.gz: 21119d3c972aa60d2f6c95dddfd66c4b1ab4cfa0
5
5
  SHA512:
6
- metadata.gz: 8c322b4404485b79b0459e518191504a53aec1ed1bc2497f0aa746ada619eed474301e04d56aef225791188c84a6664049c63254e860ecfddcb6a8d436dd43ae
7
- data.tar.gz: 2d7a631ee5080790fd365e6b25beb5f8c6ac08a9a073868ebcaba3cc96327629a2e05d617ae735176e91e6f91bf462b6425c9006c519323054c56b279b79956b
6
+ metadata.gz: 3f1dcd8583e6038c4237308b8a7488336f8d3ecb3b8a87c745622ee95719f964e832fe6cfba5a515c1cfad61ae02382875e3c93b19156e2e391b8eee15832c9a
7
+ data.tar.gz: ab18eff67be45f8d9ed3b43fcefe14a92a8f7089cd30c366b6e1f3ab8c1cd78f7727b63bfa318ea828e2ae0d9a8386336e761bb3417815cf24492f8609d2e328
@@ -0,0 +1,40 @@
1
+ module Digger
2
+ class Index < Struct.new(:pattern, :args)
3
+ class NoBlockError < ArgumentError; end
4
+
5
+ def process(cocurrence = 1, &block)
6
+ Index.batch(urls, cocurrence, block)
7
+ end
8
+
9
+ def urls
10
+ @urls ||= begin
11
+ args = self.args.map{|a| (a.respond_to? :each) ? a.to_a : [a]}
12
+ args.shift.product(*args).map{|arg| pattern_applied_url(arg)}
13
+ end
14
+ end
15
+
16
+ def pattern_applied_url(arg)
17
+ pattern.gsub('*').each_with_index{|_, i| arg[i]}
18
+ end
19
+
20
+ def self.batch(entities, cocurrence = 1, &block)
21
+ raise NoBlockError, "No block given" unless block
22
+
23
+ if cocurrence > 1
24
+ results = {}
25
+ entities.each_slice(cocurrence) do |group|
26
+ threads = []
27
+ group.each do |entity|
28
+ threads << Thread.new(entity) do |ent|
29
+ results[ent] = block.call(ent)
30
+ end
31
+ end
32
+ threads.each{|thread| thread.join}
33
+ end
34
+ entities.map{|ent| results[ent]}
35
+ else
36
+ entities.map{|ent| block.call(ent) }
37
+ end
38
+ end
39
+ end
40
+ end
data/lib/digger/model.rb CHANGED
@@ -2,11 +2,12 @@ require 'digger/pattern'
2
2
 
3
3
  module Digger
4
4
  class Model
5
- @@patterns = {}
5
+ @@digger_config = {'pattern'=>{}, 'index'=>{}}
6
6
 
7
7
  class << self
8
+ # patterns
8
9
  def pattern_config
9
- @@patterns[self.name] ||= {}
10
+ @@digger_config['pattern'][self.name] ||= {}
10
11
  end
11
12
 
12
13
  Pattern::TYPES.each do |method|
@@ -17,10 +18,17 @@ module Digger
17
18
  }
18
19
  end
19
20
 
20
- def index_page
21
+ # index page
22
+ def index_config
23
+ @@digger_config['index'][self.name]
21
24
  end
22
25
 
23
- def one_page
26
+ def index_page(pattern, *args)
27
+ @@digger_config['index'][self.name] = Index.new(pattern, args)
28
+ end
29
+
30
+ def index_page?
31
+ !index_config.nil?
24
32
  end
25
33
  end
26
34
 
@@ -32,10 +40,20 @@ module Digger
32
40
  result
33
41
  end
34
42
 
35
- def dig(url)
43
+ def dig_url(url)
36
44
  client = Digger::HTTP.new
37
45
  page = client.fetch_page(url)
38
46
  match_page(page)
39
47
  end
48
+
49
+ def dig(urls = [], cocurrence = 1)
50
+ if urls.empty?
51
+ if self.class.index_page?
52
+ self.class.index_config.process(cocurrence){|url| dig_url(url) }
53
+ end
54
+ else
55
+ Index.batch(urls, cocurrence){|url| dig_url(url) }
56
+ end
57
+ end
40
58
  end
41
59
  end
@@ -1,3 +1,3 @@
1
1
  module Digger
2
- VERSION = "0.0.2"
2
+ VERSION = "0.0.3"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: digger
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.2
4
+ version: 0.0.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - binz
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-03-27 00:00:00.000000000 Z
11
+ date: 2015-03-28 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -80,8 +80,8 @@ files:
80
80
  - Rakefile
81
81
  - digger.gemspec
82
82
  - lib/digger.rb
83
- - lib/digger/batch.rb
84
83
  - lib/digger/http.rb
84
+ - lib/digger/index.rb
85
85
  - lib/digger/model.rb
86
86
  - lib/digger/page.rb
87
87
  - lib/digger/pattern.rb
data/lib/digger/batch.rb DELETED
@@ -1,17 +0,0 @@
1
- module Digger
2
- module Batch
3
- class NoBlockError < ArgumentError; end
4
-
5
- def self.do(cocurrence, entities, &block)
6
- raise NoBlockError, "No block given" unless block
7
- entities.each_slice(cocurrence) do |group|
8
- threads = []
9
- group.each do |entity|
10
- threads << Thread.new(entity) do |ent|
11
- block.call(ent)
12
- end
13
- end
14
- threads.each{|thread| thread.join}
15
- end
16
- end
17
- end