daimon_skycrawlers 0.11.1 → 0.11.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/daimon_skycrawlers/configurable.rb +13 -0
- data/lib/daimon_skycrawlers/crawler/base.rb +2 -0
- data/lib/daimon_skycrawlers/processor/base.rb +2 -0
- data/lib/daimon_skycrawlers/processor/spider.rb +0 -10
- data/lib/daimon_skycrawlers/version.rb +1 -1
- data/sample/amazon-ranking/app/processors/amazon_ranking.rb +4 -3
- metadata +3 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 962e472bc16cd081eb3207da6244b13d7d540e13
|
|
4
|
+
data.tar.gz: cd5285f349e8a63cc2bb4c41b03c771d71c295ba
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: de0eda81777909ec77454460532b7f7efab1916ed94dd3f696b3359ebd64bc48eda957b734c8f196925589caaa6d1ee64c1ca843b58fe1646e67b3f7578bf240
|
|
7
|
+
data.tar.gz: 3f5e52d6e1b128957d4b85dcb3644a2ae90574eb3dcce9717c4be3c9ae10e73fca04175c42de22a377a6a909941370be39936c33ee184a23ea7d0994aee174a4
|
|
@@ -4,6 +4,7 @@ require "faraday"
|
|
|
4
4
|
require "daimon_skycrawlers/logger"
|
|
5
5
|
require "daimon_skycrawlers/config"
|
|
6
6
|
require "daimon_skycrawlers/callbacks"
|
|
7
|
+
require "daimon_skycrawlers/configurable"
|
|
7
8
|
require "daimon_skycrawlers/storage"
|
|
8
9
|
require "daimon_skycrawlers/processor"
|
|
9
10
|
require "daimon_skycrawlers/filter/update_checker"
|
|
@@ -18,6 +19,7 @@ module DaimonSkycrawlers
|
|
|
18
19
|
include DaimonSkycrawlers::LoggerMixin
|
|
19
20
|
include DaimonSkycrawlers::ConfigMixin
|
|
20
21
|
include DaimonSkycrawlers::Callbacks
|
|
22
|
+
include DaimonSkycrawlers::Configurable
|
|
21
23
|
|
|
22
24
|
# @!attribute [w] storage
|
|
23
25
|
# Set storage to crawler instance.
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
require "daimon_skycrawlers/logger"
|
|
2
2
|
require "daimon_skycrawlers/config"
|
|
3
3
|
require "daimon_skycrawlers/callbacks"
|
|
4
|
+
require "daimon_skycrawlers/configurable"
|
|
4
5
|
|
|
5
6
|
module DaimonSkycrawlers
|
|
6
7
|
module Processor
|
|
@@ -8,6 +9,7 @@ module DaimonSkycrawlers
|
|
|
8
9
|
include DaimonSkycrawlers::LoggerMixin
|
|
9
10
|
include DaimonSkycrawlers::ConfigMixin
|
|
10
11
|
include DaimonSkycrawlers::Callbacks
|
|
12
|
+
include DaimonSkycrawlers::Configurable
|
|
11
13
|
|
|
12
14
|
def process(message)
|
|
13
15
|
proceeding = run_before_callbacks(message)
|
|
@@ -56,16 +56,6 @@ module DaimonSkycrawlers
|
|
|
56
56
|
@next_page_link_message = {}
|
|
57
57
|
end
|
|
58
58
|
|
|
59
|
-
#
|
|
60
|
-
# Configure spider instance
|
|
61
|
-
#
|
|
62
|
-
# @return [DaimonSkycrawlers::Processor::Spider] self
|
|
63
|
-
#
|
|
64
|
-
def configure
|
|
65
|
-
yield self
|
|
66
|
-
self
|
|
67
|
-
end
|
|
68
|
-
|
|
69
59
|
#
|
|
70
60
|
# Append filter to reduce links found by link_rules
|
|
71
61
|
#
|
|
@@ -30,8 +30,9 @@ spider.configure do |s|
|
|
|
30
30
|
end
|
|
31
31
|
DaimonSkycrawlers.register_processor(spider)
|
|
32
32
|
|
|
33
|
-
processor = AmazonRanking.new
|
|
34
|
-
|
|
35
|
-
|
|
33
|
+
processor = AmazonRanking.new.configure do |s|
|
|
34
|
+
s.before_process do |message|
|
|
35
|
+
message[:next_processor] == "AmazonRanking"
|
|
36
|
+
end
|
|
36
37
|
end
|
|
37
38
|
DaimonSkycrawlers.register_processor(processor)
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: daimon_skycrawlers
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.11.
|
|
4
|
+
version: 0.11.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- daimon developers
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2016-12-
|
|
11
|
+
date: 2016-12-08 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: thor
|
|
@@ -326,6 +326,7 @@ files:
|
|
|
326
326
|
- lib/daimon_skycrawlers/commands/enqueue.rb
|
|
327
327
|
- lib/daimon_skycrawlers/commands/runner.rb
|
|
328
328
|
- lib/daimon_skycrawlers/config.rb
|
|
329
|
+
- lib/daimon_skycrawlers/configurable.rb
|
|
329
330
|
- lib/daimon_skycrawlers/consumer.rb
|
|
330
331
|
- lib/daimon_skycrawlers/consumer/base.rb
|
|
331
332
|
- lib/daimon_skycrawlers/consumer/http_response.rb
|