apollo-crawler 0.0.23 → 0.0.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/bin/apollo-crawler +13 -3
- data/lib/apollo_crawler/version.rb +1 -1
- metadata +1 -1
data/bin/apollo-crawler
CHANGED
@@ -17,6 +17,8 @@ require File.join(File.dirname(__FILE__), '..', 'lib', 'apollo_crawler', 'versio
|
|
17
17
|
|
18
18
|
module Crawler
|
19
19
|
class Program
|
20
|
+
@@PLUGIN_DIR = File.join(File.dirname(__FILE__), "..", "lib", "apollo_crawler", "plugins")
|
21
|
+
|
20
22
|
# This hash will hold all of the options
|
21
23
|
# parsed from the command-line by
|
22
24
|
# OptionParser.
|
@@ -34,6 +36,9 @@ module Crawler
|
|
34
36
|
@options = {}
|
35
37
|
@options[:verbose] = false
|
36
38
|
@options[:version] = false
|
39
|
+
@options[:plugin_dirs] = [
|
40
|
+
@@PLUGIN_DIR
|
41
|
+
]
|
37
42
|
|
38
43
|
@optparser = OptionParser.new do | opts |
|
39
44
|
# This displays the help screen, all programs are
|
@@ -47,6 +52,10 @@ module Crawler
|
|
47
52
|
@options[:run_all] = true
|
48
53
|
end
|
49
54
|
|
55
|
+
opts.on('-i', '--include [PATH]', 'Include additional plugins or plugin directories') do |path|
|
56
|
+
@options[:plugin_dirs] << path
|
57
|
+
end
|
58
|
+
|
50
59
|
opts.on('-v', '--verbose', 'Enable verbose output') do
|
51
60
|
@options[:verbose] = true
|
52
61
|
end
|
@@ -95,8 +104,7 @@ module Crawler
|
|
95
104
|
end
|
96
105
|
|
97
106
|
# Register plugins (specific crawlers)
|
98
|
-
def register_plugins()
|
99
|
-
dir = File.join(File.dirname(__FILE__), "..", "lib", "apollo_crawler", "plugins")
|
107
|
+
def register_plugins(dir)
|
100
108
|
if(@options[:verbose])
|
101
109
|
puts "Registering plugins - '#{dir}'"
|
102
110
|
end
|
@@ -141,7 +149,9 @@ module Crawler
|
|
141
149
|
load_config_file()
|
142
150
|
|
143
151
|
# Register sites which can be crawled
|
144
|
-
|
152
|
+
@options[:plugin_dirs].each do |dir|
|
153
|
+
register_plugins(dir)
|
154
|
+
end
|
145
155
|
|
146
156
|
if(@options[:list_plugins])
|
147
157
|
puts "Listing plugins"
|