apollo-crawler 0.0.23 → 0.0.25
Sign up to get free protection for your applications and to get access to all the features.
- data/bin/apollo-crawler +13 -3
- data/lib/apollo_crawler/version.rb +1 -1
- metadata +1 -1
data/bin/apollo-crawler
CHANGED
@@ -17,6 +17,8 @@ require File.join(File.dirname(__FILE__), '..', 'lib', 'apollo_crawler', 'versio
|
|
17
17
|
|
18
18
|
module Crawler
|
19
19
|
class Program
|
20
|
+
@@PLUGIN_DIR = File.join(File.dirname(__FILE__), "..", "lib", "apollo_crawler", "plugins")
|
21
|
+
|
20
22
|
# This hash will hold all of the options
|
21
23
|
# parsed from the command-line by
|
22
24
|
# OptionParser.
|
@@ -34,6 +36,9 @@ module Crawler
|
|
34
36
|
@options = {}
|
35
37
|
@options[:verbose] = false
|
36
38
|
@options[:version] = false
|
39
|
+
@options[:plugin_dirs] = [
|
40
|
+
@@PLUGIN_DIR
|
41
|
+
]
|
37
42
|
|
38
43
|
@optparser = OptionParser.new do | opts |
|
39
44
|
# This displays the help screen, all programs are
|
@@ -47,6 +52,10 @@ module Crawler
|
|
47
52
|
@options[:run_all] = true
|
48
53
|
end
|
49
54
|
|
55
|
+
opts.on('-i', '--include [PATH]', 'Include additional plugins or plugin directories') do |path|
|
56
|
+
@options[:plugin_dirs] << path
|
57
|
+
end
|
58
|
+
|
50
59
|
opts.on('-v', '--verbose', 'Enable verbose output') do
|
51
60
|
@options[:verbose] = true
|
52
61
|
end
|
@@ -95,8 +104,7 @@ module Crawler
|
|
95
104
|
end
|
96
105
|
|
97
106
|
# Register plugins (specific crawlers)
|
98
|
-
def register_plugins()
|
99
|
-
dir = File.join(File.dirname(__FILE__), "..", "lib", "apollo_crawler", "plugins")
|
107
|
+
def register_plugins(dir)
|
100
108
|
if(@options[:verbose])
|
101
109
|
puts "Registering plugins - '#{dir}'"
|
102
110
|
end
|
@@ -141,7 +149,9 @@ module Crawler
|
|
141
149
|
load_config_file()
|
142
150
|
|
143
151
|
# Register sites which can be crawled
|
144
|
-
|
152
|
+
@options[:plugin_dirs].each do |dir|
|
153
|
+
register_plugins(dir)
|
154
|
+
end
|
145
155
|
|
146
156
|
if(@options[:list_plugins])
|
147
157
|
puts "Listing plugins"
|