apollo-crawler 0.1.21 → 0.1.22

Sign up to get free protection for your applications and to get access to all the features.
Files changed (55) hide show
  1. checksums.yaml +8 -8
  2. data/bin/apollo-crawler +0 -4
  3. data/bin/apollo-platform +30 -0
  4. data/config/amqp.yml +18 -0
  5. data/config/amqp.yml.default +18 -0
  6. data/config/apollo.yml +13 -0
  7. data/config/apollo.yml.default +10 -0
  8. data/config/memcached.yml +14 -0
  9. data/config/memcached.yml.default +14 -0
  10. data/config/mongo.yml +19 -0
  11. data/config/mongo.yml.default +19 -0
  12. data/config/mongoid.yml +23 -0
  13. data/config/mongoid.yml.default +59 -0
  14. data/lib/apollo_crawler.rb +12 -3
  15. data/lib/apollo_crawler/adapter/adapters.rb +22 -0
  16. data/lib/apollo_crawler/adapter/amqp_adapter.rb +26 -0
  17. data/lib/apollo_crawler/adapter/mongo_adapter.rb +26 -0
  18. data/lib/apollo_crawler/{cli/cli.rb → agent/agents.rb} +2 -0
  19. data/lib/apollo_crawler/agent/base_agent.rb +26 -0
  20. data/lib/apollo_crawler/cache/caches.rb +20 -0
  21. data/lib/apollo_crawler/cache/sqlite_cache.rb +9 -0
  22. data/lib/apollo_crawler/config.rb +82 -72
  23. data/lib/apollo_crawler/crawler/crawlers.rb +20 -0
  24. data/lib/apollo_crawler/crawler/google_crawler.rb +2 -2
  25. data/lib/apollo_crawler/crawler/hacker_news_crawler.rb +2 -2
  26. data/lib/apollo_crawler/crawler/slashdot_crawler.rb +2 -2
  27. data/lib/apollo_crawler/crawler/stackoverflow_crawler.rb +2 -2
  28. data/lib/apollo_crawler/crawler/xkcd_crawler.rb +2 -2
  29. data/lib/apollo_crawler/crawler/youjizz_crawler.rb +2 -2
  30. data/lib/apollo_crawler/env.rb +24 -0
  31. data/lib/apollo_crawler/fetcher/base_fetcher.rb +1 -1
  32. data/lib/apollo_crawler/fetcher/fetchers.rb +20 -0
  33. data/lib/apollo_crawler/fetcher/simple_fetcher.rb +1 -1
  34. data/lib/apollo_crawler/fetcher/smart_fetcher.rb +1 -1
  35. data/lib/apollo_crawler/formatter/formatters.rb +20 -0
  36. data/lib/apollo_crawler/formatter/json_formatter.rb +5 -1
  37. data/lib/apollo_crawler/formatter/plain_formatter.rb +4 -0
  38. data/lib/apollo_crawler/formatter/table_formatter.rb +4 -0
  39. data/lib/apollo_crawler/helper/amqp_helper.rb +26 -0
  40. data/lib/apollo_crawler/helper/core_helper.rb +24 -4
  41. data/lib/apollo_crawler/helper/helpers.rb +23 -1
  42. data/lib/apollo_crawler/helper/mongo_helper.rb +26 -0
  43. data/lib/apollo_crawler/lib.rb +12 -3
  44. data/lib/apollo_crawler/logger/loggers.rb +20 -0
  45. data/lib/apollo_crawler/planner/base_planner.rb +26 -0
  46. data/lib/apollo_crawler/planner/planners.rb +22 -0
  47. data/lib/apollo_crawler/planner/smart_planner.rb +28 -0
  48. data/lib/apollo_crawler/program/base_program.rb +130 -0
  49. data/lib/apollo_crawler/program/console_program.rb +177 -0
  50. data/lib/apollo_crawler/program/crawler_program.rb +130 -183
  51. data/lib/apollo_crawler/program/platform_program.rb +137 -0
  52. data/lib/apollo_crawler/program/programs.rb +23 -1
  53. data/lib/apollo_crawler/store/stores.rb +20 -0
  54. data/lib/apollo_crawler/version.rb +2 -2
  55. metadata +55 -3
@@ -25,6 +25,10 @@ require File.join(File.dirname(__FILE__), 'base_formatter')
25
25
  module Apollo
26
26
  module Formatter
27
27
  class PlainFormatter < BaseFormatter
28
+ def name()
29
+ return "Plain"
30
+ end
31
+
28
32
  def format(obj)
29
33
  return Plain.format(obj)
30
34
  end
@@ -25,6 +25,10 @@ require File.join(File.dirname(__FILE__), 'base_formatter')
25
25
  module Apollo
26
26
  module Formatter
27
27
  class TableFormatter < BaseFormatter
28
+ def name()
29
+ return "Table"
30
+ end
31
+
28
32
  def format(obj)
29
33
  return Table.format(obj)
30
34
  end
@@ -0,0 +1,26 @@
1
+ # Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
2
+ #
3
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
4
+ # of this software and associated documentation files (the "Software"), to deal
5
+ # in the Software without restriction, including without limitation the rights
6
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7
+ # copies of the Software, and to permit persons to whom the Software is
8
+ # furnished to do so, subject to the following conditions:
9
+ #
10
+ # The above copyright notice and this permission notice shall be included in
11
+ # all copies or substantial portions of the Software.
12
+ #
13
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19
+ # THE SOFTWARE.
20
+
21
+ module Apollo
22
+ module Helper
23
+ module Amqp
24
+ end # Amqp
25
+ end # module Helper
26
+ end # module Apollo
@@ -18,12 +18,32 @@
18
18
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19
19
  # THE SOFTWARE.
20
20
 
21
+ class Class
22
+ def subclasses(base=self)
23
+ res = ObjectSpace.each_object(Class).select { |klass| klass < base }
24
+ end
25
+ end
26
+
21
27
  module Apollo
22
28
  module Helper
23
- class CoreHelper
24
- def self.name_re()
25
- return /formatter$/
29
+ module Core
30
+
31
+ # Gets a subclasses of specified class with it's hierarchy
32
+ def self.get_classes(base=nil)
33
+ res = []
34
+
35
+ if(base != nil)
36
+ res = base.subclasses
37
+ end
38
+
39
+ res.each do |tmp|
40
+ # puts tmp.inspect
41
+ s = self.get_classes(tmp)
42
+ res << s unless s.empty?
43
+ end
44
+
45
+ return res.flatten
26
46
  end
27
- end # class CoreHelper
47
+ end # Core
28
48
  end # module Helper
29
49
  end # module Apollo
@@ -1 +1,23 @@
1
- require File.join(File.dirname(__FILE__), 'core_helper')
1
+ # Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
2
+ #
3
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
4
+ # of this software and associated documentation files (the "Software"), to deal
5
+ # in the Software without restriction, including without limitation the rights
6
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7
+ # copies of the Software, and to permit persons to whom the Software is
8
+ # furnished to do so, subject to the following conditions:
9
+ #
10
+ # The above copyright notice and this permission notice shall be included in
11
+ # all copies or substantial portions of the Software.
12
+ #
13
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19
+ # THE SOFTWARE.
20
+
21
+ require File.join(File.dirname(__FILE__), 'amqp_helper')
22
+ require File.join(File.dirname(__FILE__), 'core_helper')
23
+ require File.join(File.dirname(__FILE__), 'mongo_helper')
@@ -0,0 +1,26 @@
1
+ # Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
2
+ #
3
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
4
+ # of this software and associated documentation files (the "Software"), to deal
5
+ # in the Software without restriction, including without limitation the rights
6
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7
+ # copies of the Software, and to permit persons to whom the Software is
8
+ # furnished to do so, subject to the following conditions:
9
+ #
10
+ # The above copyright notice and this permission notice shall be included in
11
+ # all copies or substantial portions of the Software.
12
+ #
13
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19
+ # THE SOFTWARE.
20
+
21
+ module Apollo
22
+ module Helper
23
+ module Mongo
24
+ end # Mongo
25
+ end # module Helper
26
+ end # module Apollo
@@ -18,12 +18,18 @@
18
18
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19
19
  # THE SOFTWARE.
20
20
 
21
+ # Environment first
22
+ require File.join(File.dirname(__FILE__), 'env')
23
+
24
+ # Adapters
25
+ require File.join(File.dirname(__FILE__), 'adapter/adapters')
26
+
27
+ # Agents
28
+ require File.join(File.dirname(__FILE__), 'agent/agents')
29
+
21
30
  # Caches
22
31
  require File.join(File.dirname(__FILE__), 'cache/caches')
23
32
 
24
- # CLI - Command Line Interface
25
- require File.join(File.dirname(__FILE__), 'cli/cli')
26
-
27
33
  # Crawlers
28
34
  require File.join(File.dirname(__FILE__), 'crawler/crawlers')
29
35
 
@@ -39,6 +45,9 @@ require File.join(File.dirname(__FILE__), 'helper/helpers')
39
45
  # Loggers
40
46
  require File.join(File.dirname(__FILE__), 'logger/loggers')
41
47
 
48
+ # Programs
49
+ require File.join(File.dirname(__FILE__), 'planner/planners')
50
+
42
51
  # Programs
43
52
  require File.join(File.dirname(__FILE__), 'program/programs')
44
53
 
@@ -1,2 +1,22 @@
1
+ # Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
2
+ #
3
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
4
+ # of this software and associated documentation files (the "Software"), to deal
5
+ # in the Software without restriction, including without limitation the rights
6
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7
+ # copies of the Software, and to permit persons to whom the Software is
8
+ # furnished to do so, subject to the following conditions:
9
+ #
10
+ # The above copyright notice and this permission notice shall be included in
11
+ # all copies or substantial portions of the Software.
12
+ #
13
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19
+ # THE SOFTWARE.
20
+
1
21
  require File.join(File.dirname(__FILE__), 'base_logger')
2
22
  require File.join(File.dirname(__FILE__), 'console_logger')
@@ -0,0 +1,26 @@
1
+ # Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
2
+ #
3
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
4
+ # of this software and associated documentation files (the "Software"), to deal
5
+ # in the Software without restriction, including without limitation the rights
6
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7
+ # copies of the Software, and to permit persons to whom the Software is
8
+ # furnished to do so, subject to the following conditions:
9
+ #
10
+ # The above copyright notice and this permission notice shall be included in
11
+ # all copies or substantial portions of the Software.
12
+ #
13
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19
+ # THE SOFTWARE.
20
+
21
+ module Apollo
22
+ module Planner
23
+ class BasePlanner
24
+ end # class BasePlanner
25
+ end # module Planner
26
+ end # module Apollo
@@ -0,0 +1,22 @@
1
+ # Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
2
+ #
3
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
4
+ # of this software and associated documentation files (the "Software"), to deal
5
+ # in the Software without restriction, including without limitation the rights
6
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7
+ # copies of the Software, and to permit persons to whom the Software is
8
+ # furnished to do so, subject to the following conditions:
9
+ #
10
+ # The above copyright notice and this permission notice shall be included in
11
+ # all copies or substantial portions of the Software.
12
+ #
13
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19
+ # THE SOFTWARE.
20
+
21
+ require File.join(File.dirname(__FILE__), 'base_planner')
22
+ require File.join(File.dirname(__FILE__), 'smart_planner')
@@ -0,0 +1,28 @@
1
+ # Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
2
+ #
3
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
4
+ # of this software and associated documentation files (the "Software"), to deal
5
+ # in the Software without restriction, including without limitation the rights
6
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7
+ # copies of the Software, and to permit persons to whom the Software is
8
+ # furnished to do so, subject to the following conditions:
9
+ #
10
+ # The above copyright notice and this permission notice shall be included in
11
+ # all copies or substantial portions of the Software.
12
+ #
13
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19
+ # THE SOFTWARE.
20
+
21
+ require File.join(File.dirname(__FILE__),'base_planner')
22
+
23
+ module Apollo
24
+ module Planner
25
+ class SmartPlanner < BasePlanner
26
+ end # class SmartPlanner
27
+ end # module Planner
28
+ end # module Apollo
@@ -18,12 +18,142 @@
18
18
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19
19
  # THE SOFTWARE.
20
20
 
21
+ require 'yaml'
22
+
21
23
  module Apollo
22
24
  class BaseProgram
25
+ CONFIG_DIR = File.join(Apollo::BASE_DIR, "config")
26
+
27
+ DEFAULT_OPTIONS = {
28
+ :env => Apollo::ENV,
29
+ :verbose => false
30
+ }
31
+
32
+ attr_accessor :config
33
+ attr_accessor :options
34
+ attr_accessor :optparser
35
+
36
+ attr_accessor :mongo
37
+ attr_accessor :mongo_db
38
+
39
+ def initialize
40
+ self.config = {}
41
+ self.options = DEFAULT_OPTIONS
42
+ self.optparser = nil
43
+
44
+ self.mongo = nil
45
+ self.mongo_db = nil
46
+ end
47
+
48
+ def self.get_config_path(config)
49
+ return File.join(CONFIG_DIR, "#{config}.yml")
50
+ end
51
+
52
+ def load_configs(dir = CONFIG_DIR, options = DEFAULT_OPTIONS)
53
+ Dir[File.join(dir, "**/*.yml")].each do |c|
54
+ puts "Loading config #{c}" if options[:verbose]
55
+ self.load_config_file(c, options)
56
+ end
57
+
58
+ return self.config
59
+ end
60
+
61
+ def load_config_file(config_file, options = DEFAULT_OPTIONS)
62
+ return nil unless File.exists?(config_file)
63
+
64
+ config_name = File.basename(config_file.downcase, ".yml")
65
+
66
+ res = YAML.load(File.read(config_file))
67
+ return nil unless res
68
+
69
+ res = {config_name => res[options[:env]]}
70
+ self.config.merge! res
71
+
72
+ if(options[:verbose])
73
+ puts "Loaded config '#{config_file}' => #{res[config_name]}"
74
+ end
75
+
76
+ return res[config_name]
77
+ end
78
+
79
+ def load_config(config_name, options = DEFAULT_OPTIONS)
80
+ path = BaseProgram.get_config_path(config_name)
81
+ return load_config_file(path, options)
82
+ end
83
+
23
84
  def self.require_files(files = [])
24
85
  Dir.glob(files).each do |file|
25
86
  require file
26
87
  end
27
88
  end
89
+
90
+ # Parse the options passed to command-line
91
+ def parse_options(args = ARGV)
92
+ res = []
93
+
94
+ # Parse the command-line. Remember there are two forms
95
+ # of the parse method. The 'parse' method simply parses
96
+ # ARGV, while the 'parse!' method parses ARGV and removes
97
+ # any options found there, as well as any parameters for
98
+ # the options. What's left is the list of files to resize.
99
+ begin
100
+ optparser.parse!(args)
101
+ rescue Exception => e
102
+ return nil
103
+ end
104
+
105
+ return res
106
+ end
107
+
108
+ def process_options(args)
109
+ return nil
110
+ end
111
+
112
+ def init_options()
113
+ return nil
114
+ end
115
+
116
+ # Init program
117
+ def init_program(args)
118
+ res = nil
119
+ res = init_options()
120
+
121
+ begin
122
+ res = parse_options(args)
123
+ rescue Exception => e
124
+ puts "Unable to parse options"
125
+ return res unless res.nil?
126
+ end
127
+
128
+ res = process_options(args)
129
+ return res unless res.nil?
130
+
131
+ confs = load_configs(BaseProgram::CONFIG_DIR, @options)
132
+ puts "Loaded configs => #{confs.inspect}" if @options[:verbose]
133
+
134
+ # Init Mongo Connection
135
+ init_mongo()
136
+
137
+ return nil
138
+ end
139
+
140
+ # Run Program
141
+ def run(args = ARGV)
142
+ res = init_program(args)
143
+ return res unless res.nil?
144
+
145
+ puts "Running environment '#{@options[:env]}'" if @options[:verbose]
146
+ end
147
+
148
+ def request_exit(code = 0)
149
+ puts "Exiting app"
150
+ begin
151
+ exit(code)
152
+ rescue SystemExit => e
153
+ # puts "rescued a SystemExit exception, reason: '#{e.to_s}'"
154
+ end
155
+
156
+ return code
157
+ end
28
158
  end # class BaseProgram
29
159
  end # module Apollo
@@ -0,0 +1,177 @@
1
+ # Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
2
+ #
3
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
4
+ # of this software and associated documentation files (the "Software"), to deal
5
+ # in the Software without restriction, including without limitation the rights
6
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7
+ # copies of the Software, and to permit persons to whom the Software is
8
+ # furnished to do so, subject to the following conditions:
9
+ #
10
+ # The above copyright notice and this permission notice shall be included in
11
+ # all copies or substantial portions of the Software.
12
+ #
13
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19
+ # THE SOFTWARE.
20
+
21
+ require 'json'
22
+
23
+ require "thor"
24
+
25
+ require "open-uri"
26
+ require "nokogiri"
27
+
28
+ require "pp"
29
+ require "optparse"
30
+
31
+ require 'active_support'
32
+ require 'active_support/inflector'
33
+
34
+ require 'terminal-table'
35
+
36
+ require 'eventmachine'
37
+ require 'em-http'
38
+
39
+ require 'fileutils'
40
+
41
+ require File.join(File.dirname(__FILE__), '..', 'version')
42
+
43
+ require File.join(File.dirname(__FILE__), 'base_program')
44
+
45
+ module Apollo
46
+ # Apollo Crawler Base Directory
47
+ APOLLO_CONSOLE_BASE_DIR = File.join(File.dirname(__FILE__), "..")
48
+
49
+ class ConsoleProgram < BaseProgram
50
+ # Load default config
51
+ require File.join(File.dirname(__FILE__), "..", "config")
52
+
53
+ # This hash will hold all of the options
54
+ # parsed from the command-line by OptionParser.
55
+ @options = nil
56
+ @optparser = nil
57
+
58
+ # Initializer - Constructor
59
+ def initialize
60
+ super
61
+
62
+ @options = {}
63
+ end
64
+
65
+ # Initialize command-line options
66
+ def init_options()
67
+ @options[:env] = Apollo::ENV
68
+
69
+ @options[:verbose] = false
70
+ @options[:version] = nil
71
+ end
72
+
73
+ def init_options_parser()
74
+ @optparser = OptionParser.new do | opts |
75
+ opts.banner = "Usage: apollo-console [OPTIONS]"
76
+
77
+ opts.separator ""
78
+ opts.separator "Specific options:"
79
+
80
+ # This displays the help screen, all programs are
81
+ # assumed to have this option.
82
+ opts.on('-h', '--help', 'Display this screen') do
83
+ @options[:show_help] = true
84
+ end
85
+
86
+ opts.on('-e', '--environment [NAME]', "Environment used, default '#{@options[:env]}'") do |name|
87
+ @options[:env] = name
88
+ end
89
+
90
+ opts.on('-v', '--verbose', 'Enable verbose output') do
91
+ @options[:verbose] = true
92
+ end
93
+
94
+ opts.on('-V', '--version', 'Show version info') do
95
+ @options[:version] = true
96
+ end
97
+
98
+ opts.on('-s', '--silent', 'Silent mode - do not print processed document') do
99
+ @options[:silent] = true
100
+ end
101
+ end
102
+ end
103
+
104
+ # Parse the options passed to command-line
105
+ def parse_options(args = ARGV)
106
+ # Parse the command-line. Remember there are two forms
107
+ # of the parse method. The 'parse' method simply parses
108
+ # ARGV, while the 'parse!' method parses ARGV and removes
109
+ # any options found there, as well as any parameters for
110
+ # the options. What's left is the list of files to resize.
111
+ @optparser.parse!(args)
112
+ end
113
+
114
+ def process_options(args)
115
+ if(@options[:version])
116
+ puts Apollo::VERSION
117
+ return 0
118
+ end
119
+
120
+ if(@options[:show_help])
121
+ puts @optparser
122
+ return 0
123
+ end
124
+
125
+ return nil
126
+ end
127
+
128
+
129
+ # Init program
130
+ def init_program(args)
131
+ init_options()
132
+ init_options_parser()
133
+
134
+
135
+ parse_options(args)
136
+
137
+ res = process_options(args)
138
+ if res != nil
139
+ return res
140
+ end
141
+
142
+ return nil
143
+ end
144
+
145
+ # Run Program
146
+ def run(args = ARGV)
147
+ res_code = init_program(args)
148
+
149
+ if res_code.nil? == false
150
+ return request_exit(res_code)
151
+ end
152
+
153
+ if(@options[:verbose])
154
+ puts "Running environment '#{@options[:env]}'"
155
+ end
156
+
157
+ # if(ARGV.length < 1)
158
+ # puts @optparser
159
+ # return 0
160
+ # end
161
+
162
+ # Here we start
163
+
164
+ return request_exit(res_code)
165
+ end
166
+
167
+ def request_exit(code = 0)
168
+ begin
169
+ exit(0)
170
+ rescue SystemExit => e
171
+ # puts "rescued a SystemExit exception, reason: '#{e.to_s}'"
172
+ end
173
+
174
+ return code
175
+ end
176
+ end
177
+ end