apollo-crawler 0.1.22 → 0.1.24

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    MjNkN2YzMjExMDc4Mjg5ZDNhNGMxYTY0MTY1NjczMDQ0MzE4YmFlOQ==
-  data.tar.gz: !binary |-
-    ZWVlM2UyZjhhZDM4Y2ExZGExNzkzNjNjNjNkNTI4OTRkOTEzY2Y5NA==
-!binary "U0hBNTEy":
-  metadata.gz: !binary |-
-    ZDc0YmI0MjEyNTgzNTM0NmY2NGE2YTY0Y2MxNTdiNTBkN2M3ZDM2MDdmNjZm
-    MjNhZDVlNmRkNDdkMDVhNzhjNzg0ZDQ2ZjRkYThhNzJlODMxNjE3NmE4MjVm
-    ZDg0ZjFlZWFjM2I0NWQ0ZmIzYmM5ZmY0MTRiOGE2YTMwZDVmYzE=
-  data.tar.gz: !binary |-
-    N2I5ZGM1NjI2M2QxN2FmMDZkMThkOGU5NDE5MTEyOTNlZWFkNGQ1N2FlZjFi
-    MTI5ZGNhNjdmZTAyZjAyYTVkZWFlNGJmZDk5YzA3ZjlhN2Q5MTc1NTIyNGVi
-    M2VjODgxNzQyYTAxNzQ5NWQ5MTQzZjUxNWY5MWZlNDQzNjg2YmQ=
+SHA1:
+  metadata.gz: 71fb379b6ae32ceb79e40cce451c8a3646278d32
+  data.tar.gz: 08fdec629298945a86993b91be3a743b880ad4a0
+SHA512:
+  metadata.gz: d0789d2ef99358144c90d148c378d9bf53e3084b326ede425ed7ff171ab450a4ed9de7a86494dbf49992ed235807f92e6a795cc52676bd61280a006408fc4e90
+  data.tar.gz: 16ac99fc7e192fe137348c6d364e730c9c0071f274f200b395746c4247bb3958c7238b315b571a730014b5f8286602bbc67eb9f26db30c217440e15401d3ac95

data/bin/apollo-console

@@ -0,0 +1,30 @@
+#! /usr/bin/env ruby
+# encoding: utf-8
+
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require "rubygems"
+require "bundler/setup"
+
+require File.join(File.dirname(__FILE__), "..", "lib", "apollo_crawler")
+
+Apollo::ConsoleProgram.new.run(ARGV)
+

data/config/amqp.yml

@@ -2,17 +2,18 @@ default: &default
   host: localhost
   username: guest
   password: guest
+  port: 5672
 
 development:
   <<: *default
-  vhost: apollo-crawler-development
+  vhost: /apollo-crawler-development
 
 test:
   <<: *default
   host: apollo-crawler.no-ip.org
-  vhost: apollo-crawler-test
+  vhost: /apollo-crawler-test
 
 production:
   <<: *default
   host: apollo-crawler.no-ip.org
-  vhost: apollo-crawler-production
+  vhost: /apollo-crawler-production

data/config/deploy.rb

@@ -0,0 +1,61 @@
+require 'capistrano'
+
+require "rubygems"
+require "bundler/setup"
+require "bundler/capistrano"
+
+# RVM integration
+require "rvm/capistrano"
+
+# Target ruby version
+set :rvm_ruby_string, '2.0.0'
+
+set :domain, "apollo-crawler.no-ip.org"
+set :application, "apollo_platform"
+# set :deploy_to, File.join(File.expand_path("~"), "/apps/#{application}")
+set :deploy_to, "/home/ubuntu/apps/#{application}"
+
+ssh_options[:keys] = [File.join(ENV["HOME"], ".ssh", "key-webs.pem")]
+
+set :user, "ubuntu"
+set :use_sudo, false
+
+set :scm, :git
+set :repository,  "https://github.com/korczis/apollo-crawler.git"
+set :branch, 'master'
+set :git_shallow_clone, 1
+
+role :web, domain
+role :app, domain
+role :db,  domain, :primary => true
+
+set :deploy_via, :remote_cache
+
+namespace :deploy do
+  def remote_cmd(cmd)
+     run "cd #{deploy_to}/current && #{cmd}"
+  end
+
+  task :start, :roles => [:web, :app] do
+    puts "Starting.."
+    remote_cmd "./bin/apollo-platform -V"
+  end
+ 
+  task :stop, :roles => [:web, :app] do
+    puts "Stopping.."
+  end
+
+  task :status, :roles => [:web, :app] do
+    puts "Statusing.."
+  end
+ 
+  task :restart, :roles => [:web, :app] do
+    puts "Restarting.."
+  end
+ 
+  # This will make sure that Capistrano doesn't try to run rake:migrate (this is not a Rails project!)
+  task :cold do
+    deploy.update
+    deploy.start
+  end
+end

data/config/mongoid.yml

@@ -1,23 +1,26 @@
-default: &default
+default: &default_options
   sessions:
     default:
       hosts:
         - apollo-crawler.no-ip.org:27017
 
 development:
-  <<: *default
   sessions:
     default:
+      hosts:
+        - localhost:27017
       database: apollo-crawler-development
 
 test:
-  <<: *default
   sessions:
     default:
-      database: apollo-crawler-test
+    hosts:
+      - apollo-crawler.no-ip.org:27017
+    database: apollo-crawler-test
 
 production:
-  <<: *default
   sessions:
     default:
-      database: apollo-crawler-production
+    hosts:
+      - apollo-crawler.no-ip.org:27017
+    database: apollo-crawler-production

data/lib/apollo_crawler.rb

@@ -48,6 +48,9 @@ require File.join(File.dirname(__FILE__), 'apollo_crawler/helper/helpers')
 # Loggers
 require File.join(File.dirname(__FILE__), 'apollo_crawler/logger/loggers')
 
+# Models
+require File.join(File.dirname(__FILE__), 'apollo_crawler/model/models')
+
 # Planner
 require File.join(File.dirname(__FILE__), 'apollo_crawler/planner/planners')
 

data/lib/apollo_crawler/agent/agents.rb

@@ -18,4 +18,5 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
-require File.join(File.dirname(__FILE__), 'base_agent')
+require File.join(File.dirname(__FILE__), 'base_agent')
+require File.join(File.dirname(__FILE__), 'fetcher_agent')

data/lib/apollo_crawler/agent/base_agent.rb

@@ -21,6 +21,9 @@
 module Apollo
 	module Agent
 		class BaseAgent
+			def run(amqp)
+				return 0
+			end
 		end # class BaseAgent
 	end # module Agent
 end # module Apollo

data/lib/apollo_crawler/agent/fetcher_agent.rb

@@ -0,0 +1,55 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require File.join(File.dirname(__FILE__), 'base_agent')
+require File.join(File.dirname(__FILE__), '../fetcher/fetchers')
+
+module Apollo
+	module Agent
+		class FetcherAgent < BaseAgent
+			attr_accessor :fetcher
+
+			def initialize(amqp, opts={})
+				self.fetcher = Apollo::Fetcher::SmartFetcher.new
+
+				if(opts[:verbose])
+					puts "Initializing fetcher agent..."
+				end
+
+				ch = amqp.create_channel
+				q  = ch.queue("fetcher", :auto_delete => false, :durable => true)
+				x  = ch.default_exchange
+
+				q.subscribe do |delivery_info, metadata, payload|
+					res = nil
+
+					puts "Received #{payload}" if opts[:verbose]
+
+					Thread.new do |t|
+						queued_url = JSON.parse(payload)
+						# puts queued_url["url"]
+						# res = Apollo::Fetcher::SmartFetcher::fetch(queued_url["url"])
+						# puts "#{queued_url['url']} - " + res.inspect
+					end
+				end
+			end
+		end # class FetcherAgent
+	end # module Agent
+end # module Apollo

data/lib/apollo_crawler/config.rb

@@ -87,8 +87,8 @@ module Apollo
 		}
 
 		# Used caching mechanism by default
-		CACHE_CLASS = Apollo::Cache::SqliteCache
-		CACHE_CLASS_OPTIONS = CACHE_CLASS_OPTIONS_MONGO
+		CACHE_CLASS = Apollo::Cache::MemcachedCache
+		CACHE_CLASS_OPTIONS = nil
 
 		############################################################
 		# Crawlers - Built-in out-of box working crawlers

data/lib/apollo_crawler/helper/amqp_helper.rb

@@ -18,9 +18,30 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+require 'amqp'
+require 'bunny'
+require 'thread'
+
 module Apollo
 	module Helper
 		module Amqp
+			def self.connect(conn, opts={})
+				res = nil
+
+				if(opts[:verbose])
+					puts "AMQP Connecting - #{conn.inspect}"
+				end
+
+				res = Bunny.new(:host => conn['host'], :user => conn['username'], :password => conn['password'], :vhost => conn['vhost'], :port => conn['port'])
+				res.start
+
+				sleep(0.001) until res
+				if(opts[:verbose])
+					puts "AMQP connected - #{res.inspect}"
+				end
+
+				return res
+			end
 		end # Amqp
 	end # module Helper
 end # module Apollo

data/lib/apollo_crawler/helper/core_helper.rb

@@ -24,6 +24,14 @@ class Class
 	end
 end
 
+class String
+  def to_class
+    self.split('::').inject(Object) do |mod, class_name|
+      mod.const_get(class_name)
+    end
+  end
+end
+
 module Apollo
 	module Helper
 		module Core

data/lib/apollo_crawler/helper/mongo_helper.rb

@@ -18,9 +18,25 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+require 'mongo'
+require 'mongoid'
+
 module Apollo
 	module Helper
 		module Mongo
+			def self.connect(conn, opts={})
+				if(opts[:verbose])
+					puts "MongoDB connecting - '#{conn.inspect}"
+				end
+
+				res = ::Mongo::Connection.new(conn['host'])
+
+				if(opts[:verbose])
+					puts "MongoDB connected: #{res.inspect}"
+				end
+
+				return res
+			end
 		end # Mongo
 	end # module Helper
 end # module Apollo

data/lib/apollo_crawler/lib.rb

@@ -45,6 +45,9 @@ require File.join(File.dirname(__FILE__), 'helper/helpers')
 # Loggers
 require File.join(File.dirname(__FILE__), 'logger/loggers')
 
+# Models
+require File.join(File.dirname(__FILE__), 'model/models')
+
 # Programs
 require File.join(File.dirname(__FILE__), 'planner/planners')
 

data/lib/apollo_crawler/model/base_model.rb

@@ -0,0 +1,29 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require 'mongo'
+require 'mongoid'
+
+module Apollo
+	module Model
+		class BaseModel
+		end # class BaseModel
+	end # module Model
+end # module Apollo

data/lib/apollo_crawler/model/crawler.rb

@@ -0,0 +1,39 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require File.join(File.dirname(__FILE__), 'base_model') 
+
+module Apollo
+	module Model
+		class Crawler < BaseModel
+			include Mongoid::Document
+			include Mongoid::Timestamps
+			
+			store_in collection: "crawlers"
+
+			field :name
+			field :class_name
+			field :source
+
+			# Indexes
+			index({ created_at: 1, updated_at: 1, name: 1, class_name: 1 })
+		end # class Crawler
+	end # module Model
+end # module Apollo

data/lib/apollo_crawler/model/models.rb

@@ -0,0 +1,24 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require File.join(File.dirname(__FILE__), 'base_model')
+require File.join(File.dirname(__FILE__), 'crawler')
+require File.join(File.dirname(__FILE__), 'queued_url')
+require File.join(File.dirname(__FILE__), 'raw_document')

data/lib/apollo_crawler/model/queued_url.rb

@@ -0,0 +1,38 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require File.join(File.dirname(__FILE__), 'base_model') 
+
+module Apollo
+	module Model
+		class QueuedUrl < BaseModel
+			include Mongoid::Document
+			include Mongoid::Timestamps
+			
+			store_in collection: "queued_urls"
+
+			field :url
+			field :state
+
+			# Indexes
+			index({ created_at: 1, updated_at: 1 })
+		end # class QueuedUrl
+	end # module Model
+end # module Apollo

data/lib/apollo_crawler/model/raw_document.rb

@@ -0,0 +1,37 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require File.join(File.dirname(__FILE__), 'base_model') 
+
+module Apollo
+	module Model
+		class RawDocument < BaseModel
+			include Mongoid::Document
+			include Mongoid::Timestamps
+
+			store_in collection: "raw_docs"
+
+			field :body
+
+			# Indexes
+			index({ created_at: 1, updated_at: 1 })
+		end # class RawDocument
+	end # module Model
+end # module Apollo

data/lib/apollo_crawler/planner/base_planner.rb

@@ -20,7 +20,10 @@
 
 module Apollo
 	module Planner
-		class BasePlanner		
+		class BasePlanner
+			def run(amqp, mongo)
+				return 0
+			end
 		end # class BasePlanner
 	end # module Planner
 end # module Apollo

data/lib/apollo_crawler/planner/smart_planner.rb

@@ -20,9 +20,58 @@
 
 require File.join(File.dirname(__FILE__),'base_planner') 
 
+require File.join(File.dirname(__FILE__),'../model/models.rb') 
+
 module Apollo
 	module Planner
 		class SmartPlanner < BasePlanner
+			attr_accessor :amqp
+			attr_accessor :mongo
+
+			def initialize(amqp=nil, mongo=nil)
+				self.amqp = amqp
+				self.mongo = mongo
+			end
+
+			def fetch_url(url, opts={})
+				puts "AMQP fetching '#{url.inspect}'"
+
+				ch = amqp.create_channel
+				x  = ch.default_exchange
+				x.publish(url.to_json, :routing_key => "fetcher")
+
+			end
+
+			def fetch_queued_urls(opts={})
+				urls = Apollo::Model::QueuedUrl.where({:state => :queued})
+				return if urls.count < 1
+
+				if(opts[:verbose])
+					puts "Fetching Queued URLS"
+				end
+
+				puts "Count of URLs in Queue: #{urls.count}" if opts[:verbose]
+
+				urls.each do |url|
+					url.state = :fetching
+					url.save
+
+					fetch_url(url, opts)
+					
+					# puts "Removing URL from Queue '#{url.inspect}'" if opts[:verbose]
+				end
+			end
+
+			def run(opts={})
+				request_exit = false
+				
+				while request_exit == false
+					fetch_queued_urls(opts)
+					sleep 1
+				end
+
+				return 0
+			end
 		end # class SmartPlanner
 	end # module Planner
 end # module Apollo

data/lib/apollo_crawler/program/base_program.rb

@@ -20,6 +20,8 @@
 
 require 'yaml'
 
+require File.join(File.dirname(__FILE__), "../model/models.rb")
+
 module Apollo
 	class BaseProgram
 		CONFIG_DIR = File.join(Apollo::BASE_DIR, "config")
@@ -33,16 +35,16 @@ module Apollo
 		attr_accessor :options
 		attr_accessor :optparser
 
+		attr_accessor :amqp
 		attr_accessor :mongo
-		attr_accessor :mongo_db
 
 		def initialize
 			self.config = {}
 			self.options = DEFAULT_OPTIONS
 			self.optparser = nil
 
+			self.amqp = nil
 			self.mongo = nil
-			self.mongo_db = nil
 		end
 
 		def self.get_config_path(config)
@@ -113,6 +115,55 @@ module Apollo
 			return nil
 		end
 
+		def init_amqp()
+			conn_opts = self.config["amqp"]
+			if(conn_opts)
+				self.amqp = Apollo::Helper::Amqp::connect(conn_opts, self.options)
+			end
+
+			return self.amqp
+		end
+
+		def init_mongo()
+			conn_opts = self.config["mongo"]
+			if(conn_opts)
+				self.mongo = Apollo::Helper::Mongo::connect(conn_opts, self.options)
+
+				# Init Mongoid
+				path = File.join(Apollo::BASE_DIR, "config/mongoid.yml")
+				Mongoid.load!(path, @options[:env])
+			end
+
+			return self.mongo
+		end
+
+		def init_seeds_crawlers(opts={})
+			objs = Apollo::Crawler::BaseCrawler.subclasses
+			objs.each do |o|
+				crawler = Apollo::Model::Crawler.new
+				i = o.new
+				crawler.name = i.name
+				crawler.class_name = o.to_s
+				
+				res = Apollo::Model::Crawler.where(class_name: crawler.class_name)
+				# puts "RES: '#{res.inspect}'"
+				if(res.nil? || res.count < 1)
+					crawler.save
+					if(opts[:verbose])
+						puts "Adding new crawler - '#{crawler.inspect}'"
+					end
+				else
+					if(opts[:verbose])
+						puts "Using crawler - '#{res[0].inspect}'"
+					end
+				end
+			end
+		end
+
+		def init_seeds(opts={})
+			init_seeds_crawlers(opts)
+		end
+
 		# Init program
 		def init_program(args)
 			res = nil
@@ -134,6 +185,12 @@ module Apollo
 			# Init Mongo Connection
 			init_mongo()
 
+			# Init AMQP
+			init_amqp()
+
+			# Init Seed data
+			init_seeds(@options)
+
 			return nil
 		end
 

data/lib/apollo_crawler/program/crawler_program.rb

@@ -45,16 +45,6 @@ require File.join(File.dirname(__FILE__), '..', 'version')
 
 require File.join(File.dirname(__FILE__),'base_program') 
 
-
-# Hack
-class String
-  def to_class
-    self.split('::').inject(Object) do |mod, class_name|
-      mod.const_get(class_name)
-    end
-  end
-end
-
 module Apollo
 	# Apollo Crawler Base Directory
 	APOLLO_CRAWLER_BASE_DIR = File.join(File.dirname(__FILE__), "..")
@@ -174,6 +164,8 @@ module Apollo
 
 				opts.on('-i', '--include [PATH]', 'Include additional crawler or crawler directory') do |path|
 					@options[:crawler_dirs] << path
+
+					init_additional_crawlers([path])
 				end
 
 				opts.on('-n', '--doc-limit [NUM]', 'Limit count of documents to be processed') do |count|
@@ -406,6 +398,22 @@ module Apollo
 			end
 		end
 
+		def init_additional_crawlers(dirs)
+			# puts "Initializing aditional crawlers ..."
+			dirs.each do |dir|
+				if(@options[:verbose])
+					puts "Registering additional crawler dir '#{dir}'"
+				end
+
+				Dir.glob("#{dir}/*.rb").each do |f| 
+					if(@options[:verbose])
+						puts "Registering crawler '#{f}'"
+					end
+					require f
+				end
+			end
+		end
+
 		# Init program
 		def init_program(args)
 			init_options()

data/lib/apollo_crawler/program/platform_program.rb

@@ -80,6 +80,10 @@ module Apollo
 					self.options[:env] = name
 				end
 
+				opts.on('-d', '--daemon', 'Run Apollo Platform daemon') do
+					self.options[:daemon] = true
+				end
+
 				opts.on('-v', '--verbose', 'Enable verbose output') do
 					self.options[:verbose] = true
 				end
@@ -90,6 +94,40 @@ module Apollo
 			end
 		end
 
+		def enqueue_crawlers_urls(amqp, crawlers=Apollo::Crawler::BaseCrawler.subclasses, opts={})
+			crawlers.each do |crawler|
+				i = crawler.new
+				puts "Queuying Crawler base URL: '#{i.url}'" if opts[:verbose]
+				qu = Apollo::Model::QueuedUrl.new(:url => i.url, :state => :queued)
+				qu.save
+			end	
+		end
+
+		def init_fetchers(amqp, opts={})
+			fetchers = []
+			fetchers << Apollo::Agent::FetcherAgent.new(amqp, self.options)
+
+			enqueue_crawlers_urls(amqp, Apollo::Crawler::BaseCrawler.subclasses, opts)
+			
+			# ch = self.amqp.create_channel
+			# x  = ch.default_exchange
+			# x.publish("Hello!", :routing_key => "fetcher")
+		end
+
+		def init_agents(amqp, opts={})
+			puts "Initializing agents"
+
+			init_fetchers(amqp, opts)	
+		end
+
+		def init_program(args)
+			res = super(args)
+			return res unless res.nil?
+
+			init_agents(self.amqp, self.options)
+			return nil
+		end
+
 		def process_options(args)
 			if(self.options[:version])
 				puts Apollo::VERSION
@@ -105,30 +143,21 @@ module Apollo
 			return nil
 		end
 
-		def init_mongo()
-			self.mongo = Mongo::Connection.new(self.config['mongo']['host'])
-			self.mongo_db = self.mongo.db(self.config['mongo']['db'])
-
-			if(self.options[:verbose])
-				puts "(Mongo) Connection Inited: #{self.mongo.inspect}"
-				puts "(Mongo) Database Inited: #{self.mongo_db.inspect}"
-			end
-
-			return self.mongo
-		end
-
 		# Run Program
 		def run(args = ARGV)
 			res = super(args)
 			return res unless res.nil?
-			
-			# Print classes
-			# puts Apollo::Crawler::BaseCrawler.subclasses.inspect
 
 			# Here we start
-			if(ARGV.length < 1)
-				puts optparser
-				return 0
+			# if(ARGV.length < 1)
+			# 	puts optparser
+			# 	return 0
+			# end
+
+			res_code = 0
+			if(self.options[:daemon])
+				planner = Apollo::Planner::SmartPlanner.new(self.amqp, self.mongo)
+				res_code = planner.run(self.options)
 			end
 
 			return request_exit(res_code)

data/lib/apollo_crawler/version.rb

@@ -19,5 +19,5 @@
 # THE SOFTWARE.
 
 module Apollo
-	VERSION = '0.1.22'
+	VERSION = '0.1.24'
 end # Apollo

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: apollo-crawler
 version: !ruby/object:Gem::Version
-  version: 0.1.22
+  version: 0.1.24
 platform: ruby
 authors:
 - Tomas Korcak
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-04 00:00:00.000000000 Z
+date: 2013-03-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: awesome_print
@@ -28,28 +28,28 @@ dependencies:
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 3.2.12
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 3.2.12
 - !ruby/object:Gem::Dependency
   name: dalli
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 2.6.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 2.6.2
 - !ruby/object:Gem::Dependency
@@ -70,42 +70,42 @@ dependencies:
   name: eventmachine
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: em-http-request
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: em-synchrony
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
@@ -154,56 +154,56 @@ dependencies:
   name: memcache-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: mongo
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.8.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.8.2
 - !ruby/object:Gem::Dependency
   name: mongoid
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 3.1.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 3.1.2
 - !ruby/object:Gem::Dependency
   name: mime-types
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
@@ -224,70 +224,70 @@ dependencies:
   name: openurl
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.4.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.4.2
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.6.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.6.2
 - !ruby/object:Gem::Dependency
   name: rack
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.5.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.5.2
 - !ruby/object:Gem::Dependency
   name: right_aws
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 3.0.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 3.0.5
 - !ruby/object:Gem::Dependency
   name: right_http_connection
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.3.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.3.0
 - !ruby/object:Gem::Dependency
@@ -336,146 +336,155 @@ dependencies:
   name: ffi
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.4.0
 - !ruby/object:Gem::Dependency
   name: guard
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.6.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.6.2
 - !ruby/object:Gem::Dependency
   name: guard-rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.0.7
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.0.7
 - !ruby/object:Gem::Dependency
   name: guard-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 2.5.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 2.5.0
 description: Gem for crawling data from external sources
 email: korczis@gmail.com
 executables:
+- apollo-console
 - apollo-crawler
 - apollo-platform
 extensions: []
 extra_rdoc_files: []
 files:
-- ./config/mongo.yml.default
-- ./config/mongo.yml
-- ./config/memcached.yml
-- ./config/mongoid.yml
-- ./config/apollo.yml.default
+- ./config/amqp.yml
 - ./config/amqp.yml.default
+- ./config/apollo.yml
+- ./config/apollo.yml.default
+- ./config/deploy.rb
+- ./config/memcached.yml
 - ./config/memcached.yml.default
-- ./config/amqp.yml
+- ./config/mongo.yml
+- ./config/mongo.yml.default
+- ./config/mongoid.yml
 - ./config/mongoid.yml.default
-- ./config/apollo.yml
-- ./lib/apollo_crawler/fetcher/smart_fetcher.rb
-- ./lib/apollo_crawler/fetcher/fetchers.rb
-- ./lib/apollo_crawler/fetcher/simple_fetcher.rb
-- ./lib/apollo_crawler/fetcher/base_fetcher.rb
-- ./lib/apollo_crawler/planner/base_planner.rb
-- ./lib/apollo_crawler/planner/planners.rb
-- ./lib/apollo_crawler/planner/smart_planner.rb
-- ./lib/apollo_crawler/lib.rb
-- ./lib/apollo_crawler/version.rb
-- ./lib/apollo_crawler/program/console_program.rb
-- ./lib/apollo_crawler/program/platform_program.rb
-- ./lib/apollo_crawler/program/crawler_program.rb
-- ./lib/apollo_crawler/program/base_program.rb
-- ./lib/apollo_crawler/program/programs.rb
-- ./lib/apollo_crawler/logger/console_logger.rb
-- ./lib/apollo_crawler/logger/base_logger.rb
-- ./lib/apollo_crawler/logger/loggers.rb
-- ./lib/apollo_crawler/helper/core_helper.rb
-- ./lib/apollo_crawler/helper/amqp_helper.rb
-- ./lib/apollo_crawler/helper/helpers.rb
-- ./lib/apollo_crawler/helper/mongo_helper.rb
+- ./lib/apollo_crawler.rb
 - ./lib/apollo_crawler/adapter/adapters.rb
-- ./lib/apollo_crawler/adapter/mongo_adapter.rb
 - ./lib/apollo_crawler/adapter/amqp_adapter.rb
-- ./lib/apollo_crawler/config.rb
-- ./lib/apollo_crawler/cache/sqlite_cache.rb
+- ./lib/apollo_crawler/adapter/mongo_adapter.rb
+- ./lib/apollo_crawler/agent/agents.rb
+- ./lib/apollo_crawler/agent/base_agent.rb
+- ./lib/apollo_crawler/agent/fetcher_agent.rb
+- ./lib/apollo_crawler/cache/base_cache.rb
+- ./lib/apollo_crawler/cache/caches.rb
 - ./lib/apollo_crawler/cache/factory.rb
-- ./lib/apollo_crawler/cache/null_cache.rb
+- ./lib/apollo_crawler/cache/memcached_cache.rb
 - ./lib/apollo_crawler/cache/memory_cache.rb
-- ./lib/apollo_crawler/cache/base_cache.rb
 - ./lib/apollo_crawler/cache/mongo_cache.rb
-- ./lib/apollo_crawler/cache/memcached_cache.rb
-- ./lib/apollo_crawler/cache/caches.rb
-- ./lib/apollo_crawler/crawler/xkcd_crawler.rb
-- ./lib/apollo_crawler/crawler/google_crawler.rb
-- ./lib/apollo_crawler/crawler/youjizz_crawler.rb
-- ./lib/apollo_crawler/crawler/slashdot_crawler.rb
-- ./lib/apollo_crawler/crawler/hacker_news_crawler.rb
+- ./lib/apollo_crawler/cache/null_cache.rb
+- ./lib/apollo_crawler/cache/sqlite_cache.rb
+- ./lib/apollo_crawler/config.rb
 - ./lib/apollo_crawler/crawler/base_crawler.rb
 - ./lib/apollo_crawler/crawler/crawlers.rb
+- ./lib/apollo_crawler/crawler/google_crawler.rb
+- ./lib/apollo_crawler/crawler/hacker_news_crawler.rb
+- ./lib/apollo_crawler/crawler/slashdot_crawler.rb
 - ./lib/apollo_crawler/crawler/stackoverflow_crawler.rb
+- ./lib/apollo_crawler/crawler/xkcd_crawler.rb
+- ./lib/apollo_crawler/crawler/youjizz_crawler.rb
 - ./lib/apollo_crawler/env.rb
-- ./lib/apollo_crawler/agent/agents.rb
-- ./lib/apollo_crawler/agent/base_agent.rb
-- ./lib/apollo_crawler/formatter/table_formatter.rb
+- ./lib/apollo_crawler/fetcher/base_fetcher.rb
+- ./lib/apollo_crawler/fetcher/fetchers.rb
+- ./lib/apollo_crawler/fetcher/simple_fetcher.rb
+- ./lib/apollo_crawler/fetcher/smart_fetcher.rb
 - ./lib/apollo_crawler/formatter/base_formatter.rb
+- ./lib/apollo_crawler/formatter/formatters.rb
 - ./lib/apollo_crawler/formatter/json_formatter.rb
 - ./lib/apollo_crawler/formatter/plain_formatter.rb
-- ./lib/apollo_crawler/formatter/formatters.rb
-- ./lib/apollo_crawler/store/stores.rb
+- ./lib/apollo_crawler/formatter/table_formatter.rb
+- ./lib/apollo_crawler/helper/amqp_helper.rb
+- ./lib/apollo_crawler/helper/core_helper.rb
+- ./lib/apollo_crawler/helper/helpers.rb
+- ./lib/apollo_crawler/helper/mongo_helper.rb
+- ./lib/apollo_crawler/lib.rb
+- ./lib/apollo_crawler/logger/base_logger.rb
+- ./lib/apollo_crawler/logger/console_logger.rb
+- ./lib/apollo_crawler/logger/loggers.rb
+- ./lib/apollo_crawler/model/base_model.rb
+- ./lib/apollo_crawler/model/crawler.rb
+- ./lib/apollo_crawler/model/models.rb
+- ./lib/apollo_crawler/model/queued_url.rb
+- ./lib/apollo_crawler/model/raw_document.rb
+- ./lib/apollo_crawler/planner/base_planner.rb
+- ./lib/apollo_crawler/planner/planners.rb
+- ./lib/apollo_crawler/planner/smart_planner.rb
+- ./lib/apollo_crawler/program/base_program.rb
+- ./lib/apollo_crawler/program/console_program.rb
+- ./lib/apollo_crawler/program/crawler_program.rb
+- ./lib/apollo_crawler/program/platform_program.rb
+- ./lib/apollo_crawler/program/programs.rb
 - ./lib/apollo_crawler/store/base_store.rb
-- ./lib/apollo_crawler.rb
+- ./lib/apollo_crawler/store/stores.rb
+- ./lib/apollo_crawler/version.rb
+- bin/apollo-console
 - bin/apollo-crawler
 - bin/apollo-platform
 homepage: http://apollocrawler.com/
 licenses:
 - MIT
 metadata: {}
-post_install_message: Thanks for installing Apollo Crawler!
+post_install_message: Thank you for installing Apollo Crawler!
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: 1.9.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: 1.8.11
 requirements: []