apollo-crawler 0.1.21 → 0.1.22

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NWM2MjFlZGYyOTI5OGI4OWJlZWM2YmM0Zjg1NTAxZTcwZDkxYTEzZA==
+    MjNkN2YzMjExMDc4Mjg5ZDNhNGMxYTY0MTY1NjczMDQ0MzE4YmFlOQ==
   data.tar.gz: !binary |-
-    NzZiMGRhMzI1ZmVjMjE2ODcwMzJlZDllMDZmNjYzYWYwNGUwNzE2Mw==
+    ZWVlM2UyZjhhZDM4Y2ExZGExNzkzNjNjNjNkNTI4OTRkOTEzY2Y5NA==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    MzY0NTQ3OTE2OTUyMWI1NDA2N2QwNGEwNzMxMDhmMTM4ODk3ZjgzODA5ZmU4
-    MmM2NWI3OTIzMTFmZDNlZjg2ODIxZDZmMjFhMDU3OTUwYjljYmJhNWE1NTkx
-    NWExNTg2OTUyMjY3ZWQ3YWQ3ODJjYzkzMDA0ODkzMjgzMjI4MGQ=
+    ZDc0YmI0MjEyNTgzNTM0NmY2NGE2YTY0Y2MxNTdiNTBkN2M3ZDM2MDdmNjZm
+    MjNhZDVlNmRkNDdkMDVhNzhjNzg0ZDQ2ZjRkYThhNzJlODMxNjE3NmE4MjVm
+    ZDg0ZjFlZWFjM2I0NWQ0ZmIzYmM5ZmY0MTRiOGE2YTMwZDVmYzE=
   data.tar.gz: !binary |-
-    ZGU1MjY3N2JmZWFkOWE5Y2Q0Y2ViNmM4YjFkZjM2NGNmMzc3OGRlNGUxMjU2
-    Yzk1MWU2NDMwOGNhZjUwNmMzZjRhZmZjOWRmYzAzZWYyMzQ1MjNlMTU4YWYz
-    ZDMzYTliMDVkYTQzYjlmMmZjNWEzMmVmMjgzZTE2OWJiOWE0Yjk=
+    N2I5ZGM1NjI2M2QxN2FmMDZkMThkOGU5NDE5MTEyOTNlZWFkNGQ1N2FlZjFi
+    MTI5ZGNhNjdmZTAyZjAyYTVkZWFlNGJmZDk5YzA3ZjlhN2Q5MTc1NTIyNGVi
+    M2VjODgxNzQyYTAxNzQ5NWQ5MTQzZjUxNWY5MWZlNDQzNjg2YmQ=

data/bin/apollo-crawler

@@ -28,7 +28,3 @@ require File.join(File.dirname(__FILE__), "..", "lib", "apollo_crawler")
 
 Apollo::CrawlerProgram.new.run(ARGV)
 
-# TODO: Maybe consider enabling this?
-# if __FILE__ == $0
-#	Apollo::CrawlerProgram.new.run(ARGV)
-# end

data/bin/apollo-platform

@@ -0,0 +1,30 @@
+#! /usr/bin/env ruby
+# encoding: utf-8
+
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require "rubygems"
+require "bundler/setup"
+
+require File.join(File.dirname(__FILE__), "..", "lib", "apollo_crawler")
+
+Apollo::PlatformProgram.new.run(ARGV)
+

data/config/amqp.yml

@@ -0,0 +1,18 @@
+default: &default
+  host: localhost
+  username: guest
+  password: guest
+
+development:
+  <<: *default
+  vhost: apollo-crawler-development
+
+test:
+  <<: *default
+  host: apollo-crawler.no-ip.org
+  vhost: apollo-crawler-test
+
+production:
+  <<: *default
+  host: apollo-crawler.no-ip.org
+  vhost: apollo-crawler-production

data/config/amqp.yml.default

@@ -0,0 +1,18 @@
+default: &default
+  host: localhost
+  username: guest
+  password: guest
+
+development:
+  <<: *default
+  vhost: apollo-crawler-development
+
+test:
+  <<: *default
+  host: apollo-crawler.no-ip.org
+  vhost: apollo-crawler-test
+
+production:
+  <<: *default
+  host: apollo-crawler.no-ip.org
+  vhost: apollo-crawler-production

data/config/apollo.yml

@@ -0,0 +1,13 @@
+default: &default
+  host: localhost
+
+development:
+  <<: *default  
+
+test:
+  <<: *default  
+  host: apollo-crawler.no-ip.org
+
+production:
+  <<: *default
+  host: apollo-crawler.no-ip.org

data/config/apollo.yml.default

@@ -0,0 +1,10 @@
+default: &default
+
+development:
+  <<: *default  
+
+test:
+  <<: *default  
+
+production:
+  <<: *default

data/config/memcached.yml

@@ -0,0 +1,14 @@
+default: &default
+  host: localhost
+  port: 11211
+
+development:
+  <<: *default  
+
+test:
+  host: apollo-crawler.no-ip.org
+  <<: *default
+
+production:
+  <<: *default
+  host: apollo-crawler.no-ip.org

data/config/memcached.yml.default

@@ -0,0 +1,14 @@
+default: &default
+  host: localhost
+  port: 11211
+
+development:
+  <<: *default  
+
+test:
+  host: apollo-crawler.no-ip.org
+  <<: *default
+
+production:
+  <<: *default
+  host: apollo-crawler.no-ip.org

data/config/mongo.yml

@@ -0,0 +1,19 @@
+default: &default
+  port: 27017  
+  username: guest
+  password: guest
+  db: apollo-crawler
+
+development:
+  <<: *default
+  db: apollo-crawler-development
+
+test:
+  <<: *default
+  host: apollo-crawler.no-ip.org
+  db: apollo-crawler-test
+
+production:
+  <<: *default
+  host: apollo-crawler.no-ip.org
+  db: apollo-crawler-production

data/config/mongo.yml.default

@@ -0,0 +1,19 @@
+default: &default
+  port: 27017  
+  username: guest
+  password: guest
+  db: apollo-crawler
+
+development:
+  <<: *default
+  db: apollo-crawler-development
+
+test:
+  <<: *default
+  host: apollo-crawler.no-ip.org
+  db: apollo-crawler-test
+
+production:
+  <<: *default
+  host: apollo-crawler.no-ip.org
+  db: apollo-crawler-production

data/config/mongoid.yml

@@ -0,0 +1,23 @@
+default: &default
+  sessions:
+    default:
+      hosts:
+        - apollo-crawler.no-ip.org:27017
+
+development:
+  <<: *default
+  sessions:
+    default:
+      database: apollo-crawler-development
+
+test:
+  <<: *default
+  sessions:
+    default:
+      database: apollo-crawler-test
+
+production:
+  <<: *default
+  sessions:
+    default:
+      database: apollo-crawler-production

data/config/mongoid.yml.default

@@ -0,0 +1,59 @@
+# Tell Mongoid which environment this configuration is for.
+production:
+  # This starts the session configuration settings. You may have as
+  # many sessions as you like, but you must have at least 1 named
+  # 'default'.
+  sessions:
+    # Define the default session.
+    default:
+      # A session can have any number of hosts. Usually 1 for a single
+      # server setup, and at least 3 for a replica set. Hosts must be
+      # an array of host:port pairs. This session is single server.
+      hosts:
+        - flame.mongohq.com:27017
+      # Define the default database name.
+      database: mongoid
+      # Since this database points at a session connected to MongoHQ, we must
+      # provide the authentication details.
+      username: user
+      password: password
+    # This defines a secondary session at a replica set.
+    replica_set:
+      # This configuration is a 3 node replica set.
+      hosts:
+        - dedicated1.myapp.com:27017
+        - dedicated2.myapp.com:27017
+        - dedicated3.myapp.com:27017
+      database: mongoid
+      # We can set session specific options, like reads executing
+      # on secondary nodes, and defaulting the session to safe mode.
+      options:
+        consistency: :eventual
+        safe: true
+    # This defines a tertiary session at a Mongos fronted shard.
+    shard:
+      # This configuration is a Mongos shard server.
+      hosts:
+        - mongos.myapp.com:27017
+      database: mongoid
+    # This configuration shows an authenticated replica set via a uri.
+    another:
+      uri: mongodb://user:pass@59.1.22.1:27017,59.1.22.2:27017/mongoid
+  # Here we put the Mongoid specific configuration options. These are explained
+  # in more detail next.
+  options:
+    allow_dynamic_fields: false
+    identity_map_enabled: true
+    include_root_in_json: true
+    include_type_for_serialization: true
+    # Note this can also be true if you want to preload everything, but this is
+    # almost never necessary. Most of the time set this to false.
+    preload_models:
+      - Canvas
+      - Browser
+      - Firefox
+    scope_overwrite_exception: true
+    raise_not_found_error: false
+    skip_version_check: false
+    use_activesupport_time_zone: false
+    use_utc: true

data/lib/apollo_crawler.rb

@@ -21,12 +21,18 @@
 # TODO: Make this work - DRY!
 # require File.join(File.dirname(__FILE__), 'apollo_crawler/lib')
 
+# Config First
+require File.join(File.dirname(__FILE__), 'apollo_crawler/env')
+
+# Adapters
+require File.join(File.dirname(__FILE__), 'apollo_crawler/adapter/adapters')
+
+# Agents
+require File.join(File.dirname(__FILE__), 'apollo_crawler/agent/agents')
+
 # Caches
 require File.join(File.dirname(__FILE__), 'apollo_crawler/cache/caches')
 
-# CLI - Command Line Interface
-require File.join(File.dirname(__FILE__), 'apollo_crawler/cli/cli')
-
 # Crawlers
 require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/crawlers')
 
@@ -42,6 +48,9 @@ require File.join(File.dirname(__FILE__), 'apollo_crawler/helper/helpers')
 # Loggers
 require File.join(File.dirname(__FILE__), 'apollo_crawler/logger/loggers')
 
+# Planner
+require File.join(File.dirname(__FILE__), 'apollo_crawler/planner/planners')
+
 # Program
 require File.join(File.dirname(__FILE__), 'apollo_crawler/program/programs')
 

data/lib/apollo_crawler/adapter/adapters.rb

@@ -0,0 +1,22 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require File.join(File.dirname(__FILE__), 'amqp_adapter')
+require File.join(File.dirname(__FILE__), 'mongo_adapter')

data/lib/apollo_crawler/adapter/amqp_adapter.rb

@@ -0,0 +1,26 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+module Apollo
+	module Adapter
+		class AmqpAdapter
+		end # class AmqpAdapter
+	end # module Adapter
+end # module Apollo

data/lib/apollo_crawler/adapter/mongo_adapter.rb

@@ -0,0 +1,26 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+module Apollo
+	module Adapter
+		class MongoAdapter
+		end # class AmqpAdapter
+	end # module MongoAdapter
+end # module Apollo

data/lib/apollo_crawler/{cli/cli.rb → agent/agents.rb}

@@ -17,3 +17,5 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
+
+require File.join(File.dirname(__FILE__), 'base_agent')

data/lib/apollo_crawler/agent/base_agent.rb

@@ -0,0 +1,26 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+module Apollo
+	module Agent
+		class BaseAgent
+		end # class BaseAgent
+	end # module Agent
+end # module Apollo

data/lib/apollo_crawler/cache/caches.rb

@@ -1,3 +1,23 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
 require File.join(File.dirname(__FILE__), 'base_cache') 
 require File.join(File.dirname(__FILE__), 'memcached_cache') 
 require File.join(File.dirname(__FILE__), 'memory_cache')