apollo-crawler 0.1.30 → 0.1.31

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4f7597400d1ab12d2ab2d1d52330ca29e4f2192f
-  data.tar.gz: 297216744ee4ea8a9af6595b1361dd3bfcf8d0b1
+  metadata.gz: c51a69932406b288595a4063140629cf633bc893
+  data.tar.gz: bb1ad85084a5b0c3188301e2339ae1656c8807c5
 SHA512:
-  metadata.gz: 71bf482ad465efa67976547c5e0a99d8a728478c5c346f987e675da8574b45e4dba44375fd501e8699c083e42eaf7e19ac1db6d2d9962f9736d408c52f896844
-  data.tar.gz: 01651a1aa4d3408ce63142a46c6e3727ceee35995520cfcc2df4b6451495b09cf5cebe1cc1244dca6ebef2138bb06b6ede97ac1720b75b8de866b3f381c2e25f
+  metadata.gz: 6bf291b011220dc67e0f5bcaad0f9f3744e0cf607638372a441fb9f63752e6b65b82b384787ea3d506862c40a10ea3799e370e4b93e0282605aa6e2af602cdec
+  data.tar.gz: 48ea2fe6c144b771f72bddfa9012d227aa8f349fd04fea3378f2590cdd6ba30ef99ddf186dd199e450d94d6ee0b15dbb0cdff94cdc77c6fe849db3ef7c622f90

data/lib/apollo_crawler.rb

@@ -24,9 +24,6 @@
 # Config First
 require File.join(File.dirname(__FILE__), 'apollo_crawler/env')
 
-# Adapters
-require File.join(File.dirname(__FILE__), 'apollo_crawler/adapter/adapters')
-
 # Agents
 require File.join(File.dirname(__FILE__), 'apollo_crawler/agent/agents')
 

data/lib/apollo_crawler/lib.rb

@@ -21,9 +21,6 @@
 # Environment first 
 require File.join(File.dirname(__FILE__), 'env')
 
-# Adapters
-require File.join(File.dirname(__FILE__), 'adapter/adapters')
-
 # Agents
 require File.join(File.dirname(__FILE__), 'agent/agents')
 

data/lib/apollo_crawler/{adapter/amqp_adapter.rb → model/data_chunk.rb}

@@ -18,9 +18,22 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+require File.join(File.dirname(__FILE__), 'base_model') 
+
 module Apollo
-	module Adapter
-		class AmqpAdapter
-		end # class AmqpAdapter
-	end # module Adapter
+	module Model
+		class DataChunk < BaseModel
+			include Mongoid::Document
+			include Mongoid::Timestamps
+			
+			store_in collection: "data_chunks"
+
+			field :data
+			field :crawler
+			field :data_hash
+
+			# Indexes
+			index({ created_at: 1, updated_at: 1, crawler: 1, data_hash: 1})
+		end # class DataSource
+	end # module Model
 end # module Apollo

data/lib/apollo_crawler/model/models.rb

@@ -20,7 +20,9 @@
 
 require File.join(File.dirname(__FILE__), 'base_model')
 require File.join(File.dirname(__FILE__), 'crawler')
+require File.join(File.dirname(__FILE__), 'data_chunk')
 require File.join(File.dirname(__FILE__), 'data_source')
 require File.join(File.dirname(__FILE__), 'domain')
 require File.join(File.dirname(__FILE__), 'queued_url')
 require File.join(File.dirname(__FILE__), 'raw_document')
+require File.join(File.dirname(__FILE__), 'user')

data/lib/apollo_crawler/{adapter/mongo_adapter.rb → model/user.rb}

@@ -18,9 +18,24 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+require File.join(File.dirname(__FILE__), 'base_model') 
+
 module Apollo
-	module Adapter
-		class MongoAdapter
-		end # class AmqpAdapter
-	end # module MongoAdapter
-end # module Apollo
+	module Model
+		class User < BaseModel
+			include Mongoid::Document
+			include Mongoid::Timestamps
+			
+			store_in collection: "users"
+
+			field :nick
+			field :email
+
+			# Indexes
+			index({ created_at: 1, updated_at: 1})
+			index({ nick: 1 }, { unique: true, background: true })
+			index({ email: 1 }, { unique: true, background: true })
+
+		end # class User
+	end # module Model
+end # module Apollo

data/lib/apollo_crawler/planner/smart_planner.rb

@@ -47,6 +47,10 @@ module Apollo
 				# Bindings
 				declarations[:queues]["planner.fetched.queue"].bind(declarations[:exchanges]["planner.fetched"]).subscribe do |delivery_info, metadata, payload|
 					msg = JSON.parse(payload)
+					puts "#{msg.inspect}" if opts[:verbose]
+
+					puts "REQ: #{msg['request']}" if opts[:verbose]
+					puts "RESP: #{msg['response']}" if opts[:verbose]
 
 					request = msg['request']
 					response = msg['response']
@@ -98,6 +102,9 @@ module Apollo
 					links = msg['links']
 					links = [] if links.nil?
 
+					data_hash = Digest::SHA256.new.update(data).hexdigest
+					puts "#{data_hash}"
+
 					links.each do |url|
 						link = url['link']
 						

data/lib/apollo_crawler/version.rb

@@ -19,5 +19,5 @@
 # THE SOFTWARE.
 
 module Apollo
-	VERSION = '0.1.30'
+	VERSION = '0.1.31'
 end # Apollo

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: apollo-crawler
 version: !ruby/object:Gem::Version
-  version: 0.1.30
+  version: 0.1.31
 platform: ruby
 authors:
 - Tomas Korcak
@@ -410,9 +410,6 @@ files:
 - ./config/mongoid.yml
 - ./config/mongoid.yml.default
 - ./lib/apollo_crawler.rb
-- ./lib/apollo_crawler/adapter/adapters.rb
-- ./lib/apollo_crawler/adapter/amqp_adapter.rb
-- ./lib/apollo_crawler/adapter/mongo_adapter.rb
 - ./lib/apollo_crawler/agent/agents.rb
 - ./lib/apollo_crawler/agent/base_agent.rb
 - ./lib/apollo_crawler/agent/crawler_agent.rb
@@ -457,11 +454,13 @@ files:
 - ./lib/apollo_crawler/logger/loggers.rb
 - ./lib/apollo_crawler/model/base_model.rb
 - ./lib/apollo_crawler/model/crawler.rb
+- ./lib/apollo_crawler/model/data_chunk.rb
 - ./lib/apollo_crawler/model/data_source.rb
 - ./lib/apollo_crawler/model/domain.rb
 - ./lib/apollo_crawler/model/models.rb
 - ./lib/apollo_crawler/model/queued_url.rb
 - ./lib/apollo_crawler/model/raw_document.rb
+- ./lib/apollo_crawler/model/user.rb
 - ./lib/apollo_crawler/planner/base_planner.rb
 - ./lib/apollo_crawler/planner/planners.rb
 - ./lib/apollo_crawler/planner/smart_planner.rb

data/lib/apollo_crawler/adapter/adapters.rb

@@ -1,22 +0,0 @@
-# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
-# 
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-# 
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-# 
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-
-require File.join(File.dirname(__FILE__), 'amqp_adapter')
-require File.join(File.dirname(__FILE__), 'mongo_adapter')