apollo-crawler 0.1.12 → 0.1.13

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MjkxYjlmY2NjMDYwODcxN2JmMzA1MDM3NzM5NzQ1ZWVhNDNiYWQ0MQ==
+    OTQxNjRiOTJiNzc5MWZkZTg1NmE0YjA2OTE3YjQ3NmM5MjBhMTA2NQ==
   data.tar.gz: !binary |-
-    N2ViMTdhNmQ1OGM3ZTczYjIxZWU1Y2NlY2NlYWMxMDM1MDkwZjBjYg==
+    NDM0OWUwMzM3OGNkNzk5YWMwNjMwYzFlNzUwNDc2MzFlZDZhMzJiMA==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    NmNiNjMzZGFjY2ZmOTNjOTU4NTAxMGQzODZlNGYyOWI2MGQzY2YwM2Q4ZmMw
-    NDMzYzM2OTNkYmU2MzJiYzNhYzMwNGNmZDI0OWZiM2ZiZjJiYjFkMWExY2Rh
-    NTA3MDkxNTA1OTk1NWE5ZWMyNGFiZjY5ODhiMDMxZDU5NjgwZDU=
+    YjY4ZTNmMmIxZmUwMmE1OWMxNzVmZTQyYmQ2NTQ0MDExM2IzM2Q4MmVlYmEz
+    MmJiNzAxMDAzNGM4ODIwODViYjk4YTU5OWY2NGU1ODU2MmYxMTI0NzdiMDg5
+    NTI5OWQ3MjkzZTRjZDAzNWYxZTc5YmNmZGM3OTc3OGFlNDNkZjM=
   data.tar.gz: !binary |-
-    MWI1YWI3NjcwYzQ1NWVkZDI2ZjM3NjY5MGRjMzZkMDQxMThlMWE2MGU3MzAx
-    ZWE5ZWIyYWExNjdjMjYyYjUxNTU5MWZlNGI5MWUzOWYwZGI2NjQ2YTNkMTIy
-    YmRiNzIzNGY5ZThlNTdkMzIwODJkNjc2ZWUyNzQ5MWNlOWZlM2I=
+    M2NjZTJmYzFkNGFkNzQzOWY0ZTRhMjBmMThkYTIxYWFkNGJjYmFhZDgyMTQ4
+    ZWNlM2M1NGU2M2E3MjAwZjcyYmVmNmM0OTVmODQxMmY1ZmM5YWEyZDAwZWYy
+    YzMyYWMyNTBhYjg0YWRiZWNlMjcxNjQzOTMyMDlhYWU5ZjBjYjc=

data/lib/apollo_crawler.rb

@@ -39,6 +39,7 @@ require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/hacker_news_cr
 require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/slashdot_crawler')
 require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/stackoverflow_crawler')
 require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/xkcd_crawler')
+require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/youjizz_crawler')
 
 # Fetchers
 require File.join(File.dirname(__FILE__), 'apollo_crawler/fetcher/base_fetcher')

data/lib/apollo_crawler/config.rb

@@ -45,7 +45,7 @@ module RbConfig
 	# CACHE_CLASS = Apollo::Cache::NullCache
 
 	# Used caching mechanism by default
-	CACHE_CLASS = Apollo::Cache::MemoryCache
+	CACHE_CLASS = Apollo::Cache::MongoCache
 
 
 

data/lib/apollo_crawler/crawler/base_crawler.rb

@@ -160,7 +160,7 @@ module Apollo
 			def self.create_metadoc(url, doc)
 				return {
 					'url' => url,
-					'doc' => doc.encode('utf-8'),
+					'doc' => doc.encode('UTF-8', {:invalid => :replace, :undef => :replace, :replace => '?'}),
 					'hash' => Digest::SHA256.new.update(doc).hexdigest,
 					'created_at' => Time.now.utc,
 					'expires_at' => nil,

data/lib/apollo_crawler/crawler/youjizz_crawler.rb

@@ -0,0 +1,59 @@
+# Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+module Apollo
+	module Crawler
+		class YoujizzCrawler < BaseCrawler
+			@@MATCHER_ITEM = "//span[@id = 'miniatura']/span/a"
+
+			def name()
+				return "Youjizz"
+			end
+
+			def url()
+				return "http://www.youjizz.com/"
+			end
+
+			def extract_data(doc)
+				res = doc.xpath(@@MATCHER_ITEM).map { | node |
+					link = BaseCrawler.try_get_url(self.url, node['href'])
+					next if link.nil?
+
+					{ 
+						:text => link,
+						:link => link
+					}
+				}
+			end
+
+			def extract_links(doc)
+				res = doc.xpath("//div[@id = 'pagination']/a").map { | node |
+					link = BaseCrawler.try_get_url(self.url, node['href'])
+					next if link.nil?
+
+					{ 
+						:link => link
+					}
+				}
+			end
+		end
+	end # Crawler
+end # Apollo
+

data/lib/apollo_crawler/fetcher/smart_fetcher.rb

@@ -26,7 +26,7 @@ require File.join(File.dirname(__FILE__), 'base_fetcher')
 module Apollo
 	module Fetcher
 		class SmartFetcher < BaseFetcher
-			@@DEFAULT_SLEEP = 3.0
+			@@DEFAULT_SLEEP = 1.0
 			@@LAST_FETCH = nil
 
 			def self.fetch(url)

data/lib/apollo_crawler/lib.rb

@@ -36,6 +36,7 @@ require File.join(File.dirname(__FILE__), 'crawler/hacker_news_crawler')
 require File.join(File.dirname(__FILE__), 'crawler/slashdot_crawler')
 require File.join(File.dirname(__FILE__), 'crawler/stackoverflow_crawler')
 require File.join(File.dirname(__FILE__), 'crawler/xkcd_crawler')
+require File.join(File.dirname(__FILE__), 'crawler/youjizz_crawler')
 
 # Fetchers
 require File.join(File.dirname(__FILE__), 'fetcher/base_fetcher')

data/lib/apollo_crawler/program.rb

@@ -43,6 +43,10 @@ require File.join(File.dirname(__FILE__), 'version')
 
 module Apollo
 	class CrawlerProgram
+		@@PROGRAM_DIR = File.expand_path("~/.apollo-crawler")
+		@@CONFIG_PATH = File.join(@@PROGRAM_DIR, "config.rb")
+
+		# Load default config
 		require File.join(File.dirname(__FILE__), "config")
 
 		# This hash will hold all of the options
@@ -130,7 +134,11 @@ module Apollo
 
 				opts.on(nil, '--list-formatters', 'List of formatters available') do
 					@options[:list_formatters] = true
-				end				
+				end			
+
+				opts.on('-s', '--silent', 'Silent mode - do not print processed document') do
+					@options[:silent] = true
+				end	
 			end
 		end
 
@@ -197,13 +205,13 @@ module Apollo
 		# Load global options first
 		# Merge it with local options (if they exists)
 		def load_config_file()
-			config = File.join(File.dirname(__FILE__), "config", "crawler.rb")
+			config = @@CONFIG_PATH
+
 			if(File.exists?(config))
 				if(@options[:verbose])
-					RbConfig::DEFAULT_LOGGER "Loading config '#{config}'"
+					puts "Loading config '#{config}'"
 				end
 				
-				# puts "Let's require '#{@options[:verbose]}'"
 				require config
 			else
 				if(@options[:verbose])
@@ -429,8 +437,10 @@ module Apollo
 						docs = [docs]
 					end
 
-					docs.each do |doc|
-						puts @formatter.format(doc)
+					if @options[:silent] != true
+						docs.each do |doc|
+							puts @formatter.format(doc)
+						end
 					end
 				}
 			end
@@ -452,12 +462,28 @@ module Apollo
 			return crawlers
 		end
 
+		def init_program_directory()
+			dir = File.expand_path("~/.apollo-crawler")
+			if(File.directory?(dir) == false)
+				FileUtils.mkpath(dir)
+			end
+
+			config_path = File.join(File.dirname(__FILE__), 'config_user.trb')
+			dest_path = File.join(dir, 'config.rb')
+
+			if(File.exists?(config_path) && File.exists?(dest_path) == false)
+				FileUtils.cp(config_path, dest_path)
+			end
+		end
+
 		# Init program
 		def init_program(args)
 			init_options()
 
 			parse_options(args)
 
+			init_program_directory()
+
 			load_config_file()
 
 			register_modules()

data/lib/apollo_crawler/version.rb

@@ -19,5 +19,5 @@
 # THE SOFTWARE.
 
 module Apollo
-	VERSION = '0.1.12'
+	VERSION = '0.1.13'
 end # Apollo

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: apollo-crawler
 version: !ruby/object:Gem::Version
-  version: 0.1.12
+  version: 0.1.13
 platform: ruby
 authors:
 - Tomas Korcak
@@ -328,6 +328,7 @@ files:
 - ./lib/apollo_crawler/cache/memcached_cache.rb
 - ./lib/apollo_crawler/crawler/xkcd_crawler.rb
 - ./lib/apollo_crawler/crawler/google_crawler.rb
+- ./lib/apollo_crawler/crawler/youjizz_crawler.rb
 - ./lib/apollo_crawler/crawler/slashdot_crawler.rb
 - ./lib/apollo_crawler/crawler/hacker_news_crawler.rb
 - ./lib/apollo_crawler/crawler/base_crawler.rb