collamine 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0ff9befa1cf2bd9dfe6fdae08b111d4555fca9a3
-  data.tar.gz: bf78076f8b51c02eb3172ff9a5c1ff98a03fd95b
+  metadata.gz: 4082cf4e4a34cca97098611238f3ad46cf6e74d0
+  data.tar.gz: 7c5302924a2c2adfcd2f6a077d93294ff073ee8c
 SHA512:
-  metadata.gz: 8a048c8f364ecbf1d5f768ffd6e439f962882a5ed128b2d72b560adee8357d753b64e52ec97f77d98f06fd19c82791c967637af4802eab72e0bfb15b109291c2
-  data.tar.gz: 2b15cbef9354411d0b18aa24a2317f58128a1e873b2a0891d60daacbe8a743719017cc6ad2a78215de17f5211719339a83708c35c34c7cc95f46844acfdfa310
+  metadata.gz: 56faf9049d65e2c9cf092c1e275820b237abb3563088e0e658476895183ae22124e6c0c2c6e528cafa649e216f1f15bb3a55e32ec50a0202cbd7dd2554f72ee7
+  data.tar.gz: 358cc2a90c568d109926ca8a6ebae9e2ad462d1819142cc10614471565ed1e5e87e72d92c456c9175b3000533d7cf575f08a338692a62d08519abe1723df7213

data/collamine.gemspec

@@ -24,6 +24,4 @@ Gem::Specification.new do |spec|
   spec.add_dependency "spidercrawl"
   spec.add_dependency "multipart-post"
   spec.add_dependency "domainatrix"
-  spec.add_dependency "mongo"
-  spec.add_dependency "bson_ext"
 end

data/lib/collamine.rb

@@ -1,19 +1,14 @@
 require 'collamine/request'
 require 'spidercrawl'
 
-require 'curb'
-require 'domainatrix'
-require 'mongo'
-
 class Collamine
-  include Mongo
-  STORE = MongoClient.new("localhost", 27017).db("smartcache").collection("html")
-
   def self.start(url, options)
+    Request.setup_collamine(download: options[:download], upload: options[:upload])
     from_collamine = []
     pages = Spiderman.shoot(url, options) do |web|
       collamine = nil
       web.before_fetch do |url|
+        @setup.yield url unless @setup.nil?
         # Try to fetch from collamine server
         puts "trying collamine: #{url}" 
         puts "fetched from collamine: #{url}" if (collamine = Request.try_collamine(url))
@@ -36,25 +31,18 @@ class Collamine
             filename += '.html' unless filename.include?('.html')
             Request.upload_to_collamine(page.url, page.content, filename, page.crawled_time.to_i)
           end
-          # Check if duplicate
-          unless STORE.find("url" => page.url).to_a.size > 0
-            # Insert into Mongodb
-            puts "Insert to db: #{page.url}"
-            source = (from_collamine.include?(page.url) ? 'collamine' : 'original')
-            doc = {:url => page.url, 
-                   :domain => Domainatrix.parse(page.url).domain,
-                   :source => source, 
-                   :content => page.content.encode('UTF-8', 'ISO-8859-15'), 
-                   :crawled_date =>  page.crawled_time.to_i,
-                   :response_time => page.response_time.to_i
-            }
-            STORE.insert(doc)
-          else
-            puts "url exists"
-          end
         end
+        @teardown.yield page, from_collamine unless @teardown.nil?
       end
     end
     return pages, from_collamine
   end
+
+  def self.before_fetch(&block)
+    @setup = block if block
+  end
+
+  def self.after_fetch(&block)
+    @teardown = block if block
+  end
 end

data/lib/collamine/request.rb

@@ -4,13 +4,17 @@ require 'domainatrix'
 
 # Makes the request to CollaMine servers
 class Request
-  COLLAMINE_DOWNLOAD_URL = 'http://172.31.22.135:9001/download/html/'
-  COLLAMINE_UPLOAD_URL   = 'http://172.31.22.135:9001/upload/html/multipart/'
+  #COLLAMINE_DOWNLOAD_URL = 'http://172.31.22.135:9001/download/html/'
+  #COLLAMINE_UPLOAD_URL   = 'http://172.31.22.135:9001/upload/html/multipart/'
+  def self.setup_collamine(options)
+    @collamine_download_url = options[:download]
+    @collamine_upload_url = options[:upload]
+  end
   #
   # Try downloading the content from CollaMine servers
   #
   def self.try_collamine(url)
-    uri = URI.parse(COLLAMINE_DOWNLOAD_URL+CGI::escape(url.to_s))
+    uri = URI.parse(@collamine_download_url+CGI::escape(url.to_s))
     Net::HTTP.start(uri.host, uri.port) do |http|
       response = http.get(uri)
       case response
@@ -25,13 +29,13 @@ class Request
   # Upload the content to Collamine servers
   #
   def self.upload_to_collamine(url, content, filename, crawltime)
-    post_request = Net::HTTP::Post::Multipart.new COLLAMINE_UPLOAD_URL,
+    post_request = Net::HTTP::Post::Multipart.new @collamine_upload_url,
                                                   'domain'      => Domainatrix.parse(url).domain,
                                                   'url'         => url,
                                                   'crawltime'   => crawltime,
                                                   'contributor' => 'belson',
                                                   'document'    => UploadIO.new(StringIO.new(content.encode('UTF-8', 'ISO-8859-15')), 'text/html', filename)                                 
-    response = Net::HTTP.start(URI.parse(COLLAMINE_UPLOAD_URL).host, URI.parse(COLLAMINE_UPLOAD_URL).port) { |http| http.request(post_request) }
+    response = Net::HTTP.start(URI.parse(@collamine_upload_url).host, URI.parse(@collamine_upload_url).port) { |http| http.request(post_request) }
     puts response.body
   end
 end

data/lib/collamine/version.rb

@@ -1,3 +1,3 @@
 module Collamine
-  VERSION = "0.2.1"
+  VERSION = "0.3.0"
 end

data/test.rb

@@ -1,5 +1,6 @@
 require 'collamine'
-
+require 'mongo'
+require 'domainatrix'
 #http://forums.hardwarezone.com.sg/hwm-magazine-publication-38/
 #http://forums.hardwarezone.com.sg/money-mind-210/
 #http://sgforums.com/forums/4
@@ -13,10 +14,37 @@ require 'collamine'
 #https://www.apple.com/sg/
 #http://forums.hardwarezone.com.sg/current-affairs-lounge-17/
 
+include Mongo
+STORE = MongoClient.new("localhost", 27017).db("smartcache").collection("html")
+
+Collamine.before_fetch do |url|
+  puts "Do what you want to the url: #{url}"
+end
+
+Collamine.after_fetch do |page, from_collamine|
+  # Check if duplicate
+  unless STORE.find("url" => page.url).to_a.size > 0
+    # Insert into Mongodb
+    puts "Insert to db: #{page.url}"
+    source = (from_collamine.include?(page.url) ? 'collamine' : 'original')
+    doc = {:url => page.url, 
+           :domain => Domainatrix.parse(page.url).domain,
+           :source => source, 
+           :content => page.content.encode('UTF-8', 'ISO-8859-15'), 
+           :crawled_date =>  page.crawled_time.to_i,
+           :response_time => page.response_time.to_i
+    }
+    STORE.insert(doc)
+  else
+    puts "url exists"
+  end
+end
+
 pages, from_collamine = Collamine.start('http://forums.hardwarezone.com.sg/money-mind-210/',
-                                       :parallel => true,
-                                       :threads => 10, 
-                                       :pattern => Regexp.new('^http:\/\/forums\.hardwarezone\.com\.sg\/money-mind-210\/?(.*\.html)?$'))
+	                                   :pattern => Regexp.new('^http:\/\/forums\.hardwarezone\.com\.sg\/money-mind-210\/?(.*\.html)?$'),
+	                                   :download => 'http://172.20.131.150:9001/download/html/',
+	                                   :upload => 'http://172.20.131.150:9001/upload/html/multipart/',
+                                       :parallel => true, :threads => 10)
 
 puts "Total pages crawled: #{pages.size}"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: collamine
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Belson Heng
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-10 00:00:00.000000000 Z
+date: 2016-03-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -80,34 +80,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: mongo
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: bson_ext
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: Collamine is a ruby gem for CollaMine client, which communicates with
   CollaMine servers to download content from their SmartCache if it exists.
 email:
@@ -146,7 +118,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Collamine lets you crawl a web site using SpiderCrawl library and share the