socializer-scraper 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bc110e79790596f048e8745e0d346317c0ef1c0a
-  data.tar.gz: 4d084ed27c23b9218f808a8084ed6accae5462be
+  metadata.gz: dca36b51da99ac68ef2d7f6dc62fb48c46ee015b
+  data.tar.gz: 03e55384ed88979807a2bddc02acd02082b31d82
 SHA512:
-  metadata.gz: df9d0eb4c18b6c1e4f0bff4abd2c93b0ebb02af8567cda2d14550734d466fdafd6b47e3848def697d73bd6b7a9b64eff306b8db2583f1fcd6251d55d3f6fe57a
-  data.tar.gz: 258073993bb483525cbe1a2c39f521bac12f8d3a4be58f7d1d48d1b299696c1c5ae19a5982308b985bbc5358c990b5b9ad56ca97aa42d4c2ad5c47a74be2e3f9
+  metadata.gz: e07091135c2fbdafe8626dea0145119522cc2262df87459fa95b526c89f91a78d046d2a868cd853a69f4c72b487aa89e07473346b15f2e0bfab907a94f4a92b2
+  data.tar.gz: fa6185bc3d4700f4b65317cbc94f328b7cd27d8312c5622d8d2a5f21212dede0a6a341fc86df699d2f4845b0043719e7659399595163d0f9a54825866e783ec1

data/.gitignore

@@ -18,3 +18,4 @@ tmp
 data/
 tags
 scripts/
+*.yml

data/bin/socializer-scraper

@@ -1,3 +1,58 @@
 #!/usr/bin/env ruby
 
+require 'yaml'
+require 'thor'
+require 'fileutils'
 require 'socializer/scraper'
+
+class Socializer::Scraper::CLI < Thor
+
+  desc "emails [URLs]", "scrape emails for a given URL and all subsequently found URLs"
+  def emails(*urls)
+    extractor = Socializer::Scraper::Extractor.new collectors: [:email]
+    urls.each do |website|
+
+      puts "=" * 100
+      puts "Current Time is : #{Time.now.utc}"
+      puts "Scraping website: #{website}"
+      puts "=" * 100
+
+      file = File.join(Dir.pwd, "#{website}.yml")
+      counter, list = 0, (File.exists?(file) ? YAML.load_file(file) : [])
+
+      extractor.url = "http://" + website unless website.start_with?("http")
+      extractor.run do |page, collector, found|
+        found  = found.map{ |email| email.strip }.accumulate - list
+        list  |= found
+
+        found  = found.count
+        found  = "+" if found > 9
+        found  = "." if found < 1
+
+        if counter % 100 == 99
+          File.open(file, "w") { |f| f.puts list.to_yaml }
+          puts found
+        else
+          print found
+        end
+
+        counter += 1
+      end
+
+      puts "=" * 100
+      puts "Finish Time is  : #{Time.now.utc}"
+      puts "Emails Found    : #{list.count}"
+    end
+  end
+
+end
+
+Socializer::Scraper::CLI.start ARGV
+
+websites = %w[
+  www.thegearpage.net
+  www.hugeracksin.com
+  www.rig-talk.com
+  www.guitariste.com
+  www.tonequest.com
+]

data/lib/socializer/scraper/version.rb

@@ -1,5 +1,5 @@
 module Socializer
   module Scraper
-    VERSION = "0.0.2"
+    VERSION = "0.0.3"
   end
 end

data/socializer-scraper.gemspec

@@ -24,7 +24,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "yard"
   spec.add_development_dependency "guard-yard"
 
-  spec.add_dependency "bson_ext"
+  spec.add_dependency "thor"
   spec.add_dependency "mongo"
   spec.add_dependency "anemone"
+  spec.add_dependency "bson_ext"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: socializer-scraper
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Nikhil Gupta
@@ -81,7 +81,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: bson_ext
+  name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
@@ -122,6 +122,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: bson_ext
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Various scrapers for the Socializer application.
 email:
 - me@nikhgupta.com