socializer-scraper 0.0.3 → 0.0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/socializer-scraper +4 -2
- data/lib/socializer/scraper/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5ff0568774a1e3bf2ce45ffee204ad10802906ab
|
4
|
+
data.tar.gz: 964535e1f772af5f02ab1b029cb3a9281251d5c1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1dbebe4c40d48807ef2675b856aa4c47d5436cf45134c429a34bc6c9451506dbe7ed28a26ec43ffcce096d833868b5fcc44b827274f7141c15551544b4b0bc9c
|
7
|
+
data.tar.gz: 31ae5ac34fc0ecb904468772e6e7d2d51286b286a42c86740ee3095688e90471f1a3b97f08787ddffb9b22794b64b447bbc514e5c1fdd4ba6d107a702c56094c
|
data/bin/socializer-scraper
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
|
+
require 'pry'
|
3
4
|
require 'yaml'
|
4
5
|
require 'thor'
|
5
6
|
require 'fileutils'
|
@@ -17,10 +18,11 @@ class Socializer::Scraper::CLI < Thor
|
|
17
18
|
puts "Scraping website: #{website}"
|
18
19
|
puts "=" * 100
|
19
20
|
|
20
|
-
|
21
|
+
website = URI.parse("http://#{website}") unless website.start_with?("http")
|
22
|
+
file = File.join(Dir.pwd, "#{website.host}.yml")
|
21
23
|
counter, list = 0, (File.exists?(file) ? YAML.load_file(file) : [])
|
22
24
|
|
23
|
-
extractor.url =
|
25
|
+
extractor.url = website.to_s
|
24
26
|
extractor.run do |page, collector, found|
|
25
27
|
found = found.map{ |email| email.strip }.accumulate - list
|
26
28
|
list |= found
|