graybook 1.0.24 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/{CHANGES.md → CHANGELOG} +13 -2
- data/README +94 -0
- data/Rakefile +3 -22
- data/VERSION +1 -1
- data/VERSION.yml +2 -2
- data/VERSION_NAME +1 -1
- data/lib/graybook/importer/aol.rb +11 -11
- data/lib/graybook/importer/gmail.rb +20 -20
- data/lib/graybook/importer/hotmail.rb +2 -4
- data/lib/graybook/importer/page_scraper.rb +4 -2
- data/lib/graybook/importer/yahoo.rb +12 -12
- data/lib/graybook.rb +20 -21
- metadata +9 -102
- data/README.md +0 -66
- data/debug_graybook.rb +0 -10
- data/lib/graybook/exporter/vcf.rb +0 -45
- data/lib/graybook/exporter/xml.rb +0 -28
- data/lib/graybook/importer/csv.rb +0 -74
- data/lib/graybook/importer/freenet.rb +0 -62
- data/lib/graybook/importer/gmx.rb +0 -68
- data/lib/graybook/importer/web.de.rb +0 -67
- data/test/config/credentials.yml.example +0 -9
- data/test/fixtures/aol_application_page.html +0 -566
- data/test/fixtures/aol_bad_login_response_stage_3.html +0 -565
- data/test/fixtures/aol_contacts.html +0 -102
- data/test/fixtures/aol_login_response_stage_1.html +0 -158
- data/test/fixtures/aol_login_response_stage_2.html +0 -559
- data/test/fixtures/aol_login_response_stage_3.html +0 -48
- data/test/fixtures/aol_login_response_stage_4.html +0 -404
- data/test/fixtures/aol_login_response_stage_5.html +0 -404
- data/test/fixtures/aol_new_contacts.html +0 -431
- data/test/fixtures/gmail.csv +0 -3
- data/test/fixtures/gmail_bad_login_response_stage_2.html +0 -560
- data/test/fixtures/gmail_contacts.html +0 -228
- data/test/fixtures/gmail_login_response_stage_1.html +0 -556
- data/test/fixtures/gmail_login_response_stage_2.html +0 -1
- data/test/fixtures/gmail_login_response_stage_2a.html +0 -1
- data/test/fixtures/gmail_login_response_stage_3.html +0 -249
- data/test/fixtures/gmail_redirect_body.html +0 -10
- data/test/fixtures/hotmail_bad_login_response_stage_2.html +0 -31
- data/test/fixtures/hotmail_contacts.html +0 -262
- data/test/fixtures/hotmail_login_response_stage_1.html +0 -31
- data/test/fixtures/hotmail_login_response_stage_2.html +0 -1
- data/test/fixtures/hotmail_login_response_stage_3.html +0 -519
- data/test/fixtures/hotmail_scrape_first_page.html +0 -77
- data/test/fixtures/hotmail_scrape_response_stage_1.html +0 -90
- data/test/fixtures/hotmail_scrape_response_stage_2.html +0 -77
- data/test/fixtures/hotmail_scrape_response_stage_3.html +0 -0
- data/test/fixtures/yahoo_bad_login_response_stage_2.html +0 -443
- data/test/fixtures/yahoo_contacts.csv +0 -3
- data/test/fixtures/yahoo_contacts_not_logged_in.html +0 -432
- data/test/fixtures/yahoo_contacts_stage_1.html +0 -399
- data/test/fixtures/yahoo_login_response_stage_1.html +0 -433
- data/test/fixtures/yahoo_login_response_stage_2.html +0 -16
- data/test/fixtures/yahoo_no_user_response_stage_2.html +0 -574
- data/test/freenet_importer_test.rb +0 -53
- data/test/gmx_importer_test.rb +0 -53
- data/test/scripts/live_test.rb +0 -25
- data/test/test_graybook.rb +0 -60
- data/test/test_graybook_exporter_base.rb +0 -16
- data/test/test_graybook_exporter_vcf.rb +0 -52
- data/test/test_graybook_exporter_xml.rb +0 -16
- data/test/test_graybook_importer_aol.rb +0 -108
- data/test/test_graybook_importer_base.rb +0 -24
- data/test/test_graybook_importer_csv.rb +0 -60
- data/test/test_graybook_importer_gmail.rb +0 -116
- data/test/test_graybook_importer_hotmail.rb +0 -165
- data/test/test_graybook_importer_page_scraper.rb +0 -51
- data/test/test_graybook_importer_yahoo.rb +0 -137
- data/test/test_helper.rb +0 -71
- data/test/web.de_importer_test.rb +0 -53
- data/vendor/plugins/graybook/lib/autotest/discover.rb +0 -3
- data/vendor/plugins/graybook/lib/autotest/graybook.rb +0 -27
data/README.md
DELETED
@@ -1,66 +0,0 @@
|
|
1
|
-
Graybook
|
2
|
-
|
3
|
-
== Description:
|
4
|
-
|
5
|
-
Pulls contact lists from EVERYWHERE! (Some sites not included.)
|
6
|
-
|
7
|
-
== Features:
|
8
|
-
|
9
|
-
The current list of supported services and file types:
|
10
|
-
|
11
|
-
import:
|
12
|
-
- AOL
|
13
|
-
- CSV files
|
14
|
-
- Gmail
|
15
|
-
- Hotmail
|
16
|
-
- Yahoo! Mail
|
17
|
-
|
18
|
-
export:
|
19
|
-
- Simple hash (default)
|
20
|
-
- JSON (soon)
|
21
|
-
|
22
|
-
If you create an additional importer or exporter - or simply find a bug - please consider submitting it as a patch to the project so the community can all benefit from your hard work and ingenuity.
|
23
|
-
|
24
|
-
== Synopsis:
|
25
|
-
|
26
|
-
# An example of fetching Gmail contacts - by default, returns an array of hashes with :name and :email
|
27
|
-
contacts = Graybook.get :username => 'me@gmail.com', :password => 'whatever'
|
28
|
-
|
29
|
-
== Requirements:
|
30
|
-
|
31
|
-
* Mechanize and its dependencies, for interacting with online providers
|
32
|
-
* Fastercsv for reading CSV, Mechanize >= 0.7.0 for page scraping
|
33
|
-
|
34
|
-
== Install:
|
35
|
-
|
36
|
-
1. gem install gemcutter
|
37
|
-
1. gem tumble
|
38
|
-
2. gem install graybook
|
39
|
-
|
40
|
-
== Thanks:
|
41
|
-
|
42
|
-
Big thanks to Contentfree for making Blackbook.
|
43
|
-
|
44
|
-
== License:
|
45
|
-
|
46
|
-
Copyright (c) 2009, kojul.com
|
47
|
-
Copyright (c) 2007, Contentfree
|
48
|
-
|
49
|
-
Permission is hereby granted, free of charge, to any person obtaining
|
50
|
-
a copy of this software and associated documentation files (the
|
51
|
-
'Software'), to deal in the Software without restriction, including
|
52
|
-
without limitation the rights to use, copy, modify, merge, publish,
|
53
|
-
distribute, sublicense, and/or sell copies of the Software, and to
|
54
|
-
permit persons to whom the Software is furnished to do so, subject to
|
55
|
-
the following conditions:
|
56
|
-
|
57
|
-
The above copyright notice and this permission notice shall be
|
58
|
-
included in all copies or substantial portions of the Software.
|
59
|
-
|
60
|
-
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
|
61
|
-
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
62
|
-
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
63
|
-
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
64
|
-
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
65
|
-
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
66
|
-
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/debug_graybook.rb
DELETED
@@ -1,10 +0,0 @@
|
|
1
|
-
require 'graybook'
|
2
|
-
|
3
|
-
aol = Graybook.get :username => 'mondragonmichael@aol.com', :password => '123aol0'
|
4
|
-
puts "===== aol #{aol.inspect}"
|
5
|
-
|
6
|
-
unless aol.detect{|c| c[:name].downcase == 'Dave Myron'.downcase && c[:email] == 'dave.myron@contentfree.com'}
|
7
|
-
puts "AOL not working"
|
8
|
-
exit 1
|
9
|
-
end
|
10
|
-
|
@@ -1,45 +0,0 @@
|
|
1
|
-
##
|
2
|
-
# exports contacts in Vcard format
|
3
|
-
class Graybook::Exporter::Vcf < Graybook::Exporter::Base
|
4
|
-
|
5
|
-
##
|
6
|
-
# representation of a vcard
|
7
|
-
|
8
|
-
class Vcard
|
9
|
-
|
10
|
-
attr_accessor :first, :last, :email
|
11
|
-
|
12
|
-
##
|
13
|
-
# initialize dynamically sets the attributes passed in as accessible
|
14
|
-
# attribute on its object
|
15
|
-
|
16
|
-
def initialize( attributes = {} )
|
17
|
-
attributes.each{ |name,value| self.send("#{name}=", value) rescue next }
|
18
|
-
end
|
19
|
-
|
20
|
-
##
|
21
|
-
# text representation of this vcard
|
22
|
-
def to_s
|
23
|
-
<<-EOVC
|
24
|
-
BEGIN:VCARD
|
25
|
-
N:#{last};#{first}
|
26
|
-
EMAIL:#{email}
|
27
|
-
END:VCARD
|
28
|
-
EOVC
|
29
|
-
end
|
30
|
-
end
|
31
|
-
|
32
|
-
##
|
33
|
-
# exports contacts as Vcards
|
34
|
-
|
35
|
-
def export( contacts = [] )
|
36
|
-
return if contacts.empty?
|
37
|
-
|
38
|
-
contacts.uniq.compact.collect do |contact|
|
39
|
-
first_name, last_name = contact[:name].split(' ', 2)
|
40
|
-
Vcard.new( :first => first_name.to_s, :last => last_name.to_s, :email => contact[:email])
|
41
|
-
end
|
42
|
-
end
|
43
|
-
|
44
|
-
Graybook.register(:vcf, self)
|
45
|
-
end
|
@@ -1,28 +0,0 @@
|
|
1
|
-
require 'rexml/document'
|
2
|
-
|
3
|
-
##
|
4
|
-
# exports contacts in xml format
|
5
|
-
|
6
|
-
class Graybook::Exporter::Xml < Graybook::Exporter::Base
|
7
|
-
|
8
|
-
##
|
9
|
-
# contacts are an array of hashes that are contacts and returns xml
|
10
|
-
|
11
|
-
def export( contacts )
|
12
|
-
doc = REXML::Document.new
|
13
|
-
doc << REXML::XMLDecl.new
|
14
|
-
|
15
|
-
root = doc.add_element 'contacts'
|
16
|
-
contacts.each do |contact|
|
17
|
-
el = root.add_element 'contact'
|
18
|
-
name = el.add_element 'name'
|
19
|
-
name.text = contact[:name]
|
20
|
-
|
21
|
-
el.add_element('email').text = contact[:email]
|
22
|
-
end
|
23
|
-
|
24
|
-
doc.to_s
|
25
|
-
end
|
26
|
-
|
27
|
-
Graybook.register(:xml, self)
|
28
|
-
end
|
@@ -1,74 +0,0 @@
|
|
1
|
-
require 'fastercsv'
|
2
|
-
|
3
|
-
##
|
4
|
-
# Imports contacts from a CSV file
|
5
|
-
|
6
|
-
class Graybook::Importer::Csv < Graybook::Importer::Base
|
7
|
-
|
8
|
-
DEFAULT_COLUMNS = [:name,:email,:misc]
|
9
|
-
DEFAULT_PATTERN = /,/
|
10
|
-
|
11
|
-
##
|
12
|
-
# Matches this importer to a file that contains CSV values
|
13
|
-
|
14
|
-
def =~(options)
|
15
|
-
options && options[:file].respond_to?(:open) ? true : false
|
16
|
-
end
|
17
|
-
|
18
|
-
##
|
19
|
-
# fetch_contacts! implementation for this importer
|
20
|
-
|
21
|
-
def fetch_contacts!
|
22
|
-
lines = IO.readlines(options[:file].path)
|
23
|
-
columns = to_columns(lines.first)
|
24
|
-
lines.shift if columns.first == :name
|
25
|
-
columns = DEFAULT_COLUMNS.dup unless columns.first == :name
|
26
|
-
|
27
|
-
contacts = Array.new
|
28
|
-
lines.each do |l|
|
29
|
-
vals = l.split(pattern)
|
30
|
-
next if vals.empty?
|
31
|
-
vals.map! {|x| x.gsub( /\A"/m, "" ).strip.gsub( /"\Z/m, "" ) }
|
32
|
-
contacts << to_hash(columns, vals)
|
33
|
-
end
|
34
|
-
|
35
|
-
contacts
|
36
|
-
end
|
37
|
-
|
38
|
-
def to_hash(cols, vals) # :nodoc:
|
39
|
-
h = Hash.new
|
40
|
-
cols.each do |c|
|
41
|
-
h[c] = (c == cols.last) ? vals.join(',') : vals.shift
|
42
|
-
end
|
43
|
-
h
|
44
|
-
end
|
45
|
-
|
46
|
-
def to_columns(line) # :nodoc:
|
47
|
-
columns = Array.new
|
48
|
-
if line.match(pattern)
|
49
|
-
tags = line.split(pattern)
|
50
|
-
else
|
51
|
-
tags = line.split(DEFAULT_PATTERN)
|
52
|
-
end
|
53
|
-
# deal with "Name,E-mail..." oddity up front
|
54
|
-
if tags.first =~ /^name$/i
|
55
|
-
tags.shift
|
56
|
-
columns << :name
|
57
|
-
if tags.first =~ /^e.?mail/i # E-mail or Email
|
58
|
-
tags.shift
|
59
|
-
columns << :email
|
60
|
-
end
|
61
|
-
end
|
62
|
-
tags.each do |v|
|
63
|
-
v = v.to_s.gsub(/\0/,"")
|
64
|
-
columns << v.strip.to_sym unless v.blank?
|
65
|
-
end
|
66
|
-
columns
|
67
|
-
end
|
68
|
-
|
69
|
-
def pattern
|
70
|
-
@pattern ||= (options[:pattern] || DEFAULT_PATTERN) rescue DEFAULT_PATTERN
|
71
|
-
end
|
72
|
-
|
73
|
-
Graybook.register(:csv, self)
|
74
|
-
end
|
@@ -1,62 +0,0 @@
|
|
1
|
-
require 'graybook/importer/page_scraper'
|
2
|
-
|
3
|
-
class Graybook::Importer::Freenet < Graybook::Importer::PageScraper
|
4
|
-
LOGIN_URL = "https://office.freenet.de"
|
5
|
-
MESSAGES_URL = "/main_overview.html"
|
6
|
-
|
7
|
-
def =~( options )
|
8
|
-
options && options[:username] =~ /@freenet\.de$/i
|
9
|
-
end
|
10
|
-
|
11
|
-
def login
|
12
|
-
username, password = options[:username], options[:password]
|
13
|
-
|
14
|
-
begin
|
15
|
-
page = agent.get LOGIN_URL
|
16
|
-
|
17
|
-
form = page.forms.with.name("loginform").first
|
18
|
-
form.username = username.split("@").first
|
19
|
-
form.password = password
|
20
|
-
page = form.submit
|
21
|
-
|
22
|
-
if page.body.match(/Fehlgeschlagene Login-Versuche/)
|
23
|
-
page = page.forms.with.name("sicherform").submit
|
24
|
-
page = agent.get MESSAGES_URL
|
25
|
-
end
|
26
|
-
|
27
|
-
if page.body.match(/Login (erneut )?fehlgeschlagen/) || page.body.match(/Ich bin bereits Mitglied/)
|
28
|
-
raise Graybook::BadCredentialsError.new
|
29
|
-
end
|
30
|
-
rescue => e
|
31
|
-
raise e || Graybook::GraybookError.new
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
35
|
-
def prepare
|
36
|
-
login
|
37
|
-
end
|
38
|
-
|
39
|
-
def scrape_contacts
|
40
|
-
contacts = ["INBOX", "INBOX.sent"].map do |folder|
|
41
|
-
page = agent.get "/messages/mail_mailbox.html?mail_folder=#{folder}"
|
42
|
-
find_contacts(page)
|
43
|
-
end
|
44
|
-
|
45
|
-
contacts.inject([]) do |memo, contact|
|
46
|
-
memo << contact unless memo.include? contact
|
47
|
-
memo
|
48
|
-
end.flatten
|
49
|
-
end
|
50
|
-
|
51
|
-
|
52
|
-
protected
|
53
|
-
def find_contacts(page)
|
54
|
-
page.search("tr[@id]/td[9]").map do |cell|
|
55
|
-
recp, fullname, email = *cell.attributes["title"].match(/"(.*)"<(.*)>/)
|
56
|
-
|
57
|
-
{ :name => fullname, :email => email }
|
58
|
-
end
|
59
|
-
end
|
60
|
-
|
61
|
-
Graybook.register :freenet, self
|
62
|
-
end
|
@@ -1,68 +0,0 @@
|
|
1
|
-
require 'graybook/importer/page_scraper'
|
2
|
-
|
3
|
-
class Graybook::Importer::GMX < Graybook::Importer::PageScraper
|
4
|
-
LOGIN_URL = "https://www.gmx.net/"
|
5
|
-
|
6
|
-
def =~( options )
|
7
|
-
options && options[:username] =~ /@gmx\.(de|net)$/i
|
8
|
-
end
|
9
|
-
|
10
|
-
def login
|
11
|
-
username, password = options[:username], options[:password]
|
12
|
-
|
13
|
-
begin
|
14
|
-
page = agent.get LOGIN_URL
|
15
|
-
|
16
|
-
form = page.forms.with.name("login").first
|
17
|
-
form.id = username
|
18
|
-
form.p = password
|
19
|
-
page = form.submit
|
20
|
-
@next = page
|
21
|
-
|
22
|
-
if (page.at("div.index") && page.at("div.index").inner_html == "Passwort vergessen?") ||
|
23
|
-
page.uri.to_s.match(/login-failed/)
|
24
|
-
raise Graybook::BadCredentialsError.new
|
25
|
-
end
|
26
|
-
rescue => e
|
27
|
-
raise e || Graybook::GraybookError.new
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
|
-
def prepare
|
32
|
-
login
|
33
|
-
end
|
34
|
-
|
35
|
-
def scrape_contacts
|
36
|
-
page = @next
|
37
|
-
|
38
|
-
# Go to E-Mails
|
39
|
-
page = page.links.select { |link| link.text =~ /E-Mail/ }.first.click
|
40
|
-
|
41
|
-
contacts = [/Posteingang/, /Archiv/, /Gesendet/].map do |folder|
|
42
|
-
# puts "folder: #{folder}"
|
43
|
-
# puts "page title: #{page.title}"
|
44
|
-
# puts "page links: #{page.links.select { |link| link.text =~ folder }}"
|
45
|
-
page = page.links.select { |link| link.text =~ folder }.first.click
|
46
|
-
find_contacts(page)
|
47
|
-
end
|
48
|
-
|
49
|
-
contacts.inject([]) do |memo, contact|
|
50
|
-
memo << contact unless memo.include? contact
|
51
|
-
memo
|
52
|
-
end.flatten
|
53
|
-
end
|
54
|
-
|
55
|
-
protected
|
56
|
-
|
57
|
-
def find_contacts(page)
|
58
|
-
links = page.search("form#MI a").select { |link| link.attributes["title"] =~ /@/ }
|
59
|
-
links.map do |link|
|
60
|
-
recp = link.attributes["title"].gsub(/\n/, "").split(/\s/)
|
61
|
-
email = recp.pop.gsub(/[<>]/, "")
|
62
|
-
fullname = recp.join(" ")
|
63
|
-
|
64
|
-
{ :name => fullname, :email => email }
|
65
|
-
end
|
66
|
-
end
|
67
|
-
Graybook.register :gmx, self
|
68
|
-
end
|
@@ -1,67 +0,0 @@
|
|
1
|
-
require 'graybook/importer/page_scraper'
|
2
|
-
|
3
|
-
class Graybook::Importer::WebDE < Graybook::Importer::PageScraper
|
4
|
-
LOGIN_URL = "https://freemail.web.de"
|
5
|
-
|
6
|
-
def =~( options )
|
7
|
-
options && options[:username] =~ /@web\.de$/i
|
8
|
-
end
|
9
|
-
|
10
|
-
def login
|
11
|
-
username, password = options[:username], options[:password]
|
12
|
-
|
13
|
-
begin
|
14
|
-
page = agent.get LOGIN_URL
|
15
|
-
|
16
|
-
form = page.forms.with.name("login").first
|
17
|
-
form.username = username
|
18
|
-
form.password = password
|
19
|
-
|
20
|
-
page = form.submit
|
21
|
-
# follow logout hint
|
22
|
-
if (continue_link = page.links.select { |link| link.text =~ /weiter zu FreeMail/ }.first)
|
23
|
-
page = continue_link.click
|
24
|
-
end
|
25
|
-
|
26
|
-
if page.uri.to_s.match(/logonfailed/)
|
27
|
-
raise Graybook::BadCredentialsError.new
|
28
|
-
end
|
29
|
-
|
30
|
-
# follow content frame
|
31
|
-
@next = agent.get page.frames[1].src
|
32
|
-
rescue => e
|
33
|
-
raise e || Graybook::GraybookError.new
|
34
|
-
end
|
35
|
-
end
|
36
|
-
|
37
|
-
def prepare
|
38
|
-
login
|
39
|
-
end
|
40
|
-
|
41
|
-
def scrape_contacts
|
42
|
-
page = @next
|
43
|
-
|
44
|
-
contacts = [/Posteingang/, /Unbekannt/].map do |folder|
|
45
|
-
page = page.links.select { |link| link.text =~ folder }.first.click
|
46
|
-
find_contacts(page)
|
47
|
-
end
|
48
|
-
|
49
|
-
contacts.inject([]) do |memo, contact|
|
50
|
-
memo << contact unless memo.include? contact
|
51
|
-
memo
|
52
|
-
end
|
53
|
-
end
|
54
|
-
|
55
|
-
protected
|
56
|
-
def find_contacts(page)
|
57
|
-
page.search("span.from a").map do |link|
|
58
|
-
recp = link.attributes["title"].gsub(/\n/, "").split(/\s/)
|
59
|
-
email = recp.pop
|
60
|
-
fullname = recp.join(" ")
|
61
|
-
|
62
|
-
{ :name => fullname, :email => email }
|
63
|
-
end
|
64
|
-
end
|
65
|
-
|
66
|
-
Graybook.register :webde, self
|
67
|
-
end
|