graybook 1.0.24 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/{CHANGES.md → CHANGELOG} +13 -2
- data/README +94 -0
- data/Rakefile +3 -22
- data/VERSION +1 -1
- data/VERSION.yml +2 -2
- data/VERSION_NAME +1 -1
- data/lib/graybook/importer/aol.rb +11 -11
- data/lib/graybook/importer/gmail.rb +20 -20
- data/lib/graybook/importer/hotmail.rb +2 -4
- data/lib/graybook/importer/page_scraper.rb +4 -2
- data/lib/graybook/importer/yahoo.rb +12 -12
- data/lib/graybook.rb +20 -21
- metadata +9 -102
- data/README.md +0 -66
- data/debug_graybook.rb +0 -10
- data/lib/graybook/exporter/vcf.rb +0 -45
- data/lib/graybook/exporter/xml.rb +0 -28
- data/lib/graybook/importer/csv.rb +0 -74
- data/lib/graybook/importer/freenet.rb +0 -62
- data/lib/graybook/importer/gmx.rb +0 -68
- data/lib/graybook/importer/web.de.rb +0 -67
- data/test/config/credentials.yml.example +0 -9
- data/test/fixtures/aol_application_page.html +0 -566
- data/test/fixtures/aol_bad_login_response_stage_3.html +0 -565
- data/test/fixtures/aol_contacts.html +0 -102
- data/test/fixtures/aol_login_response_stage_1.html +0 -158
- data/test/fixtures/aol_login_response_stage_2.html +0 -559
- data/test/fixtures/aol_login_response_stage_3.html +0 -48
- data/test/fixtures/aol_login_response_stage_4.html +0 -404
- data/test/fixtures/aol_login_response_stage_5.html +0 -404
- data/test/fixtures/aol_new_contacts.html +0 -431
- data/test/fixtures/gmail.csv +0 -3
- data/test/fixtures/gmail_bad_login_response_stage_2.html +0 -560
- data/test/fixtures/gmail_contacts.html +0 -228
- data/test/fixtures/gmail_login_response_stage_1.html +0 -556
- data/test/fixtures/gmail_login_response_stage_2.html +0 -1
- data/test/fixtures/gmail_login_response_stage_2a.html +0 -1
- data/test/fixtures/gmail_login_response_stage_3.html +0 -249
- data/test/fixtures/gmail_redirect_body.html +0 -10
- data/test/fixtures/hotmail_bad_login_response_stage_2.html +0 -31
- data/test/fixtures/hotmail_contacts.html +0 -262
- data/test/fixtures/hotmail_login_response_stage_1.html +0 -31
- data/test/fixtures/hotmail_login_response_stage_2.html +0 -1
- data/test/fixtures/hotmail_login_response_stage_3.html +0 -519
- data/test/fixtures/hotmail_scrape_first_page.html +0 -77
- data/test/fixtures/hotmail_scrape_response_stage_1.html +0 -90
- data/test/fixtures/hotmail_scrape_response_stage_2.html +0 -77
- data/test/fixtures/hotmail_scrape_response_stage_3.html +0 -0
- data/test/fixtures/yahoo_bad_login_response_stage_2.html +0 -443
- data/test/fixtures/yahoo_contacts.csv +0 -3
- data/test/fixtures/yahoo_contacts_not_logged_in.html +0 -432
- data/test/fixtures/yahoo_contacts_stage_1.html +0 -399
- data/test/fixtures/yahoo_login_response_stage_1.html +0 -433
- data/test/fixtures/yahoo_login_response_stage_2.html +0 -16
- data/test/fixtures/yahoo_no_user_response_stage_2.html +0 -574
- data/test/freenet_importer_test.rb +0 -53
- data/test/gmx_importer_test.rb +0 -53
- data/test/scripts/live_test.rb +0 -25
- data/test/test_graybook.rb +0 -60
- data/test/test_graybook_exporter_base.rb +0 -16
- data/test/test_graybook_exporter_vcf.rb +0 -52
- data/test/test_graybook_exporter_xml.rb +0 -16
- data/test/test_graybook_importer_aol.rb +0 -108
- data/test/test_graybook_importer_base.rb +0 -24
- data/test/test_graybook_importer_csv.rb +0 -60
- data/test/test_graybook_importer_gmail.rb +0 -116
- data/test/test_graybook_importer_hotmail.rb +0 -165
- data/test/test_graybook_importer_page_scraper.rb +0 -51
- data/test/test_graybook_importer_yahoo.rb +0 -137
- data/test/test_helper.rb +0 -71
- data/test/web.de_importer_test.rb +0 -53
- data/vendor/plugins/graybook/lib/autotest/discover.rb +0 -3
- data/vendor/plugins/graybook/lib/autotest/graybook.rb +0 -27
data/README.md
DELETED
@@ -1,66 +0,0 @@
|
|
1
|
-
Graybook
|
2
|
-
|
3
|
-
== Description:
|
4
|
-
|
5
|
-
Pulls contact lists from EVERYWHERE! (Some sites not included.)
|
6
|
-
|
7
|
-
== Features:
|
8
|
-
|
9
|
-
The current list of supported services and file types:
|
10
|
-
|
11
|
-
import:
|
12
|
-
- AOL
|
13
|
-
- CSV files
|
14
|
-
- Gmail
|
15
|
-
- Hotmail
|
16
|
-
- Yahoo! Mail
|
17
|
-
|
18
|
-
export:
|
19
|
-
- Simple hash (default)
|
20
|
-
- JSON (soon)
|
21
|
-
|
22
|
-
If you create an additional importer or exporter - or simply find a bug - please consider submitting it as a patch to the project so the community can all benefit from your hard work and ingenuity.
|
23
|
-
|
24
|
-
== Synopsis:
|
25
|
-
|
26
|
-
# An example of fetching Gmail contacts - by default, returns an array of hashes with :name and :email
|
27
|
-
contacts = Graybook.get :username => 'me@gmail.com', :password => 'whatever'
|
28
|
-
|
29
|
-
== Requirements:
|
30
|
-
|
31
|
-
* Mechanize and its dependencies, for interacting with online providers
|
32
|
-
* Fastercsv for reading CSV, Mechanize >= 0.7.0 for page scraping
|
33
|
-
|
34
|
-
== Install:
|
35
|
-
|
36
|
-
1. gem install gemcutter
|
37
|
-
1. gem tumble
|
38
|
-
2. gem install graybook
|
39
|
-
|
40
|
-
== Thanks:
|
41
|
-
|
42
|
-
Big thanks to Contentfree for making Blackbook.
|
43
|
-
|
44
|
-
== License:
|
45
|
-
|
46
|
-
Copyright (c) 2009, kojul.com
|
47
|
-
Copyright (c) 2007, Contentfree
|
48
|
-
|
49
|
-
Permission is hereby granted, free of charge, to any person obtaining
|
50
|
-
a copy of this software and associated documentation files (the
|
51
|
-
'Software'), to deal in the Software without restriction, including
|
52
|
-
without limitation the rights to use, copy, modify, merge, publish,
|
53
|
-
distribute, sublicense, and/or sell copies of the Software, and to
|
54
|
-
permit persons to whom the Software is furnished to do so, subject to
|
55
|
-
the following conditions:
|
56
|
-
|
57
|
-
The above copyright notice and this permission notice shall be
|
58
|
-
included in all copies or substantial portions of the Software.
|
59
|
-
|
60
|
-
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
|
61
|
-
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
62
|
-
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
63
|
-
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
64
|
-
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
65
|
-
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
66
|
-
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/debug_graybook.rb
DELETED
@@ -1,10 +0,0 @@
|
|
1
|
-
require 'graybook'
|
2
|
-
|
3
|
-
aol = Graybook.get :username => 'mondragonmichael@aol.com', :password => '123aol0'
|
4
|
-
puts "===== aol #{aol.inspect}"
|
5
|
-
|
6
|
-
unless aol.detect{|c| c[:name].downcase == 'Dave Myron'.downcase && c[:email] == 'dave.myron@contentfree.com'}
|
7
|
-
puts "AOL not working"
|
8
|
-
exit 1
|
9
|
-
end
|
10
|
-
|
@@ -1,45 +0,0 @@
|
|
1
|
-
##
|
2
|
-
# exports contacts in Vcard format
|
3
|
-
class Graybook::Exporter::Vcf < Graybook::Exporter::Base
|
4
|
-
|
5
|
-
##
|
6
|
-
# representation of a vcard
|
7
|
-
|
8
|
-
class Vcard
|
9
|
-
|
10
|
-
attr_accessor :first, :last, :email
|
11
|
-
|
12
|
-
##
|
13
|
-
# initialize dynamically sets the attributes passed in as accessible
|
14
|
-
# attribute on its object
|
15
|
-
|
16
|
-
def initialize( attributes = {} )
|
17
|
-
attributes.each{ |name,value| self.send("#{name}=", value) rescue next }
|
18
|
-
end
|
19
|
-
|
20
|
-
##
|
21
|
-
# text representation of this vcard
|
22
|
-
def to_s
|
23
|
-
<<-EOVC
|
24
|
-
BEGIN:VCARD
|
25
|
-
N:#{last};#{first}
|
26
|
-
EMAIL:#{email}
|
27
|
-
END:VCARD
|
28
|
-
EOVC
|
29
|
-
end
|
30
|
-
end
|
31
|
-
|
32
|
-
##
|
33
|
-
# exports contacts as Vcards
|
34
|
-
|
35
|
-
def export( contacts = [] )
|
36
|
-
return if contacts.empty?
|
37
|
-
|
38
|
-
contacts.uniq.compact.collect do |contact|
|
39
|
-
first_name, last_name = contact[:name].split(' ', 2)
|
40
|
-
Vcard.new( :first => first_name.to_s, :last => last_name.to_s, :email => contact[:email])
|
41
|
-
end
|
42
|
-
end
|
43
|
-
|
44
|
-
Graybook.register(:vcf, self)
|
45
|
-
end
|
@@ -1,28 +0,0 @@
|
|
1
|
-
require 'rexml/document'
|
2
|
-
|
3
|
-
##
|
4
|
-
# exports contacts in xml format
|
5
|
-
|
6
|
-
class Graybook::Exporter::Xml < Graybook::Exporter::Base
|
7
|
-
|
8
|
-
##
|
9
|
-
# contacts are an array of hashes that are contacts and returns xml
|
10
|
-
|
11
|
-
def export( contacts )
|
12
|
-
doc = REXML::Document.new
|
13
|
-
doc << REXML::XMLDecl.new
|
14
|
-
|
15
|
-
root = doc.add_element 'contacts'
|
16
|
-
contacts.each do |contact|
|
17
|
-
el = root.add_element 'contact'
|
18
|
-
name = el.add_element 'name'
|
19
|
-
name.text = contact[:name]
|
20
|
-
|
21
|
-
el.add_element('email').text = contact[:email]
|
22
|
-
end
|
23
|
-
|
24
|
-
doc.to_s
|
25
|
-
end
|
26
|
-
|
27
|
-
Graybook.register(:xml, self)
|
28
|
-
end
|
@@ -1,74 +0,0 @@
|
|
1
|
-
require 'fastercsv'
|
2
|
-
|
3
|
-
##
|
4
|
-
# Imports contacts from a CSV file
|
5
|
-
|
6
|
-
class Graybook::Importer::Csv < Graybook::Importer::Base
|
7
|
-
|
8
|
-
DEFAULT_COLUMNS = [:name,:email,:misc]
|
9
|
-
DEFAULT_PATTERN = /,/
|
10
|
-
|
11
|
-
##
|
12
|
-
# Matches this importer to a file that contains CSV values
|
13
|
-
|
14
|
-
def =~(options)
|
15
|
-
options && options[:file].respond_to?(:open) ? true : false
|
16
|
-
end
|
17
|
-
|
18
|
-
##
|
19
|
-
# fetch_contacts! implementation for this importer
|
20
|
-
|
21
|
-
def fetch_contacts!
|
22
|
-
lines = IO.readlines(options[:file].path)
|
23
|
-
columns = to_columns(lines.first)
|
24
|
-
lines.shift if columns.first == :name
|
25
|
-
columns = DEFAULT_COLUMNS.dup unless columns.first == :name
|
26
|
-
|
27
|
-
contacts = Array.new
|
28
|
-
lines.each do |l|
|
29
|
-
vals = l.split(pattern)
|
30
|
-
next if vals.empty?
|
31
|
-
vals.map! {|x| x.gsub( /\A"/m, "" ).strip.gsub( /"\Z/m, "" ) }
|
32
|
-
contacts << to_hash(columns, vals)
|
33
|
-
end
|
34
|
-
|
35
|
-
contacts
|
36
|
-
end
|
37
|
-
|
38
|
-
def to_hash(cols, vals) # :nodoc:
|
39
|
-
h = Hash.new
|
40
|
-
cols.each do |c|
|
41
|
-
h[c] = (c == cols.last) ? vals.join(',') : vals.shift
|
42
|
-
end
|
43
|
-
h
|
44
|
-
end
|
45
|
-
|
46
|
-
def to_columns(line) # :nodoc:
|
47
|
-
columns = Array.new
|
48
|
-
if line.match(pattern)
|
49
|
-
tags = line.split(pattern)
|
50
|
-
else
|
51
|
-
tags = line.split(DEFAULT_PATTERN)
|
52
|
-
end
|
53
|
-
# deal with "Name,E-mail..." oddity up front
|
54
|
-
if tags.first =~ /^name$/i
|
55
|
-
tags.shift
|
56
|
-
columns << :name
|
57
|
-
if tags.first =~ /^e.?mail/i # E-mail or Email
|
58
|
-
tags.shift
|
59
|
-
columns << :email
|
60
|
-
end
|
61
|
-
end
|
62
|
-
tags.each do |v|
|
63
|
-
v = v.to_s.gsub(/\0/,"")
|
64
|
-
columns << v.strip.to_sym unless v.blank?
|
65
|
-
end
|
66
|
-
columns
|
67
|
-
end
|
68
|
-
|
69
|
-
def pattern
|
70
|
-
@pattern ||= (options[:pattern] || DEFAULT_PATTERN) rescue DEFAULT_PATTERN
|
71
|
-
end
|
72
|
-
|
73
|
-
Graybook.register(:csv, self)
|
74
|
-
end
|
@@ -1,62 +0,0 @@
|
|
1
|
-
require 'graybook/importer/page_scraper'
|
2
|
-
|
3
|
-
class Graybook::Importer::Freenet < Graybook::Importer::PageScraper
|
4
|
-
LOGIN_URL = "https://office.freenet.de"
|
5
|
-
MESSAGES_URL = "/main_overview.html"
|
6
|
-
|
7
|
-
def =~( options )
|
8
|
-
options && options[:username] =~ /@freenet\.de$/i
|
9
|
-
end
|
10
|
-
|
11
|
-
def login
|
12
|
-
username, password = options[:username], options[:password]
|
13
|
-
|
14
|
-
begin
|
15
|
-
page = agent.get LOGIN_URL
|
16
|
-
|
17
|
-
form = page.forms.with.name("loginform").first
|
18
|
-
form.username = username.split("@").first
|
19
|
-
form.password = password
|
20
|
-
page = form.submit
|
21
|
-
|
22
|
-
if page.body.match(/Fehlgeschlagene Login-Versuche/)
|
23
|
-
page = page.forms.with.name("sicherform").submit
|
24
|
-
page = agent.get MESSAGES_URL
|
25
|
-
end
|
26
|
-
|
27
|
-
if page.body.match(/Login (erneut )?fehlgeschlagen/) || page.body.match(/Ich bin bereits Mitglied/)
|
28
|
-
raise Graybook::BadCredentialsError.new
|
29
|
-
end
|
30
|
-
rescue => e
|
31
|
-
raise e || Graybook::GraybookError.new
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
35
|
-
def prepare
|
36
|
-
login
|
37
|
-
end
|
38
|
-
|
39
|
-
def scrape_contacts
|
40
|
-
contacts = ["INBOX", "INBOX.sent"].map do |folder|
|
41
|
-
page = agent.get "/messages/mail_mailbox.html?mail_folder=#{folder}"
|
42
|
-
find_contacts(page)
|
43
|
-
end
|
44
|
-
|
45
|
-
contacts.inject([]) do |memo, contact|
|
46
|
-
memo << contact unless memo.include? contact
|
47
|
-
memo
|
48
|
-
end.flatten
|
49
|
-
end
|
50
|
-
|
51
|
-
|
52
|
-
protected
|
53
|
-
def find_contacts(page)
|
54
|
-
page.search("tr[@id]/td[9]").map do |cell|
|
55
|
-
recp, fullname, email = *cell.attributes["title"].match(/"(.*)"<(.*)>/)
|
56
|
-
|
57
|
-
{ :name => fullname, :email => email }
|
58
|
-
end
|
59
|
-
end
|
60
|
-
|
61
|
-
Graybook.register :freenet, self
|
62
|
-
end
|
@@ -1,68 +0,0 @@
|
|
1
|
-
require 'graybook/importer/page_scraper'
|
2
|
-
|
3
|
-
class Graybook::Importer::GMX < Graybook::Importer::PageScraper
|
4
|
-
LOGIN_URL = "https://www.gmx.net/"
|
5
|
-
|
6
|
-
def =~( options )
|
7
|
-
options && options[:username] =~ /@gmx\.(de|net)$/i
|
8
|
-
end
|
9
|
-
|
10
|
-
def login
|
11
|
-
username, password = options[:username], options[:password]
|
12
|
-
|
13
|
-
begin
|
14
|
-
page = agent.get LOGIN_URL
|
15
|
-
|
16
|
-
form = page.forms.with.name("login").first
|
17
|
-
form.id = username
|
18
|
-
form.p = password
|
19
|
-
page = form.submit
|
20
|
-
@next = page
|
21
|
-
|
22
|
-
if (page.at("div.index") && page.at("div.index").inner_html == "Passwort vergessen?") ||
|
23
|
-
page.uri.to_s.match(/login-failed/)
|
24
|
-
raise Graybook::BadCredentialsError.new
|
25
|
-
end
|
26
|
-
rescue => e
|
27
|
-
raise e || Graybook::GraybookError.new
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
|
-
def prepare
|
32
|
-
login
|
33
|
-
end
|
34
|
-
|
35
|
-
def scrape_contacts
|
36
|
-
page = @next
|
37
|
-
|
38
|
-
# Go to E-Mails
|
39
|
-
page = page.links.select { |link| link.text =~ /E-Mail/ }.first.click
|
40
|
-
|
41
|
-
contacts = [/Posteingang/, /Archiv/, /Gesendet/].map do |folder|
|
42
|
-
# puts "folder: #{folder}"
|
43
|
-
# puts "page title: #{page.title}"
|
44
|
-
# puts "page links: #{page.links.select { |link| link.text =~ folder }}"
|
45
|
-
page = page.links.select { |link| link.text =~ folder }.first.click
|
46
|
-
find_contacts(page)
|
47
|
-
end
|
48
|
-
|
49
|
-
contacts.inject([]) do |memo, contact|
|
50
|
-
memo << contact unless memo.include? contact
|
51
|
-
memo
|
52
|
-
end.flatten
|
53
|
-
end
|
54
|
-
|
55
|
-
protected
|
56
|
-
|
57
|
-
def find_contacts(page)
|
58
|
-
links = page.search("form#MI a").select { |link| link.attributes["title"] =~ /@/ }
|
59
|
-
links.map do |link|
|
60
|
-
recp = link.attributes["title"].gsub(/\n/, "").split(/\s/)
|
61
|
-
email = recp.pop.gsub(/[<>]/, "")
|
62
|
-
fullname = recp.join(" ")
|
63
|
-
|
64
|
-
{ :name => fullname, :email => email }
|
65
|
-
end
|
66
|
-
end
|
67
|
-
Graybook.register :gmx, self
|
68
|
-
end
|
@@ -1,67 +0,0 @@
|
|
1
|
-
require 'graybook/importer/page_scraper'
|
2
|
-
|
3
|
-
class Graybook::Importer::WebDE < Graybook::Importer::PageScraper
|
4
|
-
LOGIN_URL = "https://freemail.web.de"
|
5
|
-
|
6
|
-
def =~( options )
|
7
|
-
options && options[:username] =~ /@web\.de$/i
|
8
|
-
end
|
9
|
-
|
10
|
-
def login
|
11
|
-
username, password = options[:username], options[:password]
|
12
|
-
|
13
|
-
begin
|
14
|
-
page = agent.get LOGIN_URL
|
15
|
-
|
16
|
-
form = page.forms.with.name("login").first
|
17
|
-
form.username = username
|
18
|
-
form.password = password
|
19
|
-
|
20
|
-
page = form.submit
|
21
|
-
# follow logout hint
|
22
|
-
if (continue_link = page.links.select { |link| link.text =~ /weiter zu FreeMail/ }.first)
|
23
|
-
page = continue_link.click
|
24
|
-
end
|
25
|
-
|
26
|
-
if page.uri.to_s.match(/logonfailed/)
|
27
|
-
raise Graybook::BadCredentialsError.new
|
28
|
-
end
|
29
|
-
|
30
|
-
# follow content frame
|
31
|
-
@next = agent.get page.frames[1].src
|
32
|
-
rescue => e
|
33
|
-
raise e || Graybook::GraybookError.new
|
34
|
-
end
|
35
|
-
end
|
36
|
-
|
37
|
-
def prepare
|
38
|
-
login
|
39
|
-
end
|
40
|
-
|
41
|
-
def scrape_contacts
|
42
|
-
page = @next
|
43
|
-
|
44
|
-
contacts = [/Posteingang/, /Unbekannt/].map do |folder|
|
45
|
-
page = page.links.select { |link| link.text =~ folder }.first.click
|
46
|
-
find_contacts(page)
|
47
|
-
end
|
48
|
-
|
49
|
-
contacts.inject([]) do |memo, contact|
|
50
|
-
memo << contact unless memo.include? contact
|
51
|
-
memo
|
52
|
-
end
|
53
|
-
end
|
54
|
-
|
55
|
-
protected
|
56
|
-
def find_contacts(page)
|
57
|
-
page.search("span.from a").map do |link|
|
58
|
-
recp = link.attributes["title"].gsub(/\n/, "").split(/\s/)
|
59
|
-
email = recp.pop
|
60
|
-
fullname = recp.join(" ")
|
61
|
-
|
62
|
-
{ :name => fullname, :email => email }
|
63
|
-
end
|
64
|
-
end
|
65
|
-
|
66
|
-
Graybook.register :webde, self
|
67
|
-
end
|