graybook 1.0.22

data/.gitignore

@@ -0,0 +1 @@
+.DS_Store

data/CHANGES.md

@@ -0,0 +1,49 @@
+== 1.0.20 / 2009-09-30
+
+* lots of changes undocumented here.
+* putting on gemcutter.
+
+== 1.0.5 / 2008-11-26
+
+* moved to github
+* added patch for gmail. emails were coming in with notes. 
+* added german stuff to graybook gem
+
+== 1.0.4 / 2008-02-27
+
+* 1 minor enhancement
+  * fix 1.0.3 dud with AOL issues - Thank you Howard Wong!
+
+== 1.0.3 / 2008-02-27
+
+* 1 minor enhancement
+  * address the change in where AOL contacts are listed
+
+== 1.0.2 / 2008-02-20
+
+* 1 minor enhancement
+  * fixed sign on issue with GMail
+
+== 1.0.1 / 2008-02-08
+
+* 2 minor enhancement
+  * fix edge case sign on issue with GMail
+
+== 1.0.0 / 2008-01-30
+
+* 2 major enhancements
+  * Birthday!
+  * Mechanize patch for AOL sign-on! Thanks Mortee!
+
+* Importers
+  * AOL
+  * GMail
+  * Hotmail
+  * Yahoo!
+  * CSV
+
+* Exporters
+  * Hash
+  * XML
+  * VCard
+

data/Manifest.txt

@@ -0,0 +1,59 @@
+CHANGES.markdown
+Manifest.txt
+README.markdown
+Rakefile
+debug_graybook.rb
+init.rb
+lib/graybook.rb
+lib/graybook/exporter/base.rb
+lib/graybook/exporter/vcf.rb
+lib/graybook/exporter/xml.rb
+lib/graybook/importer/aol.rb
+lib/graybook/importer/base.rb
+lib/graybook/importer/csv.rb
+lib/graybook/importer/gmail.rb
+lib/graybook/importer/hotmail.rb
+lib/graybook/importer/page_scraper.rb
+lib/graybook/importer/yahoo.rb
+test/fixtures/aol_application_page.html
+test/fixtures/aol_bad_login_response_stage_3.html
+test/fixtures/aol_contacts.html
+test/fixtures/aol_login_response_stage_1.html
+test/fixtures/aol_login_response_stage_2.html
+test/fixtures/aol_login_response_stage_3.html
+test/fixtures/aol_login_response_stage_4.html
+test/fixtures/aol_login_response_stage_5.html
+test/fixtures/aol_new_contacts.html
+test/fixtures/gmail.csv
+test/fixtures/gmail_bad_login_response_stage_2.html
+test/fixtures/gmail_contacts.html
+test/fixtures/gmail_login_response_stage_1.html
+test/fixtures/gmail_login_response_stage_2.html
+test/fixtures/gmail_login_response_stage_2a.html
+test/fixtures/gmail_login_response_stage_3.html
+test/fixtures/hotmail_bad_login_response_stage_2.html
+test/fixtures/hotmail_contacts.html
+test/fixtures/hotmail_login_response_stage_1.html
+test/fixtures/hotmail_login_response_stage_2.html
+test/fixtures/hotmail_login_response_stage_3.html
+test/fixtures/yahoo_bad_login_response_stage_2.html
+test/fixtures/yahoo_contacts.csv
+test/fixtures/yahoo_contacts_not_logged_in.html
+test/fixtures/yahoo_contacts_stage_1.html
+test/fixtures/yahoo_login_response_stage_1.html
+test/fixtures/yahoo_login_response_stage_2.html
+test/scripts/live_test.rb
+test/test_graybook.rb
+test/test_graybook_exporter_base.rb
+test/test_graybook_exporter_vcf.rb
+test/test_graybook_exporter_xml.rb
+test/test_graybook_importer_aol.rb
+test/test_graybook_importer_base.rb
+test/test_graybook_importer_csv.rb
+test/test_graybook_importer_gmail.rb
+test/test_graybook_importer_hotmail.rb
+test/test_graybook_importer_page_scraper.rb
+test/test_graybook_importer_yahoo.rb
+test/test_helper.rb
+vendor/plugins/graybook/lib/autotest/graybook.rb
+vendor/plugins/graybook/lib/autotest/discover.rb

data/README.md

@@ -0,0 +1,66 @@
+Graybook
+
+== Description:
+  
+Pulls contact lists from EVERYWHERE! (Some sites not included.)
+
+== Features:
+  
+The current list of supported services and file types:
+
+import:
+- AOL
+- CSV files
+- Gmail
+- Hotmail
+- Yahoo! Mail
+
+export:
+- Simple hash (default)
+- JSON (soon)
+
+If you create an additional importer or exporter - or simply find a bug - please consider submitting it as a patch to the project so the community can all benefit from your hard work and ingenuity. 
+
+== Synopsis:
+
+# An example of fetching Gmail contacts - by default, returns an array of hashes with :name and :email
+  contacts = Graybook.get :username => 'me@gmail.com', :password => 'whatever'
+
+== Requirements:
+
+* Mechanize and its dependencies, for interacting with online providers
+* Fastercsv for reading CSV, Mechanize >= 0.7.0 for page scraping
+
+== Install:
+
+1. gem install gemcutter
+1. gem tumble
+2. gem install graybook
+
+== Thanks:
+
+Big thanks to Contentfree for making Blackbook.
+
+== License:
+
+Copyright (c) 2009, kojul.com
+Copyright (c) 2007, Contentfree
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,40 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'hoe'
+$LOAD_PATH.unshift 'lib'
+require 'graybook'
+
+begin
+  require 'rcov/rcovtask'
+rescue LoadError
+end
+
+begin
+  Rcov::RcovTask.new do |t|
+    t.test_files = FileList['test/test*.rb']
+    t.verbose = true
+    #t.rcov_opts << "--exclude rcov.rb,hpricot.rb,hpricot/.*\.rb"
+  end
+rescue NameError
+end
+
+require 'rubygems'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |s|
+    s.name        = "graybook"
+    s.summary     = "It does things. Like import contacts from EVERYWHERE."
+    s.email       = "kojul@kojul.com"
+    s.homepage    = "http://github.com/kojul/gray"
+    s.description = "It does things. Like import contacts from EVERYWHERE."
+    s.authors     = ["kojul"]
+    s.add_dependency('hpricot', '>= 0.7.0')
+    s.add_dependency('mechanize', '>= 0.7.0')
+    s.add_dependency('hoe', '>= 1.5.0')
+    s.add_dependency('fastercsv', '>= 1.2.0')
+  end
+rescue LoadError
+  puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+end

data/VERSION

@@ -0,0 +1 @@
+1.0.22

data/VERSION.yml

@@ -0,0 +1,4 @@
+--- 
+:major: 1
+:minor: 0
+:patch: 22

data/VERSION_NAME

@@ -0,0 +1 @@
+Gray Grenade

data/debug_graybook.rb

@@ -0,0 +1,10 @@
+require 'graybook'
+
+aol = Graybook.get :username => 'mondragonmichael@aol.com', :password => '123aol0'
+puts "===== aol #{aol.inspect}"
+
+unless aol.detect{|c| c[:name].downcase == 'Dave Myron'.downcase && c[:email] == 'dave.myron@contentfree.com'}
+  puts "AOL not working"
+  exit 1
+end
+

data/init.rb

@@ -0,0 +1 @@
+require File.dirname(__FILE__) + '/lib/graybook'

data/lib/graybook/exporter/base.rb

@@ -0,0 +1,16 @@
+##
+# base class for exporters of contact information
+
+module Graybook::Exporter
+
+  class Base
+    ##
+    # Override this to convert +contacts+ (an array of hashes) to something more useful. Here, it
+    # just returns Graybook's internal format
+    def export( contacts )
+      contacts
+    end
+    
+    Graybook.register :basic, self
+  end
+end

data/lib/graybook/exporter/vcf.rb

@@ -0,0 +1,45 @@
+##
+# exports contacts in Vcard format
+class Graybook::Exporter::Vcf < Graybook::Exporter::Base
+
+  ##
+  # representation of a vcard
+
+  class Vcard
+
+    attr_accessor :first, :last, :email
+    
+    ##
+    # initialize dynamically sets the attributes passed in as accessible 
+    # attribute on its object
+
+    def initialize( attributes = {} )
+      attributes.each{ |name,value| self.send("#{name}=", value) rescue next }
+    end
+    
+    ##
+    # text representation of this vcard
+    def to_s
+      <<-EOVC
+BEGIN:VCARD
+N:#{last};#{first}
+EMAIL:#{email}
+END:VCARD
+      EOVC
+    end
+  end
+
+  ##
+  # exports contacts as Vcards
+
+  def export( contacts = [] )
+    return if contacts.empty? 
+    
+    contacts.uniq.compact.collect do |contact|
+      first_name, last_name = contact[:name].split(' ', 2)
+      Vcard.new( :first => first_name.to_s, :last => last_name.to_s, :email => contact[:email])
+    end
+  end
+  
+  Graybook.register(:vcf, self)
+end

data/lib/graybook/exporter/xml.rb

@@ -0,0 +1,28 @@
+require 'rexml/document'
+
+##
+# exports contacts in xml format
+
+class Graybook::Exporter::Xml < Graybook::Exporter::Base
+
+  ##
+  # contacts are an array of hashes that are contacts and returns xml
+
+  def export( contacts )
+    doc = REXML::Document.new
+    doc << REXML::XMLDecl.new
+    
+    root = doc.add_element 'contacts'
+    contacts.each do |contact|
+      el = root.add_element 'contact'
+      name = el.add_element 'name' 
+      name.text = contact[:name]
+      
+      el.add_element('email').text = contact[:email]
+    end
+    
+    doc.to_s
+  end
+  
+  Graybook.register(:xml, self)
+end

data/lib/graybook/importer/aol.rb

@@ -0,0 +1,87 @@
+require 'graybook/importer/page_scraper'
+
+##
+# Imports contacts from AOL
+
+class Graybook::Importer::Aol < Graybook::Importer::PageScraper
+
+  ##
+  # Matches this importer to an user's name/address
+
+  def =~( options )
+    options && options[:username] =~ /@(aol|aim)\.com$/i ? true : false
+  end
+  
+  ##
+  # Login process:
+  # - Get mail.aol.com which redirects to a page containing a javascript redirect
+  # - Get the URL that the javascript is supposed to redirect you to
+  # - Fill out and submit the login form
+  # - Get the URL from *another* javascript redirect
+
+  def login
+    page = agent.get( 'http://webmail.aol.com/' )
+
+    form = page.forms.find{|form| form.name == 'AOLLoginForm'}
+    form.loginId = options[:username].split('@').first # Drop the domain
+    form.password = options[:password]
+    page = agent.submit(form, form.buttons.first)
+
+    case page.body
+    when /Invalid Screen Name or Password. Please try again./
+      raise( Graybook::BadCredentialsError, "That username and password was not accepted. Please check them and try again." )
+    when /Terms of Service/
+      raise( Graybook::LegacyAccount, "Your AOL account is not setup for WebMail. Please signup: http://webmail.aol.com")
+    end
+
+    # aol bumps to a wait page while logging in.  if we can't scrape out the js then its a bad login
+    extractor = proc { |var_name| page.body.scan(/var\s*#{var_name}\s*=\s*\"(.*?)\"\s*;/).first.first }
+      
+    base_uri = extractor.call( 'gSuccessPath' )
+    raise( Graybook::BadCredentialsError, "You do not appear to be signed in." ) unless base_uri
+    page = agent.get base_uri
+  end
+  
+  ##
+  # must login to prepare
+
+  def prepare
+    login
+  end
+  
+  ##
+  # The url to scrape contacts from has to be put together from the Auth cookie
+  # and a known uri that hosts their contact service. An array of hashes with
+  # :name and :email keys is returned.
+
+  def scrape_contacts    
+    unless auth_cookie = agent.cookies.find{|c| c.name =~ /^Auth/}
+      raise( Graybook::BadCredentialsError, "Must be authenticated to access contacts." )
+    end
+    
+    # jump through the hoops of formulating a request to get printable contacts
+    uri = agent.current_page.uri.dup
+    inputs = agent.current_page.search("//input")
+    user = inputs.detect{|i| i['type'] == 'hidden' && i['name'] == 'user'}
+    utoken = user['value']
+
+    path = uri.path.split('/')
+    path.pop
+    path << 'addresslist-print.aspx'
+    uri.path = path.join('/')
+    uri.query = "command=all&sort=FirstLastNick&sortDir=Ascending&nameFormat=FirstLastNick&user=#{utoken}"
+    page = agent.get uri.to_s
+
+    # Grab all the contacts
+    names = page.body.scan( /<span class="fullName">([^<]+)<\/span>/ ).flatten
+    emails = page.body.scan( /<span>Email 1:<\/span> <span>([^<]+)<\/span>/ ).flatten
+    (0...[names.size,emails.size].max).collect do |i|
+      {
+        :name => names[i],
+        :email => emails[i]
+      }
+    end
+  end
+  
+  Graybook.register :aol, self
+end

data/lib/graybook/importer/base.rb

@@ -0,0 +1,39 @@
+#
+# Provides a base template for interface and behavior of contact importers
+
+module Graybook::Importer
+  class Base
+    attr_accessor :options
+
+    ##
+    # Should return true or false/nil depending on whether the +options+ given 
+    # can be handled by this importer
+
+    def =~( options ); end # stub
+
+    ##
+    # Does the work of extracting contacts. Returns an Array of Arrays 
+    # containing the name and email as the first and second elements. Of 
+    # course, you can override this behavior to meet the needs of a 
+    # particular service.
+
+    def fetch_contacts!; end # stub
+
+    ##
+    # Imports the contacts using the given +options+. Returns an array of 
+    # hashes in the internal format (a hash with at least :name and :email 
+    # values).
+
+    def import(options = {}) 
+      self.options = options
+      fetch_contacts!
+    end
+
+    ##
+    # Name of the importer service.
+
+    def service_name
+      self.class.name.split("::").last
+    end
+  end
+end

data/lib/graybook/importer/csv.rb

@@ -0,0 +1,74 @@
+require 'fastercsv'
+
+##
+# Imports contacts from a CSV file
+
+class Graybook::Importer::Csv < Graybook::Importer::Base
+
+  DEFAULT_COLUMNS = [:name,:email,:misc]
+  DEFAULT_PATTERN = /,/
+
+  ##
+  # Matches this importer to a file that contains CSV values
+
+  def =~(options)
+    options && options[:file].respond_to?(:open) ? true : false
+  end
+
+  ##
+  # fetch_contacts! implementation for this importer
+
+  def fetch_contacts!
+    lines   = IO.readlines(options[:file].path)
+    columns = to_columns(lines.first)
+    lines.shift if columns.first == :name
+    columns = DEFAULT_COLUMNS.dup unless columns.first == :name
+
+    contacts = Array.new
+    lines.each do |l|
+      vals = l.split(pattern)
+      next if vals.empty?
+      vals.map! {|x| x.gsub( /\A"/m, "" ).strip.gsub( /"\Z/m, "" ) }
+      contacts << to_hash(columns, vals)
+    end
+
+    contacts
+  end
+
+  def to_hash(cols, vals) # :nodoc:
+    h = Hash.new
+    cols.each do |c|
+      h[c] = (c == cols.last) ? vals.join(',') : vals.shift
+    end
+    h
+  end
+
+  def to_columns(line) # :nodoc:
+    columns = Array.new
+    if line.match(pattern)
+      tags = line.split(pattern)
+    else
+      tags = line.split(DEFAULT_PATTERN)
+    end
+    # deal with "Name,E-mail..." oddity up front
+    if tags.first =~ /^name$/i
+      tags.shift
+      columns << :name
+      if tags.first =~ /^e.?mail/i # E-mail or Email
+        tags.shift
+        columns << :email
+      end
+    end
+    tags.each do |v|
+      v = v.to_s.gsub(/\0/,"")
+      columns << v.strip.to_sym unless v.blank?
+    end
+    columns
+  end
+  
+  def pattern
+    @pattern ||= (options[:pattern] || DEFAULT_PATTERN) rescue DEFAULT_PATTERN
+  end
+
+  Graybook.register(:csv, self)
+end

data/lib/graybook/importer/freenet.rb

@@ -0,0 +1,62 @@
+require 'graybook/importer/page_scraper'
+
+class Graybook::Importer::Freenet < Graybook::Importer::PageScraper
+  LOGIN_URL = "https://office.freenet.de"
+  MESSAGES_URL = "/main_overview.html"
+
+  def =~( options )
+    options && options[:username] =~ /@freenet\.de$/i
+  end
+
+  def login
+    username, password = options[:username], options[:password]
+
+    begin
+      page = agent.get LOGIN_URL
+
+      form = page.forms.with.name("loginform").first
+      form.username = username.split("@").first
+      form.password = password
+      page = form.submit
+
+      if page.body.match(/Fehlgeschlagene Login-Versuche/)
+        page = page.forms.with.name("sicherform").submit
+        page = agent.get MESSAGES_URL
+      end
+
+      if page.body.match(/Login (erneut )?fehlgeschlagen/) || page.body.match(/Ich bin bereits Mitglied/)
+        raise Graybook::BadCredentialsError.new
+      end
+    rescue => e
+      raise e || Graybook::GraybookError.new
+    end
+  end
+
+  def prepare
+    login
+  end
+
+  def scrape_contacts
+    contacts = ["INBOX", "INBOX.sent"].map do |folder|
+      page = agent.get "/messages/mail_mailbox.html?mail_folder=#{folder}"
+      find_contacts(page)
+    end
+
+    contacts.inject([]) do |memo, contact|
+      memo << contact unless memo.include? contact
+      memo
+    end.flatten
+  end
+
+
+  protected
+    def find_contacts(page)
+      page.search("tr[@id]/td[9]").map do |cell|
+        recp, fullname, email = *cell.attributes["title"].match(/"(.*)"<(.*)>/)
+
+        { :name => fullname, :email => email }
+      end
+    end
+
+    Graybook.register :freenet, self
+end

data/lib/graybook/importer/gmail.rb

@@ -0,0 +1,84 @@
+require 'graybook/importer/page_scraper'
+
+##
+# Imports contacts from GMail
+
+class Graybook::Importer::Gmail < Graybook::Importer::PageScraper
+
+  RETRY_THRESHOLD = 5
+  ##
+  # Matches this importer to an user's name/address
+  
+  def =~(options = {})
+    options && options[:username] =~ /@(gmail|googlemail).com$/i ? true : false
+  end
+  
+  def import(*args)
+    # GMail depends on Hpricot for some reason...
+    parser = WWW::Mechanize.html_parser
+    WWW::Mechanize.html_parser = Hpricot
+    return super do
+      WWW::Mechanize.html_parser = parser
+    end
+  end
+  
+  ##
+  # login to gmail
+
+  def login    
+    page = agent.get('http://mail.google.com/mail/')
+    form = page.forms.first
+    form.Email = options[:username]
+    form.Passwd = options[:password]
+    page = agent.submit(form,form.buttons.first)
+    
+    raise( Graybook::BadCredentialsError, "That username and password was not accepted. Please check them and try again." ) if page.body =~ /Username and password do not match/
+    
+    if page.search('//meta').first.attributes['content'] =~ /url='?(http.+?)'?$/i
+      page = agent.get $1
+    end
+  end
+  
+  ##
+  # prepare this importer
+
+  def prepare
+    login
+  end
+  
+  ##
+  # scrape gmail contacts for this importer
+
+  def scrape_contacts    
+    unless agent.cookies.find { |c| 
+        c.name == 'GAUSR' && c.value.match(/mail(.*?):#{options[:username]}/) 
+      }
+      raise( Graybook::BadCredentialsError, "Must be authenticated to access contacts." )
+    end
+    
+    page = agent.get('http://mail.google.com/mail/h/?v=cl&pnl=a')
+    title = page.search("//title").inner_text
+    if title == 'Redirecting'
+      redirect_text = page.search('//meta').first.attributes['content'].inner_text
+      url = redirect_text.match(/url='(http.*)'/i)[1]
+      page = agent.get(url)
+    end
+    
+    contact_rows = page.search("//input[@name='c']/../..")
+    contact_rows.collect do |row|
+      columns = row/"td"
+      email = columns[2].inner_html.gsub( /(\n|&nbsp;)/, '' ) # email
+      clean_email = email[/[a-zA-Z0-9._%+-]+@(?:[a-zA-Z0-9-]+\.)+[a-zA-Z]{2,4}/] 
+      
+      unless clean_email.empty?
+        columns = row/"td"
+        { 
+          :name  => ( columns[1] / "b" / "a" ).inner_html.strip, # name
+          :email => clean_email
+        } 
+      end
+    end.compact
+  end
+  
+  Graybook.register(:gmail, self)
+end