graybook 1.0.22

data/lib/graybook/importer/gmx.rb

@@ -0,0 +1,68 @@
+require 'graybook/importer/page_scraper'
+
+class Graybook::Importer::GMX < Graybook::Importer::PageScraper
+  LOGIN_URL = "https://www.gmx.net/"
+
+  def =~( options )
+    options && options[:username] =~ /@gmx\.(de|net)$/i
+  end
+
+  def login
+    username, password = options[:username], options[:password]
+
+    begin
+      page = agent.get LOGIN_URL
+
+      form = page.forms.with.name("login").first
+      form.id = username
+      form.p = password
+      page = form.submit
+      @next = page
+
+      if (page.at("div.index") && page.at("div.index").inner_html == "Passwort vergessen?") ||
+        page.uri.to_s.match(/login-failed/)
+        raise Graybook::BadCredentialsError.new
+      end
+    rescue => e
+      raise e || Graybook::GraybookError.new
+    end
+  end
+
+  def prepare
+    login
+  end
+
+  def scrape_contacts
+    page = @next
+
+    # Go to E-Mails
+    page = page.links.select { |link| link.text =~ /E-Mail/ }.first.click
+
+    contacts = [/Posteingang/, /Archiv/, /Gesendet/].map do |folder|
+      # puts "folder: #{folder}"
+      # puts "page title: #{page.title}"
+      # puts "page links: #{page.links.select { |link| link.text =~ folder }}"
+      page = page.links.select { |link| link.text =~ folder }.first.click
+      find_contacts(page)
+    end
+
+    contacts.inject([]) do |memo, contact|
+      memo << contact unless memo.include? contact
+      memo
+    end.flatten
+  end
+
+  protected
+
+    def find_contacts(page)
+      links = page.search("form#MI a").select { |link| link.attributes["title"] =~ /@/ }
+      links.map do |link|
+        recp = link.attributes["title"].gsub(/\n/, "").split(/\s/)
+        email = recp.pop.gsub(/[<>]/, "")
+        fullname = recp.join(" ")
+
+        { :name => fullname, :email => email }
+      end
+    end
+    Graybook.register :gmx, self
+end

data/lib/graybook/importer/hotmail.rb

@@ -0,0 +1,135 @@
+require 'graybook/importer/page_scraper'
+require 'cgi'
+
+##
+# imports contacts for MSN/Hotmail
+class Graybook::Importer::Hotmail < Graybook::Importer::PageScraper
+
+  DOMAINS = { "compaq.net"        => "https://msnia.login.live.com/ppsecure/post.srf",
+              "hotmail.co.jp"     => "https://login.live.com/ppsecure/post.srf",
+              "hotmail.co.uk"     => "https://login.live.com/ppsecure/post.srf",
+              "hotmail.com"       => "https://login.live.com/ppsecure/post.srf",
+              "hotmail.de"        => "https://login.live.com/ppsecure/post.srf",
+              "hotmail.fr"        => "https://login.live.com/ppsecure/post.srf",
+              "hotmail.it"        => "https://login.live.com/ppsecure/post.srf",
+              "live.com"          => "https://login.live.com/ppsecure/post.srf",
+              "messengeruser.com" => "https://login.live.com/ppsecure/post.srf",
+              "msn.com"           => "https://msnia.login.live.com/ppsecure/post.srf",
+              "passport.com"      => "https://login.live.com/ppsecure/post.srf",
+              "webtv.net"         => "https://login.live.com/ppsecure/post.srf" }
+              
+  ##
+  # Matches this importer to an user's name/address
+
+  def =~(options)
+    return false unless options && options[:username]
+    domain = username_domain(options[:username].downcase)
+    !domain.empty? && DOMAINS.keys.include?( domain ) ? true : false
+  end
+   
+  ##
+  # Login procedure
+  # 1. Go to login form
+  # 2. Set login and passwd
+  # 3. Set PwdPad to IfYouAreReadingThisYouHaveTooMuchFreeTime minus however many characters are in passwd (so if passwd
+  #    was 8 chars, you'd chop 8 chars of the end of IfYouAreReadingThisYouHaveTooMuchFreeTime - giving you IfYouAreReadingThisYouHaveTooMuch)
+  # 4. Set the action to the appropriate URL for the username's domain
+  # 5. Get the query string to append to the new action
+  # 5. Submit the form and parse the url from the resulting page's javascript
+  # 6. Go to that url
+
+  def login
+    page = agent.get('http://login.live.com/login.srf?id=2')
+    form = page.forms.first
+    form.login   = options[:username]
+    form.passwd  = options[:password]
+    form.PwdPad  = ( "IfYouAreReadingThisYouHaveTooMuchFreeTime"[0..(-1 - options[:password].to_s.size )])
+    query_string = page.body.scan(/g_QS="([^"]+)/).first.first rescue nil
+    form.action  = login_url + "?#{query_string.to_s}"
+    page = agent.submit(form)
+    
+    # Check for login success
+    if page.body =~ /The e-mail address or password is incorrect/ ||
+      page.body =~ /Sign in failed\./
+      raise( Graybook::BadCredentialsError, 
+        "That username and password was not accepted. Please check them and try again." )
+    end
+    
+    @first_page = agent.get( page.body.scan(/http\:\/\/[^"]+/).first )
+  end
+  
+  ##
+  # prepare this importer
+
+  def prepare
+    login
+  end
+  
+  ##
+  # Scrape contacts for Hotmail
+  # Seems like a POST to directly fetch CSV contacts from options.aspx?subsection=26&n=
+  # raises an end of file error in Net::HTTP via Mechanize.
+  # Seems like Hotmail addresses are now hosted on Windows Live.
+
+  def scrape_contacts
+    unless agent.cookies.find{|c| c.name == 'MSPPre' && c.value == options[:username]}
+      raise( Graybook::BadCredentialsError, "Must be authenticated to access contacts." )
+    end
+
+    page = agent.get('http://mail.live.com/')
+    
+    if page.iframes.detect { |f| f.src =~ /\/mail\/TodayLight.aspx/ }
+      page = agent.get( page.iframes.first.src )
+      
+      if button = page.forms.first.buttons.detect { |b| b.name == 'TakeMeToInbox' }
+        page = agent.submit( page.forms.first, button )
+      end
+    end
+    
+    page = page.link_with(:text => 'Contact list').click
+    
+    contacts = parse_contacts(page.body)
+    while link = page.link_with(:text => 'Next page')
+      page = link.click
+      contacts += parse_contacts(page.body)
+    end
+    
+    contacts
+  end
+  
+  def parse_contacts(source)
+    source.scan(/ICc.*\:\[.*?,.*?,\['ct'\],'(.*?)',.*?,.*?,'(.*?)',.*\]/).collect do |name, email|
+      { :name => (name =~ /\\x26\\x2364\\x3b/ ? nil : name), :email => email.gsub(/\\x40/, '@') }
+    end
+  end
+  
+  ##
+  # lookup for the login service that should be used based on the user's
+  # address
+
+  def login_url
+    DOMAINS[username_domain] || DOMAINS['hotmail.com']
+  end
+  
+
+  ##
+  # normalizes the host for the page that is currently being "viewed" by the
+  # Mechanize agent
+
+  def current_host
+    return nil unless agent && agent.current_page
+    uri = agent.current_page.uri
+    "#{uri.scheme}://#{uri.host}"
+  end
+  
+  ##
+  # determines the domain for the user
+
+  def username_domain(username = nil)
+    username ||= options[:username] if options
+    return unless username
+    username.to_s.split('@').last
+  end
+  
+  Graybook.register(:hotmail, self)
+end

data/lib/graybook/importer/page_scraper.rb

@@ -0,0 +1,86 @@
+require 'rubygems'
+gem 'mechanize', '>= 0.7.0'
+require 'mechanize'
+require 'generator' # for SyncEnumerator
+
+# Patch Mechanize's broken html unescaping Mechanize 0.6.11
+class WWW::Mechanize
+  def to_absolute_uri(url, cur_page=current_page())
+    unless url.is_a? URI
+      url = url.to_s.strip
+      url = URI.parse(
+              Util.html_unescape(
+                SyncEnumerator.new(
+                  url.split(/%[0-9A-Fa-f]{2}/), url.scan(/%[0-9A-Fa-f]{2}/)
+                ).map { |x,y|
+                  "#{URI.escape(x||'')}#{y}"
+                }.join('').gsub(/%23/, '#')
+              )
+            )
+      # Mechanize here uses #zip to combine the two arrays, which will ignore 
+      # excessive elements of the second array (the one which is passed as an 
+      # argument). That means if the URL ends with more than one already escaped
+      # character, then only the first one will be restored into the resulting
+      # URL.
+    end
+
+    # construct an absolute uri
+    if url.relative?
+      raise 'no history. please specify an absolute URL' unless cur_page.uri
+      url = cur_page.uri + url
+      # Strip initial "/.." bits from the path
+      url.path.sub!(/^(\/\.\.)+(?=\/)/, '')
+    end
+
+    return url
+  end
+end
+
+##
+# A base class for importers that scrape their contacts from web services
+
+class Graybook::Importer::PageScraper < Graybook::Importer::Base
+
+  attr_accessor :agent
+  
+  ##
+  # creates the Mechanize agent used to do the scraping and sets a nice
+  # user agent header for good net educate 
+
+  def create_agent
+    self.agent = WWW::Mechanize.new
+    agent.user_agent = "Mozilla/4.0 (compatible; Graybook #{Graybook::VERSION})"
+    agent.keep_alive = false
+    agent
+  end
+  
+  ##
+  # Page scrapers will follow a fairly simple pattern of instantiating the 
+  # agent, prepping for the scrape and then the actual scrape process
+
+  def fetch_contacts!
+    create_agent
+    prepare
+    scrape_contacts
+  end
+  
+  ##
+  # Providers will often require you to login or otherwise prepare to actual 
+  # scrape the contacts
+
+  def prepare; end # stub
+
+  ##
+  # Some providers have a single page you can scrape from (like Gmail's HTML 
+  # Contacts page) while others might require you to navigate several pages, 
+  # scraping as you go.
+
+  def scrape_contacts; end # stub
+  
+  ##
+  # helper to strip html from text
+
+  def strip_html( html )
+    html.gsub(/<\/?[^>]*>/, '')
+  end
+end

data/lib/graybook/importer/web.de.rb

@@ -0,0 +1,67 @@
+require 'graybook/importer/page_scraper'
+
+class Graybook::Importer::WebDE < Graybook::Importer::PageScraper
+  LOGIN_URL = "https://freemail.web.de"
+
+  def =~( options )
+    options && options[:username] =~ /@web\.de$/i
+  end
+
+  def login
+    username, password = options[:username], options[:password]
+
+    begin
+      page = agent.get LOGIN_URL
+
+      form = page.forms.with.name("login").first
+      form.username = username
+      form.password = password
+
+      page = form.submit
+      # follow logout hint
+      if (continue_link = page.links.select { |link| link.text =~ /weiter zu FreeMail/ }.first)
+        page = continue_link.click
+      end
+
+      if page.uri.to_s.match(/logonfailed/)
+        raise Graybook::BadCredentialsError.new
+      end
+
+      # follow content frame
+      @next = agent.get page.frames[1].src
+    rescue => e
+      raise e || Graybook::GraybookError.new
+    end
+  end
+
+  def prepare
+    login
+  end
+
+  def scrape_contacts
+    page = @next
+
+    contacts = [/Posteingang/, /Unbekannt/].map do |folder|
+      page = page.links.select { |link| link.text =~ folder }.first.click
+      find_contacts(page)
+    end
+
+    contacts.inject([]) do |memo, contact|
+      memo << contact unless memo.include? contact
+      memo
+    end
+  end
+
+  protected
+    def find_contacts(page)
+      page.search("span.from a").map do |link|
+        recp = link.attributes["title"].gsub(/\n/, "").split(/\s/)
+        email = recp.pop
+        fullname = recp.join(" ")
+
+        { :name => fullname, :email => email }
+      end
+    end
+
+    Graybook.register :webde, self
+end

data/lib/graybook/importer/yahoo.rb

@@ -0,0 +1,63 @@
+require 'graybook/importer/page_scraper'
+require 'fastercsv'
+
+##
+# contacts importer for Yahoo!
+
+class Graybook::Importer::Yahoo < Graybook::Importer::PageScraper
+
+  ##
+  # Matches this importer to an user's name/address
+
+  def =~(options = {})
+    options && options[:username] =~ /@yahoo.co(m|\.uk)$/i ? true : false
+  end
+  
+  ##
+  # login for Yahoo!
+
+  def login
+    page = agent.get('https://login.yahoo.com/config/login_verify2?')
+    form = page.forms.first
+    form.login = options[:username].split("@").first
+    form.passwd = options[:password]
+    page = agent.submit(form, form.buttons.first)
+    
+    if page.body =~ /Invalid ID or password./ || page.body =~ /This ID is not yet taken./
+      raise Graybook::BadCredentialsError, "That username and password was not accepted. Please check them and try again."
+    end
+    
+    true
+  end
+  
+  ##
+  # prepare the importer
+
+  def prepare
+    login
+  end
+  
+  ##
+  # scrape yahoo contacts
+
+  def scrape_contacts
+    page = agent.get("http://address.yahoo.com/?1=&VPC=import_export")
+    if page.body =~ /To access Yahoo! Address Book\.\.\..*Sign in./m
+      raise( Graybook::BadCredentialsError, "Must be authenticated to access contacts." )
+    end
+    form = page.forms.last
+    csv = agent.submit(form, form.buttons[2]) # third button is Yahoo-format CSV
+    
+    contact_rows = FasterCSV.parse(csv.body)
+    labels = contact_rows.shift # TODO: Actually use the labels to find the indexes of the data we want
+    contact_rows.collect do |row|
+      next if !row[7].empty? && options[:username] =~ /^#{Regexp.escape(row[7])}/ # Don't collect self
+      {
+        :name  => "#{row[0]} #{row[2]}".to_s, 
+        :email => (row[4] || "#{row[7]}@yahoo.com") # email is a field in the data, but will be blank for Yahoo users so we create their email address    
+      } 
+    end
+  end
+  
+  Graybook.register(:yahoo, self)
+end

data/lib/graybook.rb

@@ -0,0 +1,83 @@
+$:.unshift File.expand_path(File.join(File.dirname(__FILE__)))
+require 'singleton'
+require 'rubygems'
+
+class Graybook
+  include ::Singleton
+  VERSION = '1.0.22'
+  
+  class GraybookError < ::StandardError; end
+  class BadCredentialsError < GraybookError; end
+  class LegacyAccount < GraybookError; end
+  
+  attr_accessor :importers
+  attr_accessor :exporters
+  
+  def self.get( *args )
+    instance.get( *args )
+  end
+
+  def self.register(name, adapter_class)
+    case adapter = adapter_class.new
+      when Importer::Base
+        instance.importers[name.to_sym] = adapter
+      when Exporter::Base
+        instance.exporters[name.to_sym] = adapter
+      else
+        raise ArgumentError, "Unknown adapter"
+    end
+  end
+  
+  # Sends the vcards from the import to whatever is handling the export
+  def export( importer, exporter, options )
+    exporter.export importer.import( options )
+  end
+  
+  # Searches registered importers for one that will handle the given options
+  def find_importer( options )
+    importers.each{ |key, importer| return importer if importer =~ options }
+    nil
+  end
+  
+  # Fetches contacts from various services or filetypes. The default is to return an array
+  # of hashes - Graybook's internal format
+  # 
+  # Handles several different calls:
+  #  get( :username => 'something@gmail.com', :password => 'whatever' )
+  #  get( :as => :xml, :username => 'something@gmail.com', :password => 'whatever' )
+  #  get( :csv, :file => #<File:/path/to/file.csv> )
+  def get( *args )
+    options   = args.last.is_a?(Hash) ? args.pop : {}
+    to_format = exporters[ options[:as] || :basic ]
+    source    = (importers[args.first.to_sym] rescue nil) || find_importer(options)
+
+    raise ArgumentError, "Unknown exporter" unless to_format
+    raise ArgumentError, "Unknown source" unless source
+    
+    export source, to_format, options
+  end
+  
+  def initialize
+    self.importers = {}
+    self.exporters = {}
+  end
+end
+
+# Require all the importers/exporters
+require 'graybook/importer/base'
+require 'graybook/exporter/base'
+Dir.glob(File.join(File.dirname(__FILE__), 'graybook/importer/*.rb')).each {|f| require f }
+Dir.glob(File.join(File.dirname(__FILE__), 'graybook/exporter/*.rb')).each {|f| require f }
+
+class NilClass
+  def empty?
+    true
+  end
+end
+
+class Object
+  def blank?
+    respond_to?(:empty?) ? empty? : !self
+  end  
+end
+

data/test/config/credentials.yml.example

@@ -0,0 +1,9 @@
+gmx:
+  username: nil
+  password: nil
+freenet:
+  username: nil
+  password: nil
+webde:
+  username: nil
+  password: nil