briancollins-blackbook 1.0.15

data/lib/blackbook/importer/base.rb

@@ -0,0 +1,39 @@
+#
+# Provides a base template for interface and behavior of contact importers
+
+module Blackbook::Importer
+  class Base
+    attr_accessor :options
+
+    ##
+    # Should return true or false/nil depending on whether the +options+ given 
+    # can be handled by this importer
+
+    def =~( options ); end # stub
+
+    ##
+    # Does the work of extracting contacts. Returns an Array of Arrays 
+    # containing the name and email as the first and second elements. Of 
+    # course, you can override this behavior to meet the needs of a 
+    # particular service.
+
+    def fetch_contacts!; end # stub
+
+    ##
+    # Imports the contacts using the given +options+. Returns an array of 
+    # hashes in the internal format (a hash with at least :name and :email 
+    # values).
+
+    def import(options = {}) 
+      self.options = options
+      fetch_contacts!
+    end
+
+    ##
+    # Name of the importer service.
+
+    def service_name
+      self.class.name.split("::").last
+    end
+  end
+end

data/lib/blackbook/importer/csv.rb

@@ -0,0 +1,68 @@
+##
+# Imports contacts from a CSV file
+
+class Blackbook::Importer::Csv < Blackbook::Importer::Base
+
+  DEFAULT_COLUMNS = [:name,:email,:misc]
+  DEFAULT_PATTERN = /,/
+
+  ##
+  # Matches this importer to a file that contains CSV values
+
+  def =~(options)
+    options && options[:file].respond_to?(:open) ? true : false
+  end
+
+  ##
+  # fetch_contacts! implementation for this importer
+
+  def fetch_contacts!
+    lines   = IO.readlines(options[:file].path)
+    columns = to_columns(lines.first)
+    lines.shift if columns.first == :name
+    columns = DEFAULT_COLUMNS.dup unless columns.first == :name
+
+    contacts = Array.new
+    lines.each do |l|
+      vals = l.split(pattern)
+      next if vals.empty?
+      contacts << to_hash(columns, vals)
+    end
+
+    contacts
+  end
+
+  def to_hash(cols, vals) # :nodoc:
+    h = Hash.new
+    cols.each do |c|
+      h[c] = (c == cols.last) ? vals.join(',') : vals.shift
+    end
+    h
+  end
+
+  def to_columns(line) # :nodoc:
+    columns = Array.new
+    if line.match(pattern)
+      tags = line.split(pattern)
+    else
+      tags = line.split(DEFAULT_PATTERN)
+    end
+    # deal with "Name,E-mail..." oddity up front
+    if tags.first =~ /^name$/i
+      tags.shift
+      columns << :name
+      if tags.first =~ /^e.?mail/i # E-mail or Email
+        tags.shift
+        columns << :email
+      end
+    end
+    tags.each{|v| columns << v.strip.to_sym}
+    columns
+  end
+  
+  def pattern
+    @pattern ||= (options[:pattern] || DEFAULT_PATTERN) rescue DEFAULT_PATTERN
+  end
+
+  Blackbook.register(:csv, self)
+end

data/lib/blackbook/importer/freenet.rb

@@ -0,0 +1,62 @@
+require 'blackbook/importer/page_scraper'
+
+class Blackbook::Importer::Freenet < Blackbook::Importer::PageScraper
+  LOGIN_URL = "https://office.freenet.de"
+  MESSAGES_URL = "/main_overview.html"
+
+  def =~( options )
+    options && options[:username] =~ /@freenet\.de$/i
+  end
+
+  def login
+    username, password = options[:username], options[:password]
+
+    begin
+      page = agent.get LOGIN_URL
+
+      form = page.forms.with.name("loginform").first
+      form.username = username.split("@").first
+      form.password = password
+      page = form.submit
+
+      if page.body.match(/Fehlgeschlagene Login-Versuche/)
+        page = page.forms.with.name("sicherform").submit
+        page = agent.get MESSAGES_URL
+      end
+
+      if page.body.match(/Login (erneut )?fehlgeschlagen/) || page.body.match(/Ich bin bereits Mitglied/)
+        raise Blackbook::BadCredentialsError.new
+      end
+    rescue => e
+      raise e || Blackbook::BlackbookError.new
+    end
+  end
+
+  def prepare
+    login
+  end
+
+  def scrape_contacts
+    contacts = ["INBOX", "INBOX.sent"].map do |folder|
+      page = agent.get "/messages/mail_mailbox.html?mail_folder=#{folder}"
+      find_contacts(page)
+    end
+
+    contacts.inject([]) do |memo, contact|
+      memo << contact unless memo.include? contact
+      memo
+    end.flatten
+  end
+
+
+  protected
+    def find_contacts(page)
+      page.search("tr[@id]/td[9]").map do |cell|
+        recp, fullname, email = *cell.attributes["title"].match(/"(.*)"<(.*)>/)
+
+        { :name => fullname, :email => email }
+      end
+    end
+
+    Blackbook.register :freenet, self
+end

data/lib/blackbook/importer/gmail.rb

@@ -0,0 +1,73 @@
+require 'kconv'
+require 'blackbook/importer/page_scraper'
+
+if RUBY_VERSION > "1.9"    
+ require "csv"  
+ unless defined? FCSV
+   class Object  
+     FCSV = CSV 
+     alias_method :FCSV, :CSV
+   end  
+ end
+else
+ require "fastercsv"
+end
+
+##
+# Imports contacts from GMail
+
+class Blackbook::Importer::Gmail < Blackbook::Importer::PageScraper
+
+  RETRY_THRESHOLD = 5
+  ##
+  # Matches this importer to an user's name/address
+  
+  def =~(options = {})
+    options && options[:username] =~ /@(gmail|googlemail).com$/i ? true : false
+  end
+  
+  ##
+  # login to gmail
+
+  def login
+    page = agent.get('http://mail.google.com/mail/')
+    form = page.forms.first
+    form.Email = options[:username]
+    form.Passwd = options[:password]
+    page = agent.submit(form,form.buttons.first)
+    
+    raise( Blackbook::BadCredentialsError, "That username and password was not accepted. Please check them and try again." ) if page.body =~ /Username and password do not match/
+    
+    if page.search('//meta').first.attributes['content'] =~ /url='?(http.+?)'?$/i
+      page = agent.get $1
+    end
+  end
+  
+  ##
+  # prepare this importer
+
+  def prepare
+    login
+  end
+  
+  ##
+  # scrape gmail contacts for this importer
+
+  def scrape_contacts
+    unless agent.cookies.find{|c| c.name == 'GAUSR' && 
+                           (c.value.include? "mail:#{options[:username]}")}
+      raise( Blackbook::BadCredentialsError, "Must be authenticated to access contacts." )
+    end
+
+	contacts = []
+    csv = agent.get('https://mail.google.com/mail/contacts/data/export?exportType=ALL&out=GMAIL_CSV')
+	body = Kconv.toutf8(csv.body)
+	FCSV.parse(body) do |row|
+		next if row[0] == "Name" and row[1] == "E-mail"
+		contacts << {:name => row[0], :email => row[1]} unless row[1].blank?
+	end
+	return contacts
+  end
+  
+  Blackbook.register(:gmail, self)
+end

data/lib/blackbook/importer/gmx.rb

@@ -0,0 +1,68 @@
+require 'blackbook/importer/page_scraper'
+
+class Blackbook::Importer::GMX < Blackbook::Importer::PageScraper
+  LOGIN_URL = "https://www.gmx.net/"
+
+  def =~( options )
+    options && options[:username] =~ /@gmx\.(de|net)$/i
+  end
+
+  def login
+    username, password = options[:username], options[:password]
+
+    begin
+      page = agent.get LOGIN_URL
+
+      form = page.forms.with.name("login").first
+      form.id = username
+      form.p = password
+      page = form.submit
+      @next = page
+
+      if (page.at("div.index") && page.at("div.index").inner_html == "Passwort vergessen?") ||
+        page.uri.to_s.match(/login-failed/)
+        raise Blackbook::BadCredentialsError.new
+      end
+    rescue => e
+      raise e || Blackbook::BlackbookError.new
+    end
+  end
+
+  def prepare
+    login
+  end
+
+  def scrape_contacts
+    page = @next
+
+    # Go to E-Mails
+    page = page.links.select { |link| link.text =~ /E-Mail/ }.first.click
+
+    contacts = [/Posteingang/, /Archiv/, /Gesendet/].map do |folder|
+      # puts "folder: #{folder}"
+      # puts "page title: #{page.title}"
+      # puts "page links: #{page.links.select { |link| link.text =~ folder }}"
+      page = page.links.select { |link| link.text =~ folder }.first.click
+      find_contacts(page)
+    end
+
+    contacts.inject([]) do |memo, contact|
+      memo << contact unless memo.include? contact
+      memo
+    end.flatten
+  end
+
+  protected
+
+    def find_contacts(page)
+      links = page.search("form#MI a").select { |link| link.attributes["title"] =~ /@/ }
+      links.map do |link|
+        recp = link.attributes["title"].gsub(/\n/, "").split(/\s/)
+        email = recp.pop.gsub(/[<>]/, "")
+        fullname = recp.join(" ")
+
+        { :name => fullname, :email => email }
+      end
+    end
+    Blackbook.register :gmx, self
+end

data/lib/blackbook/importer/hotmail.rb

@@ -0,0 +1,128 @@
+require 'blackbook/importer/page_scraper'
+require 'cgi'
+
+##
+# imports contacts for MSN/Hotmail
+class Blackbook::Importer::Hotmail < Blackbook::Importer::PageScraper
+
+  DOMAINS = { "compaq.net"        => "https://msnia.login.live.com/ppsecure/post.srf",
+              "hotmail.co.jp"     => "https://login.live.com/ppsecure/post.srf",
+              "hotmail.co.uk"     => "https://login.live.com/ppsecure/post.srf",
+              "hotmail.com"       => "https://login.live.com/ppsecure/post.srf",
+              "hotmail.de"        => "https://login.live.com/ppsecure/post.srf",
+              "hotmail.fr"        => "https://login.live.com/ppsecure/post.srf",
+              "hotmail.it"        => "https://login.live.com/ppsecure/post.srf",
+              "live.com"          => "https://login.live.com/ppsecure/post.srf",
+              "messengeruser.com" => "https://login.live.com/ppsecure/post.srf",
+              "msn.com"           => "https://msnia.login.live.com/ppsecure/post.srf",
+              "passport.com"      => "https://login.live.com/ppsecure/post.srf",
+              "webtv.net"         => "https://login.live.com/ppsecure/post.srf" }
+              
+  ##
+  # Matches this importer to an user's name/address
+
+  def =~(options)
+    return false unless options && options[:username]
+    domain = username_domain(options[:username].downcase)
+    !domain.empty? && DOMAINS.keys.include?( domain ) ? true : false
+  end
+   
+  ##
+  # Login procedure
+  # 1. Go to login form
+  # 2. Set login and passwd
+  # 3. Set PwdPad to IfYouAreReadingThisYouHaveTooMuchFreeTime minus however many characters are in passwd (so if passwd
+  #    was 8 chars, you'd chop 8 chars of the end of IfYouAreReadingThisYouHaveTooMuchFreeTime - giving you IfYouAreReadingThisYouHaveTooMuch)
+  # 4. Set the action to the appropriate URL for the username's domain
+  # 5. Get the query string to append to the new action
+  # 5. Submit the form and parse the url from the resulting page's javascript
+  # 6. Go to that url
+
+  def login
+    page = agent.get('http://login.live.com/login.srf?id=2')
+    form = page.forms.first
+    form.login   = options[:username]
+    form.passwd  = options[:password]
+    form.PwdPad  = ( "IfYouAreReadingThisYouHaveTooMuchFreeTime"[0..(-1 - options[:password].to_s.size )])
+    query_string = page.body.scan(/g_QS="([^"]+)/).first.first rescue nil
+    form.action  = login_url + "?#{query_string.to_s}"
+    page = agent.submit(form)
+    
+    # Check for login success
+    if page.body =~ /The e-mail address or password is incorrect/ ||
+      page.body =~ /Sign in failed\./
+      raise( Blackbook::BadCredentialsError, 
+        "That username and password was not accepted. Please check them and try again." )
+    end
+    
+    @first_page = agent.get( page.body.scan(/http\:\/\/[^"]+/).first )
+  end
+  
+  ##
+  # prepare this importer
+
+  def prepare
+    login
+  end
+  
+  ##
+  # Scrape contacts for Hotmail
+  # Seems like a POST to directly fetch CSV contacts from options.aspx?subsection=26&n=
+  # raises an end of file error in Net::HTTP via Mechanize.
+  # Seems like Hotmail addresses are now hosted on Windows Live.
+
+  def scrape_contacts
+    unless agent.cookies.find{|c| c.name == 'MSPPre' && c.value == options[:username]}
+      raise( Blackbook::BadCredentialsError, "Must be authenticated to access contacts." )
+    end
+    page = agent.get(@first_page.iframes.first.src)
+    
+    page = agent.click(page.link_with(:text => 'Mail'))
+    page = agent.get(page.iframes.first.src)
+    page = agent.get('/mail/PrintShell.aspx?type=contact')
+        
+    rows = page.search("//div[@class='ContactsPrintPane cPrintContact BorderTop']")
+    rows.collect do |row|
+      vals = {}
+      row.search("table/tr").each do |pair|
+        key = pair.search("td[@class='TextAlignRight Label']").first.inner_text.strip rescue nil
+        next if key.nil?
+        val = pair.search("td[@class='Value']").first.inner_text.strip
+        vals[key.to_sym] = val
+      end
+      vals[:name] = vals['Name:'.to_sym] rescue ''
+      vals[:email] = (vals['Personal e-mail:'.to_sym] || vals['Work e-mail:'.to_sym] || vals['Windows Live ID:'.to_sym]).split(' ').first rescue ''
+      vals
+    end
+  end
+  
+  ##
+  # lookup for the login service that should be used based on the user's
+  # address
+
+  def login_url
+    DOMAINS[username_domain] || DOMAINS['hotmail.com']
+  end
+  
+
+  ##
+  # normalizes the host for the page that is currently being "viewed" by the
+  # Mechanize agent
+
+  def current_host
+    return nil unless agent && agent.current_page
+    uri = agent.current_page.uri
+    "#{uri.scheme}://#{uri.host}"
+  end
+  
+  ##
+  # determines the domain for the user
+
+  def username_domain(username = nil)
+    username ||= options[:username] if options
+    return unless username
+    username.to_s.split('@').last
+  end
+  
+  Blackbook.register(:hotmail, self)
+end

data/lib/blackbook/importer/page_scraper.rb

@@ -0,0 +1,52 @@
+require 'rubygems'
+gem 'mechanize', '>= 0.9.2'
+require 'mechanize'
+
+##
+# A base class for importers that scrape their contacts from web services
+
+class Blackbook::Importer::PageScraper < Blackbook::Importer::Base
+
+  attr_accessor :agent
+  
+  ##
+  # creates the Mechanize agent used to do the scraping and sets a nice
+  # user agent header for good net educate 
+
+  def create_agent
+    self.agent = WWW::Mechanize.new
+    agent.user_agent = "Mozilla/4.0 (compatible; Blackbook #{Blackbook::VERSION})"
+    agent.keep_alive = false
+    agent
+  end
+  
+  ##
+  # Page scrapers will follow a fairly simple pattern of instantiating the 
+  # agent, prepping for the scrape and then the actual scrape process
+
+  def fetch_contacts!
+    create_agent
+    prepare
+    scrape_contacts
+  end
+  
+  ##
+  # Providers will often require you to login or otherwise prepare to actual 
+  # scrape the contacts
+
+  def prepare; end # stub
+
+  ##
+  # Some providers have a single page you can scrape from (like Gmail's HTML 
+  # Contacts page) while others might require you to navigate several pages, 
+  # scraping as you go.
+
+  def scrape_contacts; end # stub
+  
+  ##
+  # helper to strip html from text
+
+  def strip_html( html )
+    html.gsub(/<\/?[^>]*>/, '')
+  end
+end