alexjp-blackbook_csv 1.0.5

data/lib/blackbook/importer/hotmail.rb

@@ -0,0 +1,124 @@
+require 'blackbook/importer/page_scraper'
+require 'cgi'
+
+##
+# imports contacts for MSN/Hotmail
+class Blackbook::Importer::Hotmail < Blackbook::Importer::PageScraper
+
+  DOMAINS = { "compaq.net"        => "https://msnia.login.live.com/ppsecure/post.srf",
+              "hotmail.co.jp"     => "https://login.live.com/ppsecure/post.srf",
+              "hotmail.co.uk"     => "https://login.live.com/ppsecure/post.srf",
+              "hotmail.com"       => "https://login.live.com/ppsecure/post.srf",
+              "hotmail.de"        => "https://login.live.com/ppsecure/post.srf",
+              "hotmail.fr"        => "https://login.live.com/ppsecure/post.srf",
+              "hotmail.it"        => "https://login.live.com/ppsecure/post.srf",
+              "messengeruser.com" => "https://login.live.com/ppsecure/post.srf",
+              "msn.com"           => "https://msnia.login.live.com/ppsecure/post.srf",
+              "passport.com"      => "https://login.live.com/ppsecure/post.srf",
+              "webtv.net"         => "https://login.live.com/ppsecure/post.srf" }
+              
+  ##
+  # Matches this importer to an user's name/address
+
+  def =~(options)
+    return false unless options && options[:username]
+    domain = username_domain(options[:username].downcase)
+    !domain.empty? && DOMAINS.keys.include?( domain ) ? true : false
+  end
+   
+  ##
+  # Login procedure
+  # 1. Go to login form
+  # 2. Set login and passwd
+  # 3. Set PwdPad to IfYouAreReadingThisYouHaveTooMuchFreeTime minus however many characters are in passwd (so if passwd
+  #    was 8 chars, you'd chop 8 chars of the end of IfYouAreReadingThisYouHaveTooMuchFreeTime - giving you IfYouAreReadingThisYouHaveTooMuch)
+  # 4. Set the action to the appropriate URL for the username's domain
+  # 5. Get the query string to append to the new action
+  # 5. Submit the form and parse the url from the resulting page's javascript
+  # 6. Go to that url
+
+  def login
+    page = agent.get('http://login.live.com/login.srf?id=2')
+    form = page.forms.first
+    form.login   = options[:username]
+    form.passwd  = options[:password]
+    form.PwdPad  = ( "IfYouAreReadingThisYouHaveTooMuchFreeTime"[0..(-1 - options[:password].to_s.size )])
+    query_string = page.body.scan(/g_QS="([^"]+)/).first.first rescue nil
+    form.action  = login_url + "?#{query_string.to_s}"
+    page = agent.submit(form)
+    
+    # Check for login success
+    if page.body =~ /The e-mail address or password is incorrect/ ||
+      page.body =~ /Sign in failed\./
+      raise( Blackbook::BadCredentialsError, 
+        "That username and password was not accepted. Please check them and try again." )
+    end
+    
+    page = agent.get( page.body.scan(/http\:\/\/[^"]+/).first )
+  end
+  
+  ##
+  # prepare this importer
+
+  def prepare
+    login
+  end
+  
+  ##
+  # Scrape contacts for Hotmail
+  # Seems like a POST to directly fetch CSV contacts from options.aspx?subsection=26&n=
+  # raises an end of file error in Net::HTTP via Mechanize.
+  # Seems like Hotmail addresses are now hosted on Windows Live.
+
+  def scrape_contacts
+    unless agent.cookies.find{|c| c.name == 'MSPPre' && c.value == options[:username]}
+      raise( Blackbook::BadCredentialsError, "Must be authenticated to access contacts." )
+    end
+
+    page = agent.get('PrintShell.aspx?type=contact')
+    rows = page.search("//div[@class='ContactsPrintPane cPrintContact BorderTop']")
+    rows.collect do |row|
+      name = row.search("//div[@class='cDisplayName']").first.innerText.strip
+      
+      vals = {}
+      row.search("//table/tr").each do |pair|
+        key = pair.search("/td[@class='TextAlignRight Label']").first.innerText.strip
+        val = pair.search("/td[@class='Value']").first.innerText.strip
+        vals[key.to_sym] = val
+      end
+      vals[:name] = name
+      vals[:email] = (vals['Personal e-mail:'.to_sym] || vals['Work e-mail:'.to_sym]).split(' ').first rescue ''
+      vals
+    end
+  end
+  
+  ##
+  # lookup for the login service that should be used based on the user's
+  # address
+
+  def login_url
+    DOMAINS[username_domain] || DOMAINS['hotmail.com']
+  end
+  
+
+  ##
+  # normalizes the host for the page that is currently being "viewed" by the
+  # Mechanize agent
+
+  def current_host
+    return nil unless agent && agent.current_page
+    uri = agent.current_page.uri
+    "#{uri.scheme}://#{uri.host}"
+  end
+  
+  ##
+  # determines the domain for the user
+
+  def username_domain(username = nil)
+    username ||= options[:username] if options
+    return unless username
+    username.to_s.split('@').last
+  end
+  
+  Blackbook.register(:hotmail, self)
+end

data/lib/blackbook/importer/page_scraper.rb

@@ -0,0 +1,86 @@
+require 'rubygems'
+gem 'mechanize', '>= 0.7.0'
+require 'mechanize'
+require 'generator' # for SyncEnumerator
+
+# Patch Mechanize's broken html unescaping Mechanize 0.6.11
+class WWW::Mechanize
+  def to_absolute_uri(url, cur_page=current_page())
+    unless url.is_a? URI
+      url = url.to_s.strip
+      url = URI.parse(
+              Util.html_unescape(
+                SyncEnumerator.new(
+                  url.split(/%[0-9A-Fa-f]{2}/), url.scan(/%[0-9A-Fa-f]{2}/)
+                ).map { |x,y|
+                  "#{URI.escape(x||'')}#{y}"
+                }.join('').gsub(/%23/, '#')
+              )
+            )
+      # Mechanize here uses #zip to combine the two arrays, which will ignore 
+      # excessive elements of the second array (the one which is passed as an 
+      # argument). That means if the URL ends with more than one already escaped
+      # character, then only the first one will be restored into the resulting
+      # URL.
+    end
+
+    # construct an absolute uri
+    if url.relative?
+      raise 'no history. please specify an absolute URL' unless cur_page.uri
+      url = cur_page.uri + url
+      # Strip initial "/.." bits from the path
+      url.path.sub!(/^(\/\.\.)+(?=\/)/, '')
+    end
+
+    return url
+  end
+end
+
+##
+# A base class for importers that scrape their contacts from web services
+
+class Blackbook::Importer::PageScraper < Blackbook::Importer::Base
+
+  attr_accessor :agent
+  
+  ##
+  # creates the Mechanize agent used to do the scraping and sets a nice
+  # user agent header for good net educate 
+
+  def create_agent
+    self.agent = WWW::Mechanize.new
+    agent.user_agent = "Mozilla/4.0 (compatible; Blackbook #{Blackbook::VERSION})"
+    agent.keep_alive = false
+    agent
+  end
+  
+  ##
+  # Page scrapers will follow a fairly simple pattern of instantiating the 
+  # agent, prepping for the scrape and then the actual scrape process
+
+  def fetch_contacts!
+    create_agent
+    prepare
+    scrape_contacts
+  end
+  
+  ##
+  # Providers will often require you to login or otherwise prepare to actual 
+  # scrape the contacts
+
+  def prepare; end # stub
+
+  ##
+  # Some providers have a single page you can scrape from (like Gmail's HTML 
+  # Contacts page) while others might require you to navigate several pages, 
+  # scraping as you go.
+
+  def scrape_contacts; end # stub
+  
+  ##
+  # helper to strip html from text
+
+  def strip_html( html )
+    html.gsub(/<\/?[^>]*>/, '')
+  end
+end

data/lib/blackbook/importer/yahoo.rb

@@ -0,0 +1,61 @@
+require 'blackbook/importer/page_scraper'
+require 'fastercsv'
+
+##
+# contacts importer for Yahoo!
+
+class Blackbook::Importer::Yahoo < Blackbook::Importer::PageScraper
+
+  ##
+  # Matches this importer to an user's name/address
+
+  def =~(options = {})
+    options && options[:username] =~ /@yahoo.com$/i ? true : false
+  end
+  
+  ##
+  # login for Yahoo!
+
+  def login
+    page = agent.get('https://login.yahoo.com/config/login_verify2?')
+    form = page.forms.first
+    form.login = options[:username].split("@").first
+    form.passwd = options[:password]
+    page = agent.submit(form, form.buttons.first)
+    
+    # Check for login success
+    raise( Blackbook::BadCredentialsError, "That username and password was not accepted. Please check them and try again." ) if page.body =~ /Invalid ID or password./
+    true
+  end
+  
+  ##
+  # prepare the importer
+
+  def prepare
+    login
+  end
+  
+  ##
+  # scrape yahoo contacts
+
+  def scrape_contacts
+    page = agent.get("http://address.yahoo.com/?1=&VPC=import_export")
+    if page.body =~ /To access Yahoo! Address Book\.\.\..*Sign in./m
+      raise( Blackbook::BadCredentialsError, "Must be authenticated to access contacts." )
+    end
+    form = page.forms.last
+    csv = agent.submit(form, form.buttons[2]) # third button is Yahoo-format CSV
+    
+    contact_rows = FasterCSV.parse(csv.body)
+    labels = contact_rows.shift # TODO: Actually use the labels to find the indexes of the data we want
+    contact_rows.collect do |row|
+      next if !row[7].empty? && options[:username] =~ /^#{row[7]}/ # Don't collect self
+      {
+        :name  => "#{row[0]} #{row[2]}".to_s, 
+        :email => (row[4] || "#{row[7]}@yahoo.com") # email is a field in the data, but will be blank for Yahoo users so we create their email address    
+      } 
+    end
+  end
+  
+  Blackbook.register(:yahoo, self)
+end

data/lib/blackbook.rb

@@ -0,0 +1,76 @@
+$:.unshift File.expand_path(File.join(File.dirname(__FILE__)))
+require 'singleton'
+require 'rubygems'
+
+class Blackbook
+  include ::Singleton
+  VERSION = '1.0.5'
+  
+  class BlackbookError < ::StandardError; end
+  class BadCredentialsError < BlackbookError; end
+  
+  attr_accessor :importers
+  attr_accessor :exporters
+  
+  def self.get( *args )
+    instance.get( *args )
+  end
+
+  def self.register(name, adapter_class)
+    case adapter = adapter_class.new
+      when Importer::Base
+        instance.importers[name.to_sym] = adapter
+      when Exporter::Base
+        instance.exporters[name.to_sym] = adapter
+      else
+        raise ArgumentError, "Unknown adapter"
+    end
+  end
+  
+  # Sends the vcards from the import to whatever is handling the export
+  def export( importer, exporter, options )
+    exporter.export importer.import( options )
+  end
+  
+  # Searches registered importers for one that will handle the given options
+  def find_importer( options )
+    importers.each{ |key, importer| return importer if importer =~ options }
+    nil
+  end
+  
+  # Fetches contacts from various services or filetypes. The default is to return an array
+  # of hashes - Blackbook's internal format
+  # 
+  # Handles several different calls:
+  #  get( :username => 'something@gmail.com', :password => 'whatever' )
+  #  get( :as => :xml, :username => 'something@gmail.com', :password => 'whatever' )
+  #  get( :csv, :file => #<File:/path/to/file.csv> )
+  def get( *args )
+    options   = args.last.is_a?(Hash) ? args.pop : {}
+    to_format = exporters[ options[:as] || :basic ]
+    source    = (importers[args.first.to_sym] rescue nil) || find_importer(options)
+
+    raise ArgumentError, "Unknown exporter" unless to_format
+    raise ArgumentError, "Unknown source" unless source
+    
+    export source, to_format, options
+  end
+  
+  def initialize
+    self.importers = {}
+    self.exporters = {}
+  end
+end
+
+# Require all the importers/exporters
+require 'blackbook/importer/base'
+require 'blackbook/exporter/base'
+Dir.glob(File.join(File.dirname(__FILE__), 'blackbook/importer/*.rb')).each {|f| require f }
+Dir.glob(File.join(File.dirname(__FILE__), 'blackbook/exporter/*.rb')).each {|f| require f }
+
+class NilClass
+  def empty?
+    true
+  end
+end
+