tweetlr 0.1.7pre → 0.1.7pre4

data/.travis.yml

@@ -4,7 +4,7 @@ bundler_args: --binstubs
 # Specify which ruby versions you wish to run your tests on, each version will be used
 rvm:
   - 1.9.2
-  - 1.8.7 # (current default)
+  #- 1.8.7 # (current default)
 
 # Define how to run your tests (defaults to `bundle exec rake` or `rake` depending on whether you have a `Gemfile`)
 script: "bundle exec rake test"    

data/README.md

@@ -15,7 +15,7 @@ tweetlr supports
 - imgly
 - twitter / photobucket
 - t.co shortened links to pictures
-- every service accessible via embed.ly (see [photo providers](http://embed.ly/providers))
+- every service accessible via embed.ly (see [photo providers](http://embed.ly/providers))  
 
 ## Installation
 
@@ -34,10 +34,11 @@ api_endpoint_twitter: 'http://search.twitter.com/search.json'
 api_endpoint_tumblr: 'http://www.tumblr.com'
 tumblr_username: YOUR_TUMBLR_EMAIL
 tumblr_password: YOUR_TUMBLR_PW
+embedly_key: '' #tweetlr uses http://embedly.com for link processing. a free plan containing an api key is available & recommended to use in order to ensure full support
 update_period: 300 #check for updates every 300 secs = 5 minutes
 shouts: 'says' # will be concatenated after the username, before the message: @mr_x says: awesome things on a photo!
 loglevel: 1 # 0: debug, 1: info (default), 2: warn, 3: error, 5: fatal
-whitelist: #twitter accounts in that list will have their tweets published immediately. post from others will be saved as drafts. blank list will publish all tweets immediately.
+whitelist: #twitter accounts in that list will have their tweets published immediately. post from others will be saved as drafts. blank list will publish all tweets immediately
   - whitey_mc_whitelist
   - sven_kr
 ```

data/bin/tweetlr

@@ -22,7 +22,9 @@ begin
    
   UPDATE_PERIOD = CONFIG['update_period']  
   
-  @tweetlr = Tweetlr.new(CONFIG['tumblr_username'], CONFIG['tumblr_password'], {
+  @tweetlr_config = {
+    :tumblr_email => CONFIG['tumblr_username'], 
+    :tumblr_password => CONFIG['tumblr_password'],
     :whitelist => CONFIG['whitelist'], 
     :shouts => CONFIG['shouts'], 
     :since_id => CONFIG['start_at_tweet_id'] , 
@@ -32,8 +34,9 @@ begin
     :api_endpoint_tumblr => CONFIG['api_endpoint_tumblr'],
     :api_endpoint_twitter => CONFIG['api_endpoint_twitter'],
     :results_per_page => CONFIG['results_per_page'],
-    :result_type => CONFIG['result_type']
-    })
+    :result_type => CONFIG['result_type'],
+    :embedly_key => CONFIG['embedly_key']
+    }
 rescue SystemCallError
   $stderr.puts "Ooops - looks like there is no ./config/tweetlr.yml found. I'm affraid tweetlr won't work properly until you introduced that configuration file."
   exit(1)
@@ -43,32 +46,11 @@ Daemons.run_proc('tweetlr', :dir_mode => :script, :dir => './', :backtrace => tr
   @log = Logger.new(STDOUT)
   @log.info "starting tweetlr daemon..." 
   @log.info "creating a new tweetlr instance using this config: #{CONFIG.inspect}"   
-  EventMachine::run { 
-    EventMachine::add_periodic_timer( UPDATE_PERIOD ) {
-     @log.info "starting tweetlr crawl..."
-     response = {}
-     response = @tweetlr.lazy_search_twitter(@tweetlr.twitter_config["refresh_url"]) #looks awkward, but the refresh url will come from the db soon and make sense then...
-     if response
-       tweets = response['results']
-       if tweets
-       tweets.each do |tweet|
-         tumblr_post = @tweetlr.generate_tumblr_photo_post tweet
-         if tumblr_post.nil? ||  tumblr_post[:source].nil?
-            @log.warn "could not get image source: tweet: #{tweet} --- tumblr post: #{tumblr_post.inspect}"
-         else
-           @log.debug "tumblr post: #{tumblr_post}"
-           res = @tweetlr.post_to_tumblr tumblr_post
-           @log.warn "tumblr response: #{res.header_str} #{res.body_str}" unless res.response_code == 201
-         end
-        end
-         # store the highest tweet id
-         @tweetlr.twitter_config[:refresh_url]=response['refresh_url']
-         File.open(tid_file, "w+") { |io| io.write(response['max_id']) }
-       end
-     else
-       @log.error "twitter search returned no response. hail the failwhale!"
+  EventMachine::run do 
+    EventMachine::add_periodic_timer( UPDATE_PERIOD ) do
+      response = Tweetlr.crawl(@tweetlr_config)
+      File.open(tid_file, "w+") { |io| io.write(response[:since_id]) }
+      @tweetlr_config.merge! response
      end
-     @log.info "finished tweetlr crawl."
-     }
-   }
+   end
 end

data/config/tweetlr.yml

@@ -6,9 +6,10 @@ api_endpoint_twitter: 'http://search.twitter.com/search.json'
 api_endpoint_tumblr: 'http://www.tumblr.com'
 tumblr_username: YOUR_TUMBLR_EMAIL
 tumblr_password: YOUR_TUMBLR_PW
+embedly_key: '' #tweetlr uses http://embedly.com for link processing. a free plan containing an api key is available & recommended to use in order to ensure full support
 update_period: 300 #check for updates every 300 secs = 5 minutes
 shouts: 'says' # will be concatenated after the username, before the message: @mr_x says: awesome things on a photo!
-loglevel: 0 # 0: debug, 1: info (default), 2: warn, 3: error, 5: fatal
+loglevel: 1 # 0: debug, 1: info (default), 2: warn, 3: error, 5: fatal
 whitelist: #twitter accounts in that list will have their tweets published immediately. post from others will be saved as drafts. blank list will publish all tweets immediately
   - whitey_mc_whitelist
-  - sven_kr
+  - sven_kr

data/lib/combinators/twitter_tumblr.rb

@@ -0,0 +1,54 @@
+require 'processors/twitter'
+require 'processors/tumblr'
+require 'processors/photo_service'
+
+require 'log_aware'
+
+module Combinators
+  module TwitterTumblr
+    include LogAware
+    def self.log
+      LogAware.log #TODO why doesn't the include make the log method accessible?
+    end
+    #extract a linked image file's url from a tweet. first found image will be used.
+    def self.extract_image_url(tweet, embedly_key=nil)
+      links = Processors::Twitter::extract_links tweet
+      image_url = nil
+      if links
+        links.each do |link|
+          image_url = Processors::PhotoService::find_image_url(link, embedly_key)
+          return image_url if Processors::PhotoService::photo? image_url
+        end
+      end
+      image_url
+    end
+    #generate the data for a tumblr photo entry by parsing a tweet
+    def self.generate_photo_post_from_tweet(tweet, options = {})
+      log.debug "#{self}.generate_photo_post_from_tweet with options: #{options.inspect}"
+      tumblr_post = nil
+      message = tweet['text']
+      whitelist = options[:whitelist]
+      whitelist.each {|entry| entry.downcase!} if whitelist
+      if !Processors::Twitter::retweet? message
+        log.debug "tweet: #{tweet}"
+        tumblr_post = {}
+        tumblr_post[:type] = 'photo'
+        tumblr_post[:date] = tweet['created_at']
+        tumblr_post[:source] = extract_image_url tweet, options[:embedly_key]
+        user = tweet['from_user']
+        tumblr_post[:tags] = user
+        tweet_id = tweet['id']
+        if !whitelist || whitelist.member?(user.downcase)
+          state = 'published'
+        else
+          state = 'draft'
+        end
+        tumblr_post[:state] = state
+        shouts = " #{@shouts}" if @shouts
+        tumblr_post[:caption] = %?<a href="http://twitter.com/#{user}/statuses/#{tweet_id}" alt="tweet">@#{user}</a>#{shouts}: #{tweet['text']}? 
+        #TODO make the caption a bigger matter of yml/ general configuration
+      end
+      tumblr_post
+    end
+  end
+end

data/lib/log_aware.rb

@@ -1,8 +1,9 @@
+#use centralized logging
 module LogAware
     def self.log=(log)
-      @@log = log #TODO think of a more elegant way of logging than a static attribute
+      @log = log
     end
     def self.log()
-      @@log
+      @log || Logger.new(STDOUT)
     end
 end

data/lib/processors/http.rb

@@ -0,0 +1,45 @@
+require 'curb'
+require 'log_aware'
+
+module Processors
+  #utilities for handling http
+  module Http
+    include LogAware
+  
+    USER_AGENT = %{Mozilla/5.0 (compatible; tweetlr; +http://tweetlr.5v3n.com)}
+    
+    def self.log
+      LogAware.log #TODO why doesn't the include make the log method accessible?
+    end
+
+    #convenience method for curl http get calls and parsing them to json.
+    def self.http_get(request)
+      tries = 3
+      begin
+        curl = Curl::Easy.new request
+        curl.useragent = USER_AGENT
+        curl.perform
+        begin
+          JSON.parse curl.body_str
+        rescue JSON::ParserError => err
+          begin
+            log.warn "#{err}: Could not parse response for #{request} - this is probably not a json response: #{curl.body_str}"
+            return nil
+          rescue Encoding::CompatibilityError => err
+            log.error "Trying to rescue a JSON::ParserError for '#{request}' we got stuck in a Encoding::CompatibilityError."
+            return nil
+          end
+        end
+      rescue Curl::Err::CurlError => err
+        log.error "Failure in Curl call: #{err}" if log
+        tries -= 1
+        sleep 3
+        if tries > 0
+            retry
+        else
+            nil
+        end
+      end 
+    end
+  end
+end

data/lib/processors/photo_service.rb

@@ -0,0 +1,126 @@
+require 'processors/http'
+require 'log_aware'
+
+module Processors
+  #utilities for dealing with photo services
+  module PhotoService
+  
+    LOCATION_START_INDICATOR = 'Location: '
+    LOCATION_STOP_INDICATOR  = "\r\n"
+    PIC_REGEXP = /(.*?)\.(jpg|jpeg|png|gif)/i 
+  
+    include LogAware
+    
+    def self.log
+      LogAware.log #TODO why doesn't the include make the log method accessible?
+    end
+  
+    def self.find_image_url(link, embedly_key=nil)
+      url = nil
+      if link && !(photo? link)
+        url = image_url_instagram link if (link.index('instagr.am') || link.index('instagram.com'))
+        url = image_url_picplz link if link.index 'picplz'
+        url = image_url_twitpic link if link.index 'twitpic'
+        url = image_url_yfrog link if link.index 'yfrog'
+        url = image_url_imgly link if link.index 'img.ly'
+        url = image_url_tco link, embedly_key if link.index 't.co'
+        url = image_url_lockerz link if link.index 'lockerz.com'
+        url = image_url_embedly link, embedly_key if url.nil? #just try embed.ly for anything else. could do all image url processing w/ embedly, but there's probably some kind of rate limit invovled.
+      elsif photo? link
+        url = link
+      end
+      url
+    end
+  
+    def self.photo?(link)
+      link =~ PIC_REGEXP
+    end
+  
+    #find the image's url via embed.ly
+    def self.image_url_embedly(link_url, key)
+      response = Processors::Http::http_get "http://api.embed.ly/1/oembed?key=#{key}&url=#{link_url}"
+      log.debug "embedly call: http://api.embed.ly/1/oembed?key=#{key}&url=#{link_url}"
+      if response && response['type'] == 'photo'
+        image_url = response['url'] 
+      end
+      image_url
+    end
+    #find the image's url for a lockerz link
+    def self.image_url_lockerz(link_url)
+      response = Processors::Http::http_get "http://api.plixi.com/api/tpapi.svc/json/metadatafromurl?details=false&url=#{link_url}"
+      response["BigImageUrl"] if response
+    end
+    #find the image's url for an twitter shortened link
+    def self.image_url_tco(link_url, embedly_key = nil)
+      service_url = link_url_redirect link_url
+      find_image_url service_url, embedly_key
+    end
+    #find the image's url for an instagram link
+    def self.image_url_instagram(link_url)
+      link_url['instagram.com'] = 'instagr.am' if link_url.index 'instagram.com' #instagram's oembed does not work for .com links
+      response = Processors::Http::http_get "http://api.instagram.com/oembed?url=#{link_url}"
+      response['url'] if response
+    end
+
+    #find the image's url for a picplz short/longlink
+    def self.image_url_picplz(link_url)
+      id = extract_id link_url
+      #try short url
+      response = Processors::Http::http_get "http://picplz.com/api/v2/pic.json?shorturl_ids=#{id}"
+      #if short url fails, try long url
+      #response = HTTParty.get "http://picplz.com/api/v2/pic.json?longurl_ids=#{id}"
+      #extract url
+      if response && response['value'] && response['value']['pics'] && response['value']['pics'].first && response['value']['pics'].first['pic_files'] && response['value']['pics'].first['pic_files']['640r']
+        response['value']['pics'].first['pic_files']['640r']['img_url'] 
+      else
+        nil
+      end
+    end
+    #find the image's url for a twitpic link
+    def self.image_url_twitpic(link_url)
+      image_url_redirect link_url, "http://twitpic.com/show/full/"
+    end
+    #find the image'S url for a yfrog link
+    def self.image_url_yfrog(link_url)
+      response = Processors::Http::http_get("http://www.yfrog.com/api/oembed?url=#{link_url}")
+      response['url'] if response
+    end
+    #find the image's url for a img.ly link
+    def self.image_url_imgly(link_url)
+      image_url_redirect link_url, "http://img.ly/show/full/", "\r\n"
+    end
+  
+    # extract image url from services like twitpic & img.ly that do not offer oembed interfaces
+    def self.image_url_redirect(link_url, service_endpoint, stop_indicator = LOCATION_STOP_INDICATOR)
+      link_url_redirect "#{service_endpoint}#{extract_id link_url}", stop_indicator
+    end
+  
+    def self.link_url_redirect(short_url, stop_indicator = LOCATION_STOP_INDICATOR)
+      tries = 3
+      begin
+        resp = Curl::Easy.http_get(short_url) { |res| res.follow_location = true }
+      rescue Curl::Err::CurlError => err
+          log.error "Curl::Easy.http_get failed: #{err}"
+          tries -= 1
+          sleep 3
+          if tries > 0
+              retry
+          else
+             return nil
+          end
+      end
+      if(resp && resp.header_str && resp.header_str.index(LOCATION_START_INDICATOR) && resp.header_str.index(stop_indicator))
+        start = resp.header_str.index(LOCATION_START_INDICATOR) + LOCATION_START_INDICATOR.size
+        stop  = resp.header_str.index(stop_indicator, start)
+        resp.header_str[start...stop]
+      else
+        nil
+      end
+    end
+  
+    #extract the pic id from a given <code>link</code>
+    def self.extract_id(link)
+      link.split('/').last if link.split('/')
+    end
+  end
+end

data/lib/processors/tumblr.rb

@@ -0,0 +1,46 @@
+require 'log_aware'
+
+module Processors
+  #utilities for handling tumblr
+  module Tumblr
+    GENERATOR = %{tweetlr - http://tweetlr.5v3n.com}
+    API_ENDPOINT_TUMBLR = 'http://www.tumblr.com'
+    include LogAware
+    def self.log
+      LogAware.log #TODO why doesn't the include make the log method accessible?
+    end
+    #post a tumblr photo entry. 
+    #
+    #required arguments are :email, :password, :type, :date, :source, :caption, :state, :source
+    #
+    #optional arguments: :api_endpoint_tumblr, :tags
+    #
+    def self.post(options={})
+      tries = 3
+      tags = options[:tags]
+      begin
+        response = Curl::Easy.http_post("#{options[:api_endpoint_tumblr] || API_ENDPOINT_TUMBLR}/api/write", 
+        Curl::PostField.content('generator', GENERATOR),
+        Curl::PostField.content('email', options[:email]), 
+        Curl::PostField.content('password', options[:password]),
+        Curl::PostField.content('type', options[:type]),
+        Curl::PostField.content('date', options[:date]),
+        Curl::PostField.content('source', options[:source]),
+        Curl::PostField.content('caption', options[:caption]),
+        Curl::PostField.content('state', options[:state]),
+        Curl::PostField.content('tags', tags)
+        )
+      rescue Curl::Err::CurlError => err
+        log.error "Failure in Curl call: #{err}"
+        tries -= 1
+        sleep 3
+        if tries > 0
+            retry
+        else
+            response = nil
+        end
+      end
+      response
+    end
+  end
+end

data/lib/processors/twitter.rb

@@ -0,0 +1,44 @@
+require 'processors/http'
+require 'log_aware'
+
+module Processors
+  #utilities for dealing with twitter
+  module Twitter
+    include LogAware
+    def self.log
+      LogAware.log #TODO why doesn't the include make the log method accessible?
+    end
+    
+    #checks if the message is a retweet
+    def self.retweet?(message)
+      message.index('RT @') || message.index(%{"@}) || message.index("\u201c@") #detect retweets
+    end
+  
+    #extract the links from a given tweet
+    def self.extract_links(tweet)
+      if tweet
+        text = tweet['text']
+        text.gsub(/https?:\/\/[\S]+/).to_a if text
+      end
+    end
+
+    #fire a new search
+    def self.search(config)
+      search_call = "#{config[:api_endpoint_twitter]}?ors=#{config[:search_term]}&result_type=#{config[:result_type]}&rpp=#{config[:results_per_page]}"
+      Processors::Http::http_get search_call
+    end
+
+    # lazy update - search for a term or refresh the search if a response is available already
+    def self.lazy_search(config)
+      response = nil
+      if config
+        search_url = "#{config[:api_endpoint_twitter]}?since_id=#{config[:since_id]}&ors=#{config[:search_term]}&result_type=#{config[:result_type]}&rpp=#{config[:results_per_page]}"
+        log.info "lazy search using '#{search_url}'"
+        response = Processors::Http::http_get search_url
+      else
+        log.error "#{self}.lazy_search: no config given!"
+      end
+      response
+    end
+  end
+end

data/lib/tweetlr.rb

@@ -3,17 +3,16 @@ require 'logger'
 require 'yaml'
 require 'curb'
 require 'json'
-require 'twitter_processor'
-require 'http_processor'
-require 'photo_service_processor'
+require 'processors/twitter'
+require 'processors/http'
+require 'processors/photo_service'
+require 'processors/tumblr'
+require 'combinators/twitter_tumblr'
 require 'log_aware'
 
 class Tweetlr
-  
-  attr_accessor :twitter_config
 
-  VERSION = '0.1.7pre'
-  GENERATOR = %{tweetlr - http://tweetlr.5v3n.com}
+  VERSION = '0.1.7pre4'
   
   API_ENDPOINT_TWITTER = 'http://search.twitter.com/search.json'
   API_ENDPOINT_TUMBLR = 'http://www.tumblr.com'
@@ -21,27 +20,23 @@ class Tweetlr
   TWITTER_RESULTS_TYPE = 'recent'
   UPDATE_PERIOD = 600 #10 minutes
   
-  def initialize(email, password, args={:terms=>nil, :whitelist => nil, :shouts => nil, :since_id=>nil, :results_per_page => nil, :loglevel=>nil, :result_type => nil})
-    @log = Logger.new(STDOUT)
+  include LogAware
+  def self.log
+    LogAware.log #TODO why doesn't the include make the log method accessible?
+  end
+  
+  def initialize(args)
+    log = Logger.new(STDOUT)
     if (Logger::DEBUG..Logger::UNKNOWN).to_a.index(args[:loglevel])
-      @log.level = args[:loglevel] 
+      log.level = args[:loglevel] 
     else
-      @log.level = Logger::INFO
+      log.level = Logger::INFO
     end
-    @log.debug "log level set to #{@log.level}"
-    LogAware.log=@log
-    @twitter_config = {
-      :since_id => args[:since_id],
-      :search_term => args[:terms],
-      :results_per_page => args[:results_per_page] || TWITTER_RESULTS_PER_PAGE,
-      :result_type => args[:result_type] || TWITTER_RESULTS_TYPE,  
-      :api_endpoint_twitter => args[:api_endpoint_twitter] || API_ENDPOINT_TWITTER
-    }
-    @twitter_config[:refresh_url] = "?ors=#{@twitter_config[:search_term]}&since_id=#{@twitter_config[:since_id]}&rpp=#{@twitter_config[:results_per_page]}&result_type=#{@twitter_config[:result_type]}" if (@twitter_config[:since_id] && @twitter_config[:search_term])
-    @twitter_config[:logger] = @log
+    log.debug "log level set to #{log.level}"
+    LogAware.log=log
     
-    @email = email
-    @password = password
+    @email = args[:tumblr_email]
+    @password = args[:tumblr_password]
     @cookie = args[:cookie]
     @api_endpoint_twitter = 
     @api_endpoint_tumblr = args[:api_endpoint_tumblr] || API_ENDPOINT_TUMBLR
@@ -51,79 +46,38 @@ class Tweetlr
     @whitelist.each {|entry| entry.downcase!} if @whitelist
   end
   
-  def lazy_search_twitter(refresh_url=nil)
-    @twitter_config[:refresh_url] = refresh_url if refresh_url
-    TwitterProcessor::lazy_search(@twitter_config)
-  end
-  
-  #post a tumblr photo entry. required arguments are :type, :date, :source, :caption, :state. optional argument: :tags 
-  def post_to_tumblr(options={})
-    tries = 3
-    if options[:type] && options[:date] && options[:source] && options[:caption] && options[:state]
-      tags = options[:tags]
-      begin
-        response = Curl::Easy.http_post("#{@api_endpoint_tumblr}/api/write", 
-        Curl::PostField.content('generator', GENERATOR),
-        Curl::PostField.content('email', @email), 
-        Curl::PostField.content('password', @password),
-        Curl::PostField.content('type', options[:type]),
-        Curl::PostField.content('date', options[:date]),
-        Curl::PostField.content('source', options[:source]),
-        Curl::PostField.content('caption', options[:caption]),
-        Curl::PostField.content('state', options[:state]),
-        Curl::PostField.content('tags', tags)
-        )
-      rescue Curl::Err::CurlError => err
-        @log.error "Failure in Curl call: #{err}"
-        tries -= 1
-        sleep 3
-        if tries > 0
-            retry
+  def self.crawl(config)
+    log.debug "#{self}.crawl() using config: #{config.inspect}"
+    twitter_config = {
+      :since_id => config[:since_id] || config[:start_at_tweet_id],
+      :search_term => config[:terms] || config[:search_term] ,
+      :results_per_page => config[:results_per_page] || TWITTER_RESULTS_PER_PAGE,
+      :result_type => config[:result_type] || TWITTER_RESULTS_TYPE,  
+      :api_endpoint_twitter => config[:api_endpoint_twitter] || API_ENDPOINT_TWITTER
+    }    
+    log.info "starting tweetlr crawl..."
+    response = {}
+    response = Processors::Twitter::lazy_search(twitter_config) #looks awkward, but the refresh url will come from the db soon and make sense then...
+    if response
+      tweets = response['results']
+      if tweets
+      tweets.each do |tweet|
+        tumblr_post = Combinators::TwitterTumblr::generate_photo_post_from_tweet(tweet, {:whitelist => config[:whitelist], :embedly_key => config[:embedly_key]}) 
+        if tumblr_post.nil? ||  tumblr_post[:source].nil?
+           log.warn "could not get image source: tweet: #{tweet} --- tumblr post: #{tumblr_post.inspect}"
         else
-            response = nil
+          log.debug "tumblr post: #{tumblr_post}"
+          res = Processors::Tumblr.post tumblr_post.merge({:password => config[:tumblr_password], :email => config[:tumblr_email]})
+          log.warn "tumblr response: #{res.header_str} #{res.body_str}" unless res.response_code == 201
         end
+       end
+        # store the highest tweet id
+        config[:since_id] = response['max_id']
       end
+    else
+      log.error "twitter search returned no response. hail the failwhale!"
     end
-    response
-  end
-  
-  #generate the data for a tumblr photo entry by parsing a tweet
-  def generate_tumblr_photo_post tweet
-    tumblr_post = nil
-    message = tweet['text']
-    if !TwitterProcessor::retweet? message
-      @log.debug "tweet: #{tweet}"
-      tumblr_post = {}
-      tumblr_post[:type] = 'photo'
-      tumblr_post[:date] = tweet['created_at']
-      tumblr_post[:source] = extract_image_url tweet
-      user = tweet['from_user']
-      tumblr_post[:tags] = user
-      tweet_id = tweet['id']
-      if !@whitelist || @whitelist.member?(user.downcase)
-        state = 'published'
-      else
-        state = 'draft'
-      end
-      tumblr_post[:state] = state
-      shouts = " #{@shouts}" if @shouts
-      tumblr_post[:caption] = %?<a href="http://twitter.com/#{user}/statuses/#{tweet_id}" alt="tweet">@#{user}</a>#{shouts}: #{tweet['text']}? 
-      #TODO make the caption a bigger matter of yml/ general configuration
-    end
-    tumblr_post
-  end
-  
-  #extract a linked image file's url from a tweet. first found image will be used.
-  def extract_image_url(tweet)
-    links = TwitterProcessor::extract_links tweet
-    image_url = nil
-    if links
-      links.each do |link|
-        image_url = PhotoServiceProcessor::find_image_url(link)
-        return image_url if PhotoServiceProcessor::photo? image_url
-      end
-    end
-    image_url
-  end
-  
+    log.info "finished tweetlr crawl."
+    return config
+  end  
 end