tweetlr 0.1.20 → 0.1.21

data/.gitignore

@@ -1,9 +1,10 @@
 *.swp
+log/
 *.log
 pkg
 *.pid
 *.output
-.rvmrc
+.ruby-version
 Gemfile.lock
 tweetlr.tid
 tweetlr.yml.dev

data/.travis.yml

@@ -5,6 +5,7 @@ bundler_args: --binstubs
 rvm:
   - 1.9.2
   - 1.9.3
+  - 2.0.0
   #- 1.8.7 # (current default)
 
 # Define how to run your tests (defaults to `bundle exec rake` or `rake` depending on whether you have a `Gemfile`)

data/Gemfile

@@ -1,2 +1,2 @@
-source :rubygems
+source 'https://rubygems.org'
 gemspec

data/README.md

@@ -10,17 +10,16 @@ There is a new [tweetlr "as-a-service"](http://tweetlr.5v3n.com) where you can e
 
 tweetlr supports
 
-- eyeem.com
+- instagram
 - twitter
   - photobucket
   - twimg
 - foursquare
 - path.com
-- picplz
 - twitpic
 - yfrog
 - imgly
-- instagram
+- eyeem.com
 - t.co shortened links to pictures
 - every photo service accessible via embed.ly (see [photo providers](http://embed.ly/providers))
 

data/lib/tweetlr/core.rb

@@ -15,24 +15,8 @@ class Tweetlr::Core
   end
   
   def initialize(args)
-    log = Logger.new(STDOUT)
-    if (Logger::DEBUG..Logger::UNKNOWN).to_a.index(args[:loglevel])
-      log.level = args[:loglevel] 
-    else
-      log.level = Logger::INFO
-    end
-    log.debug "log level set to #{log.level}"
-    Tweetlr::LogAware.log=log
-    
-    @email = args[:tumblr_email]
-    @password = args[:tumblr_password]
-    @cookie = args[:cookie]
-    @api_endpoint_twitter = args[:api_endpoint_twitter] || Tweetlr::API_ENDPOINT_TWITTER
-    @api_endpoint_tumblr = args[:api_endpoint_tumblr] || Tweetlr::API_ENDPOINT_TUMBLR
-    @whitelist = args[:whitelist]
-    @shouts = args[:shouts]
-    @update_period = args[:update_period] || Tweetlr::UPDATE_PERIOD
-    @whitelist.each {|entry| entry.downcase!} if @whitelist
+    initialize_logging(args[:loglevel])
+    initialize_attributes(args)
     log.info "Tweetlr #{Tweetlr::VERSION} initialized. Ready to roll."
   end
   
@@ -55,26 +39,51 @@ class Tweetlr::Core
     return config
   end
 private
+  def initialize_attributes(args)
+    @email = args[:tumblr_email]
+    @password = args[:tumblr_password]
+    @cookie = args[:cookie]
+    @api_endpoint_twitter = args[:api_endpoint_twitter] || Tweetlr::API_ENDPOINT_TWITTER
+    @api_endpoint_tumblr = args[:api_endpoint_tumblr] || Tweetlr::API_ENDPOINT_TUMBLR
+    @whitelist = args[:whitelist]
+    @shouts = args[:shouts]
+    @update_period = args[:update_period] || Tweetlr::UPDATE_PERIOD
+    @whitelist.each {|entry| entry.downcase!} if @whitelist
+  end
+  def initialize_logging(loglevel)
+    log = Logger.new(STDOUT)
+    if (Logger::DEBUG..Logger::UNKNOWN).to_a.index(loglevel)
+      log.level = loglevel 
+    else
+      log.level = Logger::INFO
+    end
+    log.debug "log level set to #{log.level}"
+    Tweetlr::LogAware.log=log
+  end 
   def self.process_response(response, config, tumblr_config)
     tweets = response['results']
-    if tweets
-      tweets.each do |tweet|
-        tumblr_post = Tweetlr::Combinators::TwitterTumblr::generate_photo_post_from_tweet(tweet, {:whitelist => config[:whitelist], :embedly_key => config[:embedly_key], :group => config[:group]}) 
-        if tumblr_post.nil? ||  tumblr_post[:source].nil?
-          log.warn "could not get image source: tweet: #{tweet} --- tumblr post: #{tumblr_post.inspect}"
-        else
-          log.debug "tumblr post: #{tumblr_post}"
-          res = Tweetlr::Processors::Tumblr.post tumblr_post.merge(tumblr_config)
-          log.debug "tumblr response: #{res}"
-          if res && res.code == "201"
-            log.info "tumblr post created (tumblr response: #{res.header} #{res.body}"
-          elsif res
-            log.warn "tumblr response: #{res.header} #{res.body}"
-          else
-            log.warn "there was no tumblr post response - most probably due to a missing oauth authorization"
-          end
-        end
+    process_and_post tweets, config, tumblr_config if tweets
+  end
+  def self.process_and_post(tweets, config, tumblr_config)
+    tweets.each do |tweet|
+      tumblr_post = Tweetlr::Combinators::TwitterTumblr::generate_photo_post_from_tweet(tweet, {:whitelist => config[:whitelist], :embedly_key => config[:embedly_key], :group => config[:group]}) 
+      if tumblr_post.nil? ||  tumblr_post[:source].nil?
+        log.warn "could not get image source: tweet: #{tweet} --- tumblr post: #{tumblr_post.inspect}"
+      else
+        post_to_tumblr
       end
+    end    
+  end
+  def self.post_to_tumblr(tumblr_post, tumblr_config)
+    log.debug "tumblr post: #{tumblr_post}"
+    res = Tweetlr::Processors::Tumblr.post tumblr_post.merge(tumblr_config)
+    log.debug "tumblr response: #{res}"
+    if res && res.code == "201"
+      log.info "tumblr post created (tumblr response: #{res.header} #{res.body}"
+    elsif res
+      log.warn "tumblr response: #{res.header} #{res.body}"
+    else
+      log.warn "there was no tumblr post response - most probably due to a missing oauth authorization"
     end
   end
   def self.prepare_twitter_config(config)

data/lib/tweetlr/processors/photo_service.rb

@@ -22,15 +22,13 @@ module Tweetlr::Processors
       if link && !(photo? link)
         url = image_url_eyeem link if link.index 'eyeem.com'
         url = image_url_instagram link if (link.index('instagr.am') || link.index('instagram.com'))
-        url = image_url_picplz link if link.index 'picplz'
         url = image_url_twitpic link if link.index 'twitpic'
         url = image_url_yfrog link if link.index 'yfrog'
-        url = image_url_imgly link if link.index 'img.ly'
+        url = image_url_imgly link, embedly_key if link.index 'img.ly'
         url = image_url_tco link, embedly_key if link.index 't.co'
         url = image_url_twimg link if link.index 'twitter.com'
-        url = image_url_lockerz link if link.index 'lockerz.com'
         url = image_url_path link if link.index 'path.com'
-        url = image_url_foursqaure link if link.index '4sq.com'
+        url = image_url_foursqaure link if (link.index('4sq.com') || link.index('foursquare.com'))
         url = image_url_embedly link, embedly_key if url.nil? #just try embed.ly for anything else. could do all image url processing w/ embedly, but there's probably some kind of rate limit invovled.
       elsif photo? link
         url = link
@@ -42,7 +40,7 @@ module Tweetlr::Processors
       link =~ PIC_REGEXP
     end
     def self.image_url_twimg(link_url)
-      retrieve_image_url_by_css link_url, '.twimg img'
+      retrieve_image_url_by_css link_url, '.media img'
     end
     #extract the image of an eyeem.com pic
     def self.image_url_eyeem(link_url)
@@ -50,7 +48,9 @@ module Tweetlr::Processors
     end
     #extract the image of a foursquare.com pic
     def self.image_url_foursqaure(link_url)
-      retrieve_image_url_by_css link_url, '.commentPhoto img'
+      link_url = follow_redirect(link_url)
+      image_url = retrieve_image_url_by_css link_url, 'meta[property="og:image"]', 'content'
+      image_url
     end
     #extract the image of a path.com pic
     def self.image_url_path(link_url)
@@ -59,18 +59,14 @@ module Tweetlr::Processors
   
     #find the image's url via embed.ly
     def self.image_url_embedly(link_url, key)
-      response = Tweetlr::Processors::Http::http_get_json "http://api.embed.ly/1/oembed?key=#{key}&url=#{link_url}"
+      link_url = follow_redirect(link_url)
       log.debug "embedly call: http://api.embed.ly/1/oembed?key=#{key}&url=#{link_url}"
-      if response && response['type'] == 'photo'
+      response = Tweetlr::Processors::Http::http_get_json "http://api.embed.ly/1/oembed?key=#{key}&url=#{link_url}"
+      if response && (response['type'] == 'photo' || response['type'] == 'image')
         image_url = response['url'] 
       end
       image_url
     end
-    #find the image's url for a lockerz link
-    def self.image_url_lockerz(link_url)
-      response = Tweetlr::Processors::Http::http_get_json "http://api.plixi.com/api/tpapi.svc/json/metadatafromurl?details=false&url=#{link_url}"
-      response["BigImageUrl"] if response
-    end
     #find the image's url for an twitter shortened link
     def self.image_url_tco(link_url, embedly_key = nil)
       service_url = link_url_redirect link_url
@@ -82,33 +78,17 @@ module Tweetlr::Processors
       response = Tweetlr::Processors::Http::http_get_json "http://api.instagram.com/oembed?url=#{link_url}"
       response['url'] if response
     end
-
-    #find the image's url for a picplz short/longlink
-    def self.image_url_picplz(link_url)
-      id = extract_id link_url
-      #try short url
-      response = Tweetlr::Processors::Http::http_get_json "http://picplz.com/api/v2/pic.json?shorturl_ids=#{id}"
-      #if short url fails, try long url
-      #response = HTTParty.get "http://picplz.com/api/v2/pic.json?longurl_ids=#{id}"
-      #extract url
-      if response && response['value'] && response['value']['pics'] && response['value']['pics'].first && response['value']['pics'].first['pic_files'] && response['value']['pics'].first['pic_files']['640r']
-        response['value']['pics'].first['pic_files']['640r']['img_url'] 
-      else
-        nil
-      end
-    end
     #find the image's url for a twitpic link
     def self.image_url_twitpic(link_url)
       image_url_redirect link_url, "http://twitpic.com/show/full/"
     end
     #find the image'S url for a yfrog link
     def self.image_url_yfrog(link_url)
-      response = Tweetlr::Processors::Http::http_get_json("http://www.yfrog.com/api/oembed?url=#{link_url}")
-      response['url'] if response
+      retrieve_image_url_by_css link_url, '#input-direct', 'value'
     end
     #find the image's url for a img.ly link
-    def self.image_url_imgly(link_url)
-      image_url_redirect link_url, "http://img.ly/show/full/", "\r\n"
+    def self.image_url_imgly(link_url, embedly_key)
+      retrieve_image_url_by_css link_url, '#the-image'
     end
   
     # extract image url from services like twitpic & img.ly that do not offer oembed interfaces
@@ -121,22 +101,12 @@ module Tweetlr::Processors
       begin
         resp = Curl::Easy.http_get(short_url) { |res| res.follow_location = true }
       rescue Curl::Err::CurlError => err
-          log.error "Curl::Easy.http_get failed: #{err}"
-          tries -= 1
-          sleep 3
-          if tries > 0
-              retry
-          else
-             return nil
-          end
-      end
-      if(resp && resp.header_str && resp.header_str.index(LOCATION_START_INDICATOR) && resp.header_str.index(stop_indicator))
-        start = resp.header_str.index(LOCATION_START_INDICATOR) + LOCATION_START_INDICATOR.size
-        stop  = resp.header_str.index(stop_indicator, start)
-        resp.header_str[start...stop]
-      else
-        nil
+        log.error "Curl::Easy.http_get failed: #{err}"
+        tries -= 1
+        sleep 3
+        (tries > 0) ? retry : return
       end
+      process_reponse_header resp, stop_indicator
     end
   
     #extract the pic id from a given <code>link</code>
@@ -144,22 +114,36 @@ module Tweetlr::Processors
       link.split('/').last if link.split('/')
     end
     #parse html doc for element signature
-    def self.parse_html_for(element_signature, html_doc)
+    def self.parse_html_for(element_signature, html_doc, identifier="src")
       image_url= nil
       if html_doc
         photo_container_div = html_doc.css(element_signature)
-        if photo_container_div && photo_container_div.first && photo_container_div.first.attributes["src"]
-          image_url = photo_container_div.first.attributes["src"].value
+        if photo_container_div && photo_container_div.first && photo_container_div.first.attributes[identifier]
+          image_url = photo_container_div.first.attributes[identifier].value
         end
       end
       image_url
     end
-    def self.retrieve_image_url_by_css link_url, css_path
+    def self.retrieve_image_url_by_css(link_url, css_path, selector='src')
+      link_url = follow_redirect link_url
+      response = Tweetlr::Processors::Http::http_get link_url
+      image_url = parse_html_for css_path, Nokogiri::HTML.parse(response.body_str), selector
+      return image_url
+    end
+private
+    def self.process_reponse_header(resp, stop_indicator)
+      if(resp && resp.header_str && resp.header_str.index(LOCATION_START_INDICATOR) && resp.header_str.index(stop_indicator))
+        start = resp.header_str.index(LOCATION_START_INDICATOR) + LOCATION_START_INDICATOR.size
+        stop  = resp.header_str.index(stop_indicator, start)
+        resp.header_str[start...stop]
+      else
+        nil
+      end
+    end
+    def self.follow_redirect(link_url)
       service_url = link_url_redirect link_url #follow possible redirects
-        link_url = service_url if service_url #if there's no redirect, service_url will be nil
-        response = Tweetlr::Processors::Http::http_get link_url
-        image_url = parse_html_for css_path, Nokogiri::HTML.parse(response.body_str)
-        return image_url
+      link_url = service_url if service_url #if there's no redirect, service_url will be nil
+      link_url
     end
   end
 end

data/lib/tweetlr.rb

@@ -2,7 +2,7 @@ require 'tweetlr/log_aware'
 require 'tweetlr/core'
 
 module Tweetlr
-  VERSION = '0.1.20'
+  VERSION = '0.1.21'
     
   API_ENDPOINT_TWITTER = 'http://search.twitter.com/search.json'
   API_ENDPOINT_TUMBLR = 'http://www.tumblr.com'

data/spec/combinators/twitter_tumblr_combinator_spec.rb

@@ -15,10 +15,8 @@ describe Tweetlr::Combinators::TwitterTumblr do
       :instagram => {'text' => "jadda jadda http://instagr.am/p/DzCWn/"},
       :twitpic => {'text' => "jadda jadda http://twitpic.com/449o2x"},
       :yfrog => {'text' => "jadda jadda http://yfrog.com/h4vlfp"},
-      :picplz => {'text' => "jadda jadda http://picplz.com/2hWv"},
       :imgly => {'text' => "jadda jadda http://img.ly/3M1o"},
       :tco => {'text' => "jadda jadda http://t.co/MUGNayA"},
-      :lockerz => {'text' => "jadda jadda http://lockerz.com/s/100269159"},
       :embedly => {'text' => "jadda jadda http://flic.kr/p/973hTv"},
       :twitter_pics => {'text' => "jadda jadda http://t.co/FmyBGfyY"} 
       }
@@ -26,10 +24,8 @@ describe Tweetlr::Combinators::TwitterTumblr do
       :instagram => "http://instagr.am/p/DzCWn/",
       :twitpic => "http://twitpic.com/449o2x",
       :yfrog => "http://yfrog.com/h4vlfp",
-      :picplz => "http://picplz.com/2hWv",
       :imgly => "http://img.ly/3M1o",
       :tco => 'http://t.co/MUGNayA',
-      :lockerz => 'http://lockerz.com/s/100269159',
       :embedly => 'http://flic.kr/p/973hTv',
       :twitter_pics => 'http://t.co/FmyBGfyY' 
       }

data/spec/processors/photo_services_processor_spec.rb

@@ -3,28 +3,35 @@ require 'spec_helper'
 describe Tweetlr::Processors::PhotoService do
   before :each do
     @links = {
-      :twimg => 'http://twitter.com/KSilbereisen/status/228035435237097472',
-      :eyeem => 'http://www.eyeem.com/p/326629',
       :foursquare => 'http://4sq.com/x4p87N',
+      :eyeem => 'http://www.eyeem.com/p/326629',
       :path => 'http://path.com/p/KQd57', 
       :instagram => "http://instagr.am/p/DzCWn/",
       :twitpic => "http://twitpic.com/449o2x",
       :yfrog => "http://yfrog.com/h4vlfp",
-      :picplz => "http://picplz.com/2hWv",
-      :imgly => "http://img.ly/3M1o",
       :tco => 'http://t.co/MUGNayA',
-      :lockerz => 'http://lockerz.com/s/100269159',
       :embedly => 'http://flic.kr/p/973hTv',
-      :twitter_pics => 'http://t.co/FmyBGfyY'
+      :twitter_pics => 'http://t.co/FmyBGfyY',
+      :twimg => 'http://twitter.com/KSilbereisen/status/228035435237097472',
+      :imgly => "http://img.ly/3M1o"
       }
   end
+  it "finds a picture's url from the supported services" do
+    @links.each do |service,link|
+      send "stub_#{service}"
+      #puts "checking #{service}"
+      url = Tweetlr::Processors::PhotoService::find_image_url link
+      url.should be, "service #{service} not working!"
+      check_pic_url_extraction service if [:twimg, :instagram,:yfrog,:imgly,:foursqaure,:not_listed].index service
+    end
+  end
   it "extracts images from eye em" do
     stub_eyeem
     link = Tweetlr::Processors::PhotoService::find_image_url @links[:eyeem]
     link.should be
     link.should == "http://www.eyeem.com/thumb/h/1024/e35db836c5d3f02498ef60fc3d53837fbe621561-1334126483"
   end
-  it "doesnt find images in embedly results that are not explicitly marked as 'Photo' via the response's 'thumbnail_url' attribute" do
+  it "doesnt find images in embedly results that are not explicitly marked as 'Photo' or 'Image' via the response's 'thumbnail_url' attribute" do
     stub_embedly_no_photo
     link = Tweetlr::Processors::PhotoService::find_image_url 'http://makersand.co/'
     link.should be_nil
@@ -32,16 +39,9 @@ describe Tweetlr::Processors::PhotoService do
   it "does find an image for foursquare that is not he profile pic" do
     stub_foursquare
     link = Tweetlr::Processors::PhotoService::find_image_url @links[:foursquare]
+    link.should be
     link.index('userpix_thumbs').should_not be
   end
-  it "should find a picture's url from the supported services" do
-    @links.each do |service,link|
-      send "stub_#{service}"
-      url = Tweetlr::Processors::PhotoService::find_image_url link
-      url.should be, "service #{service} not working!"
-      check_pic_url_extraction service if [:twimg, :instagram,:picplz,:yfrog,:imgly,:foursqaure,:not_listed].index service
-    end
-  end
   it "finds path images for redirected moments as well" do
     stub_path_redirected
     url = Tweetlr::Processors::PhotoService::find_image_url @links[:path]