tweetlr 0.1.5 → 0.1.6

data/README.md

@@ -13,8 +13,7 @@ tweetlr supports
 - twitpic
 - yfrog
 - imgly
-- lockerz / the service formerly known as plixi
-- foursquare
+- twitter / photobucket
 - t.co shortened links to pictures
 - every service accessible via embed.ly (see [photo providers](http://embed.ly/providers))
 
@@ -38,7 +37,7 @@ tumblr_password: YOUR_TUMBLR_PW
 update_period: 300 #check for updates every 300 secs = 5 minutes
 shouts: 'says' # will be concatenated after the username, before the message: @mr_x says: awesome things on a photo!
 loglevel: 1 # 0: debug, 1: info (default), 2: warn, 3: error, 5: fatal
-whitelist: #twitter accounts in that list will have their tweets published immediately. post from others will be saved as drafts
+whitelist: #twitter accounts in that list will have their tweets published immediately. post from others will be saved as drafts. blank list will publish all tweets immediately.
   - whitey_mc_whitelist
   - sven_kr
 ```

data/config/tweetlr.yml

@@ -9,6 +9,6 @@ tumblr_password: YOUR_TUMBLR_PW
 update_period: 300 #check for updates every 300 secs = 5 minutes
 shouts: 'says' # will be concatenated after the username, before the message: @mr_x says: awesome things on a photo!
 loglevel: 0 # 0: debug, 1: info (default), 2: warn, 3: error, 5: fatal
-whitelist: #twitter accounts in that list will have their tweets published immediately. post from others will be saved as drafts
+whitelist: #twitter accounts in that list will have their tweets published immediately. post from others will be saved as drafts. blank list will publish all tweets immediately
   - whitey_mc_whitelist
   - sven_kr

data/lib/tweetlr.rb

@@ -6,7 +6,7 @@ require 'json'
 
 class Tweetlr
 
-  VERSION = '0.1.5'
+  VERSION = '0.1.6'
   GENERATOR = %{tweetlr - http://tweetlr.5v3n.com}
   USER_AGENT = %{Mozilla/5.0 (compatible; tweetlr/#{VERSION}; +http://tweetlr.5v3n.com)}
   LOCATION_START_INDICATOR = 'Location: '
@@ -18,6 +18,8 @@ class Tweetlr
   TWITTER_RESULTS_TYPE = 'recent'
   UPDATE_PERIOD = 600 #10 minutes
   
+  PIC_REGEXP = /(.*?)\.(jpg|jpeg|png|gif)/i 
+  
   def initialize(email, password, args={:terms=>nil, :whitelist => nil, :shouts => nil, :since_id=>nil, :results_per_page => nil, :loglevel=>nil, :result_type => nil})
     @log = Logger.new(STDOUT)
     if (Logger::DEBUG..Logger::UNKNOWN).to_a.index(args[:loglevel])
@@ -35,10 +37,10 @@ class Tweetlr
     @result_type = args[:result_type] || TWITTER_RESULTS_TYPE
     @api_endpoint_twitter = args[:api_endpoint_twitter] || API_ENDPOINT_TWITTER
     @api_endpoint_tumblr = args[:api_endpoint_tumblr] || API_ENDPOINT_TUMBLR
-    @whitelist = args[:whitelist] || []
+    @whitelist = args[:whitelist]
     @shouts = args[:shouts]
     @update_period = args[:update_period] || UPDATE_PERIOD
-    @whitelist.each {|entry| entry.downcase!}
+    @whitelist.each {|entry| entry.downcase!} if @whitelist
     @refresh_url = "#{@api_endpoint_twitter}?ors=#{@search_term}&since_id=#{@since_id}&rpp=#{@results_per_page}&result_type=#{@result_type}" if (@since_id && @search_term)  
   end
   #post a tumblr photo entry. required arguments are :type, :date, :source, :caption, :state. optional argument: :tags 
@@ -85,14 +87,15 @@ class Tweetlr
       user = tweet['from_user']
       tumblr_post[:tags] = user
       tweet_id = tweet['id']
-      if @whitelist.member? user.downcase
+      if !@whitelist || @whitelist.member?(user.downcase)
         state = 'published'
       else
         state = 'draft'
       end
       tumblr_post[:state] = state
       shouts = " #{@shouts}" if @shouts
-      tumblr_post[:caption] = %?<a href="http://twitter.com/#{user}/statuses/#{tweet_id}" alt="tweet">@#{user}</a>#{shouts}: #{tweet['text']}? #TODO make this a bigger matter of yml configuration
+      tumblr_post[:caption] = %?<a href="http://twitter.com/#{user}/statuses/#{tweet_id}" alt="tweet">@#{user}</a>#{shouts}: #{tweet['text']}? 
+      #TODO make the caption a bigger matter of yml/ general configuration
     end
     tumblr_post
   end
@@ -120,10 +123,17 @@ class Tweetlr
     end
   end
   
-  #extract the linked image file's url from a tweet
+  #extract a linked image file's url from a tweet. first found image will be used.
   def extract_image_url(tweet)
-    link = extract_link tweet
-    find_image_url link
+    links = extract_links tweet
+    image_url = nil
+    if links
+      links.each do |link|
+        image_url = find_image_url(link)
+        return image_url if image_url =~ PIC_REGEXP 
+      end
+    end
+    image_url
   end
   
   #extract the linked image file's url from a tweet
@@ -206,7 +216,7 @@ class Tweetlr
     tries = 3
     begin
       resp = Curl::Easy.http_get(short_url) { |res| res.follow_location = true }
-    rescue => err
+    rescue Curl::Err => err
         @log.error "Curl::Easy.http_get failed: #{err}"
         tries -= 1
         sleep 3
@@ -230,15 +240,11 @@ class Tweetlr
     link.split('/').last if link.split('/')
   end
 
-  #extract the link from a given tweet
-  def extract_link(tweet)
+  #extract the links from a given tweet
+  def extract_links(tweet)
     if tweet
       text = tweet['text']
-      start = text.index('http') if text
-      if start
-        stop = text.index(' ', start) || 0
-        text[start..stop-1]
-      end
+      text.gsub(/https?:\/\/[\S]+/).to_a if text
     end
   end
   
@@ -262,24 +268,6 @@ class Tweetlr
           return nil
         end
       end
-    rescue Curl::Err::ConnectionFailedError => err
-      @log.error "Connection failed: #{err}"
-      tries -= 1
-      sleep 3
-      if tries > 0
-          retry
-      else
-          nil
-      end
-    rescue Curl::Err::RecvError => err
-      @log.error "Failure when receiving data from the peer: #{err}"
-      tries -= 1
-      sleep 3
-      if tries > 0
-          retry
-      else
-          nil
-      end
     rescue Curl::Err => err
       @log.error "Failure in Curl call: #{err}"
       tries -= 1

data/spec/spec_helper.rb

@@ -7,6 +7,11 @@ def stub_twitter
   Curl::Easy.any_instance.stub(:perform).and_return Curl::Easy.new
 end
 
+def stub_twitter_pics
+  Curl::Easy.any_instance.stub(:body_str).and_return %|{"url": "http://pic.twitter.com/stubbedpic.jpg:large"}|
+  Curl::Easy.any_instance.stub(:perform).and_return Curl::Easy.new
+end
+
 def stub_instagram
   Curl::Easy.any_instance.stub(:body_str).and_return %|{"provider_url": "http://instagram.com/", "title": "Curse you tweets. See what you have done to me?!!!", "url": "http://distillery.s3.amazonaws.com/media/2011/05/02/d25df62b9cec4a138967a3ad027d055b_7.jpg", "author_name": "loswhit", "height": 612, "width": 612, "version": "1.0", "author_url": "http://instagram.com/", "provider_name": "Instagram", "type": "photo"}|
   Curl::Easy.any_instance.stub(:perform).and_return Curl::Easy.new
@@ -58,7 +63,8 @@ Server: nginx/1.0.0 + Phusion Passenger 3.0.7 (mod_rails/mod_rack)
 end
 
 def stub_twitpic
-  Curl::Easy.any_instance.stub(:perform).and_return Curl::Easy.new
+  curl = Curl::Easy.new
+  Curl::Easy.any_instance.stub(:perform).and_return curl
   Curl::Easy.any_instance.stub(:header_str).and_return %|HTTP/1.1 302 Moved Temporarily
 Server: nginx
 Date: Sun, 17 Jul 2011 01:03:43 GMT
@@ -73,6 +79,9 @@ Expires: Sun, 17 Jul 2011 03:50:23 GMT
 Pragma: public
 Location: http://s3.amazonaws.com/twitpic/photos/full/249034281.jpg?AWSAccessKeyId=AKIAJF3XCCKACR3QDMOA&Expires=1310865623&Signature=KNFdFAK%2Bu0u3maMaguUjsm2MbaM%3D\r\n
 |
+  #stub redirected call's response
+  Curl::Easy.stub!(:http_get).and_return curl
+  Curl::Easy.any_instance.stub(:body_str).and_return %|http://s3.amazonaws.com/twitpic/photos/full/249034281.jpg?AWSAccessKeyId=AKIAJF3XCCKACR3QDMOA&Expires=1310865623&Signature=KNFdFAK%2Bu0u3maMaguUjsm2MbaM%3D\r\n|
 end
 
 def stub_tco

data/spec/tweetlr_spec.rb

@@ -12,10 +12,6 @@ describe Tweetlr do
   before :each do
     @credentials = {:email => USER, :password => PW}
     @searchterm = 'fail'
-    @twitter_response = {"from_user_id_str"=>"1915714", "profile_image_url"=>"http://a0.twimg.com/profile_images/386000279/2_normal.jpg", "created_at"=>"Sun, 17 Apr 2011 16:48:42 +0000", "from_user"=>"whitey_Mc_whIteLIst", "id_str"=>"59659561224765440", "metadata"=>{"result_type"=>"recent"}, "to_user_id"=>nil, "text"=>"Rigaer #wirsounterwegs   @ Augenarzt Dr. Lierow http://instagr.am/p/DzCWn/", "id"=>59659561224765440, "from_user_id"=>1915714, "geo"=>{"type"=>"Point", "coordinates"=>[52.5182, 13.454]}, "iso_language_code"=>"de", "place"=>{"id"=>"3078869807f9dd36", "type"=>"city", "full_name"=>"Berlin, Berlin"}, "to_user_id_str"=>nil, "source"=>"<a href="http://instagr.am" rel="nofollow">instagram</a>"}
-    @non_whitelist_tweet = @twitter_response.merge 'from_user' => 'nonwhitelist user' 
-    @retweet = @twitter_response.merge "text" => "bla bla RT @fgd: tueddelkram"
-    @new_style_retweet = @twitter_response.merge "text" => "and it scales! \u201c@moeffju: http://t.co/8gUSPKu #hktbl1 #origami success! :)\u201d"
     @links = {
       :instagram => "http://instagr.am/p/DzCWn/",
       :twitpic => "http://twitpic.com/449o2x",
@@ -24,12 +20,30 @@ describe Tweetlr do
       :imgly => "http://img.ly/3M1o",
       :tco => 'http://t.co/MUGNayA',
       :lockerz => 'http://lockerz.com/s/100269159',
-      :foursquare => 'http://4sq.com/mLKDdF',
-      :embedly => 'http://flic.kr/p/973hTv' 
+      :embedly => 'http://flic.kr/p/973hTv',
+      :twitter_pics => 'http://t.co/FmyBGfyY' 
       }
-    @pic_regexp = /(.*?)\.(jpg|jpeg|png|gif)$/i 
+    @tweets = {
+      :instagram => {'text' => "jadda jadda http://instagr.am/p/DzCWn/"},
+      :twitpic => {'text' => "jadda jadda http://twitpic.com/449o2x"},
+      :yfrog => {'text' => "jadda jadda http://yfrog.com/h4vlfp"},
+      :picplz => {'text' => "jadda jadda http://picplz.com/2hWv"},
+      :imgly => {'text' => "jadda jadda http://img.ly/3M1o"},
+      :tco => {'text' => "jadda jadda http://t.co/MUGNayA"},
+      :lockerz => {'text' => "jadda jadda http://lockerz.com/s/100269159"},
+      :embedly => {'text' => "jadda jadda http://flic.kr/p/973hTv"},
+      :twitter_pics => {'text' => "jadda jadda http://t.co/FmyBGfyY"} 
+      }
+    @first_link = "http://url.com"
+    @second_link = @links[:instagram]
+    @third_link = "https://imageurl.com"
+    @twitter_response = {"from_user_id_str"=>"1915714", "profile_image_url"=>"http://a0.twimg.com/profile_images/386000279/2_normal.jpg", "created_at"=>"Sun, 17 Apr 2011 16:48:42 +0000", "from_user"=>"whitey_Mc_whIteLIst", "id_str"=>"59659561224765440", "metadata"=>{"result_type"=>"recent"}, "to_user_id"=>nil, "text"=>"Rigaer #wirsounterwegs #{@first_link}  @ Augenarzt Dr. Lierow #{@second_link} #{@third_link}", "id"=>59659561224765440, "from_user_id"=>1915714, "geo"=>{"type"=>"Point", "coordinates"=>[52.5182, 13.454]}, "iso_language_code"=>"de", "place"=>{"id"=>"3078869807f9dd36", "type"=>"city", "full_name"=>"Berlin, Berlin"}, "to_user_id_str"=>nil, "source"=>"<a href="http://instagr.am" rel="nofollow">instagram</a>"}
+    @non_whitelist_tweet = @twitter_response.merge 'from_user' => 'nonwhitelist user' 
+    @retweet = @twitter_response.merge "text" => "bla bla RT @fgd: tueddelkram"
+    @new_style_retweet = @twitter_response.merge "text" => "and it scales! \u201c@moeffju: http://t.co/8gUSPKu #hktbl1 #origami success! :)\u201d"
+    @pic_regexp = /(.*?)\.(jpg|jpeg|png|gif)/i 
     @config_file = File.join( Dir.pwd, 'config', 'tweetlr.yml')
-    @tweetlr = Tweetlr.new(USER, PW, {:whitelist => WHITELIST, :results_per_page => 5, :since_id => TIMESTAMP, :terms => @searchterm, :loglevel => 4, :cookie => @cookie})
+    @tweetlr = Tweetlr.new(USER, PW, {:whitelist => WHITELIST, :results_per_page => 5, :since_id => TIMESTAMP, :terms => @searchterm, :loglevel => 4})
   end
   # it "should post to tumblr" do
   #   tumblr_post = @tweetlr.generate_tumblr_photo_post @twitter_response
@@ -46,15 +60,35 @@ describe Tweetlr do
     tweets.should be
     tweets.should_not be_empty
   end
-  it "should mark whitelist users' tweets as published" do
-    stub_instagram
-    post = @tweetlr.generate_tumblr_photo_post @twitter_response
-    post[:state].should == 'published' 
+  context "given a user whitelist" do
+    it "should mark whitelist users' tweets as published" do
+      stub_instagram
+      post = @tweetlr.generate_tumblr_photo_post @twitter_response
+      post[:state].should == 'published' 
+    end
+    it "should mark non whitelist users' tweets as drafts" do
+      stub_instagram
+      post = @tweetlr.generate_tumblr_photo_post @non_whitelist_tweet
+      post[:state].should == 'draft' 
+    end
   end
-  it "should mark non whitelist users' tweets as drafts" do
-    stub_instagram
-    post = @tweetlr.generate_tumblr_photo_post @non_whitelist_tweet
-    post[:state].should == 'draft' 
+  context "without a user whitelist" do
+    before :each do
+      @tweetlr = Tweetlr.new(USER, PW, {
+        :whitelist => nil, 
+        :results_per_page => 5, 
+        :since_id => TIMESTAMP, 
+        :terms => @searchterm, 
+        :loglevel => 4})
+    end
+    it "should mark every users' posts as published" do
+      stub_instagram
+      post = @tweetlr.generate_tumblr_photo_post @twitter_response
+      post[:state].should == 'published'
+      stub_instagram
+      post = @tweetlr.generate_tumblr_photo_post @non_whitelist_tweet
+      post[:state].should == 'published'
+    end
   end
   it "should not use retweets which would produce double blog posts" do
     post = @tweetlr.generate_tumblr_photo_post @retweet
@@ -64,13 +98,13 @@ describe Tweetlr do
     post = @tweetlr.generate_tumblr_photo_post @new_style_retweet
     post.should_not be
   end
-  describe "image url processing" do
+  context "image url processing" do
     it "should find a picture's url from the supported services" do
       @links.each do |key,value|
         send "stub_#{key}"
         url = @tweetlr.find_image_url value
         url.should be, "service #{key} not working!"
-        check_pic_url_extraction key if [:instagram,:picplz,:yfrog,:imgly,:foursquare,:not_listed].index key
+        check_pic_url_extraction key if [:instagram,:picplz,:yfrog,:imgly,:not_listed].index key
       end
     end
     it "should not crash if embedly fallback won't find a link" do
@@ -83,23 +117,36 @@ describe Tweetlr do
     end
   end
   describe "tweet api response processing" do
-    it "should extract links" do
-      tweetlr = @tweetlr
-      link = tweetlr.extract_link @twitter_response
-      link.should == @links[:instagram]
-      link = tweetlr.extract_link @twitter_response.merge 'text' => @links[:instagram].chop #check if it works w/o the trailing slash
-      link.should == @links[:instagram].chop
+    it "extracts links" do
+      links = @tweetlr.extract_links ''
+      links.should be_nil
+      links = @tweetlr.extract_links @twitter_response
+      links[0].should == @first_link
+      links[1].should == @second_link
+      links[2].should == @third_link
     end 
+    it "uses the first image link found in a tweet with multiple links" do
+      stub_instagram
+      link = @tweetlr.extract_image_url @twitter_response
+      link.should == 'http://distillery.s3.amazonaws.com/media/2011/05/02/d25df62b9cec4a138967a3ad027d055b_7.jpg'
+    end
     it "follows redirects" do
       stub_imgly
       link = @tweetlr.link_url_redirect 'im mocked anyways'
       link.should == 'http://s3.amazonaws.com/imgly_production/899582/full.jpg'
     end
+    it "extracts pictures from links" do
+      @tweets.each do |key,value|
+        send "stub_#{key}"
+        url = @tweetlr.extract_image_url value
+        url.should be, "service #{key} not working!"
+        check_pic_url_extraction key if [:instagram,:picplz,:yfrog,:imgly,:not_listed].index key
+      end
+    end
   end
   
   def check_pic_url_extraction(service)
-    tweetlr = @tweetlr
-    image_url = tweetlr.send "image_url_#{service}".to_sym, @links[service]
+    image_url = @tweetlr.send "image_url_#{service}".to_sym, @links[service]
     image_url.should =~ @pic_regexp 
   end
   

data/tweetlr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = "tweetlr"
-  s.version     = "0.1.5"
+  s.version     = "0.1.6"
   s.author      = "Sven Kraeuter"
   s.email       = "sven.kraeuter@gmail.com"
   s.homepage    = "http://tweetlr.5v3n.com"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tweetlr
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-10-04 00:00:00.000000000Z
+date: 2011-11-05 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons
-  requirement: &2153779740 !ruby/object:Gem::Requirement
+  requirement: &2156339340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2153779740
+  version_requirements: *2156339340
 - !ruby/object:Gem::Dependency
   name: eventmachine
-  requirement: &2153805660 !ruby/object:Gem::Requirement
+  requirement: &2156338900 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2153805660
+  version_requirements: *2156338900
 - !ruby/object:Gem::Dependency
   name: curb
-  requirement: &2153805240 !ruby/object:Gem::Requirement
+  requirement: &2156338480 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2153805240
+  version_requirements: *2156338480
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &2153804820 !ruby/object:Gem::Requirement
+  requirement: &2156338060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2153804820
+  version_requirements: *2156338060
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &2153804320 !ruby/object:Gem::Requirement
+  requirement: &2156337560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 0.8.7
   type: :development
   prerelease: false
-  version_requirements: *2153804320
+  version_requirements: *2156337560
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2153803900 !ruby/object:Gem::Requirement
+  requirement: &2156337140 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2153803900
+  version_requirements: *2156337140
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &2153803440 !ruby/object:Gem::Requirement
+  requirement: &2156336680 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,7 +87,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2153803440
+  version_requirements: *2156336680
 description: tweetlr crawls twitter for a given term, extracts photos out of the collected
   tweets' short urls and posts the images to tumblr.
 email: sven.kraeuter@gmail.com