tweetlr 0.1.7pre → 0.1.7pre4

data/spec/combinators/twitter_tumblr_combinator_spec.rb

@@ -0,0 +1,93 @@
+require 'spec_helper'
+
+describe Combinators::TwitterTumblr do
+  before :each do
+    @first_link = "http://url.com"
+    @second_link = "http://instagr.am/p/DzCWn/"
+    @third_link = "https://imageurl.com"
+    @twitter_response = {"from_user_id_str"=>"1915714", "profile_image_url"=>"http://a0.twimg.com/profile_images/386000279/2_normal.jpg", "created_at"=>"Sun, 17 Apr 2011 16:48:42 +0000", "from_user"=>"whitey_Mc_whIteLIst", "id_str"=>"59659561224765440", "metadata"=>{"result_type"=>"recent"}, "to_user_id"=>nil, "text"=>"Rigaer #wirsounterwegs #{@first_link}  @ Augenarzt Dr. Lierow #{@second_link} #{@third_link}", "id"=>59659561224765440, "from_user_id"=>1915714, "geo"=>{"type"=>"Point", "coordinates"=>[52.5182, 13.454]}, "iso_language_code"=>"de", "place"=>{"id"=>"3078869807f9dd36", "type"=>"city", "full_name"=>"Berlin, Berlin"}, "to_user_id_str"=>nil, "source"=>"<a href="http://instagr.am" rel="nofollow">instagram</a>"}
+    @retweet = @twitter_response.merge "text" => "bla bla RT @fgd: tueddelkram"
+    @new_style_retweet = @twitter_response.merge "text" => "and it scales! \u201c@moeffju: http://t.co/8gUSPKu #hktbl1 #origami success! :)\u201d"
+    @new_style_retweet_no_addition = @twitter_response.merge "text" => "\u201c@moeffju: http://t.co/8gUSPKu #hktbl1 #origami success! :)\u201d"
+    @non_whitelist_tweet = @twitter_response.merge 'from_user' => 'nonwhitelist user' 
+    @whitelist = ['whitey_mc_whitelist']
+    @tweets = {
+      :instagram => {'text' => "jadda jadda http://instagr.am/p/DzCWn/"},
+      :twitpic => {'text' => "jadda jadda http://twitpic.com/449o2x"},
+      :yfrog => {'text' => "jadda jadda http://yfrog.com/h4vlfp"},
+      :picplz => {'text' => "jadda jadda http://picplz.com/2hWv"},
+      :imgly => {'text' => "jadda jadda http://img.ly/3M1o"},
+      :tco => {'text' => "jadda jadda http://t.co/MUGNayA"},
+      :lockerz => {'text' => "jadda jadda http://lockerz.com/s/100269159"},
+      :embedly => {'text' => "jadda jadda http://flic.kr/p/973hTv"},
+      :twitter_pics => {'text' => "jadda jadda http://t.co/FmyBGfyY"} 
+      }
+    @links = {
+      :instagram => "http://instagr.am/p/DzCWn/",
+      :twitpic => "http://twitpic.com/449o2x",
+      :yfrog => "http://yfrog.com/h4vlfp",
+      :picplz => "http://picplz.com/2hWv",
+      :imgly => "http://img.ly/3M1o",
+      :tco => 'http://t.co/MUGNayA',
+      :lockerz => 'http://lockerz.com/s/100269159',
+      :embedly => 'http://flic.kr/p/973hTv',
+      :twitter_pics => 'http://t.co/FmyBGfyY' 
+      }
+  end
+  context "handles pictures in tweets" do
+    it "extracting their corresponding links" do
+      @tweets.each do |key,value|
+        send "stub_#{key}"
+        url = Combinators::TwitterTumblr.extract_image_url value
+        url.should be, "service #{key} not working!"
+        check_pic_url_extraction key if [:instagram,:picplz,:yfrog,:imgly,:not_listed].index key
+      end
+    end
+    it "using the first image link found in a tweet with multiple links" do
+      stub_instagram
+      link = Combinators::TwitterTumblr.extract_image_url @twitter_response
+      link.should == 'http://distillery.s3.amazonaws.com/media/2011/05/02/d25df62b9cec4a138967a3ad027d055b_7.jpg'
+    end
+    it "not returning links that do not belong to images" do
+      stub_no_image_link
+      link = Combinators::TwitterTumblr.extract_image_url @twitter_response
+      link.should_not be
+    end
+  end
+  context "given a user whitelist" do
+    it "should mark whitelist users' tweets as published" do
+      stub_instagram
+      post = Combinators::TwitterTumblr::generate_photo_post_from_tweet @twitter_response, :whitelist => @whitelist
+      post[:state].should == 'published' 
+    end
+    it "should mark non whitelist users' tweets as drafts" do
+      stub_instagram
+      post = Combinators::TwitterTumblr::generate_photo_post_from_tweet @non_whitelist_tweet, :whitelist => @whitelist
+      post[:state].should == 'draft' 
+    end
+  end
+  context "without a user whitelist" do
+    it "should mark every users' posts as published" do
+      stub_instagram
+      post = Combinators::TwitterTumblr::generate_photo_post_from_tweet @twitter_response, :whitelist => nil
+      post[:state].should == 'published'
+      stub_instagram
+      post = Combinators::TwitterTumblr::generate_photo_post_from_tweet @non_whitelist_tweet, :whitelist => nil
+      post[:state].should == 'published'
+    end
+  end
+  it "should not use retweets which would produce double blog posts" do
+    post = Combinators::TwitterTumblr::generate_photo_post_from_tweet @retweet, :whitelist => @whitelist
+    post.should_not be
+  end
+  context "should not use new style retweets which would produce double blog posts" do
+    it "for quotes in context" do
+      post = Combinators::TwitterTumblr::generate_photo_post_from_tweet @new_style_retweet, :whitelist => @whitelist
+      post.should_not be
+    end
+    it "for quotes without further text addition" do
+      post = Combinators::TwitterTumblr::generate_photo_post_from_tweet @new_style_retweet_no_addition, :whitelist => @whitelist
+      post.should_not be
+    end
+  end
+end

data/spec/{photo_services_processor_spec.rb → processors/photo_services_processor_spec.rb}

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe PhotoServiceProcessor do
+describe Processors::PhotoService do
   before :each do
     @links = {
       :instagram => "http://instagr.am/p/DzCWn/",
@@ -17,22 +17,22 @@ describe PhotoServiceProcessor do
   it "should find a picture's url from the supported services" do
     @links.each do |service,link|
       send "stub_#{service}"
-      url = PhotoServiceProcessor::find_image_url link
+      url = Processors::PhotoService::find_image_url link
       url.should be, "service #{service} not working!"
       check_pic_url_extraction service if [:instagram,:picplz,:yfrog,:imgly,:not_listed].index service
     end
   end
   it "should not crash if embedly fallback won't find a link" do
     stub_bad_request
-    url = PhotoServiceProcessor::find_image_url "http://mopskopf"     
+    url = Processors::PhotoService::find_image_url "http://mopskopf"     
   end
   it "should not crash with an encoding error when response is non-us-ascii" do
     stub_utf8_response
-    url = PhotoServiceProcessor::find_image_url "http://api.instagram.com/oembed?url=http://instagr.am/p/Gx%E2%80%946/"
+    url = Processors::PhotoService::find_image_url "http://api.instagram.com/oembed?url=http://instagr.am/p/Gx%E2%80%946/"
   end
   it "follows redirects" do
     stub_imgly
-    link = PhotoServiceProcessor::link_url_redirect 'im mocked anyways'
+    link = Processors::PhotoService::link_url_redirect 'im mocked anyways'
     link.should == 'http://s3.amazonaws.com/imgly_production/899582/full.jpg'
   end
 end

data/spec/{twitter_processor_spec.rb → processors/twitter_processor_spec.rb}

@@ -1,16 +1,30 @@
 require 'spec_helper'
 
-describe TwitterProcessor do
+describe Processors::Twitter do
   before :each do
     @first_link = "http://url.com"
     @second_link = "http://instagr.am/p/DzCWn/"
     @third_link = "https://imageurl.com"
     @twitter_response = {"from_user_id_str"=>"1915714", "profile_image_url"=>"http://a0.twimg.com/profile_images/386000279/2_normal.jpg", "created_at"=>"Sun, 17 Apr 2011 16:48:42 +0000", "from_user"=>"whitey_Mc_whIteLIst", "id_str"=>"59659561224765440", "metadata"=>{"result_type"=>"recent"}, "to_user_id"=>nil, "text"=>"Rigaer #wirsounterwegs #{@first_link}  @ Augenarzt Dr. Lierow #{@second_link} #{@third_link}", "id"=>59659561224765440, "from_user_id"=>1915714, "geo"=>{"type"=>"Point", "coordinates"=>[52.5182, 13.454]}, "iso_language_code"=>"de", "place"=>{"id"=>"3078869807f9dd36", "type"=>"city", "full_name"=>"Berlin, Berlin"}, "to_user_id_str"=>nil, "source"=>"<a href="http://instagr.am" rel="nofollow">instagram</a>"}
+    @twitter_config = {
+      :since_id => 0,
+      :search_term => 'moped',
+      :results_per_page => 100,
+      :result_type => 'recent',  
+      :api_endpoint_twitter => Tweetlr::API_ENDPOINT_TWITTER
+    }
+  end
+  it "should search twitter for a given term" do
+    stub_twitter
+    response = Processors::Twitter::lazy_search @twitter_config
+    tweets = response['results']
+    tweets.should be
+    tweets.should_not be_empty
   end
   it "extracts links" do
-    links = TwitterProcessor::extract_links ''
+    links = Processors::Twitter::extract_links ''
     links.should be_nil
-    links = TwitterProcessor::extract_links @twitter_response
+    links = Processors::Twitter::extract_links @twitter_response
     links[0].should == @first_link
     links[1].should == @second_link
     links[2].should == @third_link

data/spec/spec_helper.rb

@@ -2,9 +2,31 @@
 require "bundler"
 Bundler.require :default, :development, :test
 
+logger = Logger.new(STDOUT)
+logger.level = Logger::FATAL
+LogAware.log = logger
+
 def check_pic_url_extraction(service)
-  image_url = PhotoServiceProcessor::send "image_url_#{service}".to_sym, @links[service]
-  image_url.should =~ PhotoServiceProcessor::PIC_REGEXP
+  image_url = Processors::PhotoService::send "image_url_#{service}".to_sym, @links[service]
+  image_url.should =~ Processors::PhotoService::PIC_REGEXP
+end
+
+def stub_tumblr
+  Curl::Easy.any_instance.stub(:response_code).and_return 201
+  Curl::Easy.any_instance.stub(:header_str).and_return %|HTTP/1.1 201 Created
+Date: Sun, 13 Nov 2011 16:56:02 GMT
+Server: Apache
+P3P: CP="ALL ADM DEV PSAi COM OUR OTRo STP IND ONL"
+Vary: Accept-Encoding
+X-Tumblr-Usec: D=2600406
+Content-Length: 11
+Connection: close
+Content-Type: text/plain; charset=utf-8
+
+|
+  Curl::Easy.any_instance.stub(:body_str).and_return %|12742797055|
+  Curl::Easy.stub!(:http_post).and_return Curl::Easy.new
+  stub_instagram
 end
 
 def stub_twitter

data/spec/tweetlr_spec.rb

@@ -10,118 +10,32 @@ describe Tweetlr do
   WHITELIST = config['whitelist']
 
   before :each do
-    @credentials = {:email => USER, :password => PW}
-    @searchterm = 'fail'
-    @tweets = {
-      :instagram => {'text' => "jadda jadda http://instagr.am/p/DzCWn/"},
-      :twitpic => {'text' => "jadda jadda http://twitpic.com/449o2x"},
-      :yfrog => {'text' => "jadda jadda http://yfrog.com/h4vlfp"},
-      :picplz => {'text' => "jadda jadda http://picplz.com/2hWv"},
-      :imgly => {'text' => "jadda jadda http://img.ly/3M1o"},
-      :tco => {'text' => "jadda jadda http://t.co/MUGNayA"},
-      :lockerz => {'text' => "jadda jadda http://lockerz.com/s/100269159"},
-      :embedly => {'text' => "jadda jadda http://flic.kr/p/973hTv"},
-      :twitter_pics => {'text' => "jadda jadda http://t.co/FmyBGfyY"} 
-      }
-      @links = {
-        :instagram => "http://instagr.am/p/DzCWn/",
-        :twitpic => "http://twitpic.com/449o2x",
-        :yfrog => "http://yfrog.com/h4vlfp",
-        :picplz => "http://picplz.com/2hWv",
-        :imgly => "http://img.ly/3M1o",
-        :tco => 'http://t.co/MUGNayA',
-        :lockerz => 'http://lockerz.com/s/100269159',
-        :embedly => 'http://flic.kr/p/973hTv',
-        :twitter_pics => 'http://t.co/FmyBGfyY' 
-        }
     @first_link = "http://url.com"
     @second_link = "http://instagr.am/p/DzCWn/"
     @third_link = "https://imageurl.com"
     @twitter_response = {"from_user_id_str"=>"1915714", "profile_image_url"=>"http://a0.twimg.com/profile_images/386000279/2_normal.jpg", "created_at"=>"Sun, 17 Apr 2011 16:48:42 +0000", "from_user"=>"whitey_Mc_whIteLIst", "id_str"=>"59659561224765440", "metadata"=>{"result_type"=>"recent"}, "to_user_id"=>nil, "text"=>"Rigaer #wirsounterwegs #{@first_link}  @ Augenarzt Dr. Lierow #{@second_link} #{@third_link}", "id"=>59659561224765440, "from_user_id"=>1915714, "geo"=>{"type"=>"Point", "coordinates"=>[52.5182, 13.454]}, "iso_language_code"=>"de", "place"=>{"id"=>"3078869807f9dd36", "type"=>"city", "full_name"=>"Berlin, Berlin"}, "to_user_id_str"=>nil, "source"=>"<a href="http://instagr.am" rel="nofollow">instagram</a>"}
-    @non_whitelist_tweet = @twitter_response.merge 'from_user' => 'nonwhitelist user' 
-    @retweet = @twitter_response.merge "text" => "bla bla RT @fgd: tueddelkram"
-    @new_style_retweet = @twitter_response.merge "text" => "and it scales! \u201c@moeffju: http://t.co/8gUSPKu #hktbl1 #origami success! :)\u201d"
-    @new_style_retweet_no_addition = @twitter_response.merge "text" => "\u201c@moeffju: http://t.co/8gUSPKu #hktbl1 #origami success! :)\u201d"
-    @config_file = File.join( Dir.pwd, 'config', 'tweetlr.yml')
-    @tweetlr = Tweetlr.new(USER, PW, {:whitelist => WHITELIST, :results_per_page => 5, :since_id => TIMESTAMP, :terms => @searchterm, :loglevel => 4})
+    @tweetlr_config =  {
+      :tumblr_email => USER,
+      :tumblr_pw => PW,
+      :whitelist => WHITELIST,
+      :since_id => 0,
+      :search_term => 'moped',
+      :results_per_page => 100,
+      :result_type => 'recent',  
+      :api_endpoint_twitter => Tweetlr::API_ENDPOINT_TWITTER
+    }
   end
-  # it "should post to tumblr" do
-  #   tumblr_post = @tweetlr.generate_tumblr_photo_post @twitter_response
-  #   tumblr_post[:date] = Time.now.to_s
-  #   response = @tweetlr.post_to_tumblr tumblr_post
-  #   response.should be
-  #   response.response_code.should be 201
-  # end
-  it "should search twitter for a given term" do
-    stub_twitter
-    tweetlr = @tweetlr
-    response = tweetlr.lazy_search_twitter
-    tweets = response['results']
-    tweets.should be
-    tweets.should_not be_empty
+  it "should post to tumblr" do
+    stub_tumblr
+    tumblr_post = Combinators::TwitterTumblr::generate_photo_post_from_tweet @twitter_response
+    tumblr_post[:date] = Time.now.to_s
+    response = Processors::Tumblr::post tumblr_post.merge({:email => USER, :password => PW})
+    response.should be
+    response.response_code.should be 201
   end
-  context "given a user whitelist" do
-    it "should mark whitelist users' tweets as published" do
-      stub_instagram
-      post = @tweetlr.generate_tumblr_photo_post @twitter_response
-      post[:state].should == 'published' 
-    end
-    it "should mark non whitelist users' tweets as drafts" do
-      stub_instagram
-      post = @tweetlr.generate_tumblr_photo_post @non_whitelist_tweet
-      post[:state].should == 'draft' 
-    end
-  end
-  context "without a user whitelist" do
-    before :each do
-      @tweetlr = Tweetlr.new(USER, PW, {
-        :whitelist => nil, 
-        :results_per_page => 5, 
-        :since_id => TIMESTAMP, 
-        :terms => @searchterm, 
-        :loglevel => 4})
-    end
-    it "should mark every users' posts as published" do
-      stub_instagram
-      post = @tweetlr.generate_tumblr_photo_post @twitter_response
-      post[:state].should == 'published'
-      stub_instagram
-      post = @tweetlr.generate_tumblr_photo_post @non_whitelist_tweet
-      post[:state].should == 'published'
-    end
-  end
-  it "should not use retweets which would produce double blog posts" do
-    post = @tweetlr.generate_tumblr_photo_post @retweet
-    post.should_not be
-  end
-  context "should not use new style retweets which would produce double blog posts" do
-    it "for quotes in context" do
-      post = @tweetlr.generate_tumblr_photo_post @new_style_retweet
-      post.should_not be
-    end
-    it "for quotes without further text addition" do
-      post = @tweetlr.generate_tumblr_photo_post @new_style_retweet_no_addition
-      post.should_not be
-    end
-  end
-  context "handles pictures in tweets" do
-    it "extracting their corresponding links" do
-      @tweets.each do |key,value|
-        send "stub_#{key}"
-        url = @tweetlr.extract_image_url value
-        url.should be, "service #{key} not working!"
-        check_pic_url_extraction key if [:instagram,:picplz,:yfrog,:imgly,:not_listed].index key
-      end
-    end
-    it "using the first image link found in a tweet with multiple links" do
-      stub_instagram
-      link = @tweetlr.extract_image_url @twitter_response
-      link.should == 'http://distillery.s3.amazonaws.com/media/2011/05/02/d25df62b9cec4a138967a3ad027d055b_7.jpg'
-    end
-    it "not returning links that do not belong to images" do
-      stub_no_image_link
-      link = @tweetlr.extract_image_url @twitter_response
-      link.should_not be
-    end
+  it "crawls twitter and posts to tumblr" do 
+    Tweetlr.crawl(config)
   end
+
+
 end

data/tweetlr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = "tweetlr"
-  s.version     = "0.1.7pre"
+  s.version     = "0.1.7pre4"
   s.author      = "Sven Kraeuter"
   s.email       = "sven.kraeuter@gmail.com"
   s.homepage    = "http://tweetlr.5v3n.com"
@@ -15,7 +15,7 @@ Gem::Specification.new do |s|
   s.add_dependency "curb"
   s.add_dependency "json"
 
-  s.add_development_dependency "rake",            "~> 0.8.7"
+  s.add_development_dependency "rake"
   s.add_development_dependency "rspec"
   s.add_development_dependency "rdoc"
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tweetlr
 version: !ruby/object:Gem::Version
-  version: 0.1.7pre
+  version: 0.1.7pre4
   prerelease: 5
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-11-12 00:00:00.000000000Z
+date: 2011-11-18 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons
-  requirement: &2153597020 !ruby/object:Gem::Requirement
+  requirement: &2153761960 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2153597020
+  version_requirements: *2153761960
 - !ruby/object:Gem::Dependency
   name: eventmachine
-  requirement: &2153596600 !ruby/object:Gem::Requirement
+  requirement: &2153761540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2153596600
+  version_requirements: *2153761540
 - !ruby/object:Gem::Dependency
   name: curb
-  requirement: &2153596180 !ruby/object:Gem::Requirement
+  requirement: &2153761120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2153596180
+  version_requirements: *2153761120
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &2153595760 !ruby/object:Gem::Requirement
+  requirement: &2153760700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,21 +54,21 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2153595760
+  version_requirements: *2153760700
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &2153595260 !ruby/object:Gem::Requirement
+  requirement: &2153760280 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.8.7
+        version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2153595260
+  version_requirements: *2153760280
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2153621460 !ruby/object:Gem::Requirement
+  requirement: &2153759860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2153621460
+  version_requirements: *2153759860
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &2153621000 !ruby/object:Gem::Requirement
+  requirement: &2153759440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,7 +87,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2153621000
+  version_requirements: *2153759440
 description: tweetlr crawls twitter for a given term, extracts photos out of the collected
   tweets' short urls and posts the images to tumblr.
 email: sven.kraeuter@gmail.com
@@ -108,16 +108,18 @@ files:
 - Rakefile
 - bin/tweetlr
 - config/tweetlr.yml
-- lib/http_processor.rb
+- lib/combinators/twitter_tumblr.rb
 - lib/log_aware.rb
-- lib/photo_service_processor.rb
-- lib/tumblr_processor.rb
+- lib/processors/http.rb
+- lib/processors/photo_service.rb
+- lib/processors/tumblr.rb
+- lib/processors/twitter.rb
 - lib/tweetlr.rb
-- lib/twitter_processor.rb
-- spec/photo_services_processor_spec.rb
+- spec/combinators/twitter_tumblr_combinator_spec.rb
+- spec/processors/photo_services_processor_spec.rb
+- spec/processors/twitter_processor_spec.rb
 - spec/spec_helper.rb
 - spec/tweetlr_spec.rb
-- spec/twitter_processor_spec.rb
 - tweetlr.gemspec
 homepage: http://tweetlr.5v3n.com
 licenses: []
@@ -145,7 +147,8 @@ specification_version: 3
 summary: tweetlr crawls twitter for a given term, extracts photos out of the collected
   tweets' short urls and posts the images to tumblr.
 test_files:
-- spec/photo_services_processor_spec.rb
+- spec/combinators/twitter_tumblr_combinator_spec.rb
+- spec/processors/photo_services_processor_spec.rb
+- spec/processors/twitter_processor_spec.rb
 - spec/spec_helper.rb
 - spec/tweetlr_spec.rb
-- spec/twitter_processor_spec.rb

data/lib/http_processor.rb

@@ -1,42 +0,0 @@
-require 'curb'
-require 'log_aware'
-
-module HttpProcessor
-  include LogAware
-  
-  USER_AGENT = %{Mozilla/5.0 (compatible; tweetlr; +http://tweetlr.5v3n.com)}
-
-  #convenience method for curl http get calls and parsing them to json.
-  def HttpProcessor::http_get(request, log=nil)
-    tries = 3
-    begin
-      curl = Curl::Easy.new request
-      curl.useragent = USER_AGENT
-      curl.perform
-      begin
-        JSON.parse curl.body_str
-      rescue JSON::ParserError => err
-        begin
-          if log
-            log.warn "#{err}: Could not parse response for #{request} - this is probably not a json response: #{curl.body_str}"
-          end
-          return nil
-        rescue Encoding::CompatibilityError => err
-          if log
-            log.error "Trying to rescue a JSON::ParserError for '#{request}' we got stuck in a Encoding::CompatibilityError."
-          end
-          return nil
-        end
-      end
-    rescue Curl::Err::CurlError => err
-      log.error "Failure in Curl call: #{err}" if log
-      tries -= 1
-      sleep 3
-      if tries > 0
-          retry
-      else
-          nil
-      end
-    end 
-  end
-end

data/lib/photo_service_processor.rb

@@ -1,122 +0,0 @@
-require 'log_aware'
-
-module PhotoServiceProcessor
-  
-  LOCATION_START_INDICATOR = 'Location: '
-  LOCATION_STOP_INDICATOR  = "\r\n"
-  PIC_REGEXP = /(.*?)\.(jpg|jpeg|png|gif)/i 
-  
-  include LogAware
-  
-  def self.find_image_url(link)
-    url = nil
-    if link && !(photo? link)
-      url = image_url_instagram link if (link.index('instagr.am') || link.index('instagram.com'))
-      url = image_url_picplz link if link.index 'picplz'
-      url = image_url_twitpic link if link.index 'twitpic'
-      url = image_url_yfrog link if link.index 'yfrog'
-      url = image_url_imgly link if link.index 'img.ly'
-      url = image_url_tco link if link.index 't.co'
-      url = image_url_lockerz link if link.index 'lockerz.com'
-      url = image_url_foursquare link if link.index '4sq.com'
-      url = image_url_embedly link if url.nil? #just try embed.ly for anything else. could do all image url processing w/ embedly, but there's probably some kind of rate limit invovled.
-    elsif photo? link
-      url = link
-    end
-    url
-  end
-  
-  def self.photo?(link)
-    link =~ PIC_REGEXP
-  end
-  
-  #find the image's url via embed.ly
-  def self.image_url_embedly(link_url)
-    response = HttpProcessor::http_get "http://api.embed.ly/1/oembed?url=#{link_url}"
-    if response && response['type'] == 'photo'
-      image_url = response['url'] 
-    end
-    image_url
-  end
-  #find the image's url for a foursquare link
-  def self.image_url_foursquare(link_url)
-    image_url_embedly link_url
-  end
-  #find the image's url for a lockerz link
-  def self.image_url_lockerz(link_url)
-    response = HttpProcessor::http_get "http://api.plixi.com/api/tpapi.svc/json/metadatafromurl?details=false&url=#{link_url}"
-    response["BigImageUrl"] if response
-  end
-  #find the image's url for an twitter shortened link
-  def self.image_url_tco(link_url)
-    service_url = link_url_redirect link_url
-    find_image_url service_url
-  end
-  #find the image's url for an instagram link
-  def self.image_url_instagram(link_url)
-    link_url['instagram.com'] = 'instagr.am' if link_url.index 'instagram.com' #instagram's oembed does not work for .com links
-    response = HttpProcessor::http_get "http://api.instagram.com/oembed?url=#{link_url}"
-    response['url'] if response
-  end
-
-  #find the image's url for a picplz short/longlink
-  def self.image_url_picplz(link_url)
-    id = extract_id link_url
-    #try short url
-    response = HttpProcessor::http_get "http://picplz.com/api/v2/pic.json?shorturl_ids=#{id}"
-    #if short url fails, try long url
-    #response = HTTParty.get "http://picplz.com/api/v2/pic.json?longurl_ids=#{id}"
-    #extract url
-    if response && response['value'] && response['value']['pics'] && response['value']['pics'].first && response['value']['pics'].first['pic_files'] && response['value']['pics'].first['pic_files']['640r']
-      response['value']['pics'].first['pic_files']['640r']['img_url'] 
-    else
-      nil
-    end
-  end
-  #find the image's url for a twitpic link
-  def self.image_url_twitpic(link_url)
-    image_url_redirect link_url, "http://twitpic.com/show/full/"
-  end
-  #find the image'S url for a yfrog link
-  def self.image_url_yfrog(link_url)
-    response = HttpProcessor::http_get("http://www.yfrog.com/api/oembed?url=#{link_url}")
-    response['url'] if response
-  end
-  #find the image's url for a img.ly link
-  def self.image_url_imgly(link_url)
-    image_url_redirect link_url, "http://img.ly/show/full/", "\r\n"
-  end
-  
-  # extract image url from services like twitpic & img.ly that do not offer oembed interfaces
-  def self.image_url_redirect(link_url, service_endpoint, stop_indicator = LOCATION_STOP_INDICATOR)
-    link_url_redirect "#{service_endpoint}#{extract_id link_url}", stop_indicator
-  end
-  
-  def self.link_url_redirect(short_url, stop_indicator = LOCATION_STOP_INDICATOR)
-    tries = 3
-    begin
-      resp = Curl::Easy.http_get(short_url) { |res| res.follow_location = true }
-    rescue Curl::Err::CurlError => err
-        log.error "Curl::Easy.http_get failed: #{err}"
-        tries -= 1
-        sleep 3
-        if tries > 0
-            retry
-        else
-           return nil
-        end
-    end
-    if(resp && resp.header_str.index(LOCATION_START_INDICATOR) && resp.header_str.index(stop_indicator))
-      start = resp.header_str.index(LOCATION_START_INDICATOR) + LOCATION_START_INDICATOR.size
-      stop  = resp.header_str.index(stop_indicator, start)
-      resp.header_str[start...stop]
-    else
-      nil
-    end
-  end
-  
-  #extract the pic id from a given <code>link</code>
-  def self.extract_id(link)
-    link.split('/').last if link.split('/')
-  end
-end