tweetlr 0.1.2 → 0.1.3

data/.autotest

@@ -0,0 +1,3 @@
+# Include plugins
+require 'autotest/fsevent'
+require 'autotest/growl'

data/.gitignore

@@ -0,0 +1,9 @@
+*.swp
+*.log
+pkg
+*.pid
+config
+*.output
+.rvmrc
+Gemfile.lock
+tweetlr.tid

data/.rspec

@@ -0,0 +1 @@
+--colour

data/Gemfile

@@ -0,0 +1,2 @@
+source :rubygems
+gemspec

data/Rakefile

@@ -1,43 +1,21 @@
-require 'rubygems'
-require 'rake'
-require 'rake/clean'
-require 'rubygems/package_task'
-require 'rdoc/task'
-require 'rake/testtask'
-
-spec = Gem::Specification.new do |s|
-  s.name = 'tweetlr'
-  s.version = '0.1.2'
-  s.has_rdoc = true
-  s.extra_rdoc_files = ['README.md', 'LICENSE']
-  s.summary = %{tweetlr crawls twitter for a given term, extracts photos out of the collected tweets' short urls and posts the images to tumblr.}
-  s.description = s.summary
-  s.author = 'Sven Kraeuter'
-  s.email = 'mail@svenkraeuter.com'
-  s.homepage = "http://github.com/5v3n/#{s.name}"
-  s.files = %w(LICENSE README.md Rakefile) + Dir.glob("{bin,lib}/**/*")
-  s.require_path = "lib"
-  s.executables  = ['tweetlr']
-  s.add_dependency('daemons')
-  s.add_dependency('eventmachine')
-  s.add_dependency('curb')
-end
+require 'bundler'
+Bundler::GemHelper.install_tasks
 
-Gem::PackageTask.new(spec) do |p|
-  p.gem_spec = spec
-  p.need_tar = true
-  p.need_zip = true
-end
+require 'rdoc/task'
+require 'rspec/core/rake_task'
 
 RDoc::Task.new do |rdoc|
-  files =['README.md', 'LICENSE', 'lib/**/*.rb']
+  files = ['README.md', 'LICENSE', 'lib/**/*.rb']
   rdoc.rdoc_files.add(files)
-  rdoc.main = "README.md" # page to start on
-  rdoc.title = "tweetlr Docs" # <--- enter name manually!
-  rdoc.rdoc_dir = 'doc/rdoc' # rdoc output folder
+  rdoc.main = "README.md"           # page to start on
+  rdoc.title = "tweetlr Docs"       # <--- enter name manually!
+  rdoc.rdoc_dir = 'doc/rdoc'        # rdoc output folder
   rdoc.options << '--line-numbers'
 end
 
-Rake::TestTask.new do |t|
-  t.test_files = FileList['test/**/*.rb']
+RSpec::Core::RakeTask.new do |t|
+  t.rspec_opts = %w(-c)
 end
+
+task :default => :spec
+task :test => :spec

data/bin/tweetlr

@@ -4,15 +4,25 @@ require 'daemons'
 require 'eventmachine'
 require 'logger'
 require 'yaml'
-require_relative '../lib/tweetlr.rb'
+require 'tweetlr'
 
 begin
   config_file = File.join( Dir.pwd,  'config', 'tweetlr.yml')
+  tid_file = File.join( Dir.pwd ,"tweetlr.tid")
+  start_at_tweet_id = lambda { 
+     begin
+      File.open(tid_file, "r") { |io| io.gets.to_i } 
+    	rescue Errno::ENOENT => e
+        $stderr.puts "#{e} - we use the value from the configuration file" 
+    	  nil 
+    	end
+  }.call
   CONFIG = YAML.load_file(config_file)
+  CONFIG['start_at_tweet_id'] = start_at_tweet_id || CONFIG['start_at_tweet_id'] || CONFIG['twitter_timestamp'] #check the latter for backwards compability
   TERM = CONFIG['search_term']
   USER = CONFIG['tumblr_username']
   PW   = CONFIG['tumblr_password']
-  TIMESTAMP = CONFIG['twitter_timestamp']
+  TIMESTAMP = start_at_tweet_id || CONFIG['start_at_tweet_id'] 
   UPDATE_PERIOD = CONFIG['update_period']
   LOGLEVEL = CONFIG['loglevel'] || Logger::INFO
   @tweetlr = Tweetlr.new(USER, PW, config_file, {:since_id => TIMESTAMP, :terms => TERM, :loglevel => LOGLEVEL})
@@ -37,20 +47,18 @@ Daemons.run_proc('tweetlr', :dir_mode => :script, :dir => './', :backtrace => tr
            if tumblr_post.nil? ||  tumblr_post[:source].nil?
               @log.warn "could not get image source: tweet: #{tweet} --- tumblr post: #{tumblr_post.inspect}"
            else
-             #@log.debug tumblr_post
-             #@log.debug @tweetlr.post_to_tumblr tumblr_post
-             #puts "tumblr post: #{tumblr_post}"
+             @log.debug "tumblr post: #{tumblr_post}"
              res = @tweetlr.post_to_tumblr tumblr_post
              @log.warn "tumblr response: #{res.header_str} #{res.body_str}" unless res.response_code == 201
            end
+           # save the last received tweet id to tweetlr.tid 
+           File.open(tid_file, "w+") { |io| io.write(tweets.last['id']) }
          end
        end
      else
-       @log.error "twitter serach returned no response. hail the failwhale!"
+       @log.error "twitter search returned no response. hail the failwhale!"
      end
      @log.info "finished tweetlr crawl."
      }
    }
-
-end
-
+end

data/config/tweetlr.yml

@@ -0,0 +1,14 @@
+results_per_page: 100
+result_type: recent
+search_term: 'cat+dog+unicorn' #find tweets containing any of these terms
+start_at_tweet_id: 61847783463854082 # the tweet id to start searching at
+api_endpoint_twitter: 'http://search.twitter.com/search.json'
+api_endpoint_tumblr: 'http://www.tumblr.com'
+tumblr_username: YOUR_TUMBLR_EMAIL
+tumblr_password: YOUR_TUMBLR_PW
+update_period: 300 #check for updates every 300 secs = 5 minutes
+shouts: 'says' # will be concatenated after the username, before the message: @mr_x says: awesome things on a photo!
+loglevel: 0 # 0: debug, 1: info (default), 2: warn, 3: error, 5: fatal
+whitelist: #twitter accounts in that list will have their tweets published immediately. post from others will be saved as drafts
+  - whitey_mc_whitelist
+  - sven_kr

data/lib/tweetlr.rb

@@ -5,9 +5,9 @@ require 'json'
 
 class Tweetlr
 
-  VERSION = '0.1.2'
+  VERSION = '0.1.3'
   GENERATOR = %{tweetlr - http://github.com/5v3n/tweetlr}
-  USER_AGENT = %{Mozilla/5.0 (compatible; tweetlr/#{VERSION};)}
+  USER_AGENT = %{Mozilla/5.0 (compatible; tweetlr/#{VERSION}; +http://github.com/5v3n/tweetlr/wiki)}
   LOCATION_START_INDICATOR = 'Location: '
   LOCATION_STOP_INDICATOR  = "\r\n"
   
@@ -144,14 +144,20 @@ class Tweetlr
       url = image_url_tco link if link.index 't.co'
       url = image_url_lockerz link if link.index 'lockerz.com'
       url = image_url_foursquare link if link.index '4sq.com'
+      url = image_url_embedly link if url.nil? #just try embed.ly for anything else. could do all image url processing w/ embedly, but there's probably some kind of rate limit invovled.
     end
     url
   end
-  #find the image's url for a foursquare link
-  def image_url_foursquare(link_url)
+  
+  #find the image's url via embed.ly
+  def image_url_embedly(link_url)
     response = http_get "http://api.embed.ly/1/oembed?url=#{link_url}"
     response['url'] if response
   end
+  #find the image's url for a foursquare link
+  def image_url_foursquare(link_url)
+    image_url_embedly link_url
+  end
   #find the image's url for a lockerz link
   def image_url_lockerz(link_url)
     response = http_get "http://api.plixi.com/api/tpapi.svc/json/metadatafromurl?details=false&url=#{link_url}"
@@ -239,7 +245,12 @@ class Tweetlr
       curl = Curl::Easy.new request
       curl.useragent = USER_AGENT
       curl.perform
-      JSON.parse curl.body_str
+      begin
+        JSON.parse curl.body_str
+      rescue JSON::ParserError => err
+        @log.warn "#{err}: Could not parse response for #{request} - this is probably not a json response: #{curl.body_str}"
+        return nil
+      end
     rescue Curl::Err::ConnectionFailedError => err
       @log.error "Connection failed: #{err}"
       tries -= 1
@@ -267,6 +278,6 @@ class Tweetlr
       else
           nil
       end
-    end
+    end 
   end  
 end

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+require "bundler"
+Bundler.require :default, :development

data/spec/tweetlr_spec.rb

@@ -0,0 +1,93 @@
+require 'spec_helper'
+
+describe Tweetlr do
+
+  config_file = File.join( Dir.pwd, 'config', 'tweetlr.yml')
+  config = YAML.load_file(config_file)
+  USER = config['tumblr_username']
+  PW   = config['tumblr_password']
+  TIMESTAMP = config['twitter_timestamp']
+
+  before :each do
+    @credentials = {:email => USER, :password => PW}
+    @cookie = "tmgioct=as3u4KJr9COyJA9j4nwr6ZAn"
+    @searchterm = 'fail'
+    @twitter_response = {"from_user_id_str"=>"1915714", "profile_image_url"=>"http://a0.twimg.com/profile_images/386000279/2_normal.jpg", "created_at"=>"Sun, 17 Apr 2011 16:48:42 +0000", "from_user"=>"whitey_Mc_whIteLIst", "id_str"=>"59659561224765440", "metadata"=>{"result_type"=>"recent"}, "to_user_id"=>nil, "text"=>"Rigaer #wirsounterwegs   @ Augenarzt Dr. Lierow http://instagr.am/p/DzCWn/", "id"=>59659561224765440, "from_user_id"=>1915714, "geo"=>{"type"=>"Point", "coordinates"=>[52.5182, 13.454]}, "iso_language_code"=>"de", "place"=>{"id"=>"3078869807f9dd36", "type"=>"city", "full_name"=>"Berlin, Berlin"}, "to_user_id_str"=>nil, "source"=>"<a href="http://instagr.am" rel="nofollow">instagram</a>"}
+    @non_whitelist_tweet = @twitter_response.merge 'from_user' => 'nonwhitelist user' 
+    @retweet = @twitter_response.merge "text" => "bla bla RT @fgd: tueddelkram"
+    @new_style_retweet = @twitter_response.merge "text" => "and it scales! \u201c@moeffju: http://t.co/8gUSPKu #hktbl1 #origami success! :)\u201d"
+    @links = {
+      :instagram => "http://instagr.am/p/DzCWn/",
+      :twitpic => "http://twitpic.com/449o2x",
+      :yfrog => "http://yfrog.com/h4vlfp",
+      :picplz => "http://picplz.com/2hWv",
+      :imgly => "http://img.ly/3M1o",
+      :tco => 'http://t.co/MUGNayA',
+      :lockerz => 'http://lockerz.com/s/100269159',
+      :foursquare => 'http://4sq.com/mLKDdF',
+      :embedly => 'http://flic.kr/p/973hTv' #if no service matches, just try embedly
+      }
+    @pic_regexp = /(.*?)\.(jpg|jpeg|png|gif)$/i 
+    @config_file = File.join( Dir.pwd, 'config', 'tweetlr.yml')
+    @tweetlr = Tweetlr.new(USER, PW, @config_file, {:since_id => TIMESTAMP, :terms => @searchterm, :loglevel => 4, :cookie => @cookie})
+  end
+  # it "should post to tumblr" do
+  #   tweetlr = Tweetlr.new @credentials[:email], @credentials[:password], @cookie, nil, @searchterm, @config_file
+  #   tumblr_post = tweetlr.generate_tumblr_photo_post @twitter_response
+  #   tumblr_post[:date] = Time.now.to_s
+  #   response = tweetlr.post_to_tumblr tumblr_post
+  #   response.should be
+  #   response.response_code.should be 201
+  # end
+  it "should search twitter for a given term" do
+    tweetlr = @tweetlr
+    response = tweetlr.search_twitter
+    tweets = response['results']
+    tweets.should be
+    tweets.should_not be_empty
+  end
+  it "should mark whitelist users' tweets as published" do
+    post = @tweetlr.generate_tumblr_photo_post @twitter_response
+    post[:state].should == 'published' 
+  end
+  it "should mark non whitelist users' tweets as drafts" do
+    post = @tweetlr.generate_tumblr_photo_post @non_whitelist_tweet
+    post[:state].should == 'draft' 
+  end
+  it "should not use retweets which would produce double blog posts" do
+    post = @tweetlr.generate_tumblr_photo_post @retweet
+    post.should_not be
+  end
+  it "should not use new style retweets which would produce double blog posts" do
+    post = @tweetlr.generate_tumblr_photo_post @new_style_retweet
+    post.should_not be
+  end
+  describe "image url processing" do
+    it "should find a picture's url from the supported services" do
+      @links.each do |key,value|
+        url = @tweetlr.find_image_url value
+        url.should be, "service #{key} not working!"
+        check_pic_url_extraction key if [:instagram,:picplz,:yfrog,:tco,:foursquare, :not_listed].index key
+      end
+    end
+    it "should not crash if embedly fallback won't find a link" do
+      url = @tweetlr.find_image_url "http://mopskopf"     
+    end
+  end
+  describe "tweet api response processing" do
+    it "should extract links" do
+      tweetlr = @tweetlr
+      link = tweetlr.extract_link @twitter_response
+      link.should == @links[:instagram]
+      link = tweetlr.extract_link @twitter_response.merge 'text' => @links[:instagram].chop #check if it works w/o the trailing slash
+      link.should == @links[:instagram].chop
+    end
+  end
+  def check_pic_url_extraction(service)
+    tweetlr = @tweetlr
+    image_url = tweetlr.send "image_url_#{service}".to_sym, @links[service]
+    image_url.should =~ @pic_regexp 
+  end
+  
+end
+

data/tweetlr.gemspec

@@ -0,0 +1,28 @@
+Gem::Specification.new do |s|
+  s.name        = "tweetlr"
+  s.version     = "0.1.3"
+  s.author      = "Sven Kraeuter"
+  s.email       = "mail@svenkraeuter.com"
+  s.homepage    = "http://github.com/5v3n/#{s.name}"
+  s.summary     = "tweetlr crawls twitter for a given term, extracts photos out of the collected tweets' short urls and posts the images to tumblr."
+  s.description = s.summary
+
+  s.rubyforge_project = s.name
+  s.extra_rdoc_files = %w(README.md LICENSE)
+
+  s.add_dependency "daemons",      "~> 1.1.3"
+  s.add_dependency "eventmachine", "~> 0.12.10"
+  s.add_dependency "curb",         "~> 0.7.15"
+  s.add_dependency "json",         "~> 1.5.1"
+
+  s.add_development_dependency "rspec",            "~> 2.6.0"
+  s.add_development_dependency "autotest",         "~> 4.4.6"
+  s.add_development_dependency "autotest-growl",   "~> 0.2.9"
+  s.add_development_dependency "autotest-fsevent", "~> 0.2.5"
+  s.add_development_dependency "rdoc"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tweetlr
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,42 +9,108 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-06-06 00:00:00.000000000 +02:00
+date: 2011-06-14 00:00:00.000000000 +02:00
 default_executable: 
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons
-  requirement: &2156428160 !ruby/object:Gem::Requirement
+  requirement: &2153694000 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.1.3
   type: :runtime
   prerelease: false
-  version_requirements: *2156428160
+  version_requirements: *2153694000
 - !ruby/object:Gem::Dependency
   name: eventmachine
-  requirement: &2156427180 !ruby/object:Gem::Requirement
+  requirement: &2153693520 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.12.10
   type: :runtime
   prerelease: false
-  version_requirements: *2156427180
+  version_requirements: *2153693520
 - !ruby/object:Gem::Dependency
   name: curb
-  requirement: &2156426180 !ruby/object:Gem::Requirement
+  requirement: &2153693060 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.7.15
+  type: :runtime
+  prerelease: false
+  version_requirements: *2153693060
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: &2153692600 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.1
+  type: :runtime
+  prerelease: false
+  version_requirements: *2153692600
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &2153692140 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.6.0
+  type: :development
+  prerelease: false
+  version_requirements: *2153692140
+- !ruby/object:Gem::Dependency
+  name: autotest
+  requirement: &2153691680 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 4.4.6
+  type: :development
+  prerelease: false
+  version_requirements: *2153691680
+- !ruby/object:Gem::Dependency
+  name: autotest-growl
+  requirement: &2153691220 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.2.9
+  type: :development
+  prerelease: false
+  version_requirements: *2153691220
+- !ruby/object:Gem::Dependency
+  name: autotest-fsevent
+  requirement: &2153713280 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.2.5
+  type: :development
+  prerelease: false
+  version_requirements: *2153713280
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: &2153712900 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
-  version_requirements: *2156426180
+  version_requirements: *2153712900
 description: tweetlr crawls twitter for a given term, extracts photos out of the collected
   tweets' short urls and posts the images to tumblr.
 email: mail@svenkraeuter.com
@@ -55,11 +121,19 @@ extra_rdoc_files:
 - README.md
 - LICENSE
 files:
+- .autotest
+- .gitignore
+- .rspec
+- Gemfile
 - LICENSE
 - README.md
 - Rakefile
 - bin/tweetlr
+- config/tweetlr.yml
 - lib/tweetlr.rb
+- spec/spec_helper.rb
+- spec/tweetlr_spec.rb
+- tweetlr.gemspec
 has_rdoc: true
 homepage: http://github.com/5v3n/tweetlr
 licenses: []
@@ -80,10 +154,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
+rubyforge_project: tweetlr
 rubygems_version: 1.6.2
 signing_key: 
 specification_version: 3
 summary: tweetlr crawls twitter for a given term, extracts photos out of the collected
   tweets' short urls and posts the images to tumblr.
-test_files: []
+test_files:
+- spec/spec_helper.rb
+- spec/tweetlr_spec.rb