tweetlr 0.0.4

data/LICENSE

@@ -0,0 +1,22 @@
+== tweetlr
+
+Copyright (c) 2011 Sven Kraeuter sven.kraeuter@gmx.net
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+

data/README.md

@@ -0,0 +1,35 @@
+# tweetlr
+
+tweetlr crawls twitter for a given term, extracts photos out of the collected tweets' short urls and posts the images to tumblr. 
+
+## Installation
+
+Use `gem install tweetlr` if you're using *rubygems* or add the line `gem 'tweetlr'` to your `Gemfile` if you're using *bundler*.
+
+## Configuration
+
+It's essential that you have a directory called `config` in the directory you are starting tweetlr in, which has to contain the configuration file `tweetlr.yml`:
+
+    results_per_page: 100
+    result_type: recent
+    search_term: <the term you want to search for>
+    twitter_timestamp: 61847783463854082 # the timestamp you want to start searching at
+    api_endpoint_twitter: 'http://search.twitter.com/search.json'
+    api_endpoint_tumblr: 'http://www.tumblr.com'
+    tumblr_username: <your tumblr username / e-mail address>
+    tumblr_password: <your tumblr password>
+    update_period: 300 #check for updates every 300 secs = 5 minutes
+    shouts: 'says' # will be concatenated after the username, before the message: @mr_x <shouts>: awesome things on a photo!
+    whitelist: #twitter accounts in that list will have their tweets published immediately. post from others will be saved as drafts
+      - whitey_mc_whitelist
+      - sven_kr
+
+
+## Usage
+
+Make sure you put the configuration file in it's proper place as mentiones above, then: 
+
+start/stop tweetlr using  `tweetlr start`/`tweetlr stop`. Run `tweetlr` without arguments for a list of options concerning the daemon's options. 
+
+enjoy!
+

data/Rakefile

@@ -0,0 +1,44 @@
+require 'rubygems'
+require 'rake'
+require 'rake/clean'
+require 'rake/gempackagetask'
+require 'rake/rdoctask'
+require 'rake/testtask'
+
+spec = Gem::Specification.new do |s|
+  s.name = 'tweetlr'
+  s.version = '0.0.4'
+  s.has_rdoc = true
+  s.extra_rdoc_files = ['README.md', 'LICENSE']
+  s.summary = 'an unholy alliance between twitter and tumblr'
+  s.description = s.summary
+  s.author = 'Sven Kraeuter'
+  s.email = 'mail@svenkraeuter.com'
+  s.homepage = "http://github.com/5v3n/#{s.name}"
+  # s.executables = ['your_executable_here']
+  s.files = %w(LICENSE README.md Rakefile) + Dir.glob("{bin,lib}/**/*")
+  s.require_path = "lib"
+  s.executables  = ['tweetlr']
+  s.add_dependency('daemons')
+  s.add_dependency('eventmachine')
+  s.add_dependency('httparty')
+end
+
+Rake::GemPackageTask.new(spec) do |p|
+  p.gem_spec = spec
+  p.need_tar = true
+  p.need_zip = true
+end
+
+Rake::RDocTask.new do |rdoc|
+  files =['README.md', 'LICENSE', 'lib/**/*.rb']
+  rdoc.rdoc_files.add(files)
+  rdoc.main = "README.md" # page to start on
+  rdoc.title = "tweetlr Docs" # <--- enter name manually!
+  rdoc.rdoc_dir = 'doc/rdoc' # rdoc output folder
+  rdoc.options << '--line-numbers'
+end
+
+Rake::TestTask.new do |t|
+  t.test_files = FileList['test/**/*.rb']
+end

data/bin/tweetlr

@@ -0,0 +1,48 @@
+#!/usr/bin/env ruby
+
+require 'daemons'
+require 'eventmachine'
+require 'logger'
+require 'yaml'
+require_relative '../lib/tweetlr.rb'
+
+begin
+  config_file = File.join( Dir.pwd,  'config', 'tweetlr.yml')
+  CONFIG = YAML.load_file(config_file)
+  TERM = CONFIG['search_term']
+  USER = CONFIG['tumblr_username']
+  PW   = CONFIG['tumblr_password']
+  TIMESTAMP = CONFIG['twitter_timestamp']
+  UPDATE_PERIOD = CONFIG['update_period']
+rescue SystemCallError
+  $stderr.puts "Ooops - looks like there is no ./config/tweetlr.yml found. I'm affraid tweetlr won't work properly until you introduced that configuration file."
+  exit(1)
+end
+
+Daemons.run_proc('tweetlr') do
+  @log = Logger.new('tweetlr.log')
+  @log.info('starting tweetlr daemon...')
+  @log.info "createing a new tweetlr instance using this config: #{CONFIG.inspect}"   
+  EventMachine::run { 
+     tweetlr = Tweetlr.new(USER, PW, nil, TIMESTAMP, TERM, config_file)
+     EventMachine::add_periodic_timer( UPDATE_PERIOD ) {
+       @log.info('starting tweetlr crawl...')
+       response = tweetlr.lazy_search_twitter
+       tweets = response.parsed_response['results']
+       if tweets
+         tweets.each do |tweet|
+           tumblr_post = tweetlr.generate_tumblr_photo_post tweet
+           if tumblr_post.nil? ||  tumblr_post[:source].nil?
+              @log.error "could not get image source: #{tumblr_post.inspect}"
+           else
+             @log.debug tumblr_post
+             @log.debug tweetlr.post_to_tumblr tumblr_post
+           end
+         end
+       end
+       @log.info('finished tweetlr crawl.')
+    }
+   }
+
+end
+

data/lib/tweetlr.rb

@@ -0,0 +1,157 @@
+require 'httparty'
+require 'logger'
+require 'yaml'
+
+
+class Tweetlr
+
+  GENERATOR = %{tweetlr - http://github.com/5v3n/tweetlr}
+
+  def initialize(email, password, cookie=nil, since_id=nil, term=nil, config_file) #TODO use a hash or sth more elegant here...
+    @log = Logger.new('tweetlr.log')
+    config = YAML.load_file(config_file)
+    @results_per_page = config['results_per_page']
+    @result_type = config['result_type']
+    @api_endpoint_twitter = config['api_endpoint_twitter']
+    @api_endpoint_tumblr = config['api_endpoint_tumblr']
+    @whitelist = config['whitelist']
+    @shouts = config['shouts']
+    @since_id = since_id
+    @search_term = term
+    @whitelist.each {|entry| entry.downcase!}
+    @email = email
+    @password = password
+    @term = term
+    @refresh_url = "#{@api_endpoint_twitter}?q=#{term}&since_id=#{since_id}" if (since_id && term)
+    if !cookie
+      response = HTTParty.post(
+        "#{@api_endpoint_tumblr}/login",
+        :body => {
+          :email => @email,
+          :password => password
+        }
+      )
+      @log.debug("initial login response: #{response}")
+      @cookie = response.headers['Set-Cookie']
+      @log.debug("--------login cookie via new login: #{@cookie.inspect}")
+    else
+      @cookie = cookie
+      @log.debug("--------login cookie via argument: #{@cookie.inspect}")
+    end
+    
+  end
+
+  def post_to_tumblr(options={})
+    options[:generator] = GENERATOR
+    options[:email] = @email #TODO get cookie auth working!
+    options[:password] = @password
+    #options[:headers] = {'Cookie' => @cookie}
+    #arguments=options.collect { |key, value| "#{key}=#{value}" }.join('&')
+    @log.debug("------------********** post_to_tumblr options: #{options.inspect}")
+    @log.debug("------------********** post_to_tumblr options: #{{'Cookie' => @cookie}.inspect}")
+    response = HTTParty.post("#{@api_endpoint_tumblr}/api/write", :body => options, :headers => {'Cookie' => @cookie})
+    @log.debug("------------********** post_to_tumblr response: #{response.inspect}" )
+    response
+  end
+
+  #fire a new search
+  def search_twitter()
+    search_call = "#{@api_endpoint_twitter}?q=#{@search_term}&result_type=#{@result_type}&rpp=#{@results_per_page}"
+    @response = HTTParty.get(search_call)
+  end
+  # lazy update - search for a term or refresh the search if a response is available already
+  def lazy_search_twitter()
+    @refresh_url = "#{@api_endpoint_twitter}#{@response['refresh_url']}" unless (@response.nil? || @response['refresh_url'].nil? || @response['refresh_url'].empty?)
+    if @refresh_url
+     #FIXME persist the refresh url - server restart would be a pain elsewise
+     @log.info "lazy search using '#{@refresh_url}'"
+     @response = HTTParty.get(@refresh_url)
+    else
+      @log.debug "regular search using '#{term}'"
+      @response = search_twitter()
+    end
+  end
+  
+  #extract the linked image file's url
+  def extract_image_url(tweet)
+    link = extract_link tweet
+    url = nil
+    if link
+      url = image_url_instagram link if (link.index('instagr.am') || link.index('instagram.com'))
+      url = image_url_picplz link if link.index 'picplz'
+      url = image_url_twitpic link if link.index 'twitpic'
+      url = image_url_yfrog link if link.index 'yfrog'
+    end
+    url
+  end
+
+  #find the image's url for an instagram link
+  def image_url_instagram(link_url)
+    link_url['instagram.com'] = 'instagr.am' if link_url.index 'instagram.com' #instagram's oembed does not work for .com links
+    response = HTTParty.get "http://api.instagram.com/oembed?url=#{link_url}"
+    response.parsed_response['url']
+  end
+
+  #find the image's url for a picplz short/longlink
+  def image_url_picplz(link_url)
+    id = extract_id link_url
+    #try short url
+    response = HTTParty.get "http://picplz.com/api/v2/pic.json?shorturl_ids=#{id}"
+    #if short url fails, try long url
+    #response = HTTParty.get "http://picplz.com/api/v2/pic.json?longurl_ids=#{id}"
+    #extract url
+    response['value']['pics'].first['pic_files']['640r']['img_url']
+  end
+  #find the image's url for a twitpic link
+  def image_url_twitpic(link_url)
+    "http://twitpic.com/show/full/#{extract_id link_url}"
+  end
+  #find the image'S url for a yfrog link
+  def image_url_yfrog(link_url)
+    response = HTTParty.get("http://www.yfrog.com/api/oembed?url=#{link_url}")
+    response.parsed_response['url']
+  end
+
+  #extract the pic id from a given <code>link</code>
+  def extract_id(link)
+    link.split('/').last if link.split('/')
+  end
+
+  #extract the link from a given tweet
+  def extract_link(tweet)
+    if tweet
+      text = tweet['text']
+      start = text.index('http') if text
+      if start
+        stop = text.index(' ', start) || 0
+        text[start..stop-1]
+      end
+    end
+  end
+
+  def generate_tumblr_photo_post tweet
+    tumblr_post = nil
+    message = tweet['text']
+    if message && !message.index('RT @') #discard retweets
+      @log.debug "tweet: #{tweet}"
+      tumblr_post = {}
+      tumblr_post[:type] = 'photo'
+      tumblr_post[:date] = tweet['created_at']
+      tumblr_post[:source] = extract_image_url tweet
+      user = tweet['from_user']
+      if @whitelist.member? user.downcase
+        state = 'published'
+      else
+        state = 'draft'
+      end
+      tumblr_post[:state] = state
+      tumblr_post[:caption] = %?@#{user} #{@shouts}: #{tweet['text']}? #TODO make this a bigger matter of yml configuration
+    end
+    tumblr_post
+  end
+  
+end
+
+
+
+

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification 
+name: tweetlr
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.0.4
+platform: ruby
+authors: 
+- Sven Kraeuter
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-04-23 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: daemons
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: eventmachine
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: httparty
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id003
+description: an unholy alliance between twitter and tumblr
+email: mail@svenkraeuter.com
+executables: 
+- tweetlr
+extensions: []
+
+extra_rdoc_files: 
+- README.md
+- LICENSE
+files: 
+- LICENSE
+- README.md
+- Rakefile
+- bin/tweetlr
+- lib/tweetlr.rb
+homepage: http://github.com/5v3n/tweetlr
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.7.2
+signing_key: 
+specification_version: 3
+summary: an unholy alliance between twitter and tumblr
+test_files: []
+