tweetlr 0.0.4
Sign up to get free protection for your applications and to get access to all the features.
- data/LICENSE +22 -0
- data/README.md +35 -0
- data/Rakefile +44 -0
- data/bin/tweetlr +48 -0
- data/lib/tweetlr.rb +157 -0
- metadata +91 -0
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
== tweetlr
|
2
|
+
|
3
|
+
Copyright (c) 2011 Sven Kraeuter sven.kraeuter@gmx.net
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
22
|
+
|
data/README.md
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
# tweetlr
|
2
|
+
|
3
|
+
tweetlr crawls twitter for a given term, extracts photos out of the collected tweets' short urls and posts the images to tumblr.
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Use `gem install tweetlr` if you're using *rubygems* or add the line `gem 'tweetlr'` to your `Gemfile` if you're using *bundler*.
|
8
|
+
|
9
|
+
## Configuration
|
10
|
+
|
11
|
+
It's essential that you have a directory called `config` in the directory you are starting tweetlr in, which has to contain the configuration file `tweetlr.yml`:
|
12
|
+
|
13
|
+
results_per_page: 100
|
14
|
+
result_type: recent
|
15
|
+
search_term: <the term you want to search for>
|
16
|
+
twitter_timestamp: 61847783463854082 # the timestamp you want to start searching at
|
17
|
+
api_endpoint_twitter: 'http://search.twitter.com/search.json'
|
18
|
+
api_endpoint_tumblr: 'http://www.tumblr.com'
|
19
|
+
tumblr_username: <your tumblr username / e-mail address>
|
20
|
+
tumblr_password: <your tumblr password>
|
21
|
+
update_period: 300 #check for updates every 300 secs = 5 minutes
|
22
|
+
shouts: 'says' # will be concatenated after the username, before the message: @mr_x <shouts>: awesome things on a photo!
|
23
|
+
whitelist: #twitter accounts in that list will have their tweets published immediately. post from others will be saved as drafts
|
24
|
+
- whitey_mc_whitelist
|
25
|
+
- sven_kr
|
26
|
+
|
27
|
+
|
28
|
+
## Usage
|
29
|
+
|
30
|
+
Make sure you put the configuration file in it's proper place as mentiones above, then:
|
31
|
+
|
32
|
+
start/stop tweetlr using `tweetlr start`/`tweetlr stop`. Run `tweetlr` without arguments for a list of options concerning the daemon's options.
|
33
|
+
|
34
|
+
enjoy!
|
35
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1,44 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'rake'
|
3
|
+
require 'rake/clean'
|
4
|
+
require 'rake/gempackagetask'
|
5
|
+
require 'rake/rdoctask'
|
6
|
+
require 'rake/testtask'
|
7
|
+
|
8
|
+
spec = Gem::Specification.new do |s|
|
9
|
+
s.name = 'tweetlr'
|
10
|
+
s.version = '0.0.4'
|
11
|
+
s.has_rdoc = true
|
12
|
+
s.extra_rdoc_files = ['README.md', 'LICENSE']
|
13
|
+
s.summary = 'an unholy alliance between twitter and tumblr'
|
14
|
+
s.description = s.summary
|
15
|
+
s.author = 'Sven Kraeuter'
|
16
|
+
s.email = 'mail@svenkraeuter.com'
|
17
|
+
s.homepage = "http://github.com/5v3n/#{s.name}"
|
18
|
+
# s.executables = ['your_executable_here']
|
19
|
+
s.files = %w(LICENSE README.md Rakefile) + Dir.glob("{bin,lib}/**/*")
|
20
|
+
s.require_path = "lib"
|
21
|
+
s.executables = ['tweetlr']
|
22
|
+
s.add_dependency('daemons')
|
23
|
+
s.add_dependency('eventmachine')
|
24
|
+
s.add_dependency('httparty')
|
25
|
+
end
|
26
|
+
|
27
|
+
Rake::GemPackageTask.new(spec) do |p|
|
28
|
+
p.gem_spec = spec
|
29
|
+
p.need_tar = true
|
30
|
+
p.need_zip = true
|
31
|
+
end
|
32
|
+
|
33
|
+
Rake::RDocTask.new do |rdoc|
|
34
|
+
files =['README.md', 'LICENSE', 'lib/**/*.rb']
|
35
|
+
rdoc.rdoc_files.add(files)
|
36
|
+
rdoc.main = "README.md" # page to start on
|
37
|
+
rdoc.title = "tweetlr Docs" # <--- enter name manually!
|
38
|
+
rdoc.rdoc_dir = 'doc/rdoc' # rdoc output folder
|
39
|
+
rdoc.options << '--line-numbers'
|
40
|
+
end
|
41
|
+
|
42
|
+
Rake::TestTask.new do |t|
|
43
|
+
t.test_files = FileList['test/**/*.rb']
|
44
|
+
end
|
data/bin/tweetlr
ADDED
@@ -0,0 +1,48 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'daemons'
|
4
|
+
require 'eventmachine'
|
5
|
+
require 'logger'
|
6
|
+
require 'yaml'
|
7
|
+
require_relative '../lib/tweetlr.rb'
|
8
|
+
|
9
|
+
begin
|
10
|
+
config_file = File.join( Dir.pwd, 'config', 'tweetlr.yml')
|
11
|
+
CONFIG = YAML.load_file(config_file)
|
12
|
+
TERM = CONFIG['search_term']
|
13
|
+
USER = CONFIG['tumblr_username']
|
14
|
+
PW = CONFIG['tumblr_password']
|
15
|
+
TIMESTAMP = CONFIG['twitter_timestamp']
|
16
|
+
UPDATE_PERIOD = CONFIG['update_period']
|
17
|
+
rescue SystemCallError
|
18
|
+
$stderr.puts "Ooops - looks like there is no ./config/tweetlr.yml found. I'm affraid tweetlr won't work properly until you introduced that configuration file."
|
19
|
+
exit(1)
|
20
|
+
end
|
21
|
+
|
22
|
+
Daemons.run_proc('tweetlr') do
|
23
|
+
@log = Logger.new('tweetlr.log')
|
24
|
+
@log.info('starting tweetlr daemon...')
|
25
|
+
@log.info "createing a new tweetlr instance using this config: #{CONFIG.inspect}"
|
26
|
+
EventMachine::run {
|
27
|
+
tweetlr = Tweetlr.new(USER, PW, nil, TIMESTAMP, TERM, config_file)
|
28
|
+
EventMachine::add_periodic_timer( UPDATE_PERIOD ) {
|
29
|
+
@log.info('starting tweetlr crawl...')
|
30
|
+
response = tweetlr.lazy_search_twitter
|
31
|
+
tweets = response.parsed_response['results']
|
32
|
+
if tweets
|
33
|
+
tweets.each do |tweet|
|
34
|
+
tumblr_post = tweetlr.generate_tumblr_photo_post tweet
|
35
|
+
if tumblr_post.nil? || tumblr_post[:source].nil?
|
36
|
+
@log.error "could not get image source: #{tumblr_post.inspect}"
|
37
|
+
else
|
38
|
+
@log.debug tumblr_post
|
39
|
+
@log.debug tweetlr.post_to_tumblr tumblr_post
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
@log.info('finished tweetlr crawl.')
|
44
|
+
}
|
45
|
+
}
|
46
|
+
|
47
|
+
end
|
48
|
+
|
data/lib/tweetlr.rb
ADDED
@@ -0,0 +1,157 @@
|
|
1
|
+
require 'httparty'
|
2
|
+
require 'logger'
|
3
|
+
require 'yaml'
|
4
|
+
|
5
|
+
|
6
|
+
class Tweetlr
|
7
|
+
|
8
|
+
GENERATOR = %{tweetlr - http://github.com/5v3n/tweetlr}
|
9
|
+
|
10
|
+
def initialize(email, password, cookie=nil, since_id=nil, term=nil, config_file) #TODO use a hash or sth more elegant here...
|
11
|
+
@log = Logger.new('tweetlr.log')
|
12
|
+
config = YAML.load_file(config_file)
|
13
|
+
@results_per_page = config['results_per_page']
|
14
|
+
@result_type = config['result_type']
|
15
|
+
@api_endpoint_twitter = config['api_endpoint_twitter']
|
16
|
+
@api_endpoint_tumblr = config['api_endpoint_tumblr']
|
17
|
+
@whitelist = config['whitelist']
|
18
|
+
@shouts = config['shouts']
|
19
|
+
@since_id = since_id
|
20
|
+
@search_term = term
|
21
|
+
@whitelist.each {|entry| entry.downcase!}
|
22
|
+
@email = email
|
23
|
+
@password = password
|
24
|
+
@term = term
|
25
|
+
@refresh_url = "#{@api_endpoint_twitter}?q=#{term}&since_id=#{since_id}" if (since_id && term)
|
26
|
+
if !cookie
|
27
|
+
response = HTTParty.post(
|
28
|
+
"#{@api_endpoint_tumblr}/login",
|
29
|
+
:body => {
|
30
|
+
:email => @email,
|
31
|
+
:password => password
|
32
|
+
}
|
33
|
+
)
|
34
|
+
@log.debug("initial login response: #{response}")
|
35
|
+
@cookie = response.headers['Set-Cookie']
|
36
|
+
@log.debug("--------login cookie via new login: #{@cookie.inspect}")
|
37
|
+
else
|
38
|
+
@cookie = cookie
|
39
|
+
@log.debug("--------login cookie via argument: #{@cookie.inspect}")
|
40
|
+
end
|
41
|
+
|
42
|
+
end
|
43
|
+
|
44
|
+
def post_to_tumblr(options={})
|
45
|
+
options[:generator] = GENERATOR
|
46
|
+
options[:email] = @email #TODO get cookie auth working!
|
47
|
+
options[:password] = @password
|
48
|
+
#options[:headers] = {'Cookie' => @cookie}
|
49
|
+
#arguments=options.collect { |key, value| "#{key}=#{value}" }.join('&')
|
50
|
+
@log.debug("------------********** post_to_tumblr options: #{options.inspect}")
|
51
|
+
@log.debug("------------********** post_to_tumblr options: #{{'Cookie' => @cookie}.inspect}")
|
52
|
+
response = HTTParty.post("#{@api_endpoint_tumblr}/api/write", :body => options, :headers => {'Cookie' => @cookie})
|
53
|
+
@log.debug("------------********** post_to_tumblr response: #{response.inspect}" )
|
54
|
+
response
|
55
|
+
end
|
56
|
+
|
57
|
+
#fire a new search
|
58
|
+
def search_twitter()
|
59
|
+
search_call = "#{@api_endpoint_twitter}?q=#{@search_term}&result_type=#{@result_type}&rpp=#{@results_per_page}"
|
60
|
+
@response = HTTParty.get(search_call)
|
61
|
+
end
|
62
|
+
# lazy update - search for a term or refresh the search if a response is available already
|
63
|
+
def lazy_search_twitter()
|
64
|
+
@refresh_url = "#{@api_endpoint_twitter}#{@response['refresh_url']}" unless (@response.nil? || @response['refresh_url'].nil? || @response['refresh_url'].empty?)
|
65
|
+
if @refresh_url
|
66
|
+
#FIXME persist the refresh url - server restart would be a pain elsewise
|
67
|
+
@log.info "lazy search using '#{@refresh_url}'"
|
68
|
+
@response = HTTParty.get(@refresh_url)
|
69
|
+
else
|
70
|
+
@log.debug "regular search using '#{term}'"
|
71
|
+
@response = search_twitter()
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
#extract the linked image file's url
|
76
|
+
def extract_image_url(tweet)
|
77
|
+
link = extract_link tweet
|
78
|
+
url = nil
|
79
|
+
if link
|
80
|
+
url = image_url_instagram link if (link.index('instagr.am') || link.index('instagram.com'))
|
81
|
+
url = image_url_picplz link if link.index 'picplz'
|
82
|
+
url = image_url_twitpic link if link.index 'twitpic'
|
83
|
+
url = image_url_yfrog link if link.index 'yfrog'
|
84
|
+
end
|
85
|
+
url
|
86
|
+
end
|
87
|
+
|
88
|
+
#find the image's url for an instagram link
|
89
|
+
def image_url_instagram(link_url)
|
90
|
+
link_url['instagram.com'] = 'instagr.am' if link_url.index 'instagram.com' #instagram's oembed does not work for .com links
|
91
|
+
response = HTTParty.get "http://api.instagram.com/oembed?url=#{link_url}"
|
92
|
+
response.parsed_response['url']
|
93
|
+
end
|
94
|
+
|
95
|
+
#find the image's url for a picplz short/longlink
|
96
|
+
def image_url_picplz(link_url)
|
97
|
+
id = extract_id link_url
|
98
|
+
#try short url
|
99
|
+
response = HTTParty.get "http://picplz.com/api/v2/pic.json?shorturl_ids=#{id}"
|
100
|
+
#if short url fails, try long url
|
101
|
+
#response = HTTParty.get "http://picplz.com/api/v2/pic.json?longurl_ids=#{id}"
|
102
|
+
#extract url
|
103
|
+
response['value']['pics'].first['pic_files']['640r']['img_url']
|
104
|
+
end
|
105
|
+
#find the image's url for a twitpic link
|
106
|
+
def image_url_twitpic(link_url)
|
107
|
+
"http://twitpic.com/show/full/#{extract_id link_url}"
|
108
|
+
end
|
109
|
+
#find the image'S url for a yfrog link
|
110
|
+
def image_url_yfrog(link_url)
|
111
|
+
response = HTTParty.get("http://www.yfrog.com/api/oembed?url=#{link_url}")
|
112
|
+
response.parsed_response['url']
|
113
|
+
end
|
114
|
+
|
115
|
+
#extract the pic id from a given <code>link</code>
|
116
|
+
def extract_id(link)
|
117
|
+
link.split('/').last if link.split('/')
|
118
|
+
end
|
119
|
+
|
120
|
+
#extract the link from a given tweet
|
121
|
+
def extract_link(tweet)
|
122
|
+
if tweet
|
123
|
+
text = tweet['text']
|
124
|
+
start = text.index('http') if text
|
125
|
+
if start
|
126
|
+
stop = text.index(' ', start) || 0
|
127
|
+
text[start..stop-1]
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
def generate_tumblr_photo_post tweet
|
133
|
+
tumblr_post = nil
|
134
|
+
message = tweet['text']
|
135
|
+
if message && !message.index('RT @') #discard retweets
|
136
|
+
@log.debug "tweet: #{tweet}"
|
137
|
+
tumblr_post = {}
|
138
|
+
tumblr_post[:type] = 'photo'
|
139
|
+
tumblr_post[:date] = tweet['created_at']
|
140
|
+
tumblr_post[:source] = extract_image_url tweet
|
141
|
+
user = tweet['from_user']
|
142
|
+
if @whitelist.member? user.downcase
|
143
|
+
state = 'published'
|
144
|
+
else
|
145
|
+
state = 'draft'
|
146
|
+
end
|
147
|
+
tumblr_post[:state] = state
|
148
|
+
tumblr_post[:caption] = %?@#{user} #{@shouts}: #{tweet['text']}? #TODO make this a bigger matter of yml configuration
|
149
|
+
end
|
150
|
+
tumblr_post
|
151
|
+
end
|
152
|
+
|
153
|
+
end
|
154
|
+
|
155
|
+
|
156
|
+
|
157
|
+
|
metadata
ADDED
@@ -0,0 +1,91 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: tweetlr
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease:
|
5
|
+
version: 0.0.4
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Sven Kraeuter
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
|
13
|
+
date: 2011-04-23 00:00:00 Z
|
14
|
+
dependencies:
|
15
|
+
- !ruby/object:Gem::Dependency
|
16
|
+
name: daemons
|
17
|
+
prerelease: false
|
18
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
19
|
+
none: false
|
20
|
+
requirements:
|
21
|
+
- - ">="
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: "0"
|
24
|
+
type: :runtime
|
25
|
+
version_requirements: *id001
|
26
|
+
- !ruby/object:Gem::Dependency
|
27
|
+
name: eventmachine
|
28
|
+
prerelease: false
|
29
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
30
|
+
none: false
|
31
|
+
requirements:
|
32
|
+
- - ">="
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: "0"
|
35
|
+
type: :runtime
|
36
|
+
version_requirements: *id002
|
37
|
+
- !ruby/object:Gem::Dependency
|
38
|
+
name: httparty
|
39
|
+
prerelease: false
|
40
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ">="
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: "0"
|
46
|
+
type: :runtime
|
47
|
+
version_requirements: *id003
|
48
|
+
description: an unholy alliance between twitter and tumblr
|
49
|
+
email: mail@svenkraeuter.com
|
50
|
+
executables:
|
51
|
+
- tweetlr
|
52
|
+
extensions: []
|
53
|
+
|
54
|
+
extra_rdoc_files:
|
55
|
+
- README.md
|
56
|
+
- LICENSE
|
57
|
+
files:
|
58
|
+
- LICENSE
|
59
|
+
- README.md
|
60
|
+
- Rakefile
|
61
|
+
- bin/tweetlr
|
62
|
+
- lib/tweetlr.rb
|
63
|
+
homepage: http://github.com/5v3n/tweetlr
|
64
|
+
licenses: []
|
65
|
+
|
66
|
+
post_install_message:
|
67
|
+
rdoc_options: []
|
68
|
+
|
69
|
+
require_paths:
|
70
|
+
- lib
|
71
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
72
|
+
none: false
|
73
|
+
requirements:
|
74
|
+
- - ">="
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: "0"
|
77
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
78
|
+
none: false
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: "0"
|
83
|
+
requirements: []
|
84
|
+
|
85
|
+
rubyforge_project:
|
86
|
+
rubygems_version: 1.7.2
|
87
|
+
signing_key:
|
88
|
+
specification_version: 3
|
89
|
+
summary: an unholy alliance between twitter and tumblr
|
90
|
+
test_files: []
|
91
|
+
|