tweetlr 0.0.10 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +1 -0
- data/Rakefile +1 -1
- data/bin/tweetlr +11 -11
- data/lib/tweetlr.rb +86 -44
- metadata +2 -2
data/README.md
CHANGED
@@ -21,6 +21,7 @@ tumblr_username: YOUR_TUMBLR_EMAIL
|
|
21
21
|
tumblr_password: YOUR_TUMBLR_PW
|
22
22
|
update_period: 300 #check for updates every 300 secs = 5 minutes
|
23
23
|
shouts: 'says' # will be concatenated after the username, before the message: @mr_x says: awesome things on a photo!
|
24
|
+
loglevel: 0 # 0: debug, 1: info (default), 2: warn, 3: error, 5: fatal
|
24
25
|
whitelist: #twitter accounts in that list will have their tweets published immediately. post from others will be saved as drafts
|
25
26
|
- whitey_mc_whitelist
|
26
27
|
- sven_kr
|
data/Rakefile
CHANGED
@@ -7,7 +7,7 @@ require 'rake/testtask'
|
|
7
7
|
|
8
8
|
spec = Gem::Specification.new do |s|
|
9
9
|
s.name = 'tweetlr'
|
10
|
-
s.version = '0.0
|
10
|
+
s.version = '0.1.0'
|
11
11
|
s.has_rdoc = true
|
12
12
|
s.extra_rdoc_files = ['README.md', 'LICENSE']
|
13
13
|
s.summary = %{tweetlr crawls twitter for a given term, extracts photos out of the collected tweets' short urls and posts the images to tumblr.}
|
data/bin/tweetlr
CHANGED
@@ -8,26 +8,26 @@ require_relative '../lib/tweetlr.rb'
|
|
8
8
|
|
9
9
|
begin
|
10
10
|
config_file = File.join( Dir.pwd, 'config', 'tweetlr.yml')
|
11
|
-
@log_file = File.join( Dir.pwd, 'tweetlrd.log')
|
12
11
|
CONFIG = YAML.load_file(config_file)
|
13
12
|
TERM = CONFIG['search_term']
|
14
13
|
USER = CONFIG['tumblr_username']
|
15
14
|
PW = CONFIG['tumblr_password']
|
16
15
|
TIMESTAMP = CONFIG['twitter_timestamp']
|
17
16
|
UPDATE_PERIOD = CONFIG['update_period']
|
18
|
-
|
17
|
+
LOGLEVEL = CONFIG['loglevel'] || Logger::INFO
|
18
|
+
@tweetlr = Tweetlr.new(USER, PW, config_file, {:since_id => TIMESTAMP, :terms => TERM, :loglevel => LOGLEVEL})
|
19
19
|
rescue SystemCallError
|
20
20
|
$stderr.puts "Ooops - looks like there is no ./config/tweetlr.yml found. I'm affraid tweetlr won't work properly until you introduced that configuration file."
|
21
21
|
exit(1)
|
22
22
|
end
|
23
23
|
|
24
24
|
Daemons.run_proc('tweetlr', :dir_mode => :script, :dir => './', :backtrace => true, :log_output => true) do
|
25
|
-
|
26
|
-
|
27
|
-
|
25
|
+
@log = Logger.new(STDOUT)
|
26
|
+
@log.info "starting tweetlr daemon..."
|
27
|
+
@log.info "creating a new tweetlr instance using this config: #{CONFIG.inspect}"
|
28
28
|
EventMachine::run {
|
29
29
|
EventMachine::add_periodic_timer( UPDATE_PERIOD ) {
|
30
|
-
|
30
|
+
@log.info "starting tweetlr crawl..."
|
31
31
|
response = @tweetlr.lazy_search_twitter
|
32
32
|
if response
|
33
33
|
tweets = response['results']
|
@@ -35,20 +35,20 @@ Daemons.run_proc('tweetlr', :dir_mode => :script, :dir => './', :backtrace => tr
|
|
35
35
|
tweets.each do |tweet|
|
36
36
|
tumblr_post = @tweetlr.generate_tumblr_photo_post tweet
|
37
37
|
if tumblr_post.nil? || tumblr_post[:source].nil?
|
38
|
-
|
39
|
-
|
38
|
+
@log.warn "could not get image source: tweet: #{tweet} --- tumblr post: #{tumblr_post.inspect}"
|
39
|
+
else
|
40
40
|
#@log.debug tumblr_post
|
41
41
|
#@log.debug @tweetlr.post_to_tumblr tumblr_post
|
42
42
|
#puts "tumblr post: #{tumblr_post}"
|
43
43
|
res = @tweetlr.post_to_tumblr tumblr_post
|
44
|
-
|
44
|
+
@log.warn "tumblr response: #{res.header_str} #{res.body_str}" unless res.response_code == 201
|
45
45
|
end
|
46
46
|
end
|
47
47
|
end
|
48
48
|
else
|
49
|
-
|
49
|
+
@log.error "twitter serach returned no response. hail the failwhale!"
|
50
50
|
end
|
51
|
-
|
51
|
+
@log.info "finished tweetlr crawl."
|
52
52
|
}
|
53
53
|
}
|
54
54
|
|
data/lib/tweetlr.rb
CHANGED
@@ -9,51 +9,68 @@ class Tweetlr
|
|
9
9
|
LOCATION_START_INDICATOR = 'Location: '
|
10
10
|
LOCATION_STOP_INDICATOR = "\r\n"
|
11
11
|
|
12
|
-
def initialize(email, password, cookie
|
13
|
-
@log = Logger.new(
|
12
|
+
def initialize(email, password, config_file, args={:cookie => nil, :since_id=>nil, :terms=>nil, :loglevel=>Logger::INFO})
|
13
|
+
@log = Logger.new(STDOUT)
|
14
|
+
@log.level = args[:loglevel] if (Logger::DEBUG..Logger::UNKNOWN).to_a.index(args[:loglevel])
|
15
|
+
@log.debug "log level set to #{@log.level}"
|
14
16
|
config = YAML.load_file(config_file)
|
17
|
+
@email = email
|
18
|
+
@password = password
|
19
|
+
@since_id = args[:since_id]
|
20
|
+
@search_term = args[:terms]
|
21
|
+
@cookie = args[:cookie]
|
15
22
|
@results_per_page = config['results_per_page']
|
16
23
|
@result_type = config['result_type']
|
17
24
|
@api_endpoint_twitter = config['api_endpoint_twitter']
|
18
25
|
@api_endpoint_tumblr = config['api_endpoint_tumblr']
|
19
26
|
@whitelist = config['whitelist']
|
20
27
|
@shouts = config['shouts']
|
21
|
-
@since_id = since_id
|
22
|
-
@search_term = terms
|
23
28
|
@whitelist.each {|entry| entry.downcase!}
|
24
|
-
@
|
25
|
-
|
26
|
-
@refresh_url = "#{@api_endpoint_twitter}?ors=#{terms}&since_id=#{since_id}&rpp=#{@results_per_page}&result_type=#{@result_type}" if (since_id && terms)
|
27
|
-
if !cookie
|
29
|
+
@refresh_url = "#{@api_endpoint_twitter}?ors=#{@search_term}&since_id=#{@since_id}&rpp=#{@results_per_page}&result_type=#{@result_type}" if (@since_id && @search_term)
|
30
|
+
if !@cookie
|
28
31
|
response = Curl::Easy.http_post(
|
29
32
|
"#{@api_endpoint_tumblr}/login",
|
30
33
|
:body => {
|
31
34
|
:email => @email,
|
32
|
-
:password => password
|
35
|
+
:password => @password
|
33
36
|
}
|
34
37
|
)
|
35
|
-
@log.debug("initial login response: #{response}")
|
38
|
+
@log.debug("initial login response header: #{response.header_str}") if response
|
36
39
|
@cookie = response.headers['Set-Cookie']
|
37
|
-
@log.debug("
|
40
|
+
@log.debug("login cookie via new login: #{@cookie.inspect}")
|
38
41
|
else
|
39
|
-
@cookie = cookie
|
40
|
-
@log.debug("
|
42
|
+
@cookie = args[:cookie]
|
43
|
+
@log.debug("login cookie via argument: #{@cookie.inspect}")
|
41
44
|
end
|
42
45
|
|
43
46
|
end
|
44
|
-
#post a tumblr photo entry. required arguments are :type, :date, :source, :caption, :state
|
47
|
+
#post a tumblr photo entry. required arguments are :type, :date, :source, :caption, :state. optional argument: :tags
|
45
48
|
def post_to_tumblr(options={})
|
49
|
+
tries = 3
|
46
50
|
if options[:type] && options[:date] && options[:source] && options[:caption] && options[:state]
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
51
|
+
tags = options[:tags]
|
52
|
+
begin
|
53
|
+
response = Curl::Easy.http_post("#{@api_endpoint_tumblr}/api/write",
|
54
|
+
Curl::PostField.content('generator', GENERATOR),
|
55
|
+
Curl::PostField.content('email', @email),
|
56
|
+
Curl::PostField.content('password', @password),
|
57
|
+
Curl::PostField.content('type', options[:type]),
|
58
|
+
Curl::PostField.content('date', options[:date]),
|
59
|
+
Curl::PostField.content('source', options[:source]),
|
60
|
+
Curl::PostField.content('caption', options[:caption]),
|
61
|
+
Curl::PostField.content('state', options[:state]),
|
62
|
+
Curl::PostField.content('tags', tags)
|
63
|
+
)
|
64
|
+
rescue Curl::Err => err
|
65
|
+
@log.error "Failure in Curl call: #{err}"
|
66
|
+
tries -= 1
|
67
|
+
sleep 3
|
68
|
+
if tries > 0
|
69
|
+
retry
|
70
|
+
else
|
71
|
+
response = nil
|
72
|
+
end
|
73
|
+
end
|
57
74
|
end
|
58
75
|
response
|
59
76
|
end
|
@@ -63,13 +80,13 @@ class Tweetlr
|
|
63
80
|
tumblr_post = nil
|
64
81
|
message = tweet['text']
|
65
82
|
if !retweet? message
|
66
|
-
|
67
|
-
#puts "tweet: #{tweet}"
|
83
|
+
@log.debug "tweet: #{tweet}"
|
68
84
|
tumblr_post = {}
|
69
85
|
tumblr_post[:type] = 'photo'
|
70
86
|
tumblr_post[:date] = tweet['created_at']
|
71
87
|
tumblr_post[:source] = extract_image_url tweet
|
72
88
|
user = tweet['from_user']
|
89
|
+
tumblr_post[:tags] = user
|
73
90
|
tweet_id = tweet['id']
|
74
91
|
if @whitelist.member? user.downcase
|
75
92
|
state = 'published'
|
@@ -78,7 +95,7 @@ class Tweetlr
|
|
78
95
|
end
|
79
96
|
tumblr_post[:state] = state
|
80
97
|
shouts = " #{@shouts}" if @shouts
|
81
|
-
tumblr_post[:caption] = %?<a href="http://twitter.com/#{user}/statuses/#{tweet_id}" alt="
|
98
|
+
tumblr_post[:caption] = %?<a href="http://twitter.com/#{user}/statuses/#{tweet_id}" alt="tweet">@#{user}</a>#{shouts}: #{tweet['text']}? #TODO make this a bigger matter of yml configuration
|
82
99
|
end
|
83
100
|
tumblr_post
|
84
101
|
end
|
@@ -98,13 +115,11 @@ class Tweetlr
|
|
98
115
|
@refresh_url = "#{@api_endpoint_twitter}#{@response['refresh_url']}" unless (@response.nil? || @response['refresh_url'].nil? || @response['refresh_url'].empty?)
|
99
116
|
if @refresh_url
|
100
117
|
#FIXME persist the refresh url - server restart would be a pain elsewise
|
101
|
-
#@log.info "lazy search using '#{@refresh_url}'"
|
102
118
|
search_url = "#{@refresh_url}&result_type=#{@result_type}&rpp=#{@results_per_page}"
|
103
|
-
|
119
|
+
@log.info "lazy search using '#{search_url}'" #workaround to get refresh url logged w/ the Daemons gem
|
104
120
|
@response = http_get search_url
|
105
121
|
else
|
106
|
-
|
107
|
-
puts "regular search using '#{@search_term}'"
|
122
|
+
@log.debug "regular search using '#{@search_term}'"
|
108
123
|
@response = search_twitter()
|
109
124
|
end
|
110
125
|
end
|
@@ -128,12 +143,16 @@ class Tweetlr
|
|
128
143
|
end
|
129
144
|
url
|
130
145
|
end
|
131
|
-
|
146
|
+
#find the image's url for a lockerz link
|
147
|
+
def image_url_lockerz(link_url)
|
148
|
+
response = http_get "http://api.plixi.com/api/tpapi.svc/json/metadatafromurl?details=false&url=#{link_url}"
|
149
|
+
response['BigImageUrl'] if response
|
150
|
+
end
|
151
|
+
#find the image's url for an twitter shortened link
|
132
152
|
def image_url_tco(link_url)
|
133
153
|
service_url = link_url_redirect link_url
|
134
154
|
find_image_url service_url
|
135
155
|
end
|
136
|
-
|
137
156
|
#find the image's url for an instagram link
|
138
157
|
def image_url_instagram(link_url)
|
139
158
|
link_url['instagram.com'] = 'instagr.am' if link_url.index 'instagram.com' #instagram's oembed does not work for .com links
|
@@ -149,7 +168,11 @@ class Tweetlr
|
|
149
168
|
#if short url fails, try long url
|
150
169
|
#response = HTTParty.get "http://picplz.com/api/v2/pic.json?longurl_ids=#{id}"
|
151
170
|
#extract url
|
152
|
-
response['value']['pics'].first['
|
171
|
+
if response && response['value'] && response['value']['pics'] && response['value']['pics'].first && response['value']['pics'].first['pic_files'] && response['value']['pics'].first['pic_files']['640r']
|
172
|
+
response['value']['pics'].first['pic_files']['640r']['img_url']
|
173
|
+
else
|
174
|
+
nil
|
175
|
+
end
|
153
176
|
end
|
154
177
|
#find the image's url for a twitpic link
|
155
178
|
def image_url_twitpic(link_url)
|
@@ -202,18 +225,37 @@ class Tweetlr
|
|
202
225
|
|
203
226
|
#convenience method for curl http get calls
|
204
227
|
def http_get(request)
|
228
|
+
tries = 3
|
205
229
|
begin
|
206
230
|
res = Curl::Easy.http_get(request)
|
207
231
|
JSON.parse res.body_str
|
208
232
|
rescue Curl::Err::ConnectionFailedError => err
|
209
|
-
|
210
|
-
|
211
|
-
|
233
|
+
@log.error "Connection failed: #{err}"
|
234
|
+
tries -= 1
|
235
|
+
sleep 3
|
236
|
+
if tries > 0
|
237
|
+
retry
|
238
|
+
else
|
239
|
+
nil
|
240
|
+
end
|
241
|
+
rescue Curl::Err::RecvError => err
|
242
|
+
@log.error "Failure when receiving data from the peer: #{err}"
|
243
|
+
tries -= 1
|
244
|
+
sleep 3
|
245
|
+
if tries > 0
|
246
|
+
retry
|
247
|
+
else
|
248
|
+
nil
|
249
|
+
end
|
250
|
+
rescue Curl::Err => err
|
251
|
+
@log.error "Failure in Curl call: #{err}"
|
252
|
+
tries -= 1
|
253
|
+
sleep 3
|
254
|
+
if tries > 0
|
255
|
+
retry
|
256
|
+
else
|
257
|
+
nil
|
258
|
+
end
|
212
259
|
end
|
213
|
-
end
|
214
|
-
|
215
|
-
end
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
260
|
+
end
|
261
|
+
end
|
metadata
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
name: tweetlr
|
3
3
|
version: !ruby/object:Gem::Version
|
4
4
|
prerelease:
|
5
|
-
version: 0.0
|
5
|
+
version: 0.1.0
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- Sven Kraeuter
|
@@ -10,7 +10,7 @@ autorequire:
|
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
12
|
|
13
|
-
date: 2011-
|
13
|
+
date: 2011-06-02 00:00:00 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: daemons
|