tweetlr 0.1.20 → 0.1.21
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +2 -1
- data/.travis.yml +1 -0
- data/Gemfile +1 -1
- data/README.md +2 -3
- data/lib/tweetlr/core.rb +44 -35
- data/lib/tweetlr/processors/photo_service.rb +39 -55
- data/lib/tweetlr.rb +1 -1
- data/spec/combinators/twitter_tumblr_combinator_spec.rb +0 -4
- data/spec/processors/photo_services_processor_spec.rb +15 -15
- data/spec/spec_helper.rb +73 -594
- data/tweetlr.gemspec +1 -1
- metadata +72 -21
data/.gitignore
CHANGED
data/.travis.yml
CHANGED
data/Gemfile
CHANGED
@@ -1,2 +1,2 @@
|
|
1
|
-
source
|
1
|
+
source 'https://rubygems.org'
|
2
2
|
gemspec
|
data/README.md
CHANGED
@@ -10,17 +10,16 @@ There is a new [tweetlr "as-a-service"](http://tweetlr.5v3n.com) where you can e
|
|
10
10
|
|
11
11
|
tweetlr supports
|
12
12
|
|
13
|
-
-
|
13
|
+
- instagram
|
14
14
|
- twitter
|
15
15
|
- photobucket
|
16
16
|
- twimg
|
17
17
|
- foursquare
|
18
18
|
- path.com
|
19
|
-
- picplz
|
20
19
|
- twitpic
|
21
20
|
- yfrog
|
22
21
|
- imgly
|
23
|
-
-
|
22
|
+
- eyeem.com
|
24
23
|
- t.co shortened links to pictures
|
25
24
|
- every photo service accessible via embed.ly (see [photo providers](http://embed.ly/providers))
|
26
25
|
|
data/lib/tweetlr/core.rb
CHANGED
@@ -15,24 +15,8 @@ class Tweetlr::Core
|
|
15
15
|
end
|
16
16
|
|
17
17
|
def initialize(args)
|
18
|
-
|
19
|
-
|
20
|
-
log.level = args[:loglevel]
|
21
|
-
else
|
22
|
-
log.level = Logger::INFO
|
23
|
-
end
|
24
|
-
log.debug "log level set to #{log.level}"
|
25
|
-
Tweetlr::LogAware.log=log
|
26
|
-
|
27
|
-
@email = args[:tumblr_email]
|
28
|
-
@password = args[:tumblr_password]
|
29
|
-
@cookie = args[:cookie]
|
30
|
-
@api_endpoint_twitter = args[:api_endpoint_twitter] || Tweetlr::API_ENDPOINT_TWITTER
|
31
|
-
@api_endpoint_tumblr = args[:api_endpoint_tumblr] || Tweetlr::API_ENDPOINT_TUMBLR
|
32
|
-
@whitelist = args[:whitelist]
|
33
|
-
@shouts = args[:shouts]
|
34
|
-
@update_period = args[:update_period] || Tweetlr::UPDATE_PERIOD
|
35
|
-
@whitelist.each {|entry| entry.downcase!} if @whitelist
|
18
|
+
initialize_logging(args[:loglevel])
|
19
|
+
initialize_attributes(args)
|
36
20
|
log.info "Tweetlr #{Tweetlr::VERSION} initialized. Ready to roll."
|
37
21
|
end
|
38
22
|
|
@@ -55,26 +39,51 @@ class Tweetlr::Core
|
|
55
39
|
return config
|
56
40
|
end
|
57
41
|
private
|
42
|
+
def initialize_attributes(args)
|
43
|
+
@email = args[:tumblr_email]
|
44
|
+
@password = args[:tumblr_password]
|
45
|
+
@cookie = args[:cookie]
|
46
|
+
@api_endpoint_twitter = args[:api_endpoint_twitter] || Tweetlr::API_ENDPOINT_TWITTER
|
47
|
+
@api_endpoint_tumblr = args[:api_endpoint_tumblr] || Tweetlr::API_ENDPOINT_TUMBLR
|
48
|
+
@whitelist = args[:whitelist]
|
49
|
+
@shouts = args[:shouts]
|
50
|
+
@update_period = args[:update_period] || Tweetlr::UPDATE_PERIOD
|
51
|
+
@whitelist.each {|entry| entry.downcase!} if @whitelist
|
52
|
+
end
|
53
|
+
def initialize_logging(loglevel)
|
54
|
+
log = Logger.new(STDOUT)
|
55
|
+
if (Logger::DEBUG..Logger::UNKNOWN).to_a.index(loglevel)
|
56
|
+
log.level = loglevel
|
57
|
+
else
|
58
|
+
log.level = Logger::INFO
|
59
|
+
end
|
60
|
+
log.debug "log level set to #{log.level}"
|
61
|
+
Tweetlr::LogAware.log=log
|
62
|
+
end
|
58
63
|
def self.process_response(response, config, tumblr_config)
|
59
64
|
tweets = response['results']
|
60
|
-
if tweets
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
if res && res.code == "201"
|
70
|
-
log.info "tumblr post created (tumblr response: #{res.header} #{res.body}"
|
71
|
-
elsif res
|
72
|
-
log.warn "tumblr response: #{res.header} #{res.body}"
|
73
|
-
else
|
74
|
-
log.warn "there was no tumblr post response - most probably due to a missing oauth authorization"
|
75
|
-
end
|
76
|
-
end
|
65
|
+
process_and_post tweets, config, tumblr_config if tweets
|
66
|
+
end
|
67
|
+
def self.process_and_post(tweets, config, tumblr_config)
|
68
|
+
tweets.each do |tweet|
|
69
|
+
tumblr_post = Tweetlr::Combinators::TwitterTumblr::generate_photo_post_from_tweet(tweet, {:whitelist => config[:whitelist], :embedly_key => config[:embedly_key], :group => config[:group]})
|
70
|
+
if tumblr_post.nil? || tumblr_post[:source].nil?
|
71
|
+
log.warn "could not get image source: tweet: #{tweet} --- tumblr post: #{tumblr_post.inspect}"
|
72
|
+
else
|
73
|
+
post_to_tumblr
|
77
74
|
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
def self.post_to_tumblr(tumblr_post, tumblr_config)
|
78
|
+
log.debug "tumblr post: #{tumblr_post}"
|
79
|
+
res = Tweetlr::Processors::Tumblr.post tumblr_post.merge(tumblr_config)
|
80
|
+
log.debug "tumblr response: #{res}"
|
81
|
+
if res && res.code == "201"
|
82
|
+
log.info "tumblr post created (tumblr response: #{res.header} #{res.body}"
|
83
|
+
elsif res
|
84
|
+
log.warn "tumblr response: #{res.header} #{res.body}"
|
85
|
+
else
|
86
|
+
log.warn "there was no tumblr post response - most probably due to a missing oauth authorization"
|
78
87
|
end
|
79
88
|
end
|
80
89
|
def self.prepare_twitter_config(config)
|
@@ -22,15 +22,13 @@ module Tweetlr::Processors
|
|
22
22
|
if link && !(photo? link)
|
23
23
|
url = image_url_eyeem link if link.index 'eyeem.com'
|
24
24
|
url = image_url_instagram link if (link.index('instagr.am') || link.index('instagram.com'))
|
25
|
-
url = image_url_picplz link if link.index 'picplz'
|
26
25
|
url = image_url_twitpic link if link.index 'twitpic'
|
27
26
|
url = image_url_yfrog link if link.index 'yfrog'
|
28
|
-
url = image_url_imgly link if link.index 'img.ly'
|
27
|
+
url = image_url_imgly link, embedly_key if link.index 'img.ly'
|
29
28
|
url = image_url_tco link, embedly_key if link.index 't.co'
|
30
29
|
url = image_url_twimg link if link.index 'twitter.com'
|
31
|
-
url = image_url_lockerz link if link.index 'lockerz.com'
|
32
30
|
url = image_url_path link if link.index 'path.com'
|
33
|
-
url = image_url_foursqaure link if link.index
|
31
|
+
url = image_url_foursqaure link if (link.index('4sq.com') || link.index('foursquare.com'))
|
34
32
|
url = image_url_embedly link, embedly_key if url.nil? #just try embed.ly for anything else. could do all image url processing w/ embedly, but there's probably some kind of rate limit invovled.
|
35
33
|
elsif photo? link
|
36
34
|
url = link
|
@@ -42,7 +40,7 @@ module Tweetlr::Processors
|
|
42
40
|
link =~ PIC_REGEXP
|
43
41
|
end
|
44
42
|
def self.image_url_twimg(link_url)
|
45
|
-
retrieve_image_url_by_css link_url, '.
|
43
|
+
retrieve_image_url_by_css link_url, '.media img'
|
46
44
|
end
|
47
45
|
#extract the image of an eyeem.com pic
|
48
46
|
def self.image_url_eyeem(link_url)
|
@@ -50,7 +48,9 @@ module Tweetlr::Processors
|
|
50
48
|
end
|
51
49
|
#extract the image of a foursquare.com pic
|
52
50
|
def self.image_url_foursqaure(link_url)
|
53
|
-
|
51
|
+
link_url = follow_redirect(link_url)
|
52
|
+
image_url = retrieve_image_url_by_css link_url, 'meta[property="og:image"]', 'content'
|
53
|
+
image_url
|
54
54
|
end
|
55
55
|
#extract the image of a path.com pic
|
56
56
|
def self.image_url_path(link_url)
|
@@ -59,18 +59,14 @@ module Tweetlr::Processors
|
|
59
59
|
|
60
60
|
#find the image's url via embed.ly
|
61
61
|
def self.image_url_embedly(link_url, key)
|
62
|
-
|
62
|
+
link_url = follow_redirect(link_url)
|
63
63
|
log.debug "embedly call: http://api.embed.ly/1/oembed?key=#{key}&url=#{link_url}"
|
64
|
-
|
64
|
+
response = Tweetlr::Processors::Http::http_get_json "http://api.embed.ly/1/oembed?key=#{key}&url=#{link_url}"
|
65
|
+
if response && (response['type'] == 'photo' || response['type'] == 'image')
|
65
66
|
image_url = response['url']
|
66
67
|
end
|
67
68
|
image_url
|
68
69
|
end
|
69
|
-
#find the image's url for a lockerz link
|
70
|
-
def self.image_url_lockerz(link_url)
|
71
|
-
response = Tweetlr::Processors::Http::http_get_json "http://api.plixi.com/api/tpapi.svc/json/metadatafromurl?details=false&url=#{link_url}"
|
72
|
-
response["BigImageUrl"] if response
|
73
|
-
end
|
74
70
|
#find the image's url for an twitter shortened link
|
75
71
|
def self.image_url_tco(link_url, embedly_key = nil)
|
76
72
|
service_url = link_url_redirect link_url
|
@@ -82,33 +78,17 @@ module Tweetlr::Processors
|
|
82
78
|
response = Tweetlr::Processors::Http::http_get_json "http://api.instagram.com/oembed?url=#{link_url}"
|
83
79
|
response['url'] if response
|
84
80
|
end
|
85
|
-
|
86
|
-
#find the image's url for a picplz short/longlink
|
87
|
-
def self.image_url_picplz(link_url)
|
88
|
-
id = extract_id link_url
|
89
|
-
#try short url
|
90
|
-
response = Tweetlr::Processors::Http::http_get_json "http://picplz.com/api/v2/pic.json?shorturl_ids=#{id}"
|
91
|
-
#if short url fails, try long url
|
92
|
-
#response = HTTParty.get "http://picplz.com/api/v2/pic.json?longurl_ids=#{id}"
|
93
|
-
#extract url
|
94
|
-
if response && response['value'] && response['value']['pics'] && response['value']['pics'].first && response['value']['pics'].first['pic_files'] && response['value']['pics'].first['pic_files']['640r']
|
95
|
-
response['value']['pics'].first['pic_files']['640r']['img_url']
|
96
|
-
else
|
97
|
-
nil
|
98
|
-
end
|
99
|
-
end
|
100
81
|
#find the image's url for a twitpic link
|
101
82
|
def self.image_url_twitpic(link_url)
|
102
83
|
image_url_redirect link_url, "http://twitpic.com/show/full/"
|
103
84
|
end
|
104
85
|
#find the image'S url for a yfrog link
|
105
86
|
def self.image_url_yfrog(link_url)
|
106
|
-
|
107
|
-
response['url'] if response
|
87
|
+
retrieve_image_url_by_css link_url, '#input-direct', 'value'
|
108
88
|
end
|
109
89
|
#find the image's url for a img.ly link
|
110
|
-
def self.image_url_imgly(link_url)
|
111
|
-
|
90
|
+
def self.image_url_imgly(link_url, embedly_key)
|
91
|
+
retrieve_image_url_by_css link_url, '#the-image'
|
112
92
|
end
|
113
93
|
|
114
94
|
# extract image url from services like twitpic & img.ly that do not offer oembed interfaces
|
@@ -121,22 +101,12 @@ module Tweetlr::Processors
|
|
121
101
|
begin
|
122
102
|
resp = Curl::Easy.http_get(short_url) { |res| res.follow_location = true }
|
123
103
|
rescue Curl::Err::CurlError => err
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
retry
|
129
|
-
else
|
130
|
-
return nil
|
131
|
-
end
|
132
|
-
end
|
133
|
-
if(resp && resp.header_str && resp.header_str.index(LOCATION_START_INDICATOR) && resp.header_str.index(stop_indicator))
|
134
|
-
start = resp.header_str.index(LOCATION_START_INDICATOR) + LOCATION_START_INDICATOR.size
|
135
|
-
stop = resp.header_str.index(stop_indicator, start)
|
136
|
-
resp.header_str[start...stop]
|
137
|
-
else
|
138
|
-
nil
|
104
|
+
log.error "Curl::Easy.http_get failed: #{err}"
|
105
|
+
tries -= 1
|
106
|
+
sleep 3
|
107
|
+
(tries > 0) ? retry : return
|
139
108
|
end
|
109
|
+
process_reponse_header resp, stop_indicator
|
140
110
|
end
|
141
111
|
|
142
112
|
#extract the pic id from a given <code>link</code>
|
@@ -144,22 +114,36 @@ module Tweetlr::Processors
|
|
144
114
|
link.split('/').last if link.split('/')
|
145
115
|
end
|
146
116
|
#parse html doc for element signature
|
147
|
-
def self.parse_html_for(element_signature, html_doc)
|
117
|
+
def self.parse_html_for(element_signature, html_doc, identifier="src")
|
148
118
|
image_url= nil
|
149
119
|
if html_doc
|
150
120
|
photo_container_div = html_doc.css(element_signature)
|
151
|
-
if photo_container_div && photo_container_div.first && photo_container_div.first.attributes[
|
152
|
-
image_url = photo_container_div.first.attributes[
|
121
|
+
if photo_container_div && photo_container_div.first && photo_container_div.first.attributes[identifier]
|
122
|
+
image_url = photo_container_div.first.attributes[identifier].value
|
153
123
|
end
|
154
124
|
end
|
155
125
|
image_url
|
156
126
|
end
|
157
|
-
def self.retrieve_image_url_by_css
|
127
|
+
def self.retrieve_image_url_by_css(link_url, css_path, selector='src')
|
128
|
+
link_url = follow_redirect link_url
|
129
|
+
response = Tweetlr::Processors::Http::http_get link_url
|
130
|
+
image_url = parse_html_for css_path, Nokogiri::HTML.parse(response.body_str), selector
|
131
|
+
return image_url
|
132
|
+
end
|
133
|
+
private
|
134
|
+
def self.process_reponse_header(resp, stop_indicator)
|
135
|
+
if(resp && resp.header_str && resp.header_str.index(LOCATION_START_INDICATOR) && resp.header_str.index(stop_indicator))
|
136
|
+
start = resp.header_str.index(LOCATION_START_INDICATOR) + LOCATION_START_INDICATOR.size
|
137
|
+
stop = resp.header_str.index(stop_indicator, start)
|
138
|
+
resp.header_str[start...stop]
|
139
|
+
else
|
140
|
+
nil
|
141
|
+
end
|
142
|
+
end
|
143
|
+
def self.follow_redirect(link_url)
|
158
144
|
service_url = link_url_redirect link_url #follow possible redirects
|
159
|
-
|
160
|
-
|
161
|
-
image_url = parse_html_for css_path, Nokogiri::HTML.parse(response.body_str)
|
162
|
-
return image_url
|
145
|
+
link_url = service_url if service_url #if there's no redirect, service_url will be nil
|
146
|
+
link_url
|
163
147
|
end
|
164
148
|
end
|
165
149
|
end
|
data/lib/tweetlr.rb
CHANGED
@@ -15,10 +15,8 @@ describe Tweetlr::Combinators::TwitterTumblr do
|
|
15
15
|
:instagram => {'text' => "jadda jadda http://instagr.am/p/DzCWn/"},
|
16
16
|
:twitpic => {'text' => "jadda jadda http://twitpic.com/449o2x"},
|
17
17
|
:yfrog => {'text' => "jadda jadda http://yfrog.com/h4vlfp"},
|
18
|
-
:picplz => {'text' => "jadda jadda http://picplz.com/2hWv"},
|
19
18
|
:imgly => {'text' => "jadda jadda http://img.ly/3M1o"},
|
20
19
|
:tco => {'text' => "jadda jadda http://t.co/MUGNayA"},
|
21
|
-
:lockerz => {'text' => "jadda jadda http://lockerz.com/s/100269159"},
|
22
20
|
:embedly => {'text' => "jadda jadda http://flic.kr/p/973hTv"},
|
23
21
|
:twitter_pics => {'text' => "jadda jadda http://t.co/FmyBGfyY"}
|
24
22
|
}
|
@@ -26,10 +24,8 @@ describe Tweetlr::Combinators::TwitterTumblr do
|
|
26
24
|
:instagram => "http://instagr.am/p/DzCWn/",
|
27
25
|
:twitpic => "http://twitpic.com/449o2x",
|
28
26
|
:yfrog => "http://yfrog.com/h4vlfp",
|
29
|
-
:picplz => "http://picplz.com/2hWv",
|
30
27
|
:imgly => "http://img.ly/3M1o",
|
31
28
|
:tco => 'http://t.co/MUGNayA',
|
32
|
-
:lockerz => 'http://lockerz.com/s/100269159',
|
33
29
|
:embedly => 'http://flic.kr/p/973hTv',
|
34
30
|
:twitter_pics => 'http://t.co/FmyBGfyY'
|
35
31
|
}
|
@@ -3,28 +3,35 @@ require 'spec_helper'
|
|
3
3
|
describe Tweetlr::Processors::PhotoService do
|
4
4
|
before :each do
|
5
5
|
@links = {
|
6
|
-
:twimg => 'http://twitter.com/KSilbereisen/status/228035435237097472',
|
7
|
-
:eyeem => 'http://www.eyeem.com/p/326629',
|
8
6
|
:foursquare => 'http://4sq.com/x4p87N',
|
7
|
+
:eyeem => 'http://www.eyeem.com/p/326629',
|
9
8
|
:path => 'http://path.com/p/KQd57',
|
10
9
|
:instagram => "http://instagr.am/p/DzCWn/",
|
11
10
|
:twitpic => "http://twitpic.com/449o2x",
|
12
11
|
:yfrog => "http://yfrog.com/h4vlfp",
|
13
|
-
:picplz => "http://picplz.com/2hWv",
|
14
|
-
:imgly => "http://img.ly/3M1o",
|
15
12
|
:tco => 'http://t.co/MUGNayA',
|
16
|
-
:lockerz => 'http://lockerz.com/s/100269159',
|
17
13
|
:embedly => 'http://flic.kr/p/973hTv',
|
18
|
-
:twitter_pics => 'http://t.co/FmyBGfyY'
|
14
|
+
:twitter_pics => 'http://t.co/FmyBGfyY',
|
15
|
+
:twimg => 'http://twitter.com/KSilbereisen/status/228035435237097472',
|
16
|
+
:imgly => "http://img.ly/3M1o"
|
19
17
|
}
|
20
18
|
end
|
19
|
+
it "finds a picture's url from the supported services" do
|
20
|
+
@links.each do |service,link|
|
21
|
+
send "stub_#{service}"
|
22
|
+
#puts "checking #{service}"
|
23
|
+
url = Tweetlr::Processors::PhotoService::find_image_url link
|
24
|
+
url.should be, "service #{service} not working!"
|
25
|
+
check_pic_url_extraction service if [:twimg, :instagram,:yfrog,:imgly,:foursqaure,:not_listed].index service
|
26
|
+
end
|
27
|
+
end
|
21
28
|
it "extracts images from eye em" do
|
22
29
|
stub_eyeem
|
23
30
|
link = Tweetlr::Processors::PhotoService::find_image_url @links[:eyeem]
|
24
31
|
link.should be
|
25
32
|
link.should == "http://www.eyeem.com/thumb/h/1024/e35db836c5d3f02498ef60fc3d53837fbe621561-1334126483"
|
26
33
|
end
|
27
|
-
it "doesnt find images in embedly results that are not explicitly marked as 'Photo' via the response's 'thumbnail_url' attribute" do
|
34
|
+
it "doesnt find images in embedly results that are not explicitly marked as 'Photo' or 'Image' via the response's 'thumbnail_url' attribute" do
|
28
35
|
stub_embedly_no_photo
|
29
36
|
link = Tweetlr::Processors::PhotoService::find_image_url 'http://makersand.co/'
|
30
37
|
link.should be_nil
|
@@ -32,16 +39,9 @@ describe Tweetlr::Processors::PhotoService do
|
|
32
39
|
it "does find an image for foursquare that is not he profile pic" do
|
33
40
|
stub_foursquare
|
34
41
|
link = Tweetlr::Processors::PhotoService::find_image_url @links[:foursquare]
|
42
|
+
link.should be
|
35
43
|
link.index('userpix_thumbs').should_not be
|
36
44
|
end
|
37
|
-
it "should find a picture's url from the supported services" do
|
38
|
-
@links.each do |service,link|
|
39
|
-
send "stub_#{service}"
|
40
|
-
url = Tweetlr::Processors::PhotoService::find_image_url link
|
41
|
-
url.should be, "service #{service} not working!"
|
42
|
-
check_pic_url_extraction service if [:twimg, :instagram,:picplz,:yfrog,:imgly,:foursqaure,:not_listed].index service
|
43
|
-
end
|
44
|
-
end
|
45
45
|
it "finds path images for redirected moments as well" do
|
46
46
|
stub_path_redirected
|
47
47
|
url = Tweetlr::Processors::PhotoService::find_image_url @links[:path]
|