tweetable 0.1.10
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +8 -0
- data/History.txt +4 -0
- data/Manifest.txt +33 -0
- data/PostInstall.txt +7 -0
- data/README.rdoc +37 -0
- data/Rakefile +19 -0
- data/VERSION +1 -0
- data/lib/tweetable/authorization.rb +21 -0
- data/lib/tweetable/collection.rb +54 -0
- data/lib/tweetable/link.rb +30 -0
- data/lib/tweetable/message.rb +119 -0
- data/lib/tweetable/persistable.rb +32 -0
- data/lib/tweetable/photo.rb +6 -0
- data/lib/tweetable/queue.rb +121 -0
- data/lib/tweetable/search.rb +77 -0
- data/lib/tweetable/twitter_client.rb +36 -0
- data/lib/tweetable/twitter_streaming_client.rb +67 -0
- data/lib/tweetable/url.rb +39 -0
- data/lib/tweetable/user.rb +104 -0
- data/lib/tweetable.rb +82 -0
- data/pkg/tweetable-0.1.7.gem +0 -0
- data/script/console +10 -0
- data/script/destroy +14 -0
- data/script/generate +14 -0
- data/spec/collection_spec.rb +55 -0
- data/spec/fixtures/blank.json +1 -0
- data/spec/fixtures/flippyhead.json +1 -0
- data/spec/fixtures/follower_ids.json +1 -0
- data/spec/fixtures/friend_ids.json +1 -0
- data/spec/fixtures/friends_timeline.json +1 -0
- data/spec/fixtures/link_blank.json +1 -0
- data/spec/fixtures/link_exists.json +1 -0
- data/spec/fixtures/rate_limit_status.json +1 -0
- data/spec/fixtures/search.json +1 -0
- data/spec/fixtures/user_timeline.json +1 -0
- data/spec/fixtures/verify_credentials.json +1 -0
- data/spec/link_spec.rb +35 -0
- data/spec/message_spec.rb +148 -0
- data/spec/persistable_spec.rb +53 -0
- data/spec/queue_spec.rb +29 -0
- data/spec/search_spec.rb +60 -0
- data/spec/spec.opts +5 -0
- data/spec/spec_helper.rb +55 -0
- data/spec/tweetable_spec.rb +19 -0
- data/spec/twitter_client_spec.rb +41 -0
- data/spec/twitter_streaming_client_spec.rb +18 -0
- data/spec/user_spec.rb +143 -0
- data/tweetable.gemspec +106 -0
- metadata +165 -0
data/Gemfile
ADDED
data/History.txt
ADDED
data/Manifest.txt
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
History.txt
|
2
|
+
Manifest.txt
|
3
|
+
PostInstall.txt
|
4
|
+
README.rdoc
|
5
|
+
Rakefile
|
6
|
+
lib/tweetable.rb
|
7
|
+
lib/tweetable/authorization.rb
|
8
|
+
lib/tweetable/collection.rb
|
9
|
+
lib/tweetable/link.rb
|
10
|
+
lib/tweetable/message.rb
|
11
|
+
lib/tweetable/persistable.rb
|
12
|
+
lib/tweetable/photo.rb
|
13
|
+
lib/tweetable/queue.rb
|
14
|
+
lib/tweetable/search.rb
|
15
|
+
lib/tweetable/twitter_client.rb
|
16
|
+
lib/tweetable/twitter_streaming_client.rb
|
17
|
+
lib/tweetable/url.rb
|
18
|
+
lib/tweetable/user.rb
|
19
|
+
script/console
|
20
|
+
script/destroy
|
21
|
+
script/generate
|
22
|
+
spec/collection_spec.rb
|
23
|
+
spec/link_spec.rb
|
24
|
+
spec/message_spec.rb
|
25
|
+
spec/persistable_spec.rb
|
26
|
+
spec/queue_spec.rb
|
27
|
+
spec/search_spec.rb
|
28
|
+
spec/spec.opts
|
29
|
+
spec/spec_helper.rb
|
30
|
+
spec/twitter_client_spec.rb
|
31
|
+
spec/twitter_streaming_client_spec.rb
|
32
|
+
spec/user_spec.rb
|
33
|
+
tweetable.gemspec
|
data/PostInstall.txt
ADDED
data/README.rdoc
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
== DESCRIPTION:
|
2
|
+
|
3
|
+
== EXAMPLES:
|
4
|
+
|
5
|
+
To create a new Tweetable user:
|
6
|
+
|
7
|
+
@user = Tweetable::User.create(:screen_name => 'flippyhead')
|
8
|
+
|
9
|
+
To then grab recent messages, friend counts, and other profile data:
|
10
|
+
|
11
|
+
@user.update_all # will only grab messages since the last known message
|
12
|
+
|
13
|
+
Now you have access to stuff like:
|
14
|
+
|
15
|
+
@user.friend_ids # [34102, 23423, 67567, etc...]
|
16
|
+
@user.friend_ids.size # 102
|
17
|
+
@user.profile_image_url # http://twitter.com/...
|
18
|
+
@user.messages.size # 202
|
19
|
+
|
20
|
+
Links in messages can be extracted and expanded:
|
21
|
+
|
22
|
+
@message = @user.messages.first
|
23
|
+
@message.parse_links
|
24
|
+
|
25
|
+
@link = @message.links.first # Tweetable::Link
|
26
|
+
@link.url # http://tinyurl.com/yfuhltt
|
27
|
+
@link.long_url # http://pathable.com
|
28
|
+
|
29
|
+
And are connected to other users who mention them:
|
30
|
+
|
31
|
+
@message.links.size # 2
|
32
|
+
@link.count # 8 (uses discovered so far)
|
33
|
+
@link.users # [<Tweetable::User:0x1 @_attributes={}>, ...]
|
34
|
+
|
35
|
+
Performing a keyword search is just as easy:
|
36
|
+
|
37
|
+
# @search =
|
data/Rakefile
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler'
|
3
|
+
require 'rake'
|
4
|
+
|
5
|
+
begin
|
6
|
+
require 'jeweler'
|
7
|
+
Jeweler::Tasks.new do |gemspec|
|
8
|
+
gemspec.name = "tweetable"
|
9
|
+
gemspec.summary = "Track twitter messages and users in memory using Redis"
|
10
|
+
gemspec.description = "Track twitter messages and users in memory using Redis"
|
11
|
+
gemspec.email = "peter@flippyhead.com"
|
12
|
+
gemspec.homepage = "http://github.com/flippyhead/tweetable"
|
13
|
+
gemspec.authors = ["Peter T. Brown"]
|
14
|
+
gemspec.add_bundler_dependencies
|
15
|
+
end
|
16
|
+
Jeweler::GemcutterTasks.new
|
17
|
+
rescue LoadError
|
18
|
+
puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
|
19
|
+
end
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.1.10
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module Tweetable
|
2
|
+
class Authorization < Persistable
|
3
|
+
attribute :oauth_access_token
|
4
|
+
attribute :oauth_access_secret
|
5
|
+
|
6
|
+
index :oauth_access_token
|
7
|
+
|
8
|
+
def user_id
|
9
|
+
return if self.oauth_access_token.nil?
|
10
|
+
self.oauth_access_token.split('-')[0] # tokens start with ID as in: 13705052-bz9IrOTwWbLgWHQDKkGnVd815ybTujc0QeXMlh7ZJ
|
11
|
+
end
|
12
|
+
|
13
|
+
protected
|
14
|
+
|
15
|
+
def validate
|
16
|
+
assert_present :oauth_access_token
|
17
|
+
assert_present :oauth_access_secret
|
18
|
+
assert_unique :oauth_access_token
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
module Tweetable
|
2
|
+
class Collection < Persistable
|
3
|
+
attribute :created_at
|
4
|
+
attribute :updated_at
|
5
|
+
attribute :name # super class attributes don't get picked up in subclasses for some reason
|
6
|
+
index :name
|
7
|
+
|
8
|
+
def validate
|
9
|
+
assert_present :name
|
10
|
+
assert_unique :name
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
class UserCollection < Collection
|
15
|
+
attribute :created_at
|
16
|
+
attribute :updated_at
|
17
|
+
attribute :name
|
18
|
+
index :name
|
19
|
+
|
20
|
+
set :user_set, User
|
21
|
+
list :users, User
|
22
|
+
end
|
23
|
+
|
24
|
+
class MessageCollection < Collection
|
25
|
+
attribute :created_at
|
26
|
+
attribute :updated_at
|
27
|
+
attribute :name
|
28
|
+
index :name
|
29
|
+
|
30
|
+
set :message_set, Message
|
31
|
+
list :messages, Message
|
32
|
+
end
|
33
|
+
|
34
|
+
class SearchCollection < Collection
|
35
|
+
attribute :created_at
|
36
|
+
attribute :updated_at
|
37
|
+
attribute :name
|
38
|
+
index :name
|
39
|
+
|
40
|
+
set :search_set, Search
|
41
|
+
list :searches, Search
|
42
|
+
end
|
43
|
+
|
44
|
+
class LinkCollection < Collection
|
45
|
+
attribute :created_at
|
46
|
+
attribute :updated_at
|
47
|
+
attribute :name
|
48
|
+
index :name
|
49
|
+
|
50
|
+
set :link_set, Link
|
51
|
+
list :links, Link
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
module Tweetable
|
2
|
+
class Link < Persistable
|
3
|
+
URL_PATTERN = /(http:\S+)/ix
|
4
|
+
|
5
|
+
attribute :created_at
|
6
|
+
attribute :updated_at
|
7
|
+
attribute :url
|
8
|
+
attribute :long_url
|
9
|
+
|
10
|
+
index :url
|
11
|
+
index :long_url
|
12
|
+
|
13
|
+
# set :messages, Tweetable::Message
|
14
|
+
set :users, Tweetable::User
|
15
|
+
counter :count
|
16
|
+
|
17
|
+
def increment_usage_count(user)
|
18
|
+
return false if (user.nil? || self.users.include?(user))
|
19
|
+
users.add(user)
|
20
|
+
self.incr(:count)
|
21
|
+
end
|
22
|
+
|
23
|
+
protected
|
24
|
+
|
25
|
+
def validate
|
26
|
+
assert_present :url
|
27
|
+
assert_unique :url
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,119 @@
|
|
1
|
+
module Tweetable
|
2
|
+
class Message < Persistable
|
3
|
+
attr_accessor :owner
|
4
|
+
|
5
|
+
attribute :created_at
|
6
|
+
attribute :updated_at
|
7
|
+
attribute :message_id
|
8
|
+
attribute :text
|
9
|
+
attribute :favorited
|
10
|
+
attribute :from_user_id
|
11
|
+
attribute :from_screen_name
|
12
|
+
attribute :links_parsed
|
13
|
+
attribute :photos_parsed
|
14
|
+
attribute :sent_at
|
15
|
+
attribute :created_at
|
16
|
+
|
17
|
+
set :links, Link
|
18
|
+
set :photos, Photo
|
19
|
+
set :tags
|
20
|
+
|
21
|
+
index :message_id
|
22
|
+
index :links_parsed
|
23
|
+
index :photos_parsed
|
24
|
+
|
25
|
+
def self.create_from_timeline(message, create_user = false)
|
26
|
+
m = Message.find_or_create(:message_id, message[:id])
|
27
|
+
|
28
|
+
m.update(
|
29
|
+
:favorited => message.favorited,
|
30
|
+
:photos_parsed => '0',
|
31
|
+
:links_parsed => '0',
|
32
|
+
:created_at => Time.now.utc.to_s,
|
33
|
+
:sent_at => message.created_at,
|
34
|
+
:text => message.text,
|
35
|
+
:from_screen_name => message.user.screen_name.downcase,
|
36
|
+
:from_user_id => message.user[:id])
|
37
|
+
|
38
|
+
if create_user and m.valid?
|
39
|
+
u = User.create_from_timeline(message.user)
|
40
|
+
u.messages << m if u.valid?
|
41
|
+
end
|
42
|
+
m
|
43
|
+
end
|
44
|
+
|
45
|
+
def self.create_from_status(text, client)
|
46
|
+
self.create_from_timeline(client.update(text), true)
|
47
|
+
end
|
48
|
+
|
49
|
+
def self.purge(&block)
|
50
|
+
all.sort.each do |message|
|
51
|
+
message.delete if yield(message)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def from_user
|
56
|
+
return nil if self.from_screen_name.nil?
|
57
|
+
User.find(:screen_name => self.from_screen_name.downcase).first
|
58
|
+
end
|
59
|
+
|
60
|
+
def parse_links(force = false, longify = true)
|
61
|
+
return unless (force or self.links_parsed != '1')
|
62
|
+
|
63
|
+
urls = self.text.scan(Link::URL_PATTERN).flatten
|
64
|
+
urls.each do |url|
|
65
|
+
link = Link.find(:url => url).first
|
66
|
+
|
67
|
+
if !link
|
68
|
+
link = Link.create(:url => url, :created_at => Time.new.to_s)
|
69
|
+
next if !link.valid?
|
70
|
+
|
71
|
+
# link.messages.add(self)
|
72
|
+
long_url = URL.lookup_long_url(url)
|
73
|
+
link.update(:long_url => long_url) unless (long_url.nil? or long_url == url)
|
74
|
+
end
|
75
|
+
|
76
|
+
link.increment_usage_count(from_user)
|
77
|
+
|
78
|
+
update(:links_parsed => '1')
|
79
|
+
links.add(link)
|
80
|
+
end
|
81
|
+
|
82
|
+
links
|
83
|
+
end
|
84
|
+
|
85
|
+
def twitter_link
|
86
|
+
"http://twitter.com/#{from_screen_name}/status/#{message_id}"
|
87
|
+
end
|
88
|
+
|
89
|
+
def validate
|
90
|
+
super
|
91
|
+
assert_unique :message_id
|
92
|
+
assert_present :text
|
93
|
+
assert_format :links_parsed, /^[0,1]$/
|
94
|
+
assert_format :photos_parsed, /^[0,1]$/
|
95
|
+
end
|
96
|
+
|
97
|
+
def hash
|
98
|
+
self.id.hash
|
99
|
+
end
|
100
|
+
|
101
|
+
# Simply delegate to == in this example.
|
102
|
+
def eql?(comparee)
|
103
|
+
self == comparee
|
104
|
+
end
|
105
|
+
|
106
|
+
# Objects are equal if they have the same
|
107
|
+
# unique custom identifier.
|
108
|
+
def ==(comparee)
|
109
|
+
self.id == comparee.id
|
110
|
+
end
|
111
|
+
|
112
|
+
# It seems that, at least using streaming, message id's are not sequential anymore
|
113
|
+
# So comparisons are done on the official sent_at date/time
|
114
|
+
def <=>(o)
|
115
|
+
return 1 if o.nil?
|
116
|
+
Time.parse(self.sent_at) <=> Time.parse(o.sent_at)
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'ohm'
|
2
|
+
|
3
|
+
module Tweetable
|
4
|
+
class Persistable < Ohm::Model
|
5
|
+
attribute :created_at
|
6
|
+
attribute :updated_at
|
7
|
+
|
8
|
+
def self.find_or_create(key, value)
|
9
|
+
attributes = {key => value} # this persistable uses an old interface
|
10
|
+
models = self.find(attributes)
|
11
|
+
models.empty? ? self.create(attributes.merge(:created_at => Time.now.utc.to_s)) : models.first
|
12
|
+
end
|
13
|
+
|
14
|
+
def needs_update?(force = false)
|
15
|
+
force or self.updated_at.blank? or (Time.parse(self.updated_at) + self.config[:update_delay]) < Time.now.utc
|
16
|
+
end
|
17
|
+
|
18
|
+
def client
|
19
|
+
Tweetable.client
|
20
|
+
end
|
21
|
+
|
22
|
+
def config
|
23
|
+
Tweetable.config
|
24
|
+
end
|
25
|
+
|
26
|
+
protected
|
27
|
+
|
28
|
+
def validate
|
29
|
+
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,121 @@
|
|
1
|
+
module Tweetable
|
2
|
+
class Queue
|
3
|
+
def self.clear_search_queue(queue_name)
|
4
|
+
collection = Tweetable::SearchCollection.find(:name, queue_name).first
|
5
|
+
return true if collection.nil?
|
6
|
+
collection.searches.size.times{|i| collection.searches.pop}
|
7
|
+
true
|
8
|
+
end
|
9
|
+
|
10
|
+
def self.clear_user_queue(queue_name)
|
11
|
+
collection = Tweetable::UserCollection.find(:name, queue_name).first
|
12
|
+
return true if collection.nil?
|
13
|
+
collection.users.size.times{|i| collection.users.pop}
|
14
|
+
true
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.add_to_search_queue(queue_name, queries, &block)
|
18
|
+
queue = Tweetable::SearchCollection.find_or_create(:name, queue_name)
|
19
|
+
return unless queue.searches.empty?
|
20
|
+
|
21
|
+
queries.each do |query|
|
22
|
+
search = Tweetable::Search.find_or_create(:query, query)
|
23
|
+
|
24
|
+
yield search if block_given?
|
25
|
+
|
26
|
+
queue.searches << search.id
|
27
|
+
end
|
28
|
+
|
29
|
+
queue
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.add_to_user_queue(queue_name, screen_names, &block)
|
33
|
+
queue = Tweetable::UserCollection.find_or_create(:name, queue_name)
|
34
|
+
return unless queue.users.empty?
|
35
|
+
|
36
|
+
screen_names.each do |screen_name|
|
37
|
+
user = Tweetable::User.find_or_create(:screen_name, screen_name)
|
38
|
+
|
39
|
+
yield user if block_given?
|
40
|
+
|
41
|
+
queue.users << user.id
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def self.pull_from_search_queue(queue_name)
|
46
|
+
queue = Tweetable::SearchCollection.find(:name, queue_name).first
|
47
|
+
return 0 if (queue.nil? or queue.searches.empty?)
|
48
|
+
|
49
|
+
count = 0
|
50
|
+
while !queue.searches.empty?
|
51
|
+
search = Tweetable::Search[queue.searches.pop] # have to find object by id in List
|
52
|
+
process_search(search)
|
53
|
+
count += 1
|
54
|
+
end
|
55
|
+
|
56
|
+
return count
|
57
|
+
end
|
58
|
+
|
59
|
+
def self.pull_from_user_queue(queue_name)
|
60
|
+
queue = Tweetable::UserCollection.find(:name, queue_name).first
|
61
|
+
return if (queue.nil? or queue.users.empty?)
|
62
|
+
|
63
|
+
count = 0
|
64
|
+
while !queue.users.empty?
|
65
|
+
user = Tweetable::User[queue.users.pop] # have to find object by id in List
|
66
|
+
process_user(user)
|
67
|
+
count += 1
|
68
|
+
end
|
69
|
+
|
70
|
+
return count
|
71
|
+
end
|
72
|
+
|
73
|
+
def self.process_search(search)
|
74
|
+
pull_from_queue_safely do
|
75
|
+
return search.update_all(true)
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
def self.process_user(user)
|
80
|
+
pull_from_queue_safely do
|
81
|
+
messages = user.update_all(true)
|
82
|
+
user.tags.each do |tag|
|
83
|
+
message_collection = Tweetable::MessageCollection.find_or_create(:name, tag)
|
84
|
+
user_collection = Tweetable::UserCollection.find_or_create(:name, tag)
|
85
|
+
|
86
|
+
message_collection.update(:updated_at => Time.now.utc.to_s)
|
87
|
+
user_collection.update(:updated_at => Time.now.utc.to_s)
|
88
|
+
|
89
|
+
user_collection.user_set.add(user)
|
90
|
+
messages.each{|m| message_collection.messages << m.id}
|
91
|
+
end
|
92
|
+
return messages
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
def self.pull_from_queue_safely
|
97
|
+
begin
|
98
|
+
yield
|
99
|
+
rescue Tweetable::TweetableError => e
|
100
|
+
raise TemporaryPullFromQueueError.new("Twitter unavailable error: #{e}")
|
101
|
+
rescue Twitter::Unavailable => e
|
102
|
+
raise TemporaryPullFromQueueError.new("Twitter unavailable error: #{e}")
|
103
|
+
rescue URI::InvalidURIError => e
|
104
|
+
raise PullFromQueueError.new("Bad uri error: #{e}")
|
105
|
+
rescue ArgumentError => e
|
106
|
+
raise PullFromQueueError.new("Argument error: #{e}")
|
107
|
+
rescue Crack::ParseError => e
|
108
|
+
raise PullFromQueueError.new("Parsing error: #{e}")
|
109
|
+
rescue Twitter::NotFound => e
|
110
|
+
raise PullFromQueueError.new("Account does not exist: #{e}")
|
111
|
+
rescue Twitter::Unauthorized => e
|
112
|
+
raise PullFromQueueError.new("Not authorized error: #{e}")
|
113
|
+
rescue Errno::ETIMEDOUT => e
|
114
|
+
raise PullFromQueueError.new("Connection timed out error: #{e}")
|
115
|
+
rescue Exception => e
|
116
|
+
HoptoadNotifier.notify(e)
|
117
|
+
raise PullFromQueueError.new("General error: #{e}")
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
@@ -0,0 +1,77 @@
|
|
1
|
+
module Tweetable
|
2
|
+
class Search < Persistable
|
3
|
+
SEARCH_PER_PAGE_LIMIT = 100
|
4
|
+
SEARCH_START_PAGE = 1
|
5
|
+
|
6
|
+
attribute :created_at
|
7
|
+
attribute :updated_at
|
8
|
+
attribute :query
|
9
|
+
index :query
|
10
|
+
list :messages, Message
|
11
|
+
|
12
|
+
def update_all(force = false)
|
13
|
+
return unless needs_update?(force)
|
14
|
+
self.updated_at = Time.now.utc.to_s
|
15
|
+
self.save
|
16
|
+
update_messages
|
17
|
+
end
|
18
|
+
|
19
|
+
# Perform the search and update messages if any new exist
|
20
|
+
# Do up to 15 requests to collect as many historical messages as possible
|
21
|
+
# Because search API is different than the resto f the Twitter API this needs to be custom (i.e. cannot use xxx_from_timeline methods)
|
22
|
+
def update_messages(pages = 15)
|
23
|
+
most_recent_message = self.messages.first(:order => 'DESC', :by => :message_id)
|
24
|
+
since_id = most_recent_message.nil? ? 0 : most_recent_message.message_id
|
25
|
+
|
26
|
+
search_messages = []
|
27
|
+
pages.times do |page|
|
28
|
+
s = search(self.query, since_id, SEARCH_PER_PAGE_LIMIT, page+1)
|
29
|
+
break if s.results.nil?
|
30
|
+
search_messages += s.results
|
31
|
+
break if s.results.size < 99
|
32
|
+
end
|
33
|
+
|
34
|
+
search_messages.each do |message|
|
35
|
+
m = Message.find_or_create(:message_id, message.id)
|
36
|
+
|
37
|
+
m.update(
|
38
|
+
:message_id => message.id,
|
39
|
+
:favorited => message.favorited,
|
40
|
+
:photos_parsed => '0',
|
41
|
+
:links_parsed => '0',
|
42
|
+
:created_at => Time.now.utc.to_s,
|
43
|
+
:sent_at => message.created_at,
|
44
|
+
:text => message.text,
|
45
|
+
:from_screen_name => message.from_user) # we explicitly don't include the user_id provided by search since it's bullshit: http://code.google.com/p/twitter-api/issues/detail?id=214
|
46
|
+
|
47
|
+
next if !m.valid?
|
48
|
+
|
49
|
+
# create the user for this message
|
50
|
+
u = User.find_or_create(:screen_name, message.from_user)
|
51
|
+
u.update(
|
52
|
+
:user_id => message.from_user_id,
|
53
|
+
:profile_image_url => message.profile_image_url)
|
54
|
+
|
55
|
+
self.messages << m unless self.messages.include?(m)
|
56
|
+
end
|
57
|
+
|
58
|
+
search_messages.flatten
|
59
|
+
end
|
60
|
+
|
61
|
+
|
62
|
+
private
|
63
|
+
|
64
|
+
def validate
|
65
|
+
assert_present :query
|
66
|
+
assert_unique :query
|
67
|
+
end
|
68
|
+
|
69
|
+
def search(query, since_id, per_page = SEARCH_PER_PAGE_LIMIT, page = SEARCH_START_PAGE)
|
70
|
+
begin
|
71
|
+
Twitter::Search.new(query.strip).since(since_id).per_page(per_page).page(page).fetch
|
72
|
+
rescue NoMethodError => e
|
73
|
+
raise TweetableError.new("Temporary problem searching Twitter: #{e}")
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
module Tweetable
|
2
|
+
class TwitterClient
|
3
|
+
attr_accessor :consumer_token, :consumer_secret, :authorization #:oauth_access_token, :oauth_access_secret
|
4
|
+
|
5
|
+
def method_missing(sym, *args, &block)
|
6
|
+
raise TweetableAuthError.new('Not authorized. You must login or authorize the client.') if @client.nil?
|
7
|
+
@client.send sym, *args, &block
|
8
|
+
end
|
9
|
+
|
10
|
+
def authorize(token, secret, authorization)
|
11
|
+
raise TweetableAuthError if authorization.nil?
|
12
|
+
|
13
|
+
self.authorization = authorization
|
14
|
+
self.consumer_token = token
|
15
|
+
self.consumer_secret = secret
|
16
|
+
self.oauth.authorize_from_access(self.authorization.oauth_access_token, self.authorization.oauth_access_secret)
|
17
|
+
|
18
|
+
@client = Twitter::Base.new(self.oauth)
|
19
|
+
self
|
20
|
+
end
|
21
|
+
|
22
|
+
def login(username, password)
|
23
|
+
@client = Twitter::Base.new(Twitter::HTTPAuth.new(username, password))
|
24
|
+
self
|
25
|
+
end
|
26
|
+
|
27
|
+
def oauth
|
28
|
+
@oauth ||= Twitter::OAuth.new(self.consumer_token, self.consumer_secret)
|
29
|
+
end
|
30
|
+
|
31
|
+
def status
|
32
|
+
status = self.rate_limit_status
|
33
|
+
{:hourly_limit => status[:hourly_limit], :remaining_hits => status[:remaining_hits], :reset_time => status[:reset_time], :reset_time_in_seconds => status[:reset_time_in_seconds]}
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'tweetstream'
|
3
|
+
require 'daemons'
|
4
|
+
|
5
|
+
module Tweetable
|
6
|
+
class TwitterStreamingClient
|
7
|
+
|
8
|
+
def method_missing(sym, *args, &block)
|
9
|
+
@client.send sym, *args, &block
|
10
|
+
end
|
11
|
+
|
12
|
+
def initialize(username, password, parser = nil)
|
13
|
+
@client = TweetStream::Client.new(username, password, parser)
|
14
|
+
setup
|
15
|
+
self
|
16
|
+
end
|
17
|
+
|
18
|
+
def run(query_params)
|
19
|
+
query_params = query_params.call if query_params.kind_of?(Proc)
|
20
|
+
keywords = query_params[:track]
|
21
|
+
keywords = [keywords] unless keywords.kind_of?(Array)
|
22
|
+
|
23
|
+
Tweetable.log.debug("Tracking keywords: #{query_params.inspect}")
|
24
|
+
|
25
|
+
self.filter(query_params) do |status|
|
26
|
+
keywords.each do |keyword|
|
27
|
+
store(status, keyword)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def start(query_params = {}, daemon_options = {}) #:nodoc:
|
33
|
+
Daemons.run_proc('tweetable', daemon_options) do
|
34
|
+
Tweetable.log.debug("Starting...")
|
35
|
+
run(query_params)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def store(status, keyword)
|
40
|
+
if status.text =~ /#{keyword}/i
|
41
|
+
message = Message.create_from_timeline(status, true)
|
42
|
+
Tweetable.log.debug("[#{keyword}] #{message.sent_at} #{message.text} (#{message.message_id})")
|
43
|
+
|
44
|
+
search = Search.find_or_create(:query, keyword.downcase)
|
45
|
+
search.update(:updated_at => Time.now.utc.to_s)
|
46
|
+
search.messages << message
|
47
|
+
end
|
48
|
+
|
49
|
+
message
|
50
|
+
end
|
51
|
+
|
52
|
+
private
|
53
|
+
def setup
|
54
|
+
self.on_delete do |status_id, user_id|
|
55
|
+
# do nothing
|
56
|
+
end
|
57
|
+
|
58
|
+
self.on_limit do |skip_count|
|
59
|
+
raise TweetableError.new("Twitter streaming rate limit reached (#{skip_count})")
|
60
|
+
end
|
61
|
+
|
62
|
+
self.on_error do |message|
|
63
|
+
puts "Error: #{message}"
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|