devfu-twitter-search-watcher 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,21 @@
1
+ = TwitterSearchWatcher
2
+
3
+ Sometimes, we want to continually search Twitter for something
4
+ like a hashtag to display the latest results on our sites.
5
+
6
+ TwitterSearchWatcher is a simple gem for continually searching
7
+ Twitter for something and calling a callback whenever new
8
+ tweets are present.
9
+
10
+ == Installation
11
+
12
+ sudo gem install json
13
+ sudo gem install devfu-twitter-search-watcher -s http://gems.github.com
14
+
15
+ == Usage
16
+
17
+ TwitterSearchWatcher.watch! '#laidoffcamp', :max_pages => 1 do |tweet|
18
+ puts "#{ tweet.from_user }: #{ tweet.text }"
19
+ end
20
+
21
+ For additional usage, see RDoc at http://code.devfu.com/twitter-search-watcher or the specs at http://github.com/devfu/twitter-search-watcher/blob/master/spec/twitter_search_watcher_spec.rb
@@ -0,0 +1,68 @@
1
+ require 'rake'
2
+ require 'rubygems'
3
+ require 'rake/rdoctask'
4
+ require 'spec/rake/spectask'
5
+
6
+ puts "\nGem: twitter-search-watcher\n\n"
7
+
8
+ begin
9
+ require 'jeweler'
10
+ Jeweler::Tasks.new do |s|
11
+ s.name = 'twitter-search-watcher'
12
+ s.summary = 'for watching a Twitter search'
13
+ s.email = 'remi@remitaylor.com'
14
+ s.homepage = 'http://github.com/devfu/twitter-search-watcher'
15
+ s.description = 'for watching a particular Twitter search and calling code whenever there are new tweets'
16
+ s.authors = %w( remi )
17
+ s.files = FileList['[A-Z]*', '{lib,spec,bin,examples}/**/*']
18
+ # s.add_dependency 'person-gemname'
19
+ # s.executables << 'script'
20
+ # s.rubyforge_project = 'gemname'
21
+ # s.extra_rdoc_files = %w( README.rdoc )
22
+ end
23
+ rescue LoadError
24
+ puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
25
+ end
26
+
27
+ Spec::Rake::SpecTask.new do |t|
28
+ t.spec_files = FileList['spec/**/*_spec.rb']
29
+ end
30
+
31
+ desc "Run all examples with RCov"
32
+ Spec::Rake::SpecTask.new('rcov') do |t|
33
+ t.spec_files = FileList['spec/**/*_spec.rb']
34
+ t.rcov = true
35
+ end
36
+
37
+ # require 'hanna'
38
+ # require 'darkfish-rdoc'
39
+
40
+ Rake::RDocTask.new do |rdoc|
41
+ rdoc.rdoc_dir = 'rdoc'
42
+ rdoc.title = 'twitter-search-watcher'
43
+ rdoc.options << '--line-numbers' << '--inline-source'
44
+ # rdoc.options += ["--template=#{`allison --path`}"] # sudo gem install allison
45
+ # rdoc.options += %w( -f darkfish ) # sudo gem install darkfish-rdoc
46
+ # rdoc.options += %w( -T hanna ) # sudo gem install mislav-hanna
47
+ rdoc.options += %w( -m README.rdoc ) # the initial page displayed
48
+ rdoc.rdoc_files.include('README.rdoc')
49
+ rdoc.rdoc_files.include('lib/**/*.rb')
50
+ end
51
+
52
+ desc 'Confirm that gemspec is $SAFE'
53
+ task :safe do
54
+ require 'yaml'
55
+ require 'rubygems/specification'
56
+ data = File.read('twitter-search-watcher.gemspec')
57
+ spec = nil
58
+ if data !~ %r{!ruby/object:Gem::Specification}
59
+ Thread.new { spec = eval("$SAFE = 3\n#{data}") }.join
60
+ else
61
+ spec = YAML.load(data)
62
+ end
63
+ spec.validate
64
+ puts spec
65
+ puts "OK"
66
+ end
67
+
68
+ task :default => :spec
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 0.1.0
@@ -0,0 +1,191 @@
1
+ %w( rubygems cgi json open-uri ostruct ).each {|lib| require lib }
2
+
3
+ class TwitterSearchWatcher
4
+
5
+ TWITTER_SEARCH_URL = 'http://search.twitter.com/search.json'
6
+ DEFAULT_USER_AGENT = 'TwitterSearchWatcher RubyGem http://github.com/devfu/twitter-search-watcher'
7
+ QUERY_STRING_ATTRIBUTES = [ :q, :to, :from, :since_id, :page, :max_id, :rpp ]
8
+
9
+ # The User-Agent header value to send along with all Twitter Search API requests
10
+ attr_accessor :user_agent
11
+
12
+ # A string you want to search twitter for
13
+ attr_accessor :q
14
+
15
+ # The username of someone you want to search replies to
16
+ attr_accessor :to
17
+
18
+ # The username of someone you want to search replies from
19
+ attr_accessor :from
20
+
21
+ # Get a particular page of Twitter search results (pagination).
22
+ # Typically used in conjunction with :max_id
23
+ attr_accessor :page
24
+
25
+ # Used for pagination, so you can get page=3 where the max_id of the first page was 1234
26
+ attr_accessor :max_id
27
+
28
+ # Only get tweets with ID's greater than this ID (useful for only getting new tweets)
29
+ attr_accessor :since_id
30
+
31
+ # Number of results per page (max 100)
32
+ attr_accessor :rpp
33
+
34
+ # The number of seconds to wait between Twitter calls. Default: 60 (seconds)
35
+ attr_accessor :check_every
36
+
37
+ # The maximum number of pages to check for tweets
38
+ #
39
+ # If nil, we'll check until there are no more pages (when :next_page isn't present)
40
+ attr_accessor :max_pages
41
+
42
+ def rpp= value
43
+ raise "The maximum rpp (Results per Page) value is 100" if value > 100
44
+ @rpp = value
45
+ end
46
+
47
+ def check_every
48
+ @check_every || 60
49
+ end
50
+
51
+ # Create a new TwitterSearchWatcher
52
+ #
53
+ # TwitterSearchWatcher.new 'string to search'
54
+ # TwitterSearchWatcher.new 'string to search', :check_every => 60
55
+ # TwitterSearchWatcher.new :to => 'barackobama', :from => 'SenJohnMcCain'
56
+ #
57
+ def initialize search_string = nil, options = nil
58
+ if search_string.is_a? Hash
59
+ options = search_string
60
+ else
61
+ self.q = search_string
62
+ end
63
+
64
+ options.each {|k,v| send "#{k}=", v } if options
65
+ end
66
+
67
+ # Returns the URL we'll use to call the Twitter Search API.
68
+ #
69
+ # Without parameters, it'll generate a URL just from this TwitterSearchWatcher instance.
70
+ #
71
+ # With parameters, it'll override the TwitterSearchWatcher instance's options with
72
+ # whatever you pass, eg.
73
+ #
74
+ # >> TwitterSearchWatcher.new( 'foo', :rpp => 15 ).search_url
75
+ # => "http://search.twitter.com/search.json?q=foo&rpp=15"
76
+ #
77
+ # >> TwitterSearchWatcher.new( 'foo', :rpp => 15 ).search_url( :rpp => 99 )
78
+ # => "http://search.twitter.com/search.json?q=foo&rpp=99"
79
+ #
80
+ def search_url additional_parameters = nil
81
+ TWITTER_SEARCH_URL + build_query_string(additional_parameters)
82
+ end
83
+
84
+ def user_agent
85
+ @user_agent || DEFAULT_USER_AGENT
86
+ end
87
+
88
+ # Performs a search. Accepts the same parameters as #search_url
89
+ def search! additional_parameters = nil
90
+ JSON.parse open( search_url(additional_parameters), 'User-Agent' => user_agent ).read
91
+ end
92
+
93
+ # Performs a search, given the response from another search.
94
+ #
95
+ # If a response if given, the search will only return tweets newer than the given response's tweets.
96
+ # If a response is not given, this performs a normal search.
97
+ #
98
+ # Accepts additional parameters (same as #search_url)
99
+ def search_newer! response = nil, additional_parameters = nil
100
+ if response
101
+ search!( (additional_parameters || {}).merge( :since_id => response['max_id'] ) )
102
+ else
103
+ search! additional_parameters
104
+ end
105
+ end
106
+
107
+ # Performs a search, given the response from another search.
108
+ #
109
+ # If the response given is paginated (ie. there are additional tweets available on additional pages),
110
+ # this will return the next page. Else, this will return nil.
111
+ #
112
+ # Accepts additional parameters (same as #search_url)
113
+ def search_more! response, additional_parameters = nil
114
+ search!( (additional_parameters || {}).merge( :page => (response['page'] + 1), :max_id => response['max_id'] ) ) if response['next_page']
115
+ end
116
+
117
+ # Instantiates a new TwitterSearchWatcher given the search_string and options and then
118
+ # calls #watch on the instance using the block given.
119
+ def self.watch! search_string, options = nil, &block
120
+ watcher = TwitterSearchWatcher.new search_string, options
121
+ watcher.watch! &block
122
+ end
123
+
124
+ # Starts watching this search in a loop.
125
+ # It will wait #check_every seconds between new requests (except requests to get additional pages).
126
+ # Every time a new tweet is found, that tweet is passed to the block given.
127
+ #
128
+ # TwitterSearchWatcher.new('foo').watch! {|tweet| puts "got tweet: #{ tweet.text }" }
129
+ #
130
+ def watch! additional_parameters = nil, &block
131
+ @max_id_found_so_far = 0
132
+
133
+ trap('INT'){ puts "\nexiting ..."; exit }
134
+ puts "Watching for tweets: #{ search_url(additional_parameters) }"
135
+
136
+ loop do
137
+
138
+ @last_response = search_newer!(@last_response, additional_parameters)
139
+ call_tweet_callbacks(@last_response, block)
140
+ update_max_id @last_response
141
+
142
+ # this is kindof icky ... but it works
143
+ if @last_response['next_page']
144
+ response = @last_response
145
+ num_pages_searched = 0
146
+ while (response = search_more!(response, additional_parameters)) && (num_pages_searched <= max_pages if max_pages)
147
+ num_pages_searched += 1
148
+ call_tweet_callbacks(response, block)
149
+ update_max_id response
150
+ end
151
+ end
152
+
153
+ sleep check_every
154
+ end
155
+ end
156
+
157
+ private
158
+
159
+ def update_max_id response
160
+ @max_id_found_so_far = response['max_id'] if response['max_id'] > @max_id_found_so_far
161
+ end
162
+
163
+ def call_tweet_callbacks response, block
164
+ response['results'].each do |tweet|
165
+ block.call OpenStruct.new(tweet)
166
+ end
167
+ end
168
+
169
+ def escape string
170
+ CGI.escape(string.to_s).gsub('%22','"').gsub(' ','+')
171
+ end
172
+
173
+ def build_query_string additional_parameters = nil
174
+ parameter_values = QUERY_STRING_ATTRIBUTES.inject({}){|all, attr|
175
+ all[attr] = send(attr) if send(attr)
176
+ all
177
+ }
178
+
179
+ # if additional parameters are passed, we override the watcher's parameters with these
180
+ if additional_parameters
181
+ additional_parameter_values = QUERY_STRING_ATTRIBUTES.inject({}){|all, attr|
182
+ all[attr] = additional_parameters[attr] if additional_parameters.keys.include?(attr)
183
+ all
184
+ }
185
+ parameter_values.merge! additional_parameter_values
186
+ end
187
+
188
+ '?' + parameter_values.map {|k,v| "#{ k }=#{ escape(v) }" if v }.compact.join('&')
189
+ end
190
+
191
+ end
@@ -0,0 +1,4 @@
1
+ --colour
2
+ --format specdoc
3
+ --loadby mtime
4
+ --reverse
@@ -0,0 +1,53 @@
1
+ require File.dirname(__FILE__) + '/../lib/twitter-search-watcher'
2
+ require 'rubygems'
3
+ require 'spec'
4
+ require 'ostruct'
5
+
6
+ def readable value
7
+ OpenStruct.new({ :read => value })
8
+ end
9
+
10
+ def fake_response_for watcher
11
+
12
+ end
13
+
14
+ def fake_response options = {}
15
+ readable(
16
+ {
17
+ 'max_id' => 1234,
18
+ 'since_id' => 0,
19
+ 'total' => 15, # doesn't appear if next_page present?
20
+ 'next_page' => '?page=2&max_id=1234&q=remitaylor', # only appears if there are additional pages
21
+ 'refresh_url' => '?since_id=1234&q=remitaylor',
22
+ 'page' => 1,
23
+ 'results_per_page' => 15,
24
+ 'completed_in' => 0.1234,
25
+ 'query' => 'remitaylor',
26
+ 'results' => fake_tweets
27
+ }.merge(options).to_json
28
+ )
29
+ end
30
+
31
+ def fake_tweets
32
+ [ fake_tweet, fake_tweet ]
33
+ end
34
+
35
+ def fake_tweet
36
+ {
37
+ "created_at"=>"Wed, 05 Aug 2009 03:54:03 +0000",
38
+ "profile_image_url"=>"http://s3.amazonaws.com/twitter_production/profile_images/346217637/tiltshiftdino_normal.jpg",
39
+ "from_user"=>"BaddMann",
40
+ "to_user_id"=>549282,
41
+ "text"=>"@remitaylor What did you think of the badges at defcon Did you try anything with your badge",
42
+ "id" => (rand * 1000000000).to_i,
43
+ "from_user_id"=>160309,
44
+ "to_user"=>"remitaylor",
45
+ "iso_language_code"=>"en",
46
+ "source"=>"&lt;a href=&quot;http://twitter.com/&quot;&gt;web&lt;/a&gt;"}
47
+ end
48
+
49
+ Spec::Matchers.define :end_with do |expected|
50
+ match do |actual|
51
+ actual.end_with? expected
52
+ end
53
+ end
@@ -0,0 +1,140 @@
1
+ require File.dirname(__FILE__) + '/spec_helper'
2
+
3
+ describe TwitterSearchWatcher do
4
+
5
+ it 'should be able to initialize with a search string' do
6
+ TwitterSearchWatcher.new('foo').search_url.should == 'http://search.twitter.com/search.json?q=foo'
7
+ TwitterSearchWatcher.new('#foo').search_url.should == 'http://search.twitter.com/search.json?q=%23foo'
8
+ TwitterSearchWatcher.new('@foo').search_url.should == 'http://search.twitter.com/search.json?q=%40foo'
9
+ end
10
+
11
+ it 'should be able to search for replies :to someone' do
12
+ TwitterSearchWatcher.new( :to => 'remitaylor' ).search_url.should end_with('search.json?to=remitaylor')
13
+ end
14
+
15
+ it 'should be able to search for tweets :from someone' do
16
+ TwitterSearchWatcher.new( :from => 'remitaylor' ).search_url.should end_with('search.json?from=remitaylor')
17
+ end
18
+
19
+ it 'should be able to search for tweets :from someone that are :to someone' do
20
+ watcher = TwitterSearchWatcher.new :from => 'remitaylor', :to => 'wickd_wanda'
21
+ watcher.search_url.should include('from=remitaylor')
22
+ watcher.search_url.should include('to=wickd_wanda')
23
+ end
24
+
25
+ it 'should have a good default User-Agent' do
26
+ watcher = TwitterSearchWatcher.new
27
+ watcher.user_agent.should == 'TwitterSearchWatcher RubyGem http://github.com/devfu/twitter-search-watcher'
28
+ watcher.user_agent = 'foo'
29
+ watcher.user_agent.should == 'foo'
30
+ end
31
+
32
+ it 'should be able to override User-Agent' do
33
+ watcher = TwitterSearchWatcher.new 'x', :user_agent => 'bar'
34
+ watcher.search_url.should == 'http://search.twitter.com/search.json?q=x'
35
+ watcher.user_agent.should == 'bar'
36
+
37
+ watcher = TwitterSearchWatcher.new :user_agent => 'bar'
38
+ watcher.search_url.should == 'http://search.twitter.com/search.json?'
39
+ watcher.user_agent.should == 'bar'
40
+ end
41
+
42
+ it 'should allow quotes in query string' do
43
+ TwitterSearchWatcher.new('"hello there"').search_url.should end_with('q="hello+there"')
44
+ end
45
+
46
+ it 'should replace all spaces in query string with + signs' do
47
+ TwitterSearchWatcher.new('hello there').search_url.should end_with('q=hello+there')
48
+ end
49
+
50
+ it 'should be able to execute search' do
51
+ watcher = TwitterSearchWatcher.new 'chunky bacon'
52
+ watcher.should_receive(:open).with(watcher.search_url, 'User-Agent' => watcher.user_agent).and_return(fake_response)
53
+ response = watcher.search!
54
+ response.should be_a_kind_of(Hash)
55
+ response['query'].should == 'remitaylor' # make sure it's getting our fake date
56
+ end
57
+
58
+ it 'should be able to request a particular :page (pagination)' do
59
+ TwitterSearchWatcher.new( 'foo', :page => 2 ).search_url.should include("page=2")
60
+ TwitterSearchWatcher.new( 'foo', :page => 2 ).search_url.should include("q=foo")
61
+ end
62
+
63
+ it 'should be able to request since a particular twitter ID (:since_id)' do
64
+ TwitterSearchWatcher.new( 'foo', :since_id => 1234 ).search_url.should include("since_id=1234")
65
+ TwitterSearchWatcher.new( 'foo', :since_id => 1234 ).search_url.should include("q=foo")
66
+ end
67
+
68
+ it 'should be able to request with a particular :max_id (used with :page)' do
69
+ TwitterSearchWatcher.new( 'foo', :max_id => 4321 ).search_url.should include("max_id=4321")
70
+ TwitterSearchWatcher.new( 'foo', :max_id => 4321 ).search_url.should include("q=foo")
71
+ end
72
+
73
+ it 'should be able to request with a particular :rpp (results_per_page)' do
74
+ TwitterSearchWatcher.new( 'foo', :rpp => 90 ).search_url.should include("rpp=90")
75
+ TwitterSearchWatcher.new( 'foo', :rpp => 90 ).search_url.should include("q=foo")
76
+
77
+ lambda { TwitterSearchWatcher.new( 'foo', :rpp => 101 ) }.should raise_error(/100/)
78
+ end
79
+
80
+ it 'should be able to search!(:rpp => 15) ... inotherwords, should be able to pass additional parameters to search!' do
81
+ watcher = TwitterSearchWatcher.new 'foo'
82
+
83
+ watcher.should_receive(:open).with(/99/, 'User-Agent' => watcher.user_agent).and_return(fake_response)
84
+ watcher.search! :rpp => 99
85
+
86
+ watcher.should_receive(:open).with(/foo/, 'User-Agent' => watcher.user_agent).and_return(fake_response)
87
+ watcher.search! :rpp => 99
88
+ end
89
+
90
+ it 'should be able to search!(:rpp => nil) to override an existing parameter' do
91
+ watcher = TwitterSearchWatcher.new 'foo', :rpp => 50
92
+ watcher.search_url.should include('rpp=50')
93
+ watcher.search_url.should include('q=foo')
94
+
95
+ # double check that normal overrides are working properly
96
+ watcher.search_url( :rpp => 99 ).should_not include('rpp=50')
97
+ watcher.search_url( :rpp => 99 ).should include('rpp=99')
98
+
99
+ watcher.search_url( :rpp => nil ).should_not include('rpp=50')
100
+ watcher.search_url( :rpp => nil ).should_not include('rpp')
101
+ end
102
+
103
+ it 'should be able to search_newer!(result_set) to get results newer than a given result set' do
104
+ watcher = TwitterSearchWatcher.new 'chunky bacon'
105
+ watcher.should_receive(:open).with(watcher.search_url, 'User-Agent' => watcher.user_agent).and_return(fake_response(:max_id => 5555))
106
+ response = watcher.search!
107
+
108
+ watcher.should_receive(:open).with(/since_id=5555/, 'User-Agent' => watcher.user_agent).and_return(fake_response)
109
+ watcher.search_newer! response
110
+ end
111
+
112
+ it 'should be able to search_more!(result_set) to get paginated results' do
113
+ watcher = TwitterSearchWatcher.new 'chunky bacon'
114
+ watcher.should_receive(:open).with(watcher.search_url, 'User-Agent' => watcher.user_agent).and_return(fake_response(:page => 1, :max_id => 444, :next_page => '?page=2&max_id=444&q=chunky+bacon'))
115
+ response = watcher.search!
116
+
117
+ watcher.should_receive(:open).with(/page=2/, 'User-Agent' => watcher.user_agent).and_return(fake_response(:page => 2, :max_id => 445, :next_page => nil))
118
+ response2 = watcher.search_more! response
119
+ watcher.should_receive(:open).with(/max_id=444/, 'User-Agent' => watcher.user_agent).and_return(fake_response(:page => 2, :max_id => 445, :next_page => nil))
120
+ response2 = watcher.search_more! response
121
+
122
+ watcher.should_not_receive(:open)
123
+ watcher.search_more!( response2 ).should be_nil # because no next_page
124
+ end
125
+
126
+ it 'should be able to watch! on a watcher' do
127
+ # it's a PITA to test the loop, so i'm not currently testing that check_every and max_pages are actually used correctly
128
+ watcher = TwitterSearchWatcher.new 'chunky bacon', :check_every => 120, :max_pages => 5
129
+ watcher.check_every.should == 120
130
+ watcher.max_pages.should == 5
131
+ watcher.should respond_to(:watch!)
132
+ end
133
+
134
+ it 'default rpp should be 100 (the max)'
135
+
136
+ it 'should have a nice method to call that will make a new watcher and start watching'
137
+
138
+ it 'should be able to watch! with pagination (will search! while there is a :next_page to request)'
139
+
140
+ end
metadata ADDED
@@ -0,0 +1,61 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: devfu-twitter-search-watcher
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - remi
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+
12
+ date: 2009-08-06 00:00:00 -07:00
13
+ default_executable:
14
+ dependencies: []
15
+
16
+ description: for watching a particular Twitter search and calling code whenever there are new tweets
17
+ email: remi@remitaylor.com
18
+ executables: []
19
+
20
+ extensions: []
21
+
22
+ extra_rdoc_files:
23
+ - README.rdoc
24
+ files:
25
+ - README.rdoc
26
+ - Rakefile
27
+ - VERSION
28
+ - lib/twitter-search-watcher.rb
29
+ - spec/spec.opts
30
+ - spec/spec_helper.rb
31
+ - spec/twitter_search_watcher_spec.rb
32
+ has_rdoc: false
33
+ homepage: http://github.com/devfu/twitter-search-watcher
34
+ licenses:
35
+ post_install_message:
36
+ rdoc_options:
37
+ - --charset=UTF-8
38
+ require_paths:
39
+ - lib
40
+ required_ruby_version: !ruby/object:Gem::Requirement
41
+ requirements:
42
+ - - ">="
43
+ - !ruby/object:Gem::Version
44
+ version: "0"
45
+ version:
46
+ required_rubygems_version: !ruby/object:Gem::Requirement
47
+ requirements:
48
+ - - ">="
49
+ - !ruby/object:Gem::Version
50
+ version: "0"
51
+ version:
52
+ requirements: []
53
+
54
+ rubyforge_project:
55
+ rubygems_version: 1.3.5
56
+ signing_key:
57
+ specification_version: 3
58
+ summary: for watching a Twitter search
59
+ test_files:
60
+ - spec/twitter_search_watcher_spec.rb
61
+ - spec/spec_helper.rb