jamnagar 1.3.8

data/tracer.rb

@@ -0,0 +1,61 @@
+require 'aws/kinesis'
+require 'multi_json'
+require 'twitter'
+
+class SampleProducer
+  def initialize(data, service, stream_name, sleep_between_puts, shard_count=nil)
+    @data        = data
+    @stream_name = stream_name
+    @shard_count = shard_count
+    @sleep_between_puts = sleep_between_puts
+    @kinesis = service
+  end
+
+  def run
+     @data.each do |record|
+      put_record(record)
+    end
+  end
+
+  def put_record(record)
+    data = record
+    data_blob = MultiJson.dump(data)
+    r = @kinesis.put_record(:stream_name => @stream_name,
+                            :data => data_blob,
+                            :partition_key => data[:id_str])
+    puts "Put record to shard '#{r[:shard_id]}' (#{r[:sequence_number]})"
+  end
+end
+
+
+AWS.config(
+  :access_key_id => 'AKIAIB2JQHAQAT4T4DVQ',
+  :secret_access_key => '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'
+)
+
+# Angler's Weekly
+twitter_credentials = {:oauth_token=>"2602827816-rbQ93UFhoJHJZCoydSATlLZGGA8VdytcX5zNIM0", :oauth_token_secret=>"DNsZjcBFWZKOa1jEZRv08KwQymuhB69ObQHWNzn65FTXH", :consumer_secret=>"txblKWrEMSUTWjFeSNVdDC9XMkfwdEe49hjHSzLgEy46qqS9oX", :consumer_key=>"b3z9VvRtkUFbIvsXiMDzOUXjY"}
+
+client = Twitter::REST::Client.new do |config|
+  config.consumer_key        = twitter_credentials[:consumer_key]
+  config.consumer_secret     = twitter_credentials[:consumer_secret]
+  config.access_token        = twitter_credentials[:oauth_token]
+  config.access_token_secret = twitter_credentials[:oauth_token_secret]
+end
+
+puts "Getting Lists"
+lists = client.lists('agileweekly')
+
+puts "Getting Timelines for Lists"
+tweets = lists.flat_map{|list| client.list_timeline(list, {:count => 199})}.uniq.compact
+
+puts "Finding Tweets with URLs"
+with_urls = tweets.select{|t| t.urls? }.compact
+
+puts "Converting Tweets to Hashes"
+data = with_urls.map{|tweet| tweet.to_h }
+
+kinesis = AWS::Kinesis::Client.new({})
+producer = SampleProducer.new(data, kinesis, "jamnagar_tweets", 0, 2)
+puts "Sending to Kinesis"
+producer.run