http_streaming_client 0.8.1

data/spec/twitter_spec.rb

@@ -0,0 +1,66 @@
+require 'spec_helper'
+require 'json'
+require 'zlib'
+require 'uri'
+
+require 'http_streaming_client/credentials/twitter'
+include HttpStreamingClient::Credentials::Twitter
+
+describe HttpStreamingClient do
+
+  describe "twitter firehose streaming get test, no compression" do
+    url = "https://stream.twitter.com/1.1/statuses/sample.json"
+    authorization = HttpStreamingClient::Oauth::Twitter.generate_authorization(url, "get", {}, OAUTH_CONSUMER_KEY, OAUTH_CONSUMER_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
+
+    subject { authorization }
+    it { should_not be_nil}
+    it { should be_instance_of(String) }
+
+    count = 0
+
+    it "should successfully retrieve JSON records from the firehose" do
+      expect {
+	client = HttpStreamingClient::Client.new(compression: false)
+	response = client.get(url, {:headers => {'Authorization' => "#{authorization}" }}) { |line|
+	count = count + 1
+	if count > NUM_JSON_RECORDS_TO_RECEIVE then
+	  client.interrupt
+	  next
+	end
+	logger.debug "json: #{JSON.parse(line).to_s}" }
+      }.to_not raise_error
+    end
+
+  end
+
+  describe "twitter firehose streaming get test, GZIP compression" do
+    url = "https://stream.twitter.com/1.1/statuses/sample.json"
+    authorization = HttpStreamingClient::Oauth::Twitter.generate_authorization(url, "get", {}, OAUTH_CONSUMER_KEY, OAUTH_CONSUMER_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
+
+    subject { authorization }
+    it { should_not be_nil}
+    it { should be_instance_of(String) }
+
+    count = 0
+
+    it "should successfully retrieve decompressed JSON records from the firehose" do
+      expect {
+	client = HttpStreamingClient::Client.new(compression: true)
+	response = client.get(url, {:headers => {'Authorization' => "#{authorization}" }}) { |line|
+	count = count + 1
+	if count > NUM_JSON_RECORDS_TO_RECEIVE then
+	  client.interrupt
+	  next
+	end
+	logger.debug "json: #{JSON.parse(line).to_s}" }
+      }.to_not raise_error
+    end
+  end
+
+  describe "twitter firehose streaming get unauthorized failure" do
+    it "should fail if authorization not provided" do
+      expect { HttpStreamingClient::Client.get "https://stream.twitter.com/1.1/statuses/sample.json" }.to raise_error(HttpStreamingClient::HttpError)
+    end
+  end
+
+end

data/tools/adobe_firehose.rb

@@ -0,0 +1,40 @@
+require 'bundler/setup'
+require 'logger'
+require 'json'
+
+require 'http_streaming_client'
+
+HttpStreamingClient.logger.console = false
+HttpStreamingClient.logger.logfile = false
+
+require 'http_streaming_client/credentials/adobe'
+include HttpStreamingClient::Credentials::Adobe
+
+url = TOKENAPIHOST
+authorization = HttpStreamingClient::Oauth::Adobe.generate_authorization(url, USERNAME, PASSWORD, CLIENTID, CLIENTSECRET)
+puts "#{TOKENAPIHOST}:access token: #{authorization}"
+client = HttpStreamingClient::Client.new(compression: false)
+response = client.get(STREAMURL, {:headers => {'Authorization' => "Bearer #{authorization}" }}) { |line|
+
+    if line.nil? then
+      puts "error:nil line received"
+      next
+    end
+
+    if line.size == 0 then
+      puts "error:zero length line received"
+      next
+    end
+
+    if line.eql? "\r\n" then
+      puts "Server ping received"
+      next
+    end
+
+    begin
+      json = JSON.parse(line)
+      puts json.to_s
+    rescue Exception => e
+      puts "exception:#{e.message}:line-->#{line}<--"
+    end
+  }

data/tools/adobe_firehose_performance_test.rb

@@ -0,0 +1,83 @@
+require 'bundler/setup'
+require 'logger'
+require 'json'
+
+require 'http_streaming_client'
+
+HttpStreamingClient.logger.console = false
+HttpStreamingClient.logger.logfile = false
+
+require 'http_streaming_client/credentials/adobe'
+include HttpStreamingClient::Credentials::Adobe
+
+url = TOKENAPIHOST
+authorization = HttpStreamingClient::Oauth::Adobe.generate_authorization(url, USERNAME, PASSWORD, CLIENTID, CLIENTSECRET)
+puts "#{TOKENAPIHOST}:access token: #{authorization}"
+client = HttpStreamingClient::Client.new(compression: false)
+
+NUM_RECORDS_PER_BATCH = 1000
+MAX_RECORDS = 50000
+
+count = 0
+totalSize = 0
+intervalSize = 0
+startTime = nil
+lastTime = nil
+
+puts "starting performance test run: #{Time.new.to_s}"
+
+startTime = lastTime = Time.new 
+
+response = client.get(STREAMURL, {:headers => {'Authorization' => "Bearer #{authorization}" }}) { |line|
+
+  if line.nil? then
+    puts "error:nil line received"
+    next
+  end
+
+  if line.size == 0 then
+    puts "error:zero length line received"
+    next
+  end
+
+  if line.eql? "\r\n" then
+    puts "Server ping received"
+    next
+  end
+
+  count = count + 1
+  intervalSize = intervalSize + line.size
+
+  if count % NUM_RECORDS_PER_BATCH == 0 then
+
+    now = Time.new
+    intervalElapsedTime = now - lastTime
+    totalElapsedTime = now - startTime
+
+    totalSize = totalSize + intervalSize
+
+    stats = Hash.new
+
+    stats['total_records_received'] = count.to_s
+    stats['total_elapsed_time_sec'] = totalElapsedTime.round(2).to_s
+    stats['total_records_per_sec'] = (count / totalElapsedTime).round(2).to_s
+    stats['total_kbytes_per_sec'] = (totalSize / totalElapsedTime / 1024).round(2).to_s
+
+    stats['interval_records_received'] = NUM_RECORDS_PER_BATCH
+    stats['interval_elapsed_time_sec'] = intervalElapsedTime.round(2).to_s
+    stats['interval_records_per_sec'] = (NUM_RECORDS_PER_BATCH / intervalElapsedTime).round(2).to_s
+    stats['interval_kbytes_per_sec'] = (intervalSize / intervalElapsedTime / 1024).round(2).to_s
+
+    puts stats.to_json
+
+    lastTime = now
+    intervalSize = 0
+  end
+
+  if count == MAX_RECORDS then
+    puts "finished performance test run: #{Time.new.to_s}"
+    puts "total elapsed time: #{(Time.new - startTime).round(2).to_s} seconds"
+    exit 0
+  end
+
+}

data/tools/generate_twitter_authorization_header.rb

@@ -0,0 +1,31 @@
+require 'bundler/setup'
+require 'logger'
+
+require 'http/oauth'
+require 'http/streaming/client'
+require 'http/custom_logger'
+
+include Http::Oauth
+include Http::Streaming::Client
+include Http::CustomLogger
+
+logger.console = true
+logger.level = Logger::DEBUG
+logger.logfile = false
+
+if ARGV.size != 7 then
+  puts "Usage generate_twitter_authorization_header.rb method url params_string consumer_key consumer_secret oauth_token oauth_token_secret\n"
+  exit(1)
+end
+
+method = ARGV[0]
+url = ARGV[1]
+params_hash = Hash.new
+ARGV[2].split("&").each { |p| params_hash[p.split("=")[0]] = p.split("=")[1] } unless ARGV[2].empty?
+oauth_consumer_key = ARGV[3]
+oauth_consumer_secret = ARGV[4]
+oauth_token = ARGV[5]
+oauth_token_secret = ARGV[6]
+
+authorization = generate_twitter_authorization(url, method, params_hash, oauth_consumer_key, oauth_consumer_secret, oauth_token, oauth_token_secret)
+puts "Authorization: #{authorization}"

data/tools/generate_twitter_bearer_token.rb

@@ -0,0 +1,26 @@
+require 'bundler/setup'
+require 'logger'
+require 'base64'
+
+require 'http/streaming/client'
+require 'http/custom_logger'
+
+include Http::Streaming::Client
+include Http::CustomLogger
+
+logger.console = true
+logger.level = Logger::DEBUG
+logger.logfile = false
+
+if ARGV.size != 2 then
+  puts "Usage generate_twitter_bearer_token.rb app_key app_secret\n"
+  exit(1)
+end
+
+key = URI::encode(ARGV[0])
+secret = URI::encode(ARGV[1])
+encoded = Base64.strict_encode64("#{key}:#{secret}")
+
+# API spec states that grant_type should be in the post body...but api.twitter.com seems to see it only if it is a URL parameter...
+response = post("https://api.twitter.com/oauth2/token?grant_type=client_credentials", "grant_type=client_credentials", headers: {'Authorization' => "Basic #{encoded}" })
+puts response

data/tools/twitter_firehose.rb

@@ -0,0 +1,17 @@
+require 'bundler/setup'
+require 'logger'
+require 'json'
+
+require 'http_streaming_client'
+
+require 'http_streaming_client/credentials/twitter'
+include HttpStreamingClient::Credentials::Twitter
+
+HttpStreamingClient.logger.console = false
+HttpStreamingClient.logger.logfile = false
+
+url = "https://stream.twitter.com/1.1/statuses/sample.json"
+authorization = HttpStreamingClient::Oauth::Twitter.generate_authorization(url, "get", {}, OAUTH_CONSUMER_KEY, OAUTH_CONSUMER_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
+puts "authorization: #{authorization}"
+client = HttpStreamingClient::Client.new(compression: true)
+response = client.get(url, {:headers => {'Authorization' => "#{authorization}" }}) { |line| puts "#{JSON.parse(line).to_s}/n" }

data/tools/twitter_firehose_performance_test.rb

@@ -0,0 +1,78 @@
+require 'bundler/setup'
+require 'logger'
+require 'json'
+
+require 'http_streaming_client'
+
+require 'http_streaming_client/credentials/twitter'
+include HttpStreamingClient::Credentials::Twitter
+
+HttpStreamingClient.logger.console = false
+HttpStreamingClient.logger.logfile = false
+
+url = "https://stream.twitter.com/1.1/statuses/sample.json"
+authorization = HttpStreamingClient::Oauth::Twitter.generate_authorization(url, "get", {}, OAUTH_CONSUMER_KEY, OAUTH_CONSUMER_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
+puts "authorization: #{authorization}"
+
+NUM_RECORDS_PER_BATCH = 1000
+MAX_RECORDS = 50000
+
+count = 0
+totalSize = 0
+intervalSize = 0
+startTime = nil
+lastTime = nil
+
+puts "starting performance test run: #{Time.new.to_s}"
+
+startTime = lastTime = Time.new 
+
+client = HttpStreamingClient::Client.new(compression: true)
+response = client.get(url, {:headers => {'Authorization' => "#{authorization}" }}) { |line|
+
+  if line.nil? then
+    puts "error:nil line received"
+    next
+  end
+
+  if line.size == 0 then
+    puts "error:zero length line received"
+    next
+  end
+
+  count = count + 1
+  intervalSize = intervalSize + line.size
+
+  if count % NUM_RECORDS_PER_BATCH == 0 then
+    
+    now = Time.new
+    intervalElapsedTime = now - lastTime
+    totalElapsedTime = now - startTime
+
+    totalSize = totalSize + intervalSize
+
+    stats = Hash.new
+
+    stats['total_records_received'] = count.to_s
+    stats['total_elapsed_time_sec'] = totalElapsedTime.round(2).to_s
+    stats['total_records_per_sec'] = (count / totalElapsedTime).round(2).to_s
+    stats['total_kbytes_per_sec'] = (totalSize / totalElapsedTime / 1024).round(2).to_s
+
+    stats['interval_records_received'] = NUM_RECORDS_PER_BATCH
+    stats['interval_elapsed_time_sec'] = intervalElapsedTime.round(2).to_s
+    stats['interval_records_per_sec'] = (NUM_RECORDS_PER_BATCH / intervalElapsedTime).round(2).to_s
+    stats['interval_kbytes_per_sec'] = (intervalSize / intervalElapsedTime / 1024).round(2).to_s
+
+    puts stats.to_json
+    
+    lastTime = now
+    intervalSize = 0
+  end
+  
+  if count == MAX_RECORDS then
+    puts "finished performance test run: #{Time.new.to_s}"
+    puts "total elapsed time: #{(Time.new - startTime).round(2).to_s} seconds"
+    exit 0
+  end
+  
+}

metadata

@@ -0,0 +1,168 @@
+--- !ruby/object:Gem::Specification
+name: http_streaming_client
+version: !ruby/object:Gem::Version
+  version: 0.8.1
+platform: ruby
+authors:
+- David Tompkins
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-11-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Ruby HTTP client with streaming support for GZIP compressed streams and
+  chunked transfer encoding. Also includes extensible OAuth support for Adobe and
+  Twitter streaming APIs.
+email:
+- tompkins@adobe.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- AUTHORS
+- CHANGELOG
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- http_streaming_client.gemspec
+- lib/http_streaming_client.rb
+- lib/http_streaming_client/client.rb
+- lib/http_streaming_client/credentials/.gitignore
+- lib/http_streaming_client/credentials/adobe.rb.sample
+- lib/http_streaming_client/credentials/twitter.rb.sample
+- lib/http_streaming_client/custom_logger.rb
+- lib/http_streaming_client/decoders/gzip.rb
+- lib/http_streaming_client/errors.rb
+- lib/http_streaming_client/oauth.rb
+- lib/http_streaming_client/oauth/adobe.rb
+- lib/http_streaming_client/oauth/base.rb
+- lib/http_streaming_client/oauth/twitter.rb
+- lib/http_streaming_client/railtie.rb
+- lib/http_streaming_client/version.rb
+- spec/adobe_spec.rb
+- spec/client_spec.rb
+- spec/oauth_spec.rb
+- spec/spec_helper.rb
+- spec/twitter_spec.rb
+- tools/adobe_firehose.rb
+- tools/adobe_firehose_performance_test.rb
+- tools/generate_twitter_authorization_header.rb
+- tools/generate_twitter_bearer_token.rb
+- tools/twitter_firehose.rb
+- tools/twitter_firehose_performance_test.rb
+homepage: https://git.corp.adobe.com/tompkins/http_streaming_client
+licenses:
+- Apache 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: a streaming HTTP protocol client
+test_files:
+- spec/adobe_spec.rb
+- spec/client_spec.rb
+- spec/oauth_spec.rb
+- spec/spec_helper.rb
+- spec/twitter_spec.rb