datasift 0.2.0 → 1.0.0

data/README.md

@@ -28,7 +28,7 @@ consumer.consume(true) do |interaction|
 end
 ```
 
-See the DataSift documentation for full details of the data contained within each interaction: http://support.datasift.net/help/kb/rest-api/return-objects
+See the DataSift documentation for full details of the data contained within each interaction. See this page on our developer site for an example tweet: http://dev.datasift.com/docs/targets/twitter/twitter-output-format
 
 License
 -------

data/Rakefile

@@ -1,9 +1,7 @@
 require 'rubygems'
 require 'rake'
+
 require 'rake/testtask'
-require 'rake/clean'
-require 'rake/gempackagetask'
-require 'rake/rdoctask'
 Rake::TestTask.new(:test) do |test|
 	test.libs << 'lib' << 'test'
 	test.pattern = 'test/**/test_*.rb'

data/VERSION

@@ -1 +1 @@
-0.2.0
+1.0.0

data/examples/consume-stream.rb

@@ -0,0 +1,52 @@
+# This simple example demonstrates consuming a stream from just the stream
+# hash.
+#
+# NB: Most of the error handling (exception catching) has been removed for
+# the sake of simplicity. Nearly everything in this library may throw
+# exceptions, and production code should catch them. See the documentation
+# for full details.
+#
+
+# Make sure we have some arguments
+if ARGV.size == 0
+	puts 'ERR: Please specify the hash to consume!'
+	puts
+	puts
+	exit!
+end
+
+# Include the DataSift library
+require File.dirname(__FILE__) + '/../lib/datasift'
+
+# Include the configuration - put your username and API key in this file
+config = YAML::load(File.open(File.join(File.dirname(__FILE__), '..', 'config.yml')))
+
+# Authenticate
+puts 'Creating user...'
+user = DataSift::User.new(config['username'], config['api_key'])
+
+# Create the consumer
+puts 'Getting the consumer...'
+consumer = user.getConsumer(DataSift::StreamConsumer::TYPE_HTTP, ARGV[0])
+
+# Setting up the onStopped handler
+consumer.onStopped do |reason|
+	puts
+	puts 'Stopped: ' + reason
+	puts
+end
+
+# And start consuming
+puts 'Consuming...'
+puts '--'
+consumer.consume(true) do |interaction|
+	if interaction
+		puts 'Type: ' + interaction['interaction']['type']
+		puts 'Content: ' + interaction['interaction']['content']
+		puts '--'
+	end
+end
+
+puts
+puts 'Finished consuming'
+puts

data/examples/{cost.rb → dpu.rb}

@@ -1,10 +1,10 @@
-# This example gets the cost associated with the stream given on the command
+# This example gets the DPU associated with the stream given on the command
 # line or piped/typed into STDIN. It  presents it in a nice ASCII table.]
 # Note that the CSDL must be enclosed in quotes if given on the command line.
 #
-# ruby cost.rb 'interaction.content contains "football"'
+# ruby dpu.rb 'interaction.content contains "football"'
 #  or
-# cat football.csdl | ruby cost.rb
+# cat football.csdl | ruby dpu.rb
 #
 # NB: Most of the error handling (exception catching) has been removed for
 # the sake of simplicity. Nearly everything in this library may throw
@@ -38,43 +38,43 @@ user = DataSift::User.new(config['username'], config['api_key'])
 puts 'Creating definition...'
 definition = user.createDefinition(csdl)
 
-# Getting cost
-puts 'Getting cost...'
+# Getting DPU
+puts 'Getting DPU...'
 begin
-	cost = definition.getCostBreakdown()
+	dpu = definition.getDPUBreakdown()
 rescue DataSift::CompileFailedError => e
 	puts 'CSDL compilation failed: ' + e
 	puts
 	exit!
 end
 
-costtable = []
+dputable = []
 maxlength = {'target' => 'Target'.length, 'times used' => 'Times used'.length, 'complexity' => 'Complexity'.length};
-cost['costs'].each do |tgt,c|
+dpu['detail'].each do |tgt,c|
 	maxlength['target'] = [maxlength['target'], tgt.length].max()
 	maxlength['times used'] = [maxlength['times used'], number_with_delimiter(c['count']).length].max()
-	maxlength['complexity'] = [maxlength['complexity'], number_with_delimiter(c['cost']).length].max()
+	maxlength['complexity'] = [maxlength['complexity'], number_with_delimiter(c['dpu']).length].max()
 
-	costtable.push({
+	dputable.push({
 			'target' => tgt,
 			'times used' => number_with_delimiter(c['count']),
-			'complexity' => number_with_delimiter(c['cost']),
+			'complexity' => number_with_delimiter(c['dpu']),
 		})
 
 	c['targets'].each do |tgt2,d|
 		maxlength['target'] = [maxlength['target'], 2 + tgt2.length].max()
 		maxlength['times used'] = [maxlength['times used'], number_with_delimiter(d['count']).length].max()
-		maxlength['complexity'] = [maxlength['complexity'], number_with_delimiter(d['cost']).length].max()
+		maxlength['complexity'] = [maxlength['complexity'], number_with_delimiter(d['dpu']).length].max()
 
-		costtable.push({
+		dputable.push({
 				'target' => '  ' + tgt2,
 				'times used' => number_with_delimiter(d['count']),
-				'complexity' => number_with_delimiter(d['cost']),
+				'complexity' => number_with_delimiter(d['dpu']),
 			})
 	end
 end
 
-maxlength['complexity'] = [maxlength['complexity'], number_with_delimiter(cost['total']).length].max()
+maxlength['complexity'] = [maxlength['complexity'], number_with_delimiter(dpu['dpu']).length].max()
 
 puts
 print '/-' + ('-' * maxlength['target']) + '---'
@@ -89,7 +89,7 @@ print '|-' + ('-' * maxlength['target']) + '-+-'
 print ('-' * maxlength['times used']) + '-+-'
 puts ('-' * maxlength['complexity']) + '-|'
 
-costtable.each do |row|
+dputable.each do |row|
 	print '| ' + row['target'].ljust(maxlength['target']) + ' | '
 	print row['times used'].rjust(maxlength['times used']) + ' | '
 	puts row['complexity'].rjust(maxlength['complexity']) + ' |'
@@ -100,24 +100,10 @@ print ('-' * maxlength['times used']) + '-+-'
 puts ('-' * maxlength['complexity']) + '-|'
 
 print '| ' + 'Total'.rjust(maxlength['target'] + 3 + maxlength['times used']) + ' = '
-puts cost['total'].to_s.rjust(maxlength['complexity']) + ' |'
+puts dpu['dpu'].to_s.rjust(maxlength['complexity']) + ' |'
 
 print '\\-' + ('-' * maxlength['target']) + '---'
 print ('-' * maxlength['times used']) + '---'
 puts ('-' * maxlength['complexity']) + '-/'
 
 puts
-
-if cost['total'] > 1000
-	tiernum = 3;
-	tierdesc = 'high complexity';
-elsif cost['total'] > 100
-	tiernum = 2;
-	tierdesc = 'medium complexity';
-else
-	tiernum = 1;
-	tierdesc = 'simple complexity';
-end
-
-puts 'A total cost of ' + number_with_delimiter(cost['total']) + ' puts this stream in tier ' + tiernum.to_s + ', ' + tierdesc
-puts

data/lib/DataSift/definition.rb

@@ -18,7 +18,7 @@ module DataSift
 	# The Definition class represents a stream definition.
 	#
 	class Definition
-		attr_reader :csdl, :total_cost, :created_at
+		attr_reader :csdl, :total_dpu, :created_at
 
 		# Constructor. A User object is required, and you can optionally supply a
 		# default CSDL string.
@@ -36,12 +36,22 @@ module DataSift
 			self.csdl = csdl
 		end
 
+		# CSDL getter
+		def csdl
+			raise InvalidDataError, 'The CSDL is not available' unless !@csdl.nil?
+			@csdl
+		end
+
 		# CSDL setter. Strips the incoming string and resets the hash if it's changed.
 		def csdl=(csdl)
-			raise InvalidDataError, 'The CSDL must be a string.' unless csdl.is_a? String
-			csdl.strip!
-			clearHash() unless csdl == @csdl
-			@csdl = csdl
+			if csdl.nil?
+				@csdl = nil
+			else
+				raise InvalidDataError, 'The CSDL must be a string.' unless csdl.is_a? String
+				csdl.strip!
+				clearHash() unless csdl == @csdl
+				@csdl = csdl
+			end
 		end
 
 		# Hash getter. If the hash has not yet been obtained the CSDL will be
@@ -61,8 +71,9 @@ module DataSift
 		# Reset the hash to false. The effect of this is to mark the definition as
 		# requiring compilation.
 		def clearHash()
+			@csdl = '' unless !@csdl.nil?
 			@hash = false
-			@total_cost = false
+			@total_dpu = false
 			@created_at = false
 		end
 
@@ -80,10 +91,10 @@ module DataSift
 					raise CompileFailedError, 'Compiled successfully but no hash in the response'
 				end
 
-				if res.has_key?('cost')
-					@total_cost = Integer(res['cost'])
+				if res.has_key?('dpu')
+					@total_dpu = Float(res['dpu'])
 				else
-					raise CompileFailedError, 'Compiled successfully but no cost in the response'
+					raise CompileFailedError, 'Compiled successfully but no DPU in the response'
 				end
 
 				if res.has_key?('created_at')
@@ -103,16 +114,15 @@ module DataSift
 			end
 		end
 
-		# Call the DataSift API to get the cost for this definition. Returns an
+		# Call the DataSift API to get the DPU for this definition. Returns an
 		# array containing...
-		#   costs => The breakdown of running the rule
-		#   tags => The tags associated with the rule
-		#   total => The total cost of the rule
+		#   detail => The breakdown of running the rule
+		#   dpu => The total DPU of the rule
 		#
-		def getCostBreakdown()
-			raise InvalidDataError, "Cannot get the cost for an empty definition." unless @csdl.length > 0
+		def getDPUBreakdown()
+			raise InvalidDataError, "Cannot get the DPU for an empty definition." unless @csdl.length > 0
 
-			@user.callAPI('cost', { 'hash' => self.hash })
+			@user.callAPI('dpu', { 'hash' => self.hash })
 		end
 
 		# Call the DataSift API to get buffered interactions.

data/lib/DataSift/stream_consumer.rb

@@ -57,8 +57,8 @@ module DataSift
 			@stop_reason = 'Unknown reason'
 			@state = STATE_STOPPED
 
-			# Compile the definition to ensure it's valid for use
-			@definition.compile()
+			# Get the hash which will compile the CSDL if necessary
+			@definition.hash
 		end
 
 		# This is called when the consumer is stopped.

data/lib/DataSift/stream_consumer_http.rb

@@ -39,7 +39,9 @@ module DataSift
 							next if json.nil?
 							chunkLeft = size-json.size
 							if chunkLeft == 0
-								parser << json
+								if json.length > 100
+									parser << json
+								end
 							else
 								# received only part of the chunk, grab the rest
 								parser << @socket.read(chunkLeft)

data/lib/DataSift/user.rb

@@ -22,9 +22,9 @@ module DataSift
 	# provides factory methods for all of the functionality in the API.
 	#
 	class User
-		USER_AGENT = 'DataSiftRuby/0.1';
-		API_BASE_URL = 'api.datasift.net/';
-		STREAM_BASE_URL = 'stream.datasift.net/';
+		USER_AGENT = 'DataSiftRuby/0.3.0';
+		API_BASE_URL = 'api.datasift.com/';
+		STREAM_BASE_URL = 'stream.datasift.com/';
 
 		attr_reader :username, :api_key, :rate_limit, :rate_limit_remaining, :api_client
 
@@ -54,6 +54,32 @@ module DataSift
 			DataSift::Definition.new(self, csdl, false)
 		end
 
+		# Returns a StreamConsumer-derived object for the given hash, for the
+		# given type.
+		# === Parameters
+		#
+		# * +type+ - The consumer type for which to construct a consumer.
+		# * +hash+ - The hash to be consumed.
+		#
+		def getConsumer(type = nil, hash = nil, on_interaction = nil, on_stopped = nil)
+			StreamConsumer.factory(self, type, Definition.new(self, nil, hash))
+		end
+
+		# Returns the usage data for this user. If a hash is provided then a more
+		# detailed breakdown using interaction types is retrieved and returned.
+		# === Parameters
+		#
+		# * +period+ - An optional period for which to fetch data ('hour' or 'day')
+		def getUsage(period = 'hour')
+			if period != 'hour' and period != 'day'
+				raise EInvalidData, 'Period must be hour or day'
+			end
+
+			params = { 'period' => period }
+
+			callAPI('usage', params)
+		end
+
 		# Returns the user agent this library should use for all API calls.
 		def getUserAgent()
 			USER_AGENT

data/test/test_definition.rb

@@ -74,7 +74,7 @@ class TestDefinition < Test::Unit::TestCase
 				@user.api_client.setResponse(200, {
 					'hash'       => @testdata['definition_hash'],
 					'created_at' => Time.now.strftime('%Y-%m-%d %H:%M:%S'),
-					'cost'       => 10,
+					'dpu'       => 10,
 				}, 200, 150)
 				@definition.compile()
 			rescue InvalidDataError
@@ -90,26 +90,26 @@ class TestDefinition < Test::Unit::TestCase
 			@user.api_client.setResponse(200, {
 				'hash'       => @testdata['definition_hash'],
 				'created_at' => Time.now.strftime('%Y-%m-%d %H:%M:%S'),
-				'cost'       => 10,
+				'dpu'        => 10,
 			}, 200, 150)
 			assert_equal @testdata['definition_hash'], @definition.hash
 		end
 
-		should "have a positive cost" do
+		should "have a positive DPU" do
 			@user.api_client.setResponse(200, {
 				'hash'       => @testdata['definition_hash'],
 				'created_at' => Time.now.strftime('%Y-%m-%d %H:%M:%S'),
-				'cost'       => 10,
+				'dpu'        => 10,
 			}, 200, 150)
 			@definition.compile()
-			assert @definition.total_cost > 0
+			assert @definition.total_dpu > 0
 		end
 
 		should "have a valid created_at date" do
 			@user.api_client.setResponse(200, {
 				'hash'       => @testdata['definition_hash'],
 				'created_at' => Time.now.strftime('%Y-%m-%d %H:%M:%S'),
-				'cost'       => 10,
+				'dpu'        => 10,
 			}, 200, 150)
 			@definition.compile()
 			assert @definition.created_at
@@ -138,7 +138,7 @@ class TestDefinition < Test::Unit::TestCase
 		end
 	end
 
-	context "The cost returned from a valid Definition object" do
+	context "The DPU returned from a valid Definition object" do
 		setup do
 			init()
 			initUser()
@@ -147,35 +147,35 @@ class TestDefinition < Test::Unit::TestCase
 			@user.api_client.setResponse(200, {
 				'hash'       => @testdata['definition_hash'],
 				'created_at' => Time.now.strftime('%Y-%m-%d %H:%M:%S'),
-				'cost'       => 10,
+				'dpu'        => 10,
 			}, 200, 150)
 			@definition.compile()
-			# Now get the cost
+			# Now get the DPU
 			@user.api_client.setResponse(200, {
-				'costs' => {
+				'detail' => {
 					'contains' => {
 						'count'   => 1,
-						'cost'    => 4,
+						'dpu'     => 4,
 						'targets' => {
 							'interaction.content' => {
 								'count' => 1,
-								'cost'  => 4,
+								'dpu'   => 4,
 							},
 						},
 					},
 				},
-				'total' => 4
+				'dpu' => 4
 			}, 200, 150)
-			@cost = @definition.getCostBreakdown()
+			@dpu = @definition.getDPUBreakdown()
 		end
 
-		should "contain valid cost data" do
-			assert @cost.has_key?('costs')
-			assert @cost.has_key?('total')
+		should "contain valid DPU data" do
+			assert @dpu.has_key?('detail')
+			assert @dpu.has_key?('dpu')
 		end
 
-		should "have a positive total cost" do
-			assert @cost['total'] > 0
+		should "have a positive total DPU" do
+			assert @dpu['dpu'] > 0
 		end
 	end
 
@@ -188,7 +188,7 @@ class TestDefinition < Test::Unit::TestCase
 			@user.api_client.setResponse(200, {
 				'hash'       => @testdata['definition_hash'],
 				'created_at' => Time.now.strftime('%Y-%m-%d %H:%M:%S'),
-				'cost'       => 10,
+				'dpu'        => 10,
 			}, 200, 150)
 			@definition.compile()
 			# Now get some buffered interactions
@@ -268,7 +268,7 @@ class TestDefinition < Test::Unit::TestCase
 			@user.api_client.setResponse(200, {
 				'hash'       => @testdata['definition_hash'],
 				'created_at' => Time.now.strftime('%Y-%m-%d %H:%M:%S'),
-				'cost'       => 10,
+				'dpu'        => 10,
 			}, 200, 150)
 			@definition.compile()
 			# Now get a consumer

data/test/test_live_api.rb

@@ -26,9 +26,9 @@ class TestDefinitionLive < Test::Unit::TestCase
 			assert_equal @testdata['definition_hash'], @definition.hash
 		end
 
-		should "have a positive cost" do
+		should "have a positive DPU" do
 			@definition.compile()
-			assert @definition.total_cost > 0
+			assert @definition.total_dpu > 0
 		end
 
 		should "have a valid created_at date" do
@@ -54,21 +54,21 @@ class TestDefinitionLive < Test::Unit::TestCase
 		end
 	end
 
-	context "The cost returned from a valid Definition object" do
+	context "The DPU returned from a valid Definition object" do
 		setup do
 			init()
 			initUser(false)
 			@definition = @user.createDefinition(@testdata['definition'])
-			@cost = @definition.getCostBreakdown()
+			@dpu = @definition.getDPUBreakdown()
 		end
 
-		should "contain valid cost data" do
-			assert @cost.has_key?('costs')
-			assert @cost.has_key?('total')
+		should "contain valid DPU data" do
+			assert @dpu.has_key?('detail')
+			assert @dpu.has_key?('dpu')
 		end
 
-		should "have a positive total cost" do
-			assert @cost['total'] > 0
+		should "have a positive total DPU" do
+			assert @dpu['dpu'] > 0
 		end
 	end
 

data/test/test_user.rb

@@ -16,6 +16,28 @@ class TestUser < Test::Unit::TestCase
 			assert_not_nil @user
 			assert_equal @config['api_key'], @user.api_key
 		end
+
+		should "return valid summary usage information" do
+			@user.api_client.setResponse(200, {
+				'start' => 'Mon, 07 Nov 2011 14:20:00 +0000',
+				'streams' => {
+					'5e82aa9ac3dcf4dec1cce08a0cec914a' => {
+						'seconds' => 313,
+						'licenses' => {
+							'twitter' => 17,
+							'facebook' => 5
+						}
+					}
+				},
+				'end' => 'Mon, 07 Nov 2011 15:20:00 +0000'
+			}, 200, 150)
+			usage = @user.getUsage()
+			assert_equal "Mon, 07 Nov 2011 14:20:00 +0000", usage['start']
+			assert_equal "Mon, 07 Nov 2011 15:20:00 +0000", usage['end']
+			assert_equal 313, usage['streams']['5e82aa9ac3dcf4dec1cce08a0cec914a']['seconds']
+			assert_equal 17, usage['streams']['5e82aa9ac3dcf4dec1cce08a0cec914a']['licenses']['twitter']
+			assert_equal 5, usage['streams']['5e82aa9ac3dcf4dec1cce08a0cec914a']['licenses']['facebook']
+		end
 	end
 
 	context "Given an empty definition from the User factory" do
@@ -52,7 +74,7 @@ class TestUser < Test::Unit::TestCase
 			@user.api_client.setResponse(200, {
 				'hash'       => @testdata['definition_hash'],
 				'created_at' => Time.now.strftime('%Y-%m-%d %H:%M:%S'),
-				'cost'       => 10,
+				'dpu'        => 10,
 			}, 200, 150)
 			@definition.compile()
 		end
@@ -65,4 +87,31 @@ class TestUser < Test::Unit::TestCase
 			assert @user.rate_limit_remaining == 150
 		end
 	end
+
+	context "The exception raised by the User.getUsage method" do
+		setup do
+			init()
+			initUser()
+		end
+
+		should "be APIError given a 400 response" do
+			@user.api_client.setResponse(400, { 'error' => 'Bad request from user supplied data'}, 200, 150)
+			assert_raise(DataSift::APIError) { usage = @user.getUsage() }
+		end
+
+		should "be AccessDenied given a 401 response" do
+			@user.api_client.setResponse(401, { 'error' => 'User banned because they are a very bad person'}, 200, 150)
+			assert_raise(DataSift::AccessDeniedError) { usage = @user.getUsage() }
+		end
+
+		should "be APIError given a 404 response" do
+			@user.api_client.setResponse(404, { 'error' => 'Endpoint or data not found'}, 200, 150)
+			assert_raise(DataSift::APIError) { usage = @user.getUsage() }
+		end
+
+		should "be APIError given a 500 response" do
+			@user.api_client.setResponse(500, { 'error' => 'Problem with an internal service'}, 200, 150)
+			assert_raise(DataSift::APIError) { usage = @user.getUsage() }
+		end
+	end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: datasift
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 1.0.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-09-06 00:00:00.000000000Z
+date: 2011-11-21 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
-  requirement: &22559520 !ruby/object:Gem::Requirement
+  requirement: &8779460 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 1.6.3
   type: :runtime
   prerelease: false
-  version_requirements: *22559520
+  version_requirements: *8779460
 - !ruby/object:Gem::Dependency
   name: crack
-  requirement: &22559040 !ruby/object:Gem::Requirement
+  requirement: &8778640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *22559040
+  version_requirements: *8778640
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &22558540 !ruby/object:Gem::Requirement
+  requirement: &8777980 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 0.8.2
   type: :runtime
   prerelease: false
-  version_requirements: *22558540
+  version_requirements: *8777980
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &22558040 !ruby/object:Gem::Requirement
+  requirement: &8777260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *22558040
+  version_requirements: *8777260
 - !ruby/object:Gem::Dependency
   name: shoulda
-  requirement: &22557580 !ruby/object:Gem::Requirement
+  requirement: &8776460 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 2.11.3
   type: :development
   prerelease: false
-  version_requirements: *22557580
+  version_requirements: *8776460
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &22557000 !ruby/object:Gem::Requirement
+  requirement: &8775820 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,7 +76,7 @@ dependencies:
         version: 2.6.0
   type: :development
   prerelease: false
-  version_requirements: *22557000
+  version_requirements: *8775820
 description: The official Ruby library for accessing the DataSift API. See http://datasift.net/
   for full details and to sign up for an account.
 email:
@@ -91,7 +91,8 @@ files:
 - VERSION
 - config.yml
 - datasift.gemspec
-- examples/cost.rb
+- examples/consume-stream.rb
+- examples/dpu.rb
 - examples/football-buffered.rb
 - examples/football.rb
 - examples/twitter-track.rb