datasift 1.5.0 → 2.0.0

data/lib/datasift.rb

@@ -1,13 +1,5 @@
-#
-# datasift.rb - DataSift module base file
-#
-# Copyright (C) 2011 MediaSift Ltd
-#
-# == Overview
-#
-# This is the base file for the DataSift API library. Require this file to get
-# access to the full library functionality.
-#
+#This is the base file for the DataSift API library. Require this file to get
+#access to the full library functionality.
 require 'rubygems'
 
 dir = File.dirname(__FILE__)
@@ -15,5 +7,8 @@ require dir + '/DataSift/exceptions'
 require dir + '/DataSift/apiclient'
 require dir + '/DataSift/user'
 require dir + '/DataSift/definition'
+require dir + '/DataSift/historic'
+require dir + '/DataSift/push_definition'
+require dir + '/DataSift/push_subscription'
 require dir + '/DataSift/stream_consumer'
 require dir + '/DataSift/stream_consumer_http'

data/test/helper.rb

@@ -12,13 +12,87 @@ class Test::Unit::TestCase
 	def init()
 		@config = YAML::load(File.open(File.join(File.dirname(__FILE__), '..', 'config.yml')))
 		@testdata = YAML::load(File.open(File.join(File.dirname(__FILE__), 'testdata.yml')))
+
+		# Initialise the test data (parse the dates, etc).
+		@testdata['historic_start'] = DateTime.parse(@testdata['historic_start'])
+		@testdata['historic_end'] = DateTime.parse(@testdata['historic_end'])
+		@testdata['historic_created_at'] = DateTime.parse(@testdata['historic_created_at'])
+		@testdata['push_created_at'] = DateTime.parse(@testdata['push_created_at'])
+		@testdata['push_last_request'] = DateTime.parse(@testdata['push_last_request'])
+		@testdata['push_last_success'] = DateTime.parse(@testdata['push_last_success'])
+		@testdata['historic_sources'] = @testdata['historic_sources'].split(',')
+
+		@user = DataSift::User.new(@config['username'], @config['api_key'])
+		@user.setApiClient(DataSift::MockApiClient.new())
+		@user.api_client.clearResponse()
+	end
+
+	def set204Response()
+		@user.api_client.setResponse(204, {}, 200, 150)
+	end
+
+	def setResponseToASingleHistoric(changes = {})
+		data = {
+				'id' => @testdata['historic_playback_id'],
+				'definition_id' => @testdata['definition_hash'],
+				'name' => @testdata['historic_name'],
+				'start' => @testdata['historic_start'].strftime('%s'),
+				'end' => @testdata['historic_end'].strftime('%s'),
+				'created_at' => @testdata['historic_created_at'].strftime('%s'),
+				'status' => @testdata['historic_status'],
+				'progress' => 0,
+				'sources' => @testdata['historic_sources'],
+				'sample' => @testdata['historic_sample'],
+				'volume_info' => {
+					'digg' => 9
+				},
+			}
+		@user.api_client.setResponse(200, data.merge(changes), 200, 150)
+	end
+
+	def configurePushDefinition(push)
+		push.output_type = @testdata['push_output_type']
+		@testdata['push_output_params'].each { |k,v| push.output_params[k] = v }
+	end
+
+	def setResponseToASingleSubscription(changes = {})
+		data = {
+				'id'            => @testdata['push_id'],
+				'name'          => @testdata['push_name'],
+				'created_at'    => @testdata['push_created_at'].strftime('%s'),
+				'status'        => @testdata['push_status'],
+				'hash'          => @testdata['definition_hash'],
+				'hash_type'     => @testdata['push_hash_type'],
+				'output_type'   => @testdata['push_output_type'],
+				'output_params' => {
+					'delivery_frequency' => @testdata['push_output_params']['delivery_frequency'],
+					'url'                => @testdata['push_output_params']['url'],
+					'auth'               => {
+						'type'      => @testdata['push_output_params']['auth_type'],
+						'username'	=> @testdata['push_output_params']['auth_username'],
+						'password'	=> @testdata['push_output_params']['auth_password'],
+					},
+				},
+				'last_request'  => @testdata['push_last_request'].strftime('%s'),
+				'last_success'  => @testdata['push_last_success'].strftime('%s'),
+			}
+		@user.api_client.setResponse(200, data.merge(changes), 200, 150)
 	end
 
-	def initUser(mock = true)
-			@user = DataSift::User.new(@config['username'], @config['api_key'])
-			if mock
-				@user.setApiClient(DataSift::MockApiClient.new())
-				@user.api_client.clearResponse()
-			end
+	def checkSubscription(subscription)
+		assert_equal @testdata['push_id'],                                  subscription.id
+		assert_equal @testdata['push_name'],                                subscription.name
+		assert_equal @testdata['push_created_at'],                          subscription.created_at
+		assert_equal @testdata['push_status'],                              subscription.status
+		assert_equal @testdata['definition_hash'],                          subscription.hash
+		assert_equal @testdata['push_hash_type'],                           subscription.hash_type
+		assert_equal @testdata['push_output_type'],                         subscription.output_type
+		assert_equal @testdata['push_output_params']['delivery_frequency'], subscription.output_params['delivery_frequency']
+		assert_equal @testdata['push_output_params']['url'],                subscription.output_params['url']
+		assert_equal @testdata['push_output_params']['auth_type'],          subscription.output_params['auth.type']
+		assert_equal @testdata['push_output_params']['auth_username'],      subscription.output_params['auth.username']
+		assert_equal @testdata['push_output_params']['auth_password'],      subscription.output_params['auth.password']
+		assert_equal @testdata['push_last_request'],                        subscription.last_request
+		assert_equal @testdata['push_last_success'],                        subscription.last_success
 	end
 end

data/test/test_definition.rb

@@ -4,7 +4,6 @@ class TestDefinition < Test::Unit::TestCase
 	context "Given an empty Definition object" do
 		setup do
 			init()
-			initUser()
 			@definition = @user.createDefinition()
 		end
 
@@ -22,7 +21,6 @@ class TestDefinition < Test::Unit::TestCase
 	context "Given a Definition object with CSDL" do
 		setup do
 			init()
-			initUser()
 			@definition = @user.createDefinition(@testdata['definition'])
 		end
 
@@ -35,7 +33,6 @@ class TestDefinition < Test::Unit::TestCase
 	context "Given a Definition object with CSDL plus padding" do
 		setup do
 			init()
-			initUser()
 			@definition = @user.createDefinition("    " + @testdata['definition'] + "     ")
 		end
 
@@ -54,7 +51,6 @@ class TestDefinition < Test::Unit::TestCase
 	context "When trying to create a Definition object with an invalid CSDL" do
 		setup do
 			init()
-			initUser()
 		end
 
 		should "raise an InvalidDataError" do
@@ -65,7 +61,6 @@ class TestDefinition < Test::Unit::TestCase
 	context "Given a Definition object with a valid CSDL" do
 		setup do
 			init()
-			initUser()
 			@definition = @user.createDefinition(@testdata['definition'])
 		end
 
@@ -119,7 +114,6 @@ class TestDefinition < Test::Unit::TestCase
 	context "Given a Definition object with an invalid CSDL" do
 		setup do
 			init()
-			initUser()
 			@definition = @user.createDefinition(@testdata['invalid_definition'])
 		end
 
@@ -141,7 +135,6 @@ class TestDefinition < Test::Unit::TestCase
 	context "The DPU returned from a valid Definition object" do
 		setup do
 			init()
-			initUser()
 			@definition = @user.createDefinition(@testdata['definition'])
 			# Compile the definition first
 			@user.api_client.setResponse(200, {
@@ -182,7 +175,6 @@ class TestDefinition < Test::Unit::TestCase
 	context "Buffered data returned by a valid Definition object" do
 		setup do
 			init()
-			initUser()
 			@definition = @user.createDefinition(@testdata['definition'])
 			# Compile the definition first
 			@user.api_client.setResponse(200, {
@@ -262,7 +254,6 @@ class TestDefinition < Test::Unit::TestCase
 	context "A StreamConsumer object returned by a valid Definition object" do
 		setup do
 			init()
-			initUser()
 			@definition = @user.createDefinition(@testdata['definition'])
 			# Compile the definition first
 			@user.api_client.setResponse(200, {

data/test/test_historics.rb

@@ -0,0 +1,233 @@
+require 'helper'
+
+class TestHistorics < Test::Unit::TestCase
+	context "Given a new Historic object from a stream hash" do
+		setup do
+			init()
+			# Create the historic (API response is for compiling the definition)
+			@historic = @user.createHistoric(@testdata['definition_hash'], @testdata['historic_start'], @testdata['historic_end'], @testdata['historic_sources'], @testdata['historic_sample'], @testdata['historic_name'])
+		end
+
+		should "be a Historic object" do
+			assert_not_nil @historic
+			assert @historic.kind_of?(DataSift::Historic)
+		end
+
+		should "have the correct definition_hash" do
+			assert_equal @testdata['definition_hash'], @historic.stream_hash
+		end
+
+		should "have the correct name" do
+			assert_equal @testdata['historic_name'], @historic.name
+		end
+
+		should "have the correct start_date" do
+			assert_equal @testdata['historic_start'], @historic.start_date
+		end
+
+		should "have the correct end_date" do
+			assert_equal @testdata['historic_end'], @historic.end_date
+		end
+
+		should "have the correct status" do
+			assert_equal @testdata['historic_status'], @historic.status
+		end
+
+		should "have the correct progress" do
+			assert_equal 0, @historic.progress
+		end
+
+		should "have the correct sample" do
+			assert_equal @testdata['historic_sample'], @historic.sample
+		end
+
+		should "be able to change the name before preparing" do
+			assert_equal @testdata['historic_name'], @historic.name
+
+			@historic.name = 'new name'
+
+			assert_equal 'new name', @historic.name
+		end
+
+		should "be able to prepare the query" do
+			setResponseToASingleHistoric({
+				'dpus'         => @testdata['historic_dpus'],
+				'availability' => @testdata['historic_availability']
+			})
+			@historic.prepare()
+		end
+
+		should "not be able to prepare it more than once" do
+			setResponseToASingleHistoric({
+				'dpus'         => @testdata['historic_dpus'],
+				'availability' => @testdata['historic_availability']
+			})
+			@historic.prepare()
+			assert_raise(DataSift::InvalidDataError) { @historic.prepare() }
+		end
+
+		should "be able to change the name after preparing" do
+			setResponseToASingleHistoric({
+				'dpus'         => @testdata['historic_dpus'],
+				'availability' => @testdata['historic_availability']
+			})
+			@historic.prepare()
+
+			assert_equal @testdata['historic_name'], @historic.name
+
+			new_name = 'new name'
+			setResponseToASingleHistoric({ 'name' => new_name })
+			@historic.name = new_name
+
+			assert_equal new_name, @historic.name
+		end
+
+		should "be able to start the query" do
+			setResponseToASingleHistoric({
+				'dpus'         => @testdata['historic_dpus'],
+				'availability' => @testdata['historic_availability']
+			})
+			@historic.prepare()
+
+			set204Response()
+			@historic.start()
+		end
+
+		should "be able to stop the query" do
+			setResponseToASingleHistoric({
+				'dpus'         => @testdata['historic_dpus'],
+				'availability' => @testdata['historic_availability']
+			})
+			@historic.prepare()
+
+			set204Response()
+			@historic.stop()
+		end
+
+		should "be able to delete the query" do
+			setResponseToASingleHistoric({
+				'dpus'         => @testdata['historic_dpus'],
+				'availability' => @testdata['historic_availability']
+			})
+			@historic.prepare()
+
+			set204Response()
+			@historic.delete()
+		end
+
+		should "not be able to start the query after deletion" do
+			setResponseToASingleHistoric({
+				'dpus'         => @testdata['historic_dpus'],
+				'availability' => @testdata['historic_availability']
+			})
+			@historic.prepare()
+
+			set204Response()
+			@historic.delete()
+
+			assert_raise(DataSift::InvalidDataError) { @historic.start() }
+		end
+	end
+
+	context "Given a new Historic object from a Definition" do
+		setup do
+			init()
+			# Create the definition
+			@definition = @user.createDefinition(@testdata['definition'])
+			# Create the historic (API response is for compiling the definition)
+			@user.api_client.setResponse(200, {
+				'hash'       => @testdata['definition_hash'],
+				'created_at' => Time.now.strftime('%Y-%m-%d %H:%M:%S'),
+				'dpu'        => 10,
+			}, 200, 150)
+			@historic = @definition.createHistoric(@testdata['historic_start'], @testdata['historic_end'], @testdata['historic_sources'], @testdata['historic_sample'], @testdata['historic_name'])
+		end
+
+		should "be a Historic object" do
+			assert_not_nil @historic
+		end
+
+		should "have the correct definition_hash" do
+			assert_equal @testdata['definition_hash'], @historic.stream_hash
+		end
+
+		should "have the correct name" do
+			assert_equal @testdata['historic_name'], @historic.name
+		end
+
+		should "have the correct start_date" do
+			assert_equal @testdata['historic_start'], @historic.start_date
+		end
+
+		should "have the correct end_date" do
+			assert_equal @testdata['historic_end'], @historic.end_date
+		end
+
+		should "have the correct status" do
+			assert_equal @testdata['historic_status'], @historic.status
+		end
+
+		should "have the correct progress" do
+			assert_equal 0, @historic.progress
+		end
+
+		should "have the correct sample" do
+			assert_equal @testdata['historic_sample'], @historic.sample
+		end
+	end
+
+	context "Given a Historic object retrieved from the API" do
+		setup do
+			init()
+			# Create the historic (API response is for compiling the definition)
+			setResponseToASingleHistoric()
+			@historic = @user.getHistoric(@testdata['historic_playback_id'])
+		end
+
+		should "be a Historic object" do
+			assert_not_nil @historic
+		end
+
+		should "have the correct definition_hash" do
+			assert_equal @testdata['definition_hash'], @historic.stream_hash
+		end
+
+		should "have the correct name" do
+			assert_equal @testdata['historic_name'], @historic.name
+		end
+
+		should "have the correct start_date" do
+			assert_equal @testdata['historic_start'], @historic.start_date
+		end
+
+		should "have the correct end_date" do
+			assert_equal @testdata['historic_end'], @historic.end_date
+		end
+
+		should "have the correct status" do
+			assert_equal @testdata['historic_status'], @historic.status
+		end
+
+		should "have the correct progress" do
+			assert_equal 0, @historic.progress
+		end
+
+		should "have the correct sample" do
+			assert_equal @testdata['historic_sample'], @historic.sample
+		end
+
+		should "not be able to prepare the query" do
+			assert_raise(DataSift::InvalidDataError) { @historic.prepare() }
+		end
+
+		should "be able to change the name" do
+			assert_equal @testdata['historic_name'], @historic.name
+
+			new_name = 'new name'
+			setResponseToASingleHistoric({ 'name' => new_name })
+			@historic.name = new_name
+
+			assert_equal new_name, @historic.name
+		end
+	end
+end

data/test/test_pushdefinition.rb

@@ -0,0 +1,92 @@
+require 'helper'
+
+class TestPushDefinition < Test::Unit::TestCase
+	context "Given a new PushDefinition object" do
+		setup do
+			init()
+			# Create the historic (API response is for compiling the definition)
+			@push = @user.createPushDefinition()
+		end
+
+		should "be of the right type" do
+			assert_not_nil @push
+			assert @push.kind_of?(DataSift::PushDefinition)
+		end
+
+		should "have an empty initial status" do
+			assert_equal '', @push.initial_status
+		end
+
+		should "allow the initial status to be changed" do
+			@push.initial_status = DataSift::PushSubscription::STATUS_PAUSED
+			assert_equal DataSift::PushSubscription::STATUS_PAUSED, @push.initial_status
+			@push.initial_status = DataSift::PushSubscription::STATUS_STOPPED
+			assert_equal DataSift::PushSubscription::STATUS_STOPPED, @push.initial_status
+		end
+
+		should "have an empty output type" do
+			assert_equal '', @push.output_type
+		end
+
+		should "allow the output type to be set" do
+			@push.output_type = @testdata['push_output_type']
+			assert_equal @testdata['push_output_type'], @push.output_type
+		end
+
+		should "return nil for non-existant output parameters" do
+			assert_nil @push.output_params['url']
+		end
+
+		should "allow output parameters to be set" do
+			@push.output_params['url'] = @testdata['push_output_param_url']
+			assert_equal @testdata['push_output_param_url'], @push.output_params['url']
+			assert_equal 1, @push.output_params.size()
+		end
+
+		should "support validation of the output parameters" do
+			configurePushDefinition(@push)
+			set204Response()
+			@push.validate()
+		end
+
+		should "support subscribing to a Definition object" do
+			definition = @user.createDefinition(@testdata['definition'])
+			# Set the response to the compile method so we can get the hash
+			begin
+				@user.api_client.setResponse(200, {
+					'hash'       => @testdata['definition_hash'],
+					'created_at' => Time.now.strftime('%Y-%m-%d %H:%M:%S'),
+					'dpu'       => 10,
+				}, 200, 150)
+				assert_equal @testdata['definition_hash'], definition.hash
+			rescue InvalidDataError
+				assert false, "InvalidDataError"
+			rescue CompileFailedError
+				assert false, "CompileFailedError"
+			rescue APIError
+				assert false, "APIError"
+			end
+
+			configurePushDefinition(@push)
+
+			setResponseToASingleSubscription()
+
+			subscription = @push.subscribeDefinition(definition, @testdata['push_name'])
+
+			checkSubscription(subscription)
+		end
+
+		should "support subscribing to a Historic object" do
+			setResponseToASingleHistoric()
+			historic = @user.getHistoric(@testdata['historic_playback_id'])
+
+			configurePushDefinition(@push)
+
+			setResponseToASingleSubscription()
+
+			subscription = @push.subscribeHistoric(historic, @testdata['push_name'])
+
+			checkSubscription(subscription)
+		end
+	end
+end