datasift 0.2.0

data/lib/DataSift/definition.rb

@@ -0,0 +1,154 @@
+#
+# definition.rb - This file contains the Definition class.
+#
+# Copyright (C) 2011 MediaSift Ltd
+#
+# == Overview
+#
+# The User class represents a user of the API. Applications should start their
+# API interactions by creating an instance of this class. Once initialised it
+# provides factory methods for all of the functionality in the API.
+
+module DataSift
+
+	# Definition class.
+	#
+	# == Introduction
+	#
+	# The Definition class represents a stream definition.
+	#
+	class Definition
+		attr_reader :csdl, :total_cost, :created_at
+
+		# Constructor. A User object is required, and you can optionally supply a
+		# default CSDL string.
+		# === Parameters
+		#
+		# * +user+ - The DataSift::User object.
+		# * +csdl+ - Optional default CSDL string.
+		# * +hash+ - Optional default hash string.
+		#
+		def initialize(user, csdl = '', hash = false)
+			raise InvalidDataError, 'Please supply a valid User object when creating a Definition object.' unless user.is_a? DataSift::User
+			@user = user
+			clearHash()
+			@hash = hash
+			self.csdl = csdl
+		end
+
+		# CSDL setter. Strips the incoming string and resets the hash if it's changed.
+		def csdl=(csdl)
+			raise InvalidDataError, 'The CSDL must be a string.' unless csdl.is_a? String
+			csdl.strip!
+			clearHash() unless csdl == @csdl
+			@csdl = csdl
+		end
+
+		# Hash getter. If the hash has not yet been obtained the CSDL will be
+		# compiled first.
+		def hash
+			if @hash == false
+				begin
+					compile()
+				rescue DataSift::CompileFailedError
+					# Ignore
+				end
+			end
+
+			@hash
+		end
+
+		# Reset the hash to false. The effect of this is to mark the definition as
+		# requiring compilation.
+		def clearHash()
+			@hash = false
+			@total_cost = false
+			@created_at = false
+		end
+
+		# Call the DataSift API to compile this definition. On success it will
+		# store the returned hash.
+		def compile()
+			raise InvalidDataError, 'Cannot compile an empty definition.' unless @csdl.length > 0
+
+			begin
+				res = @user.callAPI('compile', { 'csdl' => @csdl })
+
+				if res.has_key?('hash')
+					@hash = res['hash']
+				else
+					raise CompileFailedError, 'Compiled successfully but no hash in the response'
+				end
+
+				if res.has_key?('cost')
+					@total_cost = Integer(res['cost'])
+				else
+					raise CompileFailedError, 'Compiled successfully but no cost in the response'
+				end
+
+				if res.has_key?('created_at')
+					@created_at = Date.parse(res['created_at'])
+				else
+					raise CompileFailedError, 'Compiled successfully but no created_at in the response'
+				end
+			rescue APIError => err
+				clearHash()
+
+				case err.http_code
+				when 400
+					raise CompileFailedError, err
+				else
+					raise CompileFailedError, 'Unexpected APIError code: ' + err.http_code.to_s + ' [' + err + ']'
+				end
+			end
+		end
+
+		# Call the DataSift API to get the cost for this definition. Returns an
+		# array containing...
+		#   costs => The breakdown of running the rule
+		#   tags => The tags associated with the rule
+		#   total => The total cost of the rule
+		#
+		def getCostBreakdown()
+			raise InvalidDataError, "Cannot get the cost for an empty definition." unless @csdl.length > 0
+
+			@user.callAPI('cost', { 'hash' => self.hash })
+		end
+
+		# Call the DataSift API to get buffered interactions.
+		# === Parameters
+		#
+		# * +count+ - Optional number of interactions to return (max 200).
+		# * +from_id+ - Optional start ID.
+		#
+		def getBuffered(count = false, from_id = false)
+			raise InvalidDataError, "Cannot get buffered interactions for an empty definition." unless @csdl.length > 0
+
+			params = { 'hash' => self.hash }
+
+			if count
+				params['count'] = count
+			end
+
+			if from_id
+				params['interaction_id'] = from_id
+			end
+
+			retval = @user.callAPI('stream', params)
+
+			raise APIError, 'No data in the response' unless retval.has_key?('stream')
+
+			retval['stream']
+		end
+
+		# Returns a StreamConsumer-derived object for this definition, for the
+		# given type.
+		# === Parameters
+		#
+		# * +type+ - The consumer type for which to construct a consumer.
+		#
+		def getConsumer(type = nil, on_interaction = nil, on_stopped = nil)
+			StreamConsumer.factory(@user, type, self)
+		end
+	end
+end

data/lib/DataSift/exceptions.rb

@@ -0,0 +1,16 @@
+module DataSift
+	class AccessDeniedError < StandardError; end
+	class CompileFailedError < StandardError; end
+	class InvalidDataError < StandardError; end
+	class NotYetImplementedError < StandardError; end
+	class RateLimitExceededError < StandardError; end
+	class StreamError < StandardError; end
+
+	class APIError < StandardError
+		attr_reader :http_code
+
+		def initialize(http_code)
+			@http_code = http_code
+		end
+	end
+end

data/lib/DataSift/mockapiclient.rb

@@ -0,0 +1,55 @@
+#
+# mockapiclient.rb - This file contains the MockApiClient class.
+#
+# Copyright (C) 2011 MediaSift Ltd
+#
+# == Overview
+#
+# The MockApiClient class implements a fake DataSift API interface.
+
+module DataSift
+	# MockApiCLient class.
+	#
+	# == Introduction
+	#
+	# The ApiClient class implements a fake DataSift API interface.
+	#
+	class MockApiClient
+		# Set the response to be returned by the call method
+		# === Parameters
+		#
+		# * +code+ - The HTTP response code
+		# * +data+ - The dictionary that would have come from the response body
+		# * +rate_limit+ - The new rate_limit value
+		# * +rate_limit_remaining+ - The new rate_limit_remaining value
+		def setResponse(code, data, rate_limit, rate_limit_remaining)
+			@response = {
+				'response_code' => code,
+				'data' => data,
+				'rate_limit' => rate_limit,
+				'rate_limit_remaining' => rate_limit_remaining,
+			}
+		end
+
+		# Clear the response so we throw an exception if we get called again
+		# without a new response being set.
+		#
+		def clearResponse()
+			@response = false
+		end
+
+		# Fake a call to a DataSift API endpoint.
+		# === Parameters
+		#
+		# * +endpoint+ - The endpoint of the API call.
+		# * +params+ - The parameters to be passed along with the request.
+		# * +username+ - The username for the Auth header
+		# * +api_key+ - The API key for the Auth header
+		def call(username, api_key, endpoint, params = {}, user_agent = 'DataSiftPHP/0.0')
+			if !@response
+				raise StandardError, 'Expected response not set in mock object'
+			end
+			@response
+		end
+	end
+end

data/lib/DataSift/stream_consumer.rb

@@ -0,0 +1,124 @@
+#
+# stream_consumer.rb - This file contains the StreamConsumer class.
+#
+# Copyright (C) 2011 MediaSift Ltd
+#
+# == Overview
+#
+# The StreamConsumer class is base class for various stream consumers.
+
+module DataSift
+
+	# StreamConsumer class.
+	#
+	class StreamConsumer
+		TYPE_HTTP = 'HTTP'
+
+		STATE_STOPPED = 0
+		STATE_STARTING = 1
+		STATE_RUNNING = 2
+		STATE_STOPPING = 3
+
+		# Factory function. Creates a StreamConsumer-derived object for the given
+		# type.
+		# === Parameters
+		#
+		# * +type+ - Use the TYPE_ constants
+		# * +definition+ - CSDL string or a Definition object.
+		#
+		def self.factory(user, type, definition)
+			type ||= TYPE_HTTP
+			@klass = Module.const_get('DataSift').const_get('StreamConsumer_' + type)
+			@klass.new(user, definition)
+		end
+
+		attr_accessor :auto_reconnect
+		attr_reader :state, :stop_reason
+
+		# Constructor. Do not use this directly, use the factory method instead.
+		# === Parameters
+		#
+		# * +user+ - The user this consumer will run as.
+		# * +definition+ - CSDL string or a Definition object.
+		#
+		def initialize(user, definition)
+			raise InvalidDataError, 'Please supply a valid User object when creating a Definition object.' unless user.is_a? DataSift::User
+
+			if definition.is_a? String
+				@definition = user.createDefinition(definition)
+			elsif definition.is_a? Definition
+				@definition = definition
+			else
+				raise InvalidDataError, 'The definition must be a CSDL string or a DataSift_Definition object'
+			end
+
+			@user = user
+			@auto_reconnect = true
+			@stop_reason = 'Unknown reason'
+			@state = STATE_STOPPED
+
+			# Compile the definition to ensure it's valid for use
+			@definition.compile()
+		end
+
+		# This is called when the consumer is stopped.
+		# === Parameters
+		#
+		# * +reason+ - The reason why the consumer stopped.
+		#
+		def onStopped(&block)
+			if block_given?
+				@on_stopped = block
+				self
+			else
+				@on_stopped
+			end
+		end
+
+		# Once an instance of a StreamConsumer is ready for use, call this to
+		# start consuming. Extending classes should implement onStart to handle
+		# actually starting.
+		# === Parameters
+		#
+		# * +auto_reconnect+ - Whether the consumer should automatically reconnect.
+		# * +block+ - An optional block to receive incoming interactions.
+		#
+		def consume(auto_reconnect = true, &block)
+			@auto_reconnect = auto_reconnect;
+
+			# Start consuming
+			@state = STATE_STARTING
+			onStart(&block)
+		end
+
+		# Called when the consumer should start consuming the stream.
+		#
+		def onStart()
+			puts 'onStart method has not been overridden!'
+		end
+
+		# This method can be called at any time to *request* that the consumer
+		# stop consuming. This method sets the state to STATE_STOPPING and it's
+		# up to the consumer implementation to notice that this has changed, stop
+		# consuming and call the onStopped method.
+		#
+		def stop()
+			raise InvalidDataError, 'Consumer state must be RUNNING before it can be stopped' unless @state = StreamConsumer::STATE_RUNNING
+			@state = StreamConsumer::STATE_STOPPING
+		end
+
+		# Default implementation of onStop. It's unlikely that this method will
+		# ever be used in isolation, but rather it should be called as the final
+		# step in the extending class's implementation.
+		# === Parameters
+		#
+		# * +reason+ - The reason why the consumer stopped.
+		#
+		def onStop(reason = '')
+			reason = 'Unexpected' unless @state != StreamConsumer::STATE_STOPPING and reason.length == 0
+			@state = StreamConsumer::STATE_STOPPED
+			@stop_reason = reason
+			onStopped.call(reason) unless onStopped.nil?
+		end
+	end
+end

data/lib/DataSift/stream_consumer_http.rb

@@ -0,0 +1,160 @@
+#
+# stream_consumer_http.rb - This file contains the StreamConsumer_HTTP class.
+#
+# Copyright (C) 2011 MediaSift Ltd
+#
+# == Overview
+#
+# The StreamConsumer_HTTP class implements HTTP streaming.
+
+$LOAD_PATH.unshift(File.dirname(__FILE__) + '/../')
+
+require 'uri'
+require 'socket'
+require 'yajl'
+
+module DataSift
+
+	class StreamConsumer_HTTP < StreamConsumer
+
+		# Constructor. Requires valid user and definition objects.
+		def initialize(user, definition)
+			super
+		end
+
+		def onStart(&block)
+			begin
+				reconnect() unless !@socket.nil? and !@socket.closed?
+
+				parser = Yajl::Parser.new
+				parser.on_parse_complete = block if block_given?
+				if @response_head[:headers]["Transfer-Encoding"] == 'chunked'
+					if block_given?
+						chunkLeft = 0
+						while !@socket.eof? && (line = @socket.gets)
+							break if line.match /^0.*?\r\n/
+							next if line == "\r\n"
+							size = line.hex
+							json = @socket.read(size)
+							next if json.nil?
+							chunkLeft = size-json.size
+							if chunkLeft == 0
+								parser << json
+							else
+								# received only part of the chunk, grab the rest
+								parser << @socket.read(chunkLeft)
+							end
+						end
+					else
+						raise StreamError, 'Chunked responses detected, but no block given to handle the chunks.'
+					end
+				else
+					content_type = @response_head[:headers]['Content-Type'].split(';')
+					content_type = content_type.first
+					if ALLOWED_MIME_TYPES.include?(content_type)
+						case @response_head[:headers]['Content-Encoding']
+						when 'gzip'
+							return Yajl::Gzip::StreamReader.parse(@socket, opts, &block)
+						when 'deflate'
+							return Yajl::Deflate::StreamReader.parse(@socket, opts.merge({:deflate_options => -Zlib::MAX_WBITS}), &block)
+						when 'bzip2'
+							return Yajl::Bzip2::StreamReader.parse(@socket, opts, &block)
+						else
+							return parser.parse(@socket)
+						end
+					else
+						raise StreamError, 'Unhandled response MIME type ' + content_type
+					end
+				end
+			end while @auto_reconnect and @state == StreamConsumer::STATE_RUNNING
+
+			disconnect()
+
+			if @state == StreamConsumer::STATE_STOPPING
+				@stop_reason = 'Stop requested'
+			else
+				@stop_reason = 'Connection dropped'
+			end
+
+			onStop(@stop_reason)
+		end
+
+		def reconnect()
+			uri = URI.parse('http://' + User::STREAM_BASE_URL + @definition.hash +
+											'?username=' + CGI.escape(@user.username) + '&api_key=' + CGI.escape(@user.api_key))
+
+			user_agent = @user.getUserAgent()
+
+			request = "GET #{uri.path}#{uri.query ? "?"+uri.query : nil} HTTP/1.1\r\n"
+			request << "Host: #{uri.host}\r\n"
+			request << "User-Agent: #{user_agent}\r\n"
+			request << "Accept: */*\r\n"
+			request << "\r\n"
+
+			connection_delay = 0
+
+			begin
+				# Close the socket if it's open
+				disconnect()
+
+				# Back off a bit if required
+				sleep(connection_delay) if connection_delay > 0
+
+				begin
+					@socket = TCPSocket.new(uri.host, uri.port)
+
+					@socket.write(request)
+					@response_head = {}
+					@response_head[:headers] = {}
+
+					# Read the headers
+					@socket.each_line do |line|
+						if line == "\r\n" # end of the headers
+							break
+						else
+							header = line.split(": ")
+							if header.size == 1
+								header = header[0].split(" ")
+								@response_head[:version] = header[0]
+								@response_head[:code] = header[1].to_i
+								@response_head[:msg] = header[2]
+							else
+								@response_head[:headers][header[0]] = header[1].strip
+							end
+						end
+					end
+
+					if @response_head[:code] == 200
+						# Success!
+						@state = StreamConsumer::STATE_RUNNING
+					elsif @response_head[:code] == 404
+						raise StreamError, 'Hash not found!'
+					else
+						puts 'Connection failed: ' + @response_head[:code] + ' ' + @response_head[:msg]
+						if connection_delay == 0
+							connection_delay = 10;
+						elsif connection_delay < 240
+							connection_delay *= 2;
+						else
+							raise StreamError, 'Connection failed: ' + @response_head[:code] + ' ' + @response_head[:msg]
+						end
+					end
+				#rescue
+				#	if connection_delay == 0
+				#		connection_delay = 1
+				#	elsif connection_delay <= 16
+				#		connection_delay += 1
+				#	else
+				#		raise StreamError, 'Connection failed due to a network error'
+				#	end
+				end
+			end while @state != StreamConsumer::STATE_RUNNING
+		end
+
+		def disconnect()
+			@socket.close if !@socket.nil? and !@socket.closed?
+		end
+
+	end
+
+end