datasift 2.1.1 → 3.0.0.beta

data/lib/DataSift/stream_consumer.rb

@@ -1,166 +0,0 @@
-module DataSift
-	#This is the base class for all StreamConsumer implementation.
-	class StreamConsumer
-		#Constant for the HTTP StreamConsumer implementation option.
-		TYPE_HTTP = 'HTTP'
-
-		#Constant for the "stopped" status.
-		STATE_STOPPED = 0
-		#Constant for the "starting" status.
-		STATE_STARTING = 1
-		#Constant for the "running" status.
-		STATE_RUNNING = 2
-		#Constant for the "stopping" status.
-		STATE_STOPPING = 3
-
-		#Factory function. Creates a StreamConsumer-derived object for the given
-		#type.
-		#=== Parameters
-		#* +type+ - Use the TYPE_ constants
-		#* +definition+ - CSDL string or a Definition object.
-		#=== Returns
-		#A StreamConsumer-derived object.
-		def self.factory(user, type, definition)
-			type ||= TYPE_HTTP
-			@klass = Module.const_get('DataSift').const_get('StreamConsumer_' + type)
-			@klass.new(user, definition)
-		end
-
-		#Whether the consumer should automatically try to reconnect if the
-		#connection is dropped.
-		attr_accessor :auto_reconnect
-		#The current state of the consumer.
-		attr_reader :state
-		#The reason the consumer was stopped.
-		attr_reader :stop_reason
-
-		#Constructor. Do not use this directly, use the factory method instead.
-		#=== Parameters
-		#* +user+ - The user this consumer will run as.
-		#* +definition+ - CSDL string or a Definition object.
-		def initialize(user, definition)
-			raise InvalidDataError, 'Please supply a valid User object when creating a Definition object.' unless user.is_a? DataSift::User
-
-			if definition.is_a? String
-				@definition = user.createDefinition(definition)
-			elsif definition.is_a? Definition
-				@definition = definition
-			else
-				raise InvalidDataError, 'The definition must be a CSDL string or a DataSift_Definition object'
-			end
-
-			@user = user
-			@auto_reconnect = true
-			@stop_reason = 'Unknown reason'
-			@state = STATE_STOPPED
-			@stream_timeout = 65
-
-			# Get the hash which will compile the CSDL if necessary
-			@definition.hash
-		end
-
-		#Called when a deletion notification is received.
-		#=== Parameters
-		#* +interaction+ - Minimal details about the interaction that was deleted.
-		def onDeleted(&block)
-			if block_given?
-				@on_deleted = block
-				self
-			else
-				@on_deleted
-			end
-		end
-
-		#This is called when an error message is received.
-		#=== Parameters
-		#* +message+ - The error message.
-		def onError(&block)
-			if block_given?
-				@on_error = block
-				self
-			else
-				@on_error
-			end
-		end
-
-		#This is called when an error message is received.
-		#=== Parameters
-		#* +message+ - The error message.
-		def onWarning(&block)
-			if block_given?
-				@on_warning = block
-				self
-			else
-				@on_warning
-			end
-		end
-
-		#This is called when the consumer is stopped.
-		#=== Parameters
-		#* +reason+ - The reason why the consumer stopped.
-		def onStopped(&block)
-			if block_given?
-				@on_stopped = block
-				self
-			else
-				@on_stopped
-			end
-		end
-
-		#Once an instance of a StreamConsumer is ready for use, call this to
-		#start consuming. Extending classes should implement onStart to handle
-		#actually starting.
-		#=== Parameters
-		#* +auto_reconnect+ - Whether the consumer should automatically reconnect.
-		#* +block+ - An optional block to receive incoming interactions.
-		def consume(auto_reconnect = true, &block)
-			@auto_reconnect = auto_reconnect;
-
-			# Start consuming
-			@state = STATE_STARTING
-			onStart do |interaction|
-				if interaction.has_key?('status')
-					if interaction['status'] == 'error' || interaction['status'] == 'failure'
-						onError.call(interaction['message'])
-					elsif interaction['status'] == 'warning'
-						onWarning.call(interaction['message'])
-					else
-						# Tick
-					end
-				else
-					if interaction.has_key?('deleted') and interaction['deleted']
-						onDeleted.call(interaction) unless onDeleted.nil?
-					else
-						block.call(interaction) unless block.nil?
-					end
-				end
-			end
-		end
-
-		#Called when the consumer should start consuming the stream.
-		def onStart()
-			abort('onStart method has not been overridden!')
-		end
-
-		#This method can be called at any time to *request* that the consumer
-		#stop consuming. This method sets the state to STATE_STOPPING and it's
-		#up to the consumer implementation to notice that this has changed, stop
-		#consuming and call the onStopped method.
-		def stop()
-			raise InvalidDataError, 'Consumer state must be RUNNING before it can be stopped' unless @state = StreamConsumer::STATE_RUNNING
-			@state = StreamConsumer::STATE_STOPPING
-		end
-
-		#Default implementation of onStop. It's unlikely that this method will
-		#ever be used in isolation, but rather it should be called as the final
-		#step in the extending class's implementation.
-		#=== Parameters
-		#* +reason+ - The reason why the consumer stopped.
-		def onStop(reason = '')
-			reason = 'Unexpected' unless @state != StreamConsumer::STATE_STOPPING and reason.length == 0
-			@state = StreamConsumer::STATE_STOPPED
-			@stop_reason = reason
-			onStopped.call(reason) unless onStopped.nil?
-		end
-	end
-end

data/lib/DataSift/stream_consumer_http.rb

@@ -1,188 +0,0 @@
-$LOAD_PATH.unshift(File.dirname(__FILE__) + '/../')
-
-require 'uri'
-require 'socket'
-require 'yajl'
-require 'cgi'
-
-module DataSift
-	#The HTTP implementation of the StreamConsumer.
-	class StreamConsumer_HTTP < StreamConsumer
-		#Constructor. Requires valid user and definition objects.
-		#=== Parameters
-		#* +user+ - The user consuming the data.
-		#* +definition+ - The Definition to consume.
-		def initialize(user, definition)
-			super
-		end
-
-		#Called when the consumer is started.
-		#=== Parameters
-		#* +block+ - A block to receive incoming data.
-		def onStart(&block)
-			begin
-				reconnect() unless !@socket.nil? and !@socket.closed?
-				parser = Yajl::Parser.new
-				parser.on_parse_complete = block if block_given?
-				if @response_head[:headers]["Transfer-Encoding"] == 'chunked'
-					if block_given?
-						chunkLeft = 0
-						while !@socket.eof? && (line = @socket.gets) && @state == StreamConsumer::STATE_RUNNING
-							break if line.match /^0.*?\r\n/
-
-							begin
-								ready = IO.select([@socket], nil, [@socket], @stream_timeout)
-								unless ready
-									reconnect()
-									next
-								end
-							end
-
-							next if line == "\r\n"
-							size = line.hex
-							json = @socket.read(size)
-							next if json.nil?
-							chunkLeft = size-json.size
-							if chunkLeft == 0
-								if json.length > 100
-									parser << json
-								end
-							else
-								# received only part of the chunk, grab the rest
-								received_data = @socket.read(chunkLeft)
-								if not received_data.nil?
-									parser << received_data
-								end
-							end
-						end
-					else
-						raise StreamError, 'Chunked responses detected, but no block given to handle the chunks.'
-					end
-				else
-					content_type = @response_head[:headers]['Content-Type'].split(';')
-					content_type = content_type.first
-					if ALLOWED_MIME_TYPES.include?(content_type)
-						case @response_head[:headers]['Content-Encoding']
-						when 'gzip'
-							return Yajl::Gzip::StreamReader.parse(@socket, opts, &block)
-						when 'deflate'
-							return Yajl::Deflate::StreamReader.parse(@socket, opts.merge({:deflate_options => -Zlib::MAX_WBITS}), &block)
-						when 'bzip2'
-							return Yajl::Bzip2::StreamReader.parse(@socket, opts, &block)
-						else
-							return parser.parse(@socket)
-						end
-					else
-						raise StreamError, 'Unhandled response MIME type ' + content_type
-					end
-				end
-			end while @auto_reconnect and @state == StreamConsumer::STATE_RUNNING
-
-			disconnect()
-
-			if @state == StreamConsumer::STATE_STOPPING
-				@stop_reason = 'Stop requested'
-			else
-				@stop_reason = 'Connection dropped'
-			end
-
-			onStop(@stop_reason)
-		end
-
-  private
-  	#Reconnect the stream socket.
-		def reconnect()
-			uri = URI.parse('http' + (@user.use_ssl ? 's' : '') + '://' + User::STREAM_BASE_URL + @definition.hash)
-
-			user_agent = @user.getUserAgent()
-
-			request = "GET #{uri.path}#{uri.query ? "?"+uri.query : nil} HTTP/1.1\r\n"
-			request << "Host: #{uri.host}\r\n"
-			request << "User-Agent: #{user_agent}\r\n"
-			request << "Accept: */*\r\n"
-			request << "Auth: #{@user.username}:#{@user.api_key}\r\n"
-			request << "\r\n"
-
-			connection_delay = 0
-
-			begin
-				# Close the socket if it's open
-				disconnect()
-
-				# Back off a bit if required
-				sleep(connection_delay) if connection_delay > 0
-
-				begin
-					@raw_socket = TCPSocket.new(uri.host, uri.port)
-					if @user.use_ssl
-						@socket = OpenSSL::SSL::SSLSocket.new(@raw_socket)
-						@socket.connect
-					else
-						@socket = @raw_socket
-					end
-
-					@socket.write(request)
-					@response_head = {}
-					@response_head[:headers] = {}
-
-					# Read the headers
-					@socket.each_line do |line|
-						if line == "\r\n" # end of the headers
-							break
-						else
-							header = line.split(": ")
-							if header.size == 1
-								header = header[0].split(" ")
-								@response_head[:version] = header[0]
-								@response_head[:code] = header[1].to_i
-								@response_head[:msg] = header[2]
-							else
-								@response_head[:headers][header[0]] = header[1].strip
-							end
-						end
-					end
-
-					if @response_head[:code].nil?
-						raise StreamError, 'Socket connection refused'
-					elsif @response_head[:code] == 200
-						# Success!
-						@state = StreamConsumer::STATE_RUNNING
-					elsif @response_head[:code] >= 399 && @response_head[:code] < 500 && @response_head[:code] != 420
-						line = ''
-						while !@socket.eof? && line.length < 10
-							line = @socket.gets
-						end
-						data = Yajl::Parser.parse(line)
-						if data.has_key?('message')
-							raise StreamError, data['message']
-						else
-							raise StreamError, 'Connection refused: ' + @response_head[:code] + ' ' + @response_head[:msg]
-						end
-					else
-						if connection_delay == 0
-							connection_delay = 10;
-						elsif connection_delay < 240
-							connection_delay *= 2;
-						else
-							raise StreamError, 'Connection refused: ' + @response_head[:code] + ' ' + @response_head[:msg]
-						end
-					end
-				rescue
-					if connection_delay == 0
-						connection_delay = 1
-					elsif connection_delay <= 16
-						connection_delay += 1
-					else
-						raise StreamError, 'Connection failed due to a network error'
-					end
-				end
-			end while @state != StreamConsumer::STATE_RUNNING
-		end
-
-		#Disconnect the stream socket.
-		def disconnect()
-			@socket.close if !@socket.nil? and !@socket.closed?
-			@raw_socket.close if !@raw_socket.nil? and !@raw_socket.closed?
-		end
-	end
-end

data/lib/DataSift/user.rb

@@ -1,311 +0,0 @@
-#This is the official DataSift client library for Ruby.
-module DataSift
-  #The User class represents a user of the API. Applications should start their
-  #API interactions by creating an instance of this class. Once initialised it
-  #provides factory methods for all of the functionality in the API.
-  class User
-    #The user agent to pass through with all HTTP requests.
-    USER_AGENT = 'DataSiftRuby/' + File.open(File.dirname(File.dirname(File.dirname(__FILE__))) + '/VERSION').first.strip!;
-    #The base URL for API requests.
-    API_BASE_URL = 'api.datasift.com/';
-    #The base URL for streams.
-    STREAM_BASE_URL = 'stream.datasift.com/';
-
-    #The User's DataSift username.
-    attr_reader :username
-    #The User's DataSift API key.
-    attr_reader :api_key
-    #The User's total number of available hourly API credits. This is not
-    #populated until an API request is made.
-    attr_reader :rate_limit
-    #The User's API credits remaining. This is not populated until an API
-    #request is made.
-    attr_reader :rate_limit_remaining
-    #The APIClient class to use when making API requests.
-    attr_reader :api_client
-    #True if streaming connections should use SSL.
-    attr_reader :use_ssl
-
-    #Constructor. A username and API key are required when constructing an
-    #instance of this class.
-    #=== Parameters
-    #* +username+ - The User's DataSift username
-    #* +api_key+ - The User's DataSift API key
-    def initialize(username, api_key, use_ssl = true)
-      username = username.strip
-      api_key = api_key.strip
-
-      raise EInvalidData, 'Please supply valid credentials when creating a User object.' unless username.size > 0 and api_key.size > 0
-
-      @username = username
-      @api_key = api_key
-      @rate_limit = -1;
-      @rate_limit_remaining = -1
-      @use_ssl = use_ssl
-    end
-
-    #Creates and returns a definition object.
-    #=== Parameters
-    #* +csdl+ - Optional CSDL string with which to prime the object.
-    #=== Returns
-    #A Definition object.
-    def createDefinition(csdl = '')
-      DataSift::Definition.new(self, csdl, false)
-    end
-
-    #Create a Historics query based on this Definition.
-    #=== Parameters
-    #* +hash+ - The stream hash for a new Historics query.
-    #* +start_date+ - The start date for a new Historics query.
-    #* +end_date+ - The end date for a new Historics query.
-    #* +sources+ - An array of sources for a new Historics query.
-    #* +name+ - The name for a new Historics query.
-    #* +sample+ - The sample rate for the new Historics query.
-    #=== Returns
-    #A Historic object.
-    def createHistoric(hash, start_date, end_date, sources, sample, name)
-      return Historic.new(self, hash, start_date, end_date, sources, sample, name)
-    end
-
-    #Get a Historics query from the API.
-    #=== Parameters
-    #* +playback_id+ - The playback ID of the Historics query to retrieve.
-    #=== Returns
-    #A Historic object.
-    def getHistoric(playback_id)
-      return Historic.new(self, playback_id)
-    end
-
-    # Get a list of Historics queries in your account.
-    #=== Parameters
-    #* +page+ - The page number to get.
-    #* +per_page+ - The number of items per page.
-    #=== Returns
-    #A Hash containing...
-    #* +count+ - The total number of Historics queries in your account.
-    #* +historics+ - An array of Hashes where each Hash is a Historics query.
-    def listHistorics(page = 1, per_page = 20)
-      return Historic::list(self, page, per_page)
-    end
-
-
-    #Create a Managed Source query based on this Definition.
-    #=== Options
-    #* +hash+ - Contains parameters to set up your Managed Source
-    #=== Returns
-    #A Managed Source object.
-    def createManagedSource(hash = {})
-      managed_source = ManagedSource.new(self, hash)
-      managed_source.create
-      return managed_source
-    end
-
-    #Get a Manged Source query from the API.
-    #=== Parameters
-    #* +managed_source_id+ - The ID of the Managed Source to retrieve.
-    #=== Returns
-    #A Managed Source object.
-    def getManagedSource(managed_source_id)
-      return ManagedSource.get(self, managed_source_id)
-    end
-
-    # Get a list of ManagedSources in your account.
-    #=== Parameters
-    #* +page+ - The page number to get.
-    #* +per_page+ - The number of items per page.
-    #=== Returns
-    #A Hash containing...
-    #* +count+ - The total number of Managed Sources in your account.
-    #* +managed_sources+ - An array of Hashes where each Hash is a managed source.
-    def listManagedSources(page = 1, per_page = 20, source_type = '')
-      return ManagedSource::list(self, page, per_page, source_type)
-    end
-
-    #Get the log entries for all Managed Sources or the given Managed Source.
-    #=== Parameters
-    #* +managed_source_id+ - Optional Managed Source ID.
-    #=== Returns
-    #A Hash containing...
-    #* +count+ - The total number of matching log entries.
-    #* +log_entries+ - An array of Hashes where each Hash is a log entry.
-    def getManagedSourcesLog(managed_source_id, page = 1, per_page = 20)
-      return getManagedSource(managed_source_id).getLogs(page = 1, per_page = 20)
-    end
-
-    #Create a new PushDefinition object for this user.
-    #=== Returns
-    #A PushDefinition object.
-    def createPushDefinition()
-      return PushDefinition.new(self)
-    end
-
-    #Get an existing PushSubscription from the API.
-    #=== Parameters
-    #* +subscription_id+ - The ID of the subscription to fetch.
-    #=== Returns
-    #A PushSubscription object.
-    def getPushSubscription(subscription_id)
-      return PushSubscription.get(self, subscription_id)
-    end
-
-    #Get the log entries for all push subscription or the given subscription.
-    #=== Parameters
-    #* +subscription_id+ - Optional subscription ID.
-    #=== Returns
-    #A Hash containing...
-    #* +count+ - The total number of matching log entries.
-    #* +log_entries+ - An array of Hashes where each Hash is a log entry.
-    def getPushSubscriptionLog(subscription_id = false)
-      if subscription_id
-        return getPushSubscription(subscription_id).getLog()
-      else
-        return PushSubscription.getLogs(self)
-      end
-    end
-
-    #Get a page of Push subscriptions in the given user's account, where each
-    #page contains up to per_page items. Results will be ordered according to
-    #the supplied ordering parameters.
-    #=== Parameters
-    #* +page+ - The page number to get.
-    #* +per_page+ - The number of items per page.
-    #* +order_by+ - The field by which to order the results.
-    #* +order_dir+ - Ascending or descending.
-    #* +include_finished+ - True to include subscriptions against finished Historics queries.
-    #=== Returns
-    #A Hash containing...
-    #* +count+ - The total number of matching Push subscriptions in your account.
-    #* +subscriptions+ - An array of Hashes where each Hash is a Push subscription.
-    def listPushSubscriptions(page = 1, per_page = 20, order_by = PushSubscription::ORDERBY_CREATED_AT, order_dir = PushSubscription::ORDERDIR_ASC, include_finished = false)
-      return PushSubscription.list(self, page, per_page, order_by, order_dir, include_finished)
-    end
-
-    #Get a page of Push subscriptions in the given user's account, where each
-    #page contains up to per_page items. Results will be ordered according to
-    #the supplied ordering parameters.
-    #=== Parameters
-    #* +hash+ - The stream hash.
-    #* +page+ - The page number to get.
-    #* +per_page+ - The number of items per page.
-    #* +order_by+ - The field by which to order the results.
-    #* +order_dir+ - Ascending or descending.
-    #* +include_finished+ - True to include subscriptions against finished Historics queries.
-    #=== Returns
-    #A Hash containing...
-    #* +count+ - The total number of matching Push subscriptions in your account.
-    #* +subscriptions+ - An array of Hashes where each Hash is a Push subscription.
-    def listPushSubscriptionsToStreamHash(hash, page = 1, per_page = 20, order_by = PushSubscription::ORDERBY_CREATED_AT, order_dir = PushSubscription::ORDERDIR_ASC, include_finished = false)
-      return PushSubscription.listByStreamHash(self, hash, page, per_page, order_by, order_dir)
-    end
-
-    #Get a page of Push subscriptions in the given user's account, where each
-    #page contains up to per_page items. Results will be ordered according to
-    #the supplied ordering parameters.
-    #=== Parameters
-    #* +hash+ - The stream hash.
-    #* +page+ - The page number to get.
-    #* +per_page+ - The number of items per page.
-    #* +order_by+ - The field by which to order the results.
-    #* +order_dir+ - Ascending or descending.
-    #* +include_finished+ - True to include subscriptions against finished Historics queries.
-    #=== Returns
-    #A Hash containing...
-    #* +count+ - The total number of matching Push subscriptions in your account.
-    #* +subscriptions+ - An array of Hashes where each Hash is a Push subscription.
-    def listPushSubscriptionsToPlaybackId(playback_id, page = 1, per_page = 20, order_by = PushSubscription::ORDERBY_CREATED_AT, order_dir = PushSubscription::ORDERDIR_ASC, include_finished = false)
-      return PushSubscription.listByPlaybackId(self, playback_id, page, per_page, order_by, order_dir)
-    end
-
-    #Returns a StreamConsumer-derived object for the given hash, for the
-    #given type.
-    #=== Parameters
-    #* +type+ - The consumer type for which to construct a consumer.
-    #* +hash+ - The hash to be consumed.
-    #=== Returns
-    #A StreamConsumer-derived object.
-    def getConsumer(type = nil, hash = nil, on_interaction = nil, on_stopped = nil)
-      StreamConsumer.factory(self, type, Definition.new(self, nil, hash))
-    end
-
-    #Returns the account balance information for this user.
-    #=== Returns
-    #A Hash containing the balance information.
-    def getBalance
-      return callAPI('balance')['balance']
-    end
-
-    #Returns the usage data for this user. If a hash is provided then a more
-    #detailed breakdown using interaction types is retrieved and returned.
-    #=== Parameters
-    #* +period+ - An optional period for which to fetch data ('hour' or 'day')
-    #=== Returns
-    #A Hash containing the usage information.
-    def getUsage(period = 'hour')
-      params = { 'period' => period }
-
-      return callAPI('usage', params)
-    end
-
-    #Returns the user agent this library should use for all API calls.
-    #=== Returns
-    #The user agent string.
-    def getUserAgent()
-      return USER_AGENT
-    end
-
-    #Sets the ApiClient object to use to access the API
-    #=== Parameters
-    #* +client+ - The API client object to be used.
-    def setApiClient(client)
-      @api_client = client
-    end
-
-    #Sets whether to use SSL for API and stream communication.
-    #=== Parameters
-    #* +use_ssl+ - Pass true to use SSL.
-    def enableSSL(use_ssl = true)
-      @use_ssl = use_ssl
-    end
-
-    #Make a call to a DataSift API endpoint.
-    #=== Parameters
-    #* +endpoint+ - The endpoint of the API call.
-    #* +params+ - A Hash of parameters to be passed along with the request.
-    #=== Returns
-    #A Hash containing the response data.
-    def callAPI(endpoint, params = {})
-      if !@api_client
-        @api_client = ApiClient.new()
-      end
-
-      res = @api_client.call(self, endpoint, params, getUserAgent())
-
-      # Set up the return value
-      retval = res['data']
-
-      # Update the rate limits from the headers
-      @rate_limit = res['rate_limit']
-      @rate_limit_remaining = res['rate_limit_remaining']
-
-      case res['response_code']
-      when 200
-      when 201
-      when 202
-      when 204
-        # Do nothing
-      when 401
-        # Authentication failure
-        raise AccessDeniedError, retval.has_key?('error') ? retval['error'] : 'Authentication failed'
-      when 403
-        # Check the rate limit
-        raise RateLimitExceededError, retval['comment'] if @rate_limit_remaining == 0
-        # Rate limit is ok, raise a generic exception
-        raise APIError.new(res['response_code']), retval.has_key?('error') ? retval['error'] : 'Unknown error'
-      else
-        raise APIError.new(res['response_code']), retval.has_key?('error') ? retval['error'] : 'Unknown error'
-      end
-
-      return retval
-    end
-  end
-end