datasift 2.1.1 → 3.0.0.beta

data/examples/historics_preview_eg.rb

@@ -0,0 +1,30 @@
+require './auth'
+class HistoricsPreviewApi < DataSiftExample
+  def initialize
+    super
+    run
+  end
+
+  def run
+    begin
+      puts 'Creating hash'
+      stream = @datasift.compile 'interaction.content contains "datasift"'
+      hash   = stream[:data][:hash]
+
+      puts 'Creating a preview'
+      # see http://dev.datasift.com/docs/rest-api/previewcreate for docs
+      parameters = 'interaction.author.link,targetVol,hour;interaction.type,freqDist,10'
+      start      = Time.now.to_i - (3600 * 48) # 48hrs ago
+      source     = @datasift.historics_preview.create(hash, parameters, start)
+      puts source
+
+      puts 'Getting preview data'
+      puts @datasift.historics_preview.get source[:data][:id]
+
+    rescue DataSiftError => dse
+      puts dse.message
+    end
+  end
+end
+
+HistoricsPreviewApi.new

data/examples/live_stream_eg.rb

@@ -0,0 +1,89 @@
+require './auth'
+class StreamingApi < DataSiftExample
+
+  def initialize
+    super
+    @datasift = DataSift::Client.new(@config)
+    run
+  end
+
+  def run
+    begin
+      rubyReceived   = 0
+      pythonReceived = 0
+      ruby           = 'interaction.content contains "ruby"'
+      rubyStream     = @datasift.compile ruby
+
+      python       = 'interaction.content contains "python"'
+      pythonStream = @datasift.compile python
+
+      on_delete = lambda { |stream, m| puts 'We must delete this to be compliant ==> ' + m }
+
+      on_error = lambda do |stream, e|
+        puts 'A serious error has occurred'
+        puts e.message
+      end
+
+      on_message_ruby = lambda do |message, stream, hash|
+        rubyReceived += 1
+        puts "Ruby #{rubyReceived}, #{message}"
+
+        if rubyReceived >= 10
+          puts 'un-subscribing from ruby stream '+ hash
+          stream.unsubscribe hash
+        end
+      end
+
+      on_message_python = lambda do |message, stream, hash|
+        pythonReceived += 1
+        puts "python #{pythonReceived}, #{message}"
+
+        if pythonReceived >= 10
+          puts 'un-subscribing from python stream '+ hash
+          stream.unsubscribe hash
+        end
+      end
+
+      on_connect = lambda do |stream|
+        #
+        puts 'subscribing to python stream '+ pythonStream[:data][:hash]
+        stream.subscribe(pythonStream[:data][:hash], on_message_python)
+        puts 'Subscribed to '+ pythonStream[:data][:hash]
+        sleep 1
+        #
+        puts 'subscribing to ruby stream '+ rubyStream[:data][:hash]
+        stream.subscribe(rubyStream[:data][:hash], on_message_ruby)
+        puts 'Subscribed to '+ rubyStream[:data][:hash]
+      end
+
+      on_close = lambda do |stream|
+        puts 'closed'
+      end
+
+      on_datasift_message = lambda do |stream, message, hash|
+        #not all messages have a hash
+        puts "is_success =  #{message[:is_success]}, is_failure =  #{message[:is_failure]}, is_warning =  #{message[:is_warning]}, is_tick =  #{message[:is_tick]}"
+        puts "DataSift Message #{hash} ==> #{message}"
+      end
+
+      conn                     = DataSift::new_stream(@config, on_delete, on_error, on_connect, on_close)
+      conn.on_datasift_message = on_datasift_message
+      #can do something else here now...
+      puts 'Do some other business stuff...'
+      conn.stream.read_thread.join
+        #rescue DataSiftError
+    rescue DataSiftError => dse
+      puts "Error #{dse.message}"
+      # Then match specific one to take action - All errors thrown by the client extend DataSiftError
+      case dse
+        when ConnectionError
+          # some connection error
+        when AuthError
+        when BadRequestError
+        else
+          # do something else...
+      end
+    end
+  end
+end
+StreamingApi.new

data/examples/managed_source_eg.rb

@@ -0,0 +1,56 @@
+require './auth'
+class ManagedSourceApi < DataSiftExample
+  def initialize
+    super
+    run
+  end
+
+  def run
+    begin
+      puts 'Creating a managed source'
+      parameters = {:likes           => true,
+                    :posts_by_others => true,
+                    :comments        => true
+      }
+      resources  = [{
+                        :parameters => {
+                            :url   => 'http://www.facebook.com/thegaurdian',
+                            :title => 'Some news page',
+                            :id    => :thegaurdian
+                        }
+                    }]
+      auth       = [{
+                        :parameters => {
+                            :value => 'CAAIUKbXn8xsBAN5MnINICUT9gEBsZBh3hKoSEeIMP0ZA4zadMr64X6ljvZC4VBZCyYr9tyhih5nO0R39A1FQ848v0mZA6d3ehIHuSbKb7avtfLOtL5XKDYRIXHmRWreyxxVc3jk7CIa4ZCI5AAKeUUO3GUS8EaPdYVh9rO5FvvNmIatzz6k8el'
+                        }
+                    }]
+
+      source = @datasift.managed_source.create('facebook_page', 'My managed source', parameters, resources, auth)
+      puts source
+
+      id = source[:data][:id]
+
+      puts 'Starting delivery for my private source'
+      puts @datasift.managed_source.start id
+
+      puts 'Updating'
+      puts @datasift.managed_source.update(id, 'facebook_page', 'Updated source', parameters, resources, auth)
+
+      puts 'Getting info from DataSift about my page'
+      puts @datasift.managed_source.get id
+
+      puts 'Fetching logs'
+      puts @datasift.managed_source.log id
+
+      puts 'Stopping'
+      puts @datasift.managed_source.stop id
+
+      puts 'Deleting'
+      puts @datasift.managed_source.delete id
+    rescue DataSiftError => dse
+      puts dse.message
+    end
+  end
+end
+
+ManagedSourceApi.new

data/examples/pull.rb

@@ -0,0 +1,44 @@
+require './auth'
+class PushApi < DataSiftExample
+  def initialize
+    super
+  end
+
+  def run
+    begin
+      @params = {:output_type => 'pull'}
+      puts 'Validating'
+      if @datasift.push.valid? @params
+        stream = @datasift.compile 'interaction.content contains "music"'
+        subscription = create_push(stream[:data][:hash])
+
+        subscription_id = subscription[:data][:id]
+        # Pull interactions from the push queue - this will only work if we have set
+        # the Push Subscription output_type above to 'pull'
+
+        puts 'Waiting...'
+        sleep 10
+
+        # Passing a lambda is more efficient because it is executed once for each interaction received
+        # this saves having to iterate over the array returned so the same iteration isn't done twice
+        puts 'Pulling'
+        @datasift.push.pull(subscription_id, 20971520, '', lambda{ |e| puts "on_message => #{e}" })
+
+        puts 'Waiting...'
+        sleep 10
+        puts 'Pulling'
+        @datasift.push.pull(subscription_id, 20971520, '', lambda{ |e| puts "on_message => #{e}" })
+
+        puts 'Stop Subscription'
+        @datasift.push.stop subscription_id
+
+        puts 'Delete Subscription'
+        @datasift.push.delete subscription_id
+      end
+    rescue DataSiftError => dse
+      puts dse
+    end
+  end
+
+end
+PushApi.new().run

data/examples/push_eg.rb

@@ -0,0 +1,56 @@
+require './auth'
+class PushApi < DataSiftExample
+  def initialize
+    super
+  end
+
+  def run
+    begin
+      puts 'Validating'
+      if @datasift.push.valid? @params
+        stream       = @datasift.compile 'interaction.content contains "datasift"'
+        subscription = create_push(stream[:data][:hash])
+
+        subscription_id = subscription[:data][:id]
+        #pull a bunch of interactions from the push queue - only work if we had set the output_type above to pull
+        #pull @datasift.pull subscription_id
+
+        puts 'updating subscription'
+        # update the info we just used to create
+        # id, name and output_params.* are valid
+        puts @datasift.push.update @params.merge({:id => subscription_id, :name => 'My updated awesome name'})
+
+        puts 'getting subscription info'
+        # get details for a subscription also available are
+        # push.[get, get_by_hash,get_by_historics_id]
+        puts @datasift.push.get_by_subscription subscription_id
+
+        puts 'getting logs for subscription'
+        # get log messages for a subscription id
+        #also available push.logs to fetch logs for all subscriptions
+        puts @datasift.push.logs_for subscription_id
+
+        puts 'pausing subscription'
+        #pause the subscription that was created
+        puts @datasift.push.pause subscription_id
+
+        puts 'resuming subscription'
+        # resume the subscription that was just paused
+        puts @datasift.push.resume subscription_id
+
+        puts 'stopping subscription'
+        # stop the subscription
+        puts @datasift.push.stop subscription_id
+
+        puts 'deleting subscription'
+        #and delete it
+        puts @datasift.push.delete subscription_id
+      end
+        #rescue DataSiftError
+    rescue DataSiftError => dse
+      puts dse.message
+    end
+  end
+
+end
+PushApi.new().run

data/lib/api/api_resource.rb

@@ -0,0 +1,23 @@
+module DataSift
+  class ApiResource
+    include DataSift
+
+    def initialize (config)
+      @config = config
+      config[:api_host] = 'api.datasift.com' unless config.has_key?(:api_host)
+      config[:stream_host] = 'websocket.datasift.com' unless config.has_key?(:stream_host)
+      config[:api_version] = 'v1' unless config.has_key?(:api_version)
+      config[:enable_ssl] = true unless config.has_key?(:enable_ssl)
+      # max 320 seconds retry - http://dev.datasift.com/docs/streaming-api/reconnecting
+      config[:max_retry_time] = 320 unless config.has_key?(:max_retry_time)
+    end
+
+    def requires params
+      params.each { |k, v|
+        if v == nil || v.to_s.length == 0
+          raise InvalidParamError.new "#{k} is a required parameter, it cannot be nil or empty"
+        end
+      }
+    end
+  end
+end

data/lib/datasift.rb

@@ -1,15 +1,288 @@
-#This is the base file for the DataSift API library. Require this file to get
-#access to the full library functionality.
-require 'rubygems'
-
 dir = File.dirname(__FILE__)
-require dir + '/DataSift/exceptions'
-require dir + '/DataSift/apiclient'
-require dir + '/DataSift/user'
-require dir + '/DataSift/definition'
-require dir + '/DataSift/historic'
-require dir + '/DataSift/push_definition'
-require dir + '/DataSift/push_subscription'
-require dir + '/DataSift/stream_consumer'
-require dir + '/DataSift/stream_consumer_http'
-require dir + '/DataSift/managed_source'
+
+require 'uri'
+require 'rest_client'
+require 'multi_json'
+require 'websocket_td'
+
+require dir + '/api/api_resource'
+
+require dir + '/errors'
+require dir + '/push'
+require dir + '/historics'
+require dir + '/historics_preview'
+require dir + '/managed_source'
+require dir + '/live_stream'
+
+require 'rbconfig'
+
+module DataSift
+  #
+  IS_WINDOWS = (RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/)
+  VERSION    = File.open('../VERSION').first
+
+  class Client < ApiResource
+
+    #+config+:: A hash containing configuration options for the client for e.g.
+    # {:username => 'some_user', :api_key => 'ds_api_key', :enable_ssl => true, :open_timeout => 30, :timeout => 30}
+    def initialize (config)
+      if config == nil
+        raise InvalidConfigError.new ('Config cannot be nil')
+      end
+      if config.key?(:api_key) == false || config.key?(:username) == false
+        raise InvalidConfigError.new('A valid username and API key are required')
+      end
+
+      @config            = config
+      @historics         = DataSift::Historics.new(config)
+      @push              = DataSift::Push.new(config)
+      @managed_source    = DataSift::ManagedSource.new(config)
+      @historics_preview = DataSift::HistoricsPreview.new(config)
+    end
+
+    attr_reader :historics, :push, :managed_source, :historics_preview
+
+    ##
+    # Checks if the syntax of the given CSDL is valid
+    def valid?(csdl)
+      requires({:csdl => csdl})
+      res= DataSift.request(:POST, 'validate', @config, {:csdl => csdl})
+      res[:http][:status] == 200
+    end
+
+    ##
+    # Compile CSDL code.
+    #+csdl+:: The CSDL you wish to compile
+    def compile(csdl)
+      requires({:csdl => csdl})
+      DataSift.request(:POST, 'compile', @config, {:csdl => csdl})
+    end
+
+    ##
+    # Check the number of objects processed and delivered for a given time period.
+    #+period+:: Can be "day", "hour", or "current", defaults to hour
+    def usage(period = :hour)
+      DataSift.request(:POST, 'usage', @config, {:period => period})
+    end
+
+    ##
+    # Calculate the DPU cost of consuming a stream.
+    def dpu(hash)
+      requires ({:hash => hash})
+      DataSift.request(:POST, 'dpu', @config, {:hash => hash})
+    end
+
+    ##
+    # Determine your credit balance or DPU balance.
+    def balance
+      DataSift.request(:POST, 'balance', @config, {})
+    end
+
+    ##
+    # Collect a batch of interactions from a push queue
+    def pull(id, size = 20971520, cursor='')
+      DataSift.request(:POST, 'pull', @config, {:id => id, :size => size, :cursor => cursor})
+    end
+
+  end
+
+
+  # Generates and executes an HTTP request from the params provided
+  # Params:
+  # +method+:: the HTTP method to use e.g. GET,POST
+  # +path+:: the DataSift path relevant to the base URL of the API
+  # +username+:: API username
+  # +api_key+:: DS api key
+  # +params+:: A hash representing the params to use in the request, if it's a get,head or delete request these params
+  # are used as query string params, if not they become form url encoded params
+  # +headers+:: any headers to pass to the API, Authorization header is automatically included
+  def self.request(method, path, config, params = {}, headers = {}, timeout=30, open_timeout=30, new_line_separated=false)
+    validate config
+    options = {}
+    url     = build_url(path, config)
+    case method.to_s.downcase.to_sym
+      when :get, :head, :delete
+        url     += "#{URI.parse(url).query ? '&' : '?'}#{encode params}"
+        payload = nil
+      else
+        payload = encode params
+    end
+
+    headers.update ({
+        :user_agent    => "DataSift/#{config[:api_version]} Ruby/v#{VERSION}",
+        :authorization => "#{config[:username]}:#{config[:api_key]}",
+        :content_type  => 'application/x-www-form-urlencoded'
+    })
+
+    options.update(
+        :headers      => headers,
+        :method       => method,
+        :open_timeout => open_timeout,
+        :timeout      => timeout,
+        :payload      => payload,
+        :url          => url
+    )
+
+    begin
+      response = RestClient::Request.execute options
+      if response != nil && response.length > 0
+        if new_line_separated
+          res_arr = response.split("\n")
+          data = []
+          res_arr.each { |e|
+            interaction = MultiJson.load(e, :symbolize_keys => true)
+            data.push(interaction)
+            if params.has_key? :on_interaction
+              params[:on_interaction].call(interaction)
+            end
+          }
+        else
+          data = MultiJson.load(response, :symbolize_keys => true)
+        end
+      else
+        data = {}
+      end
+      {
+          :data     => data,
+          :datasift => {
+              :x_ratelimit_limit     => response.headers[:x_ratelimit_limit],
+              :x_ratelimit_remaining => response.headers[:x_ratelimit_remaining],
+              :x_ratelimit_cost      => response.headers[:x_ratelimit_cost]
+          },
+          :http     => {
+              :status  => response.code,
+              :headers => response.headers
+          }
+      }
+    rescue MultiJson::DecodeError => de
+      raise DataSiftError.new response
+    rescue SocketError => e
+      process_client_error(e)
+    rescue RestClient::ExceptionWithResponse => e
+      begin
+        code = e.http_code
+        body = e.http_body
+        if code && body
+          error = MultiJson.load(body)
+          handle_api_error(e.http_code, error['error'] + " for URL #{url}")
+        else
+          process_client_error(e)
+        end
+      rescue MultiJson::DecodeError
+        process_client_error(e)
+      end
+    rescue RestClient::Exception, Errno::ECONNREFUSED => e
+      process_client_error (e)
+    end
+  end
+
+  def self.build_url(path, config)
+    'http' + (config[:enable_ssl] ? 's' : '') + '://' + config[:api_host] + '/' + config[:api_version] + '/' + path
+  end
+
+  #returns true if username and api key are set
+  def self.is_invalid? config
+    !config.key?(:username) || !config.key?(:api_key)
+  end
+
+  def self.validate conf
+    if is_invalid? conf
+      raise InvalidConfigError.new 'A username and api_key are required'
+    end
+  end
+
+  def self.encode params
+    URI.escape(params.collect { |k, v| "#{k}=#{v}" }.join('&'))
+  end
+
+  def self.handle_api_error(code, body)
+    case code
+      when 400
+        raise BadRequestError.new(code, body)
+      when 401
+        raise AuthError.new(code, body)
+      when 404
+        raise ApiResourceNotFoundError.new(code, body)
+      else
+        raise DataSiftError.new(code, body)
+    end
+  end
+
+  def self.process_client_error(e)
+    case e
+      when RestClient::ServerBrokeConnection, RestClient::RequestTimeout
+        message = 'Unable to connect to DataSift. Please check your connection and try again'
+      when RestClient::SSLCertificateNotVerified
+        message = 'Failed to complete SSL verification'
+      when SocketError
+        message = 'Communication with DataSift failed. Are you able to resolve api.datasift.com?'
+      else
+        message = 'Unexpected error.'
+    end
+    raise ConnectionError.new(message + " (Network error: #{e.message})")
+  end
+
+  ##
+  # a Proc/lambda callback to receive delete messages
+  # DataSift and its customers are required to process Twitter's delete request, a delete handler must be provided
+  # a Proc/lambda callback to receive errors
+  # Because EventMachine is used errors can be raised from another thread, this method will receive any such errors
+  def self.new_stream(config, on_delete, on_error, on_open = nil, on_close = nil)
+    if on_delete == nil || on_error == nil
+      raise NotConfiguredError.new 'on_delete and on_error are required before you can connect'
+    end
+
+    begin
+      stream     = WebsocketTD::Websocket.new('websocket.datasift.com', '/multi', "username=#{config[:username]}&api_key=#{config[:api_key]}")
+      connection = LiveStream.new(config, stream)
+
+      stream.on_open = lambda {
+        connection.connected     = true
+        connection.retry_timeout = 0
+        on_open.call(connection) if on_open != nil
+      }
+
+      stream.on_close   = lambda {
+        connection.connected = false
+        retry_connect(config, connection, on_delete, on_error, on_open, on_close, '', true)
+      }
+      stream.on_error   = lambda {
+        connection.connected = false
+        on_error.call(connection) if on_close != nil
+        retry_connect(config, connection, on_delete, on_error, on_open, on_close)
+      }
+      stream.on_message =lambda { |msg|
+        data = MultiJson.load(msg.data, :symbolize_keys => true)
+        if data.has_key?(:deleted)
+          on_delete.call(connection, data)
+        elsif data.has_key?(:status)
+          connection.fire_ds_message(data)
+        else
+          connection.fire_on_message(data[:hash], data[:data])
+        end
+      }
+    rescue Exception => e
+      retry_connect(config, connection, on_delete, on_error, on_open, on_close, e.message)
+    end
+    connection
+  end
+
+  def self.retry_connect(config, connection, on_delete, on_error, on_open, on_close, message = '', use_closed = false)
+    connection.retry_timeout = connection.retry_timeout == 0 ? 10 : connection.retry_timeout * 2
+    if connection.retry_timeout > config[:max_retry_time]
+      if use_closed && on_close != nil
+        on_close.call(connection)
+      else
+        on_error.call ReconnectTimeoutError.new "Connecting to DataSift has failed, re-connection was attempted but
+                                         multiple consecutive failures where encountered. As a result no further
+                                         re-connection will be automatically attempted. Manually invoke connect() after
+                                          investigating the cause of the failure, be sure to observe DataSift's
+                                          re-connect policies available at http://dev.datasift.com/docs/streaming-api/reconnecting
+                                          - Error { #{message}}"
+      end
+    else
+      sleep connection.retry_timeout
+      new_stream(config, on_delete, on_error, on_open, on_close)
+    end
+  end
+end