datasift 3.0.0.beta2 → 3.0.0.beta4

data/examples/live_stream_eg.rb

@@ -9,13 +9,13 @@ class StreamingApi < DataSiftExample
 
   def run
     begin
-      rubyReceived   = 0
-      pythonReceived = 0
+      ruby_received   = 0
+      python_received = 0
       ruby           = 'interaction.content contains "ruby"'
-      rubyStream     = @datasift.compile ruby
+      ruby_stream     = @datasift.compile ruby
 
       python       = 'interaction.content contains "python"'
-      pythonStream = @datasift.compile python
+      python_stream = @datasift.compile python
 
       on_delete = lambda { |stream, m| puts 'We must delete this to be compliant ==> ' + m }
 
@@ -25,20 +25,20 @@ class StreamingApi < DataSiftExample
       end
 
       on_message_ruby = lambda do |message, stream, hash|
-        rubyReceived += 1
-        puts "Ruby #{rubyReceived}, #{message}"
+        ruby_received += 1
+        puts "Ruby #{ruby_received}, #{message}"
 
-        if rubyReceived >= 10
+        if ruby_received >= 10
           puts 'un-subscribing from ruby stream '+ hash
           stream.unsubscribe hash
         end
       end
 
       on_message_python = lambda do |message, stream, hash|
-        pythonReceived += 1
-        puts "python #{pythonReceived}, #{message}"
+        python_received += 1
+        puts "python #{python_received}, #{message}"
 
-        if pythonReceived >= 10
+        if python_received >= 10
           puts 'un-subscribing from python stream '+ hash
           stream.unsubscribe hash
         end
@@ -46,18 +46,18 @@ class StreamingApi < DataSiftExample
 
       on_connect = lambda do |stream|
         #
-        puts 'subscribing to python stream '+ pythonStream[:data][:hash]
-        stream.subscribe(pythonStream[:data][:hash], on_message_python)
-        puts 'Subscribed to '+ pythonStream[:data][:hash]
+        puts 'subscribing to python stream '+ python_stream[:data][:hash]
+        stream.subscribe(python_stream[:data][:hash], on_message_python)
+        puts 'Subscribed to '+ python_stream[:data][:hash]
         sleep 1
         #
-        puts 'subscribing to ruby stream '+ rubyStream[:data][:hash]
-        stream.subscribe(rubyStream[:data][:hash], on_message_ruby)
-        puts 'Subscribed to '+ rubyStream[:data][:hash]
+        puts 'subscribing to ruby stream '+ ruby_stream[:data][:hash]
+        stream.subscribe(ruby_stream[:data][:hash], on_message_ruby)
+        puts 'Subscribed to '+ ruby_stream[:data][:hash]
       end
 
-      on_close = lambda do |stream|
-        puts 'closed'
+      on_close = lambda do |stream,msg|
+        puts msg
       end
 
       on_datasift_message = lambda do |stream, message, hash|

data/examples/pull.rb

@@ -13,30 +13,30 @@ class PushApi < DataSiftExample
         subscription = create_push(stream[:data][:hash])
 
         subscription_id = subscription[:data][:id]
-        # Pull interactions from the push queue - this will only work if we have set
-        # the Push Subscription output_type above to 'pull'
+        #pull a bunch of interactions from the push queue - only work if we had set the output_type above to pull
+        #pull @datasift.pull subscription_id
+
+        puts 'pullinga'
+        @datasift.push.pull(subscription_id).each { |e| puts e }
 
-        puts 'Waiting...'
         sleep 10
 
-        # Passing a lambda is more efficient because it is executed once for each interaction received
-        # this saves having to iterate over the array returned so the same iteration isn't done twice
-        puts 'Pulling'
-        @datasift.push.pull(subscription_id, 20971520, '', lambda{ |e| puts "on_message => #{e}" })
+        puts 'pullingb'
+        @datasift.push.pull(subscription_id).each { |e| puts e }
 
-        puts 'Waiting...'
         sleep 10
-        puts 'Pulling'
-        @datasift.push.pull(subscription_id, 20971520, '', lambda{ |e| puts "on_message => #{e}" })
 
-        puts 'Stop Subscription'
-        @datasift.push.stop subscription_id
+        puts 'pullingc'
+        #passing a lambda is more efficient because it is executed once for each interaction received
+        #this saves having to iterate over the array returned so the same iteration isn't done twice
+        @datasift.push.pull(subscription_id,20971520,'', lambda{ |e| puts "on_message => #{e}" })
 
-        puts 'Delete Subscription'
+        puts 'pullingdelete'
         @datasift.push.delete subscription_id
       end
+        #rescue DataSiftError
     rescue DataSiftError => dse
-      puts dse
+      puts dse.inspect
     end
   end
 

data/examples/push_eg.rb

@@ -4,53 +4,30 @@ class PushApi < DataSiftExample
     super
   end
 
-  def run
+  def run(count)
     begin
-      puts 'Validating'
-      if @datasift.push.valid? @params
-        stream       = @datasift.compile 'interaction.content contains "datasift"'
-        subscription = create_push(stream[:data][:hash])
+      subscription = create_push('5cdb0c8b4f3f6ca26f6ba1b086f22edd', count)
 
-        subscription_id = subscription[:data][:id]
-        #pull a bunch of interactions from the push queue - only work if we had set the output_type above to pull
-        #pull @datasift.pull subscription_id
+      subscription_id = subscription[:data][:id]
+      #pull a bunch of interactions from the push queue - only work if we had set the output_type above to pull
+      #pull @datasift.pull subscription_id
 
-        puts 'updating subscription'
-        # update the info we just used to create
-        # id, name and output_params.* are valid
-        puts @datasift.push.update @params.merge({:id => subscription_id, :name => 'My updated awesome name'})
-
-        puts 'getting subscription info'
-        # get details for a subscription also available are
-        # push.[get, get_by_hash,get_by_historics_id]
-        puts @datasift.push.get_by_subscription subscription_id
-
-        puts 'getting logs for subscription'
-        # get log messages for a subscription id
-        #also available push.logs to fetch logs for all subscriptions
-        puts @datasift.push.logs_for subscription_id
-
-        puts 'pausing subscription'
-        #pause the subscription that was created
-        puts @datasift.push.pause subscription_id
-
-        puts 'resuming subscription'
-        # resume the subscription that was just paused
-        puts @datasift.push.resume subscription_id
-
-        puts 'stopping subscription'
-        # stop the subscription
-        puts @datasift.push.stop subscription_id
-
-        puts 'deleting subscription'
-        #and delete it
-        puts @datasift.push.delete subscription_id
-      end
-        #rescue DataSiftError
+      puts 'getting subscription info'
+      # get details for a subscription also available are
+      # push.[get, get_by_hash,get_by_historics_id]
+      puts @datasift.push.get_by_subscription subscription_id
     rescue DataSiftError => dse
       puts dse.message
     end
   end
 
+  def get_all
+    puts MultiJson.dump(@datasift.push.get(1, 500))
+  end
 end
-PushApi.new().run
+
+p = PushApi.new()
+#for i in 1..1000
+#  p.run(i)
+#end
+p.get_all()

data/lib/api/api_resource.rb

@@ -7,7 +7,7 @@ module DataSift
       config[:api_host] = 'api.datasift.com' unless config.has_key?(:api_host)
       config[:stream_host] = 'websocket.datasift.com' unless config.has_key?(:stream_host)
       config[:api_version] = 'v1' unless config.has_key?(:api_version)
-      config[:enable_ssl] = true unless config.has_key?(:enable_ssl)
+      config[:enable_ssl] = false unless config.has_key?(:enable_ssl)
       # max 320 seconds retry - http://dev.datasift.com/docs/streaming-api/reconnecting
       config[:max_retry_time] = 320 unless config.has_key?(:max_retry_time)
     end
@@ -20,4 +20,4 @@ module DataSift
       }
     end
   end
-end
+end

data/lib/cli.rb

@@ -0,0 +1,233 @@
+require 'optparse'
+require 'optparse/time'
+require 'ostruct'
+require 'pp'
+require 'multi_json'
+require_relative '../lib/datasift'
+
+def to_output(r)
+  MultiJson.dump({
+                     :status => r[:http][:status],
+                     :headers => r[:http][:headers],
+                     :body => r[:data]
+                 },
+                 :pretty => true)
+end
+
+def opt(val, default)
+  val ? val : default
+end
+
+def err(m)
+  puts MultiJson.dump({:error => m})
+end
+
+def parse(args)
+  options = OpenStruct.new
+  options.auth = nil
+  options.endpoint = 'core'
+  options.command = nil
+  options.params = {}
+  options.api = 'api.datasift.com'
+
+  opt_parser = OptionParser.new do |opts|
+    opts.banner = 'Usage: cli.rb [-c] [--api] -a -e [-p*]'
+    opts.separator 'Specific options:'
+
+    opts.on('-a', '--auth AUTH', 'DataSift username:api_key') do |username|
+      api_key = ARGV.length>0 && ARGV[0].index('-') == 0 ? '' : ARGV[0]
+      if username == nil || api_key == nil || username.empty? || api_key.empty?
+        err 'Unable to parse username and API key, they must be in the format username api_key'
+        err parse(%w(-h))
+        exit
+      end
+      options.auth = {:username => username, :api_key => api_key}
+    end
+
+    opts.on('-e', '--endpoint ENDPOINT', 'Defaults to core, must be one of core,push,historics,preview,sources') do |e|
+      options.endpoint = e
+    end
+
+    opts.on('-c', '--command COMMAND', 'DataSift endpoint, depends on the endpoint') do |e|
+      options.command = e|| 'core'
+    end
+
+    opts.on('-p', '--param PARAM', 'Command specific parameters e.g. -p name value') do |k|
+      # value is ARGV[0] unless ARGV[0] starts with a hyphen
+      options.params[k] = ARGV.length>0 && ARGV[0].index('-') == 0 ? '' : ARGV[0]
+    end
+
+    opts.on('-u', '--url API_HOSTNAME', 'Override the API URL') do |e|
+      options.api = e
+    end
+
+    opts.on_tail('-h', '--help', 'Show this message') do
+      err opts
+      exit
+    end
+
+    opts.on_tail('--version', 'Show version') do
+      err :: DataSift::VERSION
+      exit
+    end
+  end
+
+  opt_parser.parse!(args)
+  options #.marshal_dump
+end
+
+def run_core_command (c, command, p)
+  case command
+    when 'validate'
+      c.valid?(p['csdl'], false)
+    when 'compile'
+      c.compile(p['csdl'])
+    when 'usage'
+      c.usage(usage = p['period'] ? p['period'].to_sym : :hour)
+    when 'balance'
+      c.balance
+    when 'dpu'
+      c.dpu(p['hash'])
+    else
+      err 'Unknown command for the core endpoint'
+      exit
+  end
+end
+
+def run_historics_command (c, command, p)
+  case command
+    when 'prepare'
+      c.historics.prepare(p['hash'], p['start'], p['end'], p['name'], opt(p['sources'], 'twitter'), opt(p['sample'], 10))
+    when 'start'
+      c.historics.start(p['id'])
+    when 'stop'
+      c.historics.stop(p['id'], opt(p['reason'], ''))
+    when 'status'
+      c.historics.status(p['start'], p['end'], opt(p['sources'], 'twitter'))
+    when 'update'
+      c.historics.update(p['id'], p['name'])
+    when 'delete'
+      c.historics.delete(p['id'])
+    when 'get'
+      c.historics.get(opt(p['max'], 20), opt(p['page'], 1), opt(p['with_estimate'], 1))
+    else
+      err 'Unknown command for the historics endpoint'
+      exit
+  end
+end
+
+def run_preview_command (c, command, p)
+  case command
+    when 'create'
+      c.historics_preview.create(p['hash'], p['parameters'], p['start'], opt(p['end'], nil))
+    when 'get'
+      c.historics_preview.get(p['id'])
+    else
+      err 'Unknown command for the historics preview endpoint'
+      exit
+  end
+end
+
+def run_sources_command (c, command, p)
+  case command
+    when 'create'
+      c.managed_source.create(p['source_type'],p['name'], opt(p['parameters'], {}),
+                              opt(p['resources'], []), opt(p['auth'], []))
+    when 'update'
+      c.managed_source.update(p['id'], p['source_type'], p['name'], opt(p['parameters'], {}),
+                              opt(p['resources'], []),
+                              opt(p['auth'], []))
+    when 'delete'
+      c.managed_source.delete(p['id'])
+    when 'stop'
+      c.managed_source.stop(p['id'])
+    when 'start'
+      c.managed_source.start(p['id'])
+    when 'log'
+      c.managed_source.log(p['id'], opt(p['page'], 1), opt(p['per_page'], 20))
+    when 'get'
+      c.managed_source.get(opt(p['id'], nil), opt(p['source_type'], nil), opt(p['page'], 1), opt(p['per_page'], 20))
+    else
+      err 'Unknown command for the historics preview endpoint'
+      exit
+  end
+end
+
+def run_push_command (c, command, p)
+  case command
+    when 'validate'
+      c.push.valid? p, false
+    when 'create'
+      c.push.create p
+    when 'pause'
+      c.push.pause p['id']
+    when 'resume'
+      c.push.resume p['id']
+    when 'update'
+      c.push.update p
+    when 'stop'
+      c.push.stop p['id']
+    when 'delete'
+      c.push.delete p['id']
+    when 'log'
+      p['id'] ?
+          c.push.logs_for(p['id'], opt(p['page'], 0), opt(p['per_page'], 20), opt(p['order_by'], :request_time), opt(p['order_dir'], :desc)) :
+          c.push.logs(opt(p['page'], 0), opt(p['per_page'], 20), opt(p['order_by'], :request_time), opt(p['order_dir'], :desc))
+    when 'get'
+      if p['id']
+        c.push.get_by_subscription(p['id'], opt(p['page'], 0), opt(p['per_page'], 20), opt(p['order_by'], :request_time))
+      elsif p['hash']
+        c.push.get_by_hash(p['hash'], opt(p['page'], 0), opt(p['per_page'], 20), opt(p['order_by'], :request_time), opt(p['order_dir'], :desc))
+      elsif p['playback_id']
+        c.push.get_by_historics_id(p['playback_id'], opt(p['page'], 0), opt(p['per_page'], 20), opt(p['order_by'], :request_time), opt(p['order_dir'], :desc))
+      else
+        c.push.get(opt(p['page'], 0), opt(p['per_page'], 20), opt(p['order_by'], :request_time), opt(p['order_dir'], :desc))
+      end
+    when 'pull'
+      c.push.pull(p['id'], opt(p['size'], 20971520), opt(p['cursor'], ''))
+    else
+      err 'Unknown command for the core endpoint'
+      exit
+  end
+end
+
+begin
+  options = parse(ARGV)
+  req = [:auth, :command]
+  missing = req.select { |param| options.send(param).nil? }
+  unless missing.empty?
+    err "The following options are required : #{missing.join(', ')}"
+    err parse(%w(-h))
+    exit
+  end
+  config =
+      {
+          :username => options.auth[:username],
+          :api_key => options.auth[:api_key],
+          :api_host => options.api
+      }
+  datasift = DataSift::Client.new(config)
+
+  res = case options.endpoint
+          when 'core'
+            run_core_command(datasift, options.command, options.params)
+          when 'historics'
+            run_historics_command(datasift, options.command, options.params)
+          when 'push'
+            run_push_command(datasift, options.command, options.params)
+          when 'preview'
+            run_preview_command(datasift, options.command, options.params)
+          when 'managed_sources'
+            run_sources_command(datasift, options.command, options.params)
+          else
+            err 'Unsupported/Unknown endpoint'
+            exit
+        end
+  puts to_output(res)
+rescue DataSiftError => e
+  err e.message
+rescue OptionParser::InvalidOption, OptionParser::MissingArgument
+  err $!.to_s
+  err parse(%w(-h))
+  exit
+end

data/lib/datasift.rb

@@ -1,30 +1,45 @@
 dir = File.dirname(__FILE__)
-
+#
 require 'uri'
 require 'rest_client'
 require 'multi_json'
 require 'websocket_td'
-
+#
 require dir + '/api/api_resource'
-
+#
 require dir + '/errors'
 require dir + '/push'
 require dir + '/historics'
 require dir + '/historics_preview'
 require dir + '/managed_source'
 require dir + '/live_stream'
-
+#
 require 'rbconfig'
 
 module DataSift
   #
-  IS_WINDOWS = (RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/)
-  VERSION    = File.open(File.dirname(__FILE__) + '/../VERSION').first
+  IS_WINDOWS              = (RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/)
+  VERSION                 = File.open(File.join(File.dirname(__FILE__), '../') + '/VERSION').first
+  KNOWN_SOCKETS           = {}
+  DETECT_DEAD_SOCKETS     = true
+  SOCKET_DETECTOR_TIMEOUT = 6.5
+
+  Thread.new do
+    while DETECT_DEAD_SOCKETS
+      now = Time.now.to_i
+      KNOWN_SOCKETS.clone.map { |connection, last_time|
+        if now - last_time > SOCKET_DETECTOR_TIMEOUT
+          connection.stream.reconnect
+        end
+      }
+      sleep SOCKET_DETECTOR_TIMEOUT * 10
+    end
+  end
 
   class Client < ApiResource
 
     #+config+:: A hash containing configuration options for the client for e.g.
-    # {:username => 'some_user', :api_key => 'ds_api_key', :enable_ssl => true, :open_timeout => 30, :timeout => 30}
+    # {username => 'some_user', api_key => 'ds_api_key', open_timeout => 30, timeout => 30}
     def initialize (config)
       if config == nil
         raise InvalidConfigError.new ('Config cannot be nil')
@@ -44,10 +59,12 @@ module DataSift
 
     ##
     # Checks if the syntax of the given CSDL is valid
-    def valid?(csdl)
+    #+boolResponse+ If true then a boolean is returned indicating whether the CSDL is valid, otherwise
+    # the response object itself is returned
+    def valid?(csdl, boolResponse = true)
       requires({:csdl => csdl})
       res= DataSift.request(:POST, 'validate', @config, {:csdl => csdl})
-      res[:http][:status] == 200
+      boolResponse ? res[:http][:status] == 200 : res
     end
 
     ##
@@ -120,7 +137,8 @@ module DataSift
         :open_timeout => open_timeout,
         :timeout      => timeout,
         :payload      => payload,
-        :url          => url
+        :url          => url,
+        :ssl_version => 'TLSv1'
     )
 
     begin
@@ -128,7 +146,7 @@ module DataSift
       if response != nil && response.length > 0
         if new_line_separated
           res_arr = response.split("\n")
-          data = []
+          data    = []
           res_arr.each { |e|
             interaction = MultiJson.load(e, :symbolize_keys => true)
             data.push(interaction)
@@ -164,7 +182,7 @@ module DataSift
         body = e.http_body
         if code && body
           error = MultiJson.load(body)
-          handle_api_error(e.http_code, error['error'] + " for URL #{url}")
+          handle_api_error(e.http_code, (error['error'] ? error['error'] : '') + " for URL #{url}")
         else
           process_client_error(e)
         end
@@ -215,7 +233,7 @@ module DataSift
       when RestClient::SSLCertificateNotVerified
         message = 'Failed to complete SSL verification'
       when SocketError
-        message = 'Communication with DataSift failed. Are you able to resolve api.datasift.com?'
+        message = 'Communication with DataSift failed. Are you able to resolve the API hostname?'
       else
         message = 'Unexpected error.'
     end
@@ -231,58 +249,76 @@ module DataSift
     if on_delete == nil || on_error == nil
       raise NotConfiguredError.new 'on_delete and on_error are required before you can connect'
     end
-
+    raise BadParametersError.new('on_delete - 2 parameter required') unless on_delete.arity == 2
+    raise BadParametersError.new('on_error - 2 parameter required') unless on_error.arity == 2
+    if on_open != nil
+      raise BadParametersError.new('on_open - 1 parameter required') unless on_open.arity == 1
+    end
+    if on_close != nil
+      raise BadParametersError.new('on_close - 2 parameter required') unless on_close.arity == 2
+    end
     begin
-      stream     = WebsocketTD::Websocket.new('websocket.datasift.com', '/multi', "username=#{config[:username]}&api_key=#{config[:api_key]}")
-      connection = LiveStream.new(config, stream)
-
-      stream.on_open = lambda {
+      stream                    = WebsocketTD::Websocket.new('websocket.datasift.com', '/multi', "username=#{config[:username]}&api_key=#{config[:api_key]}")
+      connection                = LiveStream.new(config, stream)
+      KNOWN_SOCKETS[connection] = Time.new.to_i
+      stream.on_ping            = lambda { |data|
+        KNOWN_SOCKETS[connection] = Time.new.to_i
+      }
+      stream.on_open            =lambda {
         connection.connected     = true
         connection.retry_timeout = 0
         on_open.call(connection) if on_open != nil
       }
 
-      stream.on_close   = lambda {
+      stream.on_close  =lambda { |message|
         connection.connected = false
-        retry_connect(config, connection, on_delete, on_error, on_open, on_close, '', true)
+        retry_connect(config, connection, on_delete, on_error, on_open, on_close, message, true)
       }
-      stream.on_error   = lambda {
+      stream.on_error  =lambda { |message|
         connection.connected = false
-        on_error.call(connection) if on_close != nil
-        retry_connect(config, connection, on_delete, on_error, on_open, on_close)
+        retry_connect(config, connection, on_delete, on_error, on_open, on_close, message)
       }
-      stream.on_message =lambda { |msg|
-        data = MultiJson.load(msg.data, :symbolize_keys => true)
+      stream.on_message=lambda { |msg|
+        data                      = MultiJson.load(msg.data, :symbolize_keys => true)
+        KNOWN_SOCKETS[connection] = Time.new.to_i
         if data.has_key?(:deleted)
           on_delete.call(connection, data)
         elsif data.has_key?(:status)
           connection.fire_ds_message(data)
+        elsif data.has_key?(:reconnect)
+          connection.stream.reconnect
         else
           connection.fire_on_message(data[:hash], data[:data])
         end
       }
     rescue Exception => e
-      retry_connect(config, connection, on_delete, on_error, on_open, on_close, e.message)
+      case e
+        when DataSiftError, ArgumentError
+          raise e
+        else
+          retry_connect(config, connection, on_delete, on_error, on_open, on_close, e.message)
+      end
     end
     connection
   end
 
   def self.retry_connect(config, connection, on_delete, on_error, on_open, on_close, message = '', use_closed = false)
-    connection.retry_timeout = connection.retry_timeout == 0 ? 10 : connection.retry_timeout * 2
-    if connection.retry_timeout > config[:max_retry_time]
+    config[:retry_timeout]   = config[:retry_timeout] == 0 || config[:retry_timeout] == nil ? 10 : config[:retry_timeout] * 2
+    connection.retry_timeout = config[:retry_timeout]
+    if config[:retry_timeout] > config[:max_retry_time]
       if use_closed && on_close != nil
-        on_close.call(connection)
+        on_close.call(connection, message)
       else
-        on_error.call ReconnectTimeoutError.new "Connecting to DataSift has failed, re-connection was attempted but
+        on_error.call(connection, ReconnectTimeoutError.new("Connecting to DataSift has failed, re-connection was attempted but
                                          multiple consecutive failures where encountered. As a result no further
                                          re-connection will be automatically attempted. Manually invoke connect() after
                                           investigating the cause of the failure, be sure to observe DataSift's
                                           re-connect policies available at http://dev.datasift.com/docs/streaming-api/reconnecting
-                                          - Error { #{message}}"
+                                          - Error { #{message}}"))
       end
     else
-      sleep connection.retry_timeout
+      sleep config[:retry_timeout]
       new_stream(config, on_delete, on_error, on_open, on_close)
     end
   end
-end
+end