datasift 2.1.1 → 3.0.0.beta

data/lib/errors.rb

@@ -0,0 +1,59 @@
+class DataSiftError < StandardError
+  attr_reader :status
+  attr_reader :body
+
+  def initialize(http_status = nil, http_body = nil)
+    @status = http_status
+    @body   = http_body
+  end
+
+  def message
+    @body == nil ? @status : @body
+  end
+
+  def to_s
+    #if both body and status were provided then message is the body otherwise the status contains the message
+    msg           = !@body.nil? && !@status.nil? ? @body : @status
+    #if body is nil then status is the message body so no status is included
+    status_string = @body.nil? ? '' : "(Status #{@status}) "
+    "#{status_string} : #{msg}"
+  end
+end
+
+class NotSupportedError < DataSiftError
+end
+
+class BadRequestError < DataSiftError
+end
+
+class AuthError < DataSiftError
+end
+
+class ConnectionError < DataSiftError
+end
+
+class ApiResourceNotFoundError < DataSiftError
+end
+
+class InvalidConfigError < DataSiftError
+end
+
+class InvalidParamError < DataSiftError
+end
+
+class NotConnectedError < DataSiftError
+end
+
+class ReconnectTimeoutError < DataSiftError
+end
+
+class NotConfiguredError < DataSiftError
+end
+
+class InvalidTypeError < DataSiftError
+end
+
+class StreamingMessageError < DataSiftError
+end
+class WebSocketOnWindowsError < DataSiftError
+end

data/lib/historics.rb

@@ -0,0 +1,76 @@
+module DataSift
+  class Historics < DataSift::ApiResource
+
+    ##
+    # Create a new historics query and return its id.
+    def prepare (hash, start, end_time, name, sources = 'twitter', sample = 100)
+      params = {
+          :hash    => hash,
+          :start   => start,
+          :end     => end_time,
+          :name    => name,
+          :sources => sources,
+          :sample  => sample
+      }
+      requires params
+      DataSift.request(:POST, 'historics/prepare', @config, params)
+    end
+
+    ##
+    # Starts historics query.
+    def start(id)
+      params = {:id => id}
+      requires params
+      DataSift.request(:POST, 'historics/start', @config, params)
+    end
+
+    ##
+    # Stop historics query.
+    def stop(id)
+      params = {:id => id}
+      requires params
+      DataSift.request(:POST, 'historics/stop', @config, params)
+    end
+
+    ##
+    # Check the data coverage in the archive for a specified interval.
+    def status(start, end_time, sources='twitter')
+      params = {:start => start, :end => end_time, :sources => sources}
+      requires params
+      DataSift.request(:GET, 'historics/status', @config, params)
+    end
+
+    ##
+    # Update a historics query's name.
+    def update(id, name)
+      params = {:id => id, :name => name}
+      requires params
+      DataSift.request(:POST, 'historics/update', @config, params)
+    end
+
+    ##
+    # Delete a historics query.
+    def delete(id)
+      params = {:id => id}
+      requires params
+      DataSift.request(:DELETE, 'historics/delete', @config, params)
+    end
+
+    ##
+    # Get details for a given historics query.
+    def get_by_id(id, with_estimate = 1)
+      params = {:id => id, :with_estimate => with_estimate}
+      requires params
+      DataSift.request(:GET, 'historics/get', @config, params)
+    end
+
+    ##
+    # Get details for a set of historics within the given page constraints.
+    def get(max=20, page=1, with_estimate = 1)
+      params = {:max => max, :page => page, :with_estimate => with_estimate}
+      requires params
+      DataSift.request(:GET, 'historics/get', @config, params)
+    end
+
+  end
+end

data/lib/historics_preview.rb

@@ -0,0 +1,20 @@
+module DataSift
+  class HistoricsPreview < DataSift::ApiResource
+
+    def create(hash, parameters, start, end_time = nil)
+      params = {
+          :hash       => hash,
+          :parameters => parameters,
+          :start      => start
+      }
+      params.merge!(end_time) if end_time != nil
+
+      DataSift.request(:POST, 'preview/create', @config, params)
+    end
+
+    def get(id)
+      DataSift.request(:POST, 'preview/get', @config, {:id => id})
+    end
+
+  end
+end

data/lib/live_stream.rb

@@ -0,0 +1,53 @@
+module DataSift
+  class LiveStream < DataSift::ApiResource
+
+    @stream              = nil
+    @on_datasift_message = lambda {}
+
+    def initialize (config, stream)
+      @config        = config
+      @stream        = stream
+      @retry_timeout = 0
+      @subscriptions = {}
+      @connected     = false
+    end
+
+    attr_reader :connected, :stream, :retry_timeout, :subscriptions
+    attr_writer :connected, :retry_timeout, :on_datasift_message
+
+    def connected?
+      @connected
+    end
+
+    def fire_ds_message(message)
+      hash = false
+      if message.has_key?(:hash)
+        hash = message[:hash]
+      end
+      message.merge!({
+                         :is_failure => message[:status] == 'failure',
+                         :is_success => message[:status] == 'success',
+                         :is_warning => message[:status] == 'warning',
+                         :is_tick    => message[:status] == 'connected'
+                     })
+      @on_datasift_message.call(self, message, hash)
+    end
+
+    def fire_on_message(hash, interaction)
+      callback = @subscriptions[hash]
+      if callback == nil
+        raise StreamingMessageError.new "no valid on_message callback provided for stream #{hash} with message #{interaction}"
+      end
+      callback.call(interaction, self, hash)
+    end
+
+    def subscribe(hash, on_message)
+      @subscriptions[hash] = on_message
+      @stream.send "{ \"action\":\"subscribe\",\"hash\":\"#{hash}\"}"
+    end
+
+    def unsubscribe hash
+      @stream.send "{ \"action\":\"unsubscribe\",\"hash\":\"#{hash}\"}"
+    end
+  end
+end

data/lib/managed_source.rb

@@ -0,0 +1,57 @@
+module DataSift
+  class ManagedSource < DataSift::ApiResource
+
+    ##
+    # Creates a new managed source
+    #+source_type+:: can be facebook_page, googleplus, instagram or yammer
+    def create(source_type, name, parameters = {}, resources = [], auth = [])
+      params = {
+          :source_type => source_type,
+          :name        => name
+      }
+      params.merge!({:auth => MultiJson.dump(auth)}) if !auth.empty?
+      params.merge!({:parameters => MultiJson.dump(parameters)}) if !parameters.empty?
+      params.merge!({:resources => MultiJson.dump(resources)}) if resources.length > 0
+
+      DataSift.request(:POST, 'source/create', @config, params)
+    end
+
+    def update(id, source_type, name, parameters = {}, resources = [], auth = [])
+      params = {
+          :id          => id,
+          :source_type => source_type,
+          :name        => name
+      }
+      params.merge!({:auth => MultiJson.dump(auth)}) if !auth.empty?
+      params.merge!({:parameters => MultiJson.dump(parameters)}) if !parameters.empty?
+      params.merge!({:resources => MultiJson.dump(resources)}) if resources.length > 0
+
+      DataSift.request(:POST, 'source/update', @config, params)
+    end
+
+    def delete(id)
+      DataSift.request(:DELETE, 'source/delete', @config, {:id => id})
+    end
+
+    def stop(id)
+      DataSift.request(:POST, 'source/stop', @config, {:id => id})
+    end
+
+    def start(id)
+      DataSift.request(:POST, 'source/start', @config, {:id => id})
+    end
+
+    def get(id = nil, source_type = nil, page = 1, per_page = 20)
+      params = {:page => page, :per_page => per_page}
+      params.merge!({:id => id}) if id != nil
+      params.merge!({:source_type => source_type}) if source_type != nil
+
+      DataSift.request(:GET, 'source/get', @config, params)
+    end
+
+    def log(id, page = 1, per_page = 20)
+      DataSift.request(:POST, 'source/get', @config, {:id => id, :page => page, :per_page => per_page})
+    end
+
+  end
+end

data/lib/push.rb

@@ -0,0 +1,156 @@
+module DataSift
+  ##
+  # Push is a simple and robust mechanism for periodically delivering your data directly to a given destination
+  # Several widely adopted storage locations are available including Amazon S3, FTP, HTTP, SFTP, DynamoDB, CouchDB
+  # MongoDB, Splunk, Elastic search and more! See http://dev.datasift.com/docs/push/connectors
+  class Push < DataSift::ApiResource
+
+    ##
+    # Check that a subscription is defined correctly
+    def valid?(params)
+      requires params
+      res = DataSift.request(:POST, 'push/validate', @config, params)
+      res[:http][:status] == 200
+    end
+
+    ##
+    #Create a new subscription to a live stream or historics query
+    # Possible params are
+    # hash, historics_id, name, output_type, initial_status, start, end and output_params.*
+    # where output_params.* depends on the output_type for specific options see the documentation at
+    # http://dev.datasift.com/docs/rest-api/pushcreate
+    # For a list of available connectors see http://dev.datasift.com/docs/push/connectors
+    def create(params)
+      DataSift.request(:POST, 'push/create', @config, params)
+    end
+
+
+    ##
+    # Update the name or output parameters for an existing subscription
+    def update (params)
+      DataSift.request(:POST, 'push/update', @config, params)
+    end
+
+    ##
+    #Pause a subscription and buffer the data for up to an hour
+    def pause(id)
+      params = {:id => id}
+      requires params
+      DataSift.request(:POST, 'push/pause', @config, params)
+    end
+
+    ##
+    #Restart a job that was previously paused
+    def resume(id)
+      params = {:id => id}
+      requires params
+      DataSift.request(:POST, 'push/resume', @config, params)
+    end
+
+    ##
+    # Stop a historics query or a live stream that is running with push
+    def stop(id)
+      params = {:id => id}
+      requires params
+      DataSift.request(:POST, 'push/stop', @config, params)
+    end
+
+    ##
+    # Deletes an existing push subscription
+    def delete(id)
+      params = {:id => id}
+      requires params
+      DataSift.request(:DELETE, 'push/delete', @config, params)
+    end
+
+    ##
+    # Retrieve log messages for a specific subscription
+    def logs_for (id, page = 1, per_page = 20, order_by = :request_time, order_dir = :desc)
+      params = {
+          :id        => id,
+          :page      => page,
+          :per_page  => per_page,
+          :order_by  => order_by,
+          :order_dir => order_dir
+      }
+      DataSift.request(:GET, 'push/log', @config, params)
+    end
+
+    ##
+    # Retrieve log messages for all subscriptions
+    def logs (page = 1, per_page = 20, order_by = :request_time, order_dir = :desc)
+      params = {
+          :page      => page,
+          :per_page  => per_page,
+          :order_by  => order_by,
+          :order_dir => order_dir
+      }
+      DataSift.request(:GET, 'push/log', @config, params)
+    end
+
+    ##
+    # Get details of the subscription with the given ID
+    def get_by_subscription(id, page = 1, per_page = 20, order_by = :request_time, order_dir = :desc)
+      params = {
+          :id        => id,
+          :page      => page,
+          :per_page  => per_page,
+          :order_by  => order_by,
+          :order_dir => order_dir
+      }
+      DataSift.request(:GET, 'push/get', @config, params)
+    end
+
+    ##
+    # Get details of the subscription with the given stream ID/hash
+    def get_by_hash(hash, page = 1, per_page = 20, order_by = :request_time, order_dir = :desc)
+      params = {
+          :hash      => hash,
+          :page      => page,
+          :per_page  => per_page,
+          :order_by  => order_by,
+          :order_dir => order_dir
+      }
+      DataSift.request(:GET, 'push/get', @config, params)
+    end
+
+    ##
+    # Get details of the subscription with the given Historics ID
+    def get_by_historics_id(id, page = 1, per_page = 20, order_by = :request_time, order_dir = :desc)
+      params = {
+          :historics_id => id,
+          :page         => page,
+          :per_page     => per_page,
+          :order_by     => order_by,
+          :order_dir    => order_dir
+      }
+      DataSift.request(:GET, 'push/get', @config, params)
+    end
+
+    ##
+    # Get details of all subscriptions within the given page constraints
+    def get(page = 1, per_page = 20, order_by = :request_time, order_dir = :desc)
+      params = {
+          :page      => page,
+          :per_page  => per_page,
+          :order_by  => order_by,
+          :order_dir => order_dir
+      }
+      DataSift.request(:GET, 'push/get', @config, params)
+    end
+
+    ##
+    # Pull data from a 'pull' type Push Subscription
+    def pull(id, size = 20971520, cursor = '', callback = nil)
+      params = {
+          :id     => id,
+          :size   => size,
+          :cursor => cursor
+      }
+      if callback
+          params.merge!({:on_interaction => callback})
+      end
+      DataSift.request(:GET, 'pull', @config, params, {}, 30, 30, true)
+    end
+  end
+end

data/tests/core_api_test.rb

@@ -0,0 +1,42 @@
+require '../examples/auth'
+require 'test/unit'
+class CoreApiTest < Test::Unit::TestCase
+
+  def setup
+    auth      = DataSiftExample.new
+    @datasift = auth.datasift
+    @csdl     = 'interaction.content contains "test"'
+  end
+
+
+  def teardown
+    # Do nothing
+  end
+
+  def test_csdl_validation
+    assert_true @datasift.valid?(@csdl)
+    assert_raise_kind_of BadRequestError do
+      datasift.valid?(@csdl+' random string')
+    end
+  end
+
+  def test_compile
+    stream = @datasift.compile(@csdl)
+    hash_asserts(stream)
+  end
+
+  def test_dpus
+    stream = @datasift.compile(@csdl)
+    hash_asserts(stream)
+
+    dpus = @datasift.dpu stream[:data][:hash]
+    assert_not_nil dpus[:data][:dpu]
+    assert_not_nil dpus[:data][:detail]
+  end
+
+  def hash_asserts(stream)
+    assert_not_nil stream[:data]
+    assert_not_nil stream[:data][:hash]
+  end
+
+end