datasift 1.5.0 → 2.0.0

data/examples/push/env.rb

@@ -0,0 +1,53 @@
+# This class is used by the Push examples to remove the noise of
+# dealing with command line arguments.
+#
+
+# Include the DataSift library
+require File.dirname(__FILE__) + '/../../lib/datasift'
+
+class Env
+	attr_reader :user, :args
+
+	def initialize(args = false)
+		if args === false
+			args = ARGV
+		end
+
+		abort('Please specify your DataSift username and API key as the first two command line arguments!') unless args.size() >= 2
+
+		username = args.shift
+		api_key = args.shift
+		@user = DataSift::User.new(username, api_key)
+
+		@args = args
+	end
+
+	def displaySubscriptionDetails(subscription)
+		puts 'ID:           ' + subscription.id
+		puts 'Name:         ' + subscription.name
+		puts 'Status:       ' + subscription.status
+		puts 'Created at:   ' + subscription.created_at.strftime('%Y-%m-%d %H:%M:%S')
+		puts 'Last request: ' + (subscription.last_request.nil? ? 'None' : subscription.last_request.strftime('%Y-%m-%d %H:%M:%S'))
+		puts 'Last success: ' + (subscription.last_success.nil? ? 'None' : subscription.last_success.strftime('%Y-%m-%d %H:%M:%S'))
+		puts 'Output type:  ' + subscription.output_type
+
+		puts 'Output Params:'
+		subscription.output_params.each do |k,v|
+			puts '  ' + k + ' = ' + String(v)
+		end
+		puts '--'
+	end
+
+	def displayHistoricDetails(historic)
+		puts 'Playback ID: ' + historic.hash
+		puts 'Stream hash: ' + historic.stream_hash
+		puts 'Name:        ' + historic.name
+		puts 'Start time:  ' + historic.start_date.strftime('%Y-%m-%d %H:%M:%S')
+		puts 'End time:    ' + historic.end_date.strftime('%Y-%m-%d %H:%M:%S')
+		puts 'Sources:     ' + historic.sources.join(', ')
+		puts 'Sample:      ' + String(historic.sample)
+		puts 'Created at:  ' + (historic.created_at.nil? ? 'None' : historic.created_at.strftime('%Y-%m-%d %H:%M:%S'))
+		puts 'Status:      ' + historic.status
+		puts '--'
+	end
+end

data/examples/push/list.rb

@@ -0,0 +1,30 @@
+# This script lists Push subscription in your account.
+#
+# NB: Most of the error handling (exception catching) has been removed for
+# the sake of simplicity. Nearly everything in this library may throw
+# exceptions, and production code should catch them. See the documentation
+# for full details.
+#
+
+# Include the shared Env class
+require File.dirname(__FILE__) + '/env'
+
+# Create the env object. This reads the command line arguments, creates the
+# user object, and provides access to both along with helper functions.
+env = Env.new()
+
+begin
+	subscriptions = env.user.listPushSubscriptions()
+
+	if subscriptions['subscriptions'].size() == 0
+		puts 'No Push subscriptions exist in your account.'
+	else
+		for subscription in subscriptions['subscriptions']
+			env.displaySubscriptionDetails(subscription)
+		end
+	end
+
+	puts 'Rate limit remainining: ' + String(env.user.rate_limit_remaining)
+rescue DataSift::DataSiftError => err
+	puts 'ERR: [' + err.class.name + '] ' + err.message
+end

data/examples/push/pause.rb

@@ -0,0 +1,33 @@
+# This script pauses Push subscriptions in your account.
+#
+# NB: Most of the error handling (exception catching) has been removed for
+# the sake of simplicity. Nearly everything in this library may throw
+# exceptions, and production code should catch them. See the documentation
+# for full details.
+#
+
+# Include the shared Env class
+require File.dirname(__FILE__) + '/env'
+
+# Create the env object. This reads the command line arguments, creates the
+# user object, and provides access to both along with helper functions.
+env = Env.new()
+
+# Make sure we have something to do
+abort('Please specify one or more subscription IDs') unless env.args.size() > 0
+
+for sub_id in env.args
+	begin
+		sub = env.user.getPushSubscription(sub_id)
+		print 'Pausing ' + sub_id + ', "' + sub.name + '"...'
+		sub.pause()
+	rescue DataSift::DataSiftError => err
+		puts 'ERR: [' + err.class.name + '] ' + err.message
+	else
+		puts 'done'
+	end
+end
+
+if env.user.rate_limit_remaining != -1
+	puts 'Rate limit remainining: ' + String(env.user.rate_limit_remaining)
+end

data/examples/push/push-from-hash.rb

@@ -0,0 +1,72 @@
+# This script creates a Push subscription for a stream hash or Historics
+# playback ID.
+#
+# NB: Most of the error handling (exception catching) has been removed for
+# the sake of simplicity. Nearly everything in this library may throw
+# exceptions, and production code should catch them. See the documentation
+# for full details.
+#
+
+# Include the shared Env class
+require File.dirname(__FILE__) + '/env'
+
+# Display usage information, with an error message if provided.
+def usage(message = '', end_of_story = true)
+	puts message + '\n' unless message.length() == 0
+	puts
+	puts 'Usage: push-from-hash.rb <username> <api_key> \\'
+	puts '                 <output_type> <hash_type> <hash> <name> ...'
+	puts
+	puts 'Where: output_type = http (currently only http is supported)'
+	puts '       hash_type   = stream | historic'
+	puts '       hash        = the hash'
+	puts '       name        = a friendly name for the subscription'
+	puts '       key=val     = output_type-specific arguments'
+	puts
+	puts 'Example'
+	puts '       push-from-hash.rb http stream <hash> PushName delivery_frequency=10 \\'
+	puts '             url=http://www.example.com/push_endpoint auth.type=none'
+	puts
+	exit 1 unless not end_of_story
+end
+
+# Create the env object. This reads the command line arguments, creates the
+# user object, and provides access to both along with helper functions.
+env = Env.new()
+
+# Check that we have enough arguments
+usage() if env.args.size() < 4
+
+# Read the arguments
+output_type = env.args.shift
+hash_type   = env.args.shift
+hash        = env.args.shift
+name        = env.args.shift
+
+begin
+	# Create the Push definition
+	pushdef = env.user.createPushDefinition()
+	pushdef.output_type = output_type
+
+	# Now add the output_type-specific args from the command line
+	while env.args.size() > 0
+		k, v = env.args.shift.split('=', 2)
+		pushdef.output_params[k] = v
+	end
+
+	# Subscribe the hash to the Push endpoint
+	if hash_type == 'stream'
+		sub = pushdef.subscribeStreamHash(hash, name)
+	elsif hash_type == 'historic'
+		sub = pushdef.subscribeHistoricPlaybackId(hash, name)
+	else
+		usage('Invalid hash_type: ' + hash_type)
+	end
+
+	# Display the details of the new subscription
+	env.displaySubscriptionDetails(sub)
+
+	puts 'Rate limit remainining: ' + String(env.user.rate_limit_remaining)
+rescue DataSift::DataSiftError => err
+	puts 'ERR: [' + err.class.name + '] ' + err.message
+end

data/examples/push/push-historic-from-csdl.rb

@@ -0,0 +1,98 @@
+# This script creates a Push subscription for a Historics query from the
+# provided CSDL.
+#
+# NB: Most of the error handling (exception catching) has been removed for
+# the sake of simplicity. Nearly everything in this library may throw
+# exceptions, and production code should catch them. See the documentation
+# for full details.
+#
+
+# Include the shared Env class
+require File.dirname(__FILE__) + '/env'
+
+# Display usage information, with an error message if provided.
+def usage(message = '', end_of_story = true)
+	puts message + '\n' unless message.length() == 0
+	puts
+	puts 'Usage: push-historic-from-csdl.rb <username> <api_key> \\'
+	puts '              <csdl_filename> <output_type> <name> ...'
+	puts
+	puts 'Where: csdl_filename = a file containing the CSDL'
+	puts '       start_date    = Historics query start date (yyyymmddhhmmss)'
+	puts '       end_date      = Historics query end date (yyyymmddhhmmss)'
+	puts '       sources       = comma separated list of sources (twitter,digg,etc)'
+	puts '       sample        = percentage of matches to return'
+	puts '       output_type   = see http://dev.datasift.com/docs/push/connectors'
+	puts '       name          = a friendly name for the subscription'
+	puts '       key=val       = output_type-specific arguments'
+	puts
+	puts 'Example'
+	puts '       push-historic-from-csdl.rb csdl.txt csdl.txt 20120101000000 \\'
+	puts '             20120101235959 twitter 100 http PushName delivery_frequency=10 \\'
+	puts '             url=http://www.example.com/push_endpoint auth.type=none'
+	puts
+	exit 1 unless not end_of_story
+end
+
+# Create the env object. This reads the command line arguments, creates the
+# user object, and provides access to both along with helper functions.
+env = Env.new()
+
+# Check that we have enough arguments
+usage() if env.args.size() < 7
+
+# Read the arguments
+csdl_filename = env.args.shift
+start_date    = env.args.shift
+end_date      = env.args.shift
+sources       = env.args.shift
+sample        = env.args.shift
+output_type   = env.args.shift
+name          = env.args.shift
+
+# Parse the dates
+date_format = '%Y%m%d%H%M%S'
+start_date = DateTime.strptime(start_date, date_format)
+end_date   = DateTime.strptime(end_date,   date_format)
+
+# Read the CSDL
+csdl = File.open(csdl_filename, 'r').read
+
+begin
+	# Create the Definition object
+	definition = env.user.createDefinition(csdl)
+
+	# Create the Historics query
+	historic = definition.createHistoric(start_date, end_date, sources.split(','), sample, name + '_historic')
+
+	# Display the details of the Historics query
+	puts 'Historics query:'
+	env.displayHistoricDetails(historic)
+
+	# Create the Push definition
+	pushdef = env.user.createPushDefinition()
+	pushdef.output_type = output_type
+
+	# Now add the output_type-specific args from the command line
+	while env.args.size() > 0
+		k, v = env.args.shift.split('=', 2)
+		pushdef.output_params[k] = v
+	end
+
+	# Subscribe the Push endpoint to the Historics query
+	sub = pushdef.subscribeHistoric(historic, name)
+
+	# Start the Historics query
+	print 'Starting Historits query...'
+	historic.start()
+	puts 'done'
+	puts '--'
+
+	# Display the details of the new subscription
+	puts 'Push subscription:'
+	env.displaySubscriptionDetails(sub)
+
+	puts 'Rate limit remainining: ' + String(env.user.rate_limit_remaining)
+rescue DataSift::DataSiftError => err
+	puts 'ERR: [' + err.class.name + '] ' + err.message
+end

data/examples/push/push-stream-from-csdl.rb

@@ -0,0 +1,70 @@
+# This script creates a Push subscription for a live stream from the provided
+# CSDL.
+#
+# NB: Most of the error handling (exception catching) has been removed for
+# the sake of simplicity. Nearly everything in this library may throw
+# exceptions, and production code should catch them. See the documentation
+# for full details.
+#
+
+# Include the shared Env class
+require File.dirname(__FILE__) + '/env'
+
+# Display usage information, with an error message if provided.
+def usage(message = '', end_of_story = true)
+	puts message + '\n' unless message.length() == 0
+	puts
+	puts 'Usage: push-stream-from-csdl.py <username> <api_key> \\'
+	puts '              <csdl_filename> <output_type> <name> ...'
+	puts
+	puts 'Where: csdl_filename = a file containing the CSDL'
+	puts '       output_type   = see http://dev.datasift.com/docs/push/connectors'
+	puts '       name          = a friendly name for the subscription'
+	puts '       key=val       = output_type-specific arguments'
+	puts
+	puts 'Example'
+	puts '       push-stream-from-csdl.py csdl.txt http PushName delivery_frequency=10 \\'
+	puts '             url=http://www.example.com/push_endpoint auth.type=none'
+	puts
+	exit 1 unless not end_of_story
+end
+
+# Create the env object. This reads the command line arguments, creates the
+# user object, and provides access to both along with helper functions.
+env = Env.new()
+
+# Check that we have enough arguments
+usage() if env.args.size() < 3
+
+# Read the arguments
+csdl_filename = env.args.shift
+output_type   = env.args.shift
+name          = env.args.shift
+
+# Read the CSDL
+csdl = File.open(csdl_filename, 'r').read
+
+begin
+	# Create the Definition object
+	definition = env.user.createDefinition(csdl)
+
+	# Create the Push definition
+	pushdef = env.user.createPushDefinition()
+	pushdef.output_type = output_type
+
+	# Now add the output_type-specific args from the command line
+	while env.args.size() > 0
+		k, v = env.args.shift.split('=', 2)
+		pushdef.output_params[k] = v
+	end
+
+	# Subscribe the Push endpoint to the Definition
+	sub = pushdef.subscribeDefinition(definition, name)
+
+	# Display the details of the new subscription
+	env.displaySubscriptionDetails(sub)
+
+	puts 'Rate limit remainining: ' + String(env.user.rate_limit_remaining)
+rescue DataSift::DataSiftError => err
+	puts 'ERR: [' + err.class.name + '] ' + err.message
+end

data/examples/push/resume.rb

@@ -0,0 +1,33 @@
+# This script resumes Push subscriptions in your account.
+#
+# NB: Most of the error handling (exception catching) has been removed for
+# the sake of simplicity. Nearly everything in this library may throw
+# exceptions, and production code should catch them. See the documentation
+# for full details.
+#
+
+# Include the shared Env class
+require File.dirname(__FILE__) + '/env'
+
+# Create the env object. This reads the command line arguments, creates the
+# user object, and provides access to both along with helper functions.
+env = Env.new()
+
+# Make sure we have something to do
+abort('Please specify one or more subscription IDs') unless env.args.size() > 0
+
+for sub_id in env.args
+	begin
+		sub = env.user.getPushSubscription(sub_id)
+		print 'Resuming ' + sub_id + ', "' + sub.name + '"...'
+		sub.resume()
+	rescue DataSift::DataSiftError => err
+		puts 'ERR: [' + err.class.name + '] ' + err.message
+	else
+		puts 'done'
+	end
+end
+
+if env.user.rate_limit_remaining != -1
+	puts 'Rate limit remainining: ' + String(env.user.rate_limit_remaining)
+end

data/examples/push/stop.rb

@@ -0,0 +1,33 @@
+# This script stops Push subscriptions in your account.
+#
+# NB: Most of the error handling (exception catching) has been removed for
+# the sake of simplicity. Nearly everything in this library may throw
+# exceptions, and production code should catch them. See the documentation
+# for full details.
+#
+
+# Include the shared Env class
+require File.dirname(__FILE__) + '/env'
+
+# Create the env object. This reads the command line arguments, creates the
+# user object, and provides access to both along with helper functions.
+env = Env.new()
+
+# Make sure we have something to do
+abort('Please specify one or more subscription IDs') unless env.args.size() > 0
+
+for sub_id in env.args
+	begin
+		sub = env.user.getPushSubscription(sub_id)
+		print 'Stopping ' + sub_id + ', "' + sub.name + '"...'
+		sub.stop()
+	rescue DataSift::DataSiftError => err
+		puts 'ERR: [' + err.class.name + '] ' + err.message
+	else
+		puts 'done'
+	end
+end
+
+if env.user.rate_limit_remaining != -1
+	puts 'Rate limit remainining: ' + String(env.user.rate_limit_remaining)
+end

data/examples/push/view-log.rb

@@ -0,0 +1,45 @@
+# This script resumes Push subscriptions in your account.
+#
+# NB: Most of the error handling (exception catching) has been removed for
+# the sake of simplicity. Nearly everything in this library may throw
+# exceptions, and production code should catch them. See the documentation
+# for full details.
+#
+
+# Include the shared Env class
+require File.dirname(__FILE__) + '/env'
+
+# Create the env object. This reads the command line arguments, creates the
+# user object, and provides access to both along with helper functions.
+env = Env.new()
+
+# Make sure we have 0 or 1 arguments
+abort('Please specify one or more subscription IDs') unless env.args.size() < 2
+
+begin
+	# Get the log
+	arg_count = env.args.size()
+	if arg_count == 0
+		log = env.user.getPushSubscriptionLog()
+	else
+		subscription_id = env.args[0]
+		log = env.user.getPushSubscriptionLog(subscription_id)
+	end
+
+	# Make sure we have some log entries
+	abort('No log entries found.') unless log['count'] > 0
+
+	# Display the log entries in reverse order
+	log['log_entries'].reverse().each do |log_entry|
+		print (DateTime.strptime(String(log_entry['request_time']), '%s')).strftime('%Y-%m-%d %H:%M:%S') + ' '
+		print '[' + log_entry['subscription_id'] + '] ' if arg_count == 0
+		print 'Success ' if log_entry['success']
+		puts log_entry['message']
+	end
+rescue DataSift::DataSiftError => err
+	puts 'ERR: [' + err.class.name + '] ' + err.message
+end
+
+if env.user.rate_limit_remaining != -1
+	puts 'Rate limit remainining: ' + String(env.user.rate_limit_remaining)
+end