datasift 1.5.0 → 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +1 -0
- data/LICENSE +1 -1
- data/README.md +6 -4
- data/VERSION +1 -1
- data/config.yml +2 -2
- data/datasift.gemspec +11 -11
- data/examples/historics.sh +2 -0
- data/examples/historics/create-from-csdl.rb +71 -0
- data/examples/historics/create-from-hash.rb +65 -0
- data/examples/historics/delete.rb +30 -0
- data/examples/historics/env.rb +37 -0
- data/examples/historics/list.rb +30 -0
- data/examples/historics/start.rb +30 -0
- data/examples/historics/stop.rb +30 -0
- data/examples/historics/view.rb +28 -0
- data/examples/push.sh +2 -0
- data/examples/push/delete.rb +33 -0
- data/examples/push/env.rb +53 -0
- data/examples/push/list.rb +30 -0
- data/examples/push/pause.rb +33 -0
- data/examples/push/push-from-hash.rb +72 -0
- data/examples/push/push-historic-from-csdl.rb +98 -0
- data/examples/push/push-stream-from-csdl.rb +70 -0
- data/examples/push/resume.rb +33 -0
- data/examples/push/stop.rb +33 -0
- data/examples/push/view-log.rb +45 -0
- data/examples/push/view.rb +31 -0
- data/lib/DataSift/apiclient.rb +20 -25
- data/lib/DataSift/definition.rb +97 -57
- data/lib/DataSift/exceptions.rb +25 -8
- data/lib/DataSift/historic.rb +321 -0
- data/lib/DataSift/mockapiclient.rb +23 -34
- data/lib/DataSift/push_definition.rb +115 -0
- data/lib/DataSift/push_subscription.rb +330 -0
- data/lib/DataSift/stream_consumer.rb +53 -70
- data/lib/DataSift/stream_consumer_http.rb +11 -15
- data/lib/DataSift/user.rb +189 -61
- data/lib/datasift.rb +5 -10
- data/test/helper.rb +80 -6
- data/test/test_definition.rb +0 -9
- data/test/test_historics.rb +233 -0
- data/test/test_pushdefinition.rb +92 -0
- data/test/test_pushsubscription.rb +17 -0
- data/test/test_user.rb +0 -6
- data/test/testdata.yml +26 -0
- metadata +38 -23
- data/test/test_live_api.rb +0 -100
data/.gitignore
CHANGED
data/LICENSE
CHANGED
data/README.md
CHANGED
@@ -2,9 +2,9 @@ DataSift
|
|
2
2
|
========
|
3
3
|
|
4
4
|
The official Ruby library for accessing the DataSift API. See
|
5
|
-
http://datasift.
|
5
|
+
http://datasift.com/ for full details and to sign up for an account.
|
6
6
|
|
7
|
-
The examples
|
7
|
+
The examples use the username and API key in config.yml unless otherwise noted.
|
8
8
|
|
9
9
|
Install Instructions
|
10
10
|
--------------------
|
@@ -19,7 +19,7 @@ If you're using the source you'll need to install the dependencies.
|
|
19
19
|
sudo gem install yajl-ruby rest-client
|
20
20
|
|
21
21
|
The library will use SSL connections by default. While we recommend using SSL
|
22
|
-
you may disable it if
|
22
|
+
you may disable it if r§equired by passing false as the third parameter when
|
23
23
|
creating a user, or by calling user.enableSSL(false) on the user object.
|
24
24
|
|
25
25
|
Simple example
|
@@ -56,7 +56,9 @@ more details.
|
|
56
56
|
Changelog
|
57
57
|
---------
|
58
58
|
|
59
|
-
* v.
|
59
|
+
* v.2.0.0 Added support for Historics queries and Push delivery (2012-08-31)
|
60
|
+
|
61
|
+
* v.1.5.0 Added getBalance to the User class [joncooper](https://github.com/joncooper) (2012-05-24)
|
60
62
|
|
61
63
|
* v.1.4.1 Fixed a minor bug in the SSL support (2012-05-15)
|
62
64
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
2.0.0
|
data/config.yml
CHANGED
@@ -1,2 +1,2 @@
|
|
1
|
-
username:
|
2
|
-
api_key:
|
1
|
+
username: 3ft9
|
2
|
+
api_key: 584b068700b016093686a8d3f95696ac
|
data/datasift.gemspec
CHANGED
@@ -2,17 +2,17 @@
|
|
2
2
|
$:.push File.expand_path("../lib", __FILE__)
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
|
-
s.name
|
6
|
-
s.version
|
5
|
+
s.name = 'datasift'
|
6
|
+
s.version = File.open('VERSION').first
|
7
7
|
|
8
|
-
s.authors
|
9
|
-
s.email
|
10
|
-
s.description
|
11
|
-
s.summary
|
12
|
-
s.homepage
|
8
|
+
s.authors = ['DataSift']
|
9
|
+
s.email = ['support@datasift.net']
|
10
|
+
s.description = %q{The official Ruby library for accessing the DataSift API. See http://datasift.com/ for full details and to sign up for an account.}
|
11
|
+
s.summary = %q{DataSift is a simple wrapper for the DataSift API.}
|
12
|
+
s.homepage = 'http://github.com/datasift/datasift-ruby'
|
13
13
|
|
14
|
-
s.platform
|
15
|
-
s.rubygems_version
|
14
|
+
s.platform = Gem::Platform::RUBY
|
15
|
+
s.rubygems_version = %q{1.3.6}
|
16
16
|
s.required_rubygems_version = Gem::Requirement.new(">= 1.3.6") if s.respond_to? :required_rubygems_version=
|
17
17
|
|
18
18
|
s.add_runtime_dependency('rest-client', '~> 1.6.3')
|
@@ -21,7 +21,7 @@ Gem::Specification.new do |s|
|
|
21
21
|
s.add_development_dependency('shoulda', '~> 2.11.3')
|
22
22
|
s.add_development_dependency('rspec', '~> 2.6.0')
|
23
23
|
|
24
|
-
s.files
|
25
|
-
s.test_files
|
24
|
+
s.files = `git ls-files`.split("\n")
|
25
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
26
26
|
s.require_paths = ["lib"]
|
27
27
|
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
# This script creates a new Historics query from a file containing CSDL.
|
2
|
+
#
|
3
|
+
# NB: Most of the error handling (exception catching) has been removed for
|
4
|
+
# the sake of simplicity. Nearly everything in this library may throw
|
5
|
+
# exceptions, and production code should catch them. See the documentation
|
6
|
+
# for full details.
|
7
|
+
#
|
8
|
+
|
9
|
+
# Function to display usage instructions with an optional error message.
|
10
|
+
def usage(message = '', end_of_story = true)
|
11
|
+
puts message + '\n' unless message.length() == 0
|
12
|
+
puts
|
13
|
+
puts 'Usage: create-from-csdl <username> <api_key> \\'
|
14
|
+
puts ' <csdl_filename> <start> <end> <sources> <name> <sample>'
|
15
|
+
puts
|
16
|
+
puts 'Where: csdl_filename = the stream hash the query should run'
|
17
|
+
puts ' start = the start date for the query (YYYYMMDDHHMMSS)'
|
18
|
+
puts ' end = the end date for the query (YYYYMMDDHHMMSS)'
|
19
|
+
puts ' sources = comma separated list of data sources (e.g. twitter)'
|
20
|
+
puts ' name = a friendly name for the query'
|
21
|
+
puts ' sample = the sample rate'
|
22
|
+
puts
|
23
|
+
puts 'Example'
|
24
|
+
puts ' create-from-csdl csdl.txt 20120101000000 20120101235959 \\'
|
25
|
+
puts ' twitter HistoricsQuery123 100'
|
26
|
+
puts
|
27
|
+
exit 1 unless not end_of_story
|
28
|
+
end
|
29
|
+
|
30
|
+
# Include the shared Env class
|
31
|
+
require File.dirname(__FILE__) + '/env'
|
32
|
+
|
33
|
+
# Create the env object. This reads the command line arguments, creates the
|
34
|
+
# user object, and provides access to both along with helper functions.
|
35
|
+
env = Env.new()
|
36
|
+
|
37
|
+
# Check that we have enough arguments
|
38
|
+
usage() unless env.args.size() == 6
|
39
|
+
|
40
|
+
# Read the arguments
|
41
|
+
csdl_filename = env.args[0]
|
42
|
+
start_date = env.args[1]
|
43
|
+
end_date = env.args[2]
|
44
|
+
sources = env.args[3].split(',')
|
45
|
+
name = env.args[4]
|
46
|
+
sample = env.args[5]
|
47
|
+
|
48
|
+
# Parse the dates
|
49
|
+
start_date = DateTime.strptime(start_date, '%Y%m%d%H%M%S')
|
50
|
+
end_date = DateTime.strptime(end_date, '%Y%m%d%H%M%S')
|
51
|
+
|
52
|
+
# Read the CSDL
|
53
|
+
csdl = File.open(csdl_filename, 'r').read
|
54
|
+
|
55
|
+
begin
|
56
|
+
# Create the definition
|
57
|
+
definition = env.user.createDefinition(csdl)
|
58
|
+
|
59
|
+
# Create the Historics query
|
60
|
+
historic = definition.createHistoric(start_date, end_date, sources, name, sample)
|
61
|
+
|
62
|
+
# Prepare the query
|
63
|
+
historic.prepare()
|
64
|
+
|
65
|
+
# Display the details
|
66
|
+
env.displayHistoricDetails(historic)
|
67
|
+
|
68
|
+
puts 'Rate limit remainining: ' + String(env.user.rate_limit_remaining)
|
69
|
+
rescue DataSift::DataSiftError => err
|
70
|
+
puts 'ERR: [' + err.class.name + '] ' + err.message
|
71
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
# This script creates a new Historics query from a stream hash.
|
2
|
+
#
|
3
|
+
# NB: Most of the error handling (exception catching) has been removed for
|
4
|
+
# the sake of simplicity. Nearly everything in this library may throw
|
5
|
+
# exceptions, and production code should catch them. See the documentation
|
6
|
+
# for full details.
|
7
|
+
#
|
8
|
+
|
9
|
+
# Function to display usage instructions with an optional error message.
|
10
|
+
def usage(message = '', end_of_story = true)
|
11
|
+
puts message + '\n' unless message.length() == 0
|
12
|
+
puts
|
13
|
+
puts 'Usage: create-from-hash \\'
|
14
|
+
puts ' <username> <api_key> <hash> <start> <end> <sources> <name> <sample>'
|
15
|
+
puts
|
16
|
+
puts 'Where: hash = the stream hash the query should run'
|
17
|
+
puts ' start = the start date for the query (YYYYMMDDHHMMSS)'
|
18
|
+
puts ' end = the end date for the query (YYYYMMDDHHMMSS)'
|
19
|
+
puts ' sources = comma separated list of data sources (e.g. twitter)'
|
20
|
+
puts ' name = a friendly name for the query'
|
21
|
+
puts ' sample = the sample rate'
|
22
|
+
puts
|
23
|
+
puts 'Example'
|
24
|
+
puts ' create-from-hash <hash> 20120101000000 20120101235959 \\'
|
25
|
+
puts ' twitter HistoricsQuery123 100'
|
26
|
+
puts
|
27
|
+
exit 1 unless not end_of_story
|
28
|
+
end
|
29
|
+
|
30
|
+
# Include the shared Env class
|
31
|
+
require File.dirname(__FILE__) + '/env'
|
32
|
+
|
33
|
+
# Create the env object. This reads the command line arguments, creates the
|
34
|
+
# user object, and provides access to both along with helper functions.
|
35
|
+
env = Env.new()
|
36
|
+
|
37
|
+
# Check that we have enough arguments
|
38
|
+
usage() unless env.args.size() == 6
|
39
|
+
|
40
|
+
# Read the arguments
|
41
|
+
stream_hash = env.args[0]
|
42
|
+
start_date = env.args[1]
|
43
|
+
end_date = env.args[2]
|
44
|
+
sources = env.args[3].split(',')
|
45
|
+
name = env.args[4]
|
46
|
+
sample = env.args[5]
|
47
|
+
|
48
|
+
# Parse the dates
|
49
|
+
start_date = DateTime.strptime(start_date, '%Y%m%d%H%M%S')
|
50
|
+
end_date = DateTime.strptime(end_date, '%Y%m%d%H%M%S')
|
51
|
+
|
52
|
+
begin
|
53
|
+
# Create the Historics query
|
54
|
+
historic = env.user.createHistoric(stream_hash, start_date, end_date, sources, name, sample)
|
55
|
+
|
56
|
+
# Prepare the query
|
57
|
+
historic.prepare()
|
58
|
+
|
59
|
+
# Display the details
|
60
|
+
env.displayHistoricDetails(historic)
|
61
|
+
|
62
|
+
puts 'Rate limit remainining: ' + String(env.user.rate_limit_remaining)
|
63
|
+
rescue DataSift::DataSiftError => err
|
64
|
+
puts 'ERR: [' + err.class.name + '] ' + err.message
|
65
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# This script deletes Historics queries from your account.
|
2
|
+
#
|
3
|
+
# NB: Most of the error handling (exception catching) has been removed for
|
4
|
+
# the sake of simplicity. Nearly everything in this library may throw
|
5
|
+
# exceptions, and production code should catch them. See the documentation
|
6
|
+
# for full details.
|
7
|
+
#
|
8
|
+
|
9
|
+
# Include the shared Env class
|
10
|
+
require File.dirname(__FILE__) + '/env'
|
11
|
+
|
12
|
+
# Create the env object. This reads the command line arguments, creates the
|
13
|
+
# user object, and provides access to both along with helper functions.
|
14
|
+
env = Env.new()
|
15
|
+
|
16
|
+
# Make sure we have something to do
|
17
|
+
abort('Please specify one or more playback IDs') unless env.args.size() > 0
|
18
|
+
|
19
|
+
begin
|
20
|
+
for playback_id in env.args
|
21
|
+
historic = env.user.getHistoric(playback_id)
|
22
|
+
print 'Deleting ' + playback_id + ', "' + historic.name + '"...'
|
23
|
+
historic.delete()
|
24
|
+
puts 'done'
|
25
|
+
end
|
26
|
+
|
27
|
+
puts 'Rate limit remainining: ' + String(env.user.rate_limit_remaining)
|
28
|
+
rescue DataSift::DataSiftError => err
|
29
|
+
puts 'ERR: [' + err.class.name + '] ' + err.message
|
30
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
# This class is used by the Historics examples to remove the noise of
|
2
|
+
# dealing with command line arguments.
|
3
|
+
#
|
4
|
+
|
5
|
+
# Include the DataSift library
|
6
|
+
require File.dirname(__FILE__) + '/../../lib/datasift'
|
7
|
+
|
8
|
+
class Env
|
9
|
+
attr_reader :user, :args
|
10
|
+
|
11
|
+
def initialize(args = false)
|
12
|
+
if args === false
|
13
|
+
args = ARGV
|
14
|
+
end
|
15
|
+
|
16
|
+
abort('Please specify your DataSift username and API key as the first two command line arguments!') unless args.size() >= 2
|
17
|
+
|
18
|
+
username = args.shift
|
19
|
+
api_key = args.shift
|
20
|
+
@user = DataSift::User.new(username, api_key)
|
21
|
+
|
22
|
+
@args = args
|
23
|
+
end
|
24
|
+
|
25
|
+
def displayHistoricDetails(historic)
|
26
|
+
puts 'Playback ID: ' + historic.hash
|
27
|
+
puts 'Stream hash: ' + historic.stream_hash
|
28
|
+
puts 'Name: ' + historic.name
|
29
|
+
puts 'Start time: ' + historic.start_date.strftime('%Y-%m-%d %H:%M:%S')
|
30
|
+
puts 'End time: ' + historic.end_date.strftime('%Y-%m-%d %H:%M:%S')
|
31
|
+
puts 'Sources: ' + historic.sources.join(', ')
|
32
|
+
puts 'Sample: ' + String(historic.sample)
|
33
|
+
puts 'Created at: ' + (historic.created_at.nil? ? 'None' : historic.created_at.strftime('%Y-%m-%d %H:%M:%S'))
|
34
|
+
puts 'Status: ' + historic.status
|
35
|
+
puts '--'
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# This script lists Historics queries in your account.
|
2
|
+
#
|
3
|
+
# NB: Most of the error handling (exception catching) has been removed for
|
4
|
+
# the sake of simplicity. Nearly everything in this library may throw
|
5
|
+
# exceptions, and production code should catch them. See the documentation
|
6
|
+
# for full details.
|
7
|
+
#
|
8
|
+
|
9
|
+
# Include the shared Env class
|
10
|
+
require File.dirname(__FILE__) + '/env'
|
11
|
+
|
12
|
+
# Create the env object. This reads the command line arguments, creates the
|
13
|
+
# user object, and provides access to both along with helper functions.
|
14
|
+
env = Env.new()
|
15
|
+
|
16
|
+
begin
|
17
|
+
historics = env.user.listHistorics()
|
18
|
+
|
19
|
+
if historics['historics'].size() == 0
|
20
|
+
puts 'No Historics queries exist in your account.'
|
21
|
+
else
|
22
|
+
for historic in historics['historics']
|
23
|
+
env.displayHistoricDetails(historic)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
puts 'Rate limit remainining: ' + String(env.user.rate_limit_remaining)
|
28
|
+
rescue DataSift::DataSiftError => err
|
29
|
+
puts 'ERR: [' + err.class.name + '] ' + err.message
|
30
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# This script starts Historics queries in your account.
|
2
|
+
#
|
3
|
+
# NB: Most of the error handling (exception catching) has been removed for
|
4
|
+
# the sake of simplicity. Nearly everything in this library may throw
|
5
|
+
# exceptions, and production code should catch them. See the documentation
|
6
|
+
# for full details.
|
7
|
+
#
|
8
|
+
|
9
|
+
# Include the shared Env class
|
10
|
+
require File.dirname(__FILE__) + '/env'
|
11
|
+
|
12
|
+
# Create the env object. This reads the command line arguments, creates the
|
13
|
+
# user object, and provides access to both along with helper functions.
|
14
|
+
env = Env.new()
|
15
|
+
|
16
|
+
# Make sure we have something to do
|
17
|
+
abort('Please specify one or more playback IDs') unless env.args.size() > 0
|
18
|
+
|
19
|
+
begin
|
20
|
+
for playback_id in env.args
|
21
|
+
historic = env.user.getHistoric(playback_id)
|
22
|
+
print 'Starting ' + playback_id + ', "' + historic.name + '"...'
|
23
|
+
historic.start()
|
24
|
+
puts 'done'
|
25
|
+
end
|
26
|
+
|
27
|
+
puts 'Rate limit remainining: ' + String(env.user.rate_limit_remaining)
|
28
|
+
rescue DataSift::DataSiftError => err
|
29
|
+
puts 'ERR: [' + err.class.name + '] ' + err.message
|
30
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# This script stops Historics queries in your account.
|
2
|
+
#
|
3
|
+
# NB: Most of the error handling (exception catching) has been removed for
|
4
|
+
# the sake of simplicity. Nearly everything in this library may throw
|
5
|
+
# exceptions, and production code should catch them. See the documentation
|
6
|
+
# for full details.
|
7
|
+
#
|
8
|
+
|
9
|
+
# Include the shared Env class
|
10
|
+
require File.dirname(__FILE__) + '/env'
|
11
|
+
|
12
|
+
# Create the env object. This reads the command line arguments, creates the
|
13
|
+
# user object, and provides access to both along with helper functions.
|
14
|
+
env = Env.new()
|
15
|
+
|
16
|
+
# Make sure we have something to do
|
17
|
+
abort('Please specify one or more playback IDs') unless env.args.size() > 0
|
18
|
+
|
19
|
+
begin
|
20
|
+
for playback_id in env.args
|
21
|
+
historic = env.user.getHistoric(playback_id)
|
22
|
+
print 'Stopping ' + playback_id + ', "' + historic.name + '"...'
|
23
|
+
historic.stop()
|
24
|
+
puts 'done'
|
25
|
+
end
|
26
|
+
|
27
|
+
puts 'Rate limit remainining: ' + String(env.user.rate_limit_remaining)
|
28
|
+
rescue DataSift::DataSiftError => err
|
29
|
+
puts 'ERR: [' + err.class.name + '] ' + err.message
|
30
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# This script views Historics queries in your account.
|
2
|
+
#
|
3
|
+
# NB: Most of the error handling (exception catching) has been removed for
|
4
|
+
# the sake of simplicity. Nearly everything in this library may throw
|
5
|
+
# exceptions, and production code should catch them. See the documentation
|
6
|
+
# for full details.
|
7
|
+
#
|
8
|
+
|
9
|
+
# Include the shared Env class
|
10
|
+
require File.dirname(__FILE__) + '/env'
|
11
|
+
|
12
|
+
# Create the env object. This reads the command line arguments, creates the
|
13
|
+
# user object, and provides access to both along with helper functions.
|
14
|
+
env = Env.new()
|
15
|
+
|
16
|
+
# Make sure we have something to do
|
17
|
+
abort('Please specify one or more playback IDs') unless env.args.size() > 0
|
18
|
+
|
19
|
+
begin
|
20
|
+
for playback_id in env.args
|
21
|
+
historic = env.user.getHistoric(playback_id)
|
22
|
+
env.displayHistoricDetails(historic)
|
23
|
+
end
|
24
|
+
|
25
|
+
puts 'Rate limit remainining: ' + String(env.user.rate_limit_remaining)
|
26
|
+
rescue DataSift::DataSiftError => err
|
27
|
+
puts 'ERR: [' + err.class.name + '] ' + err.message
|
28
|
+
end
|
data/examples/push.sh
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
# This script deletes Push subscriptions from your account.
|
2
|
+
#
|
3
|
+
# NB: Most of the error handling (exception catching) has been removed for
|
4
|
+
# the sake of simplicity. Nearly everything in this library may throw
|
5
|
+
# exceptions, and production code should catch them. See the documentation
|
6
|
+
# for full details.
|
7
|
+
#
|
8
|
+
|
9
|
+
# Include the shared Env class
|
10
|
+
require File.dirname(__FILE__) + '/env'
|
11
|
+
|
12
|
+
# Create the env object. This reads the command line arguments, creates the
|
13
|
+
# user object, and provides access to both along with helper functions.
|
14
|
+
env = Env.new()
|
15
|
+
|
16
|
+
# Make sure we have something to do
|
17
|
+
abort('Please specify one or more subscription IDs') unless env.args.size() > 0
|
18
|
+
|
19
|
+
for sub_id in env.args
|
20
|
+
begin
|
21
|
+
sub = env.user.getPushSubscription(sub_id)
|
22
|
+
print 'Deleting ' + sub_id + ', "' + sub.name + '"...'
|
23
|
+
sub.delete()
|
24
|
+
rescue DataSift::DataSiftError => err
|
25
|
+
puts 'ERR: [' + err.class.name + '] ' + err.message
|
26
|
+
else
|
27
|
+
puts 'done'
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
if env.user.rate_limit_remaining != -1
|
32
|
+
puts 'Rate limit remainining: ' + String(env.user.rate_limit_remaining)
|
33
|
+
end
|