datasift 2.1.1 → 3.0.0.beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/CHANGELOG.md +100 -0
- data/Gemfile.lock +32 -0
- data/README.md +38 -79
- data/VERSION +1 -1
- data/datasift.gemspec +21 -24
- data/examples/auth.rb +44 -0
- data/examples/core_api_eg.rb +46 -0
- data/examples/historics_eg.rb +50 -0
- data/examples/historics_preview_eg.rb +30 -0
- data/examples/live_stream_eg.rb +89 -0
- data/examples/managed_source_eg.rb +56 -0
- data/examples/pull.rb +44 -0
- data/examples/push_eg.rb +56 -0
- data/lib/api/api_resource.rb +23 -0
- data/lib/datasift.rb +287 -14
- data/lib/errors.rb +59 -0
- data/lib/historics.rb +76 -0
- data/lib/historics_preview.rb +20 -0
- data/lib/live_stream.rb +53 -0
- data/lib/managed_source.rb +57 -0
- data/lib/push.rb +156 -0
- data/tests/core_api_test.rb +42 -0
- metadata +51 -73
- data/Rakefile +0 -34
- data/config.yml +0 -2
- data/examples/consume-stream.rb +0 -63
- data/examples/deletes.rb +0 -52
- data/examples/dpu.rb +0 -115
- data/examples/football-buffered.rb +0 -51
- data/examples/football.rb +0 -53
- data/examples/historics.sh +0 -2
- data/examples/historics/create-from-csdl.rb +0 -71
- data/examples/historics/create-from-hash.rb +0 -65
- data/examples/historics/delete.rb +0 -30
- data/examples/historics/env.rb +0 -37
- data/examples/historics/list.rb +0 -30
- data/examples/historics/start.rb +0 -30
- data/examples/historics/stop.rb +0 -30
- data/examples/historics/view.rb +0 -28
- data/examples/push.sh +0 -2
- data/examples/push/delete.rb +0 -33
- data/examples/push/env.rb +0 -53
- data/examples/push/list.rb +0 -30
- data/examples/push/pause.rb +0 -33
- data/examples/push/push-from-hash.rb +0 -72
- data/examples/push/push-historic-from-csdl.rb +0 -98
- data/examples/push/push-stream-from-csdl.rb +0 -70
- data/examples/push/resume.rb +0 -33
- data/examples/push/stop.rb +0 -33
- data/examples/push/view-log.rb +0 -45
- data/examples/push/view.rb +0 -31
- data/examples/twitter-track.rb +0 -61
- data/lib/DataSift/apiclient.rb +0 -73
- data/lib/DataSift/definition.rb +0 -202
- data/lib/DataSift/exceptions.rb +0 -33
- data/lib/DataSift/historic.rb +0 -316
- data/lib/DataSift/managed_source.rb +0 -263
- data/lib/DataSift/mockapiclient.rb +0 -44
- data/lib/DataSift/push_definition.rb +0 -115
- data/lib/DataSift/push_subscription.rb +0 -330
- data/lib/DataSift/stream_consumer.rb +0 -166
- data/lib/DataSift/stream_consumer_http.rb +0 -188
- data/lib/DataSift/user.rb +0 -311
- data/test/helper.rb +0 -95
- data/test/test_definition.rb +0 -273
- data/test/test_historics.rb +0 -233
- data/test/test_pushdefinition.rb +0 -92
- data/test/test_pushsubscription.rb +0 -17
- data/test/test_user.rb +0 -130
- data/test/testdata.yml +0 -30
data/test/test_pushdefinition.rb
DELETED
@@ -1,92 +0,0 @@
|
|
1
|
-
require './' + File.dirname(__FILE__) + '/helper'
|
2
|
-
|
3
|
-
class TestPushDefinition < Test::Unit::TestCase
|
4
|
-
context "Given a new PushDefinition object" do
|
5
|
-
setup do
|
6
|
-
init()
|
7
|
-
# Create the historic (API response is for compiling the definition)
|
8
|
-
@push = @user.createPushDefinition()
|
9
|
-
end
|
10
|
-
|
11
|
-
should "be of the right type" do
|
12
|
-
assert_not_nil @push
|
13
|
-
assert @push.kind_of?(DataSift::PushDefinition)
|
14
|
-
end
|
15
|
-
|
16
|
-
should "have an empty initial status" do
|
17
|
-
assert_equal '', @push.initial_status
|
18
|
-
end
|
19
|
-
|
20
|
-
should "allow the initial status to be changed" do
|
21
|
-
@push.initial_status = DataSift::PushSubscription::STATUS_PAUSED
|
22
|
-
assert_equal DataSift::PushSubscription::STATUS_PAUSED, @push.initial_status
|
23
|
-
@push.initial_status = DataSift::PushSubscription::STATUS_STOPPED
|
24
|
-
assert_equal DataSift::PushSubscription::STATUS_STOPPED, @push.initial_status
|
25
|
-
end
|
26
|
-
|
27
|
-
should "have an empty output type" do
|
28
|
-
assert_equal '', @push.output_type
|
29
|
-
end
|
30
|
-
|
31
|
-
should "allow the output type to be set" do
|
32
|
-
@push.output_type = @testdata['push_output_type']
|
33
|
-
assert_equal @testdata['push_output_type'], @push.output_type
|
34
|
-
end
|
35
|
-
|
36
|
-
should "return nil for non-existant output parameters" do
|
37
|
-
assert_nil @push.output_params['url']
|
38
|
-
end
|
39
|
-
|
40
|
-
should "allow output parameters to be set" do
|
41
|
-
@push.output_params['url'] = @testdata['push_output_param_url']
|
42
|
-
assert_equal @testdata['push_output_param_url'], @push.output_params['url']
|
43
|
-
assert_equal 1, @push.output_params.size()
|
44
|
-
end
|
45
|
-
|
46
|
-
should "support validation of the output parameters" do
|
47
|
-
configurePushDefinition(@push)
|
48
|
-
set204Response()
|
49
|
-
@push.validate()
|
50
|
-
end
|
51
|
-
|
52
|
-
should "support subscribing to a Definition object" do
|
53
|
-
definition = @user.createDefinition(@testdata['definition'])
|
54
|
-
# Set the response to the compile method so we can get the hash
|
55
|
-
begin
|
56
|
-
@user.api_client.setResponse(200, {
|
57
|
-
'hash' => @testdata['definition_hash'],
|
58
|
-
'created_at' => Time.now.strftime('%Y-%m-%d %H:%M:%S'),
|
59
|
-
'dpu' => 10,
|
60
|
-
}, 200, 150)
|
61
|
-
assert_equal @testdata['definition_hash'], definition.hash
|
62
|
-
rescue InvalidDataError
|
63
|
-
assert false, "InvalidDataError"
|
64
|
-
rescue CompileFailedError
|
65
|
-
assert false, "CompileFailedError"
|
66
|
-
rescue APIError
|
67
|
-
assert false, "APIError"
|
68
|
-
end
|
69
|
-
|
70
|
-
configurePushDefinition(@push)
|
71
|
-
|
72
|
-
setResponseToASingleSubscription()
|
73
|
-
|
74
|
-
subscription = @push.subscribeDefinition(definition, @testdata['push_name'])
|
75
|
-
|
76
|
-
checkSubscription(subscription)
|
77
|
-
end
|
78
|
-
|
79
|
-
should "support subscribing to a Historic object" do
|
80
|
-
setResponseToASingleHistoric()
|
81
|
-
historic = @user.getHistoric(@testdata['historic_playback_id'])
|
82
|
-
|
83
|
-
configurePushDefinition(@push)
|
84
|
-
|
85
|
-
setResponseToASingleSubscription()
|
86
|
-
|
87
|
-
subscription = @push.subscribeHistoric(historic, @testdata['push_name'])
|
88
|
-
|
89
|
-
checkSubscription(subscription)
|
90
|
-
end
|
91
|
-
end
|
92
|
-
end
|
@@ -1,17 +0,0 @@
|
|
1
|
-
require './' + File.dirname(__FILE__) + '/helper'
|
2
|
-
|
3
|
-
class TestPushSubscription < Test::Unit::TestCase
|
4
|
-
context "Given a new PushSubsription object" do
|
5
|
-
setup do
|
6
|
-
init()
|
7
|
-
# Get a subscription from the API
|
8
|
-
setResponseToASingleSubscription()
|
9
|
-
@subscription = @user.getPushSubscription(@testdata['push_id'])
|
10
|
-
end
|
11
|
-
|
12
|
-
should "be of the right type" do
|
13
|
-
assert_not_nil @subscription
|
14
|
-
assert @subscription.kind_of?(DataSift::PushSubscription)
|
15
|
-
end
|
16
|
-
end
|
17
|
-
end
|
data/test/test_user.rb
DELETED
@@ -1,130 +0,0 @@
|
|
1
|
-
require './' + File.dirname(__FILE__) + '/helper'
|
2
|
-
|
3
|
-
class TestUser < Test::Unit::TestCase
|
4
|
-
context "Given a new User object" do
|
5
|
-
setup do
|
6
|
-
init()
|
7
|
-
end
|
8
|
-
|
9
|
-
should "have the correct username" do
|
10
|
-
assert_not_nil @user
|
11
|
-
assert_equal @config['username'], @user.username
|
12
|
-
end
|
13
|
-
|
14
|
-
should "have the correct API key" do
|
15
|
-
assert_not_nil @user
|
16
|
-
assert_equal @config['api_key'], @user.api_key
|
17
|
-
end
|
18
|
-
|
19
|
-
should "return valid summary usage information" do
|
20
|
-
@user.api_client.setResponse(200, {
|
21
|
-
'start' => 'Mon, 07 Nov 2011 14:20:00 +0000',
|
22
|
-
'streams' => {
|
23
|
-
'5e82aa9ac3dcf4dec1cce08a0cec914a' => {
|
24
|
-
'seconds' => 313,
|
25
|
-
'licenses' => {
|
26
|
-
'twitter' => 17,
|
27
|
-
'facebook' => 5
|
28
|
-
}
|
29
|
-
}
|
30
|
-
},
|
31
|
-
'end' => 'Mon, 07 Nov 2011 15:20:00 +0000'
|
32
|
-
}, 200, 150)
|
33
|
-
usage = @user.getUsage()
|
34
|
-
assert_equal "Mon, 07 Nov 2011 14:20:00 +0000", usage['start']
|
35
|
-
assert_equal "Mon, 07 Nov 2011 15:20:00 +0000", usage['end']
|
36
|
-
assert_equal 313, usage['streams']['5e82aa9ac3dcf4dec1cce08a0cec914a']['seconds']
|
37
|
-
assert_equal 17, usage['streams']['5e82aa9ac3dcf4dec1cce08a0cec914a']['licenses']['twitter']
|
38
|
-
assert_equal 5, usage['streams']['5e82aa9ac3dcf4dec1cce08a0cec914a']['licenses']['facebook']
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
|
-
context "Given an empty definition from the User factory" do
|
43
|
-
setup do
|
44
|
-
init()
|
45
|
-
@definition = @user.createDefinition()
|
46
|
-
end
|
47
|
-
|
48
|
-
should "have an empty CSDL" do
|
49
|
-
assert_not_nil @definition
|
50
|
-
assert_equal '', @definition.csdl
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
context "Given an new definition from the User factory" do
|
55
|
-
setup do
|
56
|
-
init()
|
57
|
-
@definition = @user.createDefinition(" " + @testdata['definition'])
|
58
|
-
end
|
59
|
-
|
60
|
-
should "have the correct CSDL" do
|
61
|
-
assert_not_nil @definition
|
62
|
-
assert_equal @testdata['definition'], @definition.csdl
|
63
|
-
end
|
64
|
-
end
|
65
|
-
|
66
|
-
context "#getBalance" do
|
67
|
-
setup do
|
68
|
-
init()
|
69
|
-
@user.api_client.setResponse(200, {
|
70
|
-
'balance' => {
|
71
|
-
'plan' => 'payg',
|
72
|
-
'credit' => 206.74
|
73
|
-
}
|
74
|
-
}, 200, 150)
|
75
|
-
end
|
76
|
-
|
77
|
-
should "return a hash containing the account balance info" do
|
78
|
-
@usage_hash = @user.getBalance()
|
79
|
-
assert @usage_hash['plan'] == 'payg'
|
80
|
-
assert @usage_hash['credit'] == 206.74
|
81
|
-
end
|
82
|
-
end
|
83
|
-
|
84
|
-
context "Given a call has been made to the API" do
|
85
|
-
setup do
|
86
|
-
init()
|
87
|
-
@definition = @user.createDefinition(@testdata['definition'])
|
88
|
-
@user.api_client.setResponse(200, {
|
89
|
-
'hash' => @testdata['definition_hash'],
|
90
|
-
'created_at' => Time.now.strftime('%Y-%m-%d %H:%M:%S'),
|
91
|
-
'dpu' => 10,
|
92
|
-
}, 200, 150)
|
93
|
-
@definition.compile()
|
94
|
-
end
|
95
|
-
|
96
|
-
should "have a rate limit value" do
|
97
|
-
assert @user.rate_limit == 200
|
98
|
-
end
|
99
|
-
|
100
|
-
should "have a rate limit remaining value" do
|
101
|
-
assert @user.rate_limit_remaining == 150
|
102
|
-
end
|
103
|
-
end
|
104
|
-
|
105
|
-
context "The exception raised by the User.getUsage method" do
|
106
|
-
setup do
|
107
|
-
init()
|
108
|
-
end
|
109
|
-
|
110
|
-
should "be APIError given a 400 response" do
|
111
|
-
@user.api_client.setResponse(400, { 'error' => 'Bad request from user supplied data'}, 200, 150)
|
112
|
-
assert_raise(DataSift::APIError) { usage = @user.getUsage() }
|
113
|
-
end
|
114
|
-
|
115
|
-
should "be AccessDenied given a 401 response" do
|
116
|
-
@user.api_client.setResponse(401, { 'error' => 'User banned because they are a very bad person'}, 200, 150)
|
117
|
-
assert_raise(DataSift::AccessDeniedError) { usage = @user.getUsage() }
|
118
|
-
end
|
119
|
-
|
120
|
-
should "be APIError given a 404 response" do
|
121
|
-
@user.api_client.setResponse(404, { 'error' => 'Endpoint or data not found'}, 200, 150)
|
122
|
-
assert_raise(DataSift::APIError) { usage = @user.getUsage() }
|
123
|
-
end
|
124
|
-
|
125
|
-
should "be APIError given a 500 response" do
|
126
|
-
@user.api_client.setResponse(500, { 'error' => 'Problem with an internal service'}, 200, 150)
|
127
|
-
assert_raise(DataSift::APIError) { usage = @user.getUsage() }
|
128
|
-
end
|
129
|
-
end
|
130
|
-
end
|
data/test/testdata.yml
DELETED
@@ -1,30 +0,0 @@
|
|
1
|
-
definition: interaction.content contains "twitter"
|
2
|
-
definition_hash: b76b06db8a679d2b4319e5d06a8b6691
|
3
|
-
|
4
|
-
invalid_definition: interactin.content contains "twitter"
|
5
|
-
|
6
|
-
# Historic
|
7
|
-
historic_playback_id: 93558e17de15072fa126370c37c5bd8f
|
8
|
-
historic_start: 2012-06-01 00:00:00
|
9
|
-
historic_end: 2012-06-01 23:59:59
|
10
|
-
historic_created_at: 2012-08-01 12:00:00
|
11
|
-
historic_sources: twitter
|
12
|
-
historic_sample: 10.0
|
13
|
-
historic_name: Historic for unit tests
|
14
|
-
|
15
|
-
# PushSubscription
|
16
|
-
push_id: b665761917bbcb7afd3102b4a645b41e
|
17
|
-
push_name: Push subscription for unit tests
|
18
|
-
push_created_at: 2012-07-20 00:00:00
|
19
|
-
push_status: active
|
20
|
-
push_hash_type: stream
|
21
|
-
push_last_request: 2012-07-20 07:10:00
|
22
|
-
push_last_success: 2012-07-20 07:00:00
|
23
|
-
push_output_type: http
|
24
|
-
push_output_params:
|
25
|
-
delivery_frequency: 60
|
26
|
-
max_size: 10240
|
27
|
-
url: http://www.example.com/push_endpoint
|
28
|
-
auth_type: basic
|
29
|
-
auth_username: myuser
|
30
|
-
auth_password: mypass
|