datasift 3.7.2 → 3.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +8 -0
  3. data/datasift.gemspec +6 -3
  4. data/lib/api/api_resource.rb +1 -1
  5. data/lib/datasift.rb +53 -16
  6. data/lib/errors.rb +8 -0
  7. data/lib/live_stream.rb +7 -7
  8. data/lib/pylon.rb +59 -26
  9. data/lib/tasks.rb +51 -0
  10. data/lib/version.rb +3 -0
  11. metadata +5 -83
  12. data/.gitignore +0 -12
  13. data/.rubocop.yml +0 -12
  14. data/.rubocop_todo.yml +0 -497
  15. data/.travis.yml +0 -17
  16. data/Gemfile +0 -15
  17. data/LICENSE +0 -19
  18. data/MIGRATING_TO_V.3.0.0.md +0 -262
  19. data/Rakefile +0 -10
  20. data/VERSION +0 -1
  21. data/examples/account_eg.rb +0 -33
  22. data/examples/account_identity_eg.rb +0 -50
  23. data/examples/account_identity_limit_eg.rb +0 -72
  24. data/examples/account_identity_token_eg.rb +0 -73
  25. data/examples/auth.rb +0 -55
  26. data/examples/cli.sh +0 -155
  27. data/examples/core_api_eg.rb +0 -50
  28. data/examples/historics_eg.rb +0 -66
  29. data/examples/historics_preview_eg.rb +0 -31
  30. data/examples/ingestion/twitter_gnip_batch_eg.rb +0 -61
  31. data/examples/live_stream_eg.rb +0 -92
  32. data/examples/managed_source_fb_eg.rb +0 -129
  33. data/examples/managed_source_ig_eg.rb +0 -126
  34. data/examples/pull.rb +0 -47
  35. data/examples/push_eg.rb +0 -51
  36. data/examples/pylon/pylon_api_v1.2_eg.rb +0 -166
  37. data/examples/pylon/pylon_api_v1.3_eg.rb +0 -201
  38. data/test/datasift/account/account_api_test.rb +0 -35
  39. data/test/datasift/core_api_test.rb +0 -179
  40. data/test/datasift/historics_preview_api_test.rb +0 -56
  41. data/test/datasift/odp/batch_upload_test.rb +0 -52
  42. data/test/datasift/push_api_test.rb +0 -238
  43. data/test/fixtures/cassettes/account/usage/default.json +0 -1
  44. data/test/fixtures/cassettes/account/usage/invalid.json +0 -1
  45. data/test/fixtures/cassettes/account/usage/valid_params.json +0 -1
  46. data/test/fixtures/cassettes/core/after_historic_dpu.json +0 -1
  47. data/test/fixtures/cassettes/core/balance_get.json +0 -1
  48. data/test/fixtures/cassettes/core/before_dpu.json +0 -1
  49. data/test/fixtures/cassettes/core/before_historic_dpu.json +0 -1
  50. data/test/fixtures/cassettes/core/compile_success.json +0 -1
  51. data/test/fixtures/cassettes/core/dpu_get_cost.json +0 -1
  52. data/test/fixtures/cassettes/core/dpu_throw_badrequest.json +0 -1
  53. data/test/fixtures/cassettes/core/historic_dpu.json +0 -1
  54. data/test/fixtures/cassettes/core/usage_success.json +0 -1
  55. data/test/fixtures/cassettes/core/validate_invalid_hash.json +0 -1
  56. data/test/fixtures/cassettes/core/validate_success_bool.json +0 -1
  57. data/test/fixtures/cassettes/core/validate_success_hash.json +0 -1
  58. data/test/fixtures/cassettes/odp/batch/after_upload.json +0 -1
  59. data/test/fixtures/cassettes/odp/batch/before_upload.json +0 -1
  60. data/test/fixtures/cassettes/odp/batch/upload_failure_no_source.json +0 -1
  61. data/test/fixtures/cassettes/odp/batch/upload_success.json +0 -1
  62. data/test/fixtures/cassettes/preview/before_preview_create.json +0 -1
  63. data/test/fixtures/cassettes/preview/before_preview_get.json +0 -1
  64. data/test/fixtures/cassettes/preview/preview_create_success.json +0 -1
  65. data/test/fixtures/cassettes/preview/preview_get_success.json +0 -1
  66. data/test/fixtures/cassettes/push/after_push_create.json +0 -1
  67. data/test/fixtures/cassettes/push/after_push_get.json +0 -1
  68. data/test/fixtures/cassettes/push/after_push_log.json +0 -1
  69. data/test/fixtures/cassettes/push/after_push_pause.json +0 -1
  70. data/test/fixtures/cassettes/push/after_push_resume.json +0 -1
  71. data/test/fixtures/cassettes/push/after_push_stop.json +0 -1
  72. data/test/fixtures/cassettes/push/after_push_update.json +0 -1
  73. data/test/fixtures/cassettes/push/before_push_create.json +0 -1
  74. data/test/fixtures/cassettes/push/before_push_delete.json +0 -1
  75. data/test/fixtures/cassettes/push/before_push_get.json +0 -1
  76. data/test/fixtures/cassettes/push/before_push_log.json +0 -1
  77. data/test/fixtures/cassettes/push/before_push_pause.json +0 -1
  78. data/test/fixtures/cassettes/push/before_push_resume.json +0 -1
  79. data/test/fixtures/cassettes/push/before_push_stop.json +0 -1
  80. data/test/fixtures/cassettes/push/before_push_update.json +0 -1
  81. data/test/fixtures/cassettes/push/push_create.json +0 -1
  82. data/test/fixtures/cassettes/push/push_delete.json +0 -1
  83. data/test/fixtures/cassettes/push/push_get_by_id.json +0 -1
  84. data/test/fixtures/cassettes/push/push_log_with_id.json +0 -1
  85. data/test/fixtures/cassettes/push/push_pause.json +0 -1
  86. data/test/fixtures/cassettes/push/push_resume.json +0 -1
  87. data/test/fixtures/cassettes/push/push_stop.json +0 -1
  88. data/test/fixtures/cassettes/push/push_update.json +0 -1
  89. data/test/fixtures/cassettes/push/push_validate.json +0 -1
  90. data/test/fixtures/data/fake_gnip_tweets.json +0 -10
  91. data/test/test_helper.rb +0 -49
@@ -1,166 +0,0 @@
1
- ##
2
- # This script runs through all PYLON API endpoints using v1.2 of the API
3
- ##
4
-
5
- require './../auth'
6
- class AnalysisApi < DataSiftExample
7
- def initialize
8
- super
9
- run_analysis
10
- end
11
-
12
- def run_analysis
13
- begin
14
- puts "Create a new identity to make PYLON API calls"
15
- identity = @datasift.account_identity.create(
16
- "RUBY_LIB_#{Time.now.to_i}",
17
- "active",
18
- false
19
- )
20
- identity_id = identity[:data][:id]
21
- puts identity[:data].to_json
22
-
23
- puts "\nCreate a Token for our Identity"
24
- token = @datasift.account_identity_token.create(
25
- identity_id,
26
- 'facebook',
27
- '125595667777713|5aef9cfdb31d8be64b87204c3bca820f'
28
- )
29
- puts token[:data].to_json
30
-
31
- puts "\nNow make PYLON API calls using the Identity's API key"
32
- @config.merge!(
33
- api_key: identity[:data][:api_key],
34
- api_version: 'v1.2'
35
- )
36
- @datasift = DataSift::Client.new(@config)
37
-
38
- csdl = "return { fb.content any \"data, #{Time.now}\" }"
39
-
40
- puts "Check this CSDL is valid: #{csdl}"
41
- puts "Valid? #{@datasift.pylon.valid?(csdl)}"
42
-
43
- puts "\nCompile my CSDL"
44
- compiled = @datasift.pylon.compile csdl
45
- hash = compiled[:data][:hash]
46
- puts "Hash: #{hash}"
47
-
48
- puts "\nStart recording filter with hash #{hash}"
49
- filter = @datasift.pylon.start(
50
- hash,
51
- 'Facebook Pylon Test Filter'
52
- )
53
- puts filter[:data].to_json
54
-
55
- puts "\nSleep for 10 seconds to record a little data"
56
- sleep(10)
57
-
58
- puts "\nGet details of our running recording"
59
- puts @datasift.pylon.get(hash)[:data].to_json
60
-
61
- puts "\nYou can also list running recordings"
62
- puts @datasift.pylon.list[:data].to_json
63
-
64
- puts "\nFrequency distribution analysis on fb.author.country"
65
- params = {
66
- analysis_type: 'freqDist',
67
- parameters: {
68
- threshold: 1,
69
- target: 'fb.author.country'
70
- }
71
- }
72
- puts @datasift.pylon.analyze(
73
- hash,
74
- params
75
- )[:data].to_json
76
-
77
- puts "\nFrequency distribution analysis on fb.author.age with filter"
78
- params = {
79
- analysis_type: 'freqDist',
80
- parameters: {
81
- threshold: 1,
82
- target: 'fb.author.age'
83
- }
84
- }
85
- filter = ''
86
- puts @datasift.pylon.analyze(
87
- hash,
88
- params,
89
- filter
90
- )[:data].to_json
91
-
92
- puts "\nTime series analysis"
93
- params = {
94
- analysis_type: 'timeSeries',
95
- parameters: {
96
- interval: 'hour',
97
- span: 12
98
- }
99
- }
100
- filter = ''
101
- start_time = Time.now.to_i - (60 * 60 * 24 * 7) # 7 days ago
102
- end_time = Time.now.to_i
103
- puts @datasift.pylon.analyze(
104
- hash,
105
- params,
106
- filter,
107
- start_time,
108
- end_time
109
- )[:data].to_json
110
-
111
- puts "\nFrequency Distribution with nested queries. Find the top three " \
112
- "age groups for each gender by country"
113
- filter = ''
114
- params = {
115
- analysis_type: 'freqDist',
116
- parameters: {
117
- threshold: 4,
118
- target: 'fb.author.country'
119
- },
120
- child: {
121
- analysis_type: 'freqDist',
122
- parameters: {
123
- threshold: 2,
124
- target: 'fb.author.gender'
125
- },
126
- child: {
127
- analysis_type: 'freqDist',
128
- parameters: {
129
- threshold: 3,
130
- target: 'fb.author.age'
131
- }
132
- }
133
- }
134
- }
135
- start_time = Time.now.to_i - (60 * 60 * 24 * 7)
136
- end_time = Time.now.to_i
137
- puts @datasift.pylon.analyze(
138
- hash,
139
- params,
140
- filter,
141
- start_time,
142
- end_time
143
- )[:data].to_json
144
-
145
- puts "\nTags analysis"
146
- puts @datasift.pylon.tags(hash)[:data].to_json
147
-
148
- puts "\nGet Public Posts"
149
- puts @datasift.pylon.sample(
150
- hash,
151
- 10,
152
- Time.now.to_i - (60 * 60), # from 1hr ago
153
- Time.now.to_i, # to 'now'
154
- 'fb.content contains_any "your, filter, terms"'
155
- )[:data].to_json
156
-
157
- puts "\nStop recording filter with hash #{hash}"
158
- puts @datasift.pylon.stop(hash)[:data].to_json
159
-
160
- rescue DataSiftError => dse
161
- puts dse.inspect
162
- end
163
- end
164
- end
165
-
166
- AnalysisApi.new
@@ -1,201 +0,0 @@
1
- ##
2
- # This script runs through all PYLON API endpoints using v1.3 of the API
3
- ##
4
-
5
- require './../auth'
6
- class AnalysisApi < DataSiftExample
7
- def initialize
8
- super
9
- run_analysis
10
- end
11
-
12
- def run_analysis
13
- begin
14
- puts "Create a new identity to make PYLON API calls"
15
- identity = @datasift.account_identity.create(
16
- "RUBY_LIB_#{Time.now.to_i}",
17
- "active",
18
- false
19
- )
20
- identity_id = identity[:data][:id]
21
- puts identity[:data].to_json
22
-
23
- puts "\nCreate a Token for our Identity"
24
- token = @datasift.account_identity_token.create(
25
- identity_id,
26
- 'facebook',
27
- '125595667777713|5aef9cfdb31d8be64b87204c3bca820f'
28
- )
29
- puts token[:data].to_json
30
-
31
- puts "\nNow make PYLON API calls using the Identity's API key"
32
- @pylon_config = @config.dup
33
- @pylon_config.merge!(
34
- api_key: identity[:data][:api_key],
35
- api_version: 'v1.3'
36
- )
37
- @datasift = DataSift::Client.new(@pylon_config)
38
-
39
- csdl = "return { fb.all.content any \"data, #{Time.now}\" }"
40
-
41
- puts "Check this CSDL is valid: #{csdl}"
42
- puts "Valid? #{@datasift.pylon.valid?(csdl)}"
43
-
44
- puts "\nCompile my CSDL"
45
- compiled = @datasift.pylon.compile csdl
46
- hash = compiled[:data][:hash]
47
- puts "Hash: #{hash}"
48
-
49
- puts "\nStart recording with hash #{hash}"
50
- recording = @datasift.pylon.start(
51
- hash,
52
- 'Facebook Pylon Test Recording'
53
- )
54
- puts recording[:data].to_json
55
-
56
- puts "\nSleep for 10 seconds to record a little data"
57
- sleep(10)
58
-
59
- puts "\nGet details of our running recording by ID"
60
- puts @datasift.pylon.get('', recording[:data][:id])[:data].to_json
61
-
62
- puts "\nYou can also list running recordings"
63
- puts @datasift.pylon.list[:data].to_json
64
-
65
- puts "\nFrequency distribution analysis on fb.author.country"
66
- params = {
67
- analysis_type: 'freqDist',
68
- parameters: {
69
- threshold: 3,
70
- target: 'fb.author.country'
71
- }
72
- }
73
- puts @datasift.pylon.analyze(
74
- '',
75
- params,
76
- '',
77
- nil,
78
- nil,
79
- recording[:data][:id]
80
- )[:data].to_json
81
-
82
- puts "\nFrequency distribution analysis on fb.author.age with filter"
83
- params = {
84
- analysis_type: 'freqDist',
85
- parameters: {
86
- threshold: 1,
87
- target: 'fb.author.age'
88
- }
89
- }
90
- filter = 'fb.parent.content any "facebook"'
91
- puts @datasift.pylon.analyze(
92
- '',
93
- params,
94
- filter,
95
- nil,
96
- nil,
97
- recording[:data][:id]
98
- )[:data].to_json
99
-
100
- puts "\nTime series analysis"
101
- params = {
102
- analysis_type: 'timeSeries',
103
- parameters: {
104
- interval: 'hour',
105
- span: 12
106
- }
107
- }
108
- filter = ''
109
- start_time = Time.now.to_i - (60 * 60 * 24 * 7) # 7 days ago
110
- end_time = Time.now.to_i
111
- puts @datasift.pylon.analyze(
112
- '',
113
- params,
114
- filter,
115
- start_time,
116
- end_time,
117
- recording[:data][:id]
118
- )[:data].to_json
119
-
120
- puts "\nFrequency Distribution with nested queries. Find the top three " \
121
- "age groups for each gender by country"
122
- filter = ''
123
- params = {
124
- analysis_type: 'freqDist',
125
- parameters: {
126
- threshold: 4,
127
- target: 'fb.author.country'
128
- },
129
- child: {
130
- analysis_type: 'freqDist',
131
- parameters: {
132
- threshold: 2,
133
- target: 'fb.author.gender'
134
- },
135
- child: {
136
- analysis_type: 'freqDist',
137
- parameters: {
138
- threshold: 3,
139
- target: 'fb.author.age'
140
- }
141
- }
142
- }
143
- }
144
- start_time = Time.now.to_i - (60 * 60 * 24 * 7)
145
- end_time = Time.now.to_i
146
- puts @datasift.pylon.analyze(
147
- '',
148
- params,
149
- filter,
150
- start_time,
151
- end_time,
152
- recording[:data][:id]
153
- )[:data].to_json
154
-
155
- puts "\nTags analysis"
156
- puts @datasift.pylon.tags('',recording[:data][:id])[:data].to_json
157
-
158
- puts "\nGet Public Posts"
159
- puts @datasift.pylon.sample(
160
- '',
161
- 10,
162
- Time.now.to_i - (60 * 60), # from 1hr ago
163
- Time.now.to_i, # to 'now'
164
- 'fb.content contains_any "your, filter, terms"',
165
- recording[:data][:id]
166
- )[:data].to_json
167
-
168
- puts "\nv1.3+ of the API allows you to update the name or hash of recordings;"
169
- puts "\nBefore update:"
170
- puts @datasift.pylon.get(recording[:data][:id])[:data].to_json
171
-
172
- new_hash = @datasift.pylon.compile("fb.content any \"data, #{Time.now}\"")[:data][:hash]
173
-
174
- puts "\nAfter update:"
175
- puts @datasift.pylon.update(
176
- recording[:data][:id],
177
- new_hash,
178
- "Updated at #{Time.now}"
179
- )[:data].to_json
180
-
181
- puts "\nStop recording filter with the recording ID #{recording[:data][:id]}"
182
- puts @datasift.pylon.stop('', recording[:data][:id])[:data].to_json
183
- sleep(3)
184
- puts "\nYou can also restart a stopped recording by recording ID #{recording[:data][:id]}"
185
- puts @datasift.pylon.restart(recording[:data][:id])[:data].to_json
186
-
187
- # Cleanup.
188
- # Stop the recording again to clean up
189
- sleep(3)
190
- @datasift.pylon.stop('', recording[:data][:id])[:data].to_json
191
- # Disable the identity created for this example
192
- @datasift = DataSift::Client.new(@config)
193
- @datasift.account_identity.update(identity_id, '', 'disabled')
194
-
195
- rescue DataSiftError => dse
196
- puts dse.inspect
197
- end
198
- end
199
- end
200
-
201
- AnalysisApi.new
@@ -1,35 +0,0 @@
1
- require File.expand_path('../../../test_helper', __FILE__)
2
-
3
- describe 'DataSift::Account' do
4
- before do
5
- auth = DataSiftExample.new
6
- @datasift = auth.datasift
7
- end
8
-
9
- ##
10
- # Account Usage
11
- #
12
- describe '#usage' do
13
- it 'can get account usage using default params' do
14
- VCR.use_cassette('account/usage/default') do
15
- response = @datasift.account.usage
16
- assert_equal STATUS.valid, response[:http][:status]
17
- end
18
- end
19
-
20
- it 'can get account usage using valid params' do
21
- VCR.use_cassette('account/usage/valid_params') do
22
- response = @datasift.account.usage('monthly')
23
- assert_equal STATUS.valid, response[:http][:status]
24
- end
25
- end
26
-
27
- it 'handles 400 when using invalid params' do
28
- VCR.use_cassette('account/usage/invalid') do
29
- assert_raises BadRequestError do
30
- @datasift.account.usage('invalid_period')
31
- end
32
- end
33
- end
34
- end
35
- end
@@ -1,179 +0,0 @@
1
- require File.expand_path('../../test_helper', __FILE__)
2
-
3
- describe 'DataSift' do
4
-
5
- before do
6
- auth = DataSiftExample.new
7
- @datasift = auth.datasift
8
- @data = OpenStruct.new
9
-
10
- @data.valid_csdl = 'interaction.content contains "test"'
11
- @data.invalid_csdl = 'interaction.nonsense is not valid'
12
- @data.invalid_hash = 'this_is_not_a_valid_stream_hash'
13
- end
14
-
15
- ##
16
- # Client
17
- #
18
- describe 'client' do
19
- it 'raises_InvalidConfigError_without_auth_credentials' do
20
- assert_raises InvalidConfigError do
21
- DataSift::Client.new({})
22
- end
23
- end
24
- end
25
-
26
- ##
27
- # /validate
28
- #
29
- describe '#validate' do
30
- it 'csdl_cant_be_nil_when_validating' do
31
- assert_raises InvalidParamError do
32
- @datasift.valid?(nil)
33
- end
34
- end
35
-
36
- it 'csdl_cant_be_empty_when_validating' do
37
- assert_raises InvalidParamError do
38
- @datasift.valid?('')
39
- end
40
- end
41
-
42
- it 'user_can_get_successful_validation_as_bool' do
43
- VCR.use_cassette('core/validate_success_bool') do
44
- assert @datasift.valid?(@data.valid_csdl), 'Valid CSDL must return true'
45
- end
46
- end
47
-
48
- it 'user_can_get_successful_validation_as_hash' do
49
- VCR.use_cassette('core/validate_success_hash') do
50
- validation = @datasift.valid?(@data.valid_csdl, false)
51
- assert_kind_of Hash, validation,
52
- "Successful validation will return a hash"
53
- assert_equal STATUS.valid, validation[:http][:status],
54
- "This request should have returned #{STATUS.valid} status"
55
- end
56
- end
57
-
58
- it 'failing_csdl_validation' do
59
- VCR.use_cassette('core/validate_invalid_hash') do
60
- assert_raises BadRequestError do
61
- @datasift.valid?(@data.invalid_csdl)
62
- end
63
- end
64
- end
65
- end
66
-
67
- ##
68
- # /compile
69
- #
70
- describe '#compile' do
71
- it 'csdl_cant_be_nil_when_compiling' do
72
- assert_raises InvalidParamError do
73
- @datasift.compile(nil)
74
- end
75
- end
76
-
77
- it 'csdl_cant_be_empty_when_compiling' do
78
- assert_raises InvalidParamError do
79
- @datasift.compile('')
80
- end
81
- end
82
-
83
- it 'successful_compilation_returns_hash' do
84
- VCR.use_cassette('core/compile_success') do
85
- response = @datasift.compile @data.valid_csdl
86
- assert_kind_of Hash, response,
87
- "Successful compilation will return a hash"
88
- assert_equal STATUS.valid, response[:http][:status],
89
- "This request should have returned #{STATUS.valid} status"
90
- end
91
- end
92
- end
93
-
94
- ##
95
- # /usage
96
- #
97
- describe '#usage' do
98
- it 'can_get_users_usage' do
99
- VCR.use_cassette('core/usage_success') do
100
- response = @datasift.usage
101
- assert_equal STATUS.valid, response[:http][:status]
102
- assert_kind_of Hash, response
103
- end
104
- end
105
- end
106
-
107
- ##
108
- # /dpu
109
- #
110
- describe '#dpu' do
111
- before do
112
- VCR.use_cassette('core/before_dpu') do
113
- @hash = @datasift.compile(@data.valid_csdl)[:data][:hash]
114
- end
115
- end
116
-
117
- it 'can_get_dpu_cost' do
118
- VCR.use_cassette('core/dpu_get_cost') do
119
- response = @datasift.dpu @hash
120
- assert_equal STATUS.valid, response[:http][:status]
121
- end
122
- end
123
-
124
- it 'cannot_get_dpu_cost_for_invalid_hash' do
125
- VCR.use_cassette('core/dpu_throw_badrequest') do
126
- assert_raises BadRequestError do
127
- @datasift.dpu @data.invalid_hash
128
- end
129
- end
130
- end
131
-
132
- it 'requires_at_least_one_param' do
133
- assert_raises ArgumentError do
134
- @datasift.dpu
135
- end
136
- end
137
- end
138
-
139
- describe '#dpu for Historics' do
140
- before do
141
- VCR.use_cassette('core/before_historic_dpu') do
142
- @hash = @datasift.compile(@data.valid_csdl)[:data][:hash]
143
- @historic = @datasift.historics.prepare(
144
- @hash,
145
- Time.now.to_i - 10800,
146
- Time.now.to_i - 7200,
147
- 'Ruby test suite',
148
- 'tumblr',
149
- 10
150
- )
151
- end
152
- end
153
-
154
- after do
155
- VCR.use_cassette('core/after_historic_dpu') do
156
- @datasift.historics.delete @historic[:data][:id]
157
- end
158
- end
159
-
160
- it 'can_get_dpu_cost_for_historic' do
161
- VCR.use_cassette('core/historic_dpu') do
162
- response = @datasift.dpu('', @historic[:data][:id])
163
- assert_equal STATUS.valid, response[:http][:status]
164
- end
165
- end
166
- end
167
-
168
- ##
169
- # /balance
170
- #
171
- describe '#balance' do
172
- it 'can get account balance' do
173
- VCR.use_cassette('core/balance_get') do
174
- response = @datasift.balance
175
- assert_equal STATUS.valid, response[:http][:status]
176
- end
177
- end
178
- end
179
- end