datasift 3.7.2 → 3.8.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (91) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +8 -0
  3. data/datasift.gemspec +6 -3
  4. data/lib/api/api_resource.rb +1 -1
  5. data/lib/datasift.rb +53 -16
  6. data/lib/errors.rb +8 -0
  7. data/lib/live_stream.rb +7 -7
  8. data/lib/pylon.rb +59 -26
  9. data/lib/tasks.rb +51 -0
  10. data/lib/version.rb +3 -0
  11. metadata +5 -83
  12. data/.gitignore +0 -12
  13. data/.rubocop.yml +0 -12
  14. data/.rubocop_todo.yml +0 -497
  15. data/.travis.yml +0 -17
  16. data/Gemfile +0 -15
  17. data/LICENSE +0 -19
  18. data/MIGRATING_TO_V.3.0.0.md +0 -262
  19. data/Rakefile +0 -10
  20. data/VERSION +0 -1
  21. data/examples/account_eg.rb +0 -33
  22. data/examples/account_identity_eg.rb +0 -50
  23. data/examples/account_identity_limit_eg.rb +0 -72
  24. data/examples/account_identity_token_eg.rb +0 -73
  25. data/examples/auth.rb +0 -55
  26. data/examples/cli.sh +0 -155
  27. data/examples/core_api_eg.rb +0 -50
  28. data/examples/historics_eg.rb +0 -66
  29. data/examples/historics_preview_eg.rb +0 -31
  30. data/examples/ingestion/twitter_gnip_batch_eg.rb +0 -61
  31. data/examples/live_stream_eg.rb +0 -92
  32. data/examples/managed_source_fb_eg.rb +0 -129
  33. data/examples/managed_source_ig_eg.rb +0 -126
  34. data/examples/pull.rb +0 -47
  35. data/examples/push_eg.rb +0 -51
  36. data/examples/pylon/pylon_api_v1.2_eg.rb +0 -166
  37. data/examples/pylon/pylon_api_v1.3_eg.rb +0 -201
  38. data/test/datasift/account/account_api_test.rb +0 -35
  39. data/test/datasift/core_api_test.rb +0 -179
  40. data/test/datasift/historics_preview_api_test.rb +0 -56
  41. data/test/datasift/odp/batch_upload_test.rb +0 -52
  42. data/test/datasift/push_api_test.rb +0 -238
  43. data/test/fixtures/cassettes/account/usage/default.json +0 -1
  44. data/test/fixtures/cassettes/account/usage/invalid.json +0 -1
  45. data/test/fixtures/cassettes/account/usage/valid_params.json +0 -1
  46. data/test/fixtures/cassettes/core/after_historic_dpu.json +0 -1
  47. data/test/fixtures/cassettes/core/balance_get.json +0 -1
  48. data/test/fixtures/cassettes/core/before_dpu.json +0 -1
  49. data/test/fixtures/cassettes/core/before_historic_dpu.json +0 -1
  50. data/test/fixtures/cassettes/core/compile_success.json +0 -1
  51. data/test/fixtures/cassettes/core/dpu_get_cost.json +0 -1
  52. data/test/fixtures/cassettes/core/dpu_throw_badrequest.json +0 -1
  53. data/test/fixtures/cassettes/core/historic_dpu.json +0 -1
  54. data/test/fixtures/cassettes/core/usage_success.json +0 -1
  55. data/test/fixtures/cassettes/core/validate_invalid_hash.json +0 -1
  56. data/test/fixtures/cassettes/core/validate_success_bool.json +0 -1
  57. data/test/fixtures/cassettes/core/validate_success_hash.json +0 -1
  58. data/test/fixtures/cassettes/odp/batch/after_upload.json +0 -1
  59. data/test/fixtures/cassettes/odp/batch/before_upload.json +0 -1
  60. data/test/fixtures/cassettes/odp/batch/upload_failure_no_source.json +0 -1
  61. data/test/fixtures/cassettes/odp/batch/upload_success.json +0 -1
  62. data/test/fixtures/cassettes/preview/before_preview_create.json +0 -1
  63. data/test/fixtures/cassettes/preview/before_preview_get.json +0 -1
  64. data/test/fixtures/cassettes/preview/preview_create_success.json +0 -1
  65. data/test/fixtures/cassettes/preview/preview_get_success.json +0 -1
  66. data/test/fixtures/cassettes/push/after_push_create.json +0 -1
  67. data/test/fixtures/cassettes/push/after_push_get.json +0 -1
  68. data/test/fixtures/cassettes/push/after_push_log.json +0 -1
  69. data/test/fixtures/cassettes/push/after_push_pause.json +0 -1
  70. data/test/fixtures/cassettes/push/after_push_resume.json +0 -1
  71. data/test/fixtures/cassettes/push/after_push_stop.json +0 -1
  72. data/test/fixtures/cassettes/push/after_push_update.json +0 -1
  73. data/test/fixtures/cassettes/push/before_push_create.json +0 -1
  74. data/test/fixtures/cassettes/push/before_push_delete.json +0 -1
  75. data/test/fixtures/cassettes/push/before_push_get.json +0 -1
  76. data/test/fixtures/cassettes/push/before_push_log.json +0 -1
  77. data/test/fixtures/cassettes/push/before_push_pause.json +0 -1
  78. data/test/fixtures/cassettes/push/before_push_resume.json +0 -1
  79. data/test/fixtures/cassettes/push/before_push_stop.json +0 -1
  80. data/test/fixtures/cassettes/push/before_push_update.json +0 -1
  81. data/test/fixtures/cassettes/push/push_create.json +0 -1
  82. data/test/fixtures/cassettes/push/push_delete.json +0 -1
  83. data/test/fixtures/cassettes/push/push_get_by_id.json +0 -1
  84. data/test/fixtures/cassettes/push/push_log_with_id.json +0 -1
  85. data/test/fixtures/cassettes/push/push_pause.json +0 -1
  86. data/test/fixtures/cassettes/push/push_resume.json +0 -1
  87. data/test/fixtures/cassettes/push/push_stop.json +0 -1
  88. data/test/fixtures/cassettes/push/push_update.json +0 -1
  89. data/test/fixtures/cassettes/push/push_validate.json +0 -1
  90. data/test/fixtures/data/fake_gnip_tweets.json +0 -10
  91. data/test/test_helper.rb +0 -49
@@ -1,92 +0,0 @@
1
- require './auth'
2
- class StreamingApi < DataSiftExample
3
-
4
- def initialize
5
- super
6
- @datasift = DataSift::Client.new(@config)
7
- run
8
- end
9
-
10
- def run
11
- begin
12
- ruby_received = 0
13
- python_received = 0
14
- ruby = 'interaction.content contains "ruby"'
15
- ruby_stream = @datasift.compile ruby
16
-
17
- python = 'interaction.content contains "python"'
18
- python_stream = @datasift.compile python
19
-
20
- on_delete = lambda { |stream, m| puts 'We must delete this to be compliant ==> ' + m }
21
-
22
- on_error = lambda do |stream, e|
23
- # This is called when an error occurs on your connection; you should probably log this event
24
- puts 'A serious error has occurred'
25
- puts e.message
26
- end
27
-
28
- on_message_ruby = lambda do |message, stream, hash|
29
- ruby_received += 1
30
- puts "Ruby #{ruby_received}, #{message}"
31
-
32
- if ruby_received >= 10
33
- puts 'un-subscribing from ruby stream '+ hash
34
- stream.unsubscribe hash
35
- end
36
- end
37
-
38
- on_message_python = lambda do |message, stream, hash|
39
- python_received += 1
40
- puts "python #{python_received}, #{message}"
41
-
42
- if python_received >= 10
43
- puts 'un-subscribing from python stream '+ hash
44
- stream.unsubscribe hash
45
- end
46
- end
47
-
48
- on_connect = lambda do |stream|
49
- #
50
- puts 'subscribing to python stream '+ python_stream[:data][:hash]
51
- stream.subscribe(python_stream[:data][:hash], on_message_python)
52
- puts 'Subscribed to '+ python_stream[:data][:hash]
53
- sleep 1
54
- #
55
- puts 'subscribing to ruby stream '+ ruby_stream[:data][:hash]
56
- stream.subscribe(ruby_stream[:data][:hash], on_message_ruby)
57
- puts 'Subscribed to '+ ruby_stream[:data][:hash]
58
- end
59
-
60
- on_close = lambda do |stream,msg|
61
- # This is called when a connection is closed; you should probably log this event
62
- puts msg
63
- end
64
-
65
- on_datasift_message = lambda do |stream, message, hash|
66
- #not all messages have a hash
67
- puts "is_success = #{message[:is_success]}, is_failure = #{message[:is_failure]}, is_warning = #{message[:is_warning]}, is_tick = #{message[:is_tick]}"
68
- puts "DataSift Message #{hash} ==> #{message}"
69
- end
70
-
71
- conn = DataSift::new_stream(@config, on_delete, on_error, on_connect, on_close)
72
- conn.on_datasift_message = on_datasift_message
73
- #can do something else here now...
74
- puts 'Do some other business stuff...'
75
- conn.stream.read_thread.join
76
- #rescue DataSiftError
77
- rescue DataSiftError => dse
78
- puts dse.inspect
79
- # Then match specific one to take action; All errors thrown by the client extend DataSiftError
80
- case dse
81
- when ConnectionError
82
- # some connection error
83
- when AuthError
84
- when BadRequestError
85
- else
86
- # do something else...
87
- end
88
- end
89
- end
90
- end
91
-
92
- StreamingApi.new
@@ -1,129 +0,0 @@
1
- require './auth'
2
- class ManagedSourceApi < DataSiftExample
3
- def initialize
4
- super
5
- run
6
- end
7
-
8
- def run
9
- puts "Creating a managed source\n--"
10
- parameters = {
11
- likes: true,
12
- posts_by_others: true,
13
- comments: true,
14
- page_likes: true
15
- }
16
- resources = [{
17
- parameters: {
18
- id: 130871750291428,
19
- title: 'DataSift'
20
- }
21
- }]
22
- auth = [{
23
- parameters: {
24
- value: 'vnHQocnOEChoOsYYOHVIn80......EOMmZ63go6s0DzxsAmJaDeE2ljdQjDqJVT'
25
- }
26
- }]
27
-
28
- source = @datasift.managed_source.create(
29
- 'facebook_page', 'Ruby test', parameters, resources, auth
30
- )
31
- puts source[:data].to_json
32
-
33
- id = source[:data][:id]
34
-
35
- puts "\nStarting delivery for my Managed Source\n--"
36
- puts @datasift.managed_source.start(id)[:data].to_json
37
-
38
- # Define new resources to be added
39
- update_resources = [{
40
- parameters: {
41
- id: 10513336322,
42
- title: 'The Guardian'
43
- }
44
- }]
45
-
46
- # Push each of the existing resources back into the new resources array
47
- source[:data][:resources].each do |resource|
48
- update_resources.push(resource)
49
- end
50
-
51
- puts "\nUpdating; adding a new resource, and changing the name\n--"
52
- puts @datasift.managed_source.update(
53
- id, 'facebook_page', 'Updated Ruby test', source[:data][:parameters],
54
- update_resources, source[:data][:auth]
55
- )[:data].to_json
56
-
57
- puts "\nGetting info from DataSift about my source\n--"
58
- puts @datasift.managed_source.get(id)[:data].to_json
59
-
60
- # Define new resources to add to Managed Source
61
- new_resources = [
62
- {
63
- parameters: {
64
- id: 5281959998,
65
- title: 'The New York Times'
66
- }
67
- },
68
- {
69
- parameters: {
70
- id: 18468761129,
71
- title: 'The Huffington Post'
72
- }
73
- },
74
- {
75
- parameters: {
76
- id: 97212224368,
77
- title: 'CNBC'
78
- }
79
- }
80
- ]
81
-
82
- new_auth = [{
83
- parameters: {
84
- value: 'CAAIUKbXn8xsBAL7eP......9hcU0b4ZVwlMe9dH5G93Nmvfi2EHJ7nXkRfc7'
85
- }
86
- }]
87
-
88
- puts "\nAdding new resources to my source (as an array of Ruby objects)\n--"
89
- puts @datasift.managed_source_resource.add(
90
- id, new_resources
91
- )[:data].to_json
92
-
93
- puts "\nAdding a new token to my source (as an array of Ruby objects)\n--"
94
- puts @datasift.managed_source_auth.add(id, new_auth)[:data].to_json
95
-
96
- puts "\nGetting info from DataSift about my source (notice the new " \
97
- "resources and tokens have been added)\n--"
98
- source = @datasift.managed_source.get id
99
- puts source[:data].to_json
100
-
101
- puts "\nRemoving a resource from my source by resource_id\n--"
102
- puts @datasift.managed_source_resource.remove(
103
- id, [source[:data][:resources][0][:resource_id]]
104
- )[:data].to_json
105
-
106
- puts "\nRemoving an auth token from my source by identity_id\n--"
107
- puts @datasift.managed_source_auth.remove(
108
- id, [source[:data][:auth][0][:identity_id]]
109
- )[:data].to_json
110
-
111
- puts "\nGetting info from DataSift about my source (notice an auth " \
112
- "token and resource have been removed)\n--"
113
- puts @datasift.managed_source.get(id)[:data].to_json
114
-
115
- puts "\nFetching logs (Any error logs for your source will appear here)\n--"
116
- puts @datasift.managed_source.log(id)[:data].to_json
117
-
118
- puts "\nStopping the Managed Source\n--"
119
- puts @datasift.managed_source.stop(id)[:data].to_json
120
-
121
- puts "\nDeleting the Managed Source\n--"
122
- puts @datasift.managed_source.delete(id)[:data].to_json
123
-
124
- rescue DataSiftError => dse
125
- puts dse.message
126
- end
127
- end
128
-
129
- ManagedSourceApi.new
@@ -1,126 +0,0 @@
1
- require './auth'
2
- class ManagedSourceApi < DataSiftExample
3
- def initialize
4
- super
5
- run
6
- end
7
-
8
- def run
9
- puts "Creating a managed source--\n"
10
- parameters = {
11
- likes: true,
12
- comments: true
13
- }
14
- resources = [{
15
- parameters: {
16
- type: 'user',
17
- value: 25025320
18
- }
19
- }]
20
- auth = [{
21
- parameters: {
22
- value: '10942122.00a3229.fff654d524854054bdb0288a05cdbdd1'
23
- }
24
- }]
25
-
26
- source = @datasift.managed_source.create(
27
- 'instagram', 'Ruby test', parameters, resources, auth
28
- )
29
- puts source[:data].to_json
30
-
31
- id = source[:data][:id]
32
-
33
- puts "\nStarting delivery for my Managed Source--\n"
34
- puts @datasift.managed_source.start(id)[:data].to_json
35
-
36
- # Define new resources to be added
37
- update_resources = [{
38
- parameters: {
39
- type: 'user',
40
- value: 8139971
41
- }
42
- }]
43
-
44
- # Push each of the existing resources back into the new resources array
45
- source[:data][:resources].each do |resource|
46
- update_resources.push(resource)
47
- end
48
-
49
- puts "\nUpdating; adding a new resource, and changing the name--\n"
50
- puts @datasift.managed_source.update(
51
- id, 'instagram', 'Updated Ruby test', source[:data][:parameters],
52
- update_resources, source[:data][:auth]
53
- )[:data].to_json
54
-
55
- puts "\nGetting info from DataSift about my source--\n"
56
- puts @datasift.managed_source.get(id)[:data].to_json
57
-
58
- # Define new resources to add to Managed Source
59
- new_resources = [
60
- {
61
- parameters: {
62
- type: 'tag',
63
- value: 'sun'
64
- }
65
- },
66
- {
67
- parameters: {
68
- type: 'tag',
69
- value: 'sea'
70
- }
71
- },
72
- {
73
- parameters: {
74
- type: 'tag',
75
- value: 'surf'
76
- }
77
- }
78
- ]
79
-
80
- new_auth = [{
81
- parameters: {
82
- value: '10942111.1f2134f.8837abb205b44ece801022f6fa989cc4'
83
- }
84
- }]
85
-
86
- puts "\nAdding a new resource to my source (as array of Ruby objects)--\n"
87
- puts @datasift.managed_source_resource.add(
88
- id, new_resources
89
- )[:data].to_json
90
-
91
- puts "\nAdding a new token to my source (as array of Ruby objects)--\n"
92
- puts @datasift.managed_source_auth.add(id, new_auth)[:data].to_json
93
-
94
- puts "\nGetting info from DataSift about my source (notice the new " \
95
- "resources and tokens have been added)--\n"
96
- source = @datasift.managed_source.get id
97
- puts source[:data].to_json
98
-
99
- puts "\nRemoving a resource from my source by resource_id--\n"
100
- puts @datasift.managed_source_resource.remove(
101
- id, [source[:data][:resources][0][:resource_id]]
102
- )[:data].to_json
103
-
104
- puts "\nRemoving an auth token from my source by identity_id--\n"
105
- puts @datasift.managed_source_auth.remove(
106
- id, [source[:data][:auth][0][:identity_id]]
107
- )[:data].to_json
108
-
109
- puts "\nGetting info from DataSift about my source (notice an auth " \
110
- "token and resource have been removed)--\n"
111
- puts @datasift.managed_source.get(id)[:data].to_json
112
-
113
- puts "\nFetching logs--\n"
114
- puts @datasift.managed_source.log(id)[:data].to_json
115
-
116
- puts "\nStopping--\n"
117
- puts @datasift.managed_source.stop(id)[:data].to_json
118
-
119
- puts "\nDeleting--\n"
120
- puts @datasift.managed_source.delete(id)[:data].to_json
121
- rescue DataSiftError => dse
122
- puts dse.message
123
- end
124
- end
125
-
126
- ManagedSourceApi.new
@@ -1,47 +0,0 @@
1
- require './auth'
2
- class PushApi < DataSiftExample
3
- def initialize
4
- super
5
- end
6
-
7
- def run
8
- begin
9
- @params = {output_type: 'pull'}
10
- puts 'Validating the Pull subscription'
11
- fail InvalidParamError unless @datasift.push.valid? @params
12
-
13
- stream = @datasift.compile 'interaction.content contains "music"'
14
- subscription = create_push(stream[:data][:hash])
15
-
16
- subscription_id = subscription[:data][:id]
17
- # Pull a bunch of interactions from the push queue. This only works if we had set the
18
- # output_type above to "pull"
19
-
20
- 2.times do
21
- puts "\nPulling data, then waiting 10 seconds"
22
- @datasift.push.pull(subscription_id).each { |e| puts e }
23
- sleep 10
24
- end
25
-
26
- puts "\nPulling data the third and final time time"
27
- # Passing a lambda is more efficient because it is executed once for each interaction
28
- # received this saves having to iterate over the array returned so the same iteration
29
- # isn't processed twice
30
- @datasift.push.pull(
31
- subscription_id,
32
- 20_971_520,
33
- '',
34
- lambda{ |e| puts "on_message => #{e}" }
35
- )
36
-
37
- puts "\nDeleting the Pull subscription"
38
- @datasift.push.delete subscription_id
39
-
40
- #rescue DataSiftError
41
- rescue DataSiftError => dse
42
- puts dse.inspect
43
- end
44
- end
45
- end
46
-
47
- PushApi.new().run
@@ -1,51 +0,0 @@
1
- require './auth'
2
- class PushApi < DataSiftExample
3
- def initialize
4
- super
5
- run
6
- end
7
-
8
- def run
9
- begin
10
- puts 'Creating Push subscription'
11
- subscription = @datasift.push.create @params.merge(
12
- hash: '54dbfc8464258de162b7f1a057e630c5',
13
- name: 'Ruby Client Example'
14
- )
15
-
16
- subscription_id = subscription[:data][:id]
17
- puts "\nPush subscription created! Push Subscription ID #{subscription_id}"
18
-
19
- puts "\nGetting subscription info"
20
- # Get details for a subscription. Also available are
21
- # push.[get, get_by_hash, get_by_historics_id]
22
- puts @datasift.push.get_by_subscription subscription_id
23
-
24
- puts "\nPausing Push subscription"
25
- # Push subscriptions can be paused for up to an hour
26
- @datasift.push.pause subscription_id
27
-
28
- puts "\nResuming Push subscription"
29
- # Push subscriptions must be resumed to continue delivering data
30
- @datasift.push.resume subscription_id
31
-
32
- puts "\nGetting subscription logs"
33
- # Get logs for a subscription. Also available is push.log to get logs for all subscriptions
34
- puts @datasift.push.log_for subscription_id
35
-
36
- puts "\nStopping Push subscription"
37
- # Push subscriptions can be stopped. Once stopped, a subscription can not be resumed
38
- @datasift.push.stop subscription_id
39
-
40
- puts "\nDeleting Push subscription"
41
- # Push subscriptions can be deleted. On delete, any undelivered data is dropped. A delete
42
- # is permenent.
43
- @datasift.push.delete subscription_id
44
-
45
- rescue DataSiftError => dse
46
- puts dse.message
47
- end
48
- end
49
- end
50
-
51
- PushApi.new()