datasift 3.7.2 → 3.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +8 -0
  3. data/datasift.gemspec +6 -3
  4. data/lib/api/api_resource.rb +1 -1
  5. data/lib/datasift.rb +53 -16
  6. data/lib/errors.rb +8 -0
  7. data/lib/live_stream.rb +7 -7
  8. data/lib/pylon.rb +59 -26
  9. data/lib/tasks.rb +51 -0
  10. data/lib/version.rb +3 -0
  11. metadata +5 -83
  12. data/.gitignore +0 -12
  13. data/.rubocop.yml +0 -12
  14. data/.rubocop_todo.yml +0 -497
  15. data/.travis.yml +0 -17
  16. data/Gemfile +0 -15
  17. data/LICENSE +0 -19
  18. data/MIGRATING_TO_V.3.0.0.md +0 -262
  19. data/Rakefile +0 -10
  20. data/VERSION +0 -1
  21. data/examples/account_eg.rb +0 -33
  22. data/examples/account_identity_eg.rb +0 -50
  23. data/examples/account_identity_limit_eg.rb +0 -72
  24. data/examples/account_identity_token_eg.rb +0 -73
  25. data/examples/auth.rb +0 -55
  26. data/examples/cli.sh +0 -155
  27. data/examples/core_api_eg.rb +0 -50
  28. data/examples/historics_eg.rb +0 -66
  29. data/examples/historics_preview_eg.rb +0 -31
  30. data/examples/ingestion/twitter_gnip_batch_eg.rb +0 -61
  31. data/examples/live_stream_eg.rb +0 -92
  32. data/examples/managed_source_fb_eg.rb +0 -129
  33. data/examples/managed_source_ig_eg.rb +0 -126
  34. data/examples/pull.rb +0 -47
  35. data/examples/push_eg.rb +0 -51
  36. data/examples/pylon/pylon_api_v1.2_eg.rb +0 -166
  37. data/examples/pylon/pylon_api_v1.3_eg.rb +0 -201
  38. data/test/datasift/account/account_api_test.rb +0 -35
  39. data/test/datasift/core_api_test.rb +0 -179
  40. data/test/datasift/historics_preview_api_test.rb +0 -56
  41. data/test/datasift/odp/batch_upload_test.rb +0 -52
  42. data/test/datasift/push_api_test.rb +0 -238
  43. data/test/fixtures/cassettes/account/usage/default.json +0 -1
  44. data/test/fixtures/cassettes/account/usage/invalid.json +0 -1
  45. data/test/fixtures/cassettes/account/usage/valid_params.json +0 -1
  46. data/test/fixtures/cassettes/core/after_historic_dpu.json +0 -1
  47. data/test/fixtures/cassettes/core/balance_get.json +0 -1
  48. data/test/fixtures/cassettes/core/before_dpu.json +0 -1
  49. data/test/fixtures/cassettes/core/before_historic_dpu.json +0 -1
  50. data/test/fixtures/cassettes/core/compile_success.json +0 -1
  51. data/test/fixtures/cassettes/core/dpu_get_cost.json +0 -1
  52. data/test/fixtures/cassettes/core/dpu_throw_badrequest.json +0 -1
  53. data/test/fixtures/cassettes/core/historic_dpu.json +0 -1
  54. data/test/fixtures/cassettes/core/usage_success.json +0 -1
  55. data/test/fixtures/cassettes/core/validate_invalid_hash.json +0 -1
  56. data/test/fixtures/cassettes/core/validate_success_bool.json +0 -1
  57. data/test/fixtures/cassettes/core/validate_success_hash.json +0 -1
  58. data/test/fixtures/cassettes/odp/batch/after_upload.json +0 -1
  59. data/test/fixtures/cassettes/odp/batch/before_upload.json +0 -1
  60. data/test/fixtures/cassettes/odp/batch/upload_failure_no_source.json +0 -1
  61. data/test/fixtures/cassettes/odp/batch/upload_success.json +0 -1
  62. data/test/fixtures/cassettes/preview/before_preview_create.json +0 -1
  63. data/test/fixtures/cassettes/preview/before_preview_get.json +0 -1
  64. data/test/fixtures/cassettes/preview/preview_create_success.json +0 -1
  65. data/test/fixtures/cassettes/preview/preview_get_success.json +0 -1
  66. data/test/fixtures/cassettes/push/after_push_create.json +0 -1
  67. data/test/fixtures/cassettes/push/after_push_get.json +0 -1
  68. data/test/fixtures/cassettes/push/after_push_log.json +0 -1
  69. data/test/fixtures/cassettes/push/after_push_pause.json +0 -1
  70. data/test/fixtures/cassettes/push/after_push_resume.json +0 -1
  71. data/test/fixtures/cassettes/push/after_push_stop.json +0 -1
  72. data/test/fixtures/cassettes/push/after_push_update.json +0 -1
  73. data/test/fixtures/cassettes/push/before_push_create.json +0 -1
  74. data/test/fixtures/cassettes/push/before_push_delete.json +0 -1
  75. data/test/fixtures/cassettes/push/before_push_get.json +0 -1
  76. data/test/fixtures/cassettes/push/before_push_log.json +0 -1
  77. data/test/fixtures/cassettes/push/before_push_pause.json +0 -1
  78. data/test/fixtures/cassettes/push/before_push_resume.json +0 -1
  79. data/test/fixtures/cassettes/push/before_push_stop.json +0 -1
  80. data/test/fixtures/cassettes/push/before_push_update.json +0 -1
  81. data/test/fixtures/cassettes/push/push_create.json +0 -1
  82. data/test/fixtures/cassettes/push/push_delete.json +0 -1
  83. data/test/fixtures/cassettes/push/push_get_by_id.json +0 -1
  84. data/test/fixtures/cassettes/push/push_log_with_id.json +0 -1
  85. data/test/fixtures/cassettes/push/push_pause.json +0 -1
  86. data/test/fixtures/cassettes/push/push_resume.json +0 -1
  87. data/test/fixtures/cassettes/push/push_stop.json +0 -1
  88. data/test/fixtures/cassettes/push/push_update.json +0 -1
  89. data/test/fixtures/cassettes/push/push_validate.json +0 -1
  90. data/test/fixtures/data/fake_gnip_tweets.json +0 -10
  91. data/test/test_helper.rb +0 -49
@@ -1,92 +0,0 @@
1
- require './auth'
2
- class StreamingApi < DataSiftExample
3
-
4
- def initialize
5
- super
6
- @datasift = DataSift::Client.new(@config)
7
- run
8
- end
9
-
10
- def run
11
- begin
12
- ruby_received = 0
13
- python_received = 0
14
- ruby = 'interaction.content contains "ruby"'
15
- ruby_stream = @datasift.compile ruby
16
-
17
- python = 'interaction.content contains "python"'
18
- python_stream = @datasift.compile python
19
-
20
- on_delete = lambda { |stream, m| puts 'We must delete this to be compliant ==> ' + m }
21
-
22
- on_error = lambda do |stream, e|
23
- # This is called when an error occurs on your connection; you should probably log this event
24
- puts 'A serious error has occurred'
25
- puts e.message
26
- end
27
-
28
- on_message_ruby = lambda do |message, stream, hash|
29
- ruby_received += 1
30
- puts "Ruby #{ruby_received}, #{message}"
31
-
32
- if ruby_received >= 10
33
- puts 'un-subscribing from ruby stream '+ hash
34
- stream.unsubscribe hash
35
- end
36
- end
37
-
38
- on_message_python = lambda do |message, stream, hash|
39
- python_received += 1
40
- puts "python #{python_received}, #{message}"
41
-
42
- if python_received >= 10
43
- puts 'un-subscribing from python stream '+ hash
44
- stream.unsubscribe hash
45
- end
46
- end
47
-
48
- on_connect = lambda do |stream|
49
- #
50
- puts 'subscribing to python stream '+ python_stream[:data][:hash]
51
- stream.subscribe(python_stream[:data][:hash], on_message_python)
52
- puts 'Subscribed to '+ python_stream[:data][:hash]
53
- sleep 1
54
- #
55
- puts 'subscribing to ruby stream '+ ruby_stream[:data][:hash]
56
- stream.subscribe(ruby_stream[:data][:hash], on_message_ruby)
57
- puts 'Subscribed to '+ ruby_stream[:data][:hash]
58
- end
59
-
60
- on_close = lambda do |stream,msg|
61
- # This is called when a connection is closed; you should probably log this event
62
- puts msg
63
- end
64
-
65
- on_datasift_message = lambda do |stream, message, hash|
66
- #not all messages have a hash
67
- puts "is_success = #{message[:is_success]}, is_failure = #{message[:is_failure]}, is_warning = #{message[:is_warning]}, is_tick = #{message[:is_tick]}"
68
- puts "DataSift Message #{hash} ==> #{message}"
69
- end
70
-
71
- conn = DataSift::new_stream(@config, on_delete, on_error, on_connect, on_close)
72
- conn.on_datasift_message = on_datasift_message
73
- #can do something else here now...
74
- puts 'Do some other business stuff...'
75
- conn.stream.read_thread.join
76
- #rescue DataSiftError
77
- rescue DataSiftError => dse
78
- puts dse.inspect
79
- # Then match specific one to take action; All errors thrown by the client extend DataSiftError
80
- case dse
81
- when ConnectionError
82
- # some connection error
83
- when AuthError
84
- when BadRequestError
85
- else
86
- # do something else...
87
- end
88
- end
89
- end
90
- end
91
-
92
- StreamingApi.new
@@ -1,129 +0,0 @@
1
- require './auth'
2
- class ManagedSourceApi < DataSiftExample
3
- def initialize
4
- super
5
- run
6
- end
7
-
8
- def run
9
- puts "Creating a managed source\n--"
10
- parameters = {
11
- likes: true,
12
- posts_by_others: true,
13
- comments: true,
14
- page_likes: true
15
- }
16
- resources = [{
17
- parameters: {
18
- id: 130871750291428,
19
- title: 'DataSift'
20
- }
21
- }]
22
- auth = [{
23
- parameters: {
24
- value: 'vnHQocnOEChoOsYYOHVIn80......EOMmZ63go6s0DzxsAmJaDeE2ljdQjDqJVT'
25
- }
26
- }]
27
-
28
- source = @datasift.managed_source.create(
29
- 'facebook_page', 'Ruby test', parameters, resources, auth
30
- )
31
- puts source[:data].to_json
32
-
33
- id = source[:data][:id]
34
-
35
- puts "\nStarting delivery for my Managed Source\n--"
36
- puts @datasift.managed_source.start(id)[:data].to_json
37
-
38
- # Define new resources to be added
39
- update_resources = [{
40
- parameters: {
41
- id: 10513336322,
42
- title: 'The Guardian'
43
- }
44
- }]
45
-
46
- # Push each of the existing resources back into the new resources array
47
- source[:data][:resources].each do |resource|
48
- update_resources.push(resource)
49
- end
50
-
51
- puts "\nUpdating; adding a new resource, and changing the name\n--"
52
- puts @datasift.managed_source.update(
53
- id, 'facebook_page', 'Updated Ruby test', source[:data][:parameters],
54
- update_resources, source[:data][:auth]
55
- )[:data].to_json
56
-
57
- puts "\nGetting info from DataSift about my source\n--"
58
- puts @datasift.managed_source.get(id)[:data].to_json
59
-
60
- # Define new resources to add to Managed Source
61
- new_resources = [
62
- {
63
- parameters: {
64
- id: 5281959998,
65
- title: 'The New York Times'
66
- }
67
- },
68
- {
69
- parameters: {
70
- id: 18468761129,
71
- title: 'The Huffington Post'
72
- }
73
- },
74
- {
75
- parameters: {
76
- id: 97212224368,
77
- title: 'CNBC'
78
- }
79
- }
80
- ]
81
-
82
- new_auth = [{
83
- parameters: {
84
- value: 'CAAIUKbXn8xsBAL7eP......9hcU0b4ZVwlMe9dH5G93Nmvfi2EHJ7nXkRfc7'
85
- }
86
- }]
87
-
88
- puts "\nAdding new resources to my source (as an array of Ruby objects)\n--"
89
- puts @datasift.managed_source_resource.add(
90
- id, new_resources
91
- )[:data].to_json
92
-
93
- puts "\nAdding a new token to my source (as an array of Ruby objects)\n--"
94
- puts @datasift.managed_source_auth.add(id, new_auth)[:data].to_json
95
-
96
- puts "\nGetting info from DataSift about my source (notice the new " \
97
- "resources and tokens have been added)\n--"
98
- source = @datasift.managed_source.get id
99
- puts source[:data].to_json
100
-
101
- puts "\nRemoving a resource from my source by resource_id\n--"
102
- puts @datasift.managed_source_resource.remove(
103
- id, [source[:data][:resources][0][:resource_id]]
104
- )[:data].to_json
105
-
106
- puts "\nRemoving an auth token from my source by identity_id\n--"
107
- puts @datasift.managed_source_auth.remove(
108
- id, [source[:data][:auth][0][:identity_id]]
109
- )[:data].to_json
110
-
111
- puts "\nGetting info from DataSift about my source (notice an auth " \
112
- "token and resource have been removed)\n--"
113
- puts @datasift.managed_source.get(id)[:data].to_json
114
-
115
- puts "\nFetching logs (Any error logs for your source will appear here)\n--"
116
- puts @datasift.managed_source.log(id)[:data].to_json
117
-
118
- puts "\nStopping the Managed Source\n--"
119
- puts @datasift.managed_source.stop(id)[:data].to_json
120
-
121
- puts "\nDeleting the Managed Source\n--"
122
- puts @datasift.managed_source.delete(id)[:data].to_json
123
-
124
- rescue DataSiftError => dse
125
- puts dse.message
126
- end
127
- end
128
-
129
- ManagedSourceApi.new
@@ -1,126 +0,0 @@
1
- require './auth'
2
- class ManagedSourceApi < DataSiftExample
3
- def initialize
4
- super
5
- run
6
- end
7
-
8
- def run
9
- puts "Creating a managed source--\n"
10
- parameters = {
11
- likes: true,
12
- comments: true
13
- }
14
- resources = [{
15
- parameters: {
16
- type: 'user',
17
- value: 25025320
18
- }
19
- }]
20
- auth = [{
21
- parameters: {
22
- value: '10942122.00a3229.fff654d524854054bdb0288a05cdbdd1'
23
- }
24
- }]
25
-
26
- source = @datasift.managed_source.create(
27
- 'instagram', 'Ruby test', parameters, resources, auth
28
- )
29
- puts source[:data].to_json
30
-
31
- id = source[:data][:id]
32
-
33
- puts "\nStarting delivery for my Managed Source--\n"
34
- puts @datasift.managed_source.start(id)[:data].to_json
35
-
36
- # Define new resources to be added
37
- update_resources = [{
38
- parameters: {
39
- type: 'user',
40
- value: 8139971
41
- }
42
- }]
43
-
44
- # Push each of the existing resources back into the new resources array
45
- source[:data][:resources].each do |resource|
46
- update_resources.push(resource)
47
- end
48
-
49
- puts "\nUpdating; adding a new resource, and changing the name--\n"
50
- puts @datasift.managed_source.update(
51
- id, 'instagram', 'Updated Ruby test', source[:data][:parameters],
52
- update_resources, source[:data][:auth]
53
- )[:data].to_json
54
-
55
- puts "\nGetting info from DataSift about my source--\n"
56
- puts @datasift.managed_source.get(id)[:data].to_json
57
-
58
- # Define new resources to add to Managed Source
59
- new_resources = [
60
- {
61
- parameters: {
62
- type: 'tag',
63
- value: 'sun'
64
- }
65
- },
66
- {
67
- parameters: {
68
- type: 'tag',
69
- value: 'sea'
70
- }
71
- },
72
- {
73
- parameters: {
74
- type: 'tag',
75
- value: 'surf'
76
- }
77
- }
78
- ]
79
-
80
- new_auth = [{
81
- parameters: {
82
- value: '10942111.1f2134f.8837abb205b44ece801022f6fa989cc4'
83
- }
84
- }]
85
-
86
- puts "\nAdding a new resource to my source (as array of Ruby objects)--\n"
87
- puts @datasift.managed_source_resource.add(
88
- id, new_resources
89
- )[:data].to_json
90
-
91
- puts "\nAdding a new token to my source (as array of Ruby objects)--\n"
92
- puts @datasift.managed_source_auth.add(id, new_auth)[:data].to_json
93
-
94
- puts "\nGetting info from DataSift about my source (notice the new " \
95
- "resources and tokens have been added)--\n"
96
- source = @datasift.managed_source.get id
97
- puts source[:data].to_json
98
-
99
- puts "\nRemoving a resource from my source by resource_id--\n"
100
- puts @datasift.managed_source_resource.remove(
101
- id, [source[:data][:resources][0][:resource_id]]
102
- )[:data].to_json
103
-
104
- puts "\nRemoving an auth token from my source by identity_id--\n"
105
- puts @datasift.managed_source_auth.remove(
106
- id, [source[:data][:auth][0][:identity_id]]
107
- )[:data].to_json
108
-
109
- puts "\nGetting info from DataSift about my source (notice an auth " \
110
- "token and resource have been removed)--\n"
111
- puts @datasift.managed_source.get(id)[:data].to_json
112
-
113
- puts "\nFetching logs--\n"
114
- puts @datasift.managed_source.log(id)[:data].to_json
115
-
116
- puts "\nStopping--\n"
117
- puts @datasift.managed_source.stop(id)[:data].to_json
118
-
119
- puts "\nDeleting--\n"
120
- puts @datasift.managed_source.delete(id)[:data].to_json
121
- rescue DataSiftError => dse
122
- puts dse.message
123
- end
124
- end
125
-
126
- ManagedSourceApi.new
@@ -1,47 +0,0 @@
1
- require './auth'
2
- class PushApi < DataSiftExample
3
- def initialize
4
- super
5
- end
6
-
7
- def run
8
- begin
9
- @params = {output_type: 'pull'}
10
- puts 'Validating the Pull subscription'
11
- fail InvalidParamError unless @datasift.push.valid? @params
12
-
13
- stream = @datasift.compile 'interaction.content contains "music"'
14
- subscription = create_push(stream[:data][:hash])
15
-
16
- subscription_id = subscription[:data][:id]
17
- # Pull a bunch of interactions from the push queue. This only works if we had set the
18
- # output_type above to "pull"
19
-
20
- 2.times do
21
- puts "\nPulling data, then waiting 10 seconds"
22
- @datasift.push.pull(subscription_id).each { |e| puts e }
23
- sleep 10
24
- end
25
-
26
- puts "\nPulling data the third and final time time"
27
- # Passing a lambda is more efficient because it is executed once for each interaction
28
- # received this saves having to iterate over the array returned so the same iteration
29
- # isn't processed twice
30
- @datasift.push.pull(
31
- subscription_id,
32
- 20_971_520,
33
- '',
34
- lambda{ |e| puts "on_message => #{e}" }
35
- )
36
-
37
- puts "\nDeleting the Pull subscription"
38
- @datasift.push.delete subscription_id
39
-
40
- #rescue DataSiftError
41
- rescue DataSiftError => dse
42
- puts dse.inspect
43
- end
44
- end
45
- end
46
-
47
- PushApi.new().run
@@ -1,51 +0,0 @@
1
- require './auth'
2
- class PushApi < DataSiftExample
3
- def initialize
4
- super
5
- run
6
- end
7
-
8
- def run
9
- begin
10
- puts 'Creating Push subscription'
11
- subscription = @datasift.push.create @params.merge(
12
- hash: '54dbfc8464258de162b7f1a057e630c5',
13
- name: 'Ruby Client Example'
14
- )
15
-
16
- subscription_id = subscription[:data][:id]
17
- puts "\nPush subscription created! Push Subscription ID #{subscription_id}"
18
-
19
- puts "\nGetting subscription info"
20
- # Get details for a subscription. Also available are
21
- # push.[get, get_by_hash, get_by_historics_id]
22
- puts @datasift.push.get_by_subscription subscription_id
23
-
24
- puts "\nPausing Push subscription"
25
- # Push subscriptions can be paused for up to an hour
26
- @datasift.push.pause subscription_id
27
-
28
- puts "\nResuming Push subscription"
29
- # Push subscriptions must be resumed to continue delivering data
30
- @datasift.push.resume subscription_id
31
-
32
- puts "\nGetting subscription logs"
33
- # Get logs for a subscription. Also available is push.log to get logs for all subscriptions
34
- puts @datasift.push.log_for subscription_id
35
-
36
- puts "\nStopping Push subscription"
37
- # Push subscriptions can be stopped. Once stopped, a subscription can not be resumed
38
- @datasift.push.stop subscription_id
39
-
40
- puts "\nDeleting Push subscription"
41
- # Push subscriptions can be deleted. On delete, any undelivered data is dropped. A delete
42
- # is permenent.
43
- @datasift.push.delete subscription_id
44
-
45
- rescue DataSiftError => dse
46
- puts dse.message
47
- end
48
- end
49
- end
50
-
51
- PushApi.new()