datasift 2.1.1 → 3.0.0.beta

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -0
  3. data/CHANGELOG.md +100 -0
  4. data/Gemfile.lock +32 -0
  5. data/README.md +38 -79
  6. data/VERSION +1 -1
  7. data/datasift.gemspec +21 -24
  8. data/examples/auth.rb +44 -0
  9. data/examples/core_api_eg.rb +46 -0
  10. data/examples/historics_eg.rb +50 -0
  11. data/examples/historics_preview_eg.rb +30 -0
  12. data/examples/live_stream_eg.rb +89 -0
  13. data/examples/managed_source_eg.rb +56 -0
  14. data/examples/pull.rb +44 -0
  15. data/examples/push_eg.rb +56 -0
  16. data/lib/api/api_resource.rb +23 -0
  17. data/lib/datasift.rb +287 -14
  18. data/lib/errors.rb +59 -0
  19. data/lib/historics.rb +76 -0
  20. data/lib/historics_preview.rb +20 -0
  21. data/lib/live_stream.rb +53 -0
  22. data/lib/managed_source.rb +57 -0
  23. data/lib/push.rb +156 -0
  24. data/tests/core_api_test.rb +42 -0
  25. metadata +51 -73
  26. data/Rakefile +0 -34
  27. data/config.yml +0 -2
  28. data/examples/consume-stream.rb +0 -63
  29. data/examples/deletes.rb +0 -52
  30. data/examples/dpu.rb +0 -115
  31. data/examples/football-buffered.rb +0 -51
  32. data/examples/football.rb +0 -53
  33. data/examples/historics.sh +0 -2
  34. data/examples/historics/create-from-csdl.rb +0 -71
  35. data/examples/historics/create-from-hash.rb +0 -65
  36. data/examples/historics/delete.rb +0 -30
  37. data/examples/historics/env.rb +0 -37
  38. data/examples/historics/list.rb +0 -30
  39. data/examples/historics/start.rb +0 -30
  40. data/examples/historics/stop.rb +0 -30
  41. data/examples/historics/view.rb +0 -28
  42. data/examples/push.sh +0 -2
  43. data/examples/push/delete.rb +0 -33
  44. data/examples/push/env.rb +0 -53
  45. data/examples/push/list.rb +0 -30
  46. data/examples/push/pause.rb +0 -33
  47. data/examples/push/push-from-hash.rb +0 -72
  48. data/examples/push/push-historic-from-csdl.rb +0 -98
  49. data/examples/push/push-stream-from-csdl.rb +0 -70
  50. data/examples/push/resume.rb +0 -33
  51. data/examples/push/stop.rb +0 -33
  52. data/examples/push/view-log.rb +0 -45
  53. data/examples/push/view.rb +0 -31
  54. data/examples/twitter-track.rb +0 -61
  55. data/lib/DataSift/apiclient.rb +0 -73
  56. data/lib/DataSift/definition.rb +0 -202
  57. data/lib/DataSift/exceptions.rb +0 -33
  58. data/lib/DataSift/historic.rb +0 -316
  59. data/lib/DataSift/managed_source.rb +0 -263
  60. data/lib/DataSift/mockapiclient.rb +0 -44
  61. data/lib/DataSift/push_definition.rb +0 -115
  62. data/lib/DataSift/push_subscription.rb +0 -330
  63. data/lib/DataSift/stream_consumer.rb +0 -166
  64. data/lib/DataSift/stream_consumer_http.rb +0 -188
  65. data/lib/DataSift/user.rb +0 -311
  66. data/test/helper.rb +0 -95
  67. data/test/test_definition.rb +0 -273
  68. data/test/test_historics.rb +0 -233
  69. data/test/test_pushdefinition.rb +0 -92
  70. data/test/test_pushsubscription.rb +0 -17
  71. data/test/test_user.rb +0 -130
  72. data/test/testdata.yml +0 -30
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: datasift
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.1.1
4
+ version: 3.0.0.beta
5
5
  platform: ruby
6
6
  authors:
7
7
  - DataSift
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-05-16 00:00:00.000000000 Z
11
+ date: 2013-11-08 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rest-client
@@ -16,28 +16,42 @@ dependencies:
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '1.6'
19
+ version: 1.6.3
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '1.6'
26
+ version: 1.6.3
27
27
  - !ruby/object:Gem::Dependency
28
- name: yajl-ruby
28
+ name: multi_json
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
31
  - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: '1.1'
33
+ version: 1.8.0
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: 1.8.0
41
+ - !ruby/object:Gem::Dependency
42
+ name: websocket-td
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: 0.0.3
34
48
  type: :runtime
35
49
  prerelease: false
36
50
  version_requirements: !ruby/object:Gem::Requirement
37
51
  requirements:
38
52
  - - "~>"
39
53
  - !ruby/object:Gem::Version
40
- version: '1.1'
54
+ version: 0.0.3
41
55
  - !ruby/object:Gem::Dependency
42
56
  name: rdoc
43
57
  requirement: !ruby/object:Gem::Requirement
@@ -56,30 +70,30 @@ dependencies:
56
70
  name: shoulda
57
71
  requirement: !ruby/object:Gem::Requirement
58
72
  requirements:
59
- - - ">="
73
+ - - "~>"
60
74
  - !ruby/object:Gem::Version
61
- version: '2.11'
75
+ version: 2.11.3
62
76
  type: :development
63
77
  prerelease: false
64
78
  version_requirements: !ruby/object:Gem::Requirement
65
79
  requirements:
66
- - - ">="
80
+ - - "~>"
67
81
  - !ruby/object:Gem::Version
68
- version: '2.11'
82
+ version: 2.11.3
69
83
  - !ruby/object:Gem::Dependency
70
- name: rspec
84
+ name: test-unit
71
85
  requirement: !ruby/object:Gem::Requirement
72
86
  requirements:
73
- - - "~>"
87
+ - - ">="
74
88
  - !ruby/object:Gem::Version
75
- version: '2.6'
89
+ version: 2.5.5
76
90
  type: :development
77
91
  prerelease: false
78
92
  version_requirements: !ruby/object:Gem::Requirement
79
93
  requirements:
80
- - - "~>"
94
+ - - ">="
81
95
  - !ruby/object:Gem::Version
82
- version: '2.6'
96
+ version: 2.5.5
83
97
  description: The official Ruby library for accessing the DataSift API. See http://datasift.com/
84
98
  for full details and to sign up for an account.
85
99
  email:
@@ -89,60 +103,31 @@ extensions: []
89
103
  extra_rdoc_files: []
90
104
  files:
91
105
  - ".gitignore"
106
+ - CHANGELOG.md
92
107
  - Gemfile
108
+ - Gemfile.lock
93
109
  - LICENSE
94
110
  - README.md
95
- - Rakefile
96
111
  - VERSION
97
- - config.yml
98
112
  - datasift.gemspec
99
- - examples/consume-stream.rb
100
- - examples/deletes.rb
101
- - examples/dpu.rb
102
- - examples/football-buffered.rb
103
- - examples/football.rb
104
- - examples/historics.sh
105
- - examples/historics/create-from-csdl.rb
106
- - examples/historics/create-from-hash.rb
107
- - examples/historics/delete.rb
108
- - examples/historics/env.rb
109
- - examples/historics/list.rb
110
- - examples/historics/start.rb
111
- - examples/historics/stop.rb
112
- - examples/historics/view.rb
113
- - examples/push.sh
114
- - examples/push/delete.rb
115
- - examples/push/env.rb
116
- - examples/push/list.rb
117
- - examples/push/pause.rb
118
- - examples/push/push-from-hash.rb
119
- - examples/push/push-historic-from-csdl.rb
120
- - examples/push/push-stream-from-csdl.rb
121
- - examples/push/resume.rb
122
- - examples/push/stop.rb
123
- - examples/push/view-log.rb
124
- - examples/push/view.rb
125
- - examples/twitter-track.rb
126
- - lib/DataSift/apiclient.rb
127
- - lib/DataSift/definition.rb
128
- - lib/DataSift/exceptions.rb
129
- - lib/DataSift/historic.rb
130
- - lib/DataSift/managed_source.rb
131
- - lib/DataSift/mockapiclient.rb
132
- - lib/DataSift/push_definition.rb
133
- - lib/DataSift/push_subscription.rb
134
- - lib/DataSift/stream_consumer.rb
135
- - lib/DataSift/stream_consumer_http.rb
136
- - lib/DataSift/user.rb
113
+ - examples/auth.rb
114
+ - examples/core_api_eg.rb
115
+ - examples/historics_eg.rb
116
+ - examples/historics_preview_eg.rb
117
+ - examples/live_stream_eg.rb
118
+ - examples/managed_source_eg.rb
119
+ - examples/pull.rb
120
+ - examples/push_eg.rb
121
+ - lib/api/api_resource.rb
137
122
  - lib/datasift.rb
138
- - test/helper.rb
139
- - test/test_definition.rb
140
- - test/test_historics.rb
141
- - test/test_pushdefinition.rb
142
- - test/test_pushsubscription.rb
143
- - test/test_user.rb
144
- - test/testdata.yml
145
- homepage: http://github.com/datasift/datasift-ruby
123
+ - lib/errors.rb
124
+ - lib/historics.rb
125
+ - lib/historics_preview.rb
126
+ - lib/live_stream.rb
127
+ - lib/managed_source.rb
128
+ - lib/push.rb
129
+ - tests/core_api_test.rb
130
+ homepage: https://github.com/datasift/datasift-ruby
146
131
  licenses:
147
132
  - BSD
148
133
  metadata: {}
@@ -162,15 +147,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
162
147
  version: 1.3.6
163
148
  requirements: []
164
149
  rubyforge_project:
165
- rubygems_version: 2.2.2
150
+ rubygems_version: 2.1.5
166
151
  signing_key:
167
152
  specification_version: 4
168
153
  summary: DataSift is a simple wrapper for the DataSift API.
169
- test_files:
170
- - test/helper.rb
171
- - test/test_definition.rb
172
- - test/test_historics.rb
173
- - test/test_pushdefinition.rb
174
- - test/test_pushsubscription.rb
175
- - test/test_user.rb
176
- - test/testdata.yml
154
+ test_files: []
data/Rakefile DELETED
@@ -1,34 +0,0 @@
1
- require 'rubygems'
2
- require 'rake'
3
-
4
- require 'rake/testtask'
5
- Rake::TestTask.new(:test) do |test|
6
- test.libs << 'lib' << 'test'
7
- test.pattern = 'test/**/test_*.rb'
8
- test.verbose = true
9
- end
10
-
11
- begin
12
- require 'rcov/rcovtask'
13
- Rcov::RcovTask.new do |test|
14
- test.libs << 'test'
15
- test.pattern = 'test/**/test_*.rb'
16
- test.verbose = true
17
- end
18
- rescue LoadError
19
- task :rcov do
20
- abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
21
- end
22
- end
23
-
24
- require 'rdoc/task'
25
- RDoc::Task.new do |rdoc|
26
- version = File.exist?('VERSION') ? File.read('VERSION') : ""
27
-
28
- rdoc.rdoc_dir = 'rdoc'
29
- rdoc.title = "datasift #{version}"
30
- rdoc.rdoc_files.include('README*')
31
- rdoc.rdoc_files.include('lib/**/*.rb')
32
- end
33
-
34
- task :default => :test
data/config.yml DELETED
@@ -1,2 +0,0 @@
1
- username: YOUR_USERNAME_HERE
2
- api_key: YOUR_API_KEY_HERE
@@ -1,63 +0,0 @@
1
- # This simple example demonstrates consuming a stream from just the stream
2
- # hash.
3
- #
4
- # NB: Most of the error handling (exception catching) has been removed for
5
- # the sake of simplicity. Nearly everything in this library may throw
6
- # exceptions, and production code should catch them. See the documentation
7
- # for full details.
8
- #
9
-
10
- # Make sure we have some arguments
11
- if ARGV.size == 0
12
- puts 'ERR: Please specify the hash to consume!'
13
- puts
14
- puts
15
- exit!
16
- end
17
-
18
- # Include the DataSift library
19
- require './' + File.dirname(__FILE__) + '/../lib/datasift'
20
-
21
- # Include the configuration - put your username and API key in this file
22
- require 'yaml'
23
- config = YAML::load(File.open(File.join(File.dirname(__FILE__), '..', 'config.yml')))
24
-
25
- # Authenticate
26
- puts 'Creating user...'
27
- user = DataSift::User.new(config['username'], config['api_key'])
28
-
29
- # Create the consumer
30
- puts 'Getting the consumer...'
31
- consumer = user.getConsumer(DataSift::StreamConsumer::TYPE_HTTP, ARGV[0])
32
-
33
- # Setting up the onStopped handler
34
- consumer.onStopped do |reason|
35
- puts
36
- puts 'Stopped: ' + reason
37
- puts
38
- end
39
-
40
- # Set up the warning event handler.
41
- consumer.onWarning do |message|
42
- puts 'WARNING: ' + message
43
- end
44
-
45
- # Set up the error event handler.
46
- consumer.onError do |message|
47
- puts 'ERROR: ' + message
48
- end
49
-
50
- # And start consuming
51
- puts 'Consuming...'
52
- puts '--'
53
- consumer.consume(true) do |interaction|
54
- if interaction
55
- puts 'Type: ' + interaction['interaction']['type']
56
- puts 'Content: ' + interaction['interaction']['content']
57
- puts '--'
58
- end
59
- end
60
-
61
- puts
62
- puts 'Finished consuming'
63
- puts
@@ -1,52 +0,0 @@
1
- # This example consumes 1% of tweets, displaying a . for each interaction
2
- # received, and an X for each delete notification.
3
- #
4
- # NB: Most of the error handling (exception catching) has been removed for
5
- # the sake of simplicity. Nearly everything in this library may throw
6
- # exceptions, and production code should catch them. See the documentation
7
- # for full details.
8
- #
9
-
10
- # Include the DataSift library
11
- require './' + File.dirname(__FILE__) + '/../lib/datasift'
12
-
13
- # Include the configuration - put your username and API key in this file
14
- require 'yaml'
15
- config = YAML::load(File.open(File.join(File.dirname(__FILE__), '..', 'config.yml')))
16
-
17
- # Authenticate
18
- puts 'Creating user...'
19
- user = DataSift::User.new(config['username'], config['api_key'])
20
-
21
- # Create the definition
22
- csdl = 'interaction.type == "twitter" AND interaction.sample < 1.0'
23
- puts 'Creating definition...'
24
- puts ' ' + csdl
25
- definition = user.createDefinition(csdl)
26
-
27
- # Create the consumer
28
- puts 'Getting the consumer...'
29
- consumer = definition.getConsumer(DataSift::StreamConsumer::TYPE_HTTP)
30
-
31
- # Set up the delete event handler. Refer to the documentation for details of
32
- # what the interaction variable will contain:
33
- # http://dev.datasift.com/docs/twitter-deletes
34
- consumer.onDeleted do |interaction|
35
- print 'X'
36
- $stdout.flush
37
- end
38
-
39
- # And start consuming
40
- puts 'Consuming...'
41
- puts '--'
42
- consumer.consume(true) do |interaction|
43
- if interaction
44
- print '.'
45
- $stdout.flush
46
- end
47
- end
48
-
49
- # This example will not stop unless it gets disconnected
50
- puts
51
- puts 'Consumer stopped'
52
- puts
@@ -1,115 +0,0 @@
1
- # This example gets the DPU associated with the stream given on the command
2
- # line or piped/typed into STDIN. It presents it in a nice ASCII table.]
3
- # Note that the CSDL must be enclosed in quotes if given on the command line.
4
- #
5
- # ruby dpu.rb 'interaction.content contains "football"'
6
- # or
7
- # cat football.csdl | ruby dpu.rb
8
- #
9
- # NB: Most of the error handling (exception catching) has been removed for
10
- # the sake of simplicity. Nearly everything in this library may throw
11
- # exceptions, and production code should catch them. See the documentation
12
- # for full details.
13
- #
14
-
15
- # Include the DataSift library
16
- require './' + File.dirname(__FILE__) + '/../lib/datasift'
17
-
18
- # Function to format a number with commas
19
- def number_with_delimiter(number, delimiter=',')
20
- number.to_s.gsub(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1#{delimiter}")
21
- end
22
-
23
- # Include the configuration - put your username and API key in this file
24
- require 'yaml'
25
- config = YAML::load(File.open(File.join(File.dirname(__FILE__), '..', 'config.yml')))
26
-
27
- # Get the CSDL from the command line or STDIN
28
- # Make sure we have some arguments
29
- if ARGV.size == 0
30
- puts 'ERR: Please specify the hash to consume!'
31
- puts
32
- exit!
33
- elsif ARGV.size > 0
34
- csdl = ARGV[0]
35
- else
36
- csdl = ARGF.read
37
- end
38
-
39
- # Authenticate
40
- puts 'Creating user...'
41
- user = DataSift::User.new(config['username'], config['api_key'])
42
-
43
- # Create the definition
44
- puts 'Creating definition...'
45
- definition = user.createDefinition(csdl)
46
-
47
- # Getting DPU
48
- puts 'Getting DPU...'
49
- begin
50
- dpu = definition.getDPUBreakdown()
51
- rescue DataSift::CompileFailedError => e
52
- puts 'CSDL compilation failed: ' + e.message
53
- puts
54
- exit!
55
- end
56
-
57
- dputable = []
58
- maxlength = {'target' => 'Target'.length, 'times used' => 'Times used'.length, 'complexity' => 'Complexity'.length};
59
- dpu['detail'].each do |tgt,c|
60
- maxlength['target'] = [maxlength['target'], tgt.length].max()
61
- maxlength['times used'] = [maxlength['times used'], number_with_delimiter(c['count']).length].max()
62
- maxlength['complexity'] = [maxlength['complexity'], number_with_delimiter(c['dpu']).length].max()
63
-
64
- dputable.push({
65
- 'target' => tgt,
66
- 'times used' => number_with_delimiter(c['count']),
67
- 'complexity' => number_with_delimiter(c['dpu']),
68
- })
69
-
70
- c['targets'].each do |tgt2,d|
71
- maxlength['target'] = [maxlength['target'], 2 + tgt2.length].max()
72
- maxlength['times used'] = [maxlength['times used'], number_with_delimiter(d['count']).length].max()
73
- maxlength['complexity'] = [maxlength['complexity'], number_with_delimiter(d['dpu']).length].max()
74
-
75
- dputable.push({
76
- 'target' => ' ' + tgt2,
77
- 'times used' => number_with_delimiter(d['count']),
78
- 'complexity' => number_with_delimiter(d['dpu']),
79
- })
80
- end
81
- end
82
-
83
- maxlength['complexity'] = [maxlength['complexity'], number_with_delimiter(dpu['dpu']).length].max()
84
-
85
- puts
86
- print '/-' + ('-' * maxlength['target']) + '---'
87
- print ('-' * maxlength['times used']) + '---'
88
- puts ('-' * maxlength['complexity']) + '-\\'
89
-
90
- print '| ' + 'Target'.ljust(maxlength['target']) + ' | '
91
- print 'Times Used'.ljust(maxlength['times used']) + ' | '
92
- puts 'Complexity'.ljust(maxlength['complexity']) + ' |'
93
-
94
- print '|-' + ('-' * maxlength['target']) + '-+-'
95
- print ('-' * maxlength['times used']) + '-+-'
96
- puts ('-' * maxlength['complexity']) + '-|'
97
-
98
- dputable.each do |row|
99
- print '| ' + row['target'].ljust(maxlength['target']) + ' | '
100
- print row['times used'].rjust(maxlength['times used']) + ' | '
101
- puts row['complexity'].rjust(maxlength['complexity']) + ' |'
102
- end
103
-
104
- print '|-' + ('-' * maxlength['target']) + '-+-'
105
- print ('-' * maxlength['times used']) + '-+-'
106
- puts ('-' * maxlength['complexity']) + '-|'
107
-
108
- print '| ' + 'Total'.rjust(maxlength['target'] + 3 + maxlength['times used']) + ' = '
109
- puts dpu['dpu'].to_s.rjust(maxlength['complexity']) + ' |'
110
-
111
- print '\\-' + ('-' * maxlength['target']) + '---'
112
- print ('-' * maxlength['times used']) + '---'
113
- puts ('-' * maxlength['complexity']) + '-/'
114
-
115
- puts