datasift 2.1.1 → 3.0.0.beta

Sign up to get free protection for your applications and to get access to all the features.
Files changed (72) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -0
  3. data/CHANGELOG.md +100 -0
  4. data/Gemfile.lock +32 -0
  5. data/README.md +38 -79
  6. data/VERSION +1 -1
  7. data/datasift.gemspec +21 -24
  8. data/examples/auth.rb +44 -0
  9. data/examples/core_api_eg.rb +46 -0
  10. data/examples/historics_eg.rb +50 -0
  11. data/examples/historics_preview_eg.rb +30 -0
  12. data/examples/live_stream_eg.rb +89 -0
  13. data/examples/managed_source_eg.rb +56 -0
  14. data/examples/pull.rb +44 -0
  15. data/examples/push_eg.rb +56 -0
  16. data/lib/api/api_resource.rb +23 -0
  17. data/lib/datasift.rb +287 -14
  18. data/lib/errors.rb +59 -0
  19. data/lib/historics.rb +76 -0
  20. data/lib/historics_preview.rb +20 -0
  21. data/lib/live_stream.rb +53 -0
  22. data/lib/managed_source.rb +57 -0
  23. data/lib/push.rb +156 -0
  24. data/tests/core_api_test.rb +42 -0
  25. metadata +51 -73
  26. data/Rakefile +0 -34
  27. data/config.yml +0 -2
  28. data/examples/consume-stream.rb +0 -63
  29. data/examples/deletes.rb +0 -52
  30. data/examples/dpu.rb +0 -115
  31. data/examples/football-buffered.rb +0 -51
  32. data/examples/football.rb +0 -53
  33. data/examples/historics.sh +0 -2
  34. data/examples/historics/create-from-csdl.rb +0 -71
  35. data/examples/historics/create-from-hash.rb +0 -65
  36. data/examples/historics/delete.rb +0 -30
  37. data/examples/historics/env.rb +0 -37
  38. data/examples/historics/list.rb +0 -30
  39. data/examples/historics/start.rb +0 -30
  40. data/examples/historics/stop.rb +0 -30
  41. data/examples/historics/view.rb +0 -28
  42. data/examples/push.sh +0 -2
  43. data/examples/push/delete.rb +0 -33
  44. data/examples/push/env.rb +0 -53
  45. data/examples/push/list.rb +0 -30
  46. data/examples/push/pause.rb +0 -33
  47. data/examples/push/push-from-hash.rb +0 -72
  48. data/examples/push/push-historic-from-csdl.rb +0 -98
  49. data/examples/push/push-stream-from-csdl.rb +0 -70
  50. data/examples/push/resume.rb +0 -33
  51. data/examples/push/stop.rb +0 -33
  52. data/examples/push/view-log.rb +0 -45
  53. data/examples/push/view.rb +0 -31
  54. data/examples/twitter-track.rb +0 -61
  55. data/lib/DataSift/apiclient.rb +0 -73
  56. data/lib/DataSift/definition.rb +0 -202
  57. data/lib/DataSift/exceptions.rb +0 -33
  58. data/lib/DataSift/historic.rb +0 -316
  59. data/lib/DataSift/managed_source.rb +0 -263
  60. data/lib/DataSift/mockapiclient.rb +0 -44
  61. data/lib/DataSift/push_definition.rb +0 -115
  62. data/lib/DataSift/push_subscription.rb +0 -330
  63. data/lib/DataSift/stream_consumer.rb +0 -166
  64. data/lib/DataSift/stream_consumer_http.rb +0 -188
  65. data/lib/DataSift/user.rb +0 -311
  66. data/test/helper.rb +0 -95
  67. data/test/test_definition.rb +0 -273
  68. data/test/test_historics.rb +0 -233
  69. data/test/test_pushdefinition.rb +0 -92
  70. data/test/test_pushsubscription.rb +0 -17
  71. data/test/test_user.rb +0 -130
  72. data/test/testdata.yml +0 -30
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: datasift
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.1.1
4
+ version: 3.0.0.beta
5
5
  platform: ruby
6
6
  authors:
7
7
  - DataSift
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-05-16 00:00:00.000000000 Z
11
+ date: 2013-11-08 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rest-client
@@ -16,28 +16,42 @@ dependencies:
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '1.6'
19
+ version: 1.6.3
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '1.6'
26
+ version: 1.6.3
27
27
  - !ruby/object:Gem::Dependency
28
- name: yajl-ruby
28
+ name: multi_json
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
31
  - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: '1.1'
33
+ version: 1.8.0
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: 1.8.0
41
+ - !ruby/object:Gem::Dependency
42
+ name: websocket-td
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: 0.0.3
34
48
  type: :runtime
35
49
  prerelease: false
36
50
  version_requirements: !ruby/object:Gem::Requirement
37
51
  requirements:
38
52
  - - "~>"
39
53
  - !ruby/object:Gem::Version
40
- version: '1.1'
54
+ version: 0.0.3
41
55
  - !ruby/object:Gem::Dependency
42
56
  name: rdoc
43
57
  requirement: !ruby/object:Gem::Requirement
@@ -56,30 +70,30 @@ dependencies:
56
70
  name: shoulda
57
71
  requirement: !ruby/object:Gem::Requirement
58
72
  requirements:
59
- - - ">="
73
+ - - "~>"
60
74
  - !ruby/object:Gem::Version
61
- version: '2.11'
75
+ version: 2.11.3
62
76
  type: :development
63
77
  prerelease: false
64
78
  version_requirements: !ruby/object:Gem::Requirement
65
79
  requirements:
66
- - - ">="
80
+ - - "~>"
67
81
  - !ruby/object:Gem::Version
68
- version: '2.11'
82
+ version: 2.11.3
69
83
  - !ruby/object:Gem::Dependency
70
- name: rspec
84
+ name: test-unit
71
85
  requirement: !ruby/object:Gem::Requirement
72
86
  requirements:
73
- - - "~>"
87
+ - - ">="
74
88
  - !ruby/object:Gem::Version
75
- version: '2.6'
89
+ version: 2.5.5
76
90
  type: :development
77
91
  prerelease: false
78
92
  version_requirements: !ruby/object:Gem::Requirement
79
93
  requirements:
80
- - - "~>"
94
+ - - ">="
81
95
  - !ruby/object:Gem::Version
82
- version: '2.6'
96
+ version: 2.5.5
83
97
  description: The official Ruby library for accessing the DataSift API. See http://datasift.com/
84
98
  for full details and to sign up for an account.
85
99
  email:
@@ -89,60 +103,31 @@ extensions: []
89
103
  extra_rdoc_files: []
90
104
  files:
91
105
  - ".gitignore"
106
+ - CHANGELOG.md
92
107
  - Gemfile
108
+ - Gemfile.lock
93
109
  - LICENSE
94
110
  - README.md
95
- - Rakefile
96
111
  - VERSION
97
- - config.yml
98
112
  - datasift.gemspec
99
- - examples/consume-stream.rb
100
- - examples/deletes.rb
101
- - examples/dpu.rb
102
- - examples/football-buffered.rb
103
- - examples/football.rb
104
- - examples/historics.sh
105
- - examples/historics/create-from-csdl.rb
106
- - examples/historics/create-from-hash.rb
107
- - examples/historics/delete.rb
108
- - examples/historics/env.rb
109
- - examples/historics/list.rb
110
- - examples/historics/start.rb
111
- - examples/historics/stop.rb
112
- - examples/historics/view.rb
113
- - examples/push.sh
114
- - examples/push/delete.rb
115
- - examples/push/env.rb
116
- - examples/push/list.rb
117
- - examples/push/pause.rb
118
- - examples/push/push-from-hash.rb
119
- - examples/push/push-historic-from-csdl.rb
120
- - examples/push/push-stream-from-csdl.rb
121
- - examples/push/resume.rb
122
- - examples/push/stop.rb
123
- - examples/push/view-log.rb
124
- - examples/push/view.rb
125
- - examples/twitter-track.rb
126
- - lib/DataSift/apiclient.rb
127
- - lib/DataSift/definition.rb
128
- - lib/DataSift/exceptions.rb
129
- - lib/DataSift/historic.rb
130
- - lib/DataSift/managed_source.rb
131
- - lib/DataSift/mockapiclient.rb
132
- - lib/DataSift/push_definition.rb
133
- - lib/DataSift/push_subscription.rb
134
- - lib/DataSift/stream_consumer.rb
135
- - lib/DataSift/stream_consumer_http.rb
136
- - lib/DataSift/user.rb
113
+ - examples/auth.rb
114
+ - examples/core_api_eg.rb
115
+ - examples/historics_eg.rb
116
+ - examples/historics_preview_eg.rb
117
+ - examples/live_stream_eg.rb
118
+ - examples/managed_source_eg.rb
119
+ - examples/pull.rb
120
+ - examples/push_eg.rb
121
+ - lib/api/api_resource.rb
137
122
  - lib/datasift.rb
138
- - test/helper.rb
139
- - test/test_definition.rb
140
- - test/test_historics.rb
141
- - test/test_pushdefinition.rb
142
- - test/test_pushsubscription.rb
143
- - test/test_user.rb
144
- - test/testdata.yml
145
- homepage: http://github.com/datasift/datasift-ruby
123
+ - lib/errors.rb
124
+ - lib/historics.rb
125
+ - lib/historics_preview.rb
126
+ - lib/live_stream.rb
127
+ - lib/managed_source.rb
128
+ - lib/push.rb
129
+ - tests/core_api_test.rb
130
+ homepage: https://github.com/datasift/datasift-ruby
146
131
  licenses:
147
132
  - BSD
148
133
  metadata: {}
@@ -162,15 +147,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
162
147
  version: 1.3.6
163
148
  requirements: []
164
149
  rubyforge_project:
165
- rubygems_version: 2.2.2
150
+ rubygems_version: 2.1.5
166
151
  signing_key:
167
152
  specification_version: 4
168
153
  summary: DataSift is a simple wrapper for the DataSift API.
169
- test_files:
170
- - test/helper.rb
171
- - test/test_definition.rb
172
- - test/test_historics.rb
173
- - test/test_pushdefinition.rb
174
- - test/test_pushsubscription.rb
175
- - test/test_user.rb
176
- - test/testdata.yml
154
+ test_files: []
data/Rakefile DELETED
@@ -1,34 +0,0 @@
1
- require 'rubygems'
2
- require 'rake'
3
-
4
- require 'rake/testtask'
5
- Rake::TestTask.new(:test) do |test|
6
- test.libs << 'lib' << 'test'
7
- test.pattern = 'test/**/test_*.rb'
8
- test.verbose = true
9
- end
10
-
11
- begin
12
- require 'rcov/rcovtask'
13
- Rcov::RcovTask.new do |test|
14
- test.libs << 'test'
15
- test.pattern = 'test/**/test_*.rb'
16
- test.verbose = true
17
- end
18
- rescue LoadError
19
- task :rcov do
20
- abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
21
- end
22
- end
23
-
24
- require 'rdoc/task'
25
- RDoc::Task.new do |rdoc|
26
- version = File.exist?('VERSION') ? File.read('VERSION') : ""
27
-
28
- rdoc.rdoc_dir = 'rdoc'
29
- rdoc.title = "datasift #{version}"
30
- rdoc.rdoc_files.include('README*')
31
- rdoc.rdoc_files.include('lib/**/*.rb')
32
- end
33
-
34
- task :default => :test
data/config.yml DELETED
@@ -1,2 +0,0 @@
1
- username: YOUR_USERNAME_HERE
2
- api_key: YOUR_API_KEY_HERE
@@ -1,63 +0,0 @@
1
- # This simple example demonstrates consuming a stream from just the stream
2
- # hash.
3
- #
4
- # NB: Most of the error handling (exception catching) has been removed for
5
- # the sake of simplicity. Nearly everything in this library may throw
6
- # exceptions, and production code should catch them. See the documentation
7
- # for full details.
8
- #
9
-
10
- # Make sure we have some arguments
11
- if ARGV.size == 0
12
- puts 'ERR: Please specify the hash to consume!'
13
- puts
14
- puts
15
- exit!
16
- end
17
-
18
- # Include the DataSift library
19
- require './' + File.dirname(__FILE__) + '/../lib/datasift'
20
-
21
- # Include the configuration - put your username and API key in this file
22
- require 'yaml'
23
- config = YAML::load(File.open(File.join(File.dirname(__FILE__), '..', 'config.yml')))
24
-
25
- # Authenticate
26
- puts 'Creating user...'
27
- user = DataSift::User.new(config['username'], config['api_key'])
28
-
29
- # Create the consumer
30
- puts 'Getting the consumer...'
31
- consumer = user.getConsumer(DataSift::StreamConsumer::TYPE_HTTP, ARGV[0])
32
-
33
- # Setting up the onStopped handler
34
- consumer.onStopped do |reason|
35
- puts
36
- puts 'Stopped: ' + reason
37
- puts
38
- end
39
-
40
- # Set up the warning event handler.
41
- consumer.onWarning do |message|
42
- puts 'WARNING: ' + message
43
- end
44
-
45
- # Set up the error event handler.
46
- consumer.onError do |message|
47
- puts 'ERROR: ' + message
48
- end
49
-
50
- # And start consuming
51
- puts 'Consuming...'
52
- puts '--'
53
- consumer.consume(true) do |interaction|
54
- if interaction
55
- puts 'Type: ' + interaction['interaction']['type']
56
- puts 'Content: ' + interaction['interaction']['content']
57
- puts '--'
58
- end
59
- end
60
-
61
- puts
62
- puts 'Finished consuming'
63
- puts
@@ -1,52 +0,0 @@
1
- # This example consumes 1% of tweets, displaying a . for each interaction
2
- # received, and an X for each delete notification.
3
- #
4
- # NB: Most of the error handling (exception catching) has been removed for
5
- # the sake of simplicity. Nearly everything in this library may throw
6
- # exceptions, and production code should catch them. See the documentation
7
- # for full details.
8
- #
9
-
10
- # Include the DataSift library
11
- require './' + File.dirname(__FILE__) + '/../lib/datasift'
12
-
13
- # Include the configuration - put your username and API key in this file
14
- require 'yaml'
15
- config = YAML::load(File.open(File.join(File.dirname(__FILE__), '..', 'config.yml')))
16
-
17
- # Authenticate
18
- puts 'Creating user...'
19
- user = DataSift::User.new(config['username'], config['api_key'])
20
-
21
- # Create the definition
22
- csdl = 'interaction.type == "twitter" AND interaction.sample < 1.0'
23
- puts 'Creating definition...'
24
- puts ' ' + csdl
25
- definition = user.createDefinition(csdl)
26
-
27
- # Create the consumer
28
- puts 'Getting the consumer...'
29
- consumer = definition.getConsumer(DataSift::StreamConsumer::TYPE_HTTP)
30
-
31
- # Set up the delete event handler. Refer to the documentation for details of
32
- # what the interaction variable will contain:
33
- # http://dev.datasift.com/docs/twitter-deletes
34
- consumer.onDeleted do |interaction|
35
- print 'X'
36
- $stdout.flush
37
- end
38
-
39
- # And start consuming
40
- puts 'Consuming...'
41
- puts '--'
42
- consumer.consume(true) do |interaction|
43
- if interaction
44
- print '.'
45
- $stdout.flush
46
- end
47
- end
48
-
49
- # This example will not stop unless it gets disconnected
50
- puts
51
- puts 'Consumer stopped'
52
- puts
@@ -1,115 +0,0 @@
1
- # This example gets the DPU associated with the stream given on the command
2
- # line or piped/typed into STDIN. It presents it in a nice ASCII table.]
3
- # Note that the CSDL must be enclosed in quotes if given on the command line.
4
- #
5
- # ruby dpu.rb 'interaction.content contains "football"'
6
- # or
7
- # cat football.csdl | ruby dpu.rb
8
- #
9
- # NB: Most of the error handling (exception catching) has been removed for
10
- # the sake of simplicity. Nearly everything in this library may throw
11
- # exceptions, and production code should catch them. See the documentation
12
- # for full details.
13
- #
14
-
15
- # Include the DataSift library
16
- require './' + File.dirname(__FILE__) + '/../lib/datasift'
17
-
18
- # Function to format a number with commas
19
- def number_with_delimiter(number, delimiter=',')
20
- number.to_s.gsub(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1#{delimiter}")
21
- end
22
-
23
- # Include the configuration - put your username and API key in this file
24
- require 'yaml'
25
- config = YAML::load(File.open(File.join(File.dirname(__FILE__), '..', 'config.yml')))
26
-
27
- # Get the CSDL from the command line or STDIN
28
- # Make sure we have some arguments
29
- if ARGV.size == 0
30
- puts 'ERR: Please specify the hash to consume!'
31
- puts
32
- exit!
33
- elsif ARGV.size > 0
34
- csdl = ARGV[0]
35
- else
36
- csdl = ARGF.read
37
- end
38
-
39
- # Authenticate
40
- puts 'Creating user...'
41
- user = DataSift::User.new(config['username'], config['api_key'])
42
-
43
- # Create the definition
44
- puts 'Creating definition...'
45
- definition = user.createDefinition(csdl)
46
-
47
- # Getting DPU
48
- puts 'Getting DPU...'
49
- begin
50
- dpu = definition.getDPUBreakdown()
51
- rescue DataSift::CompileFailedError => e
52
- puts 'CSDL compilation failed: ' + e.message
53
- puts
54
- exit!
55
- end
56
-
57
- dputable = []
58
- maxlength = {'target' => 'Target'.length, 'times used' => 'Times used'.length, 'complexity' => 'Complexity'.length};
59
- dpu['detail'].each do |tgt,c|
60
- maxlength['target'] = [maxlength['target'], tgt.length].max()
61
- maxlength['times used'] = [maxlength['times used'], number_with_delimiter(c['count']).length].max()
62
- maxlength['complexity'] = [maxlength['complexity'], number_with_delimiter(c['dpu']).length].max()
63
-
64
- dputable.push({
65
- 'target' => tgt,
66
- 'times used' => number_with_delimiter(c['count']),
67
- 'complexity' => number_with_delimiter(c['dpu']),
68
- })
69
-
70
- c['targets'].each do |tgt2,d|
71
- maxlength['target'] = [maxlength['target'], 2 + tgt2.length].max()
72
- maxlength['times used'] = [maxlength['times used'], number_with_delimiter(d['count']).length].max()
73
- maxlength['complexity'] = [maxlength['complexity'], number_with_delimiter(d['dpu']).length].max()
74
-
75
- dputable.push({
76
- 'target' => ' ' + tgt2,
77
- 'times used' => number_with_delimiter(d['count']),
78
- 'complexity' => number_with_delimiter(d['dpu']),
79
- })
80
- end
81
- end
82
-
83
- maxlength['complexity'] = [maxlength['complexity'], number_with_delimiter(dpu['dpu']).length].max()
84
-
85
- puts
86
- print '/-' + ('-' * maxlength['target']) + '---'
87
- print ('-' * maxlength['times used']) + '---'
88
- puts ('-' * maxlength['complexity']) + '-\\'
89
-
90
- print '| ' + 'Target'.ljust(maxlength['target']) + ' | '
91
- print 'Times Used'.ljust(maxlength['times used']) + ' | '
92
- puts 'Complexity'.ljust(maxlength['complexity']) + ' |'
93
-
94
- print '|-' + ('-' * maxlength['target']) + '-+-'
95
- print ('-' * maxlength['times used']) + '-+-'
96
- puts ('-' * maxlength['complexity']) + '-|'
97
-
98
- dputable.each do |row|
99
- print '| ' + row['target'].ljust(maxlength['target']) + ' | '
100
- print row['times used'].rjust(maxlength['times used']) + ' | '
101
- puts row['complexity'].rjust(maxlength['complexity']) + ' |'
102
- end
103
-
104
- print '|-' + ('-' * maxlength['target']) + '-+-'
105
- print ('-' * maxlength['times used']) + '-+-'
106
- puts ('-' * maxlength['complexity']) + '-|'
107
-
108
- print '| ' + 'Total'.rjust(maxlength['target'] + 3 + maxlength['times used']) + ' = '
109
- puts dpu['dpu'].to_s.rjust(maxlength['complexity']) + ' |'
110
-
111
- print '\\-' + ('-' * maxlength['target']) + '---'
112
- print ('-' * maxlength['times used']) + '---'
113
- puts ('-' * maxlength['complexity']) + '-/'
114
-
115
- puts