datasift 1.1.0 → 1.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +1 -0
- data/README.md +23 -5
- data/VERSION +1 -1
- data/datasift.gemspec +0 -1
- data/examples/deletes.rb +52 -0
- data/lib/DataSift/apiclient.rb +5 -6
- data/lib/DataSift/stream_consumer.rb +21 -1
- metadata +30 -27
data/.gitignore
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
sftp-config.json
|
data/README.md
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
DataSift
|
2
2
|
========
|
3
3
|
|
4
|
-
The official Ruby library for accessing the DataSift API. See
|
4
|
+
The official Ruby library for accessing the DataSift API. See
|
5
|
+
http://datasift.net for full details and to sign up for an account.
|
5
6
|
|
6
7
|
The examples and tests use the username and API key in config.yml.
|
7
8
|
|
@@ -15,12 +16,13 @@ Dependencies
|
|
15
16
|
|
16
17
|
If you're using the source you'll need to install the dependencies.
|
17
18
|
|
18
|
-
sudo gem install yajl-ruby
|
19
|
+
sudo gem install yajl-ruby rest-client
|
19
20
|
|
20
21
|
Simple example
|
21
22
|
--------------
|
22
23
|
|
23
|
-
This example looks for anything that contains the word "datasift" and simply
|
24
|
+
This example looks for anything that contains the word "datasift" and simply
|
25
|
+
prints the content to the screen as they come in.
|
24
26
|
|
25
27
|
```ruby
|
26
28
|
require 'rubygems'
|
@@ -35,11 +37,27 @@ consumer.consume(true) do |interaction|
|
|
35
37
|
end
|
36
38
|
```
|
37
39
|
|
38
|
-
See the DataSift documentation for full details of the data contained within
|
40
|
+
See the DataSift documentation for full details of the data contained within
|
41
|
+
each interaction. See this page on our developer site for an example tweet:
|
42
|
+
http://dev.datasift.com/docs/targets/twitter/twitter-output-format
|
39
43
|
|
40
44
|
License
|
41
45
|
-------
|
42
46
|
|
43
47
|
All code contained in this repository is Copyright 2011-2012 MediaSift Ltd.
|
44
48
|
|
45
|
-
This code is released under the BSD license. Please see the LICENSE file for
|
49
|
+
This code is released under the BSD license. Please see the LICENSE file for
|
50
|
+
more details.
|
51
|
+
|
52
|
+
Changelog
|
53
|
+
---------
|
54
|
+
|
55
|
+
* v.1.2.0 Twitter Compliance (2012-02-28)
|
56
|
+
|
57
|
+
The consumer now has an onDeleted method to which you can assign a block
|
58
|
+
that will be called to handle DELETE requests from Twitter. See delete.rb
|
59
|
+
in the examples folder for a sample implementation.
|
60
|
+
(@see http://dev.datasift.com/docs/twitter-deletes)
|
61
|
+
|
62
|
+
NB: if you are storing tweets you must implement this method in your code
|
63
|
+
and take appropriate action to maintain compliance with the Twitter license.
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.
|
1
|
+
1.2.0
|
data/datasift.gemspec
CHANGED
@@ -16,7 +16,6 @@ Gem::Specification.new do |s|
|
|
16
16
|
s.required_rubygems_version = Gem::Requirement.new(">= 1.3.6") if s.respond_to? :required_rubygems_version=
|
17
17
|
|
18
18
|
s.add_runtime_dependency('rest-client', '~> 1.6.3')
|
19
|
-
s.add_runtime_dependency('json', '~> 1')
|
20
19
|
s.add_runtime_dependency('yajl-ruby', '~> 0.8.2')
|
21
20
|
s.add_development_dependency('rdoc', '~> 0')
|
22
21
|
s.add_development_dependency('shoulda', '~> 2.11.3')
|
data/examples/deletes.rb
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
# This example consumes 1% of tweets, displaying a . for each interaction
|
2
|
+
# received, and an X for each delete notification.
|
3
|
+
#
|
4
|
+
# NB: Most of the error handling (exception catching) has been removed for
|
5
|
+
# the sake of simplicity. Nearly everything in this library may throw
|
6
|
+
# exceptions, and production code should catch them. See the documentation
|
7
|
+
# for full details.
|
8
|
+
#
|
9
|
+
|
10
|
+
# Include the DataSift library
|
11
|
+
require File.dirname(__FILE__) + '/../lib/datasift'
|
12
|
+
|
13
|
+
# Include the configuration - put your username and API key in this file
|
14
|
+
require 'yaml'
|
15
|
+
config = YAML::load(File.open(File.join(File.dirname(__FILE__), '..', 'config.yml')))
|
16
|
+
|
17
|
+
# Authenticate
|
18
|
+
puts 'Creating user...'
|
19
|
+
user = DataSift::User.new(config['username'], config['api_key'])
|
20
|
+
|
21
|
+
# Create the definition
|
22
|
+
csdl = 'interaction.type == "twitter" AND interaction.sample < 1.0'
|
23
|
+
puts 'Creating definition...'
|
24
|
+
puts ' ' + csdl
|
25
|
+
definition = user.createDefinition(csdl)
|
26
|
+
|
27
|
+
# Create the consumer
|
28
|
+
puts 'Getting the consumer...'
|
29
|
+
consumer = definition.getConsumer(DataSift::StreamConsumer::TYPE_HTTP)
|
30
|
+
|
31
|
+
# Set up the delete event handler. Refer to the documentation for details of
|
32
|
+
# what the interaction variable will contain:
|
33
|
+
# http://dev.datasift.com/docs/twitter-deletes
|
34
|
+
consumer.onDeleted do |interaction|
|
35
|
+
print 'X'
|
36
|
+
$stdout.flush
|
37
|
+
end
|
38
|
+
|
39
|
+
# And start consuming
|
40
|
+
puts 'Consuming...'
|
41
|
+
puts '--'
|
42
|
+
consumer.consume(true) do |interaction|
|
43
|
+
if interaction
|
44
|
+
print '.'
|
45
|
+
$stdout.flush
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
# This example will not stop unless it gets disconnected
|
50
|
+
puts
|
51
|
+
puts 'Consumer stopped'
|
52
|
+
puts
|
data/lib/DataSift/apiclient.rb
CHANGED
@@ -9,7 +9,7 @@
|
|
9
9
|
# DataSift API.
|
10
10
|
|
11
11
|
require 'rest_client'
|
12
|
-
require '
|
12
|
+
require 'yajl'
|
13
13
|
|
14
14
|
module DataSift
|
15
15
|
# ApiCLient class.
|
@@ -29,7 +29,7 @@ module DataSift
|
|
29
29
|
# * +api_key+ - The API key for the Auth header
|
30
30
|
def call(username, api_key, endpoint, params = {}, user_agent = 'DataSiftPHP/0.0')
|
31
31
|
# Build the full endpoint URL
|
32
|
-
url = 'http://' + User::API_BASE_URL + endpoint
|
32
|
+
url = 'http://' + User::API_BASE_URL + endpoint
|
33
33
|
|
34
34
|
retval = {
|
35
35
|
'response_code' => 500,
|
@@ -40,14 +40,13 @@ module DataSift
|
|
40
40
|
|
41
41
|
begin
|
42
42
|
# Make the call
|
43
|
-
res = RestClient.
|
43
|
+
res = RestClient.post(url, params, { 'Auth' => username + ':' + api_key, 'User-Agent' => user_agent })
|
44
44
|
|
45
45
|
# Success
|
46
46
|
retval['response_code'] = 200
|
47
47
|
|
48
48
|
# Parse the JSON response
|
49
|
-
retval['data'] =
|
50
|
-
|
49
|
+
retval['data'] = Yajl::Parser.parse(res)
|
51
50
|
# Rate limit headers
|
52
51
|
if (res.headers[:x_ratelimit_limit])
|
53
52
|
retval['rate_limit'] = res.headers[:x_ratelimit_limit]
|
@@ -61,7 +60,7 @@ module DataSift
|
|
61
60
|
retval['response_code'] = err.http_code
|
62
61
|
|
63
62
|
# And set the data
|
64
|
-
retval['data'] =
|
63
|
+
retval['data'] = Yajl::Parser.parse(err.response)
|
65
64
|
end
|
66
65
|
|
67
66
|
retval
|
@@ -61,6 +61,20 @@ module DataSift
|
|
61
61
|
@definition.hash
|
62
62
|
end
|
63
63
|
|
64
|
+
# This is called when a deletion notification is received.
|
65
|
+
# === Parameters
|
66
|
+
#
|
67
|
+
# * +interaction+ - Minimal details about the interaction that was deleted.
|
68
|
+
#
|
69
|
+
def onDeleted(&block)
|
70
|
+
if block_given?
|
71
|
+
@on_deleted = block
|
72
|
+
self
|
73
|
+
else
|
74
|
+
@on_deleted
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
64
78
|
# This is called when the consumer is stopped.
|
65
79
|
# === Parameters
|
66
80
|
#
|
@@ -88,7 +102,13 @@ module DataSift
|
|
88
102
|
|
89
103
|
# Start consuming
|
90
104
|
@state = STATE_STARTING
|
91
|
-
onStart
|
105
|
+
onStart do |interaction|
|
106
|
+
if interaction.has_key?('deleted') and interaction['deleted']
|
107
|
+
onDeleted.call(interaction) unless onDeleted.nil?
|
108
|
+
else
|
109
|
+
block.call(interaction) unless block.nil?
|
110
|
+
end
|
111
|
+
end
|
92
112
|
end
|
93
113
|
|
94
114
|
# Called when the consumer should start consuming the stream.
|
metadata
CHANGED
@@ -1,12 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: datasift
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
|
4
|
+
hash: 31
|
5
|
+
prerelease:
|
5
6
|
segments:
|
6
7
|
- 1
|
7
|
-
-
|
8
|
+
- 2
|
8
9
|
- 0
|
9
|
-
version: 1.
|
10
|
+
version: 1.2.0
|
10
11
|
platform: ruby
|
11
12
|
authors:
|
12
13
|
- DataSift
|
@@ -14,16 +15,17 @@ autorequire:
|
|
14
15
|
bindir: bin
|
15
16
|
cert_chain: []
|
16
17
|
|
17
|
-
date: 2012-02-
|
18
|
-
default_executable:
|
18
|
+
date: 2012-02-29 00:00:00 Z
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|
21
21
|
name: rest-client
|
22
22
|
prerelease: false
|
23
23
|
requirement: &id001 !ruby/object:Gem::Requirement
|
24
|
+
none: false
|
24
25
|
requirements:
|
25
26
|
- - ~>
|
26
27
|
- !ruby/object:Gem::Version
|
28
|
+
hash: 9
|
27
29
|
segments:
|
28
30
|
- 1
|
29
31
|
- 6
|
@@ -31,72 +33,68 @@ dependencies:
|
|
31
33
|
version: 1.6.3
|
32
34
|
type: :runtime
|
33
35
|
version_requirements: *id001
|
34
|
-
- !ruby/object:Gem::Dependency
|
35
|
-
name: json
|
36
|
-
prerelease: false
|
37
|
-
requirement: &id002 !ruby/object:Gem::Requirement
|
38
|
-
requirements:
|
39
|
-
- - ~>
|
40
|
-
- !ruby/object:Gem::Version
|
41
|
-
segments:
|
42
|
-
- 1
|
43
|
-
version: "1"
|
44
|
-
type: :runtime
|
45
|
-
version_requirements: *id002
|
46
36
|
- !ruby/object:Gem::Dependency
|
47
37
|
name: yajl-ruby
|
48
38
|
prerelease: false
|
49
|
-
requirement: &
|
39
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
40
|
+
none: false
|
50
41
|
requirements:
|
51
42
|
- - ~>
|
52
43
|
- !ruby/object:Gem::Version
|
44
|
+
hash: 59
|
53
45
|
segments:
|
54
46
|
- 0
|
55
47
|
- 8
|
56
48
|
- 2
|
57
49
|
version: 0.8.2
|
58
50
|
type: :runtime
|
59
|
-
version_requirements: *
|
51
|
+
version_requirements: *id002
|
60
52
|
- !ruby/object:Gem::Dependency
|
61
53
|
name: rdoc
|
62
54
|
prerelease: false
|
63
|
-
requirement: &
|
55
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
56
|
+
none: false
|
64
57
|
requirements:
|
65
58
|
- - ~>
|
66
59
|
- !ruby/object:Gem::Version
|
60
|
+
hash: 3
|
67
61
|
segments:
|
68
62
|
- 0
|
69
63
|
version: "0"
|
70
64
|
type: :development
|
71
|
-
version_requirements: *
|
65
|
+
version_requirements: *id003
|
72
66
|
- !ruby/object:Gem::Dependency
|
73
67
|
name: shoulda
|
74
68
|
prerelease: false
|
75
|
-
requirement: &
|
69
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
70
|
+
none: false
|
76
71
|
requirements:
|
77
72
|
- - ~>
|
78
73
|
- !ruby/object:Gem::Version
|
74
|
+
hash: 37
|
79
75
|
segments:
|
80
76
|
- 2
|
81
77
|
- 11
|
82
78
|
- 3
|
83
79
|
version: 2.11.3
|
84
80
|
type: :development
|
85
|
-
version_requirements: *
|
81
|
+
version_requirements: *id004
|
86
82
|
- !ruby/object:Gem::Dependency
|
87
83
|
name: rspec
|
88
84
|
prerelease: false
|
89
|
-
requirement: &
|
85
|
+
requirement: &id005 !ruby/object:Gem::Requirement
|
86
|
+
none: false
|
90
87
|
requirements:
|
91
88
|
- - ~>
|
92
89
|
- !ruby/object:Gem::Version
|
90
|
+
hash: 23
|
93
91
|
segments:
|
94
92
|
- 2
|
95
93
|
- 6
|
96
94
|
- 0
|
97
95
|
version: 2.6.0
|
98
96
|
type: :development
|
99
|
-
version_requirements: *
|
97
|
+
version_requirements: *id005
|
100
98
|
description: The official Ruby library for accessing the DataSift API. See http://datasift.com/ for full details and to sign up for an account.
|
101
99
|
email:
|
102
100
|
- support@datasift.net
|
@@ -107,6 +105,7 @@ extensions: []
|
|
107
105
|
extra_rdoc_files: []
|
108
106
|
|
109
107
|
files:
|
108
|
+
- .gitignore
|
110
109
|
- LICENSE
|
111
110
|
- README.md
|
112
111
|
- Rakefile
|
@@ -114,6 +113,7 @@ files:
|
|
114
113
|
- config.yml
|
115
114
|
- datasift.gemspec
|
116
115
|
- examples/consume-stream.rb
|
116
|
+
- examples/deletes.rb
|
117
117
|
- examples/dpu.rb
|
118
118
|
- examples/football-buffered.rb
|
119
119
|
- examples/football.rb
|
@@ -131,7 +131,6 @@ files:
|
|
131
131
|
- test/test_live_api.rb
|
132
132
|
- test/test_user.rb
|
133
133
|
- test/testdata.yml
|
134
|
-
has_rdoc: true
|
135
134
|
homepage: http://github.com/datasift/datasift-ruby
|
136
135
|
licenses: []
|
137
136
|
|
@@ -141,16 +140,20 @@ rdoc_options: []
|
|
141
140
|
require_paths:
|
142
141
|
- lib
|
143
142
|
required_ruby_version: !ruby/object:Gem::Requirement
|
143
|
+
none: false
|
144
144
|
requirements:
|
145
145
|
- - ">="
|
146
146
|
- !ruby/object:Gem::Version
|
147
|
+
hash: 3
|
147
148
|
segments:
|
148
149
|
- 0
|
149
150
|
version: "0"
|
150
151
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
152
|
+
none: false
|
151
153
|
requirements:
|
152
154
|
- - ">="
|
153
155
|
- !ruby/object:Gem::Version
|
156
|
+
hash: 23
|
154
157
|
segments:
|
155
158
|
- 1
|
156
159
|
- 3
|
@@ -159,7 +162,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
159
162
|
requirements: []
|
160
163
|
|
161
164
|
rubyforge_project:
|
162
|
-
rubygems_version: 1.
|
165
|
+
rubygems_version: 1.8.17
|
163
166
|
signing_key:
|
164
167
|
specification_version: 3
|
165
168
|
summary: DataSift is a simple wrapper for the DataSift API.
|