keen 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +0 -1
- data/README.md +8 -1
- data/VERSION.yml +1 -1
- data/examples.rb +2 -2
- data/features/step_definitions/keen_steps.rb +1 -37
- data/features/support/before_and_after.rb +1 -1
- data/keen.gemspec +3 -14
- data/lib/keen.rb +0 -1
- data/lib/keen/client.rb +9 -87
- data/lib/keen/keys.rb +2 -2
- metadata +15 -34
- data/features/redis_queue.feature +0 -34
- data/lib/keen/async.rb +0 -6
- data/lib/keen/async/job.rb +0 -79
- data/lib/keen/async/storage/base_storage_handler.rb +0 -51
- data/lib/keen/async/storage/flat_file_handler.rb +0 -57
- data/lib/keen/async/storage/redis_handler.rb +0 -89
- data/lib/keen/async/worker.rb +0 -62
- data/send.rb +0 -12
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -6,11 +6,18 @@ Usage
|
|
6
6
|
|
7
7
|
`gem install keen`
|
8
8
|
|
9
|
-
|
9
|
+
Ruby Client Usage Guide
|
10
|
+
http://keen.io/static/docs/clients/ruby/usage_guide.html
|
11
|
+
|
12
|
+
see Also the Cucumber and Rspec stuff in /features/ for usage details)
|
10
13
|
|
11
14
|
Contributing to keen
|
12
15
|
--------------------
|
13
16
|
|
17
|
+
* Use Jeweler!
|
18
|
+
* rake version:bump:major
|
19
|
+
* rake release
|
20
|
+
* Make sure all tests pass in 1.9.2 and 1.9.3
|
14
21
|
* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
|
15
22
|
* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it.
|
16
23
|
* Fork the project.
|
data/VERSION.yml
CHANGED
data/examples.rb
CHANGED
@@ -3,10 +3,10 @@ require 'keen'
|
|
3
3
|
|
4
4
|
# Get these from the keen.io website:
|
5
5
|
project_id = 'asdfasldkfjalsdkfalskdfj'
|
6
|
-
|
6
|
+
api_key = 'asldfjklj325tkl32jaskdlfjaf'
|
7
7
|
|
8
8
|
# First you must setup the client:
|
9
|
-
keen = Keen::Client.new(project_id,
|
9
|
+
keen = Keen::Client.new(project_id, api_key, :storage_mode => :redis)
|
10
10
|
|
11
11
|
# Then, you can use that client to send events.
|
12
12
|
|
@@ -3,23 +3,9 @@ require 'cucumber/formatter/unicode'
|
|
3
3
|
$:.unshift(File.dirname(__FILE__) + '/../../lib')
|
4
4
|
require 'keen'
|
5
5
|
|
6
|
-
Given /^a Keen Client using Redis$/ do
|
7
|
-
@client = Keen::Client.new(@project_id,
|
8
|
-
@auth_token,
|
9
|
-
:storage_class => Keen::Async::Storage::RedisHandler,
|
10
|
-
:storage_namespace => "test",
|
11
|
-
:cache_locally => true,
|
12
|
-
:logging => false )
|
13
|
-
|
14
|
-
@client.storage_handler.clear_active_queue
|
15
|
-
|
16
|
-
@starting_queue_size = @client.storage_handler.count_active_queue
|
17
|
-
end
|
18
|
-
|
19
6
|
Given /^a Keen Client in Direct Send mode$/ do
|
20
7
|
@client = Keen::Client.new(@project_id,
|
21
|
-
@
|
22
|
-
:cache_locally => false,
|
8
|
+
@api_key,
|
23
9
|
:logging => false )
|
24
10
|
end
|
25
11
|
|
@@ -27,10 +13,6 @@ When /^I post an event$/ do
|
|
27
13
|
@result = @client.add_event("cucumber_events", {:hi_from => "cucumber!", :keen_version => Keen::VERSION})
|
28
14
|
end
|
29
15
|
|
30
|
-
Then /^the size of the Redis queue should have gone up by (\d+)\.$/ do |n|
|
31
|
-
@client.storage_handler.count_active_queue.should == n.to_i + @starting_queue_size
|
32
|
-
end
|
33
|
-
|
34
16
|
Then /^the response from the server should be good\.$/ do
|
35
17
|
response = @result
|
36
18
|
response.should == {"created" => true}
|
@@ -42,21 +24,3 @@ When /^I post (\d+) events$/ do |n|
|
|
42
24
|
@client.add_event("cucumber_events", {:hi_from => "cucumber!", :keen_version => Keen::VERSION})
|
43
25
|
end
|
44
26
|
end
|
45
|
-
|
46
|
-
When /^I process the queue$/ do
|
47
|
-
worker = Keen::Async::Worker.new(@client)
|
48
|
-
@result = worker.process_queue
|
49
|
-
end
|
50
|
-
|
51
|
-
Then /^the response from Keen should be (\d+) happy smiles$/ do |n|
|
52
|
-
expectation = []
|
53
|
-
n.to_i.times do
|
54
|
-
expectation.push({"success" => true})
|
55
|
-
end
|
56
|
-
|
57
|
-
expectation = {"cucumber_events" => expectation}
|
58
|
-
end
|
59
|
-
|
60
|
-
Then /^the queue should be empty\.$/ do
|
61
|
-
@client.storage_handler.count_active_queue.should == 0
|
62
|
-
end
|
data/keen.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = "keen"
|
8
|
-
s.version = "0.2.
|
8
|
+
s.version = "0.2.2"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["dorkitude"]
|
12
|
-
s.date = "2012-10-
|
12
|
+
s.date = "2012-10-27"
|
13
13
|
s.description = "See the github repo or examples.rb for usage information."
|
14
14
|
s.email = "kyle@keen.io"
|
15
15
|
s.extra_rdoc_files = [
|
@@ -26,25 +26,17 @@ Gem::Specification.new do |s|
|
|
26
26
|
"conf/cacert.pem",
|
27
27
|
"examples.rb",
|
28
28
|
"features/add_event.feature",
|
29
|
-
"features/redis_queue.feature",
|
30
29
|
"features/step_definitions/keen_steps.rb",
|
31
30
|
"features/support/before_and_after.rb",
|
32
31
|
"features/support/env.rb",
|
33
32
|
"keen.gemspec",
|
34
33
|
"keen.gemspec.old",
|
35
34
|
"lib/keen.rb",
|
36
|
-
"lib/keen/async.rb",
|
37
|
-
"lib/keen/async/job.rb",
|
38
|
-
"lib/keen/async/storage/base_storage_handler.rb",
|
39
|
-
"lib/keen/async/storage/flat_file_handler.rb",
|
40
|
-
"lib/keen/async/storage/redis_handler.rb",
|
41
|
-
"lib/keen/async/worker.rb",
|
42
35
|
"lib/keen/client.rb",
|
43
36
|
"lib/keen/event.rb",
|
44
37
|
"lib/keen/keys.rb",
|
45
38
|
"lib/keen/utils.rb",
|
46
|
-
"lib/keen/version.rb"
|
47
|
-
"send.rb"
|
39
|
+
"lib/keen/version.rb"
|
48
40
|
]
|
49
41
|
s.homepage = "http://github.com/dorkitude/keen"
|
50
42
|
s.licenses = ["MIT"]
|
@@ -57,7 +49,6 @@ Gem::Specification.new do |s|
|
|
57
49
|
|
58
50
|
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
59
51
|
s.add_runtime_dependency(%q<json>, [">= 1.6.5"])
|
60
|
-
s.add_runtime_dependency(%q<redis>, [">= 0"])
|
61
52
|
s.add_development_dependency(%q<shoulda>, [">= 0"])
|
62
53
|
s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
|
63
54
|
s.add_development_dependency(%q<bundler>, [">= 1.1.4"])
|
@@ -66,7 +57,6 @@ Gem::Specification.new do |s|
|
|
66
57
|
s.add_development_dependency(%q<cucumber>, [">= 1.2.1"])
|
67
58
|
else
|
68
59
|
s.add_dependency(%q<json>, [">= 1.6.5"])
|
69
|
-
s.add_dependency(%q<redis>, [">= 0"])
|
70
60
|
s.add_dependency(%q<shoulda>, [">= 0"])
|
71
61
|
s.add_dependency(%q<rdoc>, ["~> 3.12"])
|
72
62
|
s.add_dependency(%q<bundler>, [">= 1.1.4"])
|
@@ -76,7 +66,6 @@ Gem::Specification.new do |s|
|
|
76
66
|
end
|
77
67
|
else
|
78
68
|
s.add_dependency(%q<json>, [">= 1.6.5"])
|
79
|
-
s.add_dependency(%q<redis>, [">= 0"])
|
80
69
|
s.add_dependency(%q<shoulda>, [">= 0"])
|
81
70
|
s.add_dependency(%q<rdoc>, ["~> 3.12"])
|
82
71
|
s.add_dependency(%q<bundler>, [">= 1.1.4"])
|
data/lib/keen.rb
CHANGED
data/lib/keen/client.rb
CHANGED
@@ -6,10 +6,8 @@ require "net/https"
|
|
6
6
|
|
7
7
|
|
8
8
|
module Keen
|
9
|
-
|
10
9
|
class Client
|
11
|
-
|
12
|
-
attr_accessor :storage_handler, :project_id, :auth_token, :options, :logging
|
10
|
+
attr_accessor :project_id, :api_key, :options, :logging
|
13
11
|
|
14
12
|
def base_url
|
15
13
|
if @options[:base_url]
|
@@ -19,50 +17,25 @@ module Keen
|
|
19
17
|
end
|
20
18
|
end
|
21
19
|
|
22
|
-
def
|
20
|
+
def ssl_cert_file
|
21
|
+
File.expand_path("../../../conf/cacert.pem", __FILE__)
|
22
|
+
end
|
23
23
|
|
24
|
+
def initialize(project_id, api_key, options = {})
|
24
25
|
raise "project_id must be string" unless project_id.kind_of? String
|
25
|
-
raise "
|
26
|
+
raise "api_key must be string" unless api_key.kind_of? String
|
26
27
|
|
27
28
|
default_options = {
|
28
29
|
:logging => true,
|
29
|
-
|
30
|
-
# should we cache events locally somewhere? if false, sends directly to Keen
|
31
|
-
:cache_locally => false,
|
32
|
-
|
33
|
-
# this is required if cache_locally is true:
|
34
|
-
:storage_class => nil,
|
35
|
-
|
36
|
-
# all keys will be prepended with this:
|
37
|
-
:storage_namespace => "default",
|
38
30
|
}
|
39
31
|
|
40
32
|
options = default_options.update(options)
|
41
33
|
|
42
34
|
@project_id = project_id
|
43
|
-
@
|
44
|
-
@cache_locally = options[:cache_locally]
|
45
|
-
|
46
|
-
if @cache_locally
|
47
|
-
raise "Local caching not supported in this version."
|
48
|
-
|
49
|
-
@storage_class = options[:storage_class]
|
50
|
-
unless @storage_class and @storage_class < Keen::Async::Storage::BaseStorageHandler
|
51
|
-
raise "The Storage Class you send must extend BaseStorageHandler. You sent: #{@storage_class}"
|
52
|
-
end
|
53
|
-
end
|
35
|
+
@api_key = api_key
|
54
36
|
|
55
37
|
@logging = options[:logging]
|
56
38
|
@options = options
|
57
|
-
|
58
|
-
end
|
59
|
-
|
60
|
-
def storage_handler
|
61
|
-
unless @storage_handler
|
62
|
-
@storage_handler = @storage_class.new(self)
|
63
|
-
end
|
64
|
-
|
65
|
-
@storage_handler
|
66
39
|
end
|
67
40
|
|
68
41
|
def add_event(event_collection, event_properties, timestamp=nil)
|
@@ -87,7 +60,7 @@ module Keen
|
|
87
60
|
uri = URI.parse(url)
|
88
61
|
http = Net::HTTP.new(uri.host, uri.port)
|
89
62
|
http.use_ssl = true
|
90
|
-
http.ca_file =
|
63
|
+
http.ca_file = ssl_cert_file
|
91
64
|
http.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
92
65
|
http.verify_depth = 5
|
93
66
|
|
@@ -104,65 +77,14 @@ module Keen
|
|
104
77
|
request.body = JSON.generate(body)
|
105
78
|
|
106
79
|
request["Content-Type"] = "application/json"
|
107
|
-
request["Authorization"] = @
|
80
|
+
request["Authorization"] = @api_key
|
108
81
|
|
109
82
|
response = http.request(request)
|
110
83
|
JSON.parse response.body
|
111
|
-
|
112
|
-
end
|
113
|
-
|
114
|
-
def send_batch(events)
|
115
|
-
# make the request body:
|
116
|
-
request_body = {}
|
117
|
-
events.each { |event|
|
118
|
-
unless request_body.has_key? event.event_collection
|
119
|
-
request_body[event.event_collection] = []
|
120
|
-
end
|
121
|
-
|
122
|
-
header = {"timestamp" => event.timestamp}
|
123
|
-
body = event.body
|
124
|
-
item = {"header" => header, "body" => body}
|
125
|
-
request_body[event.event_collection].push(item)
|
126
|
-
}
|
127
|
-
request_body = request_body.to_json
|
128
|
-
|
129
|
-
# build the request:
|
130
|
-
url = "#{base_url}/projects/#{project_id}/_events"
|
131
|
-
uri = URI.parse(url)
|
132
|
-
http = Net::HTTP.new(uri.host, uri.port)
|
133
|
-
http.use_ssl = true
|
134
|
-
http.ca_file = Keen::Async::SSL_CA_FILE
|
135
|
-
http.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
136
|
-
http.verify_depth = 5
|
137
|
-
|
138
|
-
request = Net::HTTP::Post.new(uri.path)
|
139
|
-
request.body = request_body
|
140
|
-
request["Content-Type"] = "application/json"
|
141
|
-
request["Authorization"] = auth_token
|
142
|
-
|
143
|
-
resp = http.request(request)
|
144
|
-
|
145
|
-
if @logging
|
146
|
-
puts resp.body
|
147
|
-
end
|
148
|
-
|
149
|
-
return JSON.parse resp.body
|
150
84
|
end
|
151
85
|
|
152
86
|
def validate_event_collection(event_collection)
|
153
87
|
# TODO
|
154
88
|
end
|
155
|
-
|
156
|
-
def self.create_new_storage_handler(storage_mode, client, logging)
|
157
|
-
# This is shitty as hell. We shoudl take in a class reference pointing to the storage handler, not switch on string values
|
158
|
-
case storage_mode.to_sym
|
159
|
-
|
160
|
-
when :redis
|
161
|
-
Keen::Async::Storage::RedisHandler.new(client, logging)
|
162
|
-
else
|
163
|
-
raise "Unknown storage_mode sent to client: `#{storage_mode}`"
|
164
|
-
end
|
165
|
-
end
|
166
|
-
|
167
89
|
end
|
168
90
|
end
|
data/lib/keen/keys.rb
CHANGED
metadata
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
name: keen
|
3
3
|
version: !ruby/object:Gem::Version
|
4
4
|
prerelease:
|
5
|
-
version: 0.2.
|
5
|
+
version: 0.2.2
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- dorkitude
|
@@ -10,7 +10,7 @@ autorequire:
|
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
12
|
|
13
|
-
date: 2012-10-
|
13
|
+
date: 2012-10-27 00:00:00 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: json
|
@@ -23,20 +23,9 @@ dependencies:
|
|
23
23
|
type: :runtime
|
24
24
|
prerelease: false
|
25
25
|
version_requirements: *id001
|
26
|
-
- !ruby/object:Gem::Dependency
|
27
|
-
name: redis
|
28
|
-
requirement: &id002 !ruby/object:Gem::Requirement
|
29
|
-
none: false
|
30
|
-
requirements:
|
31
|
-
- - ">="
|
32
|
-
- !ruby/object:Gem::Version
|
33
|
-
version: "0"
|
34
|
-
type: :runtime
|
35
|
-
prerelease: false
|
36
|
-
version_requirements: *id002
|
37
26
|
- !ruby/object:Gem::Dependency
|
38
27
|
name: shoulda
|
39
|
-
requirement: &
|
28
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
40
29
|
none: false
|
41
30
|
requirements:
|
42
31
|
- - ">="
|
@@ -44,10 +33,10 @@ dependencies:
|
|
44
33
|
version: "0"
|
45
34
|
type: :development
|
46
35
|
prerelease: false
|
47
|
-
version_requirements: *
|
36
|
+
version_requirements: *id002
|
48
37
|
- !ruby/object:Gem::Dependency
|
49
38
|
name: rdoc
|
50
|
-
requirement: &
|
39
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
51
40
|
none: false
|
52
41
|
requirements:
|
53
42
|
- - ~>
|
@@ -55,10 +44,10 @@ dependencies:
|
|
55
44
|
version: "3.12"
|
56
45
|
type: :development
|
57
46
|
prerelease: false
|
58
|
-
version_requirements: *
|
47
|
+
version_requirements: *id003
|
59
48
|
- !ruby/object:Gem::Dependency
|
60
49
|
name: bundler
|
61
|
-
requirement: &
|
50
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
62
51
|
none: false
|
63
52
|
requirements:
|
64
53
|
- - ">="
|
@@ -66,10 +55,10 @@ dependencies:
|
|
66
55
|
version: 1.1.4
|
67
56
|
type: :development
|
68
57
|
prerelease: false
|
69
|
-
version_requirements: *
|
58
|
+
version_requirements: *id004
|
70
59
|
- !ruby/object:Gem::Dependency
|
71
60
|
name: jeweler
|
72
|
-
requirement: &
|
61
|
+
requirement: &id005 !ruby/object:Gem::Requirement
|
73
62
|
none: false
|
74
63
|
requirements:
|
75
64
|
- - ~>
|
@@ -77,10 +66,10 @@ dependencies:
|
|
77
66
|
version: 1.8.3
|
78
67
|
type: :development
|
79
68
|
prerelease: false
|
80
|
-
version_requirements: *
|
69
|
+
version_requirements: *id005
|
81
70
|
- !ruby/object:Gem::Dependency
|
82
71
|
name: rspec
|
83
|
-
requirement: &
|
72
|
+
requirement: &id006 !ruby/object:Gem::Requirement
|
84
73
|
none: false
|
85
74
|
requirements:
|
86
75
|
- - ">="
|
@@ -88,10 +77,10 @@ dependencies:
|
|
88
77
|
version: 2.9.0
|
89
78
|
type: :development
|
90
79
|
prerelease: false
|
91
|
-
version_requirements: *
|
80
|
+
version_requirements: *id006
|
92
81
|
- !ruby/object:Gem::Dependency
|
93
82
|
name: cucumber
|
94
|
-
requirement: &
|
83
|
+
requirement: &id007 !ruby/object:Gem::Requirement
|
95
84
|
none: false
|
96
85
|
requirements:
|
97
86
|
- - ">="
|
@@ -99,7 +88,7 @@ dependencies:
|
|
99
88
|
version: 1.2.1
|
100
89
|
type: :development
|
101
90
|
prerelease: false
|
102
|
-
version_requirements: *
|
91
|
+
version_requirements: *id007
|
103
92
|
description: See the github repo or examples.rb for usage information.
|
104
93
|
email: kyle@keen.io
|
105
94
|
executables: []
|
@@ -119,25 +108,17 @@ files:
|
|
119
108
|
- conf/cacert.pem
|
120
109
|
- examples.rb
|
121
110
|
- features/add_event.feature
|
122
|
-
- features/redis_queue.feature
|
123
111
|
- features/step_definitions/keen_steps.rb
|
124
112
|
- features/support/before_and_after.rb
|
125
113
|
- features/support/env.rb
|
126
114
|
- keen.gemspec
|
127
115
|
- keen.gemspec.old
|
128
116
|
- lib/keen.rb
|
129
|
-
- lib/keen/async.rb
|
130
|
-
- lib/keen/async/job.rb
|
131
|
-
- lib/keen/async/storage/base_storage_handler.rb
|
132
|
-
- lib/keen/async/storage/flat_file_handler.rb
|
133
|
-
- lib/keen/async/storage/redis_handler.rb
|
134
|
-
- lib/keen/async/worker.rb
|
135
117
|
- lib/keen/client.rb
|
136
118
|
- lib/keen/event.rb
|
137
119
|
- lib/keen/keys.rb
|
138
120
|
- lib/keen/utils.rb
|
139
121
|
- lib/keen/version.rb
|
140
|
-
- send.rb
|
141
122
|
homepage: http://github.com/dorkitude/keen
|
142
123
|
licenses:
|
143
124
|
- MIT
|
@@ -151,7 +132,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
151
132
|
requirements:
|
152
133
|
- - ">="
|
153
134
|
- !ruby/object:Gem::Version
|
154
|
-
hash:
|
135
|
+
hash: -1089062081955943772
|
155
136
|
segments:
|
156
137
|
- 0
|
157
138
|
version: "0"
|
@@ -1,34 +0,0 @@
|
|
1
|
-
# Language: en
|
2
|
-
Feature: RedisHandler
|
3
|
-
In order to make fewer than n HTTP connections for n events
|
4
|
-
As a developer
|
5
|
-
I want to be able to batch up events in Redis and send them to Keen all at once.
|
6
|
-
|
7
|
-
Scenario Outline: Add Events to Redis queue
|
8
|
-
Given a Keen Client using Redis
|
9
|
-
When I post <n> events
|
10
|
-
Then the size of the Redis queue should have gone up by <n>.
|
11
|
-
|
12
|
-
Examples:
|
13
|
-
|n |
|
14
|
-
|1 |
|
15
|
-
|100 |
|
16
|
-
|99 |
|
17
|
-
|1000 |
|
18
|
-
|999 |
|
19
|
-
|
20
|
-
Scenario Outline: Batch a bunch of events in the Redis queue, then send them.
|
21
|
-
Given a Keen Client using Redis
|
22
|
-
When I post <n> events
|
23
|
-
And I process the queue
|
24
|
-
Then the response from Keen should be <n> happy smiles
|
25
|
-
And the queue should be empty.
|
26
|
-
|
27
|
-
Examples:
|
28
|
-
|n |
|
29
|
-
|1 |
|
30
|
-
|100 |
|
31
|
-
|99 |
|
32
|
-
|1000 |
|
33
|
-
|999 |
|
34
|
-
|
data/lib/keen/async.rb
DELETED
data/lib/keen/async/job.rb
DELETED
@@ -1,79 +0,0 @@
|
|
1
|
-
require 'keen/async/storage/redis_handler'
|
2
|
-
|
3
|
-
module Keen
|
4
|
-
module Async
|
5
|
-
class Job
|
6
|
-
# Represents one job.
|
7
|
-
#
|
8
|
-
#
|
9
|
-
|
10
|
-
attr_accessor :project_id, :auth_token, :collection_name, :event_body, :timestamp
|
11
|
-
|
12
|
-
def to_json(options=nil)
|
13
|
-
@definition.to_json
|
14
|
-
end
|
15
|
-
|
16
|
-
|
17
|
-
def to_s
|
18
|
-
self.to_json
|
19
|
-
end
|
20
|
-
|
21
|
-
def initialize(handler, definition)
|
22
|
-
# The `definition` can come from redis, a flat file, or code.
|
23
|
-
load_definition(definition)
|
24
|
-
@handler = handler
|
25
|
-
|
26
|
-
end
|
27
|
-
|
28
|
-
def load_definition(definition)
|
29
|
-
|
30
|
-
definition = Keen::Utils.symbolize_keys(definition)
|
31
|
-
|
32
|
-
# define some key lists:
|
33
|
-
required_keys = [:timestamp, :project_id, :auth_token, :collection_name, :event_body]
|
34
|
-
optional_keys = [:keen_client_version]
|
35
|
-
all_keys = required_keys + optional_keys
|
36
|
-
|
37
|
-
|
38
|
-
# don't allow them to send nil values for anything
|
39
|
-
definition.each do |key, value|
|
40
|
-
# reject unrecognized keys:
|
41
|
-
raise "Unrecognized key: #{key}" unless all_keys.include? key.to_sym
|
42
|
-
end
|
43
|
-
|
44
|
-
|
45
|
-
required_keys.each do |key|
|
46
|
-
|
47
|
-
unless definition.has_key? key
|
48
|
-
raise "You failed to send: #{key} -- you sent #{JSON.generate definition}"
|
49
|
-
end
|
50
|
-
|
51
|
-
value = definition[key]
|
52
|
-
|
53
|
-
raise "You sent a nil value for the #{key}." if value.nil?
|
54
|
-
end
|
55
|
-
|
56
|
-
|
57
|
-
all_keys.each do |key|
|
58
|
-
value = definition[key]
|
59
|
-
self.instance_variable_set("@#{key}", value)
|
60
|
-
end
|
61
|
-
|
62
|
-
unless definition.has_key? :keen_client_version
|
63
|
-
definition[:keen_client_version] = Keen::VERSION
|
64
|
-
end
|
65
|
-
|
66
|
-
@definition = definition
|
67
|
-
|
68
|
-
|
69
|
-
end
|
70
|
-
|
71
|
-
def save
|
72
|
-
@handler.record_job(self)
|
73
|
-
end
|
74
|
-
|
75
|
-
end
|
76
|
-
end
|
77
|
-
end
|
78
|
-
|
79
|
-
|
@@ -1,51 +0,0 @@
|
|
1
|
-
require 'json'
|
2
|
-
require 'time'
|
3
|
-
|
4
|
-
module Keen
|
5
|
-
module Async
|
6
|
-
module Storage
|
7
|
-
class BaseStorageHandler
|
8
|
-
|
9
|
-
def initialize(client)
|
10
|
-
@logging = client.logging
|
11
|
-
@client = client
|
12
|
-
end
|
13
|
-
|
14
|
-
# Key stuff
|
15
|
-
# ----
|
16
|
-
|
17
|
-
def global_key_prefix
|
18
|
-
"keen.#{@client.options[:storage_namespace]}"
|
19
|
-
end
|
20
|
-
|
21
|
-
def active_queue_key
|
22
|
-
"#{global_key_prefix}.active_queue"
|
23
|
-
end
|
24
|
-
|
25
|
-
def failed_queue_key
|
26
|
-
"#{global_key_prefix}.failed_queue"
|
27
|
-
end
|
28
|
-
|
29
|
-
def add_to_active_queue(value)
|
30
|
-
@redis.lpush active_queue_key, value
|
31
|
-
if @logging
|
32
|
-
puts "added #{value} to active queue; length is now #{@redis.llen active_queue_key}"
|
33
|
-
end
|
34
|
-
end
|
35
|
-
|
36
|
-
def record_job(job)
|
37
|
-
add_to_active_queue JSON.generate(job)
|
38
|
-
end
|
39
|
-
|
40
|
-
def handle_prior_failures
|
41
|
-
# TODO consume the failed_queue and do something with it (loggly? retry? flat file?)
|
42
|
-
end
|
43
|
-
|
44
|
-
def count_active_queue
|
45
|
-
raise NotImplementedError
|
46
|
-
end
|
47
|
-
|
48
|
-
end
|
49
|
-
end
|
50
|
-
end
|
51
|
-
end
|
@@ -1,57 +0,0 @@
|
|
1
|
-
require 'fileutils'
|
2
|
-
require 'base64'
|
3
|
-
require 'zlib'
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
module Keen
|
8
|
-
module Async
|
9
|
-
module Storage
|
10
|
-
class FlatFileHandler
|
11
|
-
|
12
|
-
# Paths
|
13
|
-
# -----
|
14
|
-
|
15
|
-
# Where new events go as they come in:
|
16
|
-
ACTIVE = "active.txt"
|
17
|
-
|
18
|
-
# The temporary file that will store records during processing:
|
19
|
-
PROCESSING = "processing.txt"
|
20
|
-
|
21
|
-
# Records are put here if processing fails:
|
22
|
-
FAILED = "failed.txt"
|
23
|
-
|
24
|
-
def initialize(filepath)
|
25
|
-
@filepath = filepath
|
26
|
-
|
27
|
-
is_directory?
|
28
|
-
is_writable?
|
29
|
-
is_readable?
|
30
|
-
end
|
31
|
-
|
32
|
-
def dump_contents
|
33
|
-
# move
|
34
|
-
end
|
35
|
-
|
36
|
-
def is_writable?
|
37
|
-
if not FileTest.writable? @filepath
|
38
|
-
raise "Can't write to file: " + @filepath
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
|
-
def is_readable?
|
43
|
-
if not FileTest.readable? @filepath
|
44
|
-
raise "Can't read from file: " + @filepath
|
45
|
-
end
|
46
|
-
end
|
47
|
-
|
48
|
-
def is_directory?
|
49
|
-
if not FileTest.directory? @filepath
|
50
|
-
raise "Can't read from file: " + @filepath
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
end
|
55
|
-
end
|
56
|
-
end
|
57
|
-
end
|
@@ -1,89 +0,0 @@
|
|
1
|
-
require 'fileutils'
|
2
|
-
require 'redis'
|
3
|
-
require 'json'
|
4
|
-
require 'time'
|
5
|
-
|
6
|
-
module Keen
|
7
|
-
module Async
|
8
|
-
module Storage
|
9
|
-
class RedisHandler < Keen::Async::Storage::BaseStorageHandler
|
10
|
-
|
11
|
-
# Keys
|
12
|
-
# ----
|
13
|
-
|
14
|
-
def add_to_active_queue(value)
|
15
|
-
redis.lpush active_queue_key, value
|
16
|
-
if @logging
|
17
|
-
puts "added #{value} to active queue; length is now #{redis.llen active_queue_key}"
|
18
|
-
end
|
19
|
-
end
|
20
|
-
|
21
|
-
def redis
|
22
|
-
unless @redis
|
23
|
-
@redis = Redis.new
|
24
|
-
end
|
25
|
-
|
26
|
-
@redis
|
27
|
-
end
|
28
|
-
|
29
|
-
def count_active_queue
|
30
|
-
redis.llen active_queue_key
|
31
|
-
end
|
32
|
-
|
33
|
-
def clear_active_queue
|
34
|
-
redis.del active_queue_key
|
35
|
-
end
|
36
|
-
|
37
|
-
def get_authorized_jobs(how_many, client)
|
38
|
-
|
39
|
-
handle_prior_failures
|
40
|
-
|
41
|
-
key = active_queue_key
|
42
|
-
|
43
|
-
job_definitions = []
|
44
|
-
skipped_job_definitions = []
|
45
|
-
|
46
|
-
#puts "doing the job #{how_many} times"
|
47
|
-
|
48
|
-
while true do
|
49
|
-
this = redis.lpop key
|
50
|
-
|
51
|
-
# If we're out of jobs, end the loop:
|
52
|
-
if not this
|
53
|
-
break
|
54
|
-
end
|
55
|
-
|
56
|
-
# Parse the JSON into a job definition
|
57
|
-
job_definition = JSON.parse this
|
58
|
-
job_definition = Keen::Utils.symbolize_keys(job_definition)
|
59
|
-
|
60
|
-
# Make sure this client is authorized to process this job:
|
61
|
-
unless job_definition[:project_id] == client.project_id
|
62
|
-
unless job_definition[:auth_token] == client.auth_token
|
63
|
-
# We're not authorized, so skip this job.
|
64
|
-
skipped_job_definitions.push job_definition
|
65
|
-
next
|
66
|
-
end
|
67
|
-
end
|
68
|
-
|
69
|
-
job_definitions.push job_definition
|
70
|
-
|
71
|
-
if job_definitions.length == how_many
|
72
|
-
break
|
73
|
-
end
|
74
|
-
|
75
|
-
end
|
76
|
-
|
77
|
-
# Put the skipped jobs back on the queue.
|
78
|
-
skipped_job_definitions.each do |job_definition|
|
79
|
-
redis.lpush key, job_definition
|
80
|
-
end
|
81
|
-
|
82
|
-
job_definitions
|
83
|
-
|
84
|
-
end
|
85
|
-
|
86
|
-
end
|
87
|
-
end
|
88
|
-
end
|
89
|
-
end
|
data/lib/keen/async/worker.rb
DELETED
@@ -1,62 +0,0 @@
|
|
1
|
-
require "keen/async/storage/redis_handler"
|
2
|
-
|
3
|
-
|
4
|
-
module Keen
|
5
|
-
|
6
|
-
module Async
|
7
|
-
|
8
|
-
# How many events should we send over the wire at a time?
|
9
|
-
BATCH_SIZE = 100
|
10
|
-
SSL_CA_FILE = File.dirname(__FILE__) + '../../../conf/cacert.pem'
|
11
|
-
|
12
|
-
class Worker
|
13
|
-
|
14
|
-
def initialize(client)
|
15
|
-
@client = client
|
16
|
-
@storage_handler = client.storage_handler
|
17
|
-
end
|
18
|
-
|
19
|
-
def batch_url(project_id)
|
20
|
-
if not project_id
|
21
|
-
raise "Missing project_id."
|
22
|
-
end
|
23
|
-
"https://api.keen.io/2.0/projects/#{project_id}/_events"
|
24
|
-
end
|
25
|
-
|
26
|
-
def process_queue
|
27
|
-
queue_length = @storage_handler.count_active_queue
|
28
|
-
|
29
|
-
batch_size = Keen::Async::BATCH_SIZE
|
30
|
-
|
31
|
-
|
32
|
-
events = []
|
33
|
-
|
34
|
-
|
35
|
-
responses = []
|
36
|
-
|
37
|
-
num_batches = queue_length / batch_size + 1
|
38
|
-
num_batches.times do
|
39
|
-
|
40
|
-
job_definitions = @storage_handler.get_authorized_jobs(batch_size, @client)
|
41
|
-
|
42
|
-
job_definitions.each do |job_definition|
|
43
|
-
#puts JSON.generate job_definition
|
44
|
-
job = Keen::Async::Job.new(@client, job_definition)
|
45
|
-
events.push Keen::Event.new(job.timestamp, job.collection_name, job.event_body)
|
46
|
-
end
|
47
|
-
|
48
|
-
responses.push @client.send_batch(events)
|
49
|
-
end
|
50
|
-
|
51
|
-
if @client.logging
|
52
|
-
puts responses
|
53
|
-
end
|
54
|
-
|
55
|
-
responses
|
56
|
-
|
57
|
-
end
|
58
|
-
|
59
|
-
end
|
60
|
-
end
|
61
|
-
|
62
|
-
end
|
data/send.rb
DELETED
@@ -1,12 +0,0 @@
|
|
1
|
-
require "rubygems"
|
2
|
-
require "keen"
|
3
|
-
|
4
|
-
storage_handler = Keen::Client.create_new_storage_handler(:redis)
|
5
|
-
|
6
|
-
count = storage_handler.count_active_queue
|
7
|
-
puts "we have this many jobs: #{count}"
|
8
|
-
|
9
|
-
worker = Keen::Async::Worker.new(storage_handler)
|
10
|
-
results = worker.process_queue
|
11
|
-
|
12
|
-
puts results
|