keen 0.2.0 → 0.2.2
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +0 -1
- data/README.md +8 -1
- data/VERSION.yml +1 -1
- data/examples.rb +2 -2
- data/features/step_definitions/keen_steps.rb +1 -37
- data/features/support/before_and_after.rb +1 -1
- data/keen.gemspec +3 -14
- data/lib/keen.rb +0 -1
- data/lib/keen/client.rb +9 -87
- data/lib/keen/keys.rb +2 -2
- metadata +15 -34
- data/features/redis_queue.feature +0 -34
- data/lib/keen/async.rb +0 -6
- data/lib/keen/async/job.rb +0 -79
- data/lib/keen/async/storage/base_storage_handler.rb +0 -51
- data/lib/keen/async/storage/flat_file_handler.rb +0 -57
- data/lib/keen/async/storage/redis_handler.rb +0 -89
- data/lib/keen/async/worker.rb +0 -62
- data/send.rb +0 -12
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -6,11 +6,18 @@ Usage
|
|
6
6
|
|
7
7
|
`gem install keen`
|
8
8
|
|
9
|
-
|
9
|
+
Ruby Client Usage Guide
|
10
|
+
http://keen.io/static/docs/clients/ruby/usage_guide.html
|
11
|
+
|
12
|
+
see Also the Cucumber and Rspec stuff in /features/ for usage details)
|
10
13
|
|
11
14
|
Contributing to keen
|
12
15
|
--------------------
|
13
16
|
|
17
|
+
* Use Jeweler!
|
18
|
+
* rake version:bump:major
|
19
|
+
* rake release
|
20
|
+
* Make sure all tests pass in 1.9.2 and 1.9.3
|
14
21
|
* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
|
15
22
|
* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it.
|
16
23
|
* Fork the project.
|
data/VERSION.yml
CHANGED
data/examples.rb
CHANGED
@@ -3,10 +3,10 @@ require 'keen'
|
|
3
3
|
|
4
4
|
# Get these from the keen.io website:
|
5
5
|
project_id = 'asdfasldkfjalsdkfalskdfj'
|
6
|
-
|
6
|
+
api_key = 'asldfjklj325tkl32jaskdlfjaf'
|
7
7
|
|
8
8
|
# First you must setup the client:
|
9
|
-
keen = Keen::Client.new(project_id,
|
9
|
+
keen = Keen::Client.new(project_id, api_key, :storage_mode => :redis)
|
10
10
|
|
11
11
|
# Then, you can use that client to send events.
|
12
12
|
|
@@ -3,23 +3,9 @@ require 'cucumber/formatter/unicode'
|
|
3
3
|
$:.unshift(File.dirname(__FILE__) + '/../../lib')
|
4
4
|
require 'keen'
|
5
5
|
|
6
|
-
Given /^a Keen Client using Redis$/ do
|
7
|
-
@client = Keen::Client.new(@project_id,
|
8
|
-
@auth_token,
|
9
|
-
:storage_class => Keen::Async::Storage::RedisHandler,
|
10
|
-
:storage_namespace => "test",
|
11
|
-
:cache_locally => true,
|
12
|
-
:logging => false )
|
13
|
-
|
14
|
-
@client.storage_handler.clear_active_queue
|
15
|
-
|
16
|
-
@starting_queue_size = @client.storage_handler.count_active_queue
|
17
|
-
end
|
18
|
-
|
19
6
|
Given /^a Keen Client in Direct Send mode$/ do
|
20
7
|
@client = Keen::Client.new(@project_id,
|
21
|
-
@
|
22
|
-
:cache_locally => false,
|
8
|
+
@api_key,
|
23
9
|
:logging => false )
|
24
10
|
end
|
25
11
|
|
@@ -27,10 +13,6 @@ When /^I post an event$/ do
|
|
27
13
|
@result = @client.add_event("cucumber_events", {:hi_from => "cucumber!", :keen_version => Keen::VERSION})
|
28
14
|
end
|
29
15
|
|
30
|
-
Then /^the size of the Redis queue should have gone up by (\d+)\.$/ do |n|
|
31
|
-
@client.storage_handler.count_active_queue.should == n.to_i + @starting_queue_size
|
32
|
-
end
|
33
|
-
|
34
16
|
Then /^the response from the server should be good\.$/ do
|
35
17
|
response = @result
|
36
18
|
response.should == {"created" => true}
|
@@ -42,21 +24,3 @@ When /^I post (\d+) events$/ do |n|
|
|
42
24
|
@client.add_event("cucumber_events", {:hi_from => "cucumber!", :keen_version => Keen::VERSION})
|
43
25
|
end
|
44
26
|
end
|
45
|
-
|
46
|
-
When /^I process the queue$/ do
|
47
|
-
worker = Keen::Async::Worker.new(@client)
|
48
|
-
@result = worker.process_queue
|
49
|
-
end
|
50
|
-
|
51
|
-
Then /^the response from Keen should be (\d+) happy smiles$/ do |n|
|
52
|
-
expectation = []
|
53
|
-
n.to_i.times do
|
54
|
-
expectation.push({"success" => true})
|
55
|
-
end
|
56
|
-
|
57
|
-
expectation = {"cucumber_events" => expectation}
|
58
|
-
end
|
59
|
-
|
60
|
-
Then /^the queue should be empty\.$/ do
|
61
|
-
@client.storage_handler.count_active_queue.should == 0
|
62
|
-
end
|
data/keen.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = "keen"
|
8
|
-
s.version = "0.2.
|
8
|
+
s.version = "0.2.2"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["dorkitude"]
|
12
|
-
s.date = "2012-10-
|
12
|
+
s.date = "2012-10-27"
|
13
13
|
s.description = "See the github repo or examples.rb for usage information."
|
14
14
|
s.email = "kyle@keen.io"
|
15
15
|
s.extra_rdoc_files = [
|
@@ -26,25 +26,17 @@ Gem::Specification.new do |s|
|
|
26
26
|
"conf/cacert.pem",
|
27
27
|
"examples.rb",
|
28
28
|
"features/add_event.feature",
|
29
|
-
"features/redis_queue.feature",
|
30
29
|
"features/step_definitions/keen_steps.rb",
|
31
30
|
"features/support/before_and_after.rb",
|
32
31
|
"features/support/env.rb",
|
33
32
|
"keen.gemspec",
|
34
33
|
"keen.gemspec.old",
|
35
34
|
"lib/keen.rb",
|
36
|
-
"lib/keen/async.rb",
|
37
|
-
"lib/keen/async/job.rb",
|
38
|
-
"lib/keen/async/storage/base_storage_handler.rb",
|
39
|
-
"lib/keen/async/storage/flat_file_handler.rb",
|
40
|
-
"lib/keen/async/storage/redis_handler.rb",
|
41
|
-
"lib/keen/async/worker.rb",
|
42
35
|
"lib/keen/client.rb",
|
43
36
|
"lib/keen/event.rb",
|
44
37
|
"lib/keen/keys.rb",
|
45
38
|
"lib/keen/utils.rb",
|
46
|
-
"lib/keen/version.rb"
|
47
|
-
"send.rb"
|
39
|
+
"lib/keen/version.rb"
|
48
40
|
]
|
49
41
|
s.homepage = "http://github.com/dorkitude/keen"
|
50
42
|
s.licenses = ["MIT"]
|
@@ -57,7 +49,6 @@ Gem::Specification.new do |s|
|
|
57
49
|
|
58
50
|
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
59
51
|
s.add_runtime_dependency(%q<json>, [">= 1.6.5"])
|
60
|
-
s.add_runtime_dependency(%q<redis>, [">= 0"])
|
61
52
|
s.add_development_dependency(%q<shoulda>, [">= 0"])
|
62
53
|
s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
|
63
54
|
s.add_development_dependency(%q<bundler>, [">= 1.1.4"])
|
@@ -66,7 +57,6 @@ Gem::Specification.new do |s|
|
|
66
57
|
s.add_development_dependency(%q<cucumber>, [">= 1.2.1"])
|
67
58
|
else
|
68
59
|
s.add_dependency(%q<json>, [">= 1.6.5"])
|
69
|
-
s.add_dependency(%q<redis>, [">= 0"])
|
70
60
|
s.add_dependency(%q<shoulda>, [">= 0"])
|
71
61
|
s.add_dependency(%q<rdoc>, ["~> 3.12"])
|
72
62
|
s.add_dependency(%q<bundler>, [">= 1.1.4"])
|
@@ -76,7 +66,6 @@ Gem::Specification.new do |s|
|
|
76
66
|
end
|
77
67
|
else
|
78
68
|
s.add_dependency(%q<json>, [">= 1.6.5"])
|
79
|
-
s.add_dependency(%q<redis>, [">= 0"])
|
80
69
|
s.add_dependency(%q<shoulda>, [">= 0"])
|
81
70
|
s.add_dependency(%q<rdoc>, ["~> 3.12"])
|
82
71
|
s.add_dependency(%q<bundler>, [">= 1.1.4"])
|
data/lib/keen.rb
CHANGED
data/lib/keen/client.rb
CHANGED
@@ -6,10 +6,8 @@ require "net/https"
|
|
6
6
|
|
7
7
|
|
8
8
|
module Keen
|
9
|
-
|
10
9
|
class Client
|
11
|
-
|
12
|
-
attr_accessor :storage_handler, :project_id, :auth_token, :options, :logging
|
10
|
+
attr_accessor :project_id, :api_key, :options, :logging
|
13
11
|
|
14
12
|
def base_url
|
15
13
|
if @options[:base_url]
|
@@ -19,50 +17,25 @@ module Keen
|
|
19
17
|
end
|
20
18
|
end
|
21
19
|
|
22
|
-
def
|
20
|
+
def ssl_cert_file
|
21
|
+
File.expand_path("../../../conf/cacert.pem", __FILE__)
|
22
|
+
end
|
23
23
|
|
24
|
+
def initialize(project_id, api_key, options = {})
|
24
25
|
raise "project_id must be string" unless project_id.kind_of? String
|
25
|
-
raise "
|
26
|
+
raise "api_key must be string" unless api_key.kind_of? String
|
26
27
|
|
27
28
|
default_options = {
|
28
29
|
:logging => true,
|
29
|
-
|
30
|
-
# should we cache events locally somewhere? if false, sends directly to Keen
|
31
|
-
:cache_locally => false,
|
32
|
-
|
33
|
-
# this is required if cache_locally is true:
|
34
|
-
:storage_class => nil,
|
35
|
-
|
36
|
-
# all keys will be prepended with this:
|
37
|
-
:storage_namespace => "default",
|
38
30
|
}
|
39
31
|
|
40
32
|
options = default_options.update(options)
|
41
33
|
|
42
34
|
@project_id = project_id
|
43
|
-
@
|
44
|
-
@cache_locally = options[:cache_locally]
|
45
|
-
|
46
|
-
if @cache_locally
|
47
|
-
raise "Local caching not supported in this version."
|
48
|
-
|
49
|
-
@storage_class = options[:storage_class]
|
50
|
-
unless @storage_class and @storage_class < Keen::Async::Storage::BaseStorageHandler
|
51
|
-
raise "The Storage Class you send must extend BaseStorageHandler. You sent: #{@storage_class}"
|
52
|
-
end
|
53
|
-
end
|
35
|
+
@api_key = api_key
|
54
36
|
|
55
37
|
@logging = options[:logging]
|
56
38
|
@options = options
|
57
|
-
|
58
|
-
end
|
59
|
-
|
60
|
-
def storage_handler
|
61
|
-
unless @storage_handler
|
62
|
-
@storage_handler = @storage_class.new(self)
|
63
|
-
end
|
64
|
-
|
65
|
-
@storage_handler
|
66
39
|
end
|
67
40
|
|
68
41
|
def add_event(event_collection, event_properties, timestamp=nil)
|
@@ -87,7 +60,7 @@ module Keen
|
|
87
60
|
uri = URI.parse(url)
|
88
61
|
http = Net::HTTP.new(uri.host, uri.port)
|
89
62
|
http.use_ssl = true
|
90
|
-
http.ca_file =
|
63
|
+
http.ca_file = ssl_cert_file
|
91
64
|
http.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
92
65
|
http.verify_depth = 5
|
93
66
|
|
@@ -104,65 +77,14 @@ module Keen
|
|
104
77
|
request.body = JSON.generate(body)
|
105
78
|
|
106
79
|
request["Content-Type"] = "application/json"
|
107
|
-
request["Authorization"] = @
|
80
|
+
request["Authorization"] = @api_key
|
108
81
|
|
109
82
|
response = http.request(request)
|
110
83
|
JSON.parse response.body
|
111
|
-
|
112
|
-
end
|
113
|
-
|
114
|
-
def send_batch(events)
|
115
|
-
# make the request body:
|
116
|
-
request_body = {}
|
117
|
-
events.each { |event|
|
118
|
-
unless request_body.has_key? event.event_collection
|
119
|
-
request_body[event.event_collection] = []
|
120
|
-
end
|
121
|
-
|
122
|
-
header = {"timestamp" => event.timestamp}
|
123
|
-
body = event.body
|
124
|
-
item = {"header" => header, "body" => body}
|
125
|
-
request_body[event.event_collection].push(item)
|
126
|
-
}
|
127
|
-
request_body = request_body.to_json
|
128
|
-
|
129
|
-
# build the request:
|
130
|
-
url = "#{base_url}/projects/#{project_id}/_events"
|
131
|
-
uri = URI.parse(url)
|
132
|
-
http = Net::HTTP.new(uri.host, uri.port)
|
133
|
-
http.use_ssl = true
|
134
|
-
http.ca_file = Keen::Async::SSL_CA_FILE
|
135
|
-
http.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
136
|
-
http.verify_depth = 5
|
137
|
-
|
138
|
-
request = Net::HTTP::Post.new(uri.path)
|
139
|
-
request.body = request_body
|
140
|
-
request["Content-Type"] = "application/json"
|
141
|
-
request["Authorization"] = auth_token
|
142
|
-
|
143
|
-
resp = http.request(request)
|
144
|
-
|
145
|
-
if @logging
|
146
|
-
puts resp.body
|
147
|
-
end
|
148
|
-
|
149
|
-
return JSON.parse resp.body
|
150
84
|
end
|
151
85
|
|
152
86
|
def validate_event_collection(event_collection)
|
153
87
|
# TODO
|
154
88
|
end
|
155
|
-
|
156
|
-
def self.create_new_storage_handler(storage_mode, client, logging)
|
157
|
-
# This is shitty as hell. We shoudl take in a class reference pointing to the storage handler, not switch on string values
|
158
|
-
case storage_mode.to_sym
|
159
|
-
|
160
|
-
when :redis
|
161
|
-
Keen::Async::Storage::RedisHandler.new(client, logging)
|
162
|
-
else
|
163
|
-
raise "Unknown storage_mode sent to client: `#{storage_mode}`"
|
164
|
-
end
|
165
|
-
end
|
166
|
-
|
167
89
|
end
|
168
90
|
end
|
data/lib/keen/keys.rb
CHANGED
metadata
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
name: keen
|
3
3
|
version: !ruby/object:Gem::Version
|
4
4
|
prerelease:
|
5
|
-
version: 0.2.
|
5
|
+
version: 0.2.2
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- dorkitude
|
@@ -10,7 +10,7 @@ autorequire:
|
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
12
|
|
13
|
-
date: 2012-10-
|
13
|
+
date: 2012-10-27 00:00:00 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: json
|
@@ -23,20 +23,9 @@ dependencies:
|
|
23
23
|
type: :runtime
|
24
24
|
prerelease: false
|
25
25
|
version_requirements: *id001
|
26
|
-
- !ruby/object:Gem::Dependency
|
27
|
-
name: redis
|
28
|
-
requirement: &id002 !ruby/object:Gem::Requirement
|
29
|
-
none: false
|
30
|
-
requirements:
|
31
|
-
- - ">="
|
32
|
-
- !ruby/object:Gem::Version
|
33
|
-
version: "0"
|
34
|
-
type: :runtime
|
35
|
-
prerelease: false
|
36
|
-
version_requirements: *id002
|
37
26
|
- !ruby/object:Gem::Dependency
|
38
27
|
name: shoulda
|
39
|
-
requirement: &
|
28
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
40
29
|
none: false
|
41
30
|
requirements:
|
42
31
|
- - ">="
|
@@ -44,10 +33,10 @@ dependencies:
|
|
44
33
|
version: "0"
|
45
34
|
type: :development
|
46
35
|
prerelease: false
|
47
|
-
version_requirements: *
|
36
|
+
version_requirements: *id002
|
48
37
|
- !ruby/object:Gem::Dependency
|
49
38
|
name: rdoc
|
50
|
-
requirement: &
|
39
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
51
40
|
none: false
|
52
41
|
requirements:
|
53
42
|
- - ~>
|
@@ -55,10 +44,10 @@ dependencies:
|
|
55
44
|
version: "3.12"
|
56
45
|
type: :development
|
57
46
|
prerelease: false
|
58
|
-
version_requirements: *
|
47
|
+
version_requirements: *id003
|
59
48
|
- !ruby/object:Gem::Dependency
|
60
49
|
name: bundler
|
61
|
-
requirement: &
|
50
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
62
51
|
none: false
|
63
52
|
requirements:
|
64
53
|
- - ">="
|
@@ -66,10 +55,10 @@ dependencies:
|
|
66
55
|
version: 1.1.4
|
67
56
|
type: :development
|
68
57
|
prerelease: false
|
69
|
-
version_requirements: *
|
58
|
+
version_requirements: *id004
|
70
59
|
- !ruby/object:Gem::Dependency
|
71
60
|
name: jeweler
|
72
|
-
requirement: &
|
61
|
+
requirement: &id005 !ruby/object:Gem::Requirement
|
73
62
|
none: false
|
74
63
|
requirements:
|
75
64
|
- - ~>
|
@@ -77,10 +66,10 @@ dependencies:
|
|
77
66
|
version: 1.8.3
|
78
67
|
type: :development
|
79
68
|
prerelease: false
|
80
|
-
version_requirements: *
|
69
|
+
version_requirements: *id005
|
81
70
|
- !ruby/object:Gem::Dependency
|
82
71
|
name: rspec
|
83
|
-
requirement: &
|
72
|
+
requirement: &id006 !ruby/object:Gem::Requirement
|
84
73
|
none: false
|
85
74
|
requirements:
|
86
75
|
- - ">="
|
@@ -88,10 +77,10 @@ dependencies:
|
|
88
77
|
version: 2.9.0
|
89
78
|
type: :development
|
90
79
|
prerelease: false
|
91
|
-
version_requirements: *
|
80
|
+
version_requirements: *id006
|
92
81
|
- !ruby/object:Gem::Dependency
|
93
82
|
name: cucumber
|
94
|
-
requirement: &
|
83
|
+
requirement: &id007 !ruby/object:Gem::Requirement
|
95
84
|
none: false
|
96
85
|
requirements:
|
97
86
|
- - ">="
|
@@ -99,7 +88,7 @@ dependencies:
|
|
99
88
|
version: 1.2.1
|
100
89
|
type: :development
|
101
90
|
prerelease: false
|
102
|
-
version_requirements: *
|
91
|
+
version_requirements: *id007
|
103
92
|
description: See the github repo or examples.rb for usage information.
|
104
93
|
email: kyle@keen.io
|
105
94
|
executables: []
|
@@ -119,25 +108,17 @@ files:
|
|
119
108
|
- conf/cacert.pem
|
120
109
|
- examples.rb
|
121
110
|
- features/add_event.feature
|
122
|
-
- features/redis_queue.feature
|
123
111
|
- features/step_definitions/keen_steps.rb
|
124
112
|
- features/support/before_and_after.rb
|
125
113
|
- features/support/env.rb
|
126
114
|
- keen.gemspec
|
127
115
|
- keen.gemspec.old
|
128
116
|
- lib/keen.rb
|
129
|
-
- lib/keen/async.rb
|
130
|
-
- lib/keen/async/job.rb
|
131
|
-
- lib/keen/async/storage/base_storage_handler.rb
|
132
|
-
- lib/keen/async/storage/flat_file_handler.rb
|
133
|
-
- lib/keen/async/storage/redis_handler.rb
|
134
|
-
- lib/keen/async/worker.rb
|
135
117
|
- lib/keen/client.rb
|
136
118
|
- lib/keen/event.rb
|
137
119
|
- lib/keen/keys.rb
|
138
120
|
- lib/keen/utils.rb
|
139
121
|
- lib/keen/version.rb
|
140
|
-
- send.rb
|
141
122
|
homepage: http://github.com/dorkitude/keen
|
142
123
|
licenses:
|
143
124
|
- MIT
|
@@ -151,7 +132,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
151
132
|
requirements:
|
152
133
|
- - ">="
|
153
134
|
- !ruby/object:Gem::Version
|
154
|
-
hash:
|
135
|
+
hash: -1089062081955943772
|
155
136
|
segments:
|
156
137
|
- 0
|
157
138
|
version: "0"
|
@@ -1,34 +0,0 @@
|
|
1
|
-
# Language: en
|
2
|
-
Feature: RedisHandler
|
3
|
-
In order to make fewer than n HTTP connections for n events
|
4
|
-
As a developer
|
5
|
-
I want to be able to batch up events in Redis and send them to Keen all at once.
|
6
|
-
|
7
|
-
Scenario Outline: Add Events to Redis queue
|
8
|
-
Given a Keen Client using Redis
|
9
|
-
When I post <n> events
|
10
|
-
Then the size of the Redis queue should have gone up by <n>.
|
11
|
-
|
12
|
-
Examples:
|
13
|
-
|n |
|
14
|
-
|1 |
|
15
|
-
|100 |
|
16
|
-
|99 |
|
17
|
-
|1000 |
|
18
|
-
|999 |
|
19
|
-
|
20
|
-
Scenario Outline: Batch a bunch of events in the Redis queue, then send them.
|
21
|
-
Given a Keen Client using Redis
|
22
|
-
When I post <n> events
|
23
|
-
And I process the queue
|
24
|
-
Then the response from Keen should be <n> happy smiles
|
25
|
-
And the queue should be empty.
|
26
|
-
|
27
|
-
Examples:
|
28
|
-
|n |
|
29
|
-
|1 |
|
30
|
-
|100 |
|
31
|
-
|99 |
|
32
|
-
|1000 |
|
33
|
-
|999 |
|
34
|
-
|
data/lib/keen/async.rb
DELETED
data/lib/keen/async/job.rb
DELETED
@@ -1,79 +0,0 @@
|
|
1
|
-
require 'keen/async/storage/redis_handler'
|
2
|
-
|
3
|
-
module Keen
|
4
|
-
module Async
|
5
|
-
class Job
|
6
|
-
# Represents one job.
|
7
|
-
#
|
8
|
-
#
|
9
|
-
|
10
|
-
attr_accessor :project_id, :auth_token, :collection_name, :event_body, :timestamp
|
11
|
-
|
12
|
-
def to_json(options=nil)
|
13
|
-
@definition.to_json
|
14
|
-
end
|
15
|
-
|
16
|
-
|
17
|
-
def to_s
|
18
|
-
self.to_json
|
19
|
-
end
|
20
|
-
|
21
|
-
def initialize(handler, definition)
|
22
|
-
# The `definition` can come from redis, a flat file, or code.
|
23
|
-
load_definition(definition)
|
24
|
-
@handler = handler
|
25
|
-
|
26
|
-
end
|
27
|
-
|
28
|
-
def load_definition(definition)
|
29
|
-
|
30
|
-
definition = Keen::Utils.symbolize_keys(definition)
|
31
|
-
|
32
|
-
# define some key lists:
|
33
|
-
required_keys = [:timestamp, :project_id, :auth_token, :collection_name, :event_body]
|
34
|
-
optional_keys = [:keen_client_version]
|
35
|
-
all_keys = required_keys + optional_keys
|
36
|
-
|
37
|
-
|
38
|
-
# don't allow them to send nil values for anything
|
39
|
-
definition.each do |key, value|
|
40
|
-
# reject unrecognized keys:
|
41
|
-
raise "Unrecognized key: #{key}" unless all_keys.include? key.to_sym
|
42
|
-
end
|
43
|
-
|
44
|
-
|
45
|
-
required_keys.each do |key|
|
46
|
-
|
47
|
-
unless definition.has_key? key
|
48
|
-
raise "You failed to send: #{key} -- you sent #{JSON.generate definition}"
|
49
|
-
end
|
50
|
-
|
51
|
-
value = definition[key]
|
52
|
-
|
53
|
-
raise "You sent a nil value for the #{key}." if value.nil?
|
54
|
-
end
|
55
|
-
|
56
|
-
|
57
|
-
all_keys.each do |key|
|
58
|
-
value = definition[key]
|
59
|
-
self.instance_variable_set("@#{key}", value)
|
60
|
-
end
|
61
|
-
|
62
|
-
unless definition.has_key? :keen_client_version
|
63
|
-
definition[:keen_client_version] = Keen::VERSION
|
64
|
-
end
|
65
|
-
|
66
|
-
@definition = definition
|
67
|
-
|
68
|
-
|
69
|
-
end
|
70
|
-
|
71
|
-
def save
|
72
|
-
@handler.record_job(self)
|
73
|
-
end
|
74
|
-
|
75
|
-
end
|
76
|
-
end
|
77
|
-
end
|
78
|
-
|
79
|
-
|
@@ -1,51 +0,0 @@
|
|
1
|
-
require 'json'
|
2
|
-
require 'time'
|
3
|
-
|
4
|
-
module Keen
|
5
|
-
module Async
|
6
|
-
module Storage
|
7
|
-
class BaseStorageHandler
|
8
|
-
|
9
|
-
def initialize(client)
|
10
|
-
@logging = client.logging
|
11
|
-
@client = client
|
12
|
-
end
|
13
|
-
|
14
|
-
# Key stuff
|
15
|
-
# ----
|
16
|
-
|
17
|
-
def global_key_prefix
|
18
|
-
"keen.#{@client.options[:storage_namespace]}"
|
19
|
-
end
|
20
|
-
|
21
|
-
def active_queue_key
|
22
|
-
"#{global_key_prefix}.active_queue"
|
23
|
-
end
|
24
|
-
|
25
|
-
def failed_queue_key
|
26
|
-
"#{global_key_prefix}.failed_queue"
|
27
|
-
end
|
28
|
-
|
29
|
-
def add_to_active_queue(value)
|
30
|
-
@redis.lpush active_queue_key, value
|
31
|
-
if @logging
|
32
|
-
puts "added #{value} to active queue; length is now #{@redis.llen active_queue_key}"
|
33
|
-
end
|
34
|
-
end
|
35
|
-
|
36
|
-
def record_job(job)
|
37
|
-
add_to_active_queue JSON.generate(job)
|
38
|
-
end
|
39
|
-
|
40
|
-
def handle_prior_failures
|
41
|
-
# TODO consume the failed_queue and do something with it (loggly? retry? flat file?)
|
42
|
-
end
|
43
|
-
|
44
|
-
def count_active_queue
|
45
|
-
raise NotImplementedError
|
46
|
-
end
|
47
|
-
|
48
|
-
end
|
49
|
-
end
|
50
|
-
end
|
51
|
-
end
|
@@ -1,57 +0,0 @@
|
|
1
|
-
require 'fileutils'
|
2
|
-
require 'base64'
|
3
|
-
require 'zlib'
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
module Keen
|
8
|
-
module Async
|
9
|
-
module Storage
|
10
|
-
class FlatFileHandler
|
11
|
-
|
12
|
-
# Paths
|
13
|
-
# -----
|
14
|
-
|
15
|
-
# Where new events go as they come in:
|
16
|
-
ACTIVE = "active.txt"
|
17
|
-
|
18
|
-
# The temporary file that will store records during processing:
|
19
|
-
PROCESSING = "processing.txt"
|
20
|
-
|
21
|
-
# Records are put here if processing fails:
|
22
|
-
FAILED = "failed.txt"
|
23
|
-
|
24
|
-
def initialize(filepath)
|
25
|
-
@filepath = filepath
|
26
|
-
|
27
|
-
is_directory?
|
28
|
-
is_writable?
|
29
|
-
is_readable?
|
30
|
-
end
|
31
|
-
|
32
|
-
def dump_contents
|
33
|
-
# move
|
34
|
-
end
|
35
|
-
|
36
|
-
def is_writable?
|
37
|
-
if not FileTest.writable? @filepath
|
38
|
-
raise "Can't write to file: " + @filepath
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
|
-
def is_readable?
|
43
|
-
if not FileTest.readable? @filepath
|
44
|
-
raise "Can't read from file: " + @filepath
|
45
|
-
end
|
46
|
-
end
|
47
|
-
|
48
|
-
def is_directory?
|
49
|
-
if not FileTest.directory? @filepath
|
50
|
-
raise "Can't read from file: " + @filepath
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
end
|
55
|
-
end
|
56
|
-
end
|
57
|
-
end
|
@@ -1,89 +0,0 @@
|
|
1
|
-
require 'fileutils'
|
2
|
-
require 'redis'
|
3
|
-
require 'json'
|
4
|
-
require 'time'
|
5
|
-
|
6
|
-
module Keen
|
7
|
-
module Async
|
8
|
-
module Storage
|
9
|
-
class RedisHandler < Keen::Async::Storage::BaseStorageHandler
|
10
|
-
|
11
|
-
# Keys
|
12
|
-
# ----
|
13
|
-
|
14
|
-
def add_to_active_queue(value)
|
15
|
-
redis.lpush active_queue_key, value
|
16
|
-
if @logging
|
17
|
-
puts "added #{value} to active queue; length is now #{redis.llen active_queue_key}"
|
18
|
-
end
|
19
|
-
end
|
20
|
-
|
21
|
-
def redis
|
22
|
-
unless @redis
|
23
|
-
@redis = Redis.new
|
24
|
-
end
|
25
|
-
|
26
|
-
@redis
|
27
|
-
end
|
28
|
-
|
29
|
-
def count_active_queue
|
30
|
-
redis.llen active_queue_key
|
31
|
-
end
|
32
|
-
|
33
|
-
def clear_active_queue
|
34
|
-
redis.del active_queue_key
|
35
|
-
end
|
36
|
-
|
37
|
-
def get_authorized_jobs(how_many, client)
|
38
|
-
|
39
|
-
handle_prior_failures
|
40
|
-
|
41
|
-
key = active_queue_key
|
42
|
-
|
43
|
-
job_definitions = []
|
44
|
-
skipped_job_definitions = []
|
45
|
-
|
46
|
-
#puts "doing the job #{how_many} times"
|
47
|
-
|
48
|
-
while true do
|
49
|
-
this = redis.lpop key
|
50
|
-
|
51
|
-
# If we're out of jobs, end the loop:
|
52
|
-
if not this
|
53
|
-
break
|
54
|
-
end
|
55
|
-
|
56
|
-
# Parse the JSON into a job definition
|
57
|
-
job_definition = JSON.parse this
|
58
|
-
job_definition = Keen::Utils.symbolize_keys(job_definition)
|
59
|
-
|
60
|
-
# Make sure this client is authorized to process this job:
|
61
|
-
unless job_definition[:project_id] == client.project_id
|
62
|
-
unless job_definition[:auth_token] == client.auth_token
|
63
|
-
# We're not authorized, so skip this job.
|
64
|
-
skipped_job_definitions.push job_definition
|
65
|
-
next
|
66
|
-
end
|
67
|
-
end
|
68
|
-
|
69
|
-
job_definitions.push job_definition
|
70
|
-
|
71
|
-
if job_definitions.length == how_many
|
72
|
-
break
|
73
|
-
end
|
74
|
-
|
75
|
-
end
|
76
|
-
|
77
|
-
# Put the skipped jobs back on the queue.
|
78
|
-
skipped_job_definitions.each do |job_definition|
|
79
|
-
redis.lpush key, job_definition
|
80
|
-
end
|
81
|
-
|
82
|
-
job_definitions
|
83
|
-
|
84
|
-
end
|
85
|
-
|
86
|
-
end
|
87
|
-
end
|
88
|
-
end
|
89
|
-
end
|
data/lib/keen/async/worker.rb
DELETED
@@ -1,62 +0,0 @@
|
|
1
|
-
require "keen/async/storage/redis_handler"
|
2
|
-
|
3
|
-
|
4
|
-
module Keen
|
5
|
-
|
6
|
-
module Async
|
7
|
-
|
8
|
-
# How many events should we send over the wire at a time?
|
9
|
-
BATCH_SIZE = 100
|
10
|
-
SSL_CA_FILE = File.dirname(__FILE__) + '../../../conf/cacert.pem'
|
11
|
-
|
12
|
-
class Worker
|
13
|
-
|
14
|
-
def initialize(client)
|
15
|
-
@client = client
|
16
|
-
@storage_handler = client.storage_handler
|
17
|
-
end
|
18
|
-
|
19
|
-
def batch_url(project_id)
|
20
|
-
if not project_id
|
21
|
-
raise "Missing project_id."
|
22
|
-
end
|
23
|
-
"https://api.keen.io/2.0/projects/#{project_id}/_events"
|
24
|
-
end
|
25
|
-
|
26
|
-
def process_queue
|
27
|
-
queue_length = @storage_handler.count_active_queue
|
28
|
-
|
29
|
-
batch_size = Keen::Async::BATCH_SIZE
|
30
|
-
|
31
|
-
|
32
|
-
events = []
|
33
|
-
|
34
|
-
|
35
|
-
responses = []
|
36
|
-
|
37
|
-
num_batches = queue_length / batch_size + 1
|
38
|
-
num_batches.times do
|
39
|
-
|
40
|
-
job_definitions = @storage_handler.get_authorized_jobs(batch_size, @client)
|
41
|
-
|
42
|
-
job_definitions.each do |job_definition|
|
43
|
-
#puts JSON.generate job_definition
|
44
|
-
job = Keen::Async::Job.new(@client, job_definition)
|
45
|
-
events.push Keen::Event.new(job.timestamp, job.collection_name, job.event_body)
|
46
|
-
end
|
47
|
-
|
48
|
-
responses.push @client.send_batch(events)
|
49
|
-
end
|
50
|
-
|
51
|
-
if @client.logging
|
52
|
-
puts responses
|
53
|
-
end
|
54
|
-
|
55
|
-
responses
|
56
|
-
|
57
|
-
end
|
58
|
-
|
59
|
-
end
|
60
|
-
end
|
61
|
-
|
62
|
-
end
|
data/send.rb
DELETED
@@ -1,12 +0,0 @@
|
|
1
|
-
require "rubygems"
|
2
|
-
require "keen"
|
3
|
-
|
4
|
-
storage_handler = Keen::Client.create_new_storage_handler(:redis)
|
5
|
-
|
6
|
-
count = storage_handler.count_active_queue
|
7
|
-
puts "we have this many jobs: #{count}"
|
8
|
-
|
9
|
-
worker = Keen::Async::Worker.new(storage_handler)
|
10
|
-
results = worker.process_queue
|
11
|
-
|
12
|
-
puts results
|