keen 0.0.53 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/features/add_event.feature +10 -0
- data/features/redis_queue.feature +34 -0
- data/features/step_definitions/keen_steps.rb +61 -0
- data/features/support/before_and_after.rb +4 -0
- data/keen.gemspec +5 -8
- data/lib/keen.rb +4 -4
- data/lib/keen/async.rb +6 -0
- data/lib/keen/async/job.rb +9 -4
- data/lib/keen/async/storage/base_storage_handler.rb +51 -0
- data/lib/keen/async/storage/redis_handler.rb +42 -104
- data/lib/keen/async/worker.rb +23 -47
- data/lib/keen/client.rb +112 -20
- data/lib/keen/event.rb +13 -0
- data/lib/keen/keys.rb +6 -0
- data/lib/keen/version.rb +1 -1
- data/send.rb +12 -0
- data/test/keen_spec.rb +16 -26
- metadata +27 -31
- data/Gemfile +0 -3
@@ -0,0 +1,10 @@
|
|
1
|
+
# Language: en
|
2
|
+
Feature: AddEvent
|
3
|
+
In order to send an event to Keen's servers
|
4
|
+
As a developer
|
5
|
+
I want to be able to post an event to the Keen Client as a Hash/Dictionary
|
6
|
+
|
7
|
+
Scenario: Send Event directly to Keen
|
8
|
+
Given a Keen Client using Direct
|
9
|
+
When I post an event
|
10
|
+
Then the response from the server should be good.
|
@@ -0,0 +1,34 @@
|
|
1
|
+
# Language: en
|
2
|
+
Feature: RedisHandler
|
3
|
+
In order to make fewer than n HTTP connections for n events
|
4
|
+
As a developer
|
5
|
+
I want to be able to batch up events in Redis and send them to Keen all at once.
|
6
|
+
|
7
|
+
Scenario Outline: Add Events to Redis queue
|
8
|
+
Given a Keen Client using Redis
|
9
|
+
When I post <n> events
|
10
|
+
Then the size of the Redis queue should have gone up by <n>.
|
11
|
+
|
12
|
+
Examples:
|
13
|
+
|n |
|
14
|
+
|1 |
|
15
|
+
|100 |
|
16
|
+
|99 |
|
17
|
+
|1000 |
|
18
|
+
|999 |
|
19
|
+
|
20
|
+
Scenario Outline: Batch a bunch of events in the Redis queue, then send them.
|
21
|
+
Given a Keen Client using Redis
|
22
|
+
When I post <n> events
|
23
|
+
And I process the queue
|
24
|
+
Then the response from Keen should be <n> happy smiles
|
25
|
+
And the queue should be empty.
|
26
|
+
|
27
|
+
Examples:
|
28
|
+
|n |
|
29
|
+
|1 |
|
30
|
+
|100 |
|
31
|
+
|99 |
|
32
|
+
|1000 |
|
33
|
+
|999 |
|
34
|
+
|
@@ -0,0 +1,61 @@
|
|
1
|
+
begin require 'rspec/expectations'; rescue LoadError; require 'spec/expectations'; end
|
2
|
+
require 'cucumber/formatter/unicode'
|
3
|
+
$:.unshift(File.dirname(__FILE__) + '/../../lib')
|
4
|
+
require 'keen'
|
5
|
+
|
6
|
+
Given /^a Keen Client using Redis$/ do
|
7
|
+
@client = Keen::Client.new(@project_id,
|
8
|
+
@auth_token,
|
9
|
+
:storage_class => Keen::Async::Storage::RedisHandler,
|
10
|
+
:storage_namespace => "test",
|
11
|
+
:logging => false )
|
12
|
+
|
13
|
+
@client.storage_handler.clear_active_queue
|
14
|
+
|
15
|
+
@starting_queue_size = @client.storage_handler.count_active_queue
|
16
|
+
end
|
17
|
+
|
18
|
+
Given /^a Keen Client using Direct$/ do
|
19
|
+
@client = Keen::Client.new(@project_id,
|
20
|
+
@auth_token,
|
21
|
+
:cache_locally => false,
|
22
|
+
:logging => false )
|
23
|
+
end
|
24
|
+
|
25
|
+
When /^I post an event$/ do
|
26
|
+
@result = @client.add_event("cucumber_events", {:hi_from => "cucumber!", :keen_version => Keen::VERSION})
|
27
|
+
end
|
28
|
+
|
29
|
+
Then /^the size of the Redis queue should have gone up by (\d+)\.$/ do |n|
|
30
|
+
@client.storage_handler.count_active_queue.should == n.to_i + @starting_queue_size
|
31
|
+
end
|
32
|
+
|
33
|
+
Then /^the response from the server should be good\.$/ do
|
34
|
+
response = @result
|
35
|
+
response.should == {"created" => true}
|
36
|
+
end
|
37
|
+
|
38
|
+
|
39
|
+
When /^I post (\d+) events$/ do |n|
|
40
|
+
n.to_i.times do
|
41
|
+
@client.add_event("cucumber_events", {:hi_from => "cucumber!", :keen_version => Keen::VERSION})
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
When /^I process the queue$/ do
|
46
|
+
worker = Keen::Async::Worker.new(@client)
|
47
|
+
@result = worker.process_queue
|
48
|
+
end
|
49
|
+
|
50
|
+
Then /^the response from Keen should be (\d+) happy smiles$/ do |n|
|
51
|
+
expectation = []
|
52
|
+
n.to_i.times do
|
53
|
+
expectation.push({"success" => true})
|
54
|
+
end
|
55
|
+
|
56
|
+
expectation = {"cucumber_events" => expectation}
|
57
|
+
end
|
58
|
+
|
59
|
+
Then /^the queue should be empty\.$/ do
|
60
|
+
@client.storage_handler.count_active_queue.should == 0
|
61
|
+
end
|
data/keen.gemspec
CHANGED
@@ -23,17 +23,14 @@ Gem::Specification.new do |s|
|
|
23
23
|
# s.add_runtime_dependency "rest-client"
|
24
24
|
|
25
25
|
s.add_dependency('json', '>= 1.6.5')
|
26
|
-
s.add_dependency('fakeweb', '>= 1.3.0')
|
27
26
|
s.add_dependency('rspec', '>= 2.9.0')
|
27
|
+
s.add_dependency('cucumber', '>= 1.2.1')
|
28
|
+
|
29
|
+
# This is no longer necessary, since we support several storage modes now:
|
30
|
+
# s.add_dependency('redis', '>= 2.2.2')
|
31
|
+
|
28
32
|
if RUBY_VERSION < "1.9"
|
29
33
|
s.add_dependency('system_timer', '>= 1.2.4')
|
30
34
|
end
|
31
|
-
s.add_dependency('redis', '>= 2.2.2')
|
32
35
|
|
33
|
-
# took these from Twilio library:
|
34
|
-
# TODO clean this up.
|
35
|
-
#s.add_development_dependency 'rake', '~> 0.9.0'
|
36
|
-
#s.add_development_dependency 'rspec', '~> 2.6.0'
|
37
|
-
#s.add_development_dependency 'fakeweb', '~> 1.3.0'
|
38
|
-
#s.add_development_dependency 'rack', '~> 1.3.0'
|
39
36
|
end
|
data/lib/keen.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
|
+
require 'keen/async'
|
1
2
|
require 'keen/client'
|
2
|
-
require 'keen/
|
3
|
+
require 'keen/event'
|
4
|
+
require 'keen/keys'
|
3
5
|
require 'keen/utils'
|
4
|
-
require 'keen/
|
5
|
-
require 'keen/async/worker'
|
6
|
-
require 'keen/async/storage/redis_handler'
|
6
|
+
require 'keen/version'
|
data/lib/keen/async.rb
ADDED
data/lib/keen/async/job.rb
CHANGED
@@ -7,7 +7,7 @@ module Keen
|
|
7
7
|
#
|
8
8
|
#
|
9
9
|
|
10
|
-
attr_accessor :project_id, :auth_token, :collection_name, :event_body
|
10
|
+
attr_accessor :project_id, :auth_token, :collection_name, :event_body, :timestamp
|
11
11
|
|
12
12
|
def to_json(options=nil)
|
13
13
|
@definition.to_json
|
@@ -18,8 +18,8 @@ module Keen
|
|
18
18
|
self.to_json
|
19
19
|
end
|
20
20
|
|
21
|
-
def initialize(handler, definition
|
22
|
-
|
21
|
+
def initialize(handler, definition)
|
22
|
+
# The `definition` can come from redis, a flat file, or code.
|
23
23
|
load_definition(definition)
|
24
24
|
@handler = handler
|
25
25
|
|
@@ -30,7 +30,7 @@ module Keen
|
|
30
30
|
definition = Keen::Utils.symbolize_keys(definition)
|
31
31
|
|
32
32
|
# define some key lists:
|
33
|
-
required_keys = [:project_id, :auth_token, :collection_name, :event_body]
|
33
|
+
required_keys = [:timestamp, :project_id, :auth_token, :collection_name, :event_body]
|
34
34
|
optional_keys = [:keen_client_version]
|
35
35
|
all_keys = required_keys + optional_keys
|
36
36
|
|
@@ -43,6 +43,11 @@ module Keen
|
|
43
43
|
|
44
44
|
|
45
45
|
required_keys.each do |key|
|
46
|
+
|
47
|
+
unless definition.has_key? key
|
48
|
+
raise "You failed to send: #{key} -- you sent #{JSON.generate definition}"
|
49
|
+
end
|
50
|
+
|
46
51
|
value = definition[key]
|
47
52
|
|
48
53
|
raise "You sent a nil value for the #{key}." if value.nil?
|
@@ -0,0 +1,51 @@
|
|
1
|
+
require 'json'
|
2
|
+
require 'time'
|
3
|
+
|
4
|
+
module Keen
|
5
|
+
module Async
|
6
|
+
module Storage
|
7
|
+
class BaseStorageHandler
|
8
|
+
|
9
|
+
def initialize(client)
|
10
|
+
@logging = client.options[:logging]
|
11
|
+
@client = client
|
12
|
+
end
|
13
|
+
|
14
|
+
# Key stuff
|
15
|
+
# ----
|
16
|
+
|
17
|
+
def global_key_prefix
|
18
|
+
"keen.#{@client.options[:storage_namespace]}"
|
19
|
+
end
|
20
|
+
|
21
|
+
def active_queue_key
|
22
|
+
"#{global_key_prefix}.active_queue"
|
23
|
+
end
|
24
|
+
|
25
|
+
def failed_queue_key
|
26
|
+
"#{global_key_prefix}.failed_queue"
|
27
|
+
end
|
28
|
+
|
29
|
+
def add_to_active_queue(value)
|
30
|
+
@redis.lpush active_queue_key, value
|
31
|
+
if @logging
|
32
|
+
puts "added #{value} to active queue; length is now #{@redis.llen active_queue_key}"
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def record_job(job)
|
37
|
+
add_to_active_queue JSON.generate(job)
|
38
|
+
end
|
39
|
+
|
40
|
+
def handle_prior_failures
|
41
|
+
# TODO consume the failed_queue and do something with it (loggly? retry? flat file?)
|
42
|
+
end
|
43
|
+
|
44
|
+
def count_active_queue
|
45
|
+
raise NotImplementedError
|
46
|
+
end
|
47
|
+
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -6,141 +6,79 @@ require 'time'
|
|
6
6
|
module Keen
|
7
7
|
module Async
|
8
8
|
module Storage
|
9
|
-
class RedisHandler
|
9
|
+
class RedisHandler < Keen::Async::Storage::BaseStorageHandler
|
10
10
|
|
11
11
|
# Keys
|
12
12
|
# ----
|
13
13
|
|
14
|
-
def global_key_prefix
|
15
|
-
"keen"
|
16
|
-
end
|
17
|
-
|
18
|
-
def active_queue_key
|
19
|
-
"#{global_key_prefix}.active_queue_key"
|
20
|
-
end
|
21
|
-
|
22
|
-
def failed_queue_key
|
23
|
-
"#{global_key_prefix}.failed_queue_key"
|
24
|
-
end
|
25
|
-
|
26
14
|
def add_to_active_queue(value)
|
27
|
-
|
15
|
+
redis.lpush active_queue_key, value
|
28
16
|
if @logging
|
29
|
-
puts "added #{value} to active queue; length is now #{
|
17
|
+
puts "added #{value} to active queue; length is now #{redis.llen active_queue_key}"
|
30
18
|
end
|
31
19
|
end
|
32
20
|
|
33
|
-
def
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
21
|
+
def redis
|
22
|
+
unless @redis
|
23
|
+
@redis = Redis.new
|
24
|
+
end
|
25
|
+
|
26
|
+
@redis
|
39
27
|
end
|
40
28
|
|
41
|
-
def
|
42
|
-
|
43
|
-
@logging = logging
|
44
|
-
end
|
45
|
-
|
46
|
-
def redis=(connection)
|
47
|
-
@redis = connection
|
29
|
+
def count_active_queue
|
30
|
+
redis.llen active_queue_key
|
48
31
|
end
|
49
32
|
|
50
|
-
def
|
51
|
-
|
33
|
+
def clear_active_queue
|
34
|
+
redis.del active_queue_key
|
52
35
|
end
|
53
36
|
|
54
|
-
def
|
55
|
-
|
56
|
-
# Returns a hash of the next `how_many` jobs, indexed on project_id and then collection_name.
|
57
|
-
#
|
58
|
-
# It looks like this:
|
59
|
-
#
|
60
|
-
# collated = {
|
61
|
-
# "4f5775ad163d666a6100000e" => {
|
62
|
-
# "clicks" => [
|
63
|
-
# Keen::Storage::Job.new({
|
64
|
-
# :project_id => "4f5775ad163d666a6100000e",
|
65
|
-
# :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
|
66
|
-
# :collection_name => "clicks",
|
67
|
-
# :event_body => {:user_id => "12345"},
|
68
|
-
# }),
|
69
|
-
# Keen::Storage::Job.new({
|
70
|
-
# :project_id => "4f5775ad163d666a6100000e",
|
71
|
-
# :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
|
72
|
-
# :collection_name => "clicks",
|
73
|
-
# :event_body => {:user_id => "12345"},
|
74
|
-
# }),
|
75
|
-
# Keen::Storage::Job.new({
|
76
|
-
# :project_id => "4f5775ad163d666a6100000e",
|
77
|
-
# :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
|
78
|
-
# :collection_name => "clicks",
|
79
|
-
# :event_body => {:user_id => "12345"},
|
80
|
-
# }),
|
81
|
-
# ],
|
82
|
-
# "purchases" => [
|
83
|
-
# Keen::Storage::Job.new({
|
84
|
-
# :project_id => "4f5775ad163d666a6100000e",
|
85
|
-
# :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
|
86
|
-
# :collection_name => "purchases",
|
87
|
-
# :event_body => {:user_id => "12345"},
|
88
|
-
# }),
|
89
|
-
# Keen::Storage::Job.new({
|
90
|
-
# :project_id => "4f5775ad163d666a6100000e",
|
91
|
-
# :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
|
92
|
-
# :collection_name => "purchases",
|
93
|
-
# :event_body => {:user_id => "12345"},
|
94
|
-
# }),
|
95
|
-
# Keen::Storage::Job.new({
|
96
|
-
# :project_id => "4f5775ad163d666a6100000e",
|
97
|
-
# :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
|
98
|
-
# :collection_name => "purchases",
|
99
|
-
# :event_body => {:user_id => "12345"},
|
100
|
-
# }),
|
101
|
-
# ],
|
102
|
-
# }
|
103
|
-
# }
|
37
|
+
def get_authorized_jobs(how_many, client)
|
104
38
|
|
105
39
|
handle_prior_failures
|
106
40
|
|
107
41
|
key = active_queue_key
|
108
42
|
|
109
|
-
|
43
|
+
job_definitions = []
|
44
|
+
skipped_job_definitions = []
|
110
45
|
|
111
46
|
#puts "doing the job #{how_many} times"
|
112
47
|
|
113
|
-
|
114
|
-
this =
|
115
|
-
if this
|
116
|
-
jobs.push JSON.parse this
|
117
|
-
else
|
118
|
-
#puts "couldn't process value #{this}"
|
119
|
-
end
|
120
|
-
end
|
121
|
-
|
122
|
-
collate_jobs(jobs)
|
123
|
-
end
|
124
|
-
|
125
|
-
def collate_jobs(queue)
|
126
|
-
collated = {}
|
48
|
+
while true do
|
49
|
+
this = redis.lpop key
|
127
50
|
|
128
|
-
|
51
|
+
# If we're out of jobs, end the loop:
|
52
|
+
if not this
|
53
|
+
break
|
54
|
+
end
|
55
|
+
|
56
|
+
# Parse the JSON into a job definition
|
57
|
+
job_definition = JSON.parse this
|
58
|
+
job_definition = Keen::Utils.symbolize_keys(job_definition)
|
59
|
+
|
60
|
+
# Make sure this client is authorized to process this job:
|
61
|
+
unless job_definition[:project_id] == client.project_id
|
62
|
+
unless job_definition[:auth_token] == client.auth_token
|
63
|
+
# We're not authorized, so skip this job.
|
64
|
+
skipped_job_definitions.push job_definition
|
65
|
+
next
|
66
|
+
end
|
67
|
+
end
|
129
68
|
|
130
|
-
|
69
|
+
job_definitions.push job_definition
|
131
70
|
|
132
|
-
if
|
133
|
-
|
71
|
+
if job_definitions.length == how_many
|
72
|
+
break
|
134
73
|
end
|
135
74
|
|
136
|
-
|
137
|
-
collated[job.project_id][job.collection_name] = []
|
138
|
-
end
|
75
|
+
end
|
139
76
|
|
140
|
-
|
77
|
+
# Put the skipped jobs back on the queue.
|
78
|
+
skipped_job_definitions.each do |job_definition|
|
79
|
+
redis.lpush key, job_definition
|
141
80
|
end
|
142
81
|
|
143
|
-
collated
|
144
82
|
end
|
145
83
|
|
146
84
|
end
|
data/lib/keen/async/worker.rb
CHANGED
@@ -1,7 +1,4 @@
|
|
1
1
|
require "keen/async/storage/redis_handler"
|
2
|
-
require "net/http"
|
3
|
-
require "net/https"
|
4
|
-
|
5
2
|
|
6
3
|
|
7
4
|
module Keen
|
@@ -14,67 +11,46 @@ module Keen
|
|
14
11
|
|
15
12
|
class Worker
|
16
13
|
|
17
|
-
def initialize(
|
18
|
-
@
|
14
|
+
def initialize(client)
|
15
|
+
@client = client
|
16
|
+
@storage_handler = client.storage_handler
|
19
17
|
end
|
20
18
|
|
21
19
|
def batch_url(project_id)
|
22
20
|
if not project_id
|
23
21
|
raise "Missing project_id."
|
24
22
|
end
|
25
|
-
"https://api.keen.io/
|
23
|
+
"https://api.keen.io/2.0/projects/#{project_id}/_events"
|
26
24
|
end
|
27
25
|
|
28
26
|
def process_queue
|
29
|
-
|
30
|
-
queue_length = @handler.count_active_queue
|
27
|
+
queue_length = @storage_handler.count_active_queue
|
31
28
|
|
32
29
|
batch_size = Keen::Async::BATCH_SIZE
|
33
30
|
|
34
|
-
num_batches = 1 + queue_length / batch_size
|
35
31
|
|
32
|
+
events = []
|
33
|
+
|
34
|
+
|
35
|
+
responses = []
|
36
|
+
|
37
|
+
num_batches = queue_length / batch_size + 1
|
36
38
|
num_batches.times do
|
37
|
-
|
38
|
-
|
39
|
-
|
39
|
+
|
40
|
+
job_definitions = @storage_handler.get_authorized_jobs(batch_size, @client)
|
41
|
+
|
42
|
+
job_definitions.each do |job_definition|
|
43
|
+
#puts JSON.generate job_definition
|
44
|
+
job = Keen::Async::Job.new(@client, job_definition)
|
45
|
+
events.push Keen::Event.new(job.timestamp, job.collection_name, job.event_body)
|
40
46
|
end
|
41
|
-
end
|
42
47
|
|
43
|
-
|
44
|
-
end
|
45
|
-
|
46
|
-
def send_batch(project_id, batch)
|
47
|
-
if not batch
|
48
|
-
return
|
48
|
+
responses.push @client.send_batch(events)
|
49
49
|
end
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
uri = URI.parse(batch_url(project_id))
|
56
|
-
http = Net::HTTP.new(uri.host, uri.port)
|
57
|
-
http.use_ssl = true
|
58
|
-
http.ca_file = Keen::Async::SSL_CA_FILE
|
59
|
-
http.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
60
|
-
http.verify_depth = 5
|
61
|
-
|
62
|
-
request = Net::HTTP::Post.new(uri.path)
|
63
|
-
request.body = batch.to_json
|
64
|
-
request["Content-Type"] = "application/json"
|
65
|
-
request["Authorization"] = auth_token
|
66
|
-
|
67
|
-
response = http.request(request)
|
68
|
-
|
69
|
-
#response = Net::HTTP.start(uri.host, uri.port) {|http|
|
70
|
-
#http.request(request)
|
71
|
-
#}
|
72
|
-
|
73
|
-
puts response
|
74
|
-
# TODO: If something fails, we should move the job to the
|
75
|
-
# prior_failures queue by calling, perhaps:
|
76
|
-
#
|
77
|
-
# @handler.log_failed_job(job)
|
50
|
+
|
51
|
+
|
52
|
+
responses
|
53
|
+
|
78
54
|
end
|
79
55
|
|
80
56
|
end
|
data/lib/keen/client.rb
CHANGED
@@ -1,15 +1,23 @@
|
|
1
|
-
require "keen/async/storage/redis_handler"
|
2
|
-
require "keen/async/job"
|
3
1
|
require "json"
|
4
2
|
require "uri"
|
5
3
|
require "time"
|
4
|
+
require "net/http"
|
5
|
+
require "net/https"
|
6
6
|
|
7
7
|
|
8
8
|
module Keen
|
9
9
|
|
10
10
|
class Client
|
11
11
|
|
12
|
-
attr_accessor :storage_handler, :project_id, :auth_token
|
12
|
+
attr_accessor :storage_handler, :project_id, :auth_token, :options
|
13
|
+
|
14
|
+
def base_url
|
15
|
+
if @options[:base_url]
|
16
|
+
@options[:base_url]
|
17
|
+
else
|
18
|
+
"https://api.keen.io/2.0"
|
19
|
+
end
|
20
|
+
end
|
13
21
|
|
14
22
|
def initialize(project_id, auth_token, options = {})
|
15
23
|
|
@@ -17,20 +25,39 @@ module Keen
|
|
17
25
|
raise "auth_token must be string" unless auth_token.kind_of? String
|
18
26
|
|
19
27
|
default_options = {
|
20
|
-
:
|
28
|
+
:logging => true,
|
29
|
+
|
30
|
+
# warning! not caching locally leads to bad performance:
|
31
|
+
:cache_locally => true,
|
32
|
+
|
33
|
+
# this is required if cache_locally is true:
|
34
|
+
:storage_class => nil,
|
35
|
+
|
36
|
+
# all keys will be prepended with this:
|
37
|
+
:storage_namespace => "default",
|
21
38
|
}
|
22
|
-
|
39
|
+
|
23
40
|
options = default_options.update(options)
|
24
41
|
|
25
42
|
@project_id = project_id
|
26
43
|
@auth_token = auth_token
|
27
|
-
@
|
28
|
-
end
|
44
|
+
@cache_locally = options[:cache_locally]
|
29
45
|
|
30
|
-
|
46
|
+
if @cache_locally
|
47
|
+
@storage_class = options[:storage_class]
|
48
|
+
unless @storage_class and @storage_class < Keen::Async::Storage::BaseStorageHandler
|
49
|
+
raise "The Storage Class you send must extend BaseStorageHandler. You sent: #{@storage_class}"
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
@logging = options[:logging]
|
54
|
+
@options = options
|
31
55
|
|
56
|
+
end
|
57
|
+
|
58
|
+
def storage_handler
|
32
59
|
unless @storage_handler
|
33
|
-
@storage_handler =
|
60
|
+
@storage_handler = @storage_class.new(self)
|
34
61
|
end
|
35
62
|
|
36
63
|
@storage_handler
|
@@ -44,33 +71,98 @@ module Keen
|
|
44
71
|
#
|
45
72
|
# `timestamp` is optional. If sent, it should be a Time instance.
|
46
73
|
# If it's not sent, we'll use the current time.
|
47
|
-
|
74
|
+
|
48
75
|
validate_collection_name(collection_name)
|
49
76
|
|
50
77
|
unless timestamp
|
51
78
|
timestamp = Time.now
|
52
79
|
end
|
53
80
|
|
54
|
-
|
81
|
+
timestamp = timestamp.utc.iso8601
|
55
82
|
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
83
|
+
event = Keen::Event.new(timestamp, collection_name, event_body)
|
84
|
+
|
85
|
+
if @cache_locally
|
86
|
+
job = Keen::Async::Job.new(storage_handler, {
|
87
|
+
:timestamp => event.timestamp,
|
88
|
+
:project_id => @project_id,
|
89
|
+
:auth_token => @auth_token,
|
90
|
+
:collection_name => collection_name,
|
91
|
+
:event_body => event.body,
|
92
|
+
})
|
93
|
+
|
94
|
+
job.save
|
95
|
+
else
|
96
|
+
# build the request:
|
97
|
+
url = "#{base_url}/projects/#{project_id}/#{collection_name}"
|
98
|
+
uri = URI.parse(url)
|
99
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
100
|
+
http.use_ssl = true
|
101
|
+
http.ca_file = Keen::Async::SSL_CA_FILE
|
102
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
103
|
+
http.verify_depth = 5
|
104
|
+
|
105
|
+
request = Net::HTTP::Post.new(uri.path)
|
106
|
+
request.body = JSON.generate({
|
107
|
+
:header => {
|
108
|
+
:timestamp => event.timestamp,
|
109
|
+
},
|
110
|
+
:body => event.body
|
111
|
+
})
|
112
|
+
|
113
|
+
request["Content-Type"] = "application/json"
|
114
|
+
request["Authorization"] = @auth_token
|
115
|
+
|
116
|
+
response = http.request(request)
|
117
|
+
JSON.parse response.body
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
def send_batch(events)
|
122
|
+
# make the request body:
|
123
|
+
request_body = {}
|
124
|
+
events.each { |event|
|
125
|
+
unless request_body.has_key? event.collection_name
|
126
|
+
request_body[event.collection_name] = []
|
127
|
+
end
|
128
|
+
|
129
|
+
header = {"timestamp" => event.timestamp}
|
130
|
+
body = event.body
|
131
|
+
item = {"header" => header, "body" => body}
|
132
|
+
request_body[event.collection_name].push(item)
|
133
|
+
}
|
134
|
+
request_body = request_body.to_json
|
135
|
+
|
136
|
+
# build the request:
|
137
|
+
url = "#{base_url}/projects/#{project_id}/_events"
|
138
|
+
uri = URI.parse(url)
|
139
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
140
|
+
http.use_ssl = true
|
141
|
+
http.ca_file = Keen::Async::SSL_CA_FILE
|
142
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
143
|
+
http.verify_depth = 5
|
144
|
+
|
145
|
+
request = Net::HTTP::Post.new(uri.path)
|
146
|
+
request.body = request_body
|
147
|
+
request["Content-Type"] = "application/json"
|
148
|
+
request["Authorization"] = auth_token
|
149
|
+
|
150
|
+
resp = http.request(request)
|
151
|
+
|
152
|
+
return JSON.parse resp.body
|
62
153
|
|
63
|
-
job.save
|
64
154
|
end
|
65
155
|
|
66
156
|
def validate_collection_name(collection_name)
|
67
157
|
# TODO
|
68
158
|
end
|
69
159
|
|
70
|
-
def self.create_new_storage_handler(storage_mode)
|
160
|
+
def self.create_new_storage_handler(storage_mode, client, logging)
|
161
|
+
# This is shitty as hell. We shoudl take in a class reference pointing to the storage handler, not switch on string values
|
71
162
|
case storage_mode.to_sym
|
163
|
+
|
72
164
|
when :redis
|
73
|
-
Keen::Async::Storage::RedisHandler.new
|
165
|
+
Keen::Async::Storage::RedisHandler.new(client, logging)
|
74
166
|
else
|
75
167
|
raise "Unknown storage_mode sent to client: `#{storage_mode}`"
|
76
168
|
end
|
data/lib/keen/event.rb
ADDED
data/lib/keen/keys.rb
ADDED
data/lib/keen/version.rb
CHANGED
data/send.rb
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
require "rubygems"
|
2
|
+
require "keen"
|
3
|
+
|
4
|
+
storage_handler = Keen::Client.create_new_storage_handler(:redis)
|
5
|
+
|
6
|
+
count = storage_handler.count_active_queue
|
7
|
+
puts "we have this many jobs: #{count}"
|
8
|
+
|
9
|
+
worker = Keen::Async::Worker.new(storage_handler)
|
10
|
+
results = worker.process_queue
|
11
|
+
|
12
|
+
puts results
|
data/test/keen_spec.rb
CHANGED
@@ -1,44 +1,34 @@
|
|
1
1
|
$LOAD_PATH << File.join(File.dirname(__FILE__), "..", "lib")
|
2
2
|
|
3
3
|
require "keen"
|
4
|
-
require "fakeweb"
|
5
4
|
|
6
5
|
describe Keen::Client do
|
7
6
|
|
8
|
-
#
|
9
|
-
before :all do
|
10
|
-
FakeWeb.register_uri(:any, %r/https:\/\/api.keen.io\//, :body => '{"message": "You tried to reach Keen"}')
|
11
|
-
end
|
12
|
-
|
7
|
+
# The add_event method should add stuff to Redis:
|
13
8
|
describe "#add_event" do
|
9
|
+
|
10
|
+
# set up the Keen Client instance:
|
14
11
|
project_id = "4f5775ad163d666a6100000e"
|
15
12
|
auth_token = "a5d4eaf432914823a94ecd7e0cb547b9"
|
16
13
|
|
17
|
-
|
14
|
+
# Make a client:
|
15
|
+
client = Keen::Client.new(project_id,
|
16
|
+
auth_token,
|
17
|
+
:storage_class => Keen::Async::Storage::RedisHandler,
|
18
|
+
:logging => true )
|
18
19
|
|
19
|
-
|
20
|
-
|
20
|
+
# Flush the queue first:
|
21
|
+
client.clear_active_queue()
|
22
|
+
|
23
|
+
# Send some events to the client, which should persist them in Redis
|
24
|
+
5.times do
|
25
|
+
client.add_event("rspec_clicks", {
|
21
26
|
:hi => "you",
|
22
27
|
})
|
23
28
|
end
|
24
29
|
|
25
|
-
|
26
|
-
|
27
|
-
worker.process_queue
|
30
|
+
# Make sure we have the right number of things in the queue:
|
31
|
+
client.storage_handler.count_active_queue.should == 5
|
28
32
|
|
29
33
|
end
|
30
|
-
|
31
|
-
# TODO spec it out, lazy bones!
|
32
|
-
#
|
33
|
-
#
|
34
|
-
# Each time an event is logged, we'll store a json-serialized, base64-
|
35
|
-
# encoded, and gzipped hash that loos like this:
|
36
|
-
#
|
37
|
-
# {
|
38
|
-
# :project_id => "alsdjfaldskfjadskladsklj",
|
39
|
-
# :auth_token => "aslkjflk3wjfaklsjdflkasdjflkadjflakdj211",
|
40
|
-
# :collection_name => "purchases",
|
41
|
-
# :event_body => {:prop1 => "val1", :prop2 => "val2"},
|
42
|
-
# }
|
43
|
-
|
44
34
|
end
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: keen
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 27
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 0
|
8
|
+
- 1
|
8
9
|
- 0
|
9
|
-
|
10
|
-
version: 0.0.53
|
10
|
+
version: 0.1.0
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Kyle Wild
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2012-06-
|
18
|
+
date: 2012-06-17 00:00:00 -07:00
|
19
19
|
default_executable:
|
20
20
|
dependencies:
|
21
21
|
- !ruby/object:Gem::Dependency
|
@@ -35,35 +35,35 @@ dependencies:
|
|
35
35
|
type: :runtime
|
36
36
|
version_requirements: *id001
|
37
37
|
- !ruby/object:Gem::Dependency
|
38
|
-
name:
|
38
|
+
name: rspec
|
39
39
|
prerelease: false
|
40
40
|
requirement: &id002 !ruby/object:Gem::Requirement
|
41
41
|
none: false
|
42
42
|
requirements:
|
43
43
|
- - ">="
|
44
44
|
- !ruby/object:Gem::Version
|
45
|
-
hash:
|
45
|
+
hash: 43
|
46
46
|
segments:
|
47
|
-
-
|
48
|
-
-
|
47
|
+
- 2
|
48
|
+
- 9
|
49
49
|
- 0
|
50
|
-
version:
|
50
|
+
version: 2.9.0
|
51
51
|
type: :runtime
|
52
52
|
version_requirements: *id002
|
53
53
|
- !ruby/object:Gem::Dependency
|
54
|
-
name:
|
54
|
+
name: cucumber
|
55
55
|
prerelease: false
|
56
56
|
requirement: &id003 !ruby/object:Gem::Requirement
|
57
57
|
none: false
|
58
58
|
requirements:
|
59
59
|
- - ">="
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
hash:
|
61
|
+
hash: 29
|
62
62
|
segments:
|
63
|
+
- 1
|
63
64
|
- 2
|
64
|
-
-
|
65
|
-
|
66
|
-
version: 2.9.0
|
65
|
+
- 1
|
66
|
+
version: 1.2.1
|
67
67
|
type: :runtime
|
68
68
|
version_requirements: *id003
|
69
69
|
- !ruby/object:Gem::Dependency
|
@@ -82,22 +82,6 @@ dependencies:
|
|
82
82
|
version: 1.2.4
|
83
83
|
type: :runtime
|
84
84
|
version_requirements: *id004
|
85
|
-
- !ruby/object:Gem::Dependency
|
86
|
-
name: redis
|
87
|
-
prerelease: false
|
88
|
-
requirement: &id005 !ruby/object:Gem::Requirement
|
89
|
-
none: false
|
90
|
-
requirements:
|
91
|
-
- - ">="
|
92
|
-
- !ruby/object:Gem::Version
|
93
|
-
hash: 3
|
94
|
-
segments:
|
95
|
-
- 2
|
96
|
-
- 2
|
97
|
-
- 2
|
98
|
-
version: 2.2.2
|
99
|
-
type: :runtime
|
100
|
-
version_requirements: *id005
|
101
85
|
description: See the github repo or examples.rb for usage information.
|
102
86
|
email:
|
103
87
|
- kyle@keen.io
|
@@ -110,7 +94,6 @@ extra_rdoc_files: []
|
|
110
94
|
files:
|
111
95
|
- .gitignore
|
112
96
|
- .rvmrc
|
113
|
-
- Gemfile
|
114
97
|
- README.md
|
115
98
|
- bin/keen_send
|
116
99
|
- conf/cacert.pem
|
@@ -189,15 +172,24 @@ files:
|
|
189
172
|
- examples/rails_2/CoolForums/test/unit/helpers/users_helper_test.rb
|
190
173
|
- examples/rails_2/CoolForums/test/unit/user_test.rb
|
191
174
|
- examples/rails_2/Gemfile
|
175
|
+
- features/add_event.feature
|
176
|
+
- features/redis_queue.feature
|
177
|
+
- features/step_definitions/keen_steps.rb
|
178
|
+
- features/support/before_and_after.rb
|
192
179
|
- keen.gemspec
|
193
180
|
- lib/keen.rb
|
181
|
+
- lib/keen/async.rb
|
194
182
|
- lib/keen/async/job.rb
|
183
|
+
- lib/keen/async/storage/base_storage_handler.rb
|
195
184
|
- lib/keen/async/storage/flat_file_handler.rb
|
196
185
|
- lib/keen/async/storage/redis_handler.rb
|
197
186
|
- lib/keen/async/worker.rb
|
198
187
|
- lib/keen/client.rb
|
188
|
+
- lib/keen/event.rb
|
189
|
+
- lib/keen/keys.rb
|
199
190
|
- lib/keen/utils.rb
|
200
191
|
- lib/keen/version.rb
|
192
|
+
- send.rb
|
201
193
|
- test/keen_spec.rb
|
202
194
|
has_rdoc: true
|
203
195
|
homepage: https://github.com/keenlabs/KeenClient-Ruby
|
@@ -234,4 +226,8 @@ signing_key:
|
|
234
226
|
specification_version: 3
|
235
227
|
summary: A library for sending events to the keen.io API.
|
236
228
|
test_files:
|
229
|
+
- features/add_event.feature
|
230
|
+
- features/redis_queue.feature
|
231
|
+
- features/step_definitions/keen_steps.rb
|
232
|
+
- features/support/before_and_after.rb
|
237
233
|
- test/keen_spec.rb
|
data/Gemfile
DELETED