keen 0.0.53 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/features/add_event.feature +10 -0
- data/features/redis_queue.feature +34 -0
- data/features/step_definitions/keen_steps.rb +61 -0
- data/features/support/before_and_after.rb +4 -0
- data/keen.gemspec +5 -8
- data/lib/keen.rb +4 -4
- data/lib/keen/async.rb +6 -0
- data/lib/keen/async/job.rb +9 -4
- data/lib/keen/async/storage/base_storage_handler.rb +51 -0
- data/lib/keen/async/storage/redis_handler.rb +42 -104
- data/lib/keen/async/worker.rb +23 -47
- data/lib/keen/client.rb +112 -20
- data/lib/keen/event.rb +13 -0
- data/lib/keen/keys.rb +6 -0
- data/lib/keen/version.rb +1 -1
- data/send.rb +12 -0
- data/test/keen_spec.rb +16 -26
- metadata +27 -31
- data/Gemfile +0 -3
@@ -0,0 +1,10 @@
|
|
1
|
+
# Language: en
|
2
|
+
Feature: AddEvent
|
3
|
+
In order to send an event to Keen's servers
|
4
|
+
As a developer
|
5
|
+
I want to be able to post an event to the Keen Client as a Hash/Dictionary
|
6
|
+
|
7
|
+
Scenario: Send Event directly to Keen
|
8
|
+
Given a Keen Client using Direct
|
9
|
+
When I post an event
|
10
|
+
Then the response from the server should be good.
|
@@ -0,0 +1,34 @@
|
|
1
|
+
# Language: en
|
2
|
+
Feature: RedisHandler
|
3
|
+
In order to make fewer than n HTTP connections for n events
|
4
|
+
As a developer
|
5
|
+
I want to be able to batch up events in Redis and send them to Keen all at once.
|
6
|
+
|
7
|
+
Scenario Outline: Add Events to Redis queue
|
8
|
+
Given a Keen Client using Redis
|
9
|
+
When I post <n> events
|
10
|
+
Then the size of the Redis queue should have gone up by <n>.
|
11
|
+
|
12
|
+
Examples:
|
13
|
+
|n |
|
14
|
+
|1 |
|
15
|
+
|100 |
|
16
|
+
|99 |
|
17
|
+
|1000 |
|
18
|
+
|999 |
|
19
|
+
|
20
|
+
Scenario Outline: Batch a bunch of events in the Redis queue, then send them.
|
21
|
+
Given a Keen Client using Redis
|
22
|
+
When I post <n> events
|
23
|
+
And I process the queue
|
24
|
+
Then the response from Keen should be <n> happy smiles
|
25
|
+
And the queue should be empty.
|
26
|
+
|
27
|
+
Examples:
|
28
|
+
|n |
|
29
|
+
|1 |
|
30
|
+
|100 |
|
31
|
+
|99 |
|
32
|
+
|1000 |
|
33
|
+
|999 |
|
34
|
+
|
@@ -0,0 +1,61 @@
|
|
1
|
+
begin require 'rspec/expectations'; rescue LoadError; require 'spec/expectations'; end
|
2
|
+
require 'cucumber/formatter/unicode'
|
3
|
+
$:.unshift(File.dirname(__FILE__) + '/../../lib')
|
4
|
+
require 'keen'
|
5
|
+
|
6
|
+
Given /^a Keen Client using Redis$/ do
|
7
|
+
@client = Keen::Client.new(@project_id,
|
8
|
+
@auth_token,
|
9
|
+
:storage_class => Keen::Async::Storage::RedisHandler,
|
10
|
+
:storage_namespace => "test",
|
11
|
+
:logging => false )
|
12
|
+
|
13
|
+
@client.storage_handler.clear_active_queue
|
14
|
+
|
15
|
+
@starting_queue_size = @client.storage_handler.count_active_queue
|
16
|
+
end
|
17
|
+
|
18
|
+
Given /^a Keen Client using Direct$/ do
|
19
|
+
@client = Keen::Client.new(@project_id,
|
20
|
+
@auth_token,
|
21
|
+
:cache_locally => false,
|
22
|
+
:logging => false )
|
23
|
+
end
|
24
|
+
|
25
|
+
When /^I post an event$/ do
|
26
|
+
@result = @client.add_event("cucumber_events", {:hi_from => "cucumber!", :keen_version => Keen::VERSION})
|
27
|
+
end
|
28
|
+
|
29
|
+
Then /^the size of the Redis queue should have gone up by (\d+)\.$/ do |n|
|
30
|
+
@client.storage_handler.count_active_queue.should == n.to_i + @starting_queue_size
|
31
|
+
end
|
32
|
+
|
33
|
+
Then /^the response from the server should be good\.$/ do
|
34
|
+
response = @result
|
35
|
+
response.should == {"created" => true}
|
36
|
+
end
|
37
|
+
|
38
|
+
|
39
|
+
When /^I post (\d+) events$/ do |n|
|
40
|
+
n.to_i.times do
|
41
|
+
@client.add_event("cucumber_events", {:hi_from => "cucumber!", :keen_version => Keen::VERSION})
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
When /^I process the queue$/ do
|
46
|
+
worker = Keen::Async::Worker.new(@client)
|
47
|
+
@result = worker.process_queue
|
48
|
+
end
|
49
|
+
|
50
|
+
Then /^the response from Keen should be (\d+) happy smiles$/ do |n|
|
51
|
+
expectation = []
|
52
|
+
n.to_i.times do
|
53
|
+
expectation.push({"success" => true})
|
54
|
+
end
|
55
|
+
|
56
|
+
expectation = {"cucumber_events" => expectation}
|
57
|
+
end
|
58
|
+
|
59
|
+
Then /^the queue should be empty\.$/ do
|
60
|
+
@client.storage_handler.count_active_queue.should == 0
|
61
|
+
end
|
data/keen.gemspec
CHANGED
@@ -23,17 +23,14 @@ Gem::Specification.new do |s|
|
|
23
23
|
# s.add_runtime_dependency "rest-client"
|
24
24
|
|
25
25
|
s.add_dependency('json', '>= 1.6.5')
|
26
|
-
s.add_dependency('fakeweb', '>= 1.3.0')
|
27
26
|
s.add_dependency('rspec', '>= 2.9.0')
|
27
|
+
s.add_dependency('cucumber', '>= 1.2.1')
|
28
|
+
|
29
|
+
# This is no longer necessary, since we support several storage modes now:
|
30
|
+
# s.add_dependency('redis', '>= 2.2.2')
|
31
|
+
|
28
32
|
if RUBY_VERSION < "1.9"
|
29
33
|
s.add_dependency('system_timer', '>= 1.2.4')
|
30
34
|
end
|
31
|
-
s.add_dependency('redis', '>= 2.2.2')
|
32
35
|
|
33
|
-
# took these from Twilio library:
|
34
|
-
# TODO clean this up.
|
35
|
-
#s.add_development_dependency 'rake', '~> 0.9.0'
|
36
|
-
#s.add_development_dependency 'rspec', '~> 2.6.0'
|
37
|
-
#s.add_development_dependency 'fakeweb', '~> 1.3.0'
|
38
|
-
#s.add_development_dependency 'rack', '~> 1.3.0'
|
39
36
|
end
|
data/lib/keen.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
|
+
require 'keen/async'
|
1
2
|
require 'keen/client'
|
2
|
-
require 'keen/
|
3
|
+
require 'keen/event'
|
4
|
+
require 'keen/keys'
|
3
5
|
require 'keen/utils'
|
4
|
-
require 'keen/
|
5
|
-
require 'keen/async/worker'
|
6
|
-
require 'keen/async/storage/redis_handler'
|
6
|
+
require 'keen/version'
|
data/lib/keen/async.rb
ADDED
data/lib/keen/async/job.rb
CHANGED
@@ -7,7 +7,7 @@ module Keen
|
|
7
7
|
#
|
8
8
|
#
|
9
9
|
|
10
|
-
attr_accessor :project_id, :auth_token, :collection_name, :event_body
|
10
|
+
attr_accessor :project_id, :auth_token, :collection_name, :event_body, :timestamp
|
11
11
|
|
12
12
|
def to_json(options=nil)
|
13
13
|
@definition.to_json
|
@@ -18,8 +18,8 @@ module Keen
|
|
18
18
|
self.to_json
|
19
19
|
end
|
20
20
|
|
21
|
-
def initialize(handler, definition
|
22
|
-
|
21
|
+
def initialize(handler, definition)
|
22
|
+
# The `definition` can come from redis, a flat file, or code.
|
23
23
|
load_definition(definition)
|
24
24
|
@handler = handler
|
25
25
|
|
@@ -30,7 +30,7 @@ module Keen
|
|
30
30
|
definition = Keen::Utils.symbolize_keys(definition)
|
31
31
|
|
32
32
|
# define some key lists:
|
33
|
-
required_keys = [:project_id, :auth_token, :collection_name, :event_body]
|
33
|
+
required_keys = [:timestamp, :project_id, :auth_token, :collection_name, :event_body]
|
34
34
|
optional_keys = [:keen_client_version]
|
35
35
|
all_keys = required_keys + optional_keys
|
36
36
|
|
@@ -43,6 +43,11 @@ module Keen
|
|
43
43
|
|
44
44
|
|
45
45
|
required_keys.each do |key|
|
46
|
+
|
47
|
+
unless definition.has_key? key
|
48
|
+
raise "You failed to send: #{key} -- you sent #{JSON.generate definition}"
|
49
|
+
end
|
50
|
+
|
46
51
|
value = definition[key]
|
47
52
|
|
48
53
|
raise "You sent a nil value for the #{key}." if value.nil?
|
@@ -0,0 +1,51 @@
|
|
1
|
+
require 'json'
|
2
|
+
require 'time'
|
3
|
+
|
4
|
+
module Keen
|
5
|
+
module Async
|
6
|
+
module Storage
|
7
|
+
class BaseStorageHandler
|
8
|
+
|
9
|
+
def initialize(client)
|
10
|
+
@logging = client.options[:logging]
|
11
|
+
@client = client
|
12
|
+
end
|
13
|
+
|
14
|
+
# Key stuff
|
15
|
+
# ----
|
16
|
+
|
17
|
+
def global_key_prefix
|
18
|
+
"keen.#{@client.options[:storage_namespace]}"
|
19
|
+
end
|
20
|
+
|
21
|
+
def active_queue_key
|
22
|
+
"#{global_key_prefix}.active_queue"
|
23
|
+
end
|
24
|
+
|
25
|
+
def failed_queue_key
|
26
|
+
"#{global_key_prefix}.failed_queue"
|
27
|
+
end
|
28
|
+
|
29
|
+
def add_to_active_queue(value)
|
30
|
+
@redis.lpush active_queue_key, value
|
31
|
+
if @logging
|
32
|
+
puts "added #{value} to active queue; length is now #{@redis.llen active_queue_key}"
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def record_job(job)
|
37
|
+
add_to_active_queue JSON.generate(job)
|
38
|
+
end
|
39
|
+
|
40
|
+
def handle_prior_failures
|
41
|
+
# TODO consume the failed_queue and do something with it (loggly? retry? flat file?)
|
42
|
+
end
|
43
|
+
|
44
|
+
def count_active_queue
|
45
|
+
raise NotImplementedError
|
46
|
+
end
|
47
|
+
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -6,141 +6,79 @@ require 'time'
|
|
6
6
|
module Keen
|
7
7
|
module Async
|
8
8
|
module Storage
|
9
|
-
class RedisHandler
|
9
|
+
class RedisHandler < Keen::Async::Storage::BaseStorageHandler
|
10
10
|
|
11
11
|
# Keys
|
12
12
|
# ----
|
13
13
|
|
14
|
-
def global_key_prefix
|
15
|
-
"keen"
|
16
|
-
end
|
17
|
-
|
18
|
-
def active_queue_key
|
19
|
-
"#{global_key_prefix}.active_queue_key"
|
20
|
-
end
|
21
|
-
|
22
|
-
def failed_queue_key
|
23
|
-
"#{global_key_prefix}.failed_queue_key"
|
24
|
-
end
|
25
|
-
|
26
14
|
def add_to_active_queue(value)
|
27
|
-
|
15
|
+
redis.lpush active_queue_key, value
|
28
16
|
if @logging
|
29
|
-
puts "added #{value} to active queue; length is now #{
|
17
|
+
puts "added #{value} to active queue; length is now #{redis.llen active_queue_key}"
|
30
18
|
end
|
31
19
|
end
|
32
20
|
|
33
|
-
def
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
21
|
+
def redis
|
22
|
+
unless @redis
|
23
|
+
@redis = Redis.new
|
24
|
+
end
|
25
|
+
|
26
|
+
@redis
|
39
27
|
end
|
40
28
|
|
41
|
-
def
|
42
|
-
|
43
|
-
@logging = logging
|
44
|
-
end
|
45
|
-
|
46
|
-
def redis=(connection)
|
47
|
-
@redis = connection
|
29
|
+
def count_active_queue
|
30
|
+
redis.llen active_queue_key
|
48
31
|
end
|
49
32
|
|
50
|
-
def
|
51
|
-
|
33
|
+
def clear_active_queue
|
34
|
+
redis.del active_queue_key
|
52
35
|
end
|
53
36
|
|
54
|
-
def
|
55
|
-
|
56
|
-
# Returns a hash of the next `how_many` jobs, indexed on project_id and then collection_name.
|
57
|
-
#
|
58
|
-
# It looks like this:
|
59
|
-
#
|
60
|
-
# collated = {
|
61
|
-
# "4f5775ad163d666a6100000e" => {
|
62
|
-
# "clicks" => [
|
63
|
-
# Keen::Storage::Job.new({
|
64
|
-
# :project_id => "4f5775ad163d666a6100000e",
|
65
|
-
# :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
|
66
|
-
# :collection_name => "clicks",
|
67
|
-
# :event_body => {:user_id => "12345"},
|
68
|
-
# }),
|
69
|
-
# Keen::Storage::Job.new({
|
70
|
-
# :project_id => "4f5775ad163d666a6100000e",
|
71
|
-
# :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
|
72
|
-
# :collection_name => "clicks",
|
73
|
-
# :event_body => {:user_id => "12345"},
|
74
|
-
# }),
|
75
|
-
# Keen::Storage::Job.new({
|
76
|
-
# :project_id => "4f5775ad163d666a6100000e",
|
77
|
-
# :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
|
78
|
-
# :collection_name => "clicks",
|
79
|
-
# :event_body => {:user_id => "12345"},
|
80
|
-
# }),
|
81
|
-
# ],
|
82
|
-
# "purchases" => [
|
83
|
-
# Keen::Storage::Job.new({
|
84
|
-
# :project_id => "4f5775ad163d666a6100000e",
|
85
|
-
# :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
|
86
|
-
# :collection_name => "purchases",
|
87
|
-
# :event_body => {:user_id => "12345"},
|
88
|
-
# }),
|
89
|
-
# Keen::Storage::Job.new({
|
90
|
-
# :project_id => "4f5775ad163d666a6100000e",
|
91
|
-
# :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
|
92
|
-
# :collection_name => "purchases",
|
93
|
-
# :event_body => {:user_id => "12345"},
|
94
|
-
# }),
|
95
|
-
# Keen::Storage::Job.new({
|
96
|
-
# :project_id => "4f5775ad163d666a6100000e",
|
97
|
-
# :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
|
98
|
-
# :collection_name => "purchases",
|
99
|
-
# :event_body => {:user_id => "12345"},
|
100
|
-
# }),
|
101
|
-
# ],
|
102
|
-
# }
|
103
|
-
# }
|
37
|
+
def get_authorized_jobs(how_many, client)
|
104
38
|
|
105
39
|
handle_prior_failures
|
106
40
|
|
107
41
|
key = active_queue_key
|
108
42
|
|
109
|
-
|
43
|
+
job_definitions = []
|
44
|
+
skipped_job_definitions = []
|
110
45
|
|
111
46
|
#puts "doing the job #{how_many} times"
|
112
47
|
|
113
|
-
|
114
|
-
this =
|
115
|
-
if this
|
116
|
-
jobs.push JSON.parse this
|
117
|
-
else
|
118
|
-
#puts "couldn't process value #{this}"
|
119
|
-
end
|
120
|
-
end
|
121
|
-
|
122
|
-
collate_jobs(jobs)
|
123
|
-
end
|
124
|
-
|
125
|
-
def collate_jobs(queue)
|
126
|
-
collated = {}
|
48
|
+
while true do
|
49
|
+
this = redis.lpop key
|
127
50
|
|
128
|
-
|
51
|
+
# If we're out of jobs, end the loop:
|
52
|
+
if not this
|
53
|
+
break
|
54
|
+
end
|
55
|
+
|
56
|
+
# Parse the JSON into a job definition
|
57
|
+
job_definition = JSON.parse this
|
58
|
+
job_definition = Keen::Utils.symbolize_keys(job_definition)
|
59
|
+
|
60
|
+
# Make sure this client is authorized to process this job:
|
61
|
+
unless job_definition[:project_id] == client.project_id
|
62
|
+
unless job_definition[:auth_token] == client.auth_token
|
63
|
+
# We're not authorized, so skip this job.
|
64
|
+
skipped_job_definitions.push job_definition
|
65
|
+
next
|
66
|
+
end
|
67
|
+
end
|
129
68
|
|
130
|
-
|
69
|
+
job_definitions.push job_definition
|
131
70
|
|
132
|
-
if
|
133
|
-
|
71
|
+
if job_definitions.length == how_many
|
72
|
+
break
|
134
73
|
end
|
135
74
|
|
136
|
-
|
137
|
-
collated[job.project_id][job.collection_name] = []
|
138
|
-
end
|
75
|
+
end
|
139
76
|
|
140
|
-
|
77
|
+
# Put the skipped jobs back on the queue.
|
78
|
+
skipped_job_definitions.each do |job_definition|
|
79
|
+
redis.lpush key, job_definition
|
141
80
|
end
|
142
81
|
|
143
|
-
collated
|
144
82
|
end
|
145
83
|
|
146
84
|
end
|
data/lib/keen/async/worker.rb
CHANGED
@@ -1,7 +1,4 @@
|
|
1
1
|
require "keen/async/storage/redis_handler"
|
2
|
-
require "net/http"
|
3
|
-
require "net/https"
|
4
|
-
|
5
2
|
|
6
3
|
|
7
4
|
module Keen
|
@@ -14,67 +11,46 @@ module Keen
|
|
14
11
|
|
15
12
|
class Worker
|
16
13
|
|
17
|
-
def initialize(
|
18
|
-
@
|
14
|
+
def initialize(client)
|
15
|
+
@client = client
|
16
|
+
@storage_handler = client.storage_handler
|
19
17
|
end
|
20
18
|
|
21
19
|
def batch_url(project_id)
|
22
20
|
if not project_id
|
23
21
|
raise "Missing project_id."
|
24
22
|
end
|
25
|
-
"https://api.keen.io/
|
23
|
+
"https://api.keen.io/2.0/projects/#{project_id}/_events"
|
26
24
|
end
|
27
25
|
|
28
26
|
def process_queue
|
29
|
-
|
30
|
-
queue_length = @handler.count_active_queue
|
27
|
+
queue_length = @storage_handler.count_active_queue
|
31
28
|
|
32
29
|
batch_size = Keen::Async::BATCH_SIZE
|
33
30
|
|
34
|
-
num_batches = 1 + queue_length / batch_size
|
35
31
|
|
32
|
+
events = []
|
33
|
+
|
34
|
+
|
35
|
+
responses = []
|
36
|
+
|
37
|
+
num_batches = queue_length / batch_size + 1
|
36
38
|
num_batches.times do
|
37
|
-
|
38
|
-
|
39
|
-
|
39
|
+
|
40
|
+
job_definitions = @storage_handler.get_authorized_jobs(batch_size, @client)
|
41
|
+
|
42
|
+
job_definitions.each do |job_definition|
|
43
|
+
#puts JSON.generate job_definition
|
44
|
+
job = Keen::Async::Job.new(@client, job_definition)
|
45
|
+
events.push Keen::Event.new(job.timestamp, job.collection_name, job.event_body)
|
40
46
|
end
|
41
|
-
end
|
42
47
|
|
43
|
-
|
44
|
-
end
|
45
|
-
|
46
|
-
def send_batch(project_id, batch)
|
47
|
-
if not batch
|
48
|
-
return
|
48
|
+
responses.push @client.send_batch(events)
|
49
49
|
end
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
uri = URI.parse(batch_url(project_id))
|
56
|
-
http = Net::HTTP.new(uri.host, uri.port)
|
57
|
-
http.use_ssl = true
|
58
|
-
http.ca_file = Keen::Async::SSL_CA_FILE
|
59
|
-
http.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
60
|
-
http.verify_depth = 5
|
61
|
-
|
62
|
-
request = Net::HTTP::Post.new(uri.path)
|
63
|
-
request.body = batch.to_json
|
64
|
-
request["Content-Type"] = "application/json"
|
65
|
-
request["Authorization"] = auth_token
|
66
|
-
|
67
|
-
response = http.request(request)
|
68
|
-
|
69
|
-
#response = Net::HTTP.start(uri.host, uri.port) {|http|
|
70
|
-
#http.request(request)
|
71
|
-
#}
|
72
|
-
|
73
|
-
puts response
|
74
|
-
# TODO: If something fails, we should move the job to the
|
75
|
-
# prior_failures queue by calling, perhaps:
|
76
|
-
#
|
77
|
-
# @handler.log_failed_job(job)
|
50
|
+
|
51
|
+
|
52
|
+
responses
|
53
|
+
|
78
54
|
end
|
79
55
|
|
80
56
|
end
|
data/lib/keen/client.rb
CHANGED
@@ -1,15 +1,23 @@
|
|
1
|
-
require "keen/async/storage/redis_handler"
|
2
|
-
require "keen/async/job"
|
3
1
|
require "json"
|
4
2
|
require "uri"
|
5
3
|
require "time"
|
4
|
+
require "net/http"
|
5
|
+
require "net/https"
|
6
6
|
|
7
7
|
|
8
8
|
module Keen
|
9
9
|
|
10
10
|
class Client
|
11
11
|
|
12
|
-
attr_accessor :storage_handler, :project_id, :auth_token
|
12
|
+
attr_accessor :storage_handler, :project_id, :auth_token, :options
|
13
|
+
|
14
|
+
def base_url
|
15
|
+
if @options[:base_url]
|
16
|
+
@options[:base_url]
|
17
|
+
else
|
18
|
+
"https://api.keen.io/2.0"
|
19
|
+
end
|
20
|
+
end
|
13
21
|
|
14
22
|
def initialize(project_id, auth_token, options = {})
|
15
23
|
|
@@ -17,20 +25,39 @@ module Keen
|
|
17
25
|
raise "auth_token must be string" unless auth_token.kind_of? String
|
18
26
|
|
19
27
|
default_options = {
|
20
|
-
:
|
28
|
+
:logging => true,
|
29
|
+
|
30
|
+
# warning! not caching locally leads to bad performance:
|
31
|
+
:cache_locally => true,
|
32
|
+
|
33
|
+
# this is required if cache_locally is true:
|
34
|
+
:storage_class => nil,
|
35
|
+
|
36
|
+
# all keys will be prepended with this:
|
37
|
+
:storage_namespace => "default",
|
21
38
|
}
|
22
|
-
|
39
|
+
|
23
40
|
options = default_options.update(options)
|
24
41
|
|
25
42
|
@project_id = project_id
|
26
43
|
@auth_token = auth_token
|
27
|
-
@
|
28
|
-
end
|
44
|
+
@cache_locally = options[:cache_locally]
|
29
45
|
|
30
|
-
|
46
|
+
if @cache_locally
|
47
|
+
@storage_class = options[:storage_class]
|
48
|
+
unless @storage_class and @storage_class < Keen::Async::Storage::BaseStorageHandler
|
49
|
+
raise "The Storage Class you send must extend BaseStorageHandler. You sent: #{@storage_class}"
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
@logging = options[:logging]
|
54
|
+
@options = options
|
31
55
|
|
56
|
+
end
|
57
|
+
|
58
|
+
def storage_handler
|
32
59
|
unless @storage_handler
|
33
|
-
@storage_handler =
|
60
|
+
@storage_handler = @storage_class.new(self)
|
34
61
|
end
|
35
62
|
|
36
63
|
@storage_handler
|
@@ -44,33 +71,98 @@ module Keen
|
|
44
71
|
#
|
45
72
|
# `timestamp` is optional. If sent, it should be a Time instance.
|
46
73
|
# If it's not sent, we'll use the current time.
|
47
|
-
|
74
|
+
|
48
75
|
validate_collection_name(collection_name)
|
49
76
|
|
50
77
|
unless timestamp
|
51
78
|
timestamp = Time.now
|
52
79
|
end
|
53
80
|
|
54
|
-
|
81
|
+
timestamp = timestamp.utc.iso8601
|
55
82
|
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
83
|
+
event = Keen::Event.new(timestamp, collection_name, event_body)
|
84
|
+
|
85
|
+
if @cache_locally
|
86
|
+
job = Keen::Async::Job.new(storage_handler, {
|
87
|
+
:timestamp => event.timestamp,
|
88
|
+
:project_id => @project_id,
|
89
|
+
:auth_token => @auth_token,
|
90
|
+
:collection_name => collection_name,
|
91
|
+
:event_body => event.body,
|
92
|
+
})
|
93
|
+
|
94
|
+
job.save
|
95
|
+
else
|
96
|
+
# build the request:
|
97
|
+
url = "#{base_url}/projects/#{project_id}/#{collection_name}"
|
98
|
+
uri = URI.parse(url)
|
99
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
100
|
+
http.use_ssl = true
|
101
|
+
http.ca_file = Keen::Async::SSL_CA_FILE
|
102
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
103
|
+
http.verify_depth = 5
|
104
|
+
|
105
|
+
request = Net::HTTP::Post.new(uri.path)
|
106
|
+
request.body = JSON.generate({
|
107
|
+
:header => {
|
108
|
+
:timestamp => event.timestamp,
|
109
|
+
},
|
110
|
+
:body => event.body
|
111
|
+
})
|
112
|
+
|
113
|
+
request["Content-Type"] = "application/json"
|
114
|
+
request["Authorization"] = @auth_token
|
115
|
+
|
116
|
+
response = http.request(request)
|
117
|
+
JSON.parse response.body
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
def send_batch(events)
|
122
|
+
# make the request body:
|
123
|
+
request_body = {}
|
124
|
+
events.each { |event|
|
125
|
+
unless request_body.has_key? event.collection_name
|
126
|
+
request_body[event.collection_name] = []
|
127
|
+
end
|
128
|
+
|
129
|
+
header = {"timestamp" => event.timestamp}
|
130
|
+
body = event.body
|
131
|
+
item = {"header" => header, "body" => body}
|
132
|
+
request_body[event.collection_name].push(item)
|
133
|
+
}
|
134
|
+
request_body = request_body.to_json
|
135
|
+
|
136
|
+
# build the request:
|
137
|
+
url = "#{base_url}/projects/#{project_id}/_events"
|
138
|
+
uri = URI.parse(url)
|
139
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
140
|
+
http.use_ssl = true
|
141
|
+
http.ca_file = Keen::Async::SSL_CA_FILE
|
142
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
143
|
+
http.verify_depth = 5
|
144
|
+
|
145
|
+
request = Net::HTTP::Post.new(uri.path)
|
146
|
+
request.body = request_body
|
147
|
+
request["Content-Type"] = "application/json"
|
148
|
+
request["Authorization"] = auth_token
|
149
|
+
|
150
|
+
resp = http.request(request)
|
151
|
+
|
152
|
+
return JSON.parse resp.body
|
62
153
|
|
63
|
-
job.save
|
64
154
|
end
|
65
155
|
|
66
156
|
def validate_collection_name(collection_name)
|
67
157
|
# TODO
|
68
158
|
end
|
69
159
|
|
70
|
-
def self.create_new_storage_handler(storage_mode)
|
160
|
+
def self.create_new_storage_handler(storage_mode, client, logging)
|
161
|
+
# This is shitty as hell. We shoudl take in a class reference pointing to the storage handler, not switch on string values
|
71
162
|
case storage_mode.to_sym
|
163
|
+
|
72
164
|
when :redis
|
73
|
-
Keen::Async::Storage::RedisHandler.new
|
165
|
+
Keen::Async::Storage::RedisHandler.new(client, logging)
|
74
166
|
else
|
75
167
|
raise "Unknown storage_mode sent to client: `#{storage_mode}`"
|
76
168
|
end
|
data/lib/keen/event.rb
ADDED
data/lib/keen/keys.rb
ADDED
data/lib/keen/version.rb
CHANGED
data/send.rb
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
require "rubygems"
|
2
|
+
require "keen"
|
3
|
+
|
4
|
+
storage_handler = Keen::Client.create_new_storage_handler(:redis)
|
5
|
+
|
6
|
+
count = storage_handler.count_active_queue
|
7
|
+
puts "we have this many jobs: #{count}"
|
8
|
+
|
9
|
+
worker = Keen::Async::Worker.new(storage_handler)
|
10
|
+
results = worker.process_queue
|
11
|
+
|
12
|
+
puts results
|
data/test/keen_spec.rb
CHANGED
@@ -1,44 +1,34 @@
|
|
1
1
|
$LOAD_PATH << File.join(File.dirname(__FILE__), "..", "lib")
|
2
2
|
|
3
3
|
require "keen"
|
4
|
-
require "fakeweb"
|
5
4
|
|
6
5
|
describe Keen::Client do
|
7
6
|
|
8
|
-
#
|
9
|
-
before :all do
|
10
|
-
FakeWeb.register_uri(:any, %r/https:\/\/api.keen.io\//, :body => '{"message": "You tried to reach Keen"}')
|
11
|
-
end
|
12
|
-
|
7
|
+
# The add_event method should add stuff to Redis:
|
13
8
|
describe "#add_event" do
|
9
|
+
|
10
|
+
# set up the Keen Client instance:
|
14
11
|
project_id = "4f5775ad163d666a6100000e"
|
15
12
|
auth_token = "a5d4eaf432914823a94ecd7e0cb547b9"
|
16
13
|
|
17
|
-
|
14
|
+
# Make a client:
|
15
|
+
client = Keen::Client.new(project_id,
|
16
|
+
auth_token,
|
17
|
+
:storage_class => Keen::Async::Storage::RedisHandler,
|
18
|
+
:logging => true )
|
18
19
|
|
19
|
-
|
20
|
-
|
20
|
+
# Flush the queue first:
|
21
|
+
client.clear_active_queue()
|
22
|
+
|
23
|
+
# Send some events to the client, which should persist them in Redis
|
24
|
+
5.times do
|
25
|
+
client.add_event("rspec_clicks", {
|
21
26
|
:hi => "you",
|
22
27
|
})
|
23
28
|
end
|
24
29
|
|
25
|
-
|
26
|
-
|
27
|
-
worker.process_queue
|
30
|
+
# Make sure we have the right number of things in the queue:
|
31
|
+
client.storage_handler.count_active_queue.should == 5
|
28
32
|
|
29
33
|
end
|
30
|
-
|
31
|
-
# TODO spec it out, lazy bones!
|
32
|
-
#
|
33
|
-
#
|
34
|
-
# Each time an event is logged, we'll store a json-serialized, base64-
|
35
|
-
# encoded, and gzipped hash that loos like this:
|
36
|
-
#
|
37
|
-
# {
|
38
|
-
# :project_id => "alsdjfaldskfjadskladsklj",
|
39
|
-
# :auth_token => "aslkjflk3wjfaklsjdflkasdjflkadjflakdj211",
|
40
|
-
# :collection_name => "purchases",
|
41
|
-
# :event_body => {:prop1 => "val1", :prop2 => "val2"},
|
42
|
-
# }
|
43
|
-
|
44
34
|
end
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: keen
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 27
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 0
|
8
|
+
- 1
|
8
9
|
- 0
|
9
|
-
|
10
|
-
version: 0.0.53
|
10
|
+
version: 0.1.0
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Kyle Wild
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2012-06-
|
18
|
+
date: 2012-06-17 00:00:00 -07:00
|
19
19
|
default_executable:
|
20
20
|
dependencies:
|
21
21
|
- !ruby/object:Gem::Dependency
|
@@ -35,35 +35,35 @@ dependencies:
|
|
35
35
|
type: :runtime
|
36
36
|
version_requirements: *id001
|
37
37
|
- !ruby/object:Gem::Dependency
|
38
|
-
name:
|
38
|
+
name: rspec
|
39
39
|
prerelease: false
|
40
40
|
requirement: &id002 !ruby/object:Gem::Requirement
|
41
41
|
none: false
|
42
42
|
requirements:
|
43
43
|
- - ">="
|
44
44
|
- !ruby/object:Gem::Version
|
45
|
-
hash:
|
45
|
+
hash: 43
|
46
46
|
segments:
|
47
|
-
-
|
48
|
-
-
|
47
|
+
- 2
|
48
|
+
- 9
|
49
49
|
- 0
|
50
|
-
version:
|
50
|
+
version: 2.9.0
|
51
51
|
type: :runtime
|
52
52
|
version_requirements: *id002
|
53
53
|
- !ruby/object:Gem::Dependency
|
54
|
-
name:
|
54
|
+
name: cucumber
|
55
55
|
prerelease: false
|
56
56
|
requirement: &id003 !ruby/object:Gem::Requirement
|
57
57
|
none: false
|
58
58
|
requirements:
|
59
59
|
- - ">="
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
hash:
|
61
|
+
hash: 29
|
62
62
|
segments:
|
63
|
+
- 1
|
63
64
|
- 2
|
64
|
-
-
|
65
|
-
|
66
|
-
version: 2.9.0
|
65
|
+
- 1
|
66
|
+
version: 1.2.1
|
67
67
|
type: :runtime
|
68
68
|
version_requirements: *id003
|
69
69
|
- !ruby/object:Gem::Dependency
|
@@ -82,22 +82,6 @@ dependencies:
|
|
82
82
|
version: 1.2.4
|
83
83
|
type: :runtime
|
84
84
|
version_requirements: *id004
|
85
|
-
- !ruby/object:Gem::Dependency
|
86
|
-
name: redis
|
87
|
-
prerelease: false
|
88
|
-
requirement: &id005 !ruby/object:Gem::Requirement
|
89
|
-
none: false
|
90
|
-
requirements:
|
91
|
-
- - ">="
|
92
|
-
- !ruby/object:Gem::Version
|
93
|
-
hash: 3
|
94
|
-
segments:
|
95
|
-
- 2
|
96
|
-
- 2
|
97
|
-
- 2
|
98
|
-
version: 2.2.2
|
99
|
-
type: :runtime
|
100
|
-
version_requirements: *id005
|
101
85
|
description: See the github repo or examples.rb for usage information.
|
102
86
|
email:
|
103
87
|
- kyle@keen.io
|
@@ -110,7 +94,6 @@ extra_rdoc_files: []
|
|
110
94
|
files:
|
111
95
|
- .gitignore
|
112
96
|
- .rvmrc
|
113
|
-
- Gemfile
|
114
97
|
- README.md
|
115
98
|
- bin/keen_send
|
116
99
|
- conf/cacert.pem
|
@@ -189,15 +172,24 @@ files:
|
|
189
172
|
- examples/rails_2/CoolForums/test/unit/helpers/users_helper_test.rb
|
190
173
|
- examples/rails_2/CoolForums/test/unit/user_test.rb
|
191
174
|
- examples/rails_2/Gemfile
|
175
|
+
- features/add_event.feature
|
176
|
+
- features/redis_queue.feature
|
177
|
+
- features/step_definitions/keen_steps.rb
|
178
|
+
- features/support/before_and_after.rb
|
192
179
|
- keen.gemspec
|
193
180
|
- lib/keen.rb
|
181
|
+
- lib/keen/async.rb
|
194
182
|
- lib/keen/async/job.rb
|
183
|
+
- lib/keen/async/storage/base_storage_handler.rb
|
195
184
|
- lib/keen/async/storage/flat_file_handler.rb
|
196
185
|
- lib/keen/async/storage/redis_handler.rb
|
197
186
|
- lib/keen/async/worker.rb
|
198
187
|
- lib/keen/client.rb
|
188
|
+
- lib/keen/event.rb
|
189
|
+
- lib/keen/keys.rb
|
199
190
|
- lib/keen/utils.rb
|
200
191
|
- lib/keen/version.rb
|
192
|
+
- send.rb
|
201
193
|
- test/keen_spec.rb
|
202
194
|
has_rdoc: true
|
203
195
|
homepage: https://github.com/keenlabs/KeenClient-Ruby
|
@@ -234,4 +226,8 @@ signing_key:
|
|
234
226
|
specification_version: 3
|
235
227
|
summary: A library for sending events to the keen.io API.
|
236
228
|
test_files:
|
229
|
+
- features/add_event.feature
|
230
|
+
- features/redis_queue.feature
|
231
|
+
- features/step_definitions/keen_steps.rb
|
232
|
+
- features/support/before_and_after.rb
|
237
233
|
- test/keen_spec.rb
|
data/Gemfile
DELETED