keen 0.0.2 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +2 -0
- data/bin/keen_send +5 -5
- data/examples.rb +1 -0
- data/examples/rails_2/CoolForums/app/controllers/application_controller.rb +3 -0
- data/keen.gemspec +3 -2
- data/lib/keen.rb +4 -1
- data/lib/keen/async/job.rb +70 -0
- data/lib/keen/async/storage/flat_file_handler.rb +57 -0
- data/lib/keen/async/storage/redis_handler.rb +136 -0
- data/lib/keen/async/worker.rb +81 -0
- data/lib/keen/client.rb +55 -99
- data/lib/keen/version.rb +1 -1
- data/test/keen_spec.rb +21 -3
- metadata +41 -23
- data/lib/keen/storage/flat_file_handler.rb +0 -55
- data/lib/keen/storage/redis_handler.rb +0 -155
data/Gemfile
CHANGED
data/bin/keen_send
CHANGED
@@ -1,11 +1,11 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
-
$LOAD_PATH << File.join(File.dirname(__FILE__),
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
2
|
+
$LOAD_PATH << File.join(File.dirname(__FILE__), "..", "lib")
|
3
|
+
require "rubygems"
|
4
|
+
require "optparse"
|
5
|
+
require "keen"
|
6
6
|
|
7
7
|
options = {
|
8
|
-
:env =>
|
8
|
+
:env => "production",
|
9
9
|
}
|
10
10
|
|
11
11
|
required = [
|
data/examples.rb
CHANGED
data/keen.gemspec
CHANGED
@@ -22,9 +22,10 @@ Gem::Specification.new do |s|
|
|
22
22
|
# s.add_development_dependency "rspec"
|
23
23
|
# s.add_runtime_dependency "rest-client"
|
24
24
|
|
25
|
-
s.add_dependency('
|
25
|
+
s.add_dependency('json', '>= 1.6.5')
|
26
|
+
s.add_dependency('fakeweb', '>= 1.3.0')
|
27
|
+
s.add_dependency('rspec', '>= 2.9.0')
|
26
28
|
s.add_dependency('system_timer', '>= 1.2.4')
|
27
|
-
s.add_dependency('httparty', '>= 0.8.1')
|
28
29
|
s.add_dependency('redis', '>= 2.2.2')
|
29
30
|
|
30
31
|
# took these from Twilio library:
|
data/lib/keen.rb
CHANGED
@@ -0,0 +1,70 @@
|
|
1
|
+
require 'keen/async/storage/redis_handler'
|
2
|
+
|
3
|
+
module Keen
|
4
|
+
module Async
|
5
|
+
class Job
|
6
|
+
# Represents one job.
|
7
|
+
#
|
8
|
+
#
|
9
|
+
|
10
|
+
attr_accessor :project_id, :auth_token, :collection_name, :event_body
|
11
|
+
|
12
|
+
def to_json(options=nil)
|
13
|
+
@definition.to_json
|
14
|
+
end
|
15
|
+
|
16
|
+
|
17
|
+
def to_s
|
18
|
+
self.to_json
|
19
|
+
end
|
20
|
+
|
21
|
+
def initialize(handler, definition={})
|
22
|
+
|
23
|
+
load_definition(definition)
|
24
|
+
@handler = handler
|
25
|
+
|
26
|
+
end
|
27
|
+
|
28
|
+
def load_definition(definition)
|
29
|
+
|
30
|
+
definition = Keen::Utils.symbolize_keys(definition)
|
31
|
+
|
32
|
+
# define some key lists:
|
33
|
+
required_keys = [:project_id, :auth_token, :collection_name, :event_body]
|
34
|
+
optional_keys = []
|
35
|
+
all_keys = required_keys + optional_keys
|
36
|
+
|
37
|
+
|
38
|
+
# don't allow them to send nil values for anything
|
39
|
+
definition.each do |key, value|
|
40
|
+
# reject unrecognized keys:
|
41
|
+
raise "Unrecognized key: #{key}" unless all_keys.include? key.to_sym
|
42
|
+
end
|
43
|
+
|
44
|
+
|
45
|
+
required_keys.each do |key|
|
46
|
+
value = definition[key]
|
47
|
+
|
48
|
+
raise "You sent a nil value for the #{key}." if value.nil?
|
49
|
+
end
|
50
|
+
|
51
|
+
|
52
|
+
all_keys.each do |key|
|
53
|
+
value = definition[key]
|
54
|
+
self.instance_variable_set("@#{key}", value)
|
55
|
+
end
|
56
|
+
|
57
|
+
@definition = definition
|
58
|
+
|
59
|
+
|
60
|
+
end
|
61
|
+
|
62
|
+
def save
|
63
|
+
@handler.record_job(self)
|
64
|
+
end
|
65
|
+
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
|
@@ -0,0 +1,57 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require 'base64'
|
3
|
+
require 'zlib'
|
4
|
+
|
5
|
+
|
6
|
+
|
7
|
+
module Keen
|
8
|
+
module Async
|
9
|
+
module Storage
|
10
|
+
class FlatFileHandler
|
11
|
+
|
12
|
+
# Paths
|
13
|
+
# -----
|
14
|
+
|
15
|
+
# Where new events go as they come in:
|
16
|
+
ACTIVE = "active.txt"
|
17
|
+
|
18
|
+
# The temporary file that will store records during processing:
|
19
|
+
PROCESSING = "processing.txt"
|
20
|
+
|
21
|
+
# Records are put here if processing fails:
|
22
|
+
FAILED = "failed.txt"
|
23
|
+
|
24
|
+
def initialize(filepath)
|
25
|
+
@filepath = filepath
|
26
|
+
|
27
|
+
is_directory?
|
28
|
+
is_writable?
|
29
|
+
is_readable?
|
30
|
+
end
|
31
|
+
|
32
|
+
def dump_contents
|
33
|
+
# move
|
34
|
+
end
|
35
|
+
|
36
|
+
def is_writable?
|
37
|
+
if not FileTest.writable? @filepath
|
38
|
+
raise "Can't write to file: " + @filepath
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def is_readable?
|
43
|
+
if not FileTest.readable? @filepath
|
44
|
+
raise "Can't read from file: " + @filepath
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def is_directory?
|
49
|
+
if not FileTest.directory? @filepath
|
50
|
+
raise "Can't read from file: " + @filepath
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,136 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require 'redis'
|
3
|
+
require 'json'
|
4
|
+
require 'time'
|
5
|
+
|
6
|
+
module Keen
|
7
|
+
module Async
|
8
|
+
module Storage
|
9
|
+
class RedisHandler
|
10
|
+
|
11
|
+
# Keys
|
12
|
+
# ----
|
13
|
+
|
14
|
+
def global_key_prefix
|
15
|
+
"keen_415" + Keen::VERSION
|
16
|
+
end
|
17
|
+
|
18
|
+
def active_queue_key
|
19
|
+
"#{global_key_prefix}.active_queue_key"
|
20
|
+
end
|
21
|
+
|
22
|
+
def failed_queue_key
|
23
|
+
"#{global_key_prefix}.failed_queue_key"
|
24
|
+
end
|
25
|
+
|
26
|
+
def add_to_active_queue(value)
|
27
|
+
@redis.lpush active_queue_key, value
|
28
|
+
puts "added #{value} to active queue; length is now #{@redis.llen active_queue_key}"
|
29
|
+
end
|
30
|
+
|
31
|
+
def record_job(job)
|
32
|
+
add_to_active_queue JSON.generate(job)
|
33
|
+
end
|
34
|
+
|
35
|
+
def handle_prior_failures
|
36
|
+
# TODO consume the failed_queue and do something with it (loggly? retry? flat file?)
|
37
|
+
end
|
38
|
+
|
39
|
+
def initialize
|
40
|
+
@redis = Redis.new
|
41
|
+
end
|
42
|
+
|
43
|
+
def count_active_queue
|
44
|
+
@redis.llen active_queue_key
|
45
|
+
end
|
46
|
+
|
47
|
+
def get_collated_jobs(how_many)
|
48
|
+
|
49
|
+
# Returns a hash of the next `how_many` jobs, indexed on project_id and then collection_name.
|
50
|
+
#
|
51
|
+
# It looks like this:
|
52
|
+
#
|
53
|
+
# collated = {
|
54
|
+
# "4f5775ad163d666a6100000e" => {
|
55
|
+
# "clicks" => [
|
56
|
+
# Keen::Storage::Job.new({
|
57
|
+
# :project_id => "4f5775ad163d666a6100000e",
|
58
|
+
# :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
|
59
|
+
# :collection_name => "clicks",
|
60
|
+
# :event_body => {:user_id => "12345"},
|
61
|
+
# }),
|
62
|
+
# Keen::Storage::Job.new({
|
63
|
+
# :project_id => "4f5775ad163d666a6100000e",
|
64
|
+
# :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
|
65
|
+
# :collection_name => "clicks",
|
66
|
+
# :event_body => {:user_id => "12345"},
|
67
|
+
# }),
|
68
|
+
# Keen::Storage::Job.new({
|
69
|
+
# :project_id => "4f5775ad163d666a6100000e",
|
70
|
+
# :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
|
71
|
+
# :collection_name => "clicks",
|
72
|
+
# :event_body => {:user_id => "12345"},
|
73
|
+
# }),
|
74
|
+
# ],
|
75
|
+
# "purchases" => [
|
76
|
+
# Keen::Storage::Job.new({
|
77
|
+
# :project_id => "4f5775ad163d666a6100000e",
|
78
|
+
# :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
|
79
|
+
# :collection_name => "purchases",
|
80
|
+
# :event_body => {:user_id => "12345"},
|
81
|
+
# }),
|
82
|
+
# Keen::Storage::Job.new({
|
83
|
+
# :project_id => "4f5775ad163d666a6100000e",
|
84
|
+
# :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
|
85
|
+
# :collection_name => "purchases",
|
86
|
+
# :event_body => {:user_id => "12345"},
|
87
|
+
# }),
|
88
|
+
# Keen::Storage::Job.new({
|
89
|
+
# :project_id => "4f5775ad163d666a6100000e",
|
90
|
+
# :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
|
91
|
+
# :collection_name => "purchases",
|
92
|
+
# :event_body => {:user_id => "12345"},
|
93
|
+
# }),
|
94
|
+
# ],
|
95
|
+
# }
|
96
|
+
# }
|
97
|
+
|
98
|
+
handle_prior_failures
|
99
|
+
|
100
|
+
key = active_queue_key
|
101
|
+
|
102
|
+
jobs = []
|
103
|
+
|
104
|
+
how_many.times do
|
105
|
+
this = @redis.lpop key
|
106
|
+
jobs.push JSON.parse this
|
107
|
+
end
|
108
|
+
|
109
|
+
collate_jobs(jobs)
|
110
|
+
end
|
111
|
+
|
112
|
+
def collate_jobs(queue)
|
113
|
+
collated = {}
|
114
|
+
|
115
|
+
queue.each do |job_hash|
|
116
|
+
|
117
|
+
job = Keen::Async::Job.new(self, job_hash)
|
118
|
+
|
119
|
+
if not collated.has_key? job.project_id
|
120
|
+
collated[job.project_id] = {}
|
121
|
+
end
|
122
|
+
|
123
|
+
if not collated[job.project_id].has_key? job.collection_name
|
124
|
+
collated[job.project_id][job.collection_name] = []
|
125
|
+
end
|
126
|
+
|
127
|
+
collated[job.project_id][job.collection_name].push(job)
|
128
|
+
end
|
129
|
+
|
130
|
+
collated
|
131
|
+
end
|
132
|
+
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
require "keen/async/storage/redis_handler"
|
2
|
+
require "net/http"
|
3
|
+
require "net/https"
|
4
|
+
|
5
|
+
|
6
|
+
|
7
|
+
module Keen
|
8
|
+
|
9
|
+
module Async
|
10
|
+
|
11
|
+
# How many events should we send over the wire at a time?
|
12
|
+
BATCH_SIZE = 100
|
13
|
+
SSL_CA_FILE = File.dirname(__FILE__) + '../../../conf/cacert.pem'
|
14
|
+
|
15
|
+
class Worker
|
16
|
+
|
17
|
+
def initialize(handler)
|
18
|
+
@handler = handler
|
19
|
+
end
|
20
|
+
|
21
|
+
def batch_url(project_id)
|
22
|
+
if not project_id
|
23
|
+
raise "Missing project_id."
|
24
|
+
end
|
25
|
+
"https://api.keen.io/1.0/projects/#{project_id}/_events"
|
26
|
+
end
|
27
|
+
|
28
|
+
def process_queue
|
29
|
+
|
30
|
+
queue_length = @handler.count_active_queue
|
31
|
+
|
32
|
+
batch_size = Keen::Async::BATCH_SIZE
|
33
|
+
|
34
|
+
num_batches = queue_length / batch_size
|
35
|
+
|
36
|
+
num_batches.times do
|
37
|
+
collated = @handler.get_collated_jobs(batch_size)
|
38
|
+
collated.each do |project_id, batch|
|
39
|
+
send_batch(project_id, batch)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def send_batch(project_id, batch)
|
45
|
+
if not batch
|
46
|
+
return
|
47
|
+
end
|
48
|
+
|
49
|
+
first_key = batch.keys[0]
|
50
|
+
job_list = batch[first_key]
|
51
|
+
auth_token = job_list[0].auth_token
|
52
|
+
|
53
|
+
uri = URI.parse(batch_url(project_id))
|
54
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
55
|
+
http.use_ssl = true
|
56
|
+
http.ca_file = Keen::Async::SSL_CA_FILE
|
57
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
58
|
+
http.verify_depth = 5
|
59
|
+
|
60
|
+
request = Net::HTTP::Post.new(uri.path)
|
61
|
+
request.body = batch.to_json
|
62
|
+
request["Content-Type"] = "application/json"
|
63
|
+
request["Authorization"] = auth_token
|
64
|
+
|
65
|
+
response = http.request(request)
|
66
|
+
|
67
|
+
#response = Net::HTTP.start(uri.host, uri.port) {|http|
|
68
|
+
#http.request(request)
|
69
|
+
#}
|
70
|
+
|
71
|
+
puts response
|
72
|
+
# TODO: If something fails, we should move the job to the
|
73
|
+
# prior_failures queue by calling, perhaps:
|
74
|
+
#
|
75
|
+
# @handler.log_failed_job(job)
|
76
|
+
end
|
77
|
+
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
end
|
data/lib/keen/client.rb
CHANGED
@@ -1,123 +1,79 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require "
|
1
|
+
require "keen/async/storage/redis_handler"
|
2
|
+
require "keen/async/job"
|
3
|
+
require "json"
|
4
4
|
require "uri"
|
5
|
+
require "time"
|
5
6
|
|
6
7
|
|
7
8
|
module Keen
|
9
|
+
|
8
10
|
class Client
|
9
11
|
|
10
|
-
|
11
|
-
"http://api.keen.io/1.0/projects/#{project_id}/_events"
|
12
|
-
end
|
13
|
-
|
14
|
-
def initialize(project_id, auth_token)
|
15
|
-
end
|
12
|
+
attr_accessor :storage_handler, :project_id, :auth_token
|
16
13
|
|
17
|
-
def
|
18
|
-
validate_collection_name(collection_name)
|
14
|
+
def initialize(project_id, auth_token, options = {})
|
19
15
|
|
20
|
-
|
16
|
+
raise "project_id must be string" unless project_id.kind_of? String
|
17
|
+
raise "auth_token must be string" unless auth_token.kind_of? String
|
21
18
|
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
end
|
19
|
+
default_options = {
|
20
|
+
:storage_mode => :redis,
|
21
|
+
}
|
22
|
+
|
23
|
+
options = default_options.update(options)
|
24
|
+
|
25
|
+
@project_id = project_id
|
26
|
+
@auth_token = auth_token
|
27
|
+
@storage_mode = options[:storage_mode]
|
32
28
|
end
|
33
29
|
|
34
|
-
def
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
:auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
|
46
|
-
:collection_name => "clicks",
|
47
|
-
:event_body => {:user_id => "12345"},
|
48
|
-
}),
|
49
|
-
Keen::Storage::Item.new({
|
50
|
-
:project_id => "4f5775ad163d666a6100000e",
|
51
|
-
:auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
|
52
|
-
:collection_name => "clicks",
|
53
|
-
:event_body => {:user_id => "12345"},
|
54
|
-
}),
|
55
|
-
Keen::Storage::Item.new({
|
56
|
-
:project_id => "4f5775ad163d666a6100000e",
|
57
|
-
:auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
|
58
|
-
:collection_name => "clicks",
|
59
|
-
:event_body => {:user_id => "12345"},
|
60
|
-
}),
|
61
|
-
],
|
62
|
-
"purchases" => [
|
63
|
-
Keen::Storage::Item.new({
|
64
|
-
:project_id => "4f5775ad163d666a6100000e",
|
65
|
-
:auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
|
66
|
-
:collection_name => "purchases",
|
67
|
-
:event_body => {:user_id => "12345"},
|
68
|
-
}),
|
69
|
-
Keen::Storage::Item.new({
|
70
|
-
:project_id => "4f5775ad163d666a6100000e",
|
71
|
-
:auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
|
72
|
-
:collection_name => "purchases",
|
73
|
-
:event_body => {:user_id => "12345"},
|
74
|
-
}),
|
75
|
-
Keen::Storage::Item.new({
|
76
|
-
:project_id => "4f5775ad163d666a6100000e",
|
77
|
-
:auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
|
78
|
-
:collection_name => "purchases",
|
79
|
-
:event_body => {:user_id => "12345"},
|
80
|
-
}),
|
81
|
-
],
|
82
|
-
}
|
83
|
-
}
|
30
|
+
def handler
|
31
|
+
|
32
|
+
unless @storage_handler
|
33
|
+
mode = @storage_mode
|
34
|
+
|
35
|
+
case mode
|
36
|
+
when :redis
|
37
|
+
@storage_handler = Keen::Async::Storage::RedisHandler.new
|
38
|
+
else
|
39
|
+
raise "Unknown storage_mode sent to client: `#{mode}`"
|
40
|
+
end
|
84
41
|
|
85
|
-
collated.each do |project_id, batch|
|
86
|
-
self.send_batch(project_id, batch)
|
87
42
|
end
|
43
|
+
|
44
|
+
@storage_handler
|
88
45
|
end
|
89
|
-
|
90
|
-
def self.send_batch(project_id, batch)
|
91
|
-
if not batch
|
92
|
-
return
|
93
|
-
end
|
94
46
|
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
47
|
+
def add_event(collection_name, event_body, timestamp=nil)
|
48
|
+
#
|
49
|
+
# `collection_name` should be a string
|
50
|
+
#
|
51
|
+
# `event_body` should be a JSON-serializable hash
|
52
|
+
#
|
53
|
+
# `timestamp` is optional. If sent, it should be a Time instance.
|
54
|
+
# If it's not sent, we'll use the current time.
|
101
55
|
|
102
|
-
|
103
|
-
request.body = batch.to_json
|
104
|
-
request["Content-Type"] = "application/json"
|
105
|
-
request["Authorization"] = auth_token
|
56
|
+
validate_collection_name(collection_name)
|
106
57
|
|
107
|
-
|
108
|
-
|
109
|
-
|
58
|
+
unless timestamp
|
59
|
+
timestamp = Time.now
|
60
|
+
end
|
110
61
|
|
111
|
-
|
62
|
+
event_body[:_timestamp] = timestamp.utc.iso8601
|
112
63
|
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
64
|
+
job = Keen::Async::Job.new(handler, {
|
65
|
+
:project_id => @project_id,
|
66
|
+
:auth_token => @auth_token,
|
67
|
+
:collection_name => collection_name,
|
68
|
+
:event_body => event_body,
|
69
|
+
})
|
70
|
+
|
71
|
+
job.save
|
117
72
|
end
|
118
73
|
|
119
|
-
def
|
120
|
-
|
74
|
+
def validate_collection_name(collection_name)
|
75
|
+
# TODO
|
121
76
|
end
|
77
|
+
|
122
78
|
end
|
123
79
|
end
|
data/lib/keen/version.rb
CHANGED
data/test/keen_spec.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
|
-
$LOAD_PATH << File.join(File.dirname(__FILE__),
|
1
|
+
$LOAD_PATH << File.join(File.dirname(__FILE__), "..", "lib")
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
3
|
+
require "keen"
|
4
|
+
require "fakeweb"
|
5
5
|
|
6
6
|
describe Keen::Client do
|
7
7
|
|
@@ -9,6 +9,24 @@ describe Keen::Client do
|
|
9
9
|
before :all do
|
10
10
|
FakeWeb.register_uri(:any, %r/https:\/\/api.keen.io\//, :body => '{"message": "You tried to reach Keen"}')
|
11
11
|
end
|
12
|
+
|
13
|
+
describe "#add_event" do
|
14
|
+
project_id = "4f5775ad163d666a6100000e"
|
15
|
+
auth_token = "a5d4eaf432914823a94ecd7e0cb547b9"
|
16
|
+
|
17
|
+
keen = Keen::Client.new(project_id, auth_token, :storage_mode => :redis)
|
18
|
+
|
19
|
+
310.times do
|
20
|
+
keen.add_event("rspec_clicks", {
|
21
|
+
:hi => "you",
|
22
|
+
})
|
23
|
+
end
|
24
|
+
|
25
|
+
worker = Keen::Async::Worker.new(handler = keen.storage_handler)
|
26
|
+
|
27
|
+
worker.process_queue
|
28
|
+
|
29
|
+
end
|
12
30
|
|
13
31
|
# TODO spec it out, lazy bones!
|
14
32
|
#
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: keen
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 23
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 0
|
9
|
-
-
|
10
|
-
version: 0.0.
|
9
|
+
- 4
|
10
|
+
version: 0.0.4
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Kyle Wild
|
@@ -15,61 +15,77 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2012-03-
|
18
|
+
date: 2012-03-19 00:00:00 -05:00
|
19
19
|
default_executable:
|
20
20
|
dependencies:
|
21
21
|
- !ruby/object:Gem::Dependency
|
22
|
-
name:
|
22
|
+
name: json
|
23
23
|
prerelease: false
|
24
24
|
requirement: &id001 !ruby/object:Gem::Requirement
|
25
25
|
none: false
|
26
26
|
requirements:
|
27
27
|
- - ">="
|
28
28
|
- !ruby/object:Gem::Version
|
29
|
-
hash:
|
29
|
+
hash: 5
|
30
30
|
segments:
|
31
31
|
- 1
|
32
|
-
-
|
33
|
-
-
|
34
|
-
version: 1.
|
32
|
+
- 6
|
33
|
+
- 5
|
34
|
+
version: 1.6.5
|
35
35
|
type: :runtime
|
36
36
|
version_requirements: *id001
|
37
37
|
- !ruby/object:Gem::Dependency
|
38
|
-
name:
|
38
|
+
name: fakeweb
|
39
39
|
prerelease: false
|
40
40
|
requirement: &id002 !ruby/object:Gem::Requirement
|
41
41
|
none: false
|
42
42
|
requirements:
|
43
43
|
- - ">="
|
44
44
|
- !ruby/object:Gem::Version
|
45
|
-
hash:
|
45
|
+
hash: 27
|
46
46
|
segments:
|
47
47
|
- 1
|
48
|
-
-
|
49
|
-
-
|
50
|
-
version: 1.
|
48
|
+
- 3
|
49
|
+
- 0
|
50
|
+
version: 1.3.0
|
51
51
|
type: :runtime
|
52
52
|
version_requirements: *id002
|
53
53
|
- !ruby/object:Gem::Dependency
|
54
|
-
name:
|
54
|
+
name: rspec
|
55
55
|
prerelease: false
|
56
56
|
requirement: &id003 !ruby/object:Gem::Requirement
|
57
57
|
none: false
|
58
58
|
requirements:
|
59
59
|
- - ">="
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
hash:
|
61
|
+
hash: 43
|
62
62
|
segments:
|
63
|
+
- 2
|
64
|
+
- 9
|
63
65
|
- 0
|
64
|
-
|
65
|
-
- 1
|
66
|
-
version: 0.8.1
|
66
|
+
version: 2.9.0
|
67
67
|
type: :runtime
|
68
68
|
version_requirements: *id003
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
|
-
name:
|
70
|
+
name: system_timer
|
71
71
|
prerelease: false
|
72
72
|
requirement: &id004 !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - ">="
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
hash: 23
|
78
|
+
segments:
|
79
|
+
- 1
|
80
|
+
- 2
|
81
|
+
- 4
|
82
|
+
version: 1.2.4
|
83
|
+
type: :runtime
|
84
|
+
version_requirements: *id004
|
85
|
+
- !ruby/object:Gem::Dependency
|
86
|
+
name: redis
|
87
|
+
prerelease: false
|
88
|
+
requirement: &id005 !ruby/object:Gem::Requirement
|
73
89
|
none: false
|
74
90
|
requirements:
|
75
91
|
- - ">="
|
@@ -81,7 +97,7 @@ dependencies:
|
|
81
97
|
- 2
|
82
98
|
version: 2.2.2
|
83
99
|
type: :runtime
|
84
|
-
version_requirements: *
|
100
|
+
version_requirements: *id005
|
85
101
|
description: See the github repo or examples.rb for usage information.
|
86
102
|
email:
|
87
103
|
- kyle@keen.io
|
@@ -175,9 +191,11 @@ files:
|
|
175
191
|
- examples/rails_2/Gemfile
|
176
192
|
- keen.gemspec
|
177
193
|
- lib/keen.rb
|
194
|
+
- lib/keen/async/job.rb
|
195
|
+
- lib/keen/async/storage/flat_file_handler.rb
|
196
|
+
- lib/keen/async/storage/redis_handler.rb
|
197
|
+
- lib/keen/async/worker.rb
|
178
198
|
- lib/keen/client.rb
|
179
|
-
- lib/keen/storage/flat_file_handler.rb
|
180
|
-
- lib/keen/storage/redis_handler.rb
|
181
199
|
- lib/keen/version.rb
|
182
200
|
- test/keen_spec.rb
|
183
201
|
has_rdoc: true
|
@@ -1,55 +0,0 @@
|
|
1
|
-
require 'fileutils'
|
2
|
-
require 'base64'
|
3
|
-
require 'zlib'
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
module Keen
|
8
|
-
module Storage
|
9
|
-
class FlatFileHandler
|
10
|
-
|
11
|
-
# Paths
|
12
|
-
# -----
|
13
|
-
|
14
|
-
# Where new events go as they come in:
|
15
|
-
ACTIVE = "active.txt"
|
16
|
-
|
17
|
-
# The temporary file that will store records during processing:
|
18
|
-
PROCESSING = "processing.txt"
|
19
|
-
|
20
|
-
# Records are put here if processing fails:
|
21
|
-
FAILED = "failed.txt"
|
22
|
-
|
23
|
-
def initialize(filepath)
|
24
|
-
@filepath = filepath
|
25
|
-
|
26
|
-
is_directory?
|
27
|
-
is_writable?
|
28
|
-
is_readable?
|
29
|
-
end
|
30
|
-
|
31
|
-
def dump_contents
|
32
|
-
# move
|
33
|
-
end
|
34
|
-
|
35
|
-
def is_writable?
|
36
|
-
if not FileTest.writable? @filepath
|
37
|
-
raise "Can't write to file: " + @filepath
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
|
-
def is_readable?
|
42
|
-
if not FileTest.readable? @filepath
|
43
|
-
raise "Can't read from file: " + @filepath
|
44
|
-
end
|
45
|
-
end
|
46
|
-
|
47
|
-
def is_directory?
|
48
|
-
if not FileTest.directory? @filepath
|
49
|
-
raise "Can't read from file: " + @filepath
|
50
|
-
end
|
51
|
-
end
|
52
|
-
|
53
|
-
end
|
54
|
-
end
|
55
|
-
end
|
@@ -1,155 +0,0 @@
|
|
1
|
-
require 'fileutils'
|
2
|
-
require 'redis'
|
3
|
-
require 'json'
|
4
|
-
|
5
|
-
module Keen
|
6
|
-
module Storage
|
7
|
-
class Item
|
8
|
-
# Represents one item in the Redis queue.
|
9
|
-
|
10
|
-
def to_json(options)
|
11
|
-
@event_body.to_json
|
12
|
-
end
|
13
|
-
|
14
|
-
def to_s
|
15
|
-
self.to_json
|
16
|
-
end
|
17
|
-
|
18
|
-
def auth_token
|
19
|
-
@auth_token
|
20
|
-
end
|
21
|
-
|
22
|
-
def initialize(definition={})
|
23
|
-
@project_id = definition[:project_id]
|
24
|
-
@auth_token = definition[:auth_token]
|
25
|
-
@collection_name = definition[:collection_name]
|
26
|
-
@event_body = definition[:event_body]
|
27
|
-
|
28
|
-
# TODO: type checking?
|
29
|
-
end
|
30
|
-
|
31
|
-
def save
|
32
|
-
handler = Keen::Storage::RedisHandler.new
|
33
|
-
handler.add_event(self)
|
34
|
-
end
|
35
|
-
|
36
|
-
end
|
37
|
-
|
38
|
-
class ProjectBatch
|
39
|
-
# Represents a batch of records (for a given project)
|
40
|
-
def initialize(project_id, auth_token)
|
41
|
-
@project_id = project_id
|
42
|
-
@auth_token = auth_token
|
43
|
-
@items = []
|
44
|
-
end
|
45
|
-
|
46
|
-
def add_item(item)
|
47
|
-
@items.push(item)
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
class RedisHandler
|
52
|
-
|
53
|
-
# Keys
|
54
|
-
# ----
|
55
|
-
|
56
|
-
def global_key_prefix
|
57
|
-
"keen-" + Keen::VERSION
|
58
|
-
end
|
59
|
-
|
60
|
-
def active_queue_key
|
61
|
-
"#{global_key_prefix}.active_queue_key"
|
62
|
-
end
|
63
|
-
|
64
|
-
def processing_queue_key
|
65
|
-
"#{global_key_prefix}.processing_queue_key"
|
66
|
-
end
|
67
|
-
|
68
|
-
def failed_queue_key
|
69
|
-
"#{global_key_prefix}.failed_queue_key"
|
70
|
-
end
|
71
|
-
|
72
|
-
def lock_active_queue_key
|
73
|
-
"lock" + active_queue_key
|
74
|
-
end
|
75
|
-
|
76
|
-
def lock_active_queue
|
77
|
-
# TODO: add locking
|
78
|
-
key = lock_active_queue_key
|
79
|
-
end
|
80
|
-
|
81
|
-
def unlock_active_queue
|
82
|
-
# TODO: add locking
|
83
|
-
key = lock_active_queue_key
|
84
|
-
end
|
85
|
-
|
86
|
-
def record_event(collection_name, event_body)
|
87
|
-
# TODO this is the main public method!
|
88
|
-
# must write..
|
89
|
-
end
|
90
|
-
|
91
|
-
def handle_prior_failures
|
92
|
-
# TODO consume the failed_queue and do something with it (loggly? retry? flat file?)
|
93
|
-
end
|
94
|
-
|
95
|
-
def initialize
|
96
|
-
@redis = Redis.new
|
97
|
-
end
|
98
|
-
|
99
|
-
def get_active_queue_contents
|
100
|
-
# get the list of jsonified hashes back:
|
101
|
-
list = @redis.get active_queue_key
|
102
|
-
|
103
|
-
if not list
|
104
|
-
[]
|
105
|
-
end
|
106
|
-
end
|
107
|
-
|
108
|
-
def flush_active_queue
|
109
|
-
@redis.del active_queue_key
|
110
|
-
end
|
111
|
-
|
112
|
-
def set_processing_queue(queue)
|
113
|
-
end
|
114
|
-
|
115
|
-
def process_queue
|
116
|
-
handle_prior_failures
|
117
|
-
|
118
|
-
lock_active_queue
|
119
|
-
|
120
|
-
queue = get_active_queue_contents
|
121
|
-
|
122
|
-
flush_active_queue
|
123
|
-
|
124
|
-
set_processing_queue(queue)
|
125
|
-
|
126
|
-
unlock_active_queue
|
127
|
-
|
128
|
-
# translate the queue strings into Items
|
129
|
-
items = queue.map {|json| Keen::Storage::Item.new(JSON.parse json)}.compact
|
130
|
-
|
131
|
-
collated = collate_items(items)
|
132
|
-
end
|
133
|
-
|
134
|
-
def collate_items(queue)
|
135
|
-
collated = {}
|
136
|
-
|
137
|
-
# traverse backwards so the most recent auth tokens take precedent:
|
138
|
-
queue.reverse_each do |item_hash|
|
139
|
-
if not collated.has_key? item.project_id
|
140
|
-
collated[item.project_id] = {}
|
141
|
-
end
|
142
|
-
|
143
|
-
if not collated[item.project_id].has_key? item.collection_name
|
144
|
-
collated[item.project_id][item.collection_name] = Keen::Storage::ProjectBatch.new
|
145
|
-
end
|
146
|
-
|
147
|
-
collated[item.project_id][item.collection_name].add_item(item)
|
148
|
-
end
|
149
|
-
|
150
|
-
collated
|
151
|
-
end
|
152
|
-
|
153
|
-
end
|
154
|
-
end
|
155
|
-
end
|