keen 0.0.2 → 0.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/Gemfile CHANGED
@@ -1 +1,3 @@
1
1
  source :rubygems
2
+ gem 'rspec'
3
+ gem 'fakeweb'
data/bin/keen_send CHANGED
@@ -1,11 +1,11 @@
1
1
  #!/usr/bin/env ruby
2
- $LOAD_PATH << File.join(File.dirname(__FILE__), '..', 'lib')
3
- require 'rubygems'
4
- require 'optparse'
5
- require 'keen'
2
+ $LOAD_PATH << File.join(File.dirname(__FILE__), "..", "lib")
3
+ require "rubygems"
4
+ require "optparse"
5
+ require "keen"
6
6
 
7
7
  options = {
8
- :env => 'production',
8
+ :env => "production",
9
9
  }
10
10
 
11
11
  required = [
data/examples.rb CHANGED
@@ -1,3 +1,4 @@
1
+ require 'rubygems'
1
2
  require 'keen'
2
3
 
3
4
  # Get these from the keen.io website:
@@ -1,3 +1,6 @@
1
+ require 'json'
2
+ require 'keen'
3
+
1
4
  # Filters added to this controller apply to all controllers in the application.
2
5
  # Likewise, all the methods added will be available for all controllers.
3
6
 
data/keen.gemspec CHANGED
@@ -22,9 +22,10 @@ Gem::Specification.new do |s|
22
22
  # s.add_development_dependency "rspec"
23
23
  # s.add_runtime_dependency "rest-client"
24
24
 
25
- s.add_dependency('multi_json', '>= 1.0.3')
25
+ s.add_dependency('json', '>= 1.6.5')
26
+ s.add_dependency('fakeweb', '>= 1.3.0')
27
+ s.add_dependency('rspec', '>= 2.9.0')
26
28
  s.add_dependency('system_timer', '>= 1.2.4')
27
- s.add_dependency('httparty', '>= 0.8.1')
28
29
  s.add_dependency('redis', '>= 2.2.2')
29
30
 
30
31
  # took these from Twilio library:
data/lib/keen.rb CHANGED
@@ -1,3 +1,6 @@
1
1
  require 'keen/client'
2
2
  require 'keen/version'
3
- require 'keen/storage/flat_file_handler'
3
+ require 'keen/utils'
4
+ require 'keen/async/job'
5
+ require 'keen/async/worker'
6
+ require 'keen/async/storage/redis_handler'
@@ -0,0 +1,70 @@
1
+ require 'keen/async/storage/redis_handler'
2
+
3
+ module Keen
4
+ module Async
5
+ class Job
6
+ # Represents one job.
7
+ #
8
+ #
9
+
10
+ attr_accessor :project_id, :auth_token, :collection_name, :event_body
11
+
12
+ def to_json(options=nil)
13
+ @definition.to_json
14
+ end
15
+
16
+
17
+ def to_s
18
+ self.to_json
19
+ end
20
+
21
+ def initialize(handler, definition={})
22
+
23
+ load_definition(definition)
24
+ @handler = handler
25
+
26
+ end
27
+
28
+ def load_definition(definition)
29
+
30
+ definition = Keen::Utils.symbolize_keys(definition)
31
+
32
+ # define some key lists:
33
+ required_keys = [:project_id, :auth_token, :collection_name, :event_body]
34
+ optional_keys = []
35
+ all_keys = required_keys + optional_keys
36
+
37
+
38
+ # don't allow them to send nil values for anything
39
+ definition.each do |key, value|
40
+ # reject unrecognized keys:
41
+ raise "Unrecognized key: #{key}" unless all_keys.include? key.to_sym
42
+ end
43
+
44
+
45
+ required_keys.each do |key|
46
+ value = definition[key]
47
+
48
+ raise "You sent a nil value for the #{key}." if value.nil?
49
+ end
50
+
51
+
52
+ all_keys.each do |key|
53
+ value = definition[key]
54
+ self.instance_variable_set("@#{key}", value)
55
+ end
56
+
57
+ @definition = definition
58
+
59
+
60
+ end
61
+
62
+ def save
63
+ @handler.record_job(self)
64
+ end
65
+
66
+ end
67
+ end
68
+ end
69
+
70
+
@@ -0,0 +1,57 @@
1
+ require 'fileutils'
2
+ require 'base64'
3
+ require 'zlib'
4
+
5
+
6
+
7
+ module Keen
8
+ module Async
9
+ module Storage
10
+ class FlatFileHandler
11
+
12
+ # Paths
13
+ # -----
14
+
15
+ # Where new events go as they come in:
16
+ ACTIVE = "active.txt"
17
+
18
+ # The temporary file that will store records during processing:
19
+ PROCESSING = "processing.txt"
20
+
21
+ # Records are put here if processing fails:
22
+ FAILED = "failed.txt"
23
+
24
+ def initialize(filepath)
25
+ @filepath = filepath
26
+
27
+ is_directory?
28
+ is_writable?
29
+ is_readable?
30
+ end
31
+
32
+ def dump_contents
33
+ # move
34
+ end
35
+
36
+ def is_writable?
37
+ if not FileTest.writable? @filepath
38
+ raise "Can't write to file: " + @filepath
39
+ end
40
+ end
41
+
42
+ def is_readable?
43
+ if not FileTest.readable? @filepath
44
+ raise "Can't read from file: " + @filepath
45
+ end
46
+ end
47
+
48
+ def is_directory?
49
+ if not FileTest.directory? @filepath
50
+ raise "Can't read from file: " + @filepath
51
+ end
52
+ end
53
+
54
+ end
55
+ end
56
+ end
57
+ end
@@ -0,0 +1,136 @@
1
+ require 'fileutils'
2
+ require 'redis'
3
+ require 'json'
4
+ require 'time'
5
+
6
+ module Keen
7
+ module Async
8
+ module Storage
9
+ class RedisHandler
10
+
11
+ # Keys
12
+ # ----
13
+
14
+ def global_key_prefix
15
+ "keen_415" + Keen::VERSION
16
+ end
17
+
18
+ def active_queue_key
19
+ "#{global_key_prefix}.active_queue_key"
20
+ end
21
+
22
+ def failed_queue_key
23
+ "#{global_key_prefix}.failed_queue_key"
24
+ end
25
+
26
+ def add_to_active_queue(value)
27
+ @redis.lpush active_queue_key, value
28
+ puts "added #{value} to active queue; length is now #{@redis.llen active_queue_key}"
29
+ end
30
+
31
+ def record_job(job)
32
+ add_to_active_queue JSON.generate(job)
33
+ end
34
+
35
+ def handle_prior_failures
36
+ # TODO consume the failed_queue and do something with it (loggly? retry? flat file?)
37
+ end
38
+
39
+ def initialize
40
+ @redis = Redis.new
41
+ end
42
+
43
+ def count_active_queue
44
+ @redis.llen active_queue_key
45
+ end
46
+
47
+ def get_collated_jobs(how_many)
48
+
49
+ # Returns a hash of the next `how_many` jobs, indexed on project_id and then collection_name.
50
+ #
51
+ # It looks like this:
52
+ #
53
+ # collated = {
54
+ # "4f5775ad163d666a6100000e" => {
55
+ # "clicks" => [
56
+ # Keen::Storage::Job.new({
57
+ # :project_id => "4f5775ad163d666a6100000e",
58
+ # :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
59
+ # :collection_name => "clicks",
60
+ # :event_body => {:user_id => "12345"},
61
+ # }),
62
+ # Keen::Storage::Job.new({
63
+ # :project_id => "4f5775ad163d666a6100000e",
64
+ # :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
65
+ # :collection_name => "clicks",
66
+ # :event_body => {:user_id => "12345"},
67
+ # }),
68
+ # Keen::Storage::Job.new({
69
+ # :project_id => "4f5775ad163d666a6100000e",
70
+ # :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
71
+ # :collection_name => "clicks",
72
+ # :event_body => {:user_id => "12345"},
73
+ # }),
74
+ # ],
75
+ # "purchases" => [
76
+ # Keen::Storage::Job.new({
77
+ # :project_id => "4f5775ad163d666a6100000e",
78
+ # :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
79
+ # :collection_name => "purchases",
80
+ # :event_body => {:user_id => "12345"},
81
+ # }),
82
+ # Keen::Storage::Job.new({
83
+ # :project_id => "4f5775ad163d666a6100000e",
84
+ # :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
85
+ # :collection_name => "purchases",
86
+ # :event_body => {:user_id => "12345"},
87
+ # }),
88
+ # Keen::Storage::Job.new({
89
+ # :project_id => "4f5775ad163d666a6100000e",
90
+ # :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
91
+ # :collection_name => "purchases",
92
+ # :event_body => {:user_id => "12345"},
93
+ # }),
94
+ # ],
95
+ # }
96
+ # }
97
+
98
+ handle_prior_failures
99
+
100
+ key = active_queue_key
101
+
102
+ jobs = []
103
+
104
+ how_many.times do
105
+ this = @redis.lpop key
106
+ jobs.push JSON.parse this
107
+ end
108
+
109
+ collate_jobs(jobs)
110
+ end
111
+
112
+ def collate_jobs(queue)
113
+ collated = {}
114
+
115
+ queue.each do |job_hash|
116
+
117
+ job = Keen::Async::Job.new(self, job_hash)
118
+
119
+ if not collated.has_key? job.project_id
120
+ collated[job.project_id] = {}
121
+ end
122
+
123
+ if not collated[job.project_id].has_key? job.collection_name
124
+ collated[job.project_id][job.collection_name] = []
125
+ end
126
+
127
+ collated[job.project_id][job.collection_name].push(job)
128
+ end
129
+
130
+ collated
131
+ end
132
+
133
+ end
134
+ end
135
+ end
136
+ end
@@ -0,0 +1,81 @@
1
+ require "keen/async/storage/redis_handler"
2
+ require "net/http"
3
+ require "net/https"
4
+
5
+
6
+
7
+ module Keen
8
+
9
+ module Async
10
+
11
+ # How many events should we send over the wire at a time?
12
+ BATCH_SIZE = 100
13
+ SSL_CA_FILE = File.dirname(__FILE__) + '../../../conf/cacert.pem'
14
+
15
+ class Worker
16
+
17
+ def initialize(handler)
18
+ @handler = handler
19
+ end
20
+
21
+ def batch_url(project_id)
22
+ if not project_id
23
+ raise "Missing project_id."
24
+ end
25
+ "https://api.keen.io/1.0/projects/#{project_id}/_events"
26
+ end
27
+
28
+ def process_queue
29
+
30
+ queue_length = @handler.count_active_queue
31
+
32
+ batch_size = Keen::Async::BATCH_SIZE
33
+
34
+ num_batches = queue_length / batch_size
35
+
36
+ num_batches.times do
37
+ collated = @handler.get_collated_jobs(batch_size)
38
+ collated.each do |project_id, batch|
39
+ send_batch(project_id, batch)
40
+ end
41
+ end
42
+ end
43
+
44
+ def send_batch(project_id, batch)
45
+ if not batch
46
+ return
47
+ end
48
+
49
+ first_key = batch.keys[0]
50
+ job_list = batch[first_key]
51
+ auth_token = job_list[0].auth_token
52
+
53
+ uri = URI.parse(batch_url(project_id))
54
+ http = Net::HTTP.new(uri.host, uri.port)
55
+ http.use_ssl = true
56
+ http.ca_file = Keen::Async::SSL_CA_FILE
57
+ http.verify_mode = OpenSSL::SSL::VERIFY_PEER
58
+ http.verify_depth = 5
59
+
60
+ request = Net::HTTP::Post.new(uri.path)
61
+ request.body = batch.to_json
62
+ request["Content-Type"] = "application/json"
63
+ request["Authorization"] = auth_token
64
+
65
+ response = http.request(request)
66
+
67
+ #response = Net::HTTP.start(uri.host, uri.port) {|http|
68
+ #http.request(request)
69
+ #}
70
+
71
+ puts response
72
+ # TODO: If something fails, we should move the job to the
73
+ # prior_failures queue by calling, perhaps:
74
+ #
75
+ # @handler.log_failed_job(job)
76
+ end
77
+
78
+ end
79
+ end
80
+
81
+ end
data/lib/keen/client.rb CHANGED
@@ -1,123 +1,79 @@
1
- require 'keen/storage/redis_handler'
2
- require 'json'
3
- require "net/http"
1
+ require "keen/async/storage/redis_handler"
2
+ require "keen/async/job"
3
+ require "json"
4
4
  require "uri"
5
+ require "time"
5
6
 
6
7
 
7
8
  module Keen
9
+
8
10
  class Client
9
11
 
10
- def self.batch_url(project_id)
11
- "http://api.keen.io/1.0/projects/#{project_id}/_events"
12
- end
13
-
14
- def initialize(project_id, auth_token)
15
- end
12
+ attr_accessor :storage_handler, :project_id, :auth_token
16
13
 
17
- def add_event(collection_name, event_body)
18
- validate_collection_name(collection_name)
14
+ def initialize(project_id, auth_token, options = {})
19
15
 
20
- end
16
+ raise "project_id must be string" unless project_id.kind_of? String
17
+ raise "auth_token must be string" unless auth_token.kind_of? String
21
18
 
22
- def self.process_queue(options)
23
- mode = options[:storage_mode].to_sym
24
- case mode
25
- when :flat_file
26
- self.process_queue_from_flat_file(options)
27
- when :redis
28
- self.process_queue_from_redis(options)
29
- else
30
- raise "Unknown storage_mode sent: `#{mode}`"
31
- end
19
+ default_options = {
20
+ :storage_mode => :redis,
21
+ }
22
+
23
+ options = default_options.update(options)
24
+
25
+ @project_id = project_id
26
+ @auth_token = auth_token
27
+ @storage_mode = options[:storage_mode]
32
28
  end
33
29
 
34
- def self.process_queue_from_redis(options)
35
- require 'keen/storage/redis_handler'
36
- handler = Keen::Storage::RedisHandler.new
37
- collated = handler.process_queue
38
-
39
- # TODO: remove this mock:
40
- collated = {
41
- "4f5775ad163d666a6100000e" => {
42
- "clicks" => [
43
- Keen::Storage::Item.new({
44
- :project_id => "4f5775ad163d666a6100000e",
45
- :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
46
- :collection_name => "clicks",
47
- :event_body => {:user_id => "12345"},
48
- }),
49
- Keen::Storage::Item.new({
50
- :project_id => "4f5775ad163d666a6100000e",
51
- :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
52
- :collection_name => "clicks",
53
- :event_body => {:user_id => "12345"},
54
- }),
55
- Keen::Storage::Item.new({
56
- :project_id => "4f5775ad163d666a6100000e",
57
- :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
58
- :collection_name => "clicks",
59
- :event_body => {:user_id => "12345"},
60
- }),
61
- ],
62
- "purchases" => [
63
- Keen::Storage::Item.new({
64
- :project_id => "4f5775ad163d666a6100000e",
65
- :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
66
- :collection_name => "purchases",
67
- :event_body => {:user_id => "12345"},
68
- }),
69
- Keen::Storage::Item.new({
70
- :project_id => "4f5775ad163d666a6100000e",
71
- :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
72
- :collection_name => "purchases",
73
- :event_body => {:user_id => "12345"},
74
- }),
75
- Keen::Storage::Item.new({
76
- :project_id => "4f5775ad163d666a6100000e",
77
- :auth_token => "a5d4eaf432914823a94ecd7e0cb547b9",
78
- :collection_name => "purchases",
79
- :event_body => {:user_id => "12345"},
80
- }),
81
- ],
82
- }
83
- }
30
+ def handler
31
+
32
+ unless @storage_handler
33
+ mode = @storage_mode
34
+
35
+ case mode
36
+ when :redis
37
+ @storage_handler = Keen::Async::Storage::RedisHandler.new
38
+ else
39
+ raise "Unknown storage_mode sent to client: `#{mode}`"
40
+ end
84
41
 
85
- collated.each do |project_id, batch|
86
- self.send_batch(project_id, batch)
87
42
  end
43
+
44
+ @storage_handler
88
45
  end
89
-
90
- def self.send_batch(project_id, batch)
91
- if not batch
92
- return
93
- end
94
46
 
95
- first_key = batch.keys[0]
96
- item_list = batch[first_key]
97
- puts item_list[0].class
98
- auth_token = item_list[0].auth_token
99
-
100
- uri = URI.parse(self.batch_url(project_id))
47
+ def add_event(collection_name, event_body, timestamp=nil)
48
+ #
49
+ # `collection_name` should be a string
50
+ #
51
+ # `event_body` should be a JSON-serializable hash
52
+ #
53
+ # `timestamp` is optional. If sent, it should be a Time instance.
54
+ # If it's not sent, we'll use the current time.
101
55
 
102
- request = Net::HTTP::Post.new(uri.path)
103
- request.body = batch.to_json
104
- request["Content-Type"] = "application/json"
105
- request["Authorization"] = auth_token
56
+ validate_collection_name(collection_name)
106
57
 
107
- response = Net::HTTP.start(uri.host, uri.port) {|http|
108
- http.request(request)
109
- }
58
+ unless timestamp
59
+ timestamp = Time.now
60
+ end
110
61
 
111
- puts response
62
+ event_body[:_timestamp] = timestamp.utc.iso8601
112
63
 
113
- # TODO DK: need to send this batch of Keen::Storage::Item instances
114
- # to API! we can just use the first auth_token we find on an Item.
115
- # If something fails, stick the item into the prior_failures queue
116
- # using push
64
+ job = Keen::Async::Job.new(handler, {
65
+ :project_id => @project_id,
66
+ :auth_token => @auth_token,
67
+ :collection_name => collection_name,
68
+ :event_body => event_body,
69
+ })
70
+
71
+ job.save
117
72
  end
118
73
 
119
- def self.process_queue_from_flat_file(options)
120
- raise "this feature isn't supported yet!!!"
74
+ def validate_collection_name(collection_name)
75
+ # TODO
121
76
  end
77
+
122
78
  end
123
79
  end
data/lib/keen/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Keen
2
- VERSION = "0.0.2"
2
+ VERSION = "0.0.4"
3
3
  end
data/test/keen_spec.rb CHANGED
@@ -1,7 +1,7 @@
1
- $LOAD_PATH << File.join(File.dirname(__FILE__), '..', 'lib')
1
+ $LOAD_PATH << File.join(File.dirname(__FILE__), "..", "lib")
2
2
 
3
- require 'keen'
4
- require 'fakeweb'
3
+ require "keen"
4
+ require "fakeweb"
5
5
 
6
6
  describe Keen::Client do
7
7
 
@@ -9,6 +9,24 @@ describe Keen::Client do
9
9
  before :all do
10
10
  FakeWeb.register_uri(:any, %r/https:\/\/api.keen.io\//, :body => '{"message": "You tried to reach Keen"}')
11
11
  end
12
+
13
+ describe "#add_event" do
14
+ project_id = "4f5775ad163d666a6100000e"
15
+ auth_token = "a5d4eaf432914823a94ecd7e0cb547b9"
16
+
17
+ keen = Keen::Client.new(project_id, auth_token, :storage_mode => :redis)
18
+
19
+ 310.times do
20
+ keen.add_event("rspec_clicks", {
21
+ :hi => "you",
22
+ })
23
+ end
24
+
25
+ worker = Keen::Async::Worker.new(handler = keen.storage_handler)
26
+
27
+ worker.process_queue
28
+
29
+ end
12
30
 
13
31
  # TODO spec it out, lazy bones!
14
32
  #
metadata CHANGED
@@ -1,13 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: keen
3
3
  version: !ruby/object:Gem::Version
4
- hash: 27
4
+ hash: 23
5
5
  prerelease:
6
6
  segments:
7
7
  - 0
8
8
  - 0
9
- - 2
10
- version: 0.0.2
9
+ - 4
10
+ version: 0.0.4
11
11
  platform: ruby
12
12
  authors:
13
13
  - Kyle Wild
@@ -15,61 +15,77 @@ autorequire:
15
15
  bindir: bin
16
16
  cert_chain: []
17
17
 
18
- date: 2012-03-16 00:00:00 -05:00
18
+ date: 2012-03-19 00:00:00 -05:00
19
19
  default_executable:
20
20
  dependencies:
21
21
  - !ruby/object:Gem::Dependency
22
- name: multi_json
22
+ name: json
23
23
  prerelease: false
24
24
  requirement: &id001 !ruby/object:Gem::Requirement
25
25
  none: false
26
26
  requirements:
27
27
  - - ">="
28
28
  - !ruby/object:Gem::Version
29
- hash: 17
29
+ hash: 5
30
30
  segments:
31
31
  - 1
32
- - 0
33
- - 3
34
- version: 1.0.3
32
+ - 6
33
+ - 5
34
+ version: 1.6.5
35
35
  type: :runtime
36
36
  version_requirements: *id001
37
37
  - !ruby/object:Gem::Dependency
38
- name: system_timer
38
+ name: fakeweb
39
39
  prerelease: false
40
40
  requirement: &id002 !ruby/object:Gem::Requirement
41
41
  none: false
42
42
  requirements:
43
43
  - - ">="
44
44
  - !ruby/object:Gem::Version
45
- hash: 23
45
+ hash: 27
46
46
  segments:
47
47
  - 1
48
- - 2
49
- - 4
50
- version: 1.2.4
48
+ - 3
49
+ - 0
50
+ version: 1.3.0
51
51
  type: :runtime
52
52
  version_requirements: *id002
53
53
  - !ruby/object:Gem::Dependency
54
- name: httparty
54
+ name: rspec
55
55
  prerelease: false
56
56
  requirement: &id003 !ruby/object:Gem::Requirement
57
57
  none: false
58
58
  requirements:
59
59
  - - ">="
60
60
  - !ruby/object:Gem::Version
61
- hash: 61
61
+ hash: 43
62
62
  segments:
63
+ - 2
64
+ - 9
63
65
  - 0
64
- - 8
65
- - 1
66
- version: 0.8.1
66
+ version: 2.9.0
67
67
  type: :runtime
68
68
  version_requirements: *id003
69
69
  - !ruby/object:Gem::Dependency
70
- name: redis
70
+ name: system_timer
71
71
  prerelease: false
72
72
  requirement: &id004 !ruby/object:Gem::Requirement
73
+ none: false
74
+ requirements:
75
+ - - ">="
76
+ - !ruby/object:Gem::Version
77
+ hash: 23
78
+ segments:
79
+ - 1
80
+ - 2
81
+ - 4
82
+ version: 1.2.4
83
+ type: :runtime
84
+ version_requirements: *id004
85
+ - !ruby/object:Gem::Dependency
86
+ name: redis
87
+ prerelease: false
88
+ requirement: &id005 !ruby/object:Gem::Requirement
73
89
  none: false
74
90
  requirements:
75
91
  - - ">="
@@ -81,7 +97,7 @@ dependencies:
81
97
  - 2
82
98
  version: 2.2.2
83
99
  type: :runtime
84
- version_requirements: *id004
100
+ version_requirements: *id005
85
101
  description: See the github repo or examples.rb for usage information.
86
102
  email:
87
103
  - kyle@keen.io
@@ -175,9 +191,11 @@ files:
175
191
  - examples/rails_2/Gemfile
176
192
  - keen.gemspec
177
193
  - lib/keen.rb
194
+ - lib/keen/async/job.rb
195
+ - lib/keen/async/storage/flat_file_handler.rb
196
+ - lib/keen/async/storage/redis_handler.rb
197
+ - lib/keen/async/worker.rb
178
198
  - lib/keen/client.rb
179
- - lib/keen/storage/flat_file_handler.rb
180
- - lib/keen/storage/redis_handler.rb
181
199
  - lib/keen/version.rb
182
200
  - test/keen_spec.rb
183
201
  has_rdoc: true
@@ -1,55 +0,0 @@
1
- require 'fileutils'
2
- require 'base64'
3
- require 'zlib'
4
-
5
-
6
-
7
- module Keen
8
- module Storage
9
- class FlatFileHandler
10
-
11
- # Paths
12
- # -----
13
-
14
- # Where new events go as they come in:
15
- ACTIVE = "active.txt"
16
-
17
- # The temporary file that will store records during processing:
18
- PROCESSING = "processing.txt"
19
-
20
- # Records are put here if processing fails:
21
- FAILED = "failed.txt"
22
-
23
- def initialize(filepath)
24
- @filepath = filepath
25
-
26
- is_directory?
27
- is_writable?
28
- is_readable?
29
- end
30
-
31
- def dump_contents
32
- # move
33
- end
34
-
35
- def is_writable?
36
- if not FileTest.writable? @filepath
37
- raise "Can't write to file: " + @filepath
38
- end
39
- end
40
-
41
- def is_readable?
42
- if not FileTest.readable? @filepath
43
- raise "Can't read from file: " + @filepath
44
- end
45
- end
46
-
47
- def is_directory?
48
- if not FileTest.directory? @filepath
49
- raise "Can't read from file: " + @filepath
50
- end
51
- end
52
-
53
- end
54
- end
55
- end
@@ -1,155 +0,0 @@
1
- require 'fileutils'
2
- require 'redis'
3
- require 'json'
4
-
5
- module Keen
6
- module Storage
7
- class Item
8
- # Represents one item in the Redis queue.
9
-
10
- def to_json(options)
11
- @event_body.to_json
12
- end
13
-
14
- def to_s
15
- self.to_json
16
- end
17
-
18
- def auth_token
19
- @auth_token
20
- end
21
-
22
- def initialize(definition={})
23
- @project_id = definition[:project_id]
24
- @auth_token = definition[:auth_token]
25
- @collection_name = definition[:collection_name]
26
- @event_body = definition[:event_body]
27
-
28
- # TODO: type checking?
29
- end
30
-
31
- def save
32
- handler = Keen::Storage::RedisHandler.new
33
- handler.add_event(self)
34
- end
35
-
36
- end
37
-
38
- class ProjectBatch
39
- # Represents a batch of records (for a given project)
40
- def initialize(project_id, auth_token)
41
- @project_id = project_id
42
- @auth_token = auth_token
43
- @items = []
44
- end
45
-
46
- def add_item(item)
47
- @items.push(item)
48
- end
49
- end
50
-
51
- class RedisHandler
52
-
53
- # Keys
54
- # ----
55
-
56
- def global_key_prefix
57
- "keen-" + Keen::VERSION
58
- end
59
-
60
- def active_queue_key
61
- "#{global_key_prefix}.active_queue_key"
62
- end
63
-
64
- def processing_queue_key
65
- "#{global_key_prefix}.processing_queue_key"
66
- end
67
-
68
- def failed_queue_key
69
- "#{global_key_prefix}.failed_queue_key"
70
- end
71
-
72
- def lock_active_queue_key
73
- "lock" + active_queue_key
74
- end
75
-
76
- def lock_active_queue
77
- # TODO: add locking
78
- key = lock_active_queue_key
79
- end
80
-
81
- def unlock_active_queue
82
- # TODO: add locking
83
- key = lock_active_queue_key
84
- end
85
-
86
- def record_event(collection_name, event_body)
87
- # TODO this is the main public method!
88
- # must write..
89
- end
90
-
91
- def handle_prior_failures
92
- # TODO consume the failed_queue and do something with it (loggly? retry? flat file?)
93
- end
94
-
95
- def initialize
96
- @redis = Redis.new
97
- end
98
-
99
- def get_active_queue_contents
100
- # get the list of jsonified hashes back:
101
- list = @redis.get active_queue_key
102
-
103
- if not list
104
- []
105
- end
106
- end
107
-
108
- def flush_active_queue
109
- @redis.del active_queue_key
110
- end
111
-
112
- def set_processing_queue(queue)
113
- end
114
-
115
- def process_queue
116
- handle_prior_failures
117
-
118
- lock_active_queue
119
-
120
- queue = get_active_queue_contents
121
-
122
- flush_active_queue
123
-
124
- set_processing_queue(queue)
125
-
126
- unlock_active_queue
127
-
128
- # translate the queue strings into Items
129
- items = queue.map {|json| Keen::Storage::Item.new(JSON.parse json)}.compact
130
-
131
- collated = collate_items(items)
132
- end
133
-
134
- def collate_items(queue)
135
- collated = {}
136
-
137
- # traverse backwards so the most recent auth tokens take precedent:
138
- queue.reverse_each do |item_hash|
139
- if not collated.has_key? item.project_id
140
- collated[item.project_id] = {}
141
- end
142
-
143
- if not collated[item.project_id].has_key? item.collection_name
144
- collated[item.project_id][item.collection_name] = Keen::Storage::ProjectBatch.new
145
- end
146
-
147
- collated[item.project_id][item.collection_name].add_item(item)
148
- end
149
-
150
- collated
151
- end
152
-
153
- end
154
- end
155
- end