zermelo 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +16 -0
  3. data/.rspec +10 -0
  4. data/.travis.yml +27 -0
  5. data/Gemfile +20 -0
  6. data/LICENSE.txt +22 -0
  7. data/README.md +512 -0
  8. data/Rakefile +1 -0
  9. data/lib/zermelo/associations/association_data.rb +24 -0
  10. data/lib/zermelo/associations/belongs_to.rb +115 -0
  11. data/lib/zermelo/associations/class_methods.rb +244 -0
  12. data/lib/zermelo/associations/has_and_belongs_to_many.rb +128 -0
  13. data/lib/zermelo/associations/has_many.rb +120 -0
  14. data/lib/zermelo/associations/has_one.rb +109 -0
  15. data/lib/zermelo/associations/has_sorted_set.rb +124 -0
  16. data/lib/zermelo/associations/index.rb +50 -0
  17. data/lib/zermelo/associations/index_data.rb +18 -0
  18. data/lib/zermelo/associations/unique_index.rb +44 -0
  19. data/lib/zermelo/backends/base.rb +115 -0
  20. data/lib/zermelo/backends/influxdb_backend.rb +178 -0
  21. data/lib/zermelo/backends/redis_backend.rb +281 -0
  22. data/lib/zermelo/filters/base.rb +235 -0
  23. data/lib/zermelo/filters/influxdb_filter.rb +162 -0
  24. data/lib/zermelo/filters/redis_filter.rb +558 -0
  25. data/lib/zermelo/filters/steps/base_step.rb +22 -0
  26. data/lib/zermelo/filters/steps/diff_range_step.rb +17 -0
  27. data/lib/zermelo/filters/steps/diff_step.rb +17 -0
  28. data/lib/zermelo/filters/steps/intersect_range_step.rb +17 -0
  29. data/lib/zermelo/filters/steps/intersect_step.rb +17 -0
  30. data/lib/zermelo/filters/steps/limit_step.rb +17 -0
  31. data/lib/zermelo/filters/steps/offset_step.rb +17 -0
  32. data/lib/zermelo/filters/steps/sort_step.rb +17 -0
  33. data/lib/zermelo/filters/steps/union_range_step.rb +17 -0
  34. data/lib/zermelo/filters/steps/union_step.rb +17 -0
  35. data/lib/zermelo/locks/no_lock.rb +16 -0
  36. data/lib/zermelo/locks/redis_lock.rb +221 -0
  37. data/lib/zermelo/records/base.rb +62 -0
  38. data/lib/zermelo/records/class_methods.rb +127 -0
  39. data/lib/zermelo/records/collection.rb +14 -0
  40. data/lib/zermelo/records/errors.rb +24 -0
  41. data/lib/zermelo/records/influxdb_record.rb +35 -0
  42. data/lib/zermelo/records/instance_methods.rb +224 -0
  43. data/lib/zermelo/records/key.rb +19 -0
  44. data/lib/zermelo/records/redis_record.rb +27 -0
  45. data/lib/zermelo/records/type_validator.rb +20 -0
  46. data/lib/zermelo/version.rb +3 -0
  47. data/lib/zermelo.rb +102 -0
  48. data/spec/lib/zermelo/associations/belongs_to_spec.rb +6 -0
  49. data/spec/lib/zermelo/associations/has_many_spec.rb +6 -0
  50. data/spec/lib/zermelo/associations/has_one_spec.rb +6 -0
  51. data/spec/lib/zermelo/associations/has_sorted_set.spec.rb +6 -0
  52. data/spec/lib/zermelo/associations/index_spec.rb +6 -0
  53. data/spec/lib/zermelo/associations/unique_index_spec.rb +6 -0
  54. data/spec/lib/zermelo/backends/influxdb_backend_spec.rb +0 -0
  55. data/spec/lib/zermelo/backends/moneta_backend_spec.rb +0 -0
  56. data/spec/lib/zermelo/filters/influxdb_filter_spec.rb +0 -0
  57. data/spec/lib/zermelo/filters/redis_filter_spec.rb +0 -0
  58. data/spec/lib/zermelo/locks/redis_lock_spec.rb +170 -0
  59. data/spec/lib/zermelo/records/influxdb_record_spec.rb +258 -0
  60. data/spec/lib/zermelo/records/key_spec.rb +6 -0
  61. data/spec/lib/zermelo/records/redis_record_spec.rb +1426 -0
  62. data/spec/lib/zermelo/records/type_validator_spec.rb +6 -0
  63. data/spec/lib/zermelo/version_spec.rb +6 -0
  64. data/spec/lib/zermelo_spec.rb +6 -0
  65. data/spec/spec_helper.rb +67 -0
  66. data/spec/support/profile_all_formatter.rb +44 -0
  67. data/spec/support/uncolored_doc_formatter.rb +74 -0
  68. data/zermelo.gemspec +30 -0
  69. metadata +174 -0
@@ -0,0 +1,17 @@
1
+ require 'zermelo/filters/steps/base_step'
2
+
3
+ module Zermelo
4
+ module Filters
5
+ class Steps
6
+ class SortStep < Zermelo::Filters::Steps::BaseStep
7
+ def self.accepted_types
8
+ [:set, :sorted_set]
9
+ end
10
+
11
+ def self.returns_type
12
+ :list
13
+ end
14
+ end
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,17 @@
1
+ require 'zermelo/filters/steps/base_step'
2
+
3
+ module Zermelo
4
+ module Filters
5
+ class Steps
6
+ class UnionRangeStep < Zermelo::Filters::Steps::BaseStep
7
+ def self.accepted_types
8
+ [:sorted_set]
9
+ end
10
+
11
+ def self.returns_type
12
+ :sorted_set
13
+ end
14
+ end
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,17 @@
1
+ require 'zermelo/filters/steps/base_step'
2
+
3
+ module Zermelo
4
+ module Filters
5
+ class Steps
6
+ class UnionStep < Zermelo::Filters::Steps::BaseStep
7
+ def self.accepted_types
8
+ [:set, :sorted_set]
9
+ end
10
+
11
+ def self.returns_type
12
+ :set
13
+ end
14
+ end
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,16 @@
1
+
2
+ module Zermelo
3
+
4
+ module Locks
5
+
6
+ class NoLock
7
+
8
+ def lock(*record_klasses, &block)
9
+ yield
10
+ end
11
+
12
+ end
13
+
14
+ end
15
+
16
+ end
@@ -0,0 +1,221 @@
1
+
2
+ module Zermelo
3
+
4
+ class LockNotAcquired < StandardError; end
5
+
6
+ module Locks
7
+
8
+ class RedisLock
9
+
10
+ # Adapted from https://github.com/mlanett/redis-lock,
11
+ # now covers locking multiple keys at once
12
+
13
+ attr_accessor :expires_at, :life, :sleep_in_ms
14
+
15
+ def initialize
16
+ @owner_value = Thread.current.object_id
17
+ @life = 60
18
+ @timeout = 10
19
+ @sleep_in_ms = 125
20
+ end
21
+
22
+ def lock(*record_klasses, &block)
23
+ @keys = record_klasses.map{|k| k.send(:class_key) }.sort.map{|k| "#{k}::lock" }
24
+ do_lock_with_timeout(@timeout) or raise Zermelo::LockNotAcquired.new(@keys.join(", "))
25
+ result = true
26
+ if block
27
+ begin
28
+ result = (block.arity == 1) ? block.call(self) : block.call
29
+ # rescue Exception => e
30
+ # puts e.message
31
+ # puts e.backtrace.join("\n")
32
+ # raise e
33
+ ensure
34
+ release_lock
35
+ end
36
+ end
37
+ result
38
+ end
39
+
40
+ def extend_life( new_life )
41
+ do_extend( new_life ) or raise Zermelo::LockNotAcquired.new(@keys.join(", "))
42
+ end
43
+
44
+ def unlock
45
+ release_lock
46
+ end
47
+
48
+ private
49
+
50
+ def full_keys
51
+ @full_keys ||= @keys.map {|k| ["#{k}:owner", "#{k}:expiry"] }.flatten
52
+ end
53
+
54
+ def owner_keys
55
+ @owner_keys ||= @keys.map {|k| "#{k}:owner" }
56
+ end
57
+
58
+ def expiry_keys
59
+ @expiry_keys ||= @keys.map {|k| "#{k}:expiry" }
60
+ end
61
+
62
+ def do_lock_with_timeout( timeout )
63
+ locked = false
64
+ with_timeout(timeout) { locked = do_lock }
65
+ locked
66
+ end
67
+
68
+ # @returns true if locked, false otherwise
69
+ def do_lock( tries = 3 )
70
+ # We need to set both owner and expire at the same time
71
+ # If the existing lock is stale, we delete it and try again once
72
+
73
+ locked = nil
74
+
75
+ loop do
76
+ new_xval = Time.now.to_i + @life
77
+
78
+ lock_keyvals = @keys.map {|k| ["#{k}:owner", @owner_value,
79
+ "#{k}:expiry", new_xval] }.flatten
80
+
81
+ result = Zermelo.redis.msetnx(*lock_keyvals)
82
+
83
+ if [1, true].include?(result)
84
+ # log :debug, "do_lock() success"
85
+ @expires_at = new_xval
86
+ locked = true
87
+ break
88
+ else
89
+ # log :debug, "do_lock() failed"
90
+ # consider the possibility that this lock is stale
91
+ tries -= 1
92
+ next if tries > 0 && stale_key?
93
+ locked = false
94
+ break
95
+ end
96
+ end
97
+ locked
98
+ end
99
+
100
+ def do_extend( new_life )
101
+ # We use watch and a transaction to ensure we only change a lock we own
102
+ # The transaction fails if the watched variable changed
103
+ # Use my_owner = oval to make testing easier.
104
+ new_xval = Time.now.to_i + new_life
105
+ extended = false
106
+ with_watch( *owner_keys ) do
107
+ owners = Zermelo.redis.mget( *owner_keys )
108
+ if owners == ([@owner_value.to_s] * owner_keys.size)
109
+ result = Zermelo.redis.multi do |multi|
110
+ multi.mset( *(expiry_keys.zip( [new_xval] * expiry_keys.size)) )
111
+ end
112
+ if result == ['OK']
113
+ # log :debug, "do_extend() success"
114
+ @expires_at = new_xval
115
+ extended = true
116
+ end
117
+ end
118
+ end
119
+ extended
120
+ end
121
+
122
+ # Only actually deletes it if we own it.
123
+ # There may be strange cases where we fail to delete it, in which case expiration will solve the problem.
124
+ def release_lock
125
+ released = false
126
+ with_watch( *full_keys ) do
127
+ owners = Zermelo.redis.mget( *owner_keys )
128
+ if owners == ([@owner_value.to_s] * owner_keys.size)
129
+ result = Zermelo.redis.multi do |multi|
130
+ multi.del(*full_keys)
131
+ end
132
+ if result && (result.size == 1) && (result.first == full_keys.size)
133
+ released = true
134
+ end
135
+ end
136
+ end
137
+ released
138
+ end
139
+
140
+ def stale_key?
141
+ # Check if expiration exists and is it stale?
142
+ # If so, delete it.
143
+ # watch() all keys so we can detect if they change while we do this
144
+ # multi() will fail if keys have changed after watch()
145
+ # Thus, we snapshot consistency at the time of watch()
146
+ # Note: inside a watch() we get one and only one multi()
147
+ now = Time.now.to_i
148
+ stale = false
149
+ with_watch( *full_keys ) do
150
+
151
+ owners_expires = Zermelo.redis.mget(full_keys)
152
+
153
+ if owners_expires.each_slice(2).all? {|owner, expire| is_deletable?( owner, expire, now)}
154
+ result = Zermelo.redis.multi do |multi|
155
+ multi.del(*full_keys)
156
+ end
157
+ # If anything changed then multi() fails and returns nil
158
+ if result && (result.size == 1) && (result.first == owner_keys.size)
159
+ # log :info, "Deleted stale key from #{owner}"
160
+ stale = true
161
+ end
162
+ end
163
+ end # watch
164
+ stale
165
+ end
166
+
167
+ def locked?
168
+ now = Time.now.to_i
169
+ owners_expires = Zermelo.redis.mget(full_keys)
170
+ owners_expires && (owners_expires.size == (@keys.size * 2)) &&
171
+ owners_expires.each_slice(2).all? {|owner, expiration| is_locked?(owner, expiration, now)}
172
+ end
173
+
174
+ # returns true if the lock exists and is owned by the given owner
175
+ def is_locked?(owner, expiration, now)
176
+ (owner == @owner_value) && ! is_deletable?(owner, expiration, now)
177
+ end
178
+
179
+ # returns true if this is a broken or expired lock
180
+ def is_deletable?( owner, expiration, now)
181
+ expiration = expiration.to_i
182
+ (owner || (expiration > 0)) && (!owner || (expiration < now))
183
+ end
184
+
185
+ def with_watch( *args, &block )
186
+ Zermelo.redis.watch( *args )
187
+ begin
188
+ block.call
189
+ ensure
190
+ Zermelo.redis.unwatch
191
+ end
192
+ end
193
+
194
+ # Calls block until it returns true or times out.
195
+ # @param block should return true if successful, false otherwise
196
+ # @returns true if successful, false otherwise
197
+ def with_timeout( timeout, &block )
198
+ expire = Time.now + timeout.to_f
199
+ sleepy = @sleep_in_ms / 1000.to_f()
200
+ # this looks inelegant compared to while Time.now < expire, but does not oversleep
201
+ ret = nil
202
+ loop do
203
+ if block.call
204
+ ret = true
205
+ break
206
+ end
207
+ if (Time.now + sleepy) > expire
208
+ ret = false
209
+ break
210
+ end
211
+ sleep(sleepy)
212
+ # might like a different strategy, but general goal is not use 100% cpu while contending for a lock.
213
+ end
214
+ ret
215
+ end
216
+
217
+ end
218
+
219
+ end
220
+
221
+ end
@@ -0,0 +1,62 @@
1
+ require 'monitor'
2
+
3
+ require 'active_support/concern'
4
+ require 'active_support/core_ext/object/blank'
5
+ require 'active_support/inflector'
6
+ require 'active_model'
7
+
8
+ require 'zermelo/associations/class_methods'
9
+
10
+ require 'zermelo/records/instance_methods'
11
+ require 'zermelo/records/class_methods'
12
+ require 'zermelo/records/type_validator'
13
+
14
+ # TODO escape ids and index_keys -- shouldn't allow bare :
15
+
16
+ # TODO callbacks on before/after add/delete on association?
17
+
18
+ # TODO optional sort via Redis SORT, first/last for has_many via those
19
+
20
+ # TODO get DIFF working for exclusion case against ZSETs
21
+
22
+ module Zermelo
23
+
24
+ module Records
25
+
26
+ module Base
27
+
28
+ extend ActiveSupport::Concern
29
+
30
+ include Zermelo::Records::InstMethods
31
+
32
+ included do
33
+ include ActiveModel::AttributeMethods
34
+ extend ActiveModel::Callbacks
35
+ include ActiveModel::Dirty
36
+ include ActiveModel::Validations
37
+ include ActiveModel::Validations::Callbacks
38
+
39
+ # include ActiveModel::MassAssignmentSecurity
40
+
41
+ @lock = Monitor.new
42
+
43
+ extend Zermelo::Records::ClassMethods
44
+ extend Zermelo::Associations::ClassMethods
45
+
46
+ attr_accessor :attributes
47
+
48
+ define_model_callbacks :create, :update, :destroy
49
+
50
+ attribute_method_suffix "=" # attr_writers
51
+ # attribute_method_suffix "" # attr_readers # DEPRECATED
52
+
53
+ validates_with Zermelo::Records::TypeValidator
54
+
55
+ define_attributes :id => :string
56
+ end
57
+
58
+ end
59
+
60
+ end
61
+
62
+ end
@@ -0,0 +1,127 @@
1
+ require 'forwardable'
2
+ require 'securerandom'
3
+
4
+ require 'zermelo'
5
+
6
+ require 'zermelo/backends/influxdb_backend'
7
+ require 'zermelo/backends/redis_backend'
8
+
9
+ require 'zermelo/records/key'
10
+
11
+ module Zermelo
12
+
13
+ module Records
14
+
15
+ module ClassMethods
16
+
17
+ extend Forwardable
18
+
19
+ def_delegators :filter, :intersect, :union, :diff, :sort,
20
+ :find_by_id, :find_by_ids, :find_by_id!, :find_by_ids!,
21
+ :page, :all, :each, :collect, :map,
22
+ :select, :find_all, :reject, :destroy_all,
23
+ :ids, :count, :empty?, :exists?,
24
+ :associated_ids_for
25
+
26
+ def generate_id
27
+ return SecureRandom.uuid if SecureRandom.respond_to?(:uuid)
28
+ # from 1.9 stdlib
29
+ ary = SecureRandom.random_bytes(16).unpack("NnnnnN")
30
+ ary[2] = (ary[2] & 0x0fff) | 0x4000
31
+ ary[3] = (ary[3] & 0x3fff) | 0x8000
32
+ "%08x-%04x-%04x-%04x-%04x%08x" % ary
33
+ end
34
+
35
+ def add_id(id)
36
+ backend.add(ids_key, id.to_s)
37
+ end
38
+
39
+ def delete_id(id)
40
+ backend.delete(ids_key, id.to_s)
41
+ end
42
+
43
+ def attribute_types
44
+ ret = nil
45
+ @lock.synchronize do
46
+ ret = (@attribute_types ||= {}).dup
47
+ end
48
+ ret
49
+ end
50
+
51
+ def lock(*klasses, &block)
52
+ klasses += [self] unless klasses.include?(self)
53
+ backend.lock(*klasses, &block)
54
+ end
55
+
56
+ def transaction(&block)
57
+ failed = false
58
+
59
+ backend.begin_transaction
60
+
61
+ begin
62
+ yield
63
+ rescue Exception => e
64
+ backend.abort_transaction
65
+ p e.message
66
+ puts e.backtrace.join("\n")
67
+ failed = true
68
+ ensure
69
+ backend.commit_transaction unless failed
70
+ end
71
+
72
+ # TODO include exception info
73
+ raise "Transaction failed" if failed
74
+ end
75
+
76
+ def backend
77
+ raise "No data storage backend set for #{self.name}" if @backend.nil?
78
+ @backend
79
+ end
80
+
81
+ protected
82
+
83
+ def define_attributes(options = {})
84
+ options.each_pair do |key, value|
85
+ raise "Unknown attribute type ':#{value}' for ':#{key}'" unless
86
+ Zermelo.valid_type?(value)
87
+ self.define_attribute_methods([key])
88
+ end
89
+ @lock.synchronize do
90
+ (@attribute_types ||= {}).update(options)
91
+ end
92
+ end
93
+
94
+ def set_backend(backend_type)
95
+ @backend ||= case backend_type.to_sym
96
+ when :influxdb
97
+ Zermelo::Backends::InfluxDBBackend.new
98
+ when :redis
99
+ Zermelo::Backends::RedisBackend.new
100
+ end
101
+ end
102
+
103
+ private
104
+
105
+ def ids_key
106
+ @ids_key ||= Zermelo::Records::Key.new(:klass => class_key, :name => 'ids',
107
+ :type => :set, :object => :attribute)
108
+ end
109
+
110
+ def class_key
111
+ self.name.demodulize.underscore
112
+ end
113
+
114
+ def load(id)
115
+ object = self.new
116
+ object.load(id) ? object : nil
117
+ end
118
+
119
+ def filter
120
+ backend.filter(ids_key, self)
121
+ end
122
+
123
+ end
124
+
125
+ end
126
+
127
+ end
@@ -0,0 +1,14 @@
1
+ module Zermelo
2
+ module Records
3
+ # high-level abstraction for a set or list of record ids
4
+ class Collection
5
+ attr_reader :klass, :name, :type
6
+
7
+ def initialize(opts = {})
8
+ @klass = opts[:class]
9
+ @name = opts[:name]
10
+ @type = opts[:type]
11
+ end
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,24 @@
1
+ module Zermelo
2
+ module Records
3
+ module Errors
4
+
5
+ class RecordNotFound < RuntimeError
6
+ attr_reader :klass, :id
7
+
8
+ def initialize(k, i)
9
+ @klass = k
10
+ @id = i
11
+ end
12
+ end
13
+
14
+ class RecordsNotFound < RuntimeError
15
+ attr_reader :klass, :ids
16
+
17
+ def initialize(k, i)
18
+ @klass = k
19
+ @ids = i
20
+ end
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,35 @@
1
+ require 'active_support/concern'
2
+
3
+ require 'zermelo/records/base'
4
+
5
+ # a record is a row in a time series (named for the record class)
6
+
7
+ # all attributes are stored as fields in that row
8
+
9
+ # a save will delete (if required) and create the row
10
+
11
+ # if time field does not exist, this will be created automatically by influxdb
12
+
13
+ # indexing -- not really relevant until query building has been worked on, but
14
+ # everything in the influxdb query language should be supportable, maybe those
15
+ # just indicate what should be queryable?
16
+
17
+ # TODO: ensure time_precision is set for the incoming data
18
+
19
+ # class level values are in other time series (with similar names to the
20
+ # related redis sets)
21
+
22
+ module Zermelo
23
+ module Records
24
+ module InfluxDBRecord
25
+ extend ActiveSupport::Concern
26
+
27
+ include Zermelo::Records::Base
28
+
29
+ included do
30
+ set_backend :influxdb
31
+ end
32
+
33
+ end
34
+ end
35
+ end