redlics 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +6 -0
- data/.travis.yml +24 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +21 -0
- data/README.md +64 -0
- data/Rakefile +9 -0
- data/lib/redlics.rb +94 -0
- data/lib/redlics/config.rb +62 -0
- data/lib/redlics/connection.rb +79 -0
- data/lib/redlics/counter.rb +94 -0
- data/lib/redlics/exception.rb +30 -0
- data/lib/redlics/granularity.rb +49 -0
- data/lib/redlics/key.rb +244 -0
- data/lib/redlics/lua/script.lua +82 -0
- data/lib/redlics/operators.rb +51 -0
- data/lib/redlics/query.rb +234 -0
- data/lib/redlics/query/operation.rb +135 -0
- data/lib/redlics/time_frame.rb +110 -0
- data/lib/redlics/tracker.rb +64 -0
- data/lib/redlics/version.rb +4 -0
- data/redlics.gemspec +33 -0
- data/test/redlics_test.rb +13 -0
- metadata +167 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 02052cc441c6a1ed900aca26b965f6a6bab00638
|
4
|
+
data.tar.gz: 2f372d92fec7e58e7cda634efff68c65f16bfa5a
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: ff22b8916fa232fe611123d8042ae0bed4954ff3120e499854243cc77c467b055f27f2b3971e8d4a40d534ae301cf5bd2ccbb69d6c56a1275853aec005f32f6b
|
7
|
+
data.tar.gz: 76b768e2b2a5e323186e8215e94554fcd403cf8547605d3629e6e7b99e60c1f47bd81146ce0003547e6fd26dbd6499d1bd3ffd29e8ef6c1fe5aab0a04def5fad
|
data/.gitignore
ADDED
data/.travis.yml
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
sudo: false
|
2
|
+
bundler_args: --jobs=3 --retry=3
|
3
|
+
language: ruby
|
4
|
+
cache: bundler
|
5
|
+
before_install:
|
6
|
+
- gem update --remote bundler
|
7
|
+
rvm:
|
8
|
+
- 2.0.0
|
9
|
+
- 2.1
|
10
|
+
- 2.2
|
11
|
+
- jruby-9.0.0.0
|
12
|
+
- ruby-head
|
13
|
+
- jruby-head
|
14
|
+
matrix:
|
15
|
+
allow_failures:
|
16
|
+
- rvm: ruby-head
|
17
|
+
- rvm: jruby-head
|
18
|
+
fast_finish: true
|
19
|
+
notifications:
|
20
|
+
email:
|
21
|
+
on_success: change
|
22
|
+
on_failure: always
|
23
|
+
script:
|
24
|
+
- bundle exec rake
|
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2016 Phlegx Systems OG
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,64 @@
|
|
1
|
+
# Redlics
|
2
|
+
|
3
|
+
[](https://rubygems.org/gems/redlics)
|
4
|
+
[](https://travis-ci.org/phlegx/redlics)
|
5
|
+
[](https://codeclimate.com/github/phlegx/redlics)
|
6
|
+
[](http://inch-ci.org/github/phlegx/redlics)
|
7
|
+
[](https://gemnasium.com/phlegx/redlics)
|
8
|
+
[](http://opensource.org/licenses/MIT)
|
9
|
+
|
10
|
+
Redis analytics with tracks (using bitmaps) and counts (using buckets) encoding numbers in Redis keys and values.
|
11
|
+
|
12
|
+
## Installation
|
13
|
+
|
14
|
+
Add this line to your application's Gemfile:
|
15
|
+
|
16
|
+
```ruby
|
17
|
+
gem 'redlics'
|
18
|
+
```
|
19
|
+
|
20
|
+
And then execute:
|
21
|
+
|
22
|
+
$ bundle
|
23
|
+
|
24
|
+
Or install it yourself as:
|
25
|
+
|
26
|
+
$ gem install redlics
|
27
|
+
|
28
|
+
## Features
|
29
|
+
|
30
|
+
* Tracking with bitmaps
|
31
|
+
* Counting with buckets
|
32
|
+
* High configurable
|
33
|
+
* Encode/decode numbers in Redis keys and values
|
34
|
+
* Very less memory consumption in Redis
|
35
|
+
* Support of time frames
|
36
|
+
* Uses Lua script for better performance
|
37
|
+
* Plot option for tracks and counts
|
38
|
+
* Keeps Redis clean
|
39
|
+
* and many more
|
40
|
+
|
41
|
+
## Usage
|
42
|
+
|
43
|
+
Coming soon...
|
44
|
+
|
45
|
+
## Contributors
|
46
|
+
|
47
|
+
* Inspired by Minuteman [github.com/elcuervo/minuteman](https://github.com/elcuervo/minuteman).
|
48
|
+
* Inspired by Btrack [github.com/chenfisher/Btrack](https://github.com/chenfisher/Btrack).
|
49
|
+
* Inspired by Counterman [github.com/maccman/counterman](https://github.com/maccman/counterman).
|
50
|
+
|
51
|
+
## Contributing
|
52
|
+
|
53
|
+
1. Fork it ( https://github.com/[your-username]/redlics/fork )
|
54
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
55
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
56
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
57
|
+
5. Create a new Pull Request
|
58
|
+
|
59
|
+
## License
|
60
|
+
|
61
|
+
The MIT License
|
62
|
+
|
63
|
+
Copyright (c) 2015 Phlegx Systems OG
|
64
|
+
|
data/Rakefile
ADDED
data/lib/redlics.rb
ADDED
@@ -0,0 +1,94 @@
|
|
1
|
+
require 'redlics/version'
|
2
|
+
require 'redlics/config'
|
3
|
+
require 'redlics/exception'
|
4
|
+
require 'redlics/connection'
|
5
|
+
require 'redlics/granularity'
|
6
|
+
require 'redlics/key'
|
7
|
+
require 'redlics/time_frame'
|
8
|
+
require 'redlics/counter'
|
9
|
+
require 'redlics/tracker'
|
10
|
+
require 'redlics/operators'
|
11
|
+
require 'redlics/query'
|
12
|
+
require 'redlics/query/operation'
|
13
|
+
|
14
|
+
|
15
|
+
# Redlics namespace
|
16
|
+
module Redlics
|
17
|
+
|
18
|
+
extend self
|
19
|
+
|
20
|
+
# Delegate methods to right objects.
|
21
|
+
delegate :count, to: Counter
|
22
|
+
delegate :track, to: Tracker
|
23
|
+
delegate :analyze, to: Query
|
24
|
+
|
25
|
+
|
26
|
+
# Get or initialize the Redis connection.
|
27
|
+
# @return [Object] redis connection
|
28
|
+
def redis
|
29
|
+
redis_pool.with do |conn|
|
30
|
+
retryable = true
|
31
|
+
begin
|
32
|
+
conn
|
33
|
+
rescue Redis::BaseError => e
|
34
|
+
raise e unless config.silent
|
35
|
+
rescue Redis::CommandError => ex
|
36
|
+
(conn.disconnect!; retryable = false; retry) if retryable && ex.message =~ /READONLY/
|
37
|
+
raise unless config.silent
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
|
43
|
+
# Load Lua script file and arguments in Redis.
|
44
|
+
#
|
45
|
+
# @param file [String] absolute path to the Lua script file
|
46
|
+
# @param *args [Array] list of arguments for Redis evalsha
|
47
|
+
# @return [String] Lua script result
|
48
|
+
def script(file, *args)
|
49
|
+
begin
|
50
|
+
cache = LUA_CACHE[redis.client.options[:url]]
|
51
|
+
if cache.key?(file)
|
52
|
+
sha = cache[file]
|
53
|
+
else
|
54
|
+
src = File.read(file)
|
55
|
+
sha = redis.script(:load, src)
|
56
|
+
cache[file] = sha
|
57
|
+
end
|
58
|
+
redis.evalsha(sha, *args)
|
59
|
+
rescue RuntimeError
|
60
|
+
case $!.message
|
61
|
+
when Exception::ErrorPatterns::NOSCRIPT
|
62
|
+
LUA_CACHE[redis.client.options[:url]].clear
|
63
|
+
retry
|
64
|
+
else
|
65
|
+
raise $! unless config.silent
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
|
71
|
+
# Get or initialize Redlics config.
|
72
|
+
# @return [OpenStruct] Redlics configuration
|
73
|
+
def config
|
74
|
+
@config ||= Redlics::Config.new
|
75
|
+
end
|
76
|
+
|
77
|
+
|
78
|
+
# Set configuration of Redlics in a block.
|
79
|
+
# @return [OpenStruct] Redlics configuration
|
80
|
+
def configure
|
81
|
+
yield config if block_given?
|
82
|
+
end
|
83
|
+
|
84
|
+
|
85
|
+
private
|
86
|
+
|
87
|
+
# Get or initialize the Redis connection pool.
|
88
|
+
# @return [ConnectionPool] redis connection pool
|
89
|
+
def redis_pool
|
90
|
+
@redis ||= Redlics::Connection.create(config.to_h)
|
91
|
+
end
|
92
|
+
|
93
|
+
end
|
94
|
+
|
@@ -0,0 +1,62 @@
|
|
1
|
+
module Redlics
|
2
|
+
|
3
|
+
# Redlics constants.
|
4
|
+
LUA_CACHE = Hash.new { |h, k| h[k] = Hash.new }
|
5
|
+
LUA_SCRIPT = File.expand_path('../lua/script.lua', __FILE__).freeze
|
6
|
+
CONTEXTS = { counter: { short: :c, long: :counter }, tracker: { short: :t, long: :tracker }, operation: { short: :o, long: :operation } }.freeze
|
7
|
+
|
8
|
+
|
9
|
+
# Configuration class
|
10
|
+
class Config
|
11
|
+
|
12
|
+
# Initialization with default configuration.
|
13
|
+
#
|
14
|
+
# Configure Redis:
|
15
|
+
# etc/redis/redis.conf
|
16
|
+
# hash-max-ziplist-entries 1024
|
17
|
+
# hash-max-ziplist-value 64
|
18
|
+
#
|
19
|
+
# @return [OpenStruct] default configuration
|
20
|
+
def initialize
|
21
|
+
@config = OpenStruct.new(
|
22
|
+
pool_size: 5, # Default connection pool size is 5
|
23
|
+
pool_timeout: 5, # Default connection pool timeout is 5
|
24
|
+
namespace: 'rl', # Default Redis namespace is 'rl', short name saves memory
|
25
|
+
redis: { url: 'redis://127.0.0.1:6379' }, # Default Redis configuration
|
26
|
+
silent: false, # Silent Redis errors, default is false
|
27
|
+
separator: ':', # Default Redis namespace separator, default is ':'
|
28
|
+
bucket: true, # Bucketize counter object ids, default is true
|
29
|
+
bucket_size: 1000, # Bucket size, best performance with bucket size 1000. See hash-max-ziplist-entries
|
30
|
+
auto_clean: true, # Auto remove operation keys from Redis
|
31
|
+
encode: { # Encode event ids or object ids
|
32
|
+
events: true,
|
33
|
+
ids: true
|
34
|
+
},
|
35
|
+
granularities: {
|
36
|
+
minutely: { step: 1.minute, pattern: '%Y%m%d%H%m' },
|
37
|
+
hourly: { step: 1.hour, pattern: '%Y%m%d%H' },
|
38
|
+
daily: { step: 1.day, pattern: '%Y%m%d' },
|
39
|
+
weekly: { step: 1.week, pattern: '%GW%V' },
|
40
|
+
monthly: { step: 1.month, pattern: '%Y%m' },
|
41
|
+
yearly: { step: 1.year, pattern: '%Y' }
|
42
|
+
},
|
43
|
+
counter_expirations: { minutely: 1.day, hourly: 1.week, daily: 3.months, weekly: 1.year, monthly: 1.year, yearly: 1.year },
|
44
|
+
counter_granularity: :daily..:yearly,
|
45
|
+
tracker_expirations: { minutely: 1.day, hourly: 1.week, daily: 3.months, weekly: 1.year, monthly: 1.year, yearly: 1.year },
|
46
|
+
tracker_granularity: :daily..:yearly,
|
47
|
+
operation_expiration: 1.day
|
48
|
+
)
|
49
|
+
end
|
50
|
+
|
51
|
+
|
52
|
+
# Send missing methods to the OpenStruct configuration.
|
53
|
+
#
|
54
|
+
# @param method [String] the missing method name
|
55
|
+
# @param *args [Array] list of arguments of the missing method
|
56
|
+
# @return [Object] a configuration parameter
|
57
|
+
def method_missing(method, *args, &block)
|
58
|
+
@config.send(method, *args, &block)
|
59
|
+
end
|
60
|
+
|
61
|
+
end
|
62
|
+
end
|
@@ -0,0 +1,79 @@
|
|
1
|
+
require 'connection_pool'
|
2
|
+
require 'redis'
|
3
|
+
require 'redis/namespace'
|
4
|
+
|
5
|
+
|
6
|
+
module Redlics
|
7
|
+
|
8
|
+
# Connection namespace
|
9
|
+
module Connection
|
10
|
+
|
11
|
+
extend self
|
12
|
+
|
13
|
+
|
14
|
+
# Create a new connection pool for Redis connection.
|
15
|
+
#
|
16
|
+
# @param options [Hash] configuration options
|
17
|
+
# @return [ConnectionPool] Redlics connection pool
|
18
|
+
def create(options = {})
|
19
|
+
ConnectionPool.new(pool_options(options)) do
|
20
|
+
build_connection(options)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
# Set connection pool options.
|
28
|
+
#
|
29
|
+
# @param options [Hash] configuration options
|
30
|
+
# @return [Hash] connection pool options
|
31
|
+
def pool_options(options)
|
32
|
+
{ size: options[:pool_size],
|
33
|
+
timeout: options[:pool_timeout] }
|
34
|
+
end
|
35
|
+
|
36
|
+
|
37
|
+
# Build Redis connection with options.
|
38
|
+
#
|
39
|
+
# @param options [Hash] configuration options
|
40
|
+
# @return [Redis] Redis connection
|
41
|
+
# @return [Redis::Namespace] Redis namespaced connection
|
42
|
+
def build_connection(options)
|
43
|
+
namespace = options[:namespace]
|
44
|
+
connection = Redis.new(redis_opts(options))
|
45
|
+
if namespace
|
46
|
+
Redis::Namespace.new(namespace, redis: connection)
|
47
|
+
else
|
48
|
+
connection
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
|
53
|
+
# Client options provided by redis-rb
|
54
|
+
# @see https://github.com/redis/redis-rb/blob/master/lib/redis.rb
|
55
|
+
#
|
56
|
+
# @param options [Hash] options
|
57
|
+
# @option options [String] :url (value of the environment variable REDIS_URL) a Redis URL, for a TCP connection: `redis://:[password]@[hostname]:[port]/[db]` (password, port and database are optional), for a unix socket connection: `unix://[path to Redis socket]`. This overrides all other options.
|
58
|
+
# @option options [String] :host ("127.0.0.1") server hostname
|
59
|
+
# @option options [Fixnum] :port (6379) server port
|
60
|
+
# @option options [String] :path path to server socket (overrides host and port)
|
61
|
+
# @option options [Float] :timeout (5.0) timeout in seconds
|
62
|
+
# @option options [Float] :connect_timeout (same as timeout) timeout for initial connect in seconds
|
63
|
+
# @option options [String] :password Password to authenticate against server
|
64
|
+
# @option options [Fixnum] :db (0) Database to select after initial connect
|
65
|
+
# @option options [Symbol] :driver Driver to use, currently supported: `:ruby`, `:hiredis`, `:synchrony`
|
66
|
+
# @option options [String] :id ID for the client connection, assigns name to current connection by sending `CLIENT SETNAME`
|
67
|
+
# @option options [Hash, Fixnum] :tcp_keepalive Keepalive values, if Fixnum `intvl` and `probe` are calculated based on the value, if Hash `time`, `intvl` and `probes` can be specified as a Fixnum
|
68
|
+
# @option options [Fixnum] :reconnect_attempts Number of attempts trying to connect
|
69
|
+
# @option options [Boolean] :inherit_socket (false) Whether to use socket in forked process or not
|
70
|
+
# @option options [Array] :sentinels List of sentinels to contact
|
71
|
+
# @option options [Symbol] :role (:master) Role to fetch via Sentinel, either `:master` or `:slave`
|
72
|
+
def redis_opts(options)
|
73
|
+
opts = options[:redis]
|
74
|
+
opts[:driver] ||= 'ruby'
|
75
|
+
opts
|
76
|
+
end
|
77
|
+
|
78
|
+
end
|
79
|
+
end
|
@@ -0,0 +1,94 @@
|
|
1
|
+
module Redlics
|
2
|
+
|
3
|
+
# Counter class
|
4
|
+
module Counter
|
5
|
+
|
6
|
+
# Context constant for given class.
|
7
|
+
CONTEXT = Redlics::CONTEXTS[:counter].freeze
|
8
|
+
|
9
|
+
extend self
|
10
|
+
|
11
|
+
|
12
|
+
# Count for a given event and object id with options.
|
13
|
+
#
|
14
|
+
# @param *args [Array] list of arguments for count
|
15
|
+
# @return [Array] list of counted granularities
|
16
|
+
def count(*args, &block)
|
17
|
+
return count_with_block(&block) if block_given?
|
18
|
+
return count_with_hash if args.first.is_a?(Hash)
|
19
|
+
count_with_args(*args)
|
20
|
+
end
|
21
|
+
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
# Count with hash.
|
26
|
+
#
|
27
|
+
# @param options [Hash] configuration options
|
28
|
+
# @return [Array] list of counted granularities
|
29
|
+
def count_with_hash(options)
|
30
|
+
options[:id] = options[:id].to_i unless options[:id].nil?
|
31
|
+
Granularity.validate(CONTEXT, options[:granularity]).each do |granularity|
|
32
|
+
opt = options.clone.merge(granularity: granularity)
|
33
|
+
if Redlics.config.bucket && opt[:id]
|
34
|
+
count_by_hash(opt)
|
35
|
+
else
|
36
|
+
count_by_key(opt)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
|
42
|
+
# Count with hash.
|
43
|
+
#
|
44
|
+
# @param [&Block] a block with configuration options
|
45
|
+
# @return [Array] list of counted granularities
|
46
|
+
def count_with_block
|
47
|
+
yield options = OpenStruct.new
|
48
|
+
count_with_hash(options)
|
49
|
+
end
|
50
|
+
|
51
|
+
|
52
|
+
# Count with hash.
|
53
|
+
#
|
54
|
+
# @param *args [Array] list of arguments for count
|
55
|
+
# @return [Array] list of counted granularities
|
56
|
+
def count_with_args(*args)
|
57
|
+
options = args.last.instance_of?(Hash) ? args.pop : {}
|
58
|
+
options.merge!({
|
59
|
+
event: args[0],
|
60
|
+
id: args[1]
|
61
|
+
})
|
62
|
+
count_with_hash(options)
|
63
|
+
end
|
64
|
+
|
65
|
+
|
66
|
+
# Count by hash.
|
67
|
+
#
|
68
|
+
# @param options [Hash] configuration options
|
69
|
+
# @return [Array] result of pipelined redis commands
|
70
|
+
def count_by_hash(options)
|
71
|
+
granularity = options[:granularity]
|
72
|
+
key = Key.name(CONTEXT, options[:event], granularity, options[:past], { id: options[:id], bucketized: true })
|
73
|
+
Redlics.redis.pipelined do |redis|
|
74
|
+
redis.hincrby(key[0], key[1], 1)
|
75
|
+
redis.expire(key[0], options[:expiration_for] && options[:expiration_for][granularity] || Redlics.config.counter_expirations[granularity])
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
|
80
|
+
# Count by key.
|
81
|
+
#
|
82
|
+
# @param options [Hash] configuration options
|
83
|
+
# @return [Array] result of pipelined redis commands
|
84
|
+
def count_by_key(options)
|
85
|
+
granularity = options[:granularity]
|
86
|
+
key = Key.name(CONTEXT, options[:event], granularity, options[:past], { id: options[:id], bucketized: false })
|
87
|
+
Redlics.redis.pipelined do |redis|
|
88
|
+
redis.incr(key)
|
89
|
+
redis.expire(key, options[:expiration_for] && options[:expiration_for][granularity] || Redlics.config.counter_expirations[granularity])
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
end
|
94
|
+
end
|