redlics 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 02052cc441c6a1ed900aca26b965f6a6bab00638
4
+ data.tar.gz: 2f372d92fec7e58e7cda634efff68c65f16bfa5a
5
+ SHA512:
6
+ metadata.gz: ff22b8916fa232fe611123d8042ae0bed4954ff3120e499854243cc77c467b055f27f2b3971e8d4a40d534ae301cf5bd2ccbb69d6c56a1275853aec005f32f6b
7
+ data.tar.gz: 76b768e2b2a5e323186e8215e94554fcd403cf8547605d3629e6e7b99e60c1f47bd81146ce0003547e6fd26dbd6499d1bd3ffd29e8ef6c1fe5aab0a04def5fad
data/.gitignore ADDED
@@ -0,0 +1,6 @@
1
+ *.lock
2
+ *.gem
3
+ *.swp
4
+ .project
5
+ .ruby-version
6
+ .ruby-gemset
data/.travis.yml ADDED
@@ -0,0 +1,24 @@
1
+ sudo: false
2
+ bundler_args: --jobs=3 --retry=3
3
+ language: ruby
4
+ cache: bundler
5
+ before_install:
6
+ - gem update --remote bundler
7
+ rvm:
8
+ - 2.0.0
9
+ - 2.1
10
+ - 2.2
11
+ - jruby-9.0.0.0
12
+ - ruby-head
13
+ - jruby-head
14
+ matrix:
15
+ allow_failures:
16
+ - rvm: ruby-head
17
+ - rvm: jruby-head
18
+ fast_finish: true
19
+ notifications:
20
+ email:
21
+ on_success: change
22
+ on_failure: always
23
+ script:
24
+ - bundle exec rake
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in redlics.gemspec
4
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2016 Phlegx Systems OG
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,64 @@
1
+ # Redlics
2
+
3
+ [![Gem Version](https://badge.fury.io/rb/redlics.svg)](https://rubygems.org/gems/redlics)
4
+ [![Build Status](https://secure.travis-ci.org/phlegx/redlics.svg?branch=master)](https://travis-ci.org/phlegx/redlics)
5
+ [![Code Climate](http://img.shields.io/codeclimate/github/phlegx/redlics.svg)](https://codeclimate.com/github/phlegx/redlics)
6
+ [![Inline Docs](http://inch-ci.org/github/phlegx/redlics.svg?branch=master)](http://inch-ci.org/github/phlegx/redlics)
7
+ [![Dependency Status](https://gemnasium.com/phlegx/redlics.svg)](https://gemnasium.com/phlegx/redlics)
8
+ [![License](https://img.shields.io/github/license/phlegx/redlics.svg)](http://opensource.org/licenses/MIT)
9
+
10
+ Redis analytics with tracks (using bitmaps) and counts (using buckets) encoding numbers in Redis keys and values.
11
+
12
+ ## Installation
13
+
14
+ Add this line to your application's Gemfile:
15
+
16
+ ```ruby
17
+ gem 'redlics'
18
+ ```
19
+
20
+ And then execute:
21
+
22
+ $ bundle
23
+
24
+ Or install it yourself as:
25
+
26
+ $ gem install redlics
27
+
28
+ ## Features
29
+
30
+ * Tracking with bitmaps
31
+ * Counting with buckets
32
+ * High configurable
33
+ * Encode/decode numbers in Redis keys and values
34
+ * Very less memory consumption in Redis
35
+ * Support of time frames
36
+ * Uses Lua script for better performance
37
+ * Plot option for tracks and counts
38
+ * Keeps Redis clean
39
+ * and many more
40
+
41
+ ## Usage
42
+
43
+ Coming soon...
44
+
45
+ ## Contributors
46
+
47
+ * Inspired by Minuteman [github.com/elcuervo/minuteman](https://github.com/elcuervo/minuteman).
48
+ * Inspired by Btrack [github.com/chenfisher/Btrack](https://github.com/chenfisher/Btrack).
49
+ * Inspired by Counterman [github.com/maccman/counterman](https://github.com/maccman/counterman).
50
+
51
+ ## Contributing
52
+
53
+ 1. Fork it ( https://github.com/[your-username]/redlics/fork )
54
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
55
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
56
+ 4. Push to the branch (`git push origin my-new-feature`)
57
+ 5. Create a new Pull Request
58
+
59
+ ## License
60
+
61
+ The MIT License
62
+
63
+ Copyright (c) 2015 Phlegx Systems OG
64
+
data/Rakefile ADDED
@@ -0,0 +1,9 @@
1
+ require 'rake/testtask'
2
+
3
+
4
+ Rake::TestTask.new do |t|
5
+ t.pattern = 'test/**/*_test.rb'
6
+ end
7
+
8
+
9
+ task default: :test
data/lib/redlics.rb ADDED
@@ -0,0 +1,94 @@
1
+ require 'redlics/version'
2
+ require 'redlics/config'
3
+ require 'redlics/exception'
4
+ require 'redlics/connection'
5
+ require 'redlics/granularity'
6
+ require 'redlics/key'
7
+ require 'redlics/time_frame'
8
+ require 'redlics/counter'
9
+ require 'redlics/tracker'
10
+ require 'redlics/operators'
11
+ require 'redlics/query'
12
+ require 'redlics/query/operation'
13
+
14
+
15
+ # Redlics namespace
16
+ module Redlics
17
+
18
+ extend self
19
+
20
+ # Delegate methods to right objects.
21
+ delegate :count, to: Counter
22
+ delegate :track, to: Tracker
23
+ delegate :analyze, to: Query
24
+
25
+
26
+ # Get or initialize the Redis connection.
27
+ # @return [Object] redis connection
28
+ def redis
29
+ redis_pool.with do |conn|
30
+ retryable = true
31
+ begin
32
+ conn
33
+ rescue Redis::BaseError => e
34
+ raise e unless config.silent
35
+ rescue Redis::CommandError => ex
36
+ (conn.disconnect!; retryable = false; retry) if retryable && ex.message =~ /READONLY/
37
+ raise unless config.silent
38
+ end
39
+ end
40
+ end
41
+
42
+
43
+ # Load Lua script file and arguments in Redis.
44
+ #
45
+ # @param file [String] absolute path to the Lua script file
46
+ # @param *args [Array] list of arguments for Redis evalsha
47
+ # @return [String] Lua script result
48
+ def script(file, *args)
49
+ begin
50
+ cache = LUA_CACHE[redis.client.options[:url]]
51
+ if cache.key?(file)
52
+ sha = cache[file]
53
+ else
54
+ src = File.read(file)
55
+ sha = redis.script(:load, src)
56
+ cache[file] = sha
57
+ end
58
+ redis.evalsha(sha, *args)
59
+ rescue RuntimeError
60
+ case $!.message
61
+ when Exception::ErrorPatterns::NOSCRIPT
62
+ LUA_CACHE[redis.client.options[:url]].clear
63
+ retry
64
+ else
65
+ raise $! unless config.silent
66
+ end
67
+ end
68
+ end
69
+
70
+
71
+ # Get or initialize Redlics config.
72
+ # @return [OpenStruct] Redlics configuration
73
+ def config
74
+ @config ||= Redlics::Config.new
75
+ end
76
+
77
+
78
+ # Set configuration of Redlics in a block.
79
+ # @return [OpenStruct] Redlics configuration
80
+ def configure
81
+ yield config if block_given?
82
+ end
83
+
84
+
85
+ private
86
+
87
+ # Get or initialize the Redis connection pool.
88
+ # @return [ConnectionPool] redis connection pool
89
+ def redis_pool
90
+ @redis ||= Redlics::Connection.create(config.to_h)
91
+ end
92
+
93
+ end
94
+
@@ -0,0 +1,62 @@
1
+ module Redlics
2
+
3
+ # Redlics constants.
4
+ LUA_CACHE = Hash.new { |h, k| h[k] = Hash.new }
5
+ LUA_SCRIPT = File.expand_path('../lua/script.lua', __FILE__).freeze
6
+ CONTEXTS = { counter: { short: :c, long: :counter }, tracker: { short: :t, long: :tracker }, operation: { short: :o, long: :operation } }.freeze
7
+
8
+
9
+ # Configuration class
10
+ class Config
11
+
12
+ # Initialization with default configuration.
13
+ #
14
+ # Configure Redis:
15
+ # etc/redis/redis.conf
16
+ # hash-max-ziplist-entries 1024
17
+ # hash-max-ziplist-value 64
18
+ #
19
+ # @return [OpenStruct] default configuration
20
+ def initialize
21
+ @config = OpenStruct.new(
22
+ pool_size: 5, # Default connection pool size is 5
23
+ pool_timeout: 5, # Default connection pool timeout is 5
24
+ namespace: 'rl', # Default Redis namespace is 'rl', short name saves memory
25
+ redis: { url: 'redis://127.0.0.1:6379' }, # Default Redis configuration
26
+ silent: false, # Silent Redis errors, default is false
27
+ separator: ':', # Default Redis namespace separator, default is ':'
28
+ bucket: true, # Bucketize counter object ids, default is true
29
+ bucket_size: 1000, # Bucket size, best performance with bucket size 1000. See hash-max-ziplist-entries
30
+ auto_clean: true, # Auto remove operation keys from Redis
31
+ encode: { # Encode event ids or object ids
32
+ events: true,
33
+ ids: true
34
+ },
35
+ granularities: {
36
+ minutely: { step: 1.minute, pattern: '%Y%m%d%H%m' },
37
+ hourly: { step: 1.hour, pattern: '%Y%m%d%H' },
38
+ daily: { step: 1.day, pattern: '%Y%m%d' },
39
+ weekly: { step: 1.week, pattern: '%GW%V' },
40
+ monthly: { step: 1.month, pattern: '%Y%m' },
41
+ yearly: { step: 1.year, pattern: '%Y' }
42
+ },
43
+ counter_expirations: { minutely: 1.day, hourly: 1.week, daily: 3.months, weekly: 1.year, monthly: 1.year, yearly: 1.year },
44
+ counter_granularity: :daily..:yearly,
45
+ tracker_expirations: { minutely: 1.day, hourly: 1.week, daily: 3.months, weekly: 1.year, monthly: 1.year, yearly: 1.year },
46
+ tracker_granularity: :daily..:yearly,
47
+ operation_expiration: 1.day
48
+ )
49
+ end
50
+
51
+
52
+ # Send missing methods to the OpenStruct configuration.
53
+ #
54
+ # @param method [String] the missing method name
55
+ # @param *args [Array] list of arguments of the missing method
56
+ # @return [Object] a configuration parameter
57
+ def method_missing(method, *args, &block)
58
+ @config.send(method, *args, &block)
59
+ end
60
+
61
+ end
62
+ end
@@ -0,0 +1,79 @@
1
+ require 'connection_pool'
2
+ require 'redis'
3
+ require 'redis/namespace'
4
+
5
+
6
+ module Redlics
7
+
8
+ # Connection namespace
9
+ module Connection
10
+
11
+ extend self
12
+
13
+
14
+ # Create a new connection pool for Redis connection.
15
+ #
16
+ # @param options [Hash] configuration options
17
+ # @return [ConnectionPool] Redlics connection pool
18
+ def create(options = {})
19
+ ConnectionPool.new(pool_options(options)) do
20
+ build_connection(options)
21
+ end
22
+ end
23
+
24
+
25
+ private
26
+
27
+ # Set connection pool options.
28
+ #
29
+ # @param options [Hash] configuration options
30
+ # @return [Hash] connection pool options
31
+ def pool_options(options)
32
+ { size: options[:pool_size],
33
+ timeout: options[:pool_timeout] }
34
+ end
35
+
36
+
37
+ # Build Redis connection with options.
38
+ #
39
+ # @param options [Hash] configuration options
40
+ # @return [Redis] Redis connection
41
+ # @return [Redis::Namespace] Redis namespaced connection
42
+ def build_connection(options)
43
+ namespace = options[:namespace]
44
+ connection = Redis.new(redis_opts(options))
45
+ if namespace
46
+ Redis::Namespace.new(namespace, redis: connection)
47
+ else
48
+ connection
49
+ end
50
+ end
51
+
52
+
53
+ # Client options provided by redis-rb
54
+ # @see https://github.com/redis/redis-rb/blob/master/lib/redis.rb
55
+ #
56
+ # @param options [Hash] options
57
+ # @option options [String] :url (value of the environment variable REDIS_URL) a Redis URL, for a TCP connection: `redis://:[password]@[hostname]:[port]/[db]` (password, port and database are optional), for a unix socket connection: `unix://[path to Redis socket]`. This overrides all other options.
58
+ # @option options [String] :host ("127.0.0.1") server hostname
59
+ # @option options [Fixnum] :port (6379) server port
60
+ # @option options [String] :path path to server socket (overrides host and port)
61
+ # @option options [Float] :timeout (5.0) timeout in seconds
62
+ # @option options [Float] :connect_timeout (same as timeout) timeout for initial connect in seconds
63
+ # @option options [String] :password Password to authenticate against server
64
+ # @option options [Fixnum] :db (0) Database to select after initial connect
65
+ # @option options [Symbol] :driver Driver to use, currently supported: `:ruby`, `:hiredis`, `:synchrony`
66
+ # @option options [String] :id ID for the client connection, assigns name to current connection by sending `CLIENT SETNAME`
67
+ # @option options [Hash, Fixnum] :tcp_keepalive Keepalive values, if Fixnum `intvl` and `probe` are calculated based on the value, if Hash `time`, `intvl` and `probes` can be specified as a Fixnum
68
+ # @option options [Fixnum] :reconnect_attempts Number of attempts trying to connect
69
+ # @option options [Boolean] :inherit_socket (false) Whether to use socket in forked process or not
70
+ # @option options [Array] :sentinels List of sentinels to contact
71
+ # @option options [Symbol] :role (:master) Role to fetch via Sentinel, either `:master` or `:slave`
72
+ def redis_opts(options)
73
+ opts = options[:redis]
74
+ opts[:driver] ||= 'ruby'
75
+ opts
76
+ end
77
+
78
+ end
79
+ end
@@ -0,0 +1,94 @@
1
+ module Redlics
2
+
3
+ # Counter class
4
+ module Counter
5
+
6
+ # Context constant for given class.
7
+ CONTEXT = Redlics::CONTEXTS[:counter].freeze
8
+
9
+ extend self
10
+
11
+
12
+ # Count for a given event and object id with options.
13
+ #
14
+ # @param *args [Array] list of arguments for count
15
+ # @return [Array] list of counted granularities
16
+ def count(*args, &block)
17
+ return count_with_block(&block) if block_given?
18
+ return count_with_hash if args.first.is_a?(Hash)
19
+ count_with_args(*args)
20
+ end
21
+
22
+
23
+ private
24
+
25
+ # Count with hash.
26
+ #
27
+ # @param options [Hash] configuration options
28
+ # @return [Array] list of counted granularities
29
+ def count_with_hash(options)
30
+ options[:id] = options[:id].to_i unless options[:id].nil?
31
+ Granularity.validate(CONTEXT, options[:granularity]).each do |granularity|
32
+ opt = options.clone.merge(granularity: granularity)
33
+ if Redlics.config.bucket && opt[:id]
34
+ count_by_hash(opt)
35
+ else
36
+ count_by_key(opt)
37
+ end
38
+ end
39
+ end
40
+
41
+
42
+ # Count with hash.
43
+ #
44
+ # @param [&Block] a block with configuration options
45
+ # @return [Array] list of counted granularities
46
+ def count_with_block
47
+ yield options = OpenStruct.new
48
+ count_with_hash(options)
49
+ end
50
+
51
+
52
+ # Count with hash.
53
+ #
54
+ # @param *args [Array] list of arguments for count
55
+ # @return [Array] list of counted granularities
56
+ def count_with_args(*args)
57
+ options = args.last.instance_of?(Hash) ? args.pop : {}
58
+ options.merge!({
59
+ event: args[0],
60
+ id: args[1]
61
+ })
62
+ count_with_hash(options)
63
+ end
64
+
65
+
66
+ # Count by hash.
67
+ #
68
+ # @param options [Hash] configuration options
69
+ # @return [Array] result of pipelined redis commands
70
+ def count_by_hash(options)
71
+ granularity = options[:granularity]
72
+ key = Key.name(CONTEXT, options[:event], granularity, options[:past], { id: options[:id], bucketized: true })
73
+ Redlics.redis.pipelined do |redis|
74
+ redis.hincrby(key[0], key[1], 1)
75
+ redis.expire(key[0], options[:expiration_for] && options[:expiration_for][granularity] || Redlics.config.counter_expirations[granularity])
76
+ end
77
+ end
78
+
79
+
80
+ # Count by key.
81
+ #
82
+ # @param options [Hash] configuration options
83
+ # @return [Array] result of pipelined redis commands
84
+ def count_by_key(options)
85
+ granularity = options[:granularity]
86
+ key = Key.name(CONTEXT, options[:event], granularity, options[:past], { id: options[:id], bucketized: false })
87
+ Redlics.redis.pipelined do |redis|
88
+ redis.incr(key)
89
+ redis.expire(key, options[:expiration_for] && options[:expiration_for][granularity] || Redlics.config.counter_expirations[granularity])
90
+ end
91
+ end
92
+
93
+ end
94
+ end