redlics 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 02052cc441c6a1ed900aca26b965f6a6bab00638
+  data.tar.gz: 2f372d92fec7e58e7cda634efff68c65f16bfa5a
+SHA512:
+  metadata.gz: ff22b8916fa232fe611123d8042ae0bed4954ff3120e499854243cc77c467b055f27f2b3971e8d4a40d534ae301cf5bd2ccbb69d6c56a1275853aec005f32f6b
+  data.tar.gz: 76b768e2b2a5e323186e8215e94554fcd403cf8547605d3629e6e7b99e60c1f47bd81146ce0003547e6fd26dbd6499d1bd3ffd29e8ef6c1fe5aab0a04def5fad

data/.gitignore

@@ -0,0 +1,6 @@
+*.lock
+*.gem
+*.swp
+.project
+.ruby-version
+.ruby-gemset

data/.travis.yml

@@ -0,0 +1,24 @@
+sudo: false
+bundler_args: --jobs=3 --retry=3
+language: ruby
+cache: bundler
+before_install:
+  - gem update --remote bundler
+rvm:
+  - 2.0.0
+  - 2.1
+  - 2.2
+  - jruby-9.0.0.0
+  - ruby-head
+  - jruby-head
+matrix:
+  allow_failures:
+    - rvm: ruby-head
+    - rvm: jruby-head
+  fast_finish: true
+notifications:
+  email:
+    on_success: change
+    on_failure: always
+script:
+  - bundle exec rake

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in redlics.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Phlegx Systems OG
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,64 @@
+# Redlics
+
+[![Gem Version](https://badge.fury.io/rb/redlics.svg)](https://rubygems.org/gems/redlics)
+[![Build Status](https://secure.travis-ci.org/phlegx/redlics.svg?branch=master)](https://travis-ci.org/phlegx/redlics)
+[![Code Climate](http://img.shields.io/codeclimate/github/phlegx/redlics.svg)](https://codeclimate.com/github/phlegx/redlics)
+[![Inline Docs](http://inch-ci.org/github/phlegx/redlics.svg?branch=master)](http://inch-ci.org/github/phlegx/redlics)
+[![Dependency Status](https://gemnasium.com/phlegx/redlics.svg)](https://gemnasium.com/phlegx/redlics)
+[![License](https://img.shields.io/github/license/phlegx/redlics.svg)](http://opensource.org/licenses/MIT)
+
+Redis analytics with tracks (using bitmaps) and counts (using buckets) encoding numbers in Redis keys and values.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'redlics'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install redlics
+
+## Features
+
+* Tracking with bitmaps
+* Counting with buckets
+* High configurable
+* Encode/decode numbers in Redis keys and values
+* Very less memory consumption in Redis
+* Support of time frames
+* Uses Lua script for better performance
+* Plot option for tracks and counts
+* Keeps Redis clean
+* and many more
+
+## Usage
+
+Coming soon...
+
+## Contributors
+
+* Inspired by Minuteman [github.com/elcuervo/minuteman](https://github.com/elcuervo/minuteman).
+* Inspired by Btrack [github.com/chenfisher/Btrack](https://github.com/chenfisher/Btrack).
+* Inspired by Counterman [github.com/maccman/counterman](https://github.com/maccman/counterman).
+
+## Contributing
+
+1. Fork it ( https://github.com/[your-username]/redlics/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+
+## License
+
+The MIT License
+
+Copyright (c) 2015 Phlegx Systems OG
+

data/Rakefile

@@ -0,0 +1,9 @@
+require 'rake/testtask'
+
+
+Rake::TestTask.new do |t|
+  t.pattern = 'test/**/*_test.rb'
+end
+
+
+task default: :test

data/lib/redlics.rb

@@ -0,0 +1,94 @@
+require 'redlics/version'
+require 'redlics/config'
+require 'redlics/exception'
+require 'redlics/connection'
+require 'redlics/granularity'
+require 'redlics/key'
+require 'redlics/time_frame'
+require 'redlics/counter'
+require 'redlics/tracker'
+require 'redlics/operators'
+require 'redlics/query'
+require 'redlics/query/operation'
+
+
+# Redlics namespace
+module Redlics
+
+  extend self
+
+  # Delegate methods to right objects.
+  delegate :count, to: Counter
+  delegate :track, to: Tracker
+  delegate :analyze, to: Query
+
+
+  # Get or initialize the Redis connection.
+  # @return [Object] redis connection
+  def redis
+    redis_pool.with do |conn|
+      retryable = true
+      begin
+        conn
+      rescue Redis::BaseError => e
+        raise e unless config.silent
+      rescue Redis::CommandError => ex
+        (conn.disconnect!; retryable = false; retry) if retryable && ex.message =~ /READONLY/
+        raise unless config.silent
+      end
+    end
+  end
+
+
+  # Load Lua script file and arguments in Redis.
+  #
+  # @param file [String] absolute path to the Lua script file
+  # @param *args [Array] list of arguments for Redis evalsha
+  # @return [String] Lua script result
+  def script(file, *args)
+    begin
+      cache = LUA_CACHE[redis.client.options[:url]]
+      if cache.key?(file)
+        sha = cache[file]
+      else
+        src = File.read(file)
+        sha = redis.script(:load, src)
+        cache[file] = sha
+      end
+      redis.evalsha(sha, *args)
+    rescue RuntimeError
+      case $!.message
+      when Exception::ErrorPatterns::NOSCRIPT
+        LUA_CACHE[redis.client.options[:url]].clear
+        retry
+      else
+        raise $! unless config.silent
+      end
+    end
+  end
+
+
+  # Get or initialize Redlics config.
+  # @return [OpenStruct] Redlics configuration
+  def config
+    @config ||= Redlics::Config.new
+  end
+
+
+  # Set configuration of Redlics in a block.
+  # @return [OpenStruct] Redlics configuration
+  def configure
+    yield config if block_given?
+  end
+
+
+  private
+
+    # Get or initialize the Redis connection pool.
+    # @return [ConnectionPool] redis connection pool
+    def redis_pool
+      @redis ||= Redlics::Connection.create(config.to_h)
+    end
+
+end
+

data/lib/redlics/config.rb

@@ -0,0 +1,62 @@
+module Redlics
+
+  # Redlics constants.
+  LUA_CACHE      = Hash.new { |h, k| h[k] = Hash.new }
+  LUA_SCRIPT     = File.expand_path('../lua/script.lua', __FILE__).freeze
+  CONTEXTS       = { counter: { short: :c, long: :counter }, tracker: { short: :t, long: :tracker }, operation: { short: :o, long: :operation } }.freeze
+
+
+  # Configuration class
+  class Config
+
+    # Initialization with default configuration.
+    #
+    # Configure Redis:
+    # etc/redis/redis.conf
+    # hash-max-ziplist-entries 1024
+    # hash-max-ziplist-value 64
+    #
+    # @return [OpenStruct] default configuration
+    def initialize
+      @config = OpenStruct.new(
+        pool_size: 5,                              # Default connection pool size is 5
+        pool_timeout: 5,                           # Default connection pool timeout is 5
+        namespace: 'rl',                           # Default Redis namespace is 'rl', short name saves memory
+        redis: { url: 'redis://127.0.0.1:6379' },  # Default Redis configuration
+        silent: false,                             # Silent Redis errors, default is false
+        separator: ':',                            # Default Redis namespace separator, default is ':'
+        bucket: true,                              # Bucketize counter object ids, default is true
+        bucket_size: 1000,                         # Bucket size, best performance with bucket size 1000. See hash-max-ziplist-entries
+        auto_clean: true,                          # Auto remove operation keys from Redis
+        encode: {                                  # Encode event ids or object ids
+          events: true,
+          ids: true
+        },
+        granularities: {
+          minutely: { step: 1.minute, pattern: '%Y%m%d%H%m' },
+          hourly:   { step: 1.hour,   pattern: '%Y%m%d%H' },
+          daily:    { step: 1.day,    pattern: '%Y%m%d' },
+          weekly:   { step: 1.week,   pattern: '%GW%V' },
+          monthly:  { step: 1.month,  pattern: '%Y%m' },
+          yearly:   { step: 1.year,   pattern: '%Y' }
+        },
+        counter_expirations: { minutely: 1.day, hourly: 1.week, daily: 3.months, weekly: 1.year, monthly: 1.year, yearly: 1.year },
+        counter_granularity: :daily..:yearly,
+        tracker_expirations: { minutely: 1.day, hourly: 1.week, daily: 3.months, weekly: 1.year, monthly: 1.year, yearly: 1.year },
+        tracker_granularity: :daily..:yearly,
+        operation_expiration: 1.day
+      )
+    end
+
+
+    # Send missing methods to the OpenStruct configuration.
+    #
+    # @param method [String] the missing method name
+    # @param *args [Array] list of arguments of the missing method
+    # @return [Object] a configuration parameter
+    def method_missing(method, *args, &block)
+      @config.send(method, *args, &block)
+    end
+
+  end
+end

data/lib/redlics/connection.rb

@@ -0,0 +1,79 @@
+require 'connection_pool'
+require 'redis'
+require 'redis/namespace'
+
+
+module Redlics
+
+  # Connection namespace
+  module Connection
+
+    extend self
+
+
+    # Create a new connection pool for Redis connection.
+    #
+    # @param options [Hash] configuration options
+    # @return [ConnectionPool] Redlics connection pool
+    def create(options = {})
+      ConnectionPool.new(pool_options(options)) do
+        build_connection(options)
+      end
+    end
+
+
+    private
+
+      # Set connection pool options.
+      #
+      # @param options [Hash] configuration options
+      # @return [Hash] connection pool options
+      def pool_options(options)
+        { size:    options[:pool_size],
+          timeout: options[:pool_timeout] }
+      end
+
+
+      # Build Redis connection with options.
+      #
+      # @param options [Hash] configuration options
+      # @return [Redis] Redis connection
+      # @return [Redis::Namespace] Redis namespaced connection
+      def build_connection(options)
+        namespace = options[:namespace]
+        connection = Redis.new(redis_opts(options))
+        if namespace
+          Redis::Namespace.new(namespace, redis: connection)
+        else
+          connection
+        end
+      end
+
+
+      # Client options provided by redis-rb
+      # @see https://github.com/redis/redis-rb/blob/master/lib/redis.rb
+      #
+      # @param options [Hash] options
+      # @option options [String] :url (value of the environment variable REDIS_URL) a Redis URL, for a TCP connection: `redis://:[password]@[hostname]:[port]/[db]` (password, port and database are optional), for a unix socket connection: `unix://[path to Redis socket]`. This overrides all other options.
+      # @option options [String] :host ("127.0.0.1") server hostname
+      # @option options [Fixnum] :port (6379) server port
+      # @option options [String] :path path to server socket (overrides host and port)
+      # @option options [Float] :timeout (5.0) timeout in seconds
+      # @option options [Float] :connect_timeout (same as timeout) timeout for initial connect in seconds
+      # @option options [String] :password Password to authenticate against server
+      # @option options [Fixnum] :db (0) Database to select after initial connect
+      # @option options [Symbol] :driver Driver to use, currently supported: `:ruby`, `:hiredis`, `:synchrony`
+      # @option options [String] :id ID for the client connection, assigns name to current connection by sending `CLIENT SETNAME`
+      # @option options [Hash, Fixnum] :tcp_keepalive Keepalive values, if Fixnum `intvl` and `probe` are calculated based on the value, if Hash `time`, `intvl` and `probes` can be specified as a Fixnum
+      # @option options [Fixnum] :reconnect_attempts Number of attempts trying to connect
+      # @option options [Boolean] :inherit_socket (false) Whether to use socket in forked process or not
+      # @option options [Array] :sentinels List of sentinels to contact
+      # @option options [Symbol] :role (:master) Role to fetch via Sentinel, either `:master` or `:slave`
+      def redis_opts(options)
+        opts = options[:redis]
+        opts[:driver] ||= 'ruby'
+        opts
+      end
+
+  end
+end

data/lib/redlics/counter.rb

@@ -0,0 +1,94 @@
+module Redlics
+
+  # Counter class
+  module Counter
+
+    # Context constant for given class.
+    CONTEXT = Redlics::CONTEXTS[:counter].freeze
+
+    extend self
+
+
+    # Count for a given event and object id with options.
+    #
+    # @param *args [Array] list of arguments for count
+    # @return [Array] list of counted granularities
+    def count(*args, &block)
+      return count_with_block(&block) if block_given?
+      return count_with_hash if args.first.is_a?(Hash)
+      count_with_args(*args)
+    end
+
+
+    private
+
+      # Count with hash.
+      #
+      # @param options [Hash] configuration options
+      # @return [Array] list of counted granularities
+      def count_with_hash(options)
+        options[:id] = options[:id].to_i unless options[:id].nil?
+        Granularity.validate(CONTEXT, options[:granularity]).each do |granularity|
+          opt = options.clone.merge(granularity: granularity)
+          if Redlics.config.bucket && opt[:id]
+            count_by_hash(opt)
+          else
+            count_by_key(opt)
+          end
+        end
+      end
+
+
+      # Count with hash.
+      #
+      # @param [&Block] a block with configuration options
+      # @return [Array] list of counted granularities
+      def count_with_block
+        yield options = OpenStruct.new
+        count_with_hash(options)
+      end
+
+
+      # Count with hash.
+      #
+      # @param *args [Array] list of arguments for count
+      # @return [Array] list of counted granularities
+      def count_with_args(*args)
+        options = args.last.instance_of?(Hash) ? args.pop : {}
+        options.merge!({
+          event: args[0],
+          id: args[1]
+        })
+        count_with_hash(options)
+      end
+
+
+      # Count by hash.
+      #
+      # @param options [Hash] configuration options
+      # @return [Array] result of pipelined redis commands
+      def count_by_hash(options)
+        granularity = options[:granularity]
+        key = Key.name(CONTEXT, options[:event], granularity, options[:past], { id: options[:id], bucketized: true })
+        Redlics.redis.pipelined do |redis|
+          redis.hincrby(key[0], key[1], 1)
+          redis.expire(key[0], options[:expiration_for] && options[:expiration_for][granularity] || Redlics.config.counter_expirations[granularity])
+        end
+      end
+
+
+      # Count by key.
+      #
+      # @param options [Hash] configuration options
+      # @return [Array] result of pipelined redis commands
+      def count_by_key(options)
+        granularity = options[:granularity]
+        key = Key.name(CONTEXT, options[:event], granularity, options[:past], { id: options[:id], bucketized: false })
+        Redlics.redis.pipelined do |redis|
+          redis.incr(key)
+          redis.expire(key, options[:expiration_for] && options[:expiration_for][granularity] || Redlics.config.counter_expirations[granularity])
+        end
+      end
+
+  end
+end