redis-stream 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 8e51c4737c9df5802350f4542ff2174d94b1c04d2822e7cfa8a0228add8b5064
+  data.tar.gz: f6943fea98b8deb03c2308e12c7b361e751c58f178830f7e572cfbfb9c0715ff
+SHA512:
+  metadata.gz: 79236d75161bb94c395021424287f6d8d980f580d1d24fb841ccae07b5afc5fd43054dc0d8f766db032eaca517075ced1c0dbff824818a34fa49e90c89356d8c
+  data.tar.gz: a89ed5e218e74e1a0688ec7f587466f3818d92d5f36c09e7df795ae1bf7b36a2f1a36666091d21b2b044ff6acf7d36155167185ac6838490901df367e83d1c31

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.idea
+.idea/*
+*.gem

data/.travis.yml

@@ -0,0 +1,7 @@
+---
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 2.5.0
+before_install: gem install bundler -v 2.0.1

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at mehmet@celik.be. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in redis-stream.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,28 @@
+PATH
+  remote: .
+  specs:
+    redis-stream (0.1.0)
+      moneta (~> 1.2)
+      multi_json (~> 1.14)
+      redis (= 4.1.3)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    minitest (5.13.0)
+    moneta (1.2.1)
+    multi_json (1.14.1)
+    rake (10.5.0)
+    redis (4.1.3)
+
+PLATFORMS
+  java
+
+DEPENDENCIES
+  bundler (~> 2.0)
+  minitest (~> 5.0)
+  rake (~> 10.0)
+  redis-stream!
+
+BUNDLED WITH
+   2.1.4

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 Mehmet Celik
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,97 @@
+# Redis::Stream
+### !!!Use jRuby for now. It has a weird bug in cruby
+
+Sugar coating Redis Streams
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'redis-stream'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install redis-stream
+
+## Usage
+
+#### A simple non-blocking example
+```ruby
+require 'redis-stream'
+s1 =  Redis::Stream::Client.new("test", "LIST", 't1')
+s2 =  Redis::Stream::Client.new("test", "MANIFEST", 't2')
+
+s2.on_message do |message|
+  m = message['payload']
+  puts "Hello #{m}"
+  s1.stop
+  s2.stop
+end
+
+s1.start(false)
+s2.start(false)
+
+id = s1.add("World!", "to" => "*", "group" => "MANIFEST", "type" => Redis::Stream::Type::ACTION)
+
+Timeout::timeout(10) do
+  loop do
+    break unless s1.running? || s2.running?
+    sleep 1
+    puts "checkin if still active #{s1.running?}, #{s2.running?}"
+  end
+end
+```
+
+#### Micro services example
+
+1. Sinatra as a point of entry
+2. microservice for processing
+
+# http.rb
+```ruby
+require 'sinatra'
+require 'redis-stream'
+
+configure do
+  set :redis_stream, Redis::Stream::Client.new("greetings", "HTTP", "http_client", "sync_start" => true)
+  
+get '/:name' do
+  halt 500, 'name parameter not found' unless params.include?(:name)
+  result = settings.redis_stream.sync_add(params[:name], "time_out" => 60)
+  @name = params[:name]
+  @reversed_name = result[payload]
+  erb :index
+end
+
+__END__
+@@index
+
+<p>Hello, <%= @name %>!</p>
+<p><%= @reversed_name</p>
+
+```
+
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/mehmetc/redis-stream. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the Redis::Stream project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/redis-stream/blob/master/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "redis/stream"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/config.yml

@@ -0,0 +1 @@
+data_cache: "/Users/mehmetc/Tmp/resolver/data_cache"

data/lib/redis/stream/client.rb

@@ -0,0 +1,319 @@
+#encoding: UTF-8
+require "redis"
+require "logger"
+require "json"
+require "thread"
+require "redis/stream/inspect"
+require "redis/stream/config"
+require "redis/stream/data_cache"
+
+class Redis
+  module Stream
+    class Client
+      include Redis::Stream::Inspect
+
+      attr_reader :logger, :name, :group, :consumer_id, :cache, :redis, :non_blocking
+
+      # Initialize: setup rstream
+      # @param [String] stream_name        name of the rstream
+      # @param [String] group         name of the rstream group
+      # @param [Object] options       options can contain redis[host, port, db] and logger keys
+      #
+      # Example: Redis::Stream::Client.new("resolver", "stream", {"logger" => Logger.new(STDOUT)})
+      # if group is nil or not supplied then no rstream group will be setup
+      def initialize(stream_name, group_name = nil, name = rand(36 ** 7).to_s(36), options = {})
+        default_options = {"host" => "127.0.0.1", "port" => 6379, "db" => 0, "logger" => Logger.new(STDOUT)}
+        options = default_options.merge(options)
+
+        host = options["host"]
+        port = options["port"]
+        db = options["db"]
+        @logger = options["logger"]
+        @cache = Redis::Stream::DataCache.new
+
+        @name = name
+        @state = Redis::Stream::State::IDLE
+        @stream = stream_name
+        @group = group_name
+        if options.include?('redis')
+          @redis = options['redis']
+        else
+          @redis = Redis.new(host: host, port: port, db: db)
+        end
+        @consumer_id = "#{@name}-#{@group}-#{Process.pid}"
+        @non_blocking = nil
+        #  @send_queue = []
+
+        raise "No redis" if @redis.nil?
+
+        @state = Redis::Stream::State::RUNNING if options.include?("sync_start") && options["sync_start"]
+        setup_stream
+
+        @last_id = info['last-generated-id'] rescue '0'
+        @logger.info "#{@consumer_id} - Last ID = #{@last_id}"
+      end
+
+
+      # add: add a message to the stream
+      # @param [Object] data                    Any data you want to transmit
+      # @param [String] to                      Name of the consumer can be "*" or "" or nil for any consumer
+      # @param [String] group                   Name of the consumer group can be "*" or "" or nil for any group
+      # @param [Stream::Type] type    Type of message
+      #
+      # no passthrough variable here. The passthrough is available in the start method
+      def add(data = {}, options = {})
+        raise "Client isn't running" unless @state.eql?(Redis::Stream::State::RUNNING)
+
+        default_options = {"to" => "*", "group" => "*", "type" => Redis::Stream::Type::ACTION, "cache_key" => nil}
+        options = default_options.merge(options)
+
+        type = options["type"]
+        to = options["to"]
+        group = options["group"]
+        payload = build_payload(data, options)
+        add_id = @redis.xadd(@stream, payload)
+        #  @send_queue << add_id
+
+        @logger.info("#{@consumer_id} - send to '#{to}' in group '#{group}' with id '#{add_id}' of type '#{type}'")
+        add_id
+      end
+
+      # sync_add: same as add command but synchronous. Blocks call until a message arrives
+      # @param [Object] data                    Any data you want to transmit
+      # @param [String] to                      Name of the consumer can be "*" or "" or nil for any consumer
+      # @param [String] group                   Name of the consumer group can be "*" or "" or nil for any group
+      # @param [Stream::Type] type    Type of message
+      # @param [Integer] time_out               Time out after x seconds
+      # @param [Boolean] passthrough            Receive all messages also the ones intended for other consumers
+      def sync_add(data = {}, options = {})
+        raise "Client isn't running" unless @state.eql?(Redis::Stream::State::RUNNING)
+
+        default_options = {"to" => "*", "group" => "*", "type" => Redis::Stream::Type::ACTION, "time_out" => 5, "passthrough" => false, "cache_key" => nil}
+        options = default_options.merge(options)
+
+        to = options["to"]
+        group = options["group"]
+        passthrough = options["passthrough"]
+        time_out = options["time_out"]
+
+        #@state = Redis::Stream::State::RUNNING
+        data_out = nil
+        add_id = add(data, "to" => to, "group" => group, "type" => options["type"], "cache_key" => options["cache_key"])
+
+        time = Time.now
+
+        loop do
+          timing = ((Time.now - time)).to_i
+          if timing > time_out
+            @logger.info("#{@consumer_id} - Time out(#{time_out}) for '#{to}' in group '#{group}'")
+            #@send_queue.delete(add_id) if @send_queue.include?(add_id)
+            break
+          end
+          break if (data_out = read_next_message_from_stream(false, passthrough))
+        end
+        #@state = Redis::Stream::State::STOPPED
+        data_out
+      end
+
+      # on_message: execute this block everytime a new message is received
+      def on_message(&block)
+        @on_message_callback = block
+      end
+
+      # start: start listening for stream messages
+      #
+      # @param [Boolean] block          Should the thread be blocked.
+      # @param [Boolean] passthrough    Receive all messages also the ones intended for other consumers
+      def start(block = true, passthrough = false)
+        raise "#{@consumer_id} already running" if @state == Redis::Stream::State::RUNNING
+        @state = Redis::Stream::State::RUNNING
+        #sanitize
+        if block
+          while @state == Redis::Stream::State::RUNNING
+            read_next_message_from_stream(true, passthrough)
+          end
+        else
+          @non_blocking = Thread.new do
+            while @state == Redis::Stream::State::RUNNING
+              read_next_message_from_stream(true, passthrough)
+            end
+            @logger.info("#{@consumer_id} - ending thread")
+          end
+        end
+      end
+
+      #stop: stop listening for new messages
+      def stop
+        @state = Redis::Stream::State::STOPPED
+        @logger.info("#{@consumer_id} - stopping")
+        @non_blocking.join unless @non_blocking.nil?
+      ensure
+        del_consumer
+        del_group
+      end
+
+      #running?: Are we still in the running state
+      def running?
+        t = @non_blocking.nil? ? true : @non_blocking.alive?
+        t && @state.eql?(Redis::Stream::State::RUNNING)
+      end
+
+      #remove dead and non existing consumers and groups
+      def sanitize
+        groups.each do |group|
+          consumers(group["name"]).each do |consumer|
+            if @consumer_id != consumer["name"]
+              result = sync_add({}, "to" => consumer["name"], "group" => group["name"], "type" => Redis::Stream::Type::PING, "time_out" => 1)
+              if result.nil?
+                del_consumer(group['name'], consumer['name'])
+              end
+            end
+          end
+        end
+      end
+
+
+      private
+
+      def build_payload(data, options)
+        to = options['to']
+        group = options['group']
+        type = options['type']
+
+        payload = nil
+
+        if options["cache_key"].nil?
+          cache_key = @cache.build_key(data)
+          if @cache.include?(cache_key)
+            if data && data.include?('from_cache') && data['from_cache'].eql?(0)
+              @cache.delete(cache_key)
+              @logger.info("#{@consumer_id} - invalidating cache with key #{cache_key}")
+            else
+              payload = {
+                  type: type,
+                  from: to,
+                  from_group: group,
+                  to: @consumer_id,
+                  to_group: @group,
+                  payload: @cache[cache_key].to_json
+              }
+              @logger.info("#{@consumer_id} - fetching from cache with key #{cache_key}")
+            end
+
+          end
+        else
+          @cache[options["cache_key"]] = data
+        end
+
+        if payload.nil?
+          payload = {
+              type: type,
+              from: @consumer_id,
+              from_group: @group,
+              to: to,
+              to_group: group,
+              payload: data.to_json
+          }
+        end
+        payload
+      end
+
+      #setup stream
+      def setup_stream
+        if @group
+          begin
+            @redis.xgroup(:create, @stream, @group, '$', mkstream: true)
+            @logger.info("#{@consumer_id} - Group #{@group} created")
+          rescue Redis::CommandError => e
+            @logger.error("#{@consumer_id} - Group #{@group} exists")
+            @logger.error("#{@consumer_id} - #{e.message}")
+          end
+        end
+
+        Signal.trap('INT') {
+          @logger.info("#{@consumer_id} - Caught CTRL+c")
+          stop
+        }
+
+        at_exit do
+          stop if @state == Redis::Stream::State::RUNNING
+          @logger.info("#{@consumer_id} - Done")
+        end
+
+        @logger.info("#{@consumer_id} - Listening for incoming requests")
+      end
+
+      #handle_incoming: process incoming message
+      # @param [Object] message
+      def handle_incoming(message)
+        if callback = @on_message_callback
+          timing = Time.now
+          begin
+            callback.call(message)
+          rescue Exception => e
+            @logger.error("#{@consumer_id} - #{e.message} - #{message["payload"].to_json}")
+          ensure
+            @logger.info("#{@consumer_id} - message from '#{message["from"]}' handled in #{((Time.now.to_f - timing.to_f).to_f * 1000.0).to_i}ms")
+          end
+        end
+      end
+
+      #read message from the stream
+      # @param [Boolean] async          return the message if synchronous else call handle_incoming
+      # @param [Boolean] passthrough    Receive all messages also the ones intended for other consumers
+      def read_next_message_from_stream(async = true, passthrough = false)
+        if @state == Redis::Stream::State::RUNNING
+          result = @redis.xread(@stream, @lastid, block: 1000, count: 1) if @group.nil?
+          result = @redis.xreadgroup(@group, @consumer_id, @stream, '>', block: 1000, count: 1) if @group
+
+          unless result.empty?
+            id, data_out = result[@stream][0]
+            ack_count = @redis.xack(@stream, @group, id) if @group
+
+            begin
+              data_out["payload"] = JSON.parse(data_out["payload"])
+            rescue Exception => e
+              @logger.error("#{@consumer_id} error parsing payload: #{e.message}")
+            end
+
+            # if @send_queue.include?(id)
+            #   @send_queue.delete(id)
+            #   @logger.warning("#{@consumer_id} - send queue is not empty: #{@send_queue.join(',')}") if @send_queue.length > 0
+            #   unless passthrough
+            #     #@logger.info("#{@consumer_id} - ignoring self")
+            #     return false
+            #   end
+            # end
+
+            if (data_out["from"].eql?(@consumer_id))
+              return false
+            end
+
+            unless (data_out["to"].nil? || data_out["to"].eql?('') || data_out["to"].eql?('*') || data_out["to"].eql?(@consumer_id)) &&
+                (data_out["to_group"].nil? || data_out["to_group"].eql?('') || data_out["to_group"].eql?('*') || data_out["to_group"].eql?(@group))
+              @logger.info("#{@consumer_id} - ignoring message from '#{data_out["from"]}' to '#{data_out["to"]}-#{data_out["to_group"]}'")
+
+              return false
+            end
+
+            @logger.info("#{@consumer_id} - received from '#{data_out["from"]}' of type '#{data_out['type']}' to '#{data_out["to"]}' in group '#{data_out["to_group"]}' with message id '#{id}' - with ack #{ack_count}")
+
+            if data_out["type"].eql?(Redis::Stream::Type::PING)
+              add(data_out["payload"].to_s, "to" => data_out["from"], "group" => "*", "type" => Redis::Stream::Type::PONG)
+              return false
+            end
+
+            if data_out["type"].eql?(Redis::Stream::Type::PONG)
+              return false
+            end
+
+            return data_out unless async
+            handle_incoming(data_out)
+          end
+        end
+      rescue Exception => e
+        return false
+      end
+    end
+  end
+end

data/lib/redis/stream/config.rb

@@ -0,0 +1,90 @@
+#encoding: UTF-8
+require 'yaml'
+class Redis
+  module Stream
+    # Simple way to read and manage a config.yml file
+    class Config
+      @config = {}
+      @config_file_path = ""
+
+      # return the current location of the config.yml file
+      # @return [String] path of config.yml
+      def self.path
+        @config_file_path
+      end
+
+      # set path to config file
+      # @param [String] config_file_path    path to config.yml file
+      def self.path=(config_file_path)
+        @config_file_path = config_file_path
+      end
+
+      # get the value for a key
+      # @param [String] key     key of key/value
+      # @return [Object] value of key/value pair
+      def self.[](key)
+        init
+        @config[key]
+      end
+
+      # set a value into the config.yml file
+      # @param [String] key
+      # @param [Object] value
+      # @return [Object]
+      def self.[]=(key, value)
+        init
+        @config[key] = value
+        File.open("#{path}/config.yml", 'w') do |f|
+          f.puts @config.to_yaml
+        end
+      end
+
+      #is key available in config store
+      # @param [String] key  key to lookup
+      # @return [Boolean]
+      def self.include?(key)
+        init
+        @config.include?(key)
+      end
+
+      private
+
+      # load and prepare config.yml
+      def self.init
+        discover_config_file_path
+        if @config.empty?
+          config = YAML::load_file("#{path}/config.yml")
+          @config = process(config)
+        end
+      end
+
+      private
+
+      #determine location of config.yml file
+      def self.discover_config_file_path
+        if @config_file_path.nil? || @config_file_path.empty?
+          if File.exist?('config.yml')
+            @config_file_path = '.'
+          elsif File.exist?("config/config.yml")
+            @config_file_path = 'config'
+          end
+        end
+      end
+
+      #process config.yml file
+      # @param [Object] config yaml data
+      # @return [Object]
+      def self.process(config)
+        new_config = {}
+        config.each do |k, v|
+          if config[k].is_a?(Hash)
+            v = process(v)
+          end
+          new_config.store(k.to_sym, v)
+        end
+
+        new_config
+      end
+    end
+  end
+end

data/lib/redis/stream/data_cache.rb

@@ -0,0 +1,46 @@
+#encoding: UTF-8
+require 'moneta'
+require 'redis/stream/config'
+
+class Redis
+  module Stream
+    class DataCache
+      def initialize
+        @cache = Moneta.new(:HashFile, dir: Redis::Stream::Config[:data_cache] || "/tmp/cache", serializer: :json)
+      end
+
+      def []=(key, value)
+        @cache.store(key, value)
+      end
+
+      def [](key)
+        @cache.fetch(key)
+      end
+
+      def include?(key)
+        @cache.key?(key)
+      end
+
+      def delete(key)
+        @cache.delete(key)
+      end
+
+      def build_key(data)
+        key = ""
+        if data && data.include?('payload')
+          payload_data = data['payload']
+        else
+          payload_data = data
+        end
+
+        if payload_data.include?("id")
+          action = payload_data["action"].downcase
+          id = payload_data["id"].downcase
+          key = "#{action}_#{id}"
+        end
+        key
+      end
+
+    end
+  end
+end

data/lib/redis/stream/group.rb

@@ -0,0 +1,28 @@
+#encoding: UTF-8
+class Redis
+  module Stream
+    module Group
+      THUMBNAIL = "THUMBNAIL".freeze
+      STREAM = "STREAM".freeze
+      REPRESENTATION = "REPRESENTATION".freeze
+      METADATA = "METADATA".freeze
+      CACHE = "CACHE".freeze
+      LIST = "LIST".freeze
+      MANIFEST = "MANIFEST".freeze
+
+      def self.exists?(group)
+        self.constants.include?(group.upcase.to_sym)
+      end
+
+      def self.to_s
+        self.constants.map { |m| m.to_s.downcase }.compact.join(', ')
+      end
+
+      def self.lookup(group)
+        self.constants.each { |e| return e if e.to_s.downcase.eql?(group.downcase) }
+
+        return '*'
+      end
+    end
+  end
+end

data/lib/redis/stream/inspect.rb

@@ -0,0 +1,37 @@
+#encoding: UTF-8
+class Redis
+  module Stream
+    module Inspect
+      def groups
+        @redis.xinfo("groups", @stream)
+      rescue Exception => e
+        @logger.error("#{@consumer_id} - #{e.message}")
+        {}
+      end
+
+      def info
+        @redis.xinfo("stream", @stream)
+      end
+
+      def consumers(group = @group)
+        @redis.xinfo("consumers", @stream, group)
+      end
+
+      def del_consumer(group = @group, consumer = @consumer_id)
+        @logger.info("#{@consumer_id} - deleting consumer #{group}-#{consumer}")
+        @redis.xgroup('DELCONSUMER', @stream, group, consumer)
+      end
+
+      def del_group(group = @group)
+        if consumers(group).length == 0 && groups.map { |m| m["name"] }.include?(group)
+          @logger.info("#{@consumer_id} - deleting group #{group}")
+          @redis.xgroup('DESTROY', @stream, group)
+        end
+      end
+
+      def pending_messages
+        @redis.xrange(@stream)
+      end
+    end #Inspect
+  end #Stream
+end # Redis

data/lib/redis/stream/state.rb

@@ -0,0 +1,19 @@
+#encoding: UTF-8
+class Redis
+  module Stream
+    module State
+      ERROR = -1
+      IDLE = 0
+      RUNNING = 1
+      STOPPED = 2
+
+      def self.exists?(state)
+        self.constants.include?(state.upcase.to_sym)
+      end
+
+      def self.to_s
+        self.constants.map { |m| m.to_s.downcase }.join(', ')
+      end
+    end
+  end
+end

data/lib/redis/stream/type.rb

@@ -0,0 +1,18 @@
+#encoding: UTF-8
+class Redis
+  module Stream
+    module Type
+      PING = "PING".freeze
+      PONG = "PONG".freeze
+      ACTION = "ACTION".freeze
+
+      def self.exists?(type)
+        self.constants.include?(type.upcase.to_sym)
+      end
+
+      def self.to_s
+        self.constants.map { |m| m.to_s.downcase }.join(', ')
+      end
+    end
+  end
+end

data/lib/redis/stream/version.rb

@@ -0,0 +1,6 @@
+#encoding: UTF-8
+class Redis
+  module Stream
+    VERSION = "0.1.0"
+  end
+end

data/lib/redis/stream.rb

@@ -0,0 +1,8 @@
+#encoding: UTF-8
+require "redis/stream/version"
+require "redis/stream/state"
+require "redis/stream/type"
+require "redis/stream/group"
+require "redis/stream/client"
+require "redis/stream/config"
+require "redis/stream/data_cache"

data/redis-stream.gemspec

@@ -0,0 +1,45 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "redis/stream/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "redis-stream"
+  spec.version       = Redis::Stream::VERSION
+  spec.authors       = ["Mehmet Celik"]
+  spec.email         = ["mehmet@celik.be"]
+
+  spec.summary       = %q{Sugar coating Redis Streams }
+  spec.description   = %q{Simple stream library using Redis Streams}
+  spec.homepage      = "https://github.com/mehmetc/redis-stream"
+  spec.license       = "MIT"
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata["allowed_push_host"] = "https://rubygems.org"
+
+    spec.metadata["homepage_uri"] = spec.homepage
+    spec.metadata["source_code_uri"] = "https://github.com/mehmetc/redis-stream"
+    spec.metadata["changelog_uri"] = "https://github.com/mehmetc/redis-stream"
+  else
+    raise "RubyGems 2.0 or newer is required to protect against " \
+      "public gem pushes."
+  end
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+  spec.add_dependency "redis", "4.1.3"
+  spec.add_dependency "moneta", "~> 1.2"
+  spec.add_dependency "multi_json", "~> 1.14"
+end

metadata

@@ -0,0 +1,152 @@
+--- !ruby/object:Gem::Specification
+name: redis-stream
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Mehmet Celik
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2020-01-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 4.1.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 4.1.3
+- !ruby/object:Gem::Dependency
+  name: moneta
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+description: Simple stream library using Redis Streams
+email:
+- mehmet@celik.be
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- config.yml
+- lib/redis/stream.rb
+- lib/redis/stream/client.rb
+- lib/redis/stream/config.rb
+- lib/redis/stream/data_cache.rb
+- lib/redis/stream/group.rb
+- lib/redis/stream/inspect.rb
+- lib/redis/stream/state.rb
+- lib/redis/stream/type.rb
+- lib/redis/stream/version.rb
+- redis-stream.gemspec
+homepage: https://github.com/mehmetc/redis-stream
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/mehmetc/redis-stream
+  source_code_uri: https://github.com/mehmetc/redis-stream
+  changelog_uri: https://github.com/mehmetc/redis-stream
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.6
+signing_key: 
+specification_version: 4
+summary: Sugar coating Redis Streams
+test_files: []