gitmq 0.1.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: cd7432e882137f0309ae2c8ce3c086f04b048fc9effab91ce6188699ac019d99
+  data.tar.gz: 2963ec23a3681ec4499005f7de8ecf810517fb0731091b3d375109075fcb4571
+SHA512:
+  metadata.gz: 62f9b7751214d6ccbff62987f761898dccfd922626e5d90863b04b463820ab4e95fdfdac94183492e99078d65ebf0d5fedf08f9c14530ea29f3efac5666e7a86
+  data.tar.gz: 71a7a5535992461d806aaec1a6e7da6dfff2a075f5734bf352d83830ea59896abba616ccaeca97aef5e44f3747c58abc2d2ba33fabcb35b286bad1de332d2cfd

data/.github/workflows/gempush.yml

@@ -0,0 +1,31 @@
+name: Ruby Gem
+
+on:
+  pull_request:
+    branches:
+      - master
+  push:
+    branches:
+      - master
+
+jobs:
+  build:
+    name: Build + Publish
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@master
+    - name: Set up Ruby 2.6
+      uses: actions/setup-ruby@v1
+      with:
+        version: 2.6.x
+    - name: Publish to RubyGems
+      run: |
+        mkdir -p $HOME/.gem
+        touch $HOME/.gem/credentials
+        chmod 0600 $HOME/.gem/credentials
+        printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+        gem build *.gemspec
+        gem push *.gem
+      env:
+        GEM_HOST_API_KEY: ${{secrets.RUBYGEMS_AUTH_TOKEN}}

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.reek.yml

@@ -0,0 +1,3 @@
+detectors:
+  IrresponsibleModule:
+    enabled: false

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,6 @@
+AllCops:
+  DefaultFormatter: simple
+Style/Documentation:
+  Enabled: false
+Naming/RescuedExceptionsVariableName:
+  Enabled: false

data/.ruby-version

@@ -0,0 +1 @@
+2.6.3

data/.travis.yml

@@ -0,0 +1,7 @@
+---
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 2.5.3
+before_install: gem install bundler -v 2.0.2

data/Gemfile

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in gitmq.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,45 @@
+PATH
+  remote: .
+  specs:
+    gitmq (0.1.2)
+      listen (~> 3.2.0)
+      rugged (~> 0.28)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.3)
+    ffi (1.11.3)
+    listen (3.2.0)
+      rb-fsevent (~> 0.10, >= 0.10.3)
+      rb-inotify (~> 0.9, >= 0.9.10)
+    rake (10.5.0)
+    rb-fsevent (0.10.3)
+    rb-inotify (0.10.0)
+      ffi (~> 1.0)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.0)
+      rspec-support (~> 3.9.0)
+    rspec-expectations (3.9.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.0)
+    rugged (0.28.3.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 2.0)
+  gitmq!
+  rake (~> 10.0)
+  rspec (~> 3.0)
+
+BUNDLED WITH
+   2.0.2

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2019 Emad Elsaid
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,174 @@
+# GitMQ : Git Message Queue
+
+Hey, Crazy idea bad pitch, lets use Git as a messaging queue! lets use github to
+synchronize a cluster of them together.
+
+![Screenshot-2019-11-24_22-41-55](https://user-images.githubusercontent.com/54403/69502136-09672400-0f0c-11ea-8707-df3164411c4e.png)
+
+
+Lets get the obvious out of the way.
+
+1. You already know that we use Git to keep track of our code changes, If you're a
+developer you most probably use it every day, and you probably know how to share
+your code changes with everybody else using some central repo server like Github
+or Bitbucket.
+
+1. Alright, now lets introduce another idea: **Git commits doesn't need to include
+changes**, You don't have to create files then commit these files to
+your git repo, you can create an empty commit with a command like this:
+
+```
+git commit --allow-empty -m "commit message"
+```
+
+1. OK, so the commit message can be any text right? that means you can have JSON
+messages or YAML or any other machine readable format as a commit message.
+
+## Now imagine this use case:
+
+1. Create a repository on github
+1. Clone it to your server
+1. Have your application commit a `message` to this repo every time you need to
+1. Push the repo every period say 10 seconds
+1. Clone same repo to another server
+1. Have an application pull periodically and checks for new commits and for each
+   one of them extracts the message and process it
+1. Put a git tag on the last processed message with that consumer identifier
+1. Push that label to keep track of the last processed message (pull it when you
+   deploy that consumer again)
+
+Now you have a queue of messages that can't be altered, communicated between
+servers and all data saved in a git repository.
+
+This idea is flexible and can be altered for several use cases:
+
+1. the central server doesn't have to be Github or a like, you can use your
+   own server as a git server.
+1. You can have more than one queue for messages by creating many branches each
+   for a type of messages or a different messages purpose.
+1. You can have the same branch consumed from different servers each message
+   processed multiple times in different ways like RabbitMQ fanout strategy
+1. You can keep browse your message queue with any git client UI or CLI
+1. You can clean your servers git local repo so that it deletes old commit
+   history to save space.
+1. You can have many messages producers write to different branches in the same
+   repository
+1. In the future you can join 2 branches or split a branch and have your
+   consumers adapt to that
+
+
+There are many advantages to use this approach, which is inherited from how Git
+is designed
+
+1. Every message has an address (git commit hash)
+1. Every message can be traced to an author which is the application that
+   published it (you can use different names for each server or application
+   version)
+1. Every message has a creation date
+
+This idea can as simple as invoking git commands from your application with a
+system call, or use libgit to manipulate the repository for you, or even a small
+server application that communicate over a socket, or a repo service that
+communicate over even HTTP, there are so many ways to implement that concept.
+
+The current repository is an implementation of this concept in Ruby.
+
+# Installation
+
+Install the gem with
+```
+gem install gitmq
+```
+
+or add gitmq to your `Gemfile`
+```
+gem 'gitmq', '~> 0.1'
+```
+
+
+# Producing messages
+
+First create an instance of `GitMQ::Storage`, to specify your storage place.
+
+```ruby
+storage = GitMQ::Storage.new('/tmp/gitmq')
+```
+
+This is where your git repository will live, message producer will use this
+storage instance to write messages to any branch you wish.
+
+```ruby
+producer = GitMQ::Producer.new(storage: storage, branch: 'master')
+```
+
+Now you can use `producer` to publish message to this branch.
+
+```ruby
+producer.publish('Hello, world!')
+```
+
+It will create a commit with `Hello, world!` as a message in the `master`
+branch.
+
+You can have as many producers as you wish for any number of branches, make sure
+to have one producer per branch to prevent race conditions between producers.
+
+# Consuming messages
+
+You need to create storage instance as we did with the producer, then
+instanciate your consumer
+
+```ruby
+consumer = GitMQ::Consumer.new(storage: storage, name: 'consumer', branch:
+'master')
+```
+
+And start the consume process, this function will take a block of code that
+accept one parameter which is your message, this function is non-blocking
+it will execute the block of code each time a new commit appear in the branch
+
+```ruby
+consumer.consume do |message|
+    puts message
+end
+```
+
+After your block finish execution the consumer will tag this commit with a git
+tag, the tag name will be the consumer name, if your consumer name is
+`consumer1`, the tag will be `consumer1`, as git tags are global to all the
+repository you need to make the consumer name unique not just for the branch but
+for all the repository, so if you want to consume 2 branches from one consumer
+you'll have to choose to different names so that tags are not overriden be every
+consumer instance, you can adopt a naming scheme that involve the branch name,
+while implementing this gem we had that name tag name as `branch.consumer` but
+then thought it'll be better for the user to make this decision to allow for a
+more flexible naming, the tag will be overritten after each time the consumer
+process a message.
+
+# Pushing and Pulling your data
+
+As for now there is no way to push/pull from the gem, so this part is up to you,
+until it's implemented in the gem.
+
+# Gem executables
+
+## gitmq-produce
+
+Starts a process that waits you to enter any line and publish that as a message
+
+Usage:
+
+```
+gitmq-produce /path/to/rpo branch-name
+```
+
+## gitmq-consume
+
+Starts a process that waits for messages, each message will be written to the
+terminal
+
+Usage:
+
+```
+gitmq-consume /path/to/repo branch-name consumer-name
+```

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/bin/gitmq-consume

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'gitmq'
+
+path = ARGV.shift
+branch = ARGV.shift
+consumer = ARGV.shift
+
+consumer = GitMQ::Consumer.new(
+  storage: GitMQ::Storage.new(path),
+  name: consumer,
+  branch: branch
+)
+
+consumer.consume { |m| puts m }
+
+begin
+  sleep
+rescue Interrupt
+  puts 'Bye.'
+end

data/bin/gitmq-produce

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'gitmq'
+
+path = ARGV.shift
+branch = ARGV.shift
+
+producer = GitMQ::Producer.new(
+  storage: GitMQ::Storage.new(path),
+  branch: branch
+)
+
+loop do
+  producer.publish(gets.strip)
+rescue Interrupt
+  puts 'Bye.'
+  exit
+end

data/gitmq.gemspec

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'gitmq'
+  spec.version       = GitMQ::VERSION
+  spec.authors       = ['Emad Elsaid']
+  spec.email         = ['emad.elsaid.hamed@gmail.com']
+
+  spec.summary       = 'Git Message Queue'
+  spec.description   = <<-DESC
+  An interface to use Git repositories as a message queue to communicate between services'
+  DESC
+  spec.homepage      = 'https://www.github.com/emad-elsaid/gitmq'
+  spec.license       = 'MIT'
+
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = spec.homepage
+
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
+  end
+  spec.bindir        = 'bin'
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'rugged', '~> 0.28'
+  spec.add_dependency 'listen', '~> 3.2.0'
+  spec.add_development_dependency 'bundler', '~> 2.0'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+end

data/lib/consumer.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'listen'
+
+module GitMQ
+  class Consumer
+    def initialize(storage:, name:, branch:)
+      @storage = storage
+      @branch = branch
+      @name = name
+    end
+
+    def branch_file
+      @branch_file ||= File.join(@storage.path, 'refs', 'heads')
+    end
+
+    def consume(&block)
+      @block = block
+      consume_commits
+      listener.start
+    end
+
+    def stop
+      listener.stop
+    end
+
+    private
+
+    def listener
+      @listener ||= Listen.to(branch_file, only: /#{Regexp.escape(@branch)}/) do
+        consume_commits
+      end
+    end
+
+    def consume_commits
+      commits = @storage.commits(@branch, @name)
+      commits.each do |commit|
+        @block.call commit.message
+        @storage.tag(@name, commit)
+      end
+    end
+  end
+end

data/lib/gitmq.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require_relative './version'
+require_relative './consumer'
+require_relative './producer'
+require_relative './storage'
+
+module GitMQ
+end

data/lib/producer.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module GitMQ
+  class Producer
+    def initialize(storage:, branch:)
+      @storage = storage
+      @branch = branch
+    end
+
+    def publish(event)
+      commit = Rugged::Commit.create(
+        @storage.repo,
+        tree: @storage.tree,
+        message: event.to_s,
+        parents: [@storage.branch(@branch)&.target].compact
+      )
+      @storage.branches.create(@branch, commit, force: true)
+    end
+  end
+end

data/lib/storage.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'rugged'
+
+module GitMQ
+  class Storage
+    attr_reader :repo, :path
+
+    def initialize(path)
+      @path = path
+      @repo = read_or_create_repo
+    end
+
+    def branches
+      repo.branches
+    end
+
+    def branch(name)
+      branches[name]
+    end
+
+    def tree
+      Rugged::Tree::Builder.new(repo).write
+    end
+
+    def commits(branch, tag)
+      walker = Rugged::Walker.new(repo)
+      walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE)
+      walker.push(branch(branch).target)
+
+      tag = repo.tags[tag]
+      walker.hide(tag.target) if tag
+      walker
+    end
+
+    def tag(label, commit)
+      repo.tags.create(label, commit, true)
+    end
+
+    private
+
+    def read_or_create_repo
+      FileUtils.mkdir_p(path) unless File.exist?(path)
+      begin
+        Rugged::Repository.new(path)
+      rescue Rugged::RepositoryError
+        Rugged::Repository.init_at(path, :bare)
+      end
+    end
+  end
+end

data/lib/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module GitMQ
+  VERSION = '0.1.2'
+end

metadata

@@ -0,0 +1,138 @@
+--- !ruby/object:Gem::Specification
+name: gitmq
+version: !ruby/object:Gem::Version
+  version: 0.1.2
+platform: ruby
+authors:
+- Emad Elsaid
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-11-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rugged
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.28'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.28'
+- !ruby/object:Gem::Dependency
+  name: listen
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.2.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: "  An interface to use Git repositories as a message queue to communicate
+  between services'\n"
+email:
+- emad.elsaid.hamed@gmail.com
+executables:
+- gitmq-consume
+- gitmq-produce
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/workflows/gempush.yml"
+- ".gitignore"
+- ".reek.yml"
+- ".rspec"
+- ".rubocop.yml"
+- ".ruby-version"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/gitmq-consume
+- bin/gitmq-produce
+- gitmq.gemspec
+- lib/consumer.rb
+- lib/gitmq.rb
+- lib/producer.rb
+- lib/storage.rb
+- lib/version.rb
+homepage: https://www.github.com/emad-elsaid/gitmq
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://www.github.com/emad-elsaid/gitmq
+  source_code_uri: https://www.github.com/emad-elsaid/gitmq
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Git Message Queue
+test_files: []