gitmqueue 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 7e42924f796ddd596ee51c69e7d837da4988ce63db27688998e43260570ad9ec
+  data.tar.gz: 8314d9f06183362f39f4831dc8442dfaaeb879bb23860bab6bc8cdf9ea7ebb98
+SHA512:
+  metadata.gz: c9459c7b8990fe2c62f24d5bdcdd0c50be3e1afe14fb6dc5ac920aace0ecc79581eeccf57d892bd32f29d7585bcbea895fa0e3b909e850796bb25d5075cb4c14
+  data.tar.gz: 541f339f68307c55e27cbc182e716dcd591c8d9bc5a018be4e7a7563d1a7bec1239ac85303b2c419791a7142cade0d1b6fc586875adb29717903aefafe34a915

data/.github/workflows/gempush.yml

@@ -0,0 +1,31 @@
+name: Ruby Gem
+
+on:
+  pull_request:
+    branches:
+      - master
+  push:
+    branches:
+      - master
+
+jobs:
+  build:
+    name: Build + Publish
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@master
+    - name: Set up Ruby 2.6
+      uses: actions/setup-ruby@v1
+      with:
+        version: 2.6.x
+    - name: Publish to RubyGems
+      run: |
+        mkdir -p $HOME/.gem
+        touch $HOME/.gem/credentials
+        chmod 0600 $HOME/.gem/credentials
+        printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+        gem build *.gemspec
+        gem push *.gem
+      env:
+        GEM_HOST_API_KEY: ${{secrets.RUBYGEMS_AUTH_TOKEN}}

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.reek.yml

@@ -0,0 +1,3 @@
+detectors:
+  IrresponsibleModule:
+    enabled: false

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,6 @@
+AllCops:
+  DefaultFormatter: simple
+Style/Documentation:
+  Enabled: false
+Naming/RescuedExceptionsVariableName:
+  Enabled: false

data/.ruby-version

@@ -0,0 +1 @@
+2.6.3

data/.travis.yml

@@ -0,0 +1,7 @@
+---
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 2.5.3
+before_install: gem install bundler -v 2.0.2

data/Gemfile

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in gitmq.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,37 @@
+PATH
+  remote: .
+  specs:
+    gitmqueue (0.1.0)
+      rugged (~> 0.28)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.3)
+    rake (10.5.0)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.0)
+      rspec-support (~> 3.9.0)
+    rspec-expectations (3.9.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.0)
+    rugged (0.28.3.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 2.0)
+  gitmqueue!
+  rake (~> 10.0)
+  rspec (~> 3.0)
+
+BUNDLED WITH
+   2.0.2

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2019 Emad Elsaid
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,68 @@
+# GitMQueue : Git Message Queue
+
+Hey, Good idea bad pitch, lets use Git as a messaging queue! lets use github to
+synchronize a cluster of them together.
+
+Lets get the obvious out of the way, You already know that we use Git to keep
+track of our code changes, and we need a central point to pull and push these
+changes (although it's designed to be decentralized), this is why we use
+github/lab/bitbucket...etc.
+
+Alright, now lets introduce another idea: Git commits doesn't need to include
+changes, so you don't have to create files then commit these files to
+your git repo, you can create an empty commit with a command like this:
+
+```
+git commit --allow-empty -m "commit message"
+```
+
+OK, so the commit message can be any text right? that means you can have JSON
+messages or YAML or any other machine readable format as a commit message.
+
+So now imagine this scenario:
+
+1. Create a repo on github
+1. Clone it to your server
+1. Have your application commit a `message` to this repo every time you need to
+1. Push the repo every period say 10 seconds
+1. Clone same repo to another server
+1. Have an application there (consumer) pull periodically
+1. Have Another thread there that checks for new commits and for each one of
+   them extracts the message and does the job
+1. Put a git tag on the last processed message with that consumer identifier
+1. Push that label to keep track of the last processed message (pull it when you
+   deploy that consumer again)
+
+Now you have a queue of messages that can't be altered, communicated between
+servers and all data saved in a git repository.
+
+This idea is flexible and can be modified for several use cases:
+
+1. the central server doesn't have to be a github or a like, you can use your
+   own server as a git server.
+1. You can have more than one queue for messages by creating many branches each
+   for a type of messages or a different messages purpose.
+1. You can have the same branch consumed from different servers each message
+   processed multiple times in different ways like RabbitMQ fan out strategy
+1. You can keep browse your message queue with any git client UI or CLI
+1. You can clean your servers git local repo so that it deletes old commit
+   history to save space.
+1. You can have many messages producers write to different branches in the same
+   repository
+1. In the future you can join 2 branches or split a branch and have your
+   consumers adapt to that
+
+
+There are many advantages to use this approach, which is inherited from how Git
+is designed
+
+1. Every message has an address (git commit hash)
+1. Every message can be traced to an author which is the application that
+   published it (you can use different names for each server or application
+   version)
+1. Every message has a creation date
+
+This idea can as simple as invoking git commands from your application with a
+system call, or use libgit to manipulate the repository for you, or even a small
+server application that communicate over a socket, or a repo service that
+communicate over even HTTP, there are so many ways to implement that concept.

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/bin/gitmqueue-consume

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'gitmqueue'
+
+path = ARGV.shift
+branch = ARGV.shift
+consumer = ARGV.shift
+
+consumer = GitMQueue::Consumer.new(
+  storage: GitMQueue::Storage.new(path),
+  name: consumer,
+  branch: branch
+)
+
+begin
+  consumer.consume { |m| puts m }
+rescue Interrupt
+  puts 'Bye.'
+end

data/bin/gitmqueue-produce

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'gitmqueue'
+
+path = ARGV.shift
+branch = ARGV.shift
+
+producer = GitMQueue::Producer.new(
+  storage: GitMQueue::Storage.new(path),
+  branch: branch
+)
+
+loop do
+  producer.publish(gets.strip)
+rescue Interrupt
+  puts 'Bye.'
+  exit
+end

data/gitmqueue.gemspec

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'gitmqueue'
+  spec.version       = GitMQueue::VERSION
+  spec.authors       = ['Emad Elsaid']
+  spec.email         = ['emad.elsaid.hamed@gmail.com']
+
+  spec.summary       = 'Git Message Queue'
+  spec.description   = <<-DESC
+  An interface to use Git repositories as a message queue to communicate between services'
+  DESC
+  spec.homepage      = 'https://www.github.com/emad-elsaid/gitmqueue'
+  spec.license       = 'MIT'
+
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = spec.homepage
+
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
+  end
+  spec.bindir        = 'bin'
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'rugged', '~> 0.28'
+  spec.add_development_dependency 'bundler', '~> 2.0'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+end

data/lib/consumer.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module GitMQueue
+  class Consumer
+    def initialize(storage:, name:, branch:)
+      @storage = storage
+      @branch = branch
+      @name = name
+      @label = "#{branch}.#{name}"
+    end
+
+    def consume(&block)
+      @storage.wait_branch @branch
+
+      loop do
+        commit = @storage.poll(@branch, @label)
+        block.call commit.message
+        @storage.tag(@label, commit)
+      end
+    end
+  end
+end

data/lib/gitmqueue.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require_relative './version'
+require_relative './consumer'
+require_relative './producer'
+require_relative './storage'
+
+module GitMQueue
+end

data/lib/producer.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module GitMQueue
+  class Producer
+    def initialize(storage:, branch:)
+      @storage = storage
+      @branch = branch
+    end
+
+    def publish(event)
+      commit = Rugged::Commit.create(
+        @storage.repo,
+        tree: @storage.tree,
+        message: event.to_s,
+        parents: [@storage.branch(@branch)&.target].compact
+      )
+      @storage.branches.create(@branch, commit, force: true)
+    end
+  end
+end

data/lib/storage.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'rugged'
+
+module GitMQueue
+  class Storage
+    WAIT_NON_EXISTING_BRANCH = 1 # seconds
+    WAIT_FOR_COMMIT = 1 # seconds
+
+    attr_reader :repo
+
+    def initialize(path)
+      @path = path
+      @repo = read_or_create_repo
+    end
+
+    def branches
+      repo.branches
+    end
+
+    def branch(name)
+      branches[name]
+    end
+
+    def tree
+      Rugged::Tree::Builder.new(repo).write
+    end
+
+    def poll(branch, tag)
+      wait_for_commit(branch, tag)
+      walker(branch, tag).first
+    end
+
+    def tag(label, commit)
+      repo.tags.create(label, commit, true)
+    end
+
+    def wait_branch(branch)
+      sleep WAIT_NON_EXISTING_BRANCH until repo.branches.exist?(branch)
+    end
+
+    private
+
+    attr_reader :path
+
+    def walker(branch, tag)
+      walker = Rugged::Walker.new(repo)
+      walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE)
+      walker.push(branch(branch).target)
+
+      tag = repo.tags[tag]
+      walker.hide(tag.target) if tag
+
+      walker
+    end
+
+    def read_or_create_repo
+      FileUtils.mkdir_p(path) unless File.exist?(path)
+      begin
+        Rugged::Repository.new(path)
+      rescue Rugged::RepositoryError
+        Rugged::Repository.init_at(path, :bare)
+      end
+    end
+
+    def wait_for_commit(branch, tag)
+      sleep WAIT_FOR_COMMIT until branch(branch).target.oid != repo.tags[tag]&.target&.oid
+    end
+  end
+end

data/lib/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module GitMQueue
+  VERSION = '0.1.0'
+end

metadata

@@ -0,0 +1,124 @@
+--- !ruby/object:Gem::Specification
+name: gitmqueue
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Emad Elsaid
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-11-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rugged
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.28'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.28'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: "  An interface to use Git repositories as a message queue to communicate
+  between services'\n"
+email:
+- emad.elsaid.hamed@gmail.com
+executables:
+- gitmqueue-consume
+- gitmqueue-produce
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/workflows/gempush.yml"
+- ".gitignore"
+- ".reek.yml"
+- ".rspec"
+- ".rubocop.yml"
+- ".ruby-version"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/gitmqueue-consume
+- bin/gitmqueue-produce
+- gitmqueue.gemspec
+- lib/consumer.rb
+- lib/gitmqueue.rb
+- lib/producer.rb
+- lib/storage.rb
+- lib/version.rb
+homepage: https://www.github.com/emad-elsaid/gitmqueue
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://www.github.com/emad-elsaid/gitmqueue
+  source_code_uri: https://www.github.com/emad-elsaid/gitmqueue
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Git Message Queue
+test_files: []