gitmqueue 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7e42924f796ddd596ee51c69e7d837da4988ce63db27688998e43260570ad9ec
-  data.tar.gz: 8314d9f06183362f39f4831dc8442dfaaeb879bb23860bab6bc8cdf9ea7ebb98
+  metadata.gz: fef08ee9fcfc3c24da047d2c1a0aa5e760127bf47ab665e7962590c2be239744
+  data.tar.gz: c49766c95c5d7b3848c54d1a06cacbc594abaaaab88004129602a0824a70756b
 SHA512:
-  metadata.gz: c9459c7b8990fe2c62f24d5bdcdd0c50be3e1afe14fb6dc5ac920aace0ecc79581eeccf57d892bd32f29d7585bcbea895fa0e3b909e850796bb25d5075cb4c14
-  data.tar.gz: 541f339f68307c55e27cbc182e716dcd591c8d9bc5a018be4e7a7563d1a7bec1239ac85303b2c419791a7142cade0d1b6fc586875adb29717903aefafe34a915
+  metadata.gz: 0b987755d7ff9dc3133c7a4ba8e67ede3fbfdd9967a44b955f60e775ba8ffeb2db4a5d6682c1296a28b233b7f9de80103cc3245476261fc04cedd6482e7334db
+  data.tar.gz: feb8ddc0b1f77d4ad37d61ba88e46acad0791f576d05fefb7bdd1a216214211989ad4f7735ad93f2ad4de54ec30ccabf6a3482d4c036f0aaf9c2e86e1d08b564

data/Gemfile.lock

@@ -2,13 +2,21 @@ PATH
   remote: .
   specs:
     gitmqueue (0.1.0)
+      listen (~> 3.2.0)
       rugged (~> 0.28)
 
 GEM
   remote: https://rubygems.org/
   specs:
     diff-lcs (1.3)
+    ffi (1.11.3)
+    listen (3.2.0)
+      rb-fsevent (~> 0.10, >= 0.10.3)
+      rb-inotify (~> 0.9, >= 0.9.10)
     rake (10.5.0)
+    rb-fsevent (0.10.3)
+    rb-inotify (0.10.0)
+      ffi (~> 1.0)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)

data/README.md

@@ -1,34 +1,38 @@
 # GitMQueue : Git Message Queue
 
-Hey, Good idea bad pitch, lets use Git as a messaging queue! lets use github to
+Hey, Crazy idea bad pitch, lets use Git as a messaging queue! lets use github to
 synchronize a cluster of them together.
 
-Lets get the obvious out of the way, You already know that we use Git to keep
-track of our code changes, and we need a central point to pull and push these
-changes (although it's designed to be decentralized), this is why we use
-github/lab/bitbucket...etc.
+![Screenshot-2019-11-24_22-41-55](https://user-images.githubusercontent.com/54403/69502136-09672400-0f0c-11ea-8707-df3164411c4e.png)
 
-Alright, now lets introduce another idea: Git commits doesn't need to include
-changes, so you don't have to create files then commit these files to
+
+Lets get the obvious out of the way.
+
+1. You already know that we use Git to keep track of our code changes, If you're a
+developer you most probably use it every day, and you probably know how to share
+your code changes with everybody else using some central repo server like Github
+or Bitbucket.
+
+1. Alright, now lets introduce another idea: **Git commits doesn't need to include
+changes**, You don't have to create files then commit these files to
 your git repo, you can create an empty commit with a command like this:
 
 ```
 git commit --allow-empty -m "commit message"
 ```
 
-OK, so the commit message can be any text right? that means you can have JSON
+1. OK, so the commit message can be any text right? that means you can have JSON
 messages or YAML or any other machine readable format as a commit message.
 
-So now imagine this scenario:
+## Now imagine this use case:
 
-1. Create a repo on github
+1. Create a repository on github
 1. Clone it to your server
 1. Have your application commit a `message` to this repo every time you need to
 1. Push the repo every period say 10 seconds
 1. Clone same repo to another server
-1. Have an application there (consumer) pull periodically
-1. Have Another thread there that checks for new commits and for each one of
-   them extracts the message and does the job
+1. Have an application pull periodically and checks for new commits and for each
+   one of them extracts the message and process it
 1. Put a git tag on the last processed message with that consumer identifier
 1. Push that label to keep track of the last processed message (pull it when you
    deploy that consumer again)
@@ -36,14 +40,14 @@ So now imagine this scenario:
 Now you have a queue of messages that can't be altered, communicated between
 servers and all data saved in a git repository.
 
-This idea is flexible and can be modified for several use cases:
+This idea is flexible and can be altered for several use cases:
 
-1. the central server doesn't have to be a github or a like, you can use your
+1. the central server doesn't have to be Github or a like, you can use your
    own server as a git server.
 1. You can have more than one queue for messages by creating many branches each
    for a type of messages or a different messages purpose.
 1. You can have the same branch consumed from different servers each message
-   processed multiple times in different ways like RabbitMQ fan out strategy
+   processed multiple times in different ways like RabbitMQ fanout strategy
 1. You can keep browse your message queue with any git client UI or CLI
 1. You can clean your servers git local repo so that it deletes old commit
    history to save space.
@@ -66,3 +70,105 @@ This idea can as simple as invoking git commands from your application with a
 system call, or use libgit to manipulate the repository for you, or even a small
 server application that communicate over a socket, or a repo service that
 communicate over even HTTP, there are so many ways to implement that concept.
+
+The current repository is an implementation of this concept in Ruby.
+
+# Installation
+
+Install the gem with
+```
+gem install gitmqueue
+```
+
+or add gitmqueue to your `Gemfile`
+```
+gem 'gitmqueue', '~> 0.1'
+```
+
+
+# Producing messages
+
+First create an instance of `GitMQueue::Storage`, to specify your storage place.
+
+```ruby
+storage = GitMQueue::Storage.new('/tmp/gitmqueue')
+```
+
+This is where your git repository will live, message producer will use this
+storage instance to write messages to any branch you wish.
+
+```ruby
+producer = GitMQueue::Producer.new(storage: storage, branch: 'master')
+```
+
+Now you can use `producer` to publish message to this branch.
+
+```ruby
+producer.publish('Hello, world!')
+```
+
+It will create a commit with `Hello, world!` as a message in the `master`
+branch.
+
+You can have as many producers as you wish for any number of branches, make sure
+to have one producer per branch to prevent race conditions between producers.
+
+# Consuming messages
+
+You need to create storage instance as we did with the producer, then
+instanciate your consumer
+
+```ruby
+consumer = GitMQueue::Consumer.new(storage: storage, name: 'consumer', branch:
+'master')
+```
+
+And start the consume process, this function will take a block of code that
+accept one parameter which is your message, this function is non-blocking
+it will execute the block of code each time a new commit appear in the branch
+
+```ruby
+consumer.consume do |message|
+    puts message
+end
+```
+
+After your block finish execution the consumer will tag this commit with a git
+tag, the tag name will be the consumer name, if your consumer name is
+`consumer1`, the tag will be `consumer1`, as git tags are global to all the
+repository you need to make the consumer name unique not just for the branch but
+for all the repository, so if you want to consume 2 branches from one consumer
+you'll have to choose to different names so that tags are not overriden be every
+consumer instance, you can adopt a naming scheme that involve the branch name,
+while implementing this gem we had that name tag name as `branch.consumer` but
+then thought it'll be better for the user to make this decision to allow for a
+more flexible naming, the tag will be overritten after each time the consumer
+process a message.
+
+# Pushing and Pulling your data
+
+As for now there is no way to push/pull from the gem, so this part is up to you,
+until it's implemented in the gem.
+
+# Gem executables
+
+## gitmqueue-produce
+
+Starts a process that waits you to enter any line and publish that as a message
+
+Usage:
+
+```
+gitmqueue-produce /path/to/rpo branch-name
+```
+
+## gitmqueue-consume
+
+Starts a process that waits for messages, each message will be written to the
+terminal
+
+Usage:
+
+```
+gitmqueue-consume /path/to/repo branch-name consumer-name
+```

data/bin/gitmqueue-consume

@@ -13,8 +13,10 @@ consumer = GitMQueue::Consumer.new(
   branch: branch
 )
 
+consumer.consume { |m| puts m }
+
 begin
-  consumer.consume { |m| puts m }
+  sleep
 rescue Interrupt
   puts 'Bye.'
 end

data/gitmqueue.gemspec

@@ -28,6 +28,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'rugged', '~> 0.28'
+  spec.add_dependency 'listen', '~> 3.2.0'
   spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'rspec', '~> 3.0'

data/lib/consumer.rb

@@ -1,21 +1,37 @@
 # frozen_string_literal: true
 
+require 'listen'
+
 module GitMQueue
   class Consumer
     def initialize(storage:, name:, branch:)
       @storage = storage
       @branch = branch
       @name = name
-      @label = "#{branch}.#{name}"
+    end
+
+    def branch_file
+      @branch_file ||= File.join(@storage.path, 'refs', 'heads')
     end
 
     def consume(&block)
-      @storage.wait_branch @branch
+      @block = block
+      listener.start
+    end
+
+    def stop
+      listener.stop
+    end
+
+    private
 
-      loop do
-        commit = @storage.poll(@branch, @label)
-        block.call commit.message
-        @storage.tag(@label, commit)
+    def listener
+      @listener ||= Listen.to(branch_file, only: /#{Regexp.escape(@branch)}/) do
+        commits = @storage.commits(@branch, @name)
+        commits.each do |commit|
+          @block.call commit.message
+          @storage.tag(@name, commit)
+        end
       end
     end
   end

data/lib/storage.rb

@@ -5,10 +5,7 @@ require 'rugged'
 
 module GitMQueue
   class Storage
-    WAIT_NON_EXISTING_BRANCH = 1 # seconds
-    WAIT_FOR_COMMIT = 1 # seconds
-
-    attr_reader :repo
+    attr_reader :repo, :path
 
     def initialize(path)
       @path = path
@@ -27,34 +24,22 @@ module GitMQueue
       Rugged::Tree::Builder.new(repo).write
     end
 
-    def poll(branch, tag)
-      wait_for_commit(branch, tag)
-      walker(branch, tag).first
-    end
-
-    def tag(label, commit)
-      repo.tags.create(label, commit, true)
-    end
-
-    def wait_branch(branch)
-      sleep WAIT_NON_EXISTING_BRANCH until repo.branches.exist?(branch)
-    end
-
-    private
-
-    attr_reader :path
-
-    def walker(branch, tag)
+    def commits(branch, tag)
       walker = Rugged::Walker.new(repo)
       walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE)
       walker.push(branch(branch).target)
 
       tag = repo.tags[tag]
       walker.hide(tag.target) if tag
-
       walker
     end
 
+    def tag(label, commit)
+      repo.tags.create(label, commit, true)
+    end
+
+    private
+
     def read_or_create_repo
       FileUtils.mkdir_p(path) unless File.exist?(path)
       begin
@@ -63,9 +48,5 @@ module GitMQueue
         Rugged::Repository.init_at(path, :bare)
       end
     end
-
-    def wait_for_commit(branch, tag)
-      sleep WAIT_FOR_COMMIT until branch(branch).target.oid != repo.tags[tag]&.target&.oid
-    end
   end
 end

data/lib/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module GitMQueue
-  VERSION = '0.1.0'
+  VERSION = '0.1.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitmqueue
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Emad Elsaid
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-24 00:00:00.000000000 Z
+date: 2019-11-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rugged
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.28'
+- !ruby/object:Gem::Dependency
+  name: listen
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.2.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement