hoze 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 13ccaea71486a72b78f2c505eee8c33f6262ef22
-  data.tar.gz: 101857b73c29bfc24f96b8be3392fe76f2708da2
+  metadata.gz: 53fef088aa9de3cf52f33960c03effe4f2f20be1
+  data.tar.gz: 9d779070f50f18d770c5738bdd70045c4bdff96c
 SHA512:
-  metadata.gz: bf093a19e8b9f0633d73e19ae631dad346a323fd8ec5cc4174ae1040520d4ab3b98b1831fbac6cb91d0df1610796b67533ced7d8377f8f50d80a6d74b8d6f72c
-  data.tar.gz: b820e09cf639dc65e49cf919ad0859d49a13a2bdc6919125dde6765ad2d01991530809ac847b3ce419e1666d834749b5ceb7f9f3d95ec1a98092dcb55c6cdf26
+  metadata.gz: b24fcb64027938e64d59d40e7d2801ac474d62bff4527cdb68364f1e89b3039bec9f9b2d219685eb4afbe1f6437d217c1916b14dec9c89df95aea90b208d3009
+  data.tar.gz: 58e8cd2d9fd52c54542055a3c341ebf1455ebe2f0d4d0c7f20ca63142348d9064511cc210f09f51eadd5b2734c9fdf31d92e46e36d1799368305973ebe323124

data/README.md

@@ -16,11 +16,13 @@ And then execute:
 
 ## Usage
 
+### Basic usage
+
 Write a simple worker file `worker.rb`:
 
 ```ruby
 Hoze.worker
-.configure do |config|
+.listen_to do |config| # Define where to find messages
   # You can use env variable for each property to overload hardcoded values
   config.channel = "my.pubsub.topic" # HOZE_CONFIG_CHANNEL
   config.key = "my.subscription.name" # HOZE_CONFIG_KEY
@@ -28,9 +30,18 @@ Hoze.worker
   config.connector.project = "my-gcp-project" # HOZE_CONFIG_CONNECTOR_PROJECT
   config.connector.path_to_key = "/path/to/keyfile.json" # HOZE_CONFIG_CONNECTOR_PATH_TO_KEY
 end
-.process do |message|
+.process do |message| # Define job function
   puts "Hello #{message.payload} !"
+  "my result"
+end
+.push_to do |config| # Define where to push job result
+  # You can use env variable for each property to overload hardcoded values
+  config.channel = "my.other.pubsub.topic" # HOZE_CONFIG_DEST_CHANNEL
+  config.connector.type = "pubsub" # HOZE_CONFIG_CONNECTOR_DEST_TYPE
+  config.connector.project = "my-gcp-project" # HOZE_CONFIG_CONNECTOR_DEST_PROJECT
+  config.connector.path_to_key = "/path/to/keyfile.json" # HOZE_CONFIG_CONNECTOR_DEST_PATH_TO_KEY
 end
+.go!
 ```
 
 And now run it:
@@ -39,6 +50,95 @@ And now run it:
 bundle exec hoze worker.rb
 ```
 
+It'll now listen to `my.subscription.name` for any message, print the payload to `STDOUT` then push the message `my result` to `my.other.pubsub.topic`.
+
+### Acknowledgement, delay and rejection
+
+By default, messages must be acknowledged explicitly. If messages are not acked before a time depending on the source implementation, they'll be sent again to the subscribers. Your worker may ask for more time before the timeout expires if the processing is known to be long. It may also `reject` a message, telling the source it won't process it and then sending it bck to its queue.
+You may also define an automatic acknowledgement if you don't want to handle this by hand.
+
+#### Auto-ack
+
+```ruby
+Hoze.worker
+.listen_to do |config|
+    # ...
+    config.auto_ack = true # Will ack the message as soon as it is received
+end
+.process do |message|
+    # ...
+end
+.go!
+```
+
+#### Manual ack
+
+```ruby
+Hoze.worker
+.listen_to do |config|
+    # ...
+end
+.process do |message|
+    # ...
+    message.ack!
+end
+.go!
+```
+
+#### Delay and rejection
+
+```ruby
+Hoze.worker
+.listen_to do |config|
+    # ...
+end
+.process do |message|
+    # pretty long processing
+    message.delay! 10 # asking for 10 more seconds
+    # continue long processing
+    message.reject! # finally reject the message
+end
+.go!
+```
+
+### Retries
+
+Failed processing may be retried. Set the config variable `max_tries` to tell how many times it's gonna be tried before it's considered dead.
+
+```ruby
+Hoze.worker
+.listen_to do |config|
+    # ...
+    config.max_tries = 5
+end
+.process do |message|
+    # Failable processing
+end
+.go!
+```
+
+*Beware* the retry system is still alpha. Messages will be pushed back immediately in the queue with a `tries` metadata recording the tries count. Dead messages will just be logged.
+
+### Metadata transfer
+
+When the result is pushed to another topic, original message's metadata are transfered along. They can be modified in the `process` block:
+
+```ruby
+Hoze.worker
+.listen_to do |config|
+    # ...
+end
+.process do |message|
+  message.metadata[:who_is_the_best] = "I am the best"
+  "my result"
+end
+.push_to do |config| 
+    # ...
+end
+.go!
+```
+
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/hoze/configuration.rb

@@ -5,8 +5,9 @@ module Hoze
   class Configuration
     attr_accessor :channel, :key, :auto_ack, :connector, :max_tries
 
-    def initialize
-      @connector = ConnectorConfiguration.new
+    def initialize prefix = nil
+      @connector = ConnectorConfiguration.new prefix
+      @prefix = prefix
     end
 
     def channel
@@ -20,7 +21,8 @@ module Hoze
     private
 
     def env_config name
-      return ENV["HOZE_CONFIG_#{name}"]
+      key = ["HOZE","CONFIG",@prefix,name].join('_')
+      return ENV[key]
     end
 
   end
@@ -28,6 +30,10 @@ module Hoze
   class ConnectorConfiguration
     attr_accessor :type, :project, :path_to_key
 
+    def initialize prefix = nil
+      @prefix = prefix
+    end
+
     def type
       env_config('TYPE') || @type
     end
@@ -43,7 +49,8 @@ module Hoze
     private
 
     def env_config name
-      return ENV["HOZE_CONFIG_CONNECTOR_#{name}"]
+      key = ["HOZE","CONFIG","CONNECTOR",@prefix,name].join('_')
+      return ENV[key]
     end
   end
 

data/lib/hoze/interface/source.rb

@@ -7,6 +7,10 @@ module Hoze
     def listen &block
       raise NotImplementedError.new("This source implementation doesn't support listen method.")
     end
+
+    def push payload, metadata
+      raise NotImplementedError.new("This source implementation doesn't support push method.")
+    end
     
   end
 

data/lib/hoze/interface/source_error.rb

@@ -0,0 +1,4 @@
+module Hoze
+  class SourceError < Exception
+  end
+end

data/lib/hoze/pubsub/source.rb

@@ -1,4 +1,5 @@
 require "hoze/interface/source"
+require "hoze/interface/source_error"
 require "hoze/pubsub/message"
 
 module Hoze
@@ -24,7 +25,7 @@ module Hoze
 
 
     def listen &block
-      #puts @subscription.inspect
+      raise HozeSourceError.new("Tryng to listen source but no key configured") if @key.nil?
       subscriber = @subscription.listen do |received_message|
         begin
           msg = Hoze::PubSubMessage.new(received_message, self)
@@ -46,15 +47,22 @@ module Hoze
         subscriber.stop.wait!
       end
     end
+
+    def push payload, metadata
+      @topic.publish_async payload, metadata
+    end
     
     private
 
     def ensure_topic
-      @engine.topic(@channel)||@engine.create_topic(@channel)
+      raise HozeSourceError.new("No topic configured for source") if @channel.nil?
+      @engine.topic(@channel) || @engine.create_topic(@channel)
     end
 
     def ensure_subscription
-      @topic.subscription(@key)||@topic.subscribe(@key)
+      return nil if @key.nil?
+      subname = [@channel,@key].join('+')
+      @topic.subscription(subname) || @topic.subscribe(subname)
     end
 
   end

data/lib/hoze/version.rb

@@ -1,3 +1,3 @@
 module Hoze
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/hoze/worker.rb

@@ -11,23 +11,38 @@ module Hoze
     attr_accessor :source_factory
 
     def initialize
-      @configuration = Hoze::Configuration.new
+      @source_config = Hoze::Configuration.new
       @source_factory = Hoze::SourceFactory.new
     end
 
-    def configure &block
-      yield @configuration
-      puts "Configuration is: #{@configuration.inspect}"
+    def listen_to &block
+      yield @source_config
+      puts "Source Configuration is: #{@source_config.inspect}"
       self
     end
 
-    def process &block   
-      source = @source_factory.build(@configuration)
-      
-      source.listen do |msg|
+    def push_to &block
+      @destination_config = Hoze::Configuration.new 'DEST'
+      yield @destination_config
+      puts "Destination Configuration is: #{@destination_config.inspect}"
+      self
+    end
+
+    def process &block
+      @process_fn = Proc.new do |msg|
+        yield msg
+      end
+      self
+    end
+
+    def go!
+      @source = @source_factory.build(@source_config)
+      @destination = @source_factory.build(@destination_config) if @destination_config
+      @source.listen do |msg|
         begin
-          msg.ack! if @configuration.auto_ack
-          result = yield msg
+          msg.ack! if @source_config.auto_ack
+          result = @process_fn.call msg
+          @destination.push result, msg.metadata if @destination
         rescue Exception => e
           msg.retry!
           raise e

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hoze
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - pluce
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-03-05 00:00:00.000000000 Z
+date: 2018-03-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -90,6 +90,7 @@ files:
 - lib/hoze/configuration.rb
 - lib/hoze/interface/message.rb
 - lib/hoze/interface/source.rb
+- lib/hoze/interface/source_error.rb
 - lib/hoze/pubsub/message.rb
 - lib/hoze/pubsub/source.rb
 - lib/hoze/source_factory.rb