rservicebus2 0.1.3 → 0.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 0c20d8a34264145fc0f492c81cb8981042334f0d
-  data.tar.gz: c1b5e70c54069938537d663f0ca90d630611beb0
+SHA256:
+  metadata.gz: c1e12570efafa3e17329ee11b4f458e86ba6d61700dea27f5552e3ef54945cd4
+  data.tar.gz: 0ad1c7926d644e24f795d6baaacba9b18bbb60c51c6f8d2018b7829c2f02433a
 SHA512:
-  metadata.gz: abd57f30d3ec6f0bb29edefb353939e8fb186931ccbdde24e5d05469842cf7e4bd95c79a434698dd9a5eb2442493fa43f8a2d7f6fe2d09c753e5240e288f65eb
-  data.tar.gz: e1eeba5be8ba48df45110188b51d6b66d5f44e8c361c9189007bfb39853e5c8c1b12e76f1f631fd03cb658b36c038b76b27a9fdfd2f6b4280d915a4317450b65
+  metadata.gz: 2db9257198bff90b072fc80f1561e18373b831064f321e3d3238a37904a1026a506276f64d51eb575d21d9d7a5f58d5be4c27246e6e49608b569eb722f7e70fc
+  data.tar.gz: 88f37e1a0d0837baf786ad2dfaf685f4bce862f62eebeca837275e06d37954a4681f0549b54b6e10757d9cce38a4fc1c99e68c58bb59c88be009584c1c0865cc

data/README.md

@@ -0,0 +1,50 @@
+# rservicebus2
+
+Use the examples directory as the starting point.
+
+## General Approach
+
+1. Convention over Configuration
+2. Configuration via Environment Variables
+3. Working examples provided for configuration
+
+## General Config
+
+### APPNAME
+1. Name of process
+2. Doubles as default queueName
+3. Defaults to directory name
+
+### MAX_RETRIES
+1. If an exception is generated while processing a message, the message can be
+retried. This allows the number of retries to be set.
+2. Defaults to 5
+
+### ERROR_QUEUE_NAME
+1. If an exception is generated while processing a message, the message can be
+retried. Once all retries are exhausted, the message will be put in the error queue.
+2. This allows the name of the queue to be set.
+3. Defaults to error
+
+### WORKING_DIR
+1. Where message handlers are looked for.
+2. Defaults to current directory
+
+### VERBOSE
+1. Send out more logging to track message processing
+
+### AUDIT_QUEUE_NAME
+1. When set, a copy of all messages sent and received will be add to this queue
+
+### RSBCRON_[msg name]
+1. An empty message will be created and sent based on the cron string
+
+## RSBMQ - Message queue
+1. Environment Variable Name: RSBMQ
+
+### Beanstalk
+
+### Redis
+
+### SQS
+RSBMQ=aws://[region]/[queue_name]

data/lib/rservicebus2/appresource/awsdynamodb.rb

@@ -0,0 +1,16 @@
+require 'aws-sdk-dynamodb'
+
+module RServiceBus2
+  # AppResourceAWSDynamoDb
+  class AppResourceAWSDynamoDb < AppResource
+    # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+    def connect(uri)
+      region = uri.host
+
+      Aws::DynamoDB::Client.new(region: region)
+    end
+
+    def finished
+    end
+  end
+end

data/lib/rservicebus2/appresource/awss3.rb

@@ -0,0 +1,16 @@
+require 'aws-sdk-s3'
+
+module RServiceBus2
+  # AppResourceAWSDynamoDb
+  class AppResourceAWSS3 < AppResource
+    # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+    def connect(uri)
+      region = uri.host
+
+      Aws::S3::Client.new(region: region)
+    end
+
+    def finished
+    end
+  end
+end

data/lib/rservicebus2/appresource_configure.rb

@@ -31,6 +31,12 @@ module RServiceBus2
           when 'file'
             require 'rservicebus2/appresource/file'
             rm.add k.sub('RSB_', ''), AppResourceFile.new(host, uri)
+          when 'awsdynamodb'
+            require 'rservicebus2/appresource/awsdynamodb'
+            rm.add k.sub('RSB_', ''), AppResourceAWSDynamoDb.new(host, uri)
+          when 'awss3'
+            require 'rservicebus2/appresource/awss3'
+            rm.add k.sub('RSB_', ''), AppResourceAWSS3.new(host, uri)
           else
             abort("Scheme, #{uri.scheme}, not recognised when configuring
                   app resource, #{k}=#{v}")

data/lib/rservicebus2/host.rb

@@ -43,8 +43,8 @@ module RServiceBus2
 
     # Thin veneer for Configuring state
     def configure_saga_storage
-      string = RServiceBus2.get_value('SAGA_URI')
-      string = 'dir:///tmp' if string.nil?
+      string = RServiceBus2.get_value('SAGA_URI', 'dir:///tmp/saga')
+      # string = 'dir:///tmp' if string.nil?
 
       uri = URI.parse(string)
       @saga_storage = SagaStorage.get(uri)

data/lib/rservicebus2/monitor.rb

@@ -26,6 +26,7 @@ module RServiceBus2
     def initialize(bus, name, uri)
       @bus = bus
       new_anonymous_class = Class.new(MonitorMessage)
+      puts "name: #{name}"
       Object.const_set(name, new_anonymous_class)
       @msg_type = Object.const_get(name)
 

data/lib/rservicebus2/monitor/awss3.rb

@@ -0,0 +1,73 @@
+require 'aws-sdk-s3'
+
+module RServiceBus2
+  # Monitor S3 Bucket for objects
+  class MonitorAWSS3 < Monitor
+    def connect(uri)
+      @bucket_name = uri.path
+      @bucket_name[0] = ''
+
+      @region = uri.host
+
+      @s3_client = Aws::S3::Client.new(region: @region)
+      @input_filter = []
+    end
+
+    def process_content(content)
+      content
+    end
+
+    # rubocop:disable Metrics/MethodLength
+    def read_content_from_file(file_path)
+      content = ''
+      if @input_filter.length > 0
+        if @input_filter[0] == 'ZIP'
+          content = read_content_from_zip_file(file_path)
+        elsif @input_filter[0] == 'GZ'
+          content = read_content_from_gz_file(file_path)
+        elsif @input_filter[0] == 'TAR'
+          fail 'TAR reader not implemented'
+        end
+
+      else
+        content = IO.read(file_path)
+      end
+
+      content
+    end
+
+    def process_path(object_key)
+      # content = read_content_from_object(object_key)
+      content = object_key
+      payload = process_content(content)
+
+      send(payload, URI.parse(URI.encode("s3://#{@region}/#{@bucket_name}/#{object_key}")))
+      content
+    end
+
+    def look
+      file_processed = 0
+      max_files_processed = 1
+
+      objects = @s3_client.list_objects_v2(
+          bucket: @bucket_name,
+          max_keys: max_files_processed
+        ).contents
+
+      return if objects.count.zero?
+
+      objects.each do |object|
+        RServiceBus2.log "Ready to process, #{object.key}"
+        process_path(object.key)
+        # content = process_path(object.key)
+
+        # File.unlink(file_path)
+
+        file_processed += 1
+        RServiceBus2.log "Processed #{file_processed} of #{objects.length}."
+        RServiceBus2.log "Allow system tick #{self.class.name}"
+        break if file_processed >= max_files_processed
+      end
+    end
+  end
+end

data/lib/rservicebus2/monitor_configure.rb

@@ -48,6 +48,9 @@ module RServiceBus2
           when 'dir'
             require 'rservicebus2/monitor/dir'
             monitor = MonitorDir.new(@host, name, uri)
+          when 'awss3'
+            require 'rservicebus2/monitor/awss3'
+            monitor = MonitorAWSS3.new(@host, name, uri)
           when 'dirnotifier'
             require 'rservicebus2/monitor/dirnotifier'
             monitor = MonitorDirNotifier.new(@host, name, uri)

data/lib/rservicebus2/mq.rb

@@ -20,7 +20,13 @@ module RServiceBus2
         mq = MQBeanstalk.new(uri)
       when 'file'
         require 'rservicebus2/mq/file'
-	mq = MQFile.new(uri)
+        mq = MQFile.new(uri)
+      when 'redis'
+        require 'rservicebus2/mq/redis'
+        mq = MQRedis.new(uri)
+      when 'aws'
+        require 'rservicebus2/mq/aws'
+        mq = MQAWS.new(uri)
       else
         abort("Scheme, #{uri.scheme}, not recognised when configuring mq,
           #{string}")

data/lib/rservicebus2/mq/aws.rb

@@ -0,0 +1,85 @@
+require 'aws-sdk-sqs'
+require 'aws-sdk-sts'
+require 'rservicebus2/mq'
+
+module RServiceBus2
+  # Beanstalk client implementation.
+  class MQAWS < MQ
+    # Connect to the broker
+    # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+    def connect(region, port)
+      begin
+        @max_job_size = 4_194_304
+        @region = region
+
+        sts_client = Aws::STS::Client.new(region: region)
+        @caller_identity_account = sts_client.get_caller_identity.account
+      rescue StandardError => e
+        puts 'Error connecting to AWS'
+        puts "Host string, #{region}"
+        puts e.message
+        puts e.backtrace
+        abort
+      end
+    end
+
+    def subscribe(queuename)
+      # For example:
+      # 'https://sqs.us-east-1.amazonaws.com/111111111111/my-queue'
+      @queue_url = 'https://sqs.' + @region + '.amazonaws.com/' +
+        @caller_identity_account + '/' + queuename
+      @sqs_client = Aws::SQS::Client.new(region: @region)
+    end
+
+    # Get next msg from queue
+    def pop
+      begin
+        response = @sqs_client.receive_message(
+          queue_url: @queue_url,
+          max_number_of_messages: 1
+        )
+
+        if response.messages.count.zero?
+          raise NoMsgToProcess
+        end
+
+        response.messages.each do |message|
+          @job = message
+        end
+      rescue StandardError => e
+        raise e
+      end
+      @job.body
+    end
+
+    def return_to_queue
+#      @job.release if @job
+      @job = nil
+    end
+
+    def ack
+      @sqs_client.delete_message({
+        queue_url: @queue_url,
+        receipt_handle: @job.receipt_handle
+      })
+      @job = nil
+    end
+
+    def send(queue_name, msg)
+      if msg.length > @max_job_size
+        puts '***Attempting to send a msg which will not fit on queue.'
+        puts "***Msg size, #{msg.length}, max msg size, #{@max_job_size}."
+        fail JobTooBigError, "Msg size, #{msg.length}, max msg size,
+              #{@max_job_size}"
+      end
+
+      queue_url = 'https://sqs.' + @region + '.amazonaws.com/' +
+        @caller_identity_account + '/' + queue_name
+
+      @sqs_client.send_message(
+        queue_url: queue_url,
+        message_body: msg
+      )
+    end
+  end
+end

data/lib/rservicebus2/mq/redis.rb

@@ -0,0 +1,53 @@
+require 'redis'
+require 'rservicebus2/mq'
+
+module RServiceBus2
+  # Redis client implementation.
+  class MQRedis < MQ
+    # Connect to the broker
+    # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+    def connect(host, port)
+      port ||= 6379
+      string = "#{host}:#{port}"
+
+      begin
+        @redis = Redis.new(:host => host, :port => port)
+
+      rescue Exception => e
+        puts e.message
+        puts 'Error connecting to Redis for mq'
+        puts "Host string, #{string}"
+        abort()
+      end
+    end
+
+    # Connect to the queue
+    def subscribe( queuename )
+      @queuename = queuename
+    end
+
+    # Get next msg from queue
+    def pop
+      if @redis.llen( @queuename ) == 0 then
+        sleep @timeout
+        raise NoMsgToProcess.new
+      end
+
+      return @redis.lindex @queuename, 0
+    end
+
+    def returnToQueue
+    end
+
+    # "Commit" queue
+    def ack
+      @redis.lpop @queuename
+    end
+
+    # At least called in the Host rescue block, to ensure all network links are healthy
+    def send( queueName, msg )
+      @redis.rpush queueName, msg
+    end
+
+  end
+end

data/lib/rservicebus2/saga/manager.rb

@@ -24,7 +24,7 @@ module RServiceBus2
     end
 
     def get_start_with_method_names(saga)
-      get_methods_by_prefix(saga, 'startwith_')
+      get_methods_by_prefix(saga, 'start_with_')
     end
 
     # setBusAttributeIfRequested
@@ -63,7 +63,7 @@ module RServiceBus2
         @start_with[msg_name] = [] if @start_with[msg_name].nil?
         @start_with[msg_name] << s
 
-        RServiceBus2.log "Registered, #{saga.name}, to startwith, #{msg_name}", true
+        RServiceBus2.log "Registered, #{saga.name}, to start_with, #{msg_name}", true
       end
 
       @saga[saga.name] = s
@@ -88,12 +88,13 @@ module RServiceBus2
       msg_class_name = msg.class.name.downcase
 
       RServiceBus2.log "SagaManager, started processing, #{msg_class_name}", true
-      unless @start_with[msg_class_name].nil?
+      if @start_with.key?(msg_class_name)
+#      unless @start_with[msg_class_name].nil?
         @start_with[msg_class_name].each do |saga|
           data = SagaData.new(saga)
           @saga_storage.set(data)
 
-          method_name = "startwith_#{msg_class_name}"
+          method_name = "start_with_#{msg_class_name}"
           process_msg(saga, data, method_name, msg)
 
           handled = true
@@ -102,8 +103,10 @@ module RServiceBus2
       return handled if handled == true
 
       return false if rmsg.correlation_id.nil?
+
       data = @saga_storage.get(rmsg.correlation_id)
-      return handled if data.nil?
+      return false if data.nil?
+
       method_name = "handle_#{msg_class_name}"
       saga = @saga[data.saga_class_name]
       process_msg(saga, data, method_name, msg)

data/lib/rservicebus2/saga_loader.rb

@@ -16,21 +16,21 @@ module RServiceBus2
     # Cleans the given path to ensure it can be used for as a parameter for the
     #  require statement.
     # @param [String] file_path the path to be cleaned
-    def get_require_path(filePath)
+    def get_require_path(file_path)
       file_path = './' + file_path unless file_path.start_with?('/')
 
       return file_path.sub('.rb', '') if File.exist?(file_path)
 
-      abort('Filepath, ' + filePath + ", given for Saga require doesn't exist")
+      abort('Filepath, ' + file_path + ", given for Saga require doesn't exist")
     end
 
-    # Instantiate the saga named in sagaName from the file name in filePath
+    # Instantiate the saga named in sagaName from the file name in file_path
     # Exceptions will be raised if encountered when loading sagas. This is a
     # load time activity, so sagas should load correctly. As much information
     # as possible is returned to enable the saga to be fixed, or configuration
     # corrected.
     # @param [String] sagaName name of the saga to instantiate
-    # @param [String] filePath the path to the file to be loaded
+    # @param [String] file_path the path to the file to be loaded
     # @return [RServiceBus2::Saga] the loader
     def load_saga_from_file(saga_name, file_path)
       require_path = get_require_path(file_path)
@@ -50,7 +50,7 @@ module RServiceBus2
     end
 
     # Wrapper function
-    # @param [String] filePath
+    # @param [String] file_path
     # @param [String] sagaName
     # @returns [RServiceBus2::Saga] saga
     def load_saga(file_path, saga_name)
@@ -97,7 +97,7 @@ module RServiceBus2
 
       saga_name = base_name.sub(ext_name, '')
 
-      "Saga_#{saga_name}"
+      "saga_#{saga_name}".gsub(/(?<=_|^)(\w)/){$1.upcase}.gsub(/(?:_)(\w)/,'\1')
     end
 
     # Entry point for loading Sagas
@@ -106,7 +106,7 @@ module RServiceBus2
       RServiceBus2.rlog "SagaLoader.loadSagasFromPath. base_dir: #{base_dir}"
 
       get_list_of_files_for_dir(base_dir).each do |file_path|
-        unless filePath.end_with?('.')
+        unless file_path.end_with?('.')
           saga_name = get_saga_name(file_path)
           load_saga(file_path, saga_name)
         end

data/lib/rservicebus2/saga_storage/dir.rb

@@ -75,7 +75,7 @@ module RServiceBus2
     end
 
     def load(path)
-      return {} unless File.exist?(path)
+      return nil unless File.exist?(path)
 
       content = IO.read(path)
 

metadata

@@ -1,85 +1,85 @@
 --- !ruby/object:Gem::Specification
 name: rservicebus2
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.5
 platform: ruby
 authors:
 - Guy Irvine
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-06-15 00:00:00.000000000 Z
+date: 2021-04-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: uuidtools
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.2.0
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.5.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.5.1
 - !ruby/object:Gem::Dependency
   name: beanstalk-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.1.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.1.1
 - !ruby/object:Gem::Dependency
   name: fluiddb
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.1.19
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.1.19
 - !ruby/object:Gem::Dependency
   name: parse-cron
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.1.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.1.4
 description: A Ruby interpretation of NServiceBus
 email: guy@guyirvine.com
 executables:
@@ -93,6 +93,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - LICENSE
+- README.md
 - bin/return_messages_to_source_queue
 - bin/rsb_ctl
 - bin/rservicebus2
@@ -103,6 +104,8 @@ files:
 - lib/rservicebus2.rb
 - lib/rservicebus2/agent.rb
 - lib/rservicebus2/appresource.rb
+- lib/rservicebus2/appresource/awsdynamodb.rb
+- lib/rservicebus2/appresource/awss3.rb
 - lib/rservicebus2/appresource/dir.rb
 - lib/rservicebus2/appresource/file.rb
 - lib/rservicebus2/appresource/fluiddb.rb
@@ -123,14 +126,17 @@ files:
 - lib/rservicebus2/message/subscription.rb
 - lib/rservicebus2/message/verboseoutput.rb
 - lib/rservicebus2/monitor.rb
+- lib/rservicebus2/monitor/awss3.rb
 - lib/rservicebus2/monitor/dir.rb
 - lib/rservicebus2/monitor/dirnotifier.rb
 - lib/rservicebus2/monitor/message.rb
 - lib/rservicebus2/monitor/xmldir.rb
 - lib/rservicebus2/monitor_configure.rb
 - lib/rservicebus2/mq.rb
+- lib/rservicebus2/mq/aws.rb
 - lib/rservicebus2/mq/beanstalk.rb
 - lib/rservicebus2/mq/file.rb
+- lib/rservicebus2/mq/redis.rb
 - lib/rservicebus2/resource_manager.rb
 - lib/rservicebus2/saga/base.rb
 - lib/rservicebus2/saga/data.rb
@@ -161,7 +167,7 @@ homepage: http://rubygems.org/gems/rservicebus2
 licenses:
 - LGPL-3.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -176,9 +182,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2.1
-signing_key: 
+rubygems_version: 3.0.9
+signing_key:
 specification_version: 4
 summary: RServiceBus
 test_files: []