thunderwire 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 985b8c17c616f7c9a76e56313dc46e186875b479483ffa426c0745a9255f6514
+  data.tar.gz: 02a422c11d63251542d90466d52d2cf56fe292e6437d60d5b65b66373fdb3654
+SHA512:
+  metadata.gz: 76635e57f908b2594521bcdd9965c5295a10b76649d0116713d7b26a7c05ba8d2b544a843c16264bcfc12f8a5450ce0dc2d9284f9994cf558b43a2353510e4e0
+  data.tar.gz: 609fdc9e8f8325ed53cebe16afe84d36d9c4579ff4f407998e74d87676e3708fb6bafca977c7be146598f2dfa8cc7511a8fd55f02cbf96fe55bc24c7d99003b8

data/README.md

@@ -0,0 +1,3 @@
+# thunderwire
+
+External of the thunderbolt ecosystem.

data/lib/thunderwire/background_task.rb

@@ -0,0 +1,11 @@
+module ThunderWire
+  class BackgroundTask
+    def start
+      raise NotImplementedError
+    end
+
+    def stop
+      raise NotImplementedError
+    end
+  end
+end

data/lib/thunderwire/handler.rb

@@ -0,0 +1,17 @@
+module ThunderWire
+  class Handler
+    class << self
+      # WARN: AI MADE
+      def get!(broker, name)
+        file = File.join(__dir__, 'message_brokers', broker.name.split('::').last.underscore, 'handlers',
+                         "#{name.underscore}.rb")
+        raise(Error, 'This handler does not exists') unless File.exist?(file)
+
+        require file
+
+        broker::Handlers.const_get(name.camelize)
+      end
+      # WARN: ---
+    end
+  end
+end

data/lib/thunderwire/message_broker.rb

@@ -0,0 +1,16 @@
+module ThunderWire
+  class MessageBroker
+    class << self
+      # WARN: AI MADE
+      def get!(message_broker)
+        files = Dir[File.join(__dir__, 'message_brokers', message_broker.to_s, '*.rb')]
+        raise(Error, 'This broker does not exists') if files.empty?
+
+        files.sort.each { |file| require file }
+
+        ThunderWire::MessageBrokers.const_get(message_broker.to_s.camelize)
+      end
+      # WARN: ---
+    end
+  end
+end

data/lib/thunderwire/message_brokers/rabbitmq/consumer.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require 'bunny'
+
+# WARN: AI MADE
+module ThunderWire
+  module MessageBrokers
+    module Rabbitmq
+      # Consumes the queue of the chunk: declares it with one binding per event the chunk subscribes to, and hands each
+      # message to the handler. A message requeued past the delivery limit, or rejected, goes to the dead exchange.
+      class Consumer < BackgroundTask
+        def initialize(url:, queue:, keys:, handler:, exchange: 'thunderbolt.events', dead_exchange: 'thunderbolt.dead',
+                       prefetch: 10, concurrency: 5, delivery_limit: 5)
+          @url = url
+          @queue_name = queue
+          @keys = keys
+          @handler = handler
+          @exchange_name = exchange
+          @dead_exchange_name = dead_exchange
+          @prefetch = prefetch
+          @concurrency = concurrency
+          @delivery_limit = delivery_limit
+          @stopped = Queue.new
+
+          super()
+        end
+
+        def start
+          @connection = Bunny.new(@url).start
+          channel = @connection.create_channel(nil, @concurrency)
+          channel.prefetch(@prefetch)
+
+          events = channel.exchange(@exchange_name, type: :topic, durable: true)
+          dead = channel.exchange(@dead_exchange_name, type: :fanout, durable: true)
+          channel.queue(@dead_exchange_name, durable: true, arguments: { 'x-queue-type' => 'quorum' }).bind(dead)
+
+          queue = channel.queue(@queue_name, durable: true, arguments: {
+                                  'x-queue-type' => 'quorum',
+                                  'x-dead-letter-exchange' => @dead_exchange_name,
+                                  'x-delivery-limit' => @delivery_limit
+                                })
+          @keys.each { |key| queue.bind(events, routing_key: key) }
+
+          consumer = queue.subscribe(manual_ack: true) do |delivery, _properties, body|
+            case @handler.handle(delivery.routing_key, body)
+            when :ack then channel.ack(delivery.delivery_tag)
+            when :requeue then channel.nack(delivery.delivery_tag, false, true)
+            else channel.reject(delivery.delivery_tag, false)
+            end
+          rescue StandardError => e
+            Console.error(self, e)
+            channel.nack(delivery.delivery_tag, false, true)
+          end
+
+          @stopped.pop
+
+          consumer.cancel
+          channel.work_pool.join(30)
+        ensure
+          @connection&.close
+        end
+
+        # Stops taking messages; the ones in progress finish, and the ones not acked go back to the queue.
+        def stop
+          @stopped << true
+        end
+      end
+    end
+  end
+end

data/lib/thunderwire/message_brokers/rabbitmq/handlers/direct.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require 'json'
+
+# WARN: AI MADE
+module ThunderWire
+  module MessageBrokers
+    module Rabbitmq
+      module Handlers
+        # Runs the subscriber of the event inside the consumer, and answers what the consumer does with the message.
+        # The keys of the payload the subscriber does not declare are left out; the ones it declares and the payload
+        # leaves out arrive as nil.
+        #
+        # A problem with the message itself (no subscriber, body that is not JSON, param of the wrong type) is logged
+        # and the message acked: delivering it again gives the same result. Any other failure of the subscriber is
+        # logged and the message requeued.
+        #
+        # FIX: no event is ever rejected: the ones with a problem of their own are acked, never sent to the dead
+        # exchange.
+        class Direct
+          def initialize(exporter)
+            @exporter = exporter
+          end
+
+          def self.configurations
+            {}
+          end
+
+          def handle(routing_key, body)
+            unless @exporter.exports?(routing_key)
+              Console.error(self, 'Event Without Subscriber', routing_key:)
+
+              return :ack
+            end
+
+            params = JSON.parse(body, symbolize_names: true)
+            declared = @exporter.get(routing_key).base_parameter.configuration.keys
+            unknown = params.keys - declared
+            missing = declared - params.keys
+
+            if unknown.any? || missing.any?
+              Console.error(self, 'Event Payload Off Contract', routing_key:, unknown:, missing:)
+            end
+
+            response = @exporter.call(routing_key, params: params.slice(*declared))
+
+            if response.success?
+              Console.info(self, 'Event Handled', routing_key:)
+
+              :ack
+            elsif response.error.is_a?(ThunderOperation::Exceptions::ParameterError)
+              Console.error(self, 'Event Payload Off Contract', routing_key:, error: response.error)
+
+              :ack
+            else
+              Console.error(self, 'Event Failed', routing_key:, error: response.error)
+
+              :requeue
+            end
+          rescue JSON::ParserError => e
+            Console.error(self, 'Event Body Is Not JSON', routing_key:, error: e)
+
+            :ack
+          end
+        end
+      end
+    end
+  end
+end

data/lib/thunderwire/message_brokers/rabbitmq/outbox.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require 'bunny'
+
+module ThunderWire
+  module MessageBrokers
+    module Rabbitmq
+      class Outbox < BackgroundTask
+        def initialize(url:, chunk:, exchange: 'thunderbolt.events', batch_size: 100, idle: 0.5)
+          @url = url
+          @chunk = chunk
+          @exchange_name = exchange
+          @batch_size = batch_size
+          @idle = idle
+          @running = false
+
+          super()
+        end
+
+        # WARN: AI MADE
+        def start
+          @running = true
+          @connection = Bunny.new(@url).start
+          channel = @connection.create_channel
+          channel.confirm_select
+          exchange = channel.exchange(@exchange_name, type: :topic, durable: true)
+
+          while @running
+            begin
+              sent = ThunderChunk::Outbox.transaction do
+                messages = ThunderChunk::Outbox.pending.limit(@batch_size).lock('FOR UPDATE SKIP LOCKED').to_a
+
+                messages.each do |message|
+                  routing_key = "#{@chunk}/#{message.routing_key}"
+
+                  Console.info(self, 'Event Published', message_id: message.message_id,
+                                                        routing_key:,
+                                                        payload: message.payload)
+                  exchange.publish(message.payload.to_json, routing_key:, message_id: message.message_id,
+                                                            persistent: true, content_type: 'application/json')
+                end
+
+                raise Error, 'publish not confirmed by the broker' if messages.any? && !channel.wait_for_confirms
+
+                if messages.any?
+                  ThunderChunk::Outbox.where(id: messages.map(&:id)).update_all(published_at: Time.current)
+                end
+
+                messages.size
+              end
+
+              sleep @idle if sent.zero?
+            rescue StandardError => e
+              Console.error(self, e)
+              sleep 1
+            end
+          end
+        ensure
+          @connection&.close
+        end
+
+        def stop
+          @running = false
+        end
+      end
+    end
+  end
+end

data/lib/thunderwire/message_brokers/rabbitmq/publishers/outbox.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+
+module ThunderWire
+  module MessageBrokers
+    module Rabbitmq
+      module Publishers
+        class Outbox
+          def initialize(table_name: 'thunder_outbox')
+            ThunderChunk::Outbox.table_name = table_name
+          end
+
+          def self.configurations
+            { table_name: 'messaging.outbox.table_name' }
+          end
+
+          def publish(key, params)
+            ThunderChunk::Outbox.create!(
+              message_id: SecureRandom.uuid,
+              routing_key: key,
+              payload: params
+            ).message_id
+          end
+        end
+      end
+    end
+  end
+end

data/lib/thunderwire/protocol.rb

@@ -0,0 +1,13 @@
+module ThunderWire
+  class Protocol
+    class << self
+      def get!(protocol)
+        Dir[File.join(__dir__, 'protocols', protocol.to_s, '*.rb')].sort.each { |file| require file }
+
+        ThunderWire::Protocols.const_get(protocol.camelize).tap do |v|
+          raise(Error, 'This protocol does not exists') unless v
+        end
+      end
+    end
+  end
+end

data/lib/thunderwire/protocols/http/request.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module ThunderWire
+  module Protocols
+    module Http
+      # The Rack request as the adapters read it. The body stream can be consumed only once, so it is read on the
+      # first call and kept; the parsed JSON and query are kept too.
+      class Request < ::Protocol::Rack::Request
+        ENV_KEY = 'thunderwire.request'
+
+        class InvalidJSON < StandardError; end
+
+        attr_reader :env
+
+        def self.[](env)
+          env[ENV_KEY] ||= new(env)
+        end
+
+        def body
+          return @raw_body if defined?(@raw_body)
+
+          @raw_body = super&.join.to_s
+        end
+
+        def parsed_body
+          return @parsed_body if defined?(@parsed_body)
+          return @parsed_body = body unless json?
+
+          @parsed_body = body.empty? ? {} : JSON.parse(body)
+        rescue JSON::ParserError => e
+          raise InvalidJSON, e.message
+        end
+
+        def query_string = @env['QUERY_STRING'].to_s
+
+        def query
+          @query ||= ::Rack::Utils.parse_nested_query(query_string)
+        end
+
+        def content_type = @env['CONTENT_TYPE']
+
+        def content_length = @env['CONTENT_LENGTH']&.to_i
+
+        def media_type = content_type&.split(';')&.first&.strip&.downcase
+
+        def json? = media_type == 'application/json'
+
+        def post? = method == 'POST'
+
+        def remote_ip = @env['REMOTE_ADDR']
+      end
+    end
+  end
+end

data/lib/thunderwire/protocols/http/server.rb

@@ -0,0 +1,27 @@
+module ThunderWire
+  module Protocols
+    module Http
+      class Server < ThunderWire::Server
+        attr_reader :async_server
+
+        def create_task(ip, port)
+          endpoint = Async::HTTP::Endpoint.for('http', ip, port:)
+
+          @async_server = Async::HTTP::Server.new(
+            ::Protocol::Rack::Adapter.new(
+              lambda do |request|
+                # Console.info(self, 'Http Request Received', **request)
+                @wire.call(ThunderWire::Protocols::Http::Request[request]).tap do |(status, headers, body)|
+                  # Console.info(self, 'Http Response Sent', status:, headers:, body:)
+                end
+              end
+            ),
+            endpoint
+          )
+
+          @async_server.run
+        end
+      end
+    end
+  end
+end

data/lib/thunderwire/protocols/http/wires/operation/receiver.rb

@@ -0,0 +1,56 @@
+module ThunderWire
+  module Protocols
+    module Http
+      module Wires
+        module Operation
+          class Receiver
+            def initialize(chunk)
+              @chunk = chunk
+            end
+
+            # FIX: Imrpove the code
+            def call(request)
+              key = path_to_key(request.path)
+              exporter = @chunk.exporter.get(:operation)
+
+              request.post? || raise(ProtocolError, "#{request.method} is not allowed; use POST")
+              exporter.exports?(key) || raise(ProtocolError, "No operation at #{request.path}")
+              request.json? || raise(ProtocolError, 'The body must be application/json')
+
+              params = request.parsed_body
+              params.is_a?(Hash) || raise(ProtocolError, 'The body must be a JSON object')
+
+              Console.info(self, 'Http Request Received', params:)
+
+              result = exporter.call(key, params: params.deep_symbolize_keys)
+
+              Console.info(self, 'Http Response Sent', result: result.to_h)
+
+              unless result.payload.nil? || result.payload.is_a?(Hash) || result.payload.is_a?(Array)
+                raise Error,
+                      'result.payload must be a hash or array'
+              end
+
+              [200, {}, [result.to_json]]
+            rescue ProtocolError, Request::InvalidJSON => e
+              [400, {}, [{ error: { type: 'ProtocolError', message: e.message } }.to_json]]
+            rescue Error => e
+              [500, {}, [{ error: { type: 'InternalError', message: e.message } }.to_json]]
+            end
+
+            # /_operations/books/show or /library/_operations/books/show -> 'books.show'. The chunk is known by the
+            # url; a prefix with its name is accepted for a router (ingress) that keeps it in the path.
+            def path_to_key(path)
+              segments = path.split('/').reject(&:empty?)
+              segments.shift if segments.first == @chunk.identifier.to_s
+              segment, *path_array = segments
+              return unless segment == '_operations' && path_array.any?
+
+              path_array.join('.')
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/thunderwire/protocols/http/wires/operation/sender.rb

@@ -0,0 +1,50 @@
+module ThunderWire
+  module Protocols
+    module Http
+      module Wires
+        module Operation
+          class Sender
+            def initialize(chunk_identifier, base_url)
+              @chunk_identifier = chunk_identifier
+              @endpoint = Async::HTTP::Endpoint.parse(base_url)
+              @client = Async::HTTP::Client.new(@endpoint)
+            end
+
+            # FIX: Imrpove the code
+            def call(key, params)
+              Sync do
+                Console.info(self, 'Http Request Sent', chunk: @chunk_identifier, params:)
+
+                answer = @client.post(
+                  "#{@endpoint.path.chomp('/')}/_operations/#{key.tr('.', '/')}",
+                  [['content-type', 'application/json']],
+                  params.to_json
+                )
+
+                body = JSON.parse(answer.read)
+                failure = body['failure_data']
+
+                Console.info(self, 'Http Response Received', chunk: @chunk_identifier, status: answer.status,
+                                                             body:)
+
+                error = if failure
+                          ThunderChunk::Operations::Remote::RemoteResponseFailure.new(
+                            failure['failure_message'],
+                            error_class: failure['error_class'],
+                            extras: failure['failure_extras'],
+                            trail: failure['trail']
+                          )
+                        end
+
+                ThunderOperation::Response.new(
+                  payload: body['payload'], was_success: body['was_success'], error:,
+                  status: body['status'], failed_step: body['failed_step'], operation_name: body['operation_name']
+                )
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/thunderwire/protocols/http/wires/operation.rb

@@ -0,0 +1,14 @@
+module ThunderWire
+  module Protocols
+    module Http
+      module Wires
+        module Operation
+          extend ThunderWire::Wire
+        end
+      end
+    end
+  end
+end
+
+require_relative 'operation/receiver'
+require_relative 'operation/sender'

data/lib/thunderwire/server.rb

@@ -0,0 +1,39 @@
+module ThunderWire
+  class Server
+    def initialize(ip, port, wire)
+      @ip = ip
+      @port = port
+      @wire = wire
+    end
+
+    def up
+      Signal.trap('TERM') { Thread.main.raise(Interrupt) }
+
+      Sync do
+        @server_task = create_task(@ip, @port)
+
+        Console.info(self, 'Server Started!')
+
+        @server_task.wait
+      rescue Interrupt
+        # graceful shutdown handled by ensure
+      ensure
+        down
+      end
+    end
+
+    def down
+      @server_task&.stop
+
+      Console.info(self, 'Server Stopped!')
+    end
+
+    class << self
+      def build(chunk, protocol_module, wire_klass, ip, port)
+        wire = wire_klass::Receiver.new(chunk)
+
+        protocol_module::Server.new(ip, port, wire)
+      end
+    end
+  end
+end

data/lib/thunderwire/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module ThunderWire
+  VERSION = '0.2.0'
+end

data/lib/thunderwire/wire.rb

@@ -0,0 +1,21 @@
+module ThunderWire
+  # Extended by each wire module (Protocols::Http::Wires::Operation), which holds that protocol's Receiver and Sender
+  # for one kind of export.
+  module Wire
+    def self.get!(protocol, name)
+      require "thunderwire/protocols/#{protocol.name.split('::').last.underscore}/wires/#{name.underscore}"
+
+      protocol::Wires.const_get(name.camelize).tap do |v|
+        raise(Error, 'This wire does not exists') unless v
+      end
+    end
+
+    def exporter
+      wire_name = name.split('::').last
+
+      require "thunderchunk/exporters/#{wire_name.underscore.pluralize}"
+
+      ThunderChunk::Exporters.const_get(wire_name.pluralize)
+    end
+  end
+end

data/lib/thunderwire.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'active_support'
+require 'active_support/core_ext/string/inflections'
+require 'active_support/core_ext/hash/keys'
+require 'async'
+require 'async/http/server'
+require 'async/http/endpoint'
+require 'async/http/client'
+require 'protocol/rack'
+require 'rack/utils'
+require 'json'
+require 'thunderchunk'
+
+require_relative 'thunderwire/version'
+require_relative 'thunderwire/server'
+require_relative 'thunderwire/wire'
+require_relative 'thunderwire/protocol'
+require_relative 'thunderwire/background_task'
+require_relative 'thunderwire/message_broker'
+require_relative 'thunderwire/handler'
+
+module ThunderWire
+  class Error < StandardError; end
+
+  class ProtocolError < Error; end
+
+  # WARN: AI MADE
+  class << self
+    def sender(protocol, wire)
+      Wire.get!(Protocol.get!(protocol.to_s), wire.to_s)::Sender
+    end
+
+    def publisher(broker, publisher)
+      require "thunderwire/message_brokers/#{broker}/publishers/#{publisher}"
+
+      MessageBrokers.const_get(broker.to_s.camelize)::Publishers.const_get(publisher.to_s.camelize)
+    end
+  end
+  # WARN: ---
+end

metadata

@@ -0,0 +1,152 @@
+--- !ruby/object:Gem::Specification
+name: thunderwire
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- Albert Alef
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '8.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '8.1'
+- !ruby/object:Gem::Dependency
+  name: thunderchunk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+- !ruby/object:Gem::Dependency
+  name: async
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: async-http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.103'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.103'
+- !ruby/object:Gem::Dependency
+  name: bunny
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: protocol-rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.22'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.22'
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/thunderwire.rb
+- lib/thunderwire/background_task.rb
+- lib/thunderwire/handler.rb
+- lib/thunderwire/message_broker.rb
+- lib/thunderwire/message_brokers/rabbitmq/consumer.rb
+- lib/thunderwire/message_brokers/rabbitmq/handlers/direct.rb
+- lib/thunderwire/message_brokers/rabbitmq/outbox.rb
+- lib/thunderwire/message_brokers/rabbitmq/publishers/outbox.rb
+- lib/thunderwire/protocol.rb
+- lib/thunderwire/protocols/http/request.rb
+- lib/thunderwire/protocols/http/server.rb
+- lib/thunderwire/protocols/http/wires/operation.rb
+- lib/thunderwire/protocols/http/wires/operation/receiver.rb
+- lib/thunderwire/protocols/http/wires/operation/sender.rb
+- lib/thunderwire/server.rb
+- lib/thunderwire/version.rb
+- lib/thunderwire/wire.rb
+licenses: []
+metadata:
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.4'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: External of the thunderbolt ecosystem
+test_files: []