totoro 0.6.1 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aa6dd8c493beb2f3161254c4d0c207969183d3cbe0da84dc8782c63acba3f073
-  data.tar.gz: 50e0437780093c06a2c928b49fff7c8d09a094cf906a5c9ea4cd6cf54a070132
+  metadata.gz: 46971dbec9f717c43e7cd0886b94b4391374786557286ee0e0854e424999ded6
+  data.tar.gz: 3b512d046ed53091adc36a4809e128d8d436a24e0e49b2ad1086031b45ae7b41
 SHA512:
-  metadata.gz: '08d3b721c6d06504d37d6ebf153377df3354e446d79852b090e1d93920eb301fbb65cf925eba8adf10d1c0303e44230e91d5bdb89418f6e186f40b0b4ca716b0'
-  data.tar.gz: 883d9a3d0ceabd7bff3a1eebe6eda1b9700f384dea8895fa2ba628d36c587d5a29f365d8a43f5dffd8b85796f2eb1f9515fd8a5767bb81ff38beedadd83f22a4
+  metadata.gz: 61e5a11bb89c80a98d46ee66ba20c6410da252fb3c5030d65ef3b944719ffa3feee561f0227a2e4cf953cea8b5c70a0d826de0655d95e0e81ea5a5840e0f7c61
+  data.tar.gz: 5a77beb59c2f8521c3b42738d212ef66c76b14d62fb4f04782684b9ad6c62761f55be4edad078e4e701119c9e4eea9378ba416d1a91147fdc72494a6d07a5f0d

data/lib/generators/totoro/templates/create_totoro_failed_messages.rb

@@ -5,6 +5,7 @@ class CreateTotoroFailedMessages < ActiveRecord::Migration[5.1]
       t.string :class_name
       t.string :queue_id
       t.jsonb :payload
+      t.string :group
       t.timestamps
     end
   end

data/lib/generators/totoro/templates/update_totoro_failed_messages.rb

@@ -0,0 +1,6 @@
+class UpdateTotoroFailedMessages < ActiveRecord::Migration[5.1]
+  def change
+    add_column :totoro_failed_messages, :group, :string
+    Totoro::TotoroFailedMessage.all.update(group: 'enqueue')
+  end
+end

data/lib/generators/totoro/update_generator.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Totoro
+  class UpdateGenerator < Rails::Generators::Base
+    source_root File.expand_path('templates', __dir__)
+    desc 'Update totoro to 1.0.0'
+
+    def copy_config_file
+      template 'update_totoro_failed_messages.rb', File.join('db/migrate', "#{Time.now.strftime('%Y%m%d%H%M%S')}_update_totoro_failed_messages.rb")
+    end
+  end
+end

data/lib/totoro/base_queue.rb

@@ -10,59 +10,35 @@ module Totoro
       end
 
       def connection
-        @connection ||= Bunny.new(config.connect.merge(threaded: false)).tap(&:start)
+        @connection ||= Bunny.new(config.connect.merge(threaded: false))
       end
 
-      def subscribe_channel
-        @subscribe_channel ||= Bunny.new(config.connect).tap(&:start).create_channel
+      def broadcast(id, payload)
+        Totoro::BroadcastService.new(connection, config).broadcast(id, payload)
+      rescue Totoro::ConnectionBreakError => error
+        handle_failed_msg(id, payload, error, :broadcast)
       end
 
       def enqueue(id, payload)
-        channel = connection.create_channel
-        queue =  channel.queue(*config.queue(id))
-        payload = JSON.dump payload
-        channel.default_exchange.publish(payload, routing_key: queue.name)
-        Rails.logger.info "send message to #{queue.name}"
-        STDOUT.flush
-        channel.close
-      rescue Bunny::TCPConnectionFailedForAllHosts,
-             Bunny::NetworkErrorWrapper,
-             Bunny::ChannelAlreadyClosed,
-             AMQ::Protocol::EmptyResponseError => error
-        handle_failed_msg(id, payload, error)
-      end
-
-      def subscribe(id)
-        queue = subscribe_channel.queue(*config.queue(id))
-        queue.subscribe do |delivery_info, metadata, payload|
-          yield(delivery_info, metadata, payload)
-        end
-      end
-
-      def get_worker(worker_class)
-        ::Worker.const_get(worker_class.to_s.camelize).new
+        Totoro::EnqueueService.new(connection, config).enqueue(id, payload)
+      rescue Totoro::ConnectionBreakError => error
+        handle_failed_msg(id, payload, error, :enqueue)
       end
 
       private
 
-      def handle_failed_msg(id, payload, error)
+      def handle_failed_msg(id, payload, error, group)
         Rails.logger.error error.message
         Rails.logger.info 'Add failed message to resend list'
         STDOUT.flush
-        clear_connection
+        @connection = nil
         Totoro::TotoroFailedMessage.create(
           class_name: to_s,
           queue_id: id,
+          group: group,
           payload: payload
         )
       end
-
-      def clear_connection
-        @exchange = nil
-        @channel = nil
-        @subscribe_channel = nil
-        @connection = nil
-      end
     end
   end
 end

data/lib/totoro/base_worker.rb

@@ -7,6 +7,7 @@ module Totoro
       queue_name = attrs[:queue_name]
       define_method('setup') do
         raise(Totoro::NeedQueueNameError) if queue_name.nil?
+
         @prefix = prefix
         @queue_name = queue_name
       end
@@ -14,22 +15,21 @@ module Totoro
 
     def initialize
       setup
-      @queue_class = queue_class
     end
 
     def execute
-      @queue_class.subscribe(@queue_name) do |delivery_info, metadata, payload|
+      Rails.logger.info 'Listening to the Rabbitmq'
+      STDOUT.flush
+      subscribe_service.subscribe(@queue_name) do |delivery_info, metadata, payload|
         Rails.logger.info "#{@queue_name} received message"
         STDOUT.flush
         payload_hash = JSON.parse(payload).with_indifferent_access
         process(payload_hash, metadata, delivery_info)
       end
-      Rails.logger.info 'Listening to the Rabbitmq'
-      STDOUT.flush
-      @queue_class.subscribe_channel.work_pool.join
+      subscribe_service.channel.work_pool.join
     rescue SignalException
       puts 'Terminating process ..'
-      @queue_class.subscribe_channel.work_pool.shutdown(true)
+      subscribe_service.channel.work_pool.shutdown(true)
       puts 'Stopped.'
     end
 
@@ -37,14 +37,12 @@ module Totoro
 
     private
 
-    def queue_class
-      if @prefix == :default
-        Totoro::Queue
-      else
-        "Totoro::#{@prefix.to_s.camelize}::Queue".constantize
-      end
+    def config
+      @config ||= Totoro::Config.new(@prefix)
     end
-  end
 
-  class NeedQueueNameError < RuntimeError; end
+    def subscribe_service
+      @subscribe_service ||= Totoro::SubscribeService.new(config)
+    end
+  end
 end

data/lib/totoro/config.rb

@@ -15,6 +15,14 @@ module Totoro
       @data[:connect]
     end
 
+    def exchange(id)
+      @data[:exchange][id][:name]
+    end
+
+    def exchange_name_for_queue(queue_id)
+      @data[:queue][queue_id][:exchange]
+    end
+
     def queue(id)
       name = @data[:queue][id][:name]
       settings = { durable: @data[:queue][id][:durable] }

data/lib/totoro/errors/connection_break_error.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Totoro
+  class ConnectionBreakError < StandardError
+  end
+end

data/lib/totoro/errors/need_queue_name_error.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Totoro
+  class NeedQueueNameError < RuntimeError; end
+end

data/lib/totoro/message_resender.rb

@@ -8,13 +8,7 @@ module Totoro
     run_every 10.second
     queue 'totoro'
     def perform
-      Totoro::Queue.connection
-      Totoro::TotoroFailedMessage.find_in_batches(batch_size: 100) do |message_group|
-        message_group.each do |m|
-          m.class_name.constantize.enqueue(m.queue_id, m.payload)
-          m.destroy
-        end
-      end
+      Totoro::ResendService.new.resend_all_messages
     rescue StandardError => error
       Rails.logger.error error.message
       STDOUT.flush

data/lib/totoro/services/broadcast_service.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Totoro
+  class BroadcastService
+    def initialize(connection, config)
+      @connection = connection
+      @config = config
+    end
+
+    def broadcast(exchange_id, payload)
+      @connection.start unless @connection.connected?
+      exchange = channel.fanout(@config.exchange(exchange_id))
+      payload = JSON.dump payload
+      exchange.publish(payload)
+      Rails.logger.info "send message to exchange #{@config.exchange(exchange_id)}"
+      STDOUT.flush
+      channel.close
+    rescue Bunny::TCPConnectionFailedForAllHosts,
+           Bunny::NetworkErrorWrapper,
+           Bunny::ChannelAlreadyClosed,
+           Bunny::ConnectionAlreadyClosed,
+           AMQ::Protocol::EmptyResponseError => error
+      @channel.close if @channel.present?
+      raise(Totoro::ConnectionBreakError, "type: #{error.class}, message: #{error.message}")
+    end
+
+    private
+
+    def channel
+      @channel ||= @connection.create_channel
+    end
+  end
+end

data/lib/totoro/services/enqueue_service.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Totoro
+  class EnqueueService
+    def initialize(connection, config)
+      @connection = connection
+      @config = config
+    end
+
+    def enqueue(id, payload)
+      @connection.start unless @connection.connected?
+      queue = channel.queue(*@config.queue(id))
+      payload = JSON.dump payload
+      exchange.publish(payload, routing_key: queue.name)
+      Rails.logger.info "send message to #{queue.name}"
+      STDOUT.flush
+      channel.close
+    rescue Bunny::TCPConnectionFailedForAllHosts,
+           Bunny::NetworkErrorWrapper,
+           Bunny::ChannelAlreadyClosed,
+           Bunny::ConnectionAlreadyClosed,
+           AMQ::Protocol::EmptyResponseError => error
+      @channel.close if @channel.present?
+      raise(Totoro::ConnectionBreakError, "type: #{error.class}, message: #{error.message}")
+    end
+
+    private
+
+    def channel
+      @channel ||= @connection.create_channel
+    end
+
+    # default exchange is a direct exchange
+    def exchange
+      @exchange ||= channel.default_exchange
+    end
+  end
+end

data/lib/totoro/services/resend_service.rb

@@ -0,0 +1,17 @@
+module Totoro
+  class ResendService
+    def resend_all_messages
+      Totoro::TotoroFailedMessage.find_in_batches(batch_size: 100) do |message_group|
+        message_group.each { |m| resend_message(m) }
+      end
+    end
+
+    private
+
+    def resend_message(failed_message)
+      queue_class = failed_message.class_name.constantize
+      queue_class.method(failed_message.group).call(failed_message.queue_id, failed_message.payload)
+      failed_message.destroy
+    end
+  end
+end

data/lib/totoro/services/subscribe_service.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Totoro
+  class SubscribeService
+    def initialize(config)
+      @config = config
+    end
+
+    def subscribe(id)
+      queue = bind_queue(id)
+      queue.subscribe do |delivery_info, metadata, payload|
+        yield(delivery_info, metadata, payload)
+      end
+    end
+
+    def channel
+      @channel ||= Bunny.new(@config.connect).tap(&:start).create_channel
+    end
+
+    private
+
+    def bind_queue(id)
+      exchange_name = @config.exchange_name_for_queue(id)
+      if exchange_name.nil?
+        channel.queue(*@config.queue(id))
+      else
+        channel.queue(*@config.queue(id)).bind(exchange_name)
+      end
+    end
+  end
+end

data/lib/totoro/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Totoro
-  VERSION = '0.6.1'
+  VERSION = '1.0.0'
 end

data/lib/totoro.rb

@@ -4,9 +4,15 @@ require 'totoro/version'
 require 'totoro/config'
 require 'totoro/base_queue'
 require 'totoro/base_worker'
+require 'totoro/services/enqueue_service'
+require 'totoro/services/broadcast_service'
+require 'totoro/services/subscribe_service'
+require 'totoro/services/resend_service'
+require 'totoro/errors/connection_break_error'
+require 'totoro/errors/need_queue_name_error'
 require 'totoro/initializer'
 require 'totoro/message_resender'
-require 'totoro/totoro_failed_message'
+require 'totoro/models/totoro_failed_message'
 require 'totoro/railtie' if defined?(Rails)
 
 module Totoro

data/test/totoro_test/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: /totoro
   specs:
-    totoro (0.6.0)
+    totoro (0.6.1)
       bunny (~> 2.10)
       delayed_job_active_record (~> 4.1.3)
       delayed_job_recurring (~> 0.3.7)
@@ -49,8 +49,8 @@ GEM
     amq-protocol (2.3.0)
     arel (8.0.0)
     builder (3.2.3)
-    bunny (2.11.0)
-      amq-protocol (~> 2.3.0)
+    bunny (2.13.0)
+      amq-protocol (~> 2.3, >= 2.3.0)
     byebug (10.0.2)
     concurrent-ruby (1.0.5)
     crass (1.0.4)

data/test/totoro_test/app/models/worker/exchange_queue.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Worker
+  class ExchangeQueue < Totoro::BaseWorker
+    setup queue_name: 'exchange_queue'
+    def process(payload, metadata, delivery_info)
+      p payload
+    end
+  end
+end

data/test/totoro_test/app/models/worker/shu_queue.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Worker
+  class ShuQueue < Totoro::BaseWorker
+    setup queue_name: 'shu_queue'
+    def process(payload, metadata, delivery_info)
+      p payload
+    end
+  end
+end

data/test/totoro_test/config/totoro.yml

@@ -9,6 +9,17 @@ default: &default
       example_queue:
         name: real.queue.name
         durable: true
+      exchange_queue:
+        name: real.exchange.queue
+        durable: true
+        exchange: real.exchange.name
+      shu_queue:
+        name: real.shu.queue
+        durable: true
+        exchange: real.exchange.name
+    exchange:
+      example_exchange:
+        name: real.exchange.name
 
   custom:
     connect:

data/test/totoro_test/db/migrate/20181228033422_update_totoro_failed_messages.rb

@@ -0,0 +1,5 @@
+class UpdateTotoroFailedMessages < ActiveRecord::Migration[5.1]
+  def change
+    add_column :totoro_failed_messages, :group, :string
+  end
+end

data/test/totoro_test/db/schema.rb

@@ -10,30 +10,33 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20_181_019_070_846) do
+ActiveRecord::Schema.define(version: 20181228033422) do
+
   # These are extensions that must be enabled in order to support this database
-  enable_extension 'plpgsql'
+  enable_extension "plpgsql"
 
-  create_table 'delayed_jobs', force: :cascade do |t|
-    t.integer 'priority', default: 0, null: false
-    t.integer 'attempts', default: 0, null: false
-    t.text 'handler', null: false
-    t.text 'last_error'
-    t.datetime 'run_at'
-    t.datetime 'locked_at'
-    t.datetime 'failed_at'
-    t.string 'locked_by'
-    t.string 'queue'
-    t.datetime 'created_at'
-    t.datetime 'updated_at'
-    t.index %w[priority run_at], name: 'delayed_jobs_priority'
+  create_table "delayed_jobs", force: :cascade do |t|
+    t.integer "priority", default: 0, null: false
+    t.integer "attempts", default: 0, null: false
+    t.text "handler", null: false
+    t.text "last_error"
+    t.datetime "run_at"
+    t.datetime "locked_at"
+    t.datetime "failed_at"
+    t.string "locked_by"
+    t.string "queue"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+    t.index ["priority", "run_at"], name: "delayed_jobs_priority"
   end
 
-  create_table 'totoro_failed_messages', force: :cascade do |t|
-    t.string 'class_name'
-    t.string 'queue_id'
-    t.jsonb 'payload'
-    t.datetime 'created_at', null: false
-    t.datetime 'updated_at', null: false
+  create_table "totoro_failed_messages", force: :cascade do |t|
+    t.string "class_name"
+    t.string "queue_id"
+    t.jsonb "payload"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.string "group"
   end
+
 end

data/totoro.gemspec

@@ -1,4 +1,3 @@
-
 # frozen_string_literal: true
 
 lib = File.expand_path('lib', __dir__)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: totoro
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 1.0.0
 platform: ruby
 authors:
 - ShuHui18
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-08 00:00:00.000000000 Z
+date: 2018-12-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -109,7 +109,6 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- TODO
 - bin/console
 - bin/setup
 - bin/totoro
@@ -119,19 +118,28 @@ files:
 - lib/generators/totoro/templates/create_totoro_failed_messages.rb
 - lib/generators/totoro/templates/initializer.rb
 - lib/generators/totoro/templates/totoro.yml
+- lib/generators/totoro/templates/update_totoro_failed_messages.rb
 - lib/generators/totoro/templates/worker.rb.erb
+- lib/generators/totoro/update_generator.rb
 - lib/generators/totoro/worker_generator.rb
 - lib/totoro.rb
 - lib/totoro/base_queue.rb
 - lib/totoro/base_worker.rb
 - lib/totoro/config.rb
+- lib/totoro/errors/connection_break_error.rb
+- lib/totoro/errors/need_queue_name_error.rb
 - lib/totoro/initializer.rb
 - lib/totoro/message_resender.rb
+- lib/totoro/models/totoro_failed_message.rb
 - lib/totoro/railtie.rb
+- lib/totoro/services/broadcast_service.rb
+- lib/totoro/services/enqueue_service.rb
+- lib/totoro/services/resend_service.rb
+- lib/totoro/services/subscribe_service.rb
 - lib/totoro/tasks/resend_message.rake
-- lib/totoro/totoro_failed_message.rb
 - lib/totoro/version.rb
 - pkg/totoro-0.6.0.gem
+- pkg/totoro-0.6.1.gem
 - spec/spec_helper.rb
 - spec/totoro_spec.rb
 - test/rabbitmq_commands.txt
@@ -146,6 +154,8 @@ files:
 - test/totoro_test/app/mailers/application_mailer.rb
 - test/totoro_test/app/models/application_record.rb
 - test/totoro_test/app/models/worker/example_queue.rb
+- test/totoro_test/app/models/worker/exchange_queue.rb
+- test/totoro_test/app/models/worker/shu_queue.rb
 - test/totoro_test/app/services/bench_mark.txt
 - test/totoro_test/app/services/load_test.rb
 - test/totoro_test/app/views/layouts/mailer.html.erb
@@ -183,6 +193,7 @@ files:
 - test/totoro_test/db/development.sqlite3
 - test/totoro_test/db/migrate/20181019041208_create_delayed_jobs.rb
 - test/totoro_test/db/migrate/20181019070846_create_totoro_failed_messages.rb
+- test/totoro_test/db/migrate/20181228033422_update_totoro_failed_messages.rb
 - test/totoro_test/db/schema.rb
 - test/totoro_test/db/seeds.rb
 - test/totoro_test/db/test.sqlite3

data/TODO

@@ -1,4 +0,0 @@
-
-Totoro:
-  ☐ Add log
-  ☐ flush log