insque 0.6.1 → 0.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 41594a38f5f9ff47f8ed7bab78c5d4f488f7b733
-  data.tar.gz: 95c6383a58ce039840906c5684b336a9f273e659
+  metadata.gz: 4fb5eb7a565496e70aee0f071c982f638f9b879a
+  data.tar.gz: 74447dc1cd8aecb247fcf1821a5baee070f02132
 SHA512:
-  metadata.gz: 2c518a09723ab1b90f04b693c154cceac146a47e664ba24376ee802c5770aa7285602e96019d0edbe2b76469ca16bb1de4d555a09d5435a91a00e419bedf6038
-  data.tar.gz: f29dd77fbf9d9cd0b0cc5efbf39e07a1aa5fa9864770a4d34fa3f53be35dfb8fc6a8df76d75a10864e68e897ae7a0bbf1a4ab6a636bce45634cc9c76147b0e00
+  metadata.gz: 9eb282aa17671257da5d114e78e36c90b772d049e7d6bd4712a27981122b26513fb0698003dc68bde1aa0e8abecafd76419fec268a9af7740f57b7f3c8b4cd05
+  data.tar.gz: 51f3e733493a17e601f53ab14003fa60f1accf7514c3f8fe4b864c95a2892487911dc01abdfbebcf92a72d598a8a3b9514903edd73f85fdece6f266eb72d3a1d

data/.gitignore

@@ -15,3 +15,4 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+db/*

data/insque.gemspec

@@ -17,4 +17,9 @@ Gem::Specification.new do |gem|
   gem.version       = Insque::VERSION
   
   gem.add_dependency('redis', '> 2')
+  gem.add_dependency('json')
+  gem.add_development_dependency('rspec')
+  gem.add_development_dependency('activesupport')
+  gem.add_development_dependency('activerecord')
+  gem.add_development_dependency('sqlite3')
 end

data/insque.local.yml

@@ -0,0 +1,10 @@
+version: '3.2'
+
+services:
+  redis:
+    image: redis
+    ports:
+      - target: 6379
+        published: 63790
+        protocol: tcp
+        mode: host

data/lib/insque/version.rb

@@ -1,3 +1,3 @@
 module Insque
-  VERSION = "0.6.1"
+  VERSION = "0.6.2"
 end

data/lib/insque.rb

@@ -1,9 +1,11 @@
 require "insque/version"
 require "redis"
+require 'json'
 require "insque/railtie" if defined?(Rails)
 
 module Insque
   DEFAULT_INBOX_TTL = 10800 # seconds
+  DEFAULT_PROCESSING_TTL = 3600 # seconds
 
   def self.inbox_ttl= val
     @inbox_ttl = val
@@ -13,6 +15,14 @@ module Insque
     @inbox_ttl || DEFAULT_INBOX_TTL
   end
 
+  def self.processing_ttl= val
+    @processing_ttl = val
+  end
+
+  def self.processing_ttl
+    @processing_ttl || DEFAULT_PROCESSING_TTL
+  end
+
   def self.debug= debug
     @debug = debug
   end
@@ -78,7 +88,7 @@ module Insque
   end
 
   def self.janitor redis=nil
-    real_janitor @inbox, @processing, (redis || create_redis_connection)
+    real_janitor @inbox, @processing, (redis || create_redis_connection), @inbox_pointer
   end
 
   def self.slow_janitor redis=nil
@@ -91,8 +101,8 @@ private
     loop do
       redis.setex(pointer, inbox_ttl, inbox) if pointer
       message = redis.brpoplpush(inbox, processing, 0)
-      log "#{worker_name} RECEIVING: #{message}" if @debug
       begin
+        log "#{worker_name} RECEIVING: #{message}" if @debug
         parsed_message = JSON.parse message
         send(parsed_message['message'], parsed_message) 
       rescue NoMethodError
@@ -100,13 +110,15 @@ private
         log "#{worker_name} ========== BROKEN_MESSAGE: #{message} =========="
         log e.inspect
         log e.backtrace
+      ensure
+        redis.lrem processing, 0, message
       end
-      redis.lrem processing, 0, message
     end
   end
 
-  def self.real_janitor inbox, processing, redis
+  def self.real_janitor inbox, processing, redis, pointer=nil
     loop do
+      redis.setex(pointer, inbox_ttl, inbox) if pointer
       redis.watch processing
       errors = []
       restart = []
@@ -114,15 +126,17 @@ private
       redis.lrange(processing, 0, -1).each do |m|
         begin
           parsed_message = JSON.parse(m)
-          if parsed_message['restarted_at'] && DateTime.parse(parsed_message['restarted_at']) < 1.hour.ago.utc
+          if parsed_message['restarted_at'] && Time.now.to_i - Time.parse(parsed_message['restarted_at']).to_i > processing_ttl
             errors << parsed_message 
             delete << m
-          elsif DateTime.parse(parsed_message['broadcasted_at']) < 1.hour.ago.utc
+          elsif Time.now.to_i - Time.parse(parsed_message['broadcasted_at']).to_i > processing_ttl
             restart << parsed_message.merge(restarted_at: Time.now.utc)
             delete << m
           end
-        rescue
+        rescue => e
           log "========== JANITOR_BROKEN_MESSAGE: #{m} =========="
+          log e.inspect
+          log e.backtrace
         end
       end
       result = redis.multi do |r|
@@ -136,7 +150,7 @@ private
       else
         log "CLEANING #{inbox} FAILED"
       end
-      sleep(Random.rand * 300)
+      sleep(Random.rand((inbox_ttl.to_f / 10).ceil) + 1)
     end
   end
 
@@ -161,10 +175,10 @@ if defined?(ActiveRecord::Base)
     def send_later(method, *args)
       Insque.broadcast :send_later, { class: self.class.name, id: id, method: method, args: args }, :slow
     end
+
     def self.acts_as_insque_crud(*args)
       options = args.extract_options!
       excluded = (options[:exclude] || []).map(&:to_s)
-
       set_callback :commit, :after do
         action = [:create, :update, :destroy].map {|a| a if transaction_include_any_action?([a]) }.compact.first
         params = self.serializable_hash(options).delete_if {|key| (['created_at', 'updated_at'] + excluded).include? key}

data/spec/insque_spec.rb

@@ -0,0 +1,110 @@
+require 'redis'
+require 'json'
+require 'active_support/all'
+require 'active_record'
+require 'insque'
+
+class User < ActiveRecord::Base
+  acts_as_insque_crud
+
+  def set_status value
+    update_column :status, value
+  end
+end
+
+module Insque
+  def self.myapp_test msg
+    Insque.redis.set msg['message'], msg['params']['value'] 
+  end
+
+  def self.myapp_user_update msg
+    Insque.redis.set msg['message'], msg['params']['name'] 
+  end
+
+  def self.myapp_user_create msg
+    Insque.redis.set msg['message'], msg['params']['name'] 
+  end
+
+  def self.myapp_user_destroy msg
+    Insque.redis.set msg['message'], msg['params']['name'] 
+  end
+end
+
+RSpec.describe 'streamer' do
+  before(:all) do
+    system "docker swarm init || true"
+    system "docker stack deploy -c insque.local.yml insque"
+    sleep 10
+    Thread.abort_on_exception=true
+    Insque.debug = true
+    Insque.sender = 'myapp'
+    Insque.inbox_ttl = 3
+    Insque.redis_config = { host: 'localhost', port: 63790 }
+    ActiveRecord::Base.establish_connection(
+      :adapter   => 'sqlite3',
+      :database  => 'db/test.db'
+    )
+    ActiveRecord::Base.connection.execute('create table users(id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT, status TEXT);')
+  end
+
+  after(:all) do
+    system "docker stack rm insque"
+    ActiveRecord::Base.connection.execute('drop table users;')
+  end
+
+  before(:each) do
+    Insque.redis.flushall
+  end
+
+  it "can broadcast without listeners" do
+    Insque.broadcast :test, value: '123'
+  end
+
+  it "sends and recives a message" do
+    listener = Thread.new { Insque.listen }
+    sleep 1
+    expect(Insque.redis.get '{insque}inbox_pointer_myapp').to eq('{insque}inbox_myapp') 
+    Insque.broadcast :test, value: '123'
+    sleep 1
+    listener.exit
+    expect(Insque.redis.get 'myapp_test').to eq('123')
+    expect(Insque.redis.llen '{insque}processing_myapp').to eq(0)
+    sleep 4
+    expect(Insque.redis.get '{insque}inbox_pointer_myapp').to be_nil
+  end
+
+  it "restarts broken message" do
+    janitor = Thread.new { Insque.janitor }
+    Insque.redis.lpush('{insque}processing_myapp', { message: 'myapp_test', broadcasted_at: 1.hour.ago }.to_json)
+    sleep 3
+    expect(Insque.redis.llen '{insque}processing_myapp').to eq(0)
+    expect(Insque.redis.llen '{insque}inbox_myapp').to eq(1)
+    janitor.exit
+  end
+
+  it "executes model method in background" do
+    listener = Thread.new { Insque.slow_listen }
+    u = User.create! name: 'Jon Doe'
+    u.send_later :set_status, 'processed'
+    sleep 1
+    listener.exit
+    u.reload
+    expect(u.status).to eq('processed')
+  end
+
+  it "broadcasts model changes" do
+    listener = Thread.new { Insque.listen }
+    sleep 1
+    u = User.create! name: 'Test'
+    sleep 1
+    expect(Insque.redis.get 'myapp_user_create').to eq('Test')
+    u.name = 'Test2'
+    u.save!
+    sleep 1
+    expect(Insque.redis.get 'myapp_user_update').to eq('Test2')
+    u.destroy
+    sleep 1
+    expect(Insque.redis.get 'myapp_user_destroy').to eq('Test2')
+    listener.exit
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,100 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause
+# this file to always be loaded, without a need to explicitly require it in any
+# files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, consider making
+# a separate helper file that requires the additional dependencies and performs
+# the additional setup, and require it from the spec files that actually need
+# it.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4. It makes the `description`
+    # and `failure_message` of custom matchers include text for helper methods
+    # defined using `chain`, e.g.:
+    #     be_bigger_than(2).and_smaller_than(4).description
+    #     # => "be bigger than 2 and smaller than 4"
+    # ...rather than:
+    #     # => "be bigger than 2"
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+    mocks.verify_partial_doubles = true
+  end
+
+  # This option will default to `:apply_to_host_groups` in RSpec 4 (and will
+  # have no way to turn it off -- the option exists only for backwards
+  # compatibility in RSpec 3). It causes shared context metadata to be
+  # inherited by the metadata hash of host groups and examples, rather than
+  # triggering implicit auto-inclusion in groups with matching metadata.
+  config.shared_context_metadata_behavior = :apply_to_host_groups
+
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+=begin
+  # This allows you to limit a spec run to individual examples or groups
+  # you care about by tagging them with `:focus` metadata. When nothing
+  # is tagged with `:focus`, all examples get run. RSpec also provides
+  # aliases for `it`, `describe`, and `context` that include `:focus`
+  # metadata: `fit`, `fdescribe` and `fcontext`, respectively.
+  config.filter_run_when_matching :focus
+
+  # Allows RSpec to persist some state between runs in order to support
+  # the `--only-failures` and `--next-failure` CLI options. We recommend
+  # you configure your source control system to ignore this file.
+  config.example_status_persistence_file_path = "spec/examples.txt"
+
+  # Limits the available syntax to the non-monkey patched syntax that is
+  # recommended. For more details, see:
+  #   - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/
+  #   - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+  #   - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode
+  config.disable_monkey_patching!
+
+  # This setting enables warnings. It's recommended, but in some cases may
+  # be too noisy due to issues in dependencies.
+  config.warnings = true
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = "doc"
+  end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+=end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: insque
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.6.2
 platform: ruby
 authors:
 - Yuri Gomozov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-13 00:00:00.000000000 Z
+date: 2018-04-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -24,6 +24,76 @@ dependencies:
     - - ">"
       - !ruby/object:Gem::Version
         version: '2'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Instant queue. Background processing and message driven communication
   tool. Faster and simplier alternative to Resque.
 email:
@@ -38,6 +108,7 @@ files:
 - README.md
 - Rakefile
 - insque.gemspec
+- insque.local.yml
 - lib/generators/insque/initializer_generator.rb
 - lib/generators/insque/templates/insque.erb
 - lib/generators/insque/templates/redis.yml
@@ -45,6 +116,8 @@ files:
 - lib/insque/railtie.rb
 - lib/insque/version.rb
 - lib/tasks/insque.rake
+- spec/insque_spec.rb
+- spec/spec_helper.rb
 homepage: https://github.com/Gropher/insque
 licenses:
 - MIT
@@ -69,4 +142,6 @@ rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Redis-based multi-threaded queue
-test_files: []
+test_files:
+- spec/insque_spec.rb
+- spec/spec_helper.rb