clustered_rpc 0.1.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b9bc292ea42805f08ade35dbd5ccc1856fecfdebff8443c943ee9bc4f9dd4a72
-  data.tar.gz: 3d18c96eecd6b8fe288a9130a9f15b660cc4031495d196c0fed5ea1a7aad1fb6
+  metadata.gz: 904505b16c185f8541a99935ce6f1e5238069f3f3d69bc95c440d7c7a6669357
+  data.tar.gz: 6557b5bf8f8a58120ab25f70301bf53ef8f7959d57f478b1b0589327a0d1302f
 SHA512:
-  metadata.gz: 54f0b6a5c19a38ef87c0acd1c75449a307fc4605812baa6986fb5ae0e4f1ac6abce3211e5b4989329212b885f7de9f9d99b89f115287ab4c6f5b89bbfe2d628d
-  data.tar.gz: 1e9fe2a6c30c8c8e970ee648b6fd57e8f9a31900507a820b5bddccccd140d68c905798fc0d40c1ae7da61f637b2c01dbc35886cd865187e2f59e65759376cbd3
+  metadata.gz: 0155fd98b49b9351d9cf4a3086fc9145e1095bc41ff8a2638a800a61ca73a19caefbf43c9a0b9ef458a1e847928077b7b01f5b9da27bf591395f697930fcab8b
+  data.tar.gz: 9facf71c00b00091591e2261e1ccb46e302ee0fa8f3e1d3f9b362305a48c53986a33ba6a797baf07881548261b358d81a95ac050992e49d2e027170aee7f47f7

data/.github/workflows/ruby.yml

@@ -0,0 +1,51 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
+# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
+
+name: Ruby
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version: [ '2.5', '2.6', '2.7']
+
+    # Service containers to run with `runner-job`
+    services:
+      # Label used to access the service container
+      redis:
+        # Docker Hub image
+        image: redis
+        # Set health checks to wait until redis has started
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps port 6379 on service container to the host
+          - 6379:6379
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+    # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
+    # change this to (see https://github.com/ruby/setup-ruby#versioning):
+    # uses: ruby/setup-ruby@v1
+      uses: ruby/setup-ruby@473e4d8fe5dd94ee328fdfca9f8c9c7afc9dae5e
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+    - name: Run tests
+      run: bundle exec rake

data/.travis.yml

@@ -3,5 +3,14 @@ sudo: false
 language: ruby
 cache: bundler
 rvm:
-  - 2.5.1
+  - 2.5
+  - 2.6
+  - 2.7
+services:
+  - redis-server
+  
 before_install: gem install bundler -v 2.0.2
+
+#addons:
+#  code_climate:
+#    repo_token: 

data/Gemfile.lock

@@ -1,13 +1,13 @@
 PATH
   remote: .
   specs:
-    clustered_rpc (0.1.0)
+    clustered_rpc (0.3.1)
       activesupport
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.1.3.1)
+    activesupport (6.1.3.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -16,14 +16,26 @@ GEM
     awesome_print (1.8.0)
     byebug (11.1.3)
     concurrent-ruby (1.1.8)
+    coveralls (0.7.2)
+      multi_json (~> 1.3)
+      rest-client (= 1.6.7)
+      simplecov (>= 0.7)
+      term-ansicolor (= 1.2.2)
+      thor (= 0.18.1)
     diff-lcs (1.4.4)
     docile (1.3.2)
     i18n (1.8.10)
       concurrent-ruby (~> 1.0)
     json (2.3.0)
+    mime-types (3.3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2021.0225)
     minitest (5.14.4)
+    multi_json (1.15.0)
     rake (10.5.0)
     redis (4.2.2)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
     rspec (3.10.0)
       rspec-core (~> 3.10.0)
       rspec-expectations (~> 3.10.0)
@@ -42,6 +54,10 @@ GEM
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
+    term-ansicolor (1.2.2)
+      tins (~> 0.8)
+    thor (0.18.1)
+    tins (0.13.2)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
     zeitwerk (2.4.2)
@@ -54,6 +70,7 @@ DEPENDENCIES
   bundler (~> 2.0)
   byebug
   clustered_rpc!
+  coveralls (>= 0.7.0)
   rake (~> 10.0)
   redis
   rspec (~> 3.0)

data/README.md

@@ -1,8 +1,16 @@
 # ClusteredRpc
+[![RubyGems][gem_version_badge]][ruby_gems]
+[![Travis CI][travis_ci_badge]][travis_ci]
+[![Coveralls][coveralls_badge]][coveralls]
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/clustered`. To experiment with that code, run `bin/console` for an interactive prompt.
+_RPC = Remote Procedure Calls_
+
+ClusteredRpc allows you to run code on every ruby process running within your cluster.
+
+Clusters are defined using a shared pubsub broker and a common namespace.
+
+Currently only Redis PubSub is supported by ClusteredRpc.
 
-TODO: Delete this and the text above, and describe your gem
 
 ## Installation
 
@@ -20,24 +28,110 @@ Or install it yourself as:
 
     $ gem install clustered_rpc
 
+## Configuration
+```ruby
+# Using Redis PubSub
+ClusteredRpc.config do |c|
+  c.transport_class = ClusteredRpc::Transport::RedisCluster
+  c.cluster_namespace = "myapplication"
+  c.options = {redis_url: "redis://127.0.0.1:6379/3"}
+end
+
+```
+If you are using the same redis server for multiple deployments of your application, then use different namespaces for each.
+```ruby
+ClusteredRpc.config do |c|
+  c.transport_class = ClusteredRpc::Transport::RedisCluster
+  # Using the same redis database for the development environment as well...
+  c.cluster_namespace = "myapplication_dev"
+  c.options = {redis_url: "redis://127.0.0.1:6379/3"}
+end
+```
 ## Usage
 
-Works on *static* class methods!!!
+ClusteredRpc allows static (class) methods to be run on every process within the cluster!
 
 ```ruby
 class MyClass
+  # makes all static-methods available via 'clustered_rpc' proxy method
   include ClusteredRpc::Methods
 
   def self.do_the_thing
     # important code living on many servers in the cluster
-    puts "I'm important!"
+    return "I'm important!"
   end
 end
 
 # Run the method on every process running in the cluster
+# `do_the_thing` is run on each process and the results are returned in a Hash
 MyClass.clustered_rpc.do_the_thing
+=> {
+    :request_id => "f030b020e058d7a4",
+       :success => true,
+       :results => {
+        "1ca8a7be5e" => {
+            "seconds" => 3.3e-05,
+             "result" => "I'm important"
+        },
+        "293a7be9ac" => {
+            "seconds" => 4.2e-05,
+             "result" => "I'm important"
+        }        
+    }
+}
 ```
+The keys in the results Hash (`1ca8a7be5e` and `293a7be9ac`) are the unique instance_ids assigned to each process by ClusteredRpc
+
+```ruby
+# Get the stats for your entire cluster
+ClusteredRpc::Info.clustered_rpc.stats
+=> { 
+  :request_id => "c634e6347b98a0",
+     :success => true,
+     :results => {
+      "ica817be5e" => {
+              :instance_id => "1ca8a7be5e",
+          :transport_class => "ClusteredRpc::Transport::LocalProcess",
+                  :options => {},
+               :process_id => 23566,
+                   :uptime => "03:14",
+                  :used_mb => 35.03,
+          :startup_command => "ruby-2.5.1/bin/rails console",
+             :process_type => "Rails Console",
+              :count_nodes => {}
+      },
+      {
+              :instance_id => "293a7be9ac",
+          :transport_class => "ClusteredRpc::Transport::LocalProcess",
+                  :options => {},
+               :process_id => 23736,
+                   :uptime => "42:22",
+                  :used_mb => 127.69,
+          :startup_command => "ruby-2.5.1/bin/puma",
+             :process_type => "Web Server",
+              :count_nodes => {}
+      }
+
+   }
+}
 
+```
+Of course, methods can be run locally (without `.clustered_rpc`) as well
+```ruby
+# Get the stats for the local process
+ClusteredRpc::Info.stats
+=> {
+        :instance_id => "1ca8a7be5e",
+    :transport_class => "ClusteredRpc::Transport::LocalProcess",
+            :options => {},
+         :process_id => 23566,
+             :uptime => "00:01",
+            :used_mb => 35.03,
+    :startup_command => "ruby-2.5.1/bin/rails console",
+       :process_type => "Rails Console",
+        :count_nodes => {}
+}
+```
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
@@ -46,8 +140,15 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/clustered.
+Bug reports and pull requests are welcome on GitHub at https://github.com/megalithtracers/clustered_rpc.
 
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+[gem_version_badge]: https://img.shields.io/gem/v/clustered_rpc.svg?style=flat
+[ruby_gems]: http://rubygems.org/gems/clustered_rpc
+[travis_ci]: http://travis-ci.org/megalithtracers/clustered_rpc
+[travis_ci_badge]: https://travis-ci.com/megalithtracers/clustered_rpc.svg?branch=main
+[coveralls_badge]: https://img.shields.io/coveralls/github/megalithtracers/clustered_rpc/main
+[coveralls]: https://coveralls.io/github/megalithtracers/clustered_rpc

data/clustered_rpc.gemspec

@@ -33,6 +33,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "awesome_print"
   spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "byebug"
+  spec.add_development_dependency "coveralls", ">= 0.7.0"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "redis"
   spec.add_development_dependency "rspec", "~> 3.0"

data/lib/clustered_rpc.rb

@@ -5,12 +5,14 @@ require "json"
 require "active_support/concern"
 require "clustered_rpc/proxy"
 require "clustered_rpc/transport/base"
+require "clustered_rpc/transport/local_process"
 
 module ClusteredRpc
   class Error < StandardError; end
 
   @@instance_id = SecureRandom.hex(5)
   @@logger = ::Logger.new(STDOUT)
+  @@cluster_namespace = "clustered_rpc"
   @@transport_class = nil
   @@transport = nil
   @@options = {}
@@ -21,6 +23,9 @@ module ClusteredRpc
   def self.instance_id=(instance_id); @@instance_id = instance_id; end
   def self.instance_id; @@instance_id; end
 
+  def self.cluster_namespace=(cluster_namespace); @@cluster_namespace = cluster_namespace; end
+  def self.cluster_namespace; @@cluster_namespace; end
+
   def self.transport_class=(transport_class); @@transport_class = transport_class; end
   def self.transport_class; @@transport_class; end
 
@@ -29,7 +34,8 @@ module ClusteredRpc
 
   # request_id should have been returned from the call to #cluster_send
   def self.get_result(request_id, wait_seconds = 1.0)
-    sleep(wait_seconds) if wait_seconds
+    # Don't wait at all when using LocalProcess
+    sleep(wait_seconds) if wait_seconds && transport_class != ClusteredRpc::Transport::LocalProcess
     results = @@transport.get_result(request_id)
     results.keys.each{|k| results[k] = JSON.parse(results[k])}
     results # ??? Rails anyone? .with_indifferent_access
@@ -38,14 +44,11 @@ module ClusteredRpc
 
 
   def self.config(force=false, &block)
+    @@transport = nil
     block.call(self)
-
     @@instance_id ||= SecureRandom.hex(5)
-    raise "Please set transport_class in ClusteredRpc.config" if transport_class.nil? 
-    logger.info "Clustered using #{@@transport_class}"
-    @@transport = @@transport_class.new
+    ensure_transport
     @@transport.connect
-
   end
 
   def self.reconnect
@@ -57,9 +60,20 @@ module ClusteredRpc
   def self.publish(payload={})
     # if :request_id is already present, then we're responding with a process-level response
     # otherwise we're creating a new clustered_request and should generate a :request_io
+    ensure_transport
     payload[:request_id] ||= SecureRandom.hex(8)
     @@transport.publish payload
     payload[:request_id]
   end
 
+  private
+  def self.ensure_transport
+    return if @@transport
+    if @@transport_class.nil?
+      require "clustered_rpc/transport/local_process"
+      @@transport_class = ClusteredRpc::Transport::LocalProcess
+    end
+    logger.info "Clustered using #{@@transport_class}[#{@@cluster_namespace}]"
+    @@transport = @@transport_class.new
+  end
 end

data/lib/clustered_rpc/info.rb

@@ -1,12 +1,13 @@
-require 'clustered/rpc_methods'
+require 'clustered_rpc/methods'
 require 'open3'
 
 module ClusteredRpc
   class Info
     include ClusteredRpc::Methods
 
-    def self.stats
+    def self.stats(detailed_memory_stats = false)
       require 'objspace'
+      r = 
       { instance_id: ClusteredRpc.instance_id, 
         transport_class: ClusteredRpc.transport_class.name.to_s, 
         options: ClusteredRpc.options,
@@ -15,11 +16,13 @@ module ClusteredRpc
         used_mb: memory_used,
         startup_command: startup_command,
         process_type: lookup_process_type(startup_command),
-        count_objects_size: ObjectSpace.count_objects_size,
-        gc: GC.stat,
-        count_nodes: ObjectSpace.count_nodes
+        count_nodes: ObjectSpace.count_nodes,
       }
-
+      if detailed_memory_stats
+        r[:count_objects_size] = ObjectSpace.count_objects_size
+        r[:gc] = GC.stat
+      end
+      r
     end
 
     def self.startup_command

data/lib/clustered_rpc/proxy.rb

@@ -10,7 +10,7 @@ module ClusteredRpc
       request_id = ::ClusteredRpc.publish({'klass' => @target.name, 'method' => method, 'args' => args, 'kwargs' => kwargs}.merge(@options))
       {request_id: request_id, success: true, results: ::ClusteredRpc.get_result(request_id, wait_seconds)}
     rescue => e 
-      ClusteredRpc.logger.error "ClusteredRpc::Proxy encountered errror: #{e.message}"
+      ClusteredRpc.logger.error "ClusteredRpc::Proxy encountered error: #{e.message}"
       {request_id: "Error", success: false, results: e.message}
     end
     

data/lib/clustered_rpc/transport/redis_cluster.rb

@@ -7,14 +7,14 @@ module ClusteredRpc
       def initialize
         @redis_subscriber = nil
         @redis_publish = nil
-        @redis_message_pubsub_key = "__cluster_messages"
+        @redis_message_pubsub_key = "__#{ClusteredRpc.cluster_namespace}_messages"
         connect
       end
 
       def publish(payload={})
         @redis_publish.publish @redis_message_pubsub_key, payload.to_json
       rescue => e 
-        ClusteredRpc.logger.error "ClusteredRpc.publish encountered errror: #{e.message}"
+        ClusteredRpc.logger.error "ClusteredRpc.publish encountered error: #{e.message}"
         raise e
       end
 
@@ -27,6 +27,8 @@ module ClusteredRpc
         if @subscriber_thread
           ClusteredRpc.logger.warn "ClusteredRpc: killing subscriber thread"
           @subscriber_thread.kill
+          # https://stackoverflow.com/questions/49490278/wait-for-a-thread-to-die-in-ruby
+          sleep 0.01 while @subscriber_thread.alive?
           @redis_subscriber = nil
         end
         connect

data/lib/clustered_rpc/version.rb

@@ -1,3 +1,3 @@
 module ClusteredRpc
-  VERSION = "0.1.0"
+  VERSION = "0.3.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: clustered_rpc
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.1
 platform: ruby
 authors:
 - megalithtracers@gmail.com
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-04-29 00:00:00.000000000 Z
+date: 2021-07-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.7.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.7.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -130,6 +144,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ruby.yml"
 - ".rspec"
 - ".travis.yml"
 - Gemfile