soda-core 0.0.1 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 64b5b8109a7a40030971f07e90281ded5f2611c7f46eb02602c8d4ed73c57a20
-  data.tar.gz: 69a55b5427f9b3c21207dced6b0b5a755a707d20e8eec69784cbfb49269d15d0
+  metadata.gz: 88762defaf2339fa04bbe565b9238d6edf1b6a3df2f68bd85c4bee62aa1e5397
+  data.tar.gz: b6b0934e5f9c7038859eed8dc28c8f6280223384e06f640c9ed907d8900f0392
 SHA512:
-  metadata.gz: e6c98fe461e363df472afd2377d2ad311a7c76dfa883985ee1584da75bead837f7891fd97176774ceacccb5f91cae298a61000cd905d1ce1d6c049cac355f712
-  data.tar.gz: 6ed06d08b7fc9a6d01cabcd3156d88a26dd259a1354b2f651afaa626fe23c9eb8abae1dc3b0d1183d6ec55d2dab42f90fb070c945573783b9e3be69ed5f124b1
+  metadata.gz: b1ccfad1b68e0f0e63dfd92e625a76896693b77a5314ae557a486af21a31a6f7c48c46f8495a3e8d4d088687bacff85636016181b51a2dcd24c6c29395372e79
+  data.tar.gz: 22a7dc79c110eae266bb0b4323ec474e22cc38ef02dd79bf58c1f9c26233fe0c8a10ace9906ec7bd99e17b3cc00a9fc4e245f6bd103dc2c98bd9d3876e455432

data/lib/soda.rb

@@ -1,8 +1,10 @@
 require "aws-sdk-sqs"
+require "erb"
 require "json"
 require "logger"
 require "securerandom"
 require "set"
+require "yaml"
 
 require "soda/tools"
 
@@ -18,6 +20,7 @@ require "soda/retrier"
 require "soda/worker"
 
 module Soda
+  NAME = "Soda"
   DEFAULTS = {
     concurrency: 10,
   }
@@ -64,6 +67,40 @@ module Soda
       end
     end
 
+    def queues=(configs)
+      queues do |registry|
+        names = []
+        configs.each do |cfg|
+          # Find or create the queue.
+          queue = registry.select(cfg.delete(:name))
+
+          # Update the attributes
+          name = queue.name
+          url  = cfg.delete(:url) || queue.url
+
+          registry.register(
+            name,
+            url,
+            queue.options.merge(cfg),
+          )
+
+          names << queue.name
+        end
+
+        # For queues that are not included in the command, set their weight to
+        # zero so they can still be accessed.
+        registry.each do |queue|
+          unless names.include?(queue.name)
+            registry.register(
+              queue.name,
+              queue.url,
+              queue.options.merge(weight: 0),
+            )
+          end
+        end
+      end
+    end
+
     def queue(name)
       queues.select(name).tap do |queue|
         yield(queue) if block_given?

data/lib/soda/cli.rb

@@ -12,6 +12,8 @@ module Soda
       INT,
     ].freeze
 
+    DEFAULT_CONFIG = "config/soda.yml"
+
     def self.start
       new.run
     end
@@ -23,6 +25,21 @@ module Soda
     def run
       build_options
 
+      logger.info("🥤  %s v%s" % [Soda::NAME, Soda::VERSION])
+
+      if rails?
+        if Rails::VERSION::MAJOR >= 5
+          require "./config/application.rb"
+          require "./config/environment.rb"
+          require "soda/rails"
+          require "soda/extensions/active_job"
+
+          logger.info("Loaded Rails v%s application." % ::Rails.version)
+        else
+          raise "Not compatible with Rails v%s!" % Rails.version
+        end
+      end
+
       manager = Manager.new
       manager.start
 
@@ -70,12 +87,20 @@ module Soda
       parser  = build_option_parser(opts)
       parser.parse!(argv)
 
+      if File.exists?(default_config = File.expand_path(DEFAULT_CONFIG))
+        opts[:config] ||= default_config
+      end
+
+      if (file = opts.delete(:config_file))
+        parse_config_file(opts, opts.delete(:config))
+      end
+
       if (req = opts.delete(:require))
         require(req)
       end
 
-      if (queues_opt = opts.delete(:queues))
-        parse_queues(queues_opt)
+      if (queues = opts.delete(:queues))
+        Soda.queues = queues
       end
 
       options = Soda.options
@@ -88,46 +113,38 @@ module Soda
           opts.merge!(require: val)
         end
 
-        o.on("-q", "--queues [PATH]", "Queue to listen to, with optional weights") do |val|
-          opts.merge!(queues: opts.fetch(:queues, []).push(val.split(/\,+/)))
+        o.on("-q", "--queue QUEUE[,WEIGHT]", "Queue to listen to, with optional weights") do |val|
+          name, weight = val.split(/,/)
+          opts.merge!(queues: opts.fetch(:queues, []).push(name: name, weight: weight))
+        end
+
+        o.on("-c", "--concurrency [INT]", "Number of processor threads") do |val|
+          opts.merge!(concurrency: Integer(val))
         end
       end
     end
 
-    def parse_queues(opt)
-      Soda.queues do |registry|
-        opt.each do |name, weight|
-          # Find or create the queue.
-          queue = registry.select(name)
-
-          if weight
-            # Replace the queue with the same one, except mutate the options to
-            # include the specified weight.
-            registry.register(
-              queue.name,
-              queue.url,
-              queue.options.merge(weight: weight.to_i),
-            )
-          end
-        end
+    def parse_config_file(opts = {}, file)
+      path = File.expand_path(file)
 
-        # For queues that are not included in the command, set their weight to
-        # zero so they can still be accessed.
-        names = opt.map(&:first)
-        registry.each do |queue|
-          unless names.include?(queue.name)
-            registry.register(
-              queue.name,
-              queue.url,
-              queue.options.merge(weight: 0),
-            )
-          end
-        end
+      unless File.exists?(path)
+        raise "File does not exist: %s"
       end
+
+      opts.merge!(
+        deep_symbolize_keys(
+          YAML.load(
+            ERB.new(File.read(path)).result,
+          ),
+        ),
+      )
     end
 
-    def options
-      Soda.options
+    def rails?
+      require "rails"
+      defined?(::Rails)
+    rescue LoadError
+      false
     end
   end
 end

data/lib/soda/client.rb

@@ -1,11 +1,27 @@
 module Soda
   class Client
+    DEFAULTS = {
+      "retry" => true,
+      "delay" => 0,
+    }
+
+    class << self
+      def push(*args)
+        new.push(*args)
+      end
+
+      def middleware
+        Soda.client_middleware
+      end
+    end
+
     def push(item)
-      copy  = normalize!(item)
-      mw    = Soda.client_middleware
+      copy = normalize!(item)
 
-      mw.use(item["klass"], copy, copy["queue"]) do
-        Soda.queue(copy["queue"]) do |queue|
+      self.class.middleware.use(item["klass"], copy, copy["queue"]) do
+        jid = copy["id"]
+        jid.tap do
+          queue = Soda.queue(copy["queue"])
           queue.push_in(copy["delay"], Soda.dump_json(copy))
         end
       end
@@ -14,19 +30,20 @@ module Soda
     private
 
     def normalize!(item)
-      item.dup.tap do |copy|
-        copy.keys.each do |key|
-          copy.merge!(String(key) => copy[key])
+      item = DEFAULTS.merge(item)
+      item.tap do
+        item.keys.each do |key|
+          item.merge!(String(key) => item.delete(key))
         end
 
         id      = SecureRandom.base64(10)
-        klass   = copy["klass"].to_s
-        delay   = Integer(copy["delay"]) || 0
-        queue   = copy["queue"] || Soda.default_queue!.name
+        klass   = item["klass"].to_s
+        delay   = item["delay"].to_i
+        queue   = item["queue"] || Soda.default_queue!.name
 
         # TODO: add validation
         #
-        copy.merge!(
+        item.merge!(
           "id"    => id,
           "klass" => klass,
           "delay" => delay,

data/lib/soda/extensions/active_job.rb

@@ -0,0 +1,27 @@
+module ActiveJob
+  module QueueAdapters
+    class SodaAdapter
+      def enqueue(job)
+        enqueue_at(job, Time.now)
+      end
+
+      def enqueue_at(job, ts)
+        job.provider_job_id = ::Soda::Client.push(
+          "klass"   => JobWrapper,
+          "wrapped" => job.class,
+          "queue"   => job.queue_name,
+          "delay"   => [0, (ts - Time.now).to_i].max,
+          "args"    => [job.serialize],
+        )
+      end
+
+      class JobWrapper
+        include ::Soda::Worker
+
+        def perform(data = {})
+          Base.execute(data.merge("provider_job_id" => id))
+        end
+      end
+    end
+  end
+end

data/lib/soda/fetcher.rb

@@ -37,7 +37,7 @@ module Soda
       start = now
       logger.debug(%(fetching from "%s") % queue.name)
 
-      queue.pop.tap do |msgs|
+      (queue.pop || []).tap do |msgs|
         logger.debug(%(fetched %d message(s) from "%s" (%fms)) % [msgs.count, queue.name, (now - start)])
       end
     end

data/lib/soda/middleware/chain.rb

@@ -45,7 +45,7 @@ module Soda
           yield
         else
           entry = copy.shift
-          inst  = entry.klass.new(*entry.args)
+          inst  = entry.build
 
           inst.call(*args) do
             traverse(copy, args) { yield }

data/lib/soda/processor.rb

@@ -2,6 +2,12 @@ module Soda
   class Processor
     include Tools
 
+    class << self
+      def middleware
+        Soda.server_middleware
+      end
+    end
+
     def initialize(manager)
       @manager      = manager
       @retrier      = Retrier.new
@@ -31,7 +37,7 @@ module Soda
 
     def run
       until stopped?
-        msgs = fetch
+        msgs = fetch || []
         msgs.each(&method(:process))
       end
     rescue Exception => ex
@@ -50,7 +56,9 @@ module Soda
     def process(msg)
       if (job_hash = parse_job(msg.str))
         job_logger.with(job_hash) do
-          execute(job_hash, msg)
+          reloader.wrap do
+            execute(job_hash, msg)
+          end
         end
       else
         # We can't process the work because the JSON is invalid, so we have to
@@ -69,8 +77,7 @@ module Soda
       worker  = constantize(klass)
 
       retrier.retry(job_hash, msg) do
-        middleware = Soda.server_middleware
-        middleware.use(worker, job_hash, queue.name, msg) do
+        self.class.middleware.use(worker, job_hash, queue.name, msg) do
           instance = worker.new(job_hash)
           instance.perform(*job_hash["args"])
         end
@@ -98,5 +105,21 @@ module Soda
     def constantize(str)
       Object.const_get(str)
     end
+
+    # To support Rails reloading from the CLI context, the following code find
+    # the Rails reloader or stubs a no-op one.
+    class StubReloader
+      def wrap; yield; end
+    end
+
+    def reloader
+      @reloader ||=
+        if defined?(::Soda::Rails)
+          application = ::Rails.application
+          application.reloader
+        else
+          StubReloader.new
+        end
+    end
   end
 end

data/lib/soda/rails.rb

@@ -0,0 +1,6 @@
+module Soda
+  module Rails
+    class Engine < ::Rails::Engine
+    end
+  end
+end

data/lib/soda/retrier.rb

@@ -24,7 +24,7 @@ module Soda
       if ret.is_a?(Numeric)
         ret < msg.receive_count
       else
-        ret
+        !!ret
       end
     end
 

data/lib/soda/tools.rb

@@ -19,5 +19,24 @@ module Soda
     def now
       Process.clock_gettime(Process::CLOCK_MONOTONIC)
     end
+
+    def deep_symbolize_keys(hash)
+      transform_value = -> (value) {
+        case value
+        when Hash
+          deep_symbolize_keys(value)
+        when Array
+          value.map { |val| transform_value.call(val) }
+        else
+          value
+        end
+      }
+
+      {}.tap do |memo|
+        hash.each do |key, value|
+          memo.merge!(key.to_sym => transform_value.call(value))
+        end
+      end
+    end
   end
 end

data/lib/soda/version.rb

@@ -1,3 +1,3 @@
 module Soda
-  VERSION = "0.0.1".freeze
+  VERSION = "0.0.8".freeze
 end

data/lib/soda/worker.rb

@@ -28,6 +28,7 @@ module Soda
           )
         end
       end
+      alias_method :perform_at, :perform_in
 
       private
 
@@ -70,6 +71,12 @@ module Soda
       @options = options
     end
 
+    %i[id].each do |method|
+      define_method(method) do
+        options.fetch(String(method))
+      end
+    end
+
     private
 
     attr_reader :options

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: soda-core
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.8
 platform: ruby
 authors:
 - Noah Portes Chaikin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-15 00:00:00.000000000 Z
+date: 2020-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sqs
@@ -63,6 +63,7 @@ files:
 - lib/soda.rb
 - lib/soda/cli.rb
 - lib/soda/client.rb
+- lib/soda/extensions/active_job.rb
 - lib/soda/fetcher.rb
 - lib/soda/logger.rb
 - lib/soda/manager.rb
@@ -70,6 +71,7 @@ files:
 - lib/soda/processor.rb
 - lib/soda/queue.rb
 - lib/soda/queues/registry.rb
+- lib/soda/rails.rb
 - lib/soda/retrier.rb
 - lib/soda/tools.rb
 - lib/soda/version.rb