funktor 0.4.7 → 0.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8d94ef31a3eeefcc0da81890f83168ab12d427ed53144bb7b65af723b5b2a39f
-  data.tar.gz: bf7e0c0d492a5561ac7d90b429d594c64d3b6fdadde4abc75b5d937e60561b12
+  metadata.gz: 99aa84e15abe58236ad036f0ead087630ab418a99f6240ad438e7e1809e097c7
+  data.tar.gz: 3ac97ae4967ba9abff96aa5073e8bb29be144443d8dcbebe27437f9fd12ec57a
 SHA512:
-  metadata.gz: 19dc49ea92ff06d03126e50fd2a871fe09b7e2552a8757391b761758772373bd2d99568928f6d967011185a12ba35ecb39d6f8275976fc1cc3c38258f157f4e9
-  data.tar.gz: d7eaf3c888e0a6741999199f56b2203ca3a35f539efab69852c83ef80f69adf46f6efdffe2fb1ce9342e3d09cb57c0732a68ba84376027bd71b5fec89e9e7b19
+  metadata.gz: dce4b51048490c5a2ee4c8bc794cfe3d23e3fc30ed90e1efd2fb2a9bf49f64eff3fa2656c2784a5b712752398e73caa3ee4e4079da185cbd0d80de8edf41285f
+  data.tar.gz: 747d3ed016dff5b1e5fc63deb1e332b29ac53602f40d0f4a88711f59883d2571f72f267021a285decf3c887a542800a8dcb0f50da4c4215f980b4cce80324856

data/.tool-versions

@@ -0,0 +1,2 @@
+ruby 2.7.3
+nodejs 14.16.1

data/Gemfile.lock

@@ -1,32 +1,36 @@
 PATH
   remote: .
   specs:
-    funktor (0.4.7)
+    funktor (0.6.2)
       activesupport
+      aws-sdk-dynamodb (~> 1.62)
       aws-sdk-sqs (~> 1.37)
       thor
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (6.1.4)
-      activesupport (= 6.1.4)
+    activejob (6.1.4.1)
+      activesupport (= 6.1.4.1)
       globalid (>= 0.3.6)
-    activesupport (6.1.4)
+    activesupport (6.1.4.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
       zeitwerk (~> 2.3)
-    addressable (2.7.0)
+    addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
     aws-eventstream (1.1.1)
-    aws-partitions (1.482.0)
-    aws-sdk-core (3.119.0)
+    aws-partitions (1.491.0)
+    aws-sdk-core (3.119.1)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.239.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
+    aws-sdk-dynamodb (1.62.0)
+      aws-sdk-core (~> 3, >= 3.119.0)
+      aws-sigv4 (~> 1.1)
     aws-sdk-sqs (1.42.0)
       aws-sdk-core (~> 3, >= 3.119.0)
       aws-sigv4 (~> 1.1)
@@ -38,8 +42,8 @@ GEM
     crack (0.4.5)
       rexml
     diff-lcs (1.4.4)
-    docile (1.3.5)
-    globalid (0.5.1)
+    docile (1.4.0)
+    globalid (0.5.2)
       activesupport (>= 5.0)
     hashdiff (1.0.1)
     i18n (1.8.10)
@@ -47,6 +51,8 @@ GEM
     jmespath (1.4.0)
     method_source (1.0.0)
     minitest (5.14.4)
+    mustermann (1.1.1)
+      ruby2_keywords (~> 0.0.1)
     pry (0.13.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
@@ -54,6 +60,9 @@ GEM
       byebug (~> 11.0)
       pry (~> 0.13.0)
     public_suffix (4.0.6)
+    rack (2.2.3)
+    rack-protection (2.1.0)
+      rack
     rake (12.3.3)
     rexml (3.2.5)
     rspec (3.10.0)
@@ -69,17 +78,25 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
     rspec-support (3.10.2)
+    ruby2_keywords (0.0.5)
     simplecov (0.21.2)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
-    simplecov_json_formatter (0.1.2)
+    simplecov_json_formatter (0.1.3)
+    sinatra (2.1.0)
+      mustermann (~> 1.0)
+      rack (~> 2.2)
+      rack-protection (= 2.1.0)
+      tilt (~> 2.0)
     thor (1.1.0)
+    tilt (2.0.10)
+    timecop (0.9.4)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    webmock (3.12.2)
-      addressable (>= 2.3.6)
+    webmock (3.14.0)
+      addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
     zeitwerk (2.4.2)
@@ -94,6 +111,8 @@ DEPENDENCIES
   rake (~> 12.0)
   rspec (~> 3.0)
   simplecov
+  sinatra
+  timecop
   webmock
 
 BUNDLED WITH

data/funktor-testapp/.envrc

@@ -0,0 +1 @@
+dotenv

data/funktor-testapp/.gitignore

@@ -0,0 +1,7 @@
+.env
+.serverless
+node_modules
+
+# This is our copy of the development stuff
+funktor
+

data/funktor-testapp/Gemfile

@@ -0,0 +1,25 @@
+source 'https://rubygems.org'
+git_source(:github) { |repo| "https://github.com/#{repo}.git" }
+
+ruby '~> 2.7'
+
+# We use the 1.0 branch for it's circular dependency resolution,
+# just because it's easy, and the classes in this project don't do
+# anything fancy that would cause problems for it.
+gem 'require_all', '~> 1.0'
+
+
+# We're not using funktor as a regular gem because this is an app for testing funktor
+# during development.
+#
+# gem 'funktor'
+#
+# Instead we're going to recreate the dependencies here so that we can use a local copy
+# of the gem directly. TODO - Figure out if we can have this happen automatically so we
+# don't have to keep things in sync.
+
+gem 'aws-sdk-sqs', '~> 1.37'
+gem 'aws-sdk-dynamodb', '~> 1.62'
+gem "activesupport"
+gem "thor" # Thor drives the CLI TODO - Maybe we don't need it here?
+

data/funktor-testapp/Gemfile.lock

@@ -0,0 +1,51 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (6.1.4.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    aws-eventstream (1.1.1)
+    aws-partitions (1.488.0)
+    aws-sdk-core (3.119.0)
+      aws-eventstream (~> 1, >= 1.0.2)
+      aws-partitions (~> 1, >= 1.239.0)
+      aws-sigv4 (~> 1.1)
+      jmespath (~> 1.0)
+    aws-sdk-dynamodb (1.62.0)
+      aws-sdk-core (~> 3, >= 3.119.0)
+      aws-sigv4 (~> 1.1)
+    aws-sdk-sqs (1.42.0)
+      aws-sdk-core (~> 3, >= 3.119.0)
+      aws-sigv4 (~> 1.1)
+    aws-sigv4 (1.2.4)
+      aws-eventstream (~> 1, >= 1.0.2)
+    concurrent-ruby (1.1.9)
+    i18n (1.8.10)
+      concurrent-ruby (~> 1.0)
+    jmespath (1.4.0)
+    minitest (5.14.4)
+    require_all (1.5.0)
+    thor (1.1.0)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    zeitwerk (2.4.2)
+
+PLATFORMS
+  x86_64-darwin-20
+  x86_64-linux
+
+DEPENDENCIES
+  activesupport
+  aws-sdk-dynamodb (~> 1.62)
+  aws-sdk-sqs (~> 1.37)
+  require_all (~> 1.0)
+  thor
+
+RUBY VERSION
+   ruby 2.7.3p183
+
+BUNDLED WITH
+   2.2.25

data/funktor-testapp/app/services/job_flood.rb

@@ -0,0 +1,38 @@
+class JobFlood
+  attr_accessor :length_in_minutes
+  attr_accessor :min_jobs_per_minute
+  attr_accessor :max_jobs_per_minute
+  attr_accessor :max_job_length_in_seconds
+  attr_accessor :error_percentage
+  attr_accessor :error_percentage
+
+  def initialize length_in_minutes: 5, min_jobs_per_minute: 30, max_jobs_per_minute: 120, max_job_length_in_seconds: 3, error_percentage: 25
+    @length_in_minutes = length_in_minutes
+    @min_jobs_per_minute = min_jobs_per_minute
+    @max_jobs_per_minute = max_jobs_per_minute
+    @max_job_length_in_seconds = max_job_length_in_seconds
+    @error_percentage = error_percentage
+  end
+
+  def flood
+    total_jobs = 0
+    length_in_minutes.times do |minute|
+      jobs_to_generate = rand(min_jobs_per_minute..max_jobs_per_minute)
+      jobs_to_generate.times do
+        total_jobs += 1
+        job_target_time = Time.now + (minute * 60) + rand(60)
+        job_sleep = rand(0.0..max_job_length_in_seconds.to_f)
+        puts job_target_time
+        [AuditWorker, AuditWorker, SingleThreadAuditWorker].sample.perform_at(job_target_time, {
+          mode: 'later',
+          message: 'msg: from random JobFlood - ' + SecureRandom.hex,
+          target_time: job_target_time,
+          error_percentage: error_percentage,
+          job_sleep: job_sleep
+        })
+      end
+    end
+    puts "generated #{total_jobs} jobs over the next #{length_in_minutes} minutes"
+  end
+
+end

data/funktor-testapp/app/workers/audit_worker.rb

@@ -0,0 +1,49 @@
+class AuditWorker
+  include Funktor::Worker
+
+  def perform(arg_hash)
+    time_now = Time.now
+    target_time = Time.parse arg_hash['target_time']
+    time_diff = time_now - target_time
+    Funktor.raw_logger.unknown Funktor.dump_json(metric_hash(time_diff))
+
+    puts "Greetings from the #{self.class.name}! Time diff = #{time_diff}"
+    puts arg_hash.class.name
+    puts arg_hash
+
+    if arg_hash["error_percentage"] && rand(100) < arg_hash["error_percentage"].to_i
+      raise "Oops, we encountered a 'random error'"
+    end
+    if arg_hash["job_sleep"]
+      puts "Working (sleeping) for #{arg_hash["job_sleep"]} seconds"
+      sleep arg_hash["job_sleep"]
+    end
+    puts "So long from the #{self.class.name}, and thanks for all the fish!"
+  end
+
+  def metric_hash(time_diff)
+    {
+      "_aws": {
+        "Timestamp": Time.now.strftime('%s%3N').to_i,
+        "CloudWatchMetrics": [
+          {
+            "Namespace": ENV['FUNKTOR_APP_NAME'],
+            "Dimensions": [["WorkerClassName"]],
+            "Metrics": [ # CPU, Memory, Duration, etc...
+                         {
+                           "Name": "TimeDiff",
+                           "Unit": "Seconds"
+                         }
+            ]
+          }
+        ]
+      },
+      "WorkerClassName": self.class.name,
+      "TimeDiff": time_diff
+      #"count": value,
+      #"requestId": "989ffbf8-9ace-4817-a57c-e4dd734019ee"
+    }
+    #data[key] = value
+  end
+end
+

data/funktor-testapp/app/workers/greetings_worker.rb

@@ -0,0 +1,3 @@
+class GreetingsWorker < HelloWorker
+
+end

data/funktor-testapp/app/workers/hello_worker.rb

@@ -0,0 +1,18 @@
+class HelloWorker
+  include Funktor::Worker
+
+  def perform(arg_hash)
+    puts "Greetings from the #{self.class.name}!"
+    puts arg_hash.class.name
+    puts arg_hash
+    if arg_hash["error_percentage"] && rand(100) < arg_hash["error_percentage"].to_i
+      raise "Oops, we encountered a 'random error'"
+    end
+    if arg_hash["job_sleep"]
+      puts "Working (sleeping) for #{arg_hash["job_sleep"]} seconds"
+      sleep arg_hash["job_sleep"]
+    end
+    puts "So long from the #{self.class.name}, and thanks for all the fish!"
+  end
+end
+

data/funktor-testapp/app/workers/single_thread_audit_worker.rb

@@ -0,0 +1,3 @@
+class SingleThreadAuditWorker < AuditWorker
+  funktor_options queue: :low_concurrency
+end

data/funktor-testapp/deploy-dev.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+cp -r ../lib ./funktor
+
+serverless deploy --stage dev -v

data/funktor-testapp/funktor_config/boot.rb

@@ -0,0 +1,17 @@
+# Point at our Gemfile
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
+
+# load rubygems & bundler
+require "rubygems"
+require 'bundler/setup'
+
+# Set up gems listed in the Gemfile.
+Bundler.require(:default, :production)
+
+# Here we load our development copy of funktor that we copy in using deploy-dev.sh
+$LOAD_PATH.unshift 'funktor/lib'
+require 'funktor'
+
+# Load all ruby files in the app directory
+require_rel File.join('..', 'app')
+

data/funktor-testapp/funktor_config/environment.yml

@@ -0,0 +1,15 @@
+BUNDLE_WITHOUT: development:test:assets
+BUNDLE_PLUGINS: false
+BUNDLE_FROZEN: true
+SERVERLESS_STAGE: ${self:custom.stage}
+FUNKTOR_APP_NAME: funktor-testapp
+FUNKTOR_INCOMING_JOB_QUEUE:
+  Ref: IncomingJobQueue
+FUNKTOR_DEFAULT_QUEUE:
+  Ref: DefaultQueue
+FUNKTOR_LOW_CONCURRENCY_QUEUE:
+  Ref: LowConcurrencyQueue
+FUNKTOR_ACTIVITY_TABLE:
+  Ref: ActivityTable
+FUNKTOR_JOBS_TABLE:
+  Ref: JobsTable

data/funktor-testapp/funktor_config/function_definitions/default_queue_handler.yml

@@ -0,0 +1,13 @@
+handler: lambda_event_handlers/default_queue_handler.call
+timeout: ${self:custom.funktor.DefaultQueueHandler.functionTimeout, 900}
+reservedConcurrency: ${self:custom.funktor.DefaultQueueHandler.reservedConcurrency, null}
+provisionedConcurrency: ${self:custom.funktor.DefaultQueueHandler.provisionedConcurrency, null}
+memorySize: ${self:custom.funktor.DefaultQueueHandler.memorySize, 256}
+events:
+  - sqs:
+      batchSize: ${self:custom.funktor.DefaultQueueHandler.batchSize, 1}
+      maximumBatchingWindow: ${self:custom.funktor.DefaultQueueHandler.maximumBatchingWindow, 0}
+      arn:
+        Fn::GetAtt:
+          - DefaultQueue
+          - Arn

data/funktor-testapp/funktor_config/function_definitions/incoming_job_handler.yml

@@ -0,0 +1,13 @@
+handler: lambda_event_handlers/incoming_job_handler.call
+timeout: ${self:custom.funktor.IncomingJobHandler.functionTimeout, 30}
+reservedConcurrency: ${self:custom.funktor.IncomingJobHandler.reservedConcurrency, null}
+provisionedConcurrency: ${self:custom.funktor.IncomingJobHandler.provisionedConcurrency, null}
+memorySize: ${self:custom.funktor.IncomingJobHandler.memorySize, 256}
+events:
+  - sqs:
+      batchSize: ${self:custom.funktor.IncomingJobHandler.batchSize, 1}
+      maximumBatchingWindow: ${self:custom.funktor.IncomingJobHandler.maximumBatchingWindow, 0}
+      arn:
+        Fn::GetAtt:
+          - IncomingJobQueue
+          - Arn

data/funktor-testapp/funktor_config/function_definitions/job_activator.yml

@@ -0,0 +1,7 @@
+handler: lambda_event_handlers/job_activator.call
+timeout: ${self:custom.funktor.JobActivator.functionTimeout, 30}
+reservedConcurrency: ${self:custom.funktor.JobActivator.reservedConcurrency, null}
+provisionedConcurrency: ${self:custom.funktor.JobActivator.provisionedConcurrency, null}
+memorySize: ${self:custom.funktor.JobActivator.memorySize, 256}
+events:
+  - schedule: rate(1 minute)

data/funktor-testapp/funktor_config/function_definitions/low_concurrency_queue_handler.yml

@@ -0,0 +1,13 @@
+handler: lambda_event_handlers/low_concurrency_queue_handler.call
+timeout: ${self:custom.funktor.LowConcurrencyQueueHandler.functionTimeout, 900}
+reservedConcurrency: ${self:custom.funktor.LowConcurrencyQueueHandler.reservedConcurrency, null}
+provisionedConcurrency: ${self:custom.funktor.LowConcurrencyQueueHandler.provisionedConcurrency, null}
+memorySize: ${self:custom.funktor.LowConcurrencyQueueHandler.memorySize, 256}
+events:
+  - sqs:
+      batchSize: ${self:custom.funktor.LowConcurrencyQueueHandler.batchSize, 1}
+      maximumBatchingWindow: ${self:custom.funktor.LowConcurrencyQueueHandler.maximumBatchingWindow, 0}
+      arn:
+        Fn::GetAtt:
+          - LowConcurrencyQueue
+          - Arn

data/funktor-testapp/funktor_config/function_definitions/random_job_generator.yml

@@ -0,0 +1,18 @@
+handler: lambda_event_handlers/random_job_generator.RandomJobGenerator.call
+timeout: 58
+reservedConcurrency: 0
+events:
+  - schedule: rate(1 minute)
+environment:
+  # MAX_JOB_DELAY_IN_SECONDS controls the maximum delay that will be put on a randomly generated job.
+  MAX_JOB_DELAY_IN_SECONDS: 1200 # 20 minutes
+  # These values control how many random jobs will be generated per second. With extremely high values
+  # the generator may become limited by how quickly it can push to SQS. If you need to test LOTS of
+  # jobs you may want to set the tiemout to be longer and increase reservedConcurrency so that you
+  # can have multiple random generators running at one time.
+  MIN_RANDOM_JOBS_PER_MINUTE: 30
+  MAX_RANDOM_JOBS_PER_MINUTE: 120
+  # MAX_JOB_LENGTH_IN_SECONDS controls how long the slowest job will take to execute (by sleeping)
+  MAX_JOB_LENGTH_IN_SECONDS: 3
+  # ERROR_PERCENTAGE is the percent of requests that should raise an error and be retried.
+  ERROR_PERCENTAGE: 25