funktor 0.4.7 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8d94ef31a3eeefcc0da81890f83168ab12d427ed53144bb7b65af723b5b2a39f
-  data.tar.gz: bf7e0c0d492a5561ac7d90b429d594c64d3b6fdadde4abc75b5d937e60561b12
+  metadata.gz: 6216b831ca8e28aa8d17cb86295d740bf63e95df6ce36ec966f435b6df7f427c
+  data.tar.gz: 7ade52e2f087fce2b5f61461bcbec625f7a72a82557500e3c2efb98606deadb7
 SHA512:
-  metadata.gz: 19dc49ea92ff06d03126e50fd2a871fe09b7e2552a8757391b761758772373bd2d99568928f6d967011185a12ba35ecb39d6f8275976fc1cc3c38258f157f4e9
-  data.tar.gz: d7eaf3c888e0a6741999199f56b2203ca3a35f539efab69852c83ef80f69adf46f6efdffe2fb1ce9342e3d09cb57c0732a68ba84376027bd71b5fec89e9e7b19
+  metadata.gz: 6a9b429d66bbf60a0bd64a040616eb7743b861466646550f98151dc99320ed882233e131a82d21badcacfc2f7916b05c5b6c52168bb8899c8c81823289b24ab3
+  data.tar.gz: c594578835a62eda89537a42c55021badcdb1a5e240abb1a493d36dea7e3aed5e4a3cf7126afd2aef75bcae17c7619c0e404268488308c6f5e1974d2e2ffc322

data/.tool-versions

@@ -0,0 +1,2 @@
+ruby 2.7.3
+nodejs 14.16.1

data/Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    funktor (0.4.7)
+    funktor (0.5.0)
       activesupport
+      aws-sdk-dynamodb (~> 1.62)
       aws-sdk-sqs (~> 1.37)
       thor
 
@@ -18,15 +19,18 @@ GEM
       minitest (>= 5.1)
       tzinfo (~> 2.0)
       zeitwerk (~> 2.3)
-    addressable (2.7.0)
+    addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
     aws-eventstream (1.1.1)
-    aws-partitions (1.482.0)
+    aws-partitions (1.487.0)
     aws-sdk-core (3.119.0)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.239.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
+    aws-sdk-dynamodb (1.62.0)
+      aws-sdk-core (~> 3, >= 3.119.0)
+      aws-sigv4 (~> 1.1)
     aws-sdk-sqs (1.42.0)
       aws-sdk-core (~> 3, >= 3.119.0)
       aws-sigv4 (~> 1.1)

data/funktor-testapp/.envrc

@@ -0,0 +1 @@
+dotenv

data/funktor-testapp/.gitignore

@@ -0,0 +1,7 @@
+.env
+.serverless
+node_modules
+
+# This is our copy of the development stuff
+funktor
+

data/funktor-testapp/Gemfile

@@ -0,0 +1,25 @@
+source 'https://rubygems.org'
+git_source(:github) { |repo| "https://github.com/#{repo}.git" }
+
+ruby '~> 2.7'
+
+# We use the 1.0 branch for it's circular dependency resolution,
+# just because it's easy, and the classes in this project don't do
+# anything fancy that would cause problems for it.
+gem 'require_all', '~> 1.0'
+
+
+# We're not using funktor as a regular gem because this is an app for testing funktor
+# during development.
+#
+# gem 'funktor'
+#
+# Instead we're going to recreate the dependencies here so that we can use a local copy
+# of the gem directly. TODO - Figure out if we can have this happen automatically so we
+# don't have to keep things in sync.
+
+gem 'aws-sdk-sqs', '~> 1.37'
+gem 'aws-sdk-dynamodb', '~> 1.62'
+gem "activesupport"
+gem "thor" # Thor drives the CLI TODO - Maybe we don't need it here?
+

data/funktor-testapp/Gemfile.lock

@@ -0,0 +1,51 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (6.1.4)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    aws-eventstream (1.1.1)
+    aws-partitions (1.483.0)
+    aws-sdk-core (3.119.0)
+      aws-eventstream (~> 1, >= 1.0.2)
+      aws-partitions (~> 1, >= 1.239.0)
+      aws-sigv4 (~> 1.1)
+      jmespath (~> 1.0)
+    aws-sdk-dynamodb (1.62.0)
+      aws-sdk-core (~> 3, >= 3.119.0)
+      aws-sigv4 (~> 1.1)
+    aws-sdk-sqs (1.42.0)
+      aws-sdk-core (~> 3, >= 3.119.0)
+      aws-sigv4 (~> 1.1)
+    aws-sigv4 (1.2.4)
+      aws-eventstream (~> 1, >= 1.0.2)
+    concurrent-ruby (1.1.9)
+    i18n (1.8.10)
+      concurrent-ruby (~> 1.0)
+    jmespath (1.4.0)
+    minitest (5.14.4)
+    require_all (1.5.0)
+    thor (1.1.0)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    zeitwerk (2.4.2)
+
+PLATFORMS
+  x86_64-darwin-20
+  x86_64-linux
+
+DEPENDENCIES
+  activesupport
+  aws-sdk-dynamodb (~> 1.62)
+  aws-sdk-sqs (~> 1.37)
+  require_all (~> 1.0)
+  thor
+
+RUBY VERSION
+   ruby 2.7.3p183
+
+BUNDLED WITH
+   2.2.25

data/funktor-testapp/app/services/job_flood.rb

@@ -0,0 +1,38 @@
+class JobFlood
+  attr_accessor :length_in_minutes
+  attr_accessor :min_jobs_per_minute
+  attr_accessor :max_jobs_per_minute
+  attr_accessor :max_job_length_in_seconds
+  attr_accessor :error_percentage
+  attr_accessor :error_percentage
+
+  def initialize length_in_minutes: 5, min_jobs_per_minute: 30, max_jobs_per_minute: 120, max_job_length_in_seconds: 3, error_percentage: 25
+    @length_in_minutes = length_in_minutes
+    @min_jobs_per_minute = min_jobs_per_minute
+    @max_jobs_per_minute = max_jobs_per_minute
+    @max_job_length_in_seconds = max_job_length_in_seconds
+    @error_percentage = error_percentage
+  end
+
+  def flood
+    total_jobs = 0
+    length_in_minutes.times do |minute|
+      jobs_to_generate = rand(min_jobs_per_minute..max_jobs_per_minute)
+      jobs_to_generate.times do
+        total_jobs += 1
+        job_target_time = Time.now + (minute * 60) + rand(60)
+        job_sleep = rand(0.0..max_job_length_in_seconds.to_f)
+        puts job_target_time
+        AuditWorker.perform_at(job_target_time, {
+          mode: 'later',
+          message: 'msg: from random JobFlood - ' + SecureRandom.hex,
+          target_time: job_target_time,
+          error_percentage: error_percentage,
+          job_sleep: job_sleep
+        })
+      end
+    end
+    puts "generated #{total_jobs} jobs over the next #{length_in_minutes} minutes"
+  end
+
+end

data/funktor-testapp/app/workers/audit_worker.rb

@@ -0,0 +1,49 @@
+class AuditWorker
+  include Funktor::Worker
+
+  def perform(arg_hash)
+    time_now = Time.now
+    target_time = Time.parse arg_hash['target_time']
+    time_diff = time_now - target_time
+    Funktor.raw_logger.unknown Funktor.dump_json(metric_hash(time_diff))
+
+    puts "Greetings from the #{self.class.name}! Time diff = #{time_diff}"
+    puts arg_hash.class.name
+    puts arg_hash
+
+    if arg_hash["error_percentage"] && rand(100) < arg_hash["error_percentage"].to_i
+      raise "Oops, we encountered a 'random error'"
+    end
+    if arg_hash["job_sleep"]
+      puts "Working (sleeping) for #{arg_hash["job_sleep"]} seconds"
+      sleep arg_hash["job_sleep"]
+    end
+    puts "So long from the #{self.class.name}, and thanks for all the fish!"
+  end
+
+  def metric_hash(time_diff)
+    {
+      "_aws": {
+        "Timestamp": Time.now.strftime('%s%3N').to_i,
+        "CloudWatchMetrics": [
+          {
+            "Namespace": ENV['FUNKTOR_APP_NAME'],
+            "Dimensions": [["WorkerClassName"]],
+            "Metrics": [ # CPU, Memory, Duration, etc...
+                         {
+                           "Name": "TimeDiff",
+                           "Unit": "Seconds"
+                         }
+            ]
+          }
+        ]
+      },
+      "WorkerClassName": self.class.name,
+      "TimeDiff": time_diff
+      #"count": value,
+      #"requestId": "989ffbf8-9ace-4817-a57c-e4dd734019ee"
+    }
+    #data[key] = value
+  end
+end
+

data/funktor-testapp/app/workers/greetings_worker.rb

@@ -0,0 +1,3 @@
+class GreetingsWorker < HelloWorker
+
+end

data/funktor-testapp/app/workers/hello_worker.rb

@@ -0,0 +1,18 @@
+class HelloWorker
+  include Funktor::Worker
+
+  def perform(arg_hash)
+    puts "Greetings from the #{self.class.name}!"
+    puts arg_hash.class.name
+    puts arg_hash
+    if arg_hash["error_percentage"] && rand(100) < arg_hash["error_percentage"].to_i
+      raise "Oops, we encountered a 'random error'"
+    end
+    if arg_hash["job_sleep"]
+      puts "Working (sleeping) for #{arg_hash["job_sleep"]} seconds"
+      sleep arg_hash["job_sleep"]
+    end
+    puts "So long from the #{self.class.name}, and thanks for all the fish!"
+  end
+end
+

data/funktor-testapp/deploy-dev.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+cp -r ../lib ./funktor
+
+serverless deploy --stage dev -v

data/funktor-testapp/funktor_config/boot.rb

@@ -0,0 +1,17 @@
+# Point at our Gemfile
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
+
+# load rubygems & bundler
+require "rubygems"
+require 'bundler/setup'
+
+# Set up gems listed in the Gemfile.
+Bundler.require(:default, :production)
+
+# Here we load our development copy of funktor that we copy in using deploy-dev.sh
+$LOAD_PATH.unshift 'funktor/lib'
+require 'funktor'
+
+# Load all ruby files in the app directory
+require_rel File.join('..', 'app')
+

data/funktor-testapp/funktor_config/environment.yml

@@ -0,0 +1,15 @@
+BUNDLE_WITHOUT: development:test:assets
+BUNDLE_PLUGINS: false
+BUNDLE_FROZEN: true
+SERVERLESS_STAGE: ${self:custom.stage}
+FUNKTOR_APP_NAME: funktor-testapp
+FUNKTOR_INCOMING_JOB_QUEUE:
+  Ref: IncomingJobQueue
+FUNKTOR_DEFAULT_QUEUE:
+  Ref: DefaultQueue
+FUNKTOR_SINGLE_THREAD_QUEUE:
+  Ref: SingleThreadQueue
+FUNKTOR_ACTIVITY_TABLE:
+  Ref: ActivityTable
+FUNKTOR_JOBS_TABLE:
+  Ref: JobsTable

data/funktor-testapp/funktor_config/function_definitions/default_queue_handler.yml

@@ -0,0 +1,11 @@
+handler: lambda_event_handlers/default_queue_handler.call
+timeout: ${self:custom.funktor.DefaultQueueHandler.timeout, 900}
+reservedConcurrency: ${self:custom.funktor.DefaultQueueHandler.reservedConcurrency, null}
+provisionedConcurrency: ${self:custom.funktor.DefaultQueueHandler.provisionedConcurrency, null}
+memorySize: ${self:custom.funktor.DefaultQueueHandler.memorySize, 256}
+events:
+  - sqs:
+      arn:
+        Fn::GetAtt:
+          - DefaultQueue
+          - Arn

data/funktor-testapp/funktor_config/function_definitions/incoming_job_handler.yml

@@ -0,0 +1,11 @@
+handler: lambda_event_handlers/incoming_job_handler.call
+timeout: ${self:custom.funktor.IncomingJobHandler.timeout, 30}
+reservedConcurrency: ${self:custom.funktor.IncomingJobHandler.reservedConcurrency, null}
+provisionedConcurrency: ${self:custom.funktor.IncomingJobHandler.provisionedConcurrency, null}
+memorySize: ${self:custom.funktor.IncomingJobHandler.memorySize, 256}
+events:
+  - sqs:
+      arn:
+        Fn::GetAtt:
+          - IncomingJobQueue
+          - Arn

data/funktor-testapp/funktor_config/function_definitions/job_activator.yml

@@ -0,0 +1,8 @@
+handler: lambda_event_handlers/job_activator.call
+timeout: ${self:custom.funktor.JobActivator.timeout, 30}
+reservedConcurrency: ${self:custom.funktor.JobActivator.reservedConcurrency, null}
+provisionedConcurrency: ${self:custom.funktor.JobActivator.provisionedConcurrency, null}
+memorySize: ${self:custom.funktor.JobActivator.memorySize, 256}
+#reservedConcurrency: 1
+events:
+  - schedule: rate(1 minute)

data/funktor-testapp/funktor_config/function_definitions/random_job_generator.yml

@@ -0,0 +1,18 @@
+handler: lambda_event_handlers/random_job_generator.RandomJobGenerator.call
+timeout: 58
+reservedConcurrency: 0
+events:
+  - schedule: rate(1 minute)
+environment:
+  # MAX_JOB_DELAY_IN_SECONDS controls the maximum delay that will be put on a randomly generated job.
+  MAX_JOB_DELAY_IN_SECONDS: 1200 # 20 minutes
+  # These values control how many random jobs will be generated per second. With extremely high values
+  # the generator may become limited by how quickly it can push to SQS. If you need to test LOTS of
+  # jobs you may want to set the tiemout to be longer and increase reservedConcurrency so that you
+  # can have multiple random generators running at one time.
+  MIN_RANDOM_JOBS_PER_MINUTE: 30
+  MAX_RANDOM_JOBS_PER_MINUTE: 120
+  # MAX_JOB_LENGTH_IN_SECONDS controls how long the slowest job will take to execute (by sleeping)
+  MAX_JOB_LENGTH_IN_SECONDS: 3
+  # ERROR_PERCENTAGE is the percent of requests that should raise an error and be retried.
+  ERROR_PERCENTAGE: 25

data/funktor-testapp/funktor_config/function_definitions/single_thread_queue_handler.yml

@@ -0,0 +1,11 @@
+handler: lambda_event_handlers/single_thread_queue_handler.call
+timeout: ${self:custom.funktor.SingleThreadQueueHandler.timeout, 900}
+reservedConcurrency: ${self:custom.funktor.SingleThreadQueueHandler.reservedConcurrency, null}
+provisionedConcurrency: ${self:custom.funktor.SingleThreadQueueHandler.provisionedConcurrency, null}
+memorySize: ${self:custom.funktor.SingleThreadQueueHandler.memorySize, 256}
+events:
+  - sqs:
+      arn:
+        Fn::GetAtt:
+          - SingleThreadQueue
+          - Arn

data/funktor-testapp/funktor_config/funktor.yml

@@ -0,0 +1,114 @@
+IncomingJobHandler:
+  # timeout is how long the handler can possibly run. Up to 10 messages may be delivered
+  # to a handler at one time, so you'll want this to be at least 10x the maximum time you
+  # expect to spend for one message. The incoming job handler usually will be pretty fast,
+  # but we default to a high number here to allow for the times when things go weird.
+  timeout: 300
+  # reservedConcurrency represents the maximum number of concurrent executions.
+  # For the incoming job handler you probably don't want to limit it because you
+  # want to get things onto work queues as quickly as possible.
+  reservedConcurrency: null
+  # provisionedConcurrency represents the number of lambda functions that will always
+  # be available. For the incoming jobs handler you probably don't need to set this
+  # unless your jobs are very bursty AND very time sensitive.
+  provisionedConcurrency: null
+  # Use memory_size to adjust the reousrces (both memory and CPU) available.
+  # For the incoming jobs handler you probably don't need this to be too large,
+  # but if you're seeing large delays at this stage it might help to bump it up.
+  memorySize: 256
+  batchSize: 10
+  maximumBatchingWindow: 1
+  visibilityTimeout: 900
+  logRetentionInDays: 30
+
+JobActivator:
+  # timeout is how long the handler can possibly run. Up to 10 messages may be delivered
+  # to a handler at one time, so you'll want this to be at least 10x the maximum time you
+  # expect to spend for one message. The job activator usually will be pretty fast,
+  # but we default to a high number here to allow for the times when things go weird.
+  timeout: 300
+  # reservedConcurrency represents the maximum number of concurrent executions.
+  # For the job activator you probably don't want to limit it because you
+  # want to get things onto work queues as quickly as possible when they're ready.
+  reservedConcurrency: null
+  # provisionedConcurrency represents the number of lambda functions that will always
+  # be available. For the job activator you probably don't need to set this
+  # since it will be running on a schedule
+  provisionedConcurrency: null
+  # Use memory_size to adjust the reousrces (both memory and CPU) available.
+  # For the job activator you probably don't need this to be too large,
+  # but if you're seeing large delays at this stage it might help to bump it up.
+  memorySize: 256
+  batchSize: 10
+  maximumBatchingWindow: 1
+  visibilityTimeout: 900
+  logRetentionInDays: 30
+
+
+
+DefaultQueueHandler:
+  # timeout is how long the handler can possibly run. Up to 10 messages may be delivered
+  # to a handler at one time, so you'll want this to be at least 10x the maximum time you
+  # expect to spend for one message. The active job handler may be slow if your jobs are
+  # doing a lot of work, so we default to the maximum here.
+  timeout: 300
+  # reservedConcurrency represents the maximum number of concurrent executions.
+  # For the active job handler you may want to limit it if you have resource limitations
+  # like database connections that you need to avoid exhausting.
+  reservedConcurrency: null
+  # provisionedConcurrency represents the number of lambda functions that will always
+  # be available. For the active job handler you probably don't need to set this
+  # unless your jobs are very bursty AND very time sensitive.
+  provisionedConcurrency: null
+  # Use memory_size to adjust the reousrces (both memory and CPU) available.
+  # For the active jobs handler you'll want this to be at least as large as the memory
+  # required to actually do your jobs. You can choose an even higher number to increase
+  # the available CPU to make the jobs run faster.
+  memorySize: 256
+  batchSize: 10
+  maximumBatchingWindow: 1
+  visibilityTimeout: 900
+  logRetentionInDays: 30
+
+SingleThreadQueueHandler:
+  # timeout is how long the handler can possibly run. Up to 10 messages may be delivered
+  # to a handler at one time, so you'll want this to be at least 10x the maximum time you
+  # expect to spend for one message. The active job handler may be slow if your jobs are
+  # doing a lot of work, so we default to the maximum here.
+  timeout: 300
+  # reservedConcurrency represents the maximum number of concurrent executions.
+  # For the active job handler you may want to limit it if you have resource limitations
+  # like database connections that you need to avoid exhausting.
+  reservedConcurrency: 1
+  # provisionedConcurrency represents the number of lambda functions that will always
+  # be available. For the active job handler you probably don't need to set this
+  # unless your jobs are very bursty AND very time sensitive.
+  provisionedConcurrency: null
+  # Use memory_size to adjust the reousrces (both memory and CPU) available.
+  # For the active jobs handler you'll want this to be at least as large as the memory
+  # required to actually do your jobs. You can choose an even higher number to increase
+  # the available CPU to make the jobs run faster.
+  memorySize: 256
+  batchSize: 10
+  maximumBatchingWindow: 1
+  visibilityTimeout: 900
+  logRetentionInDays: 30
+
+
+
+# You shouldn't need to mess with these under most circumstances. But you could if you want to change
+# the name of some of your resources in AWS.
+IncomingJobQueueName: ${self:service}-${self:custom.stage}-incoming-jobs
+IncomingDeadJobQueueName: ${self:service}-${self:custom.stage}-incoming-dead
+IncomingJobHandlerName: ${self:service}-${self:custom.stage}-IncomingJobHandler
+IncomingJobQueueAccessPolicyName: ${self:service}-${self:custom.stage}-incoming-job-queue-access
+DashboardName: ${self:service}-${self:custom.stage}-dashboard
+DefaultQueueName: ${self:service}-${self:custom.stage}-default
+DefaultDeadJobQueueName: ${self:service}-${self:custom.stage}-default-dead
+DefaultQueueHandlerName: ${self:service}-${self:custom.stage}-DefaultQueueHandler
+SingleThreadQueueName: ${self:service}-${self:custom.stage}-single-thread
+SingleThreadDeadJobQueueName: ${self:service}-${self:custom.stage}-single-thread-dead
+SingleThreadQueueHandlerName: ${self:service}-${self:custom.stage}-SingleThreadQueueHandler
+JobsTableName: ${self:service}-${self:custom.stage}-jobs
+ActivityTableName: ${self:service}-${self:custom.stage}-activity
+JobActivatorName: ${self:service}-${self:custom.stage}-JobActivator