funktor 0.5.0 → 0.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +26 -11
- data/funktor-testapp/Gemfile.lock +2 -2
- data/funktor-testapp/app/services/job_flood.rb +1 -1
- data/funktor-testapp/app/workers/single_thread_audit_worker.rb +3 -0
- data/funktor-testapp/funktor_config/environment.yml +2 -2
- data/funktor-testapp/funktor_config/function_definitions/default_queue_handler.yml +3 -1
- data/funktor-testapp/funktor_config/function_definitions/incoming_job_handler.yml +3 -1
- data/funktor-testapp/funktor_config/function_definitions/job_activator.yml +1 -2
- data/funktor-testapp/funktor_config/function_definitions/low_concurrency_queue_handler.yml +13 -0
- data/funktor-testapp/funktor_config/funktor.yml +25 -25
- data/funktor-testapp/funktor_config/iam_permissions/{single_thread_queue.yml → low_concurrency_queue.yml} +1 -1
- data/funktor-testapp/funktor_config/resources/cloudwatch_dashboard.yml +22 -17
- data/funktor-testapp/funktor_config/resources/default_queue.yml +2 -2
- data/funktor-testapp/funktor_config/resources/incoming_job_queue.yml +2 -2
- data/funktor-testapp/funktor_config/resources/jobs_table.yml +16 -4
- data/funktor-testapp/funktor_config/resources/low_concurrency_queue.yml +22 -0
- data/funktor-testapp/funktor_init.yml +16 -8
- data/funktor-testapp/lambda_event_handlers/{single_thread_queue_handler.rb → low_concurrency_queue_handler.rb} +0 -0
- data/funktor-testapp/serverless.yml +4 -3
- data/funktor.gemspec +3 -1
- data/lib/funktor/activity_tracker.rb +6 -2
- data/lib/funktor/cli/templates/funktor_config/function_definitions/incoming_job_handler.yml +3 -1
- data/lib/funktor/cli/templates/funktor_config/function_definitions/job_activator.yml +1 -2
- data/lib/funktor/cli/templates/funktor_config/function_definitions/work_queue_handler.yml +3 -1
- data/lib/funktor/cli/templates/funktor_config/funktor.yml +6 -6
- data/lib/funktor/cli/templates/funktor_config/resources/cloudwatch_dashboard.yml +3 -2
- data/lib/funktor/cli/templates/funktor_config/resources/incoming_job_queue.yml +2 -2
- data/lib/funktor/cli/templates/funktor_config/resources/jobs_table.yml +16 -4
- data/lib/funktor/cli/templates/funktor_config/resources/work_queue.yml +2 -2
- data/lib/funktor/cli/templates/funktor_init.yml.tt +14 -8
- data/lib/funktor/cli/templates/serverless.yml +1 -0
- data/lib/funktor/incoming_job_handler.rb +11 -15
- data/lib/funktor/job.rb +50 -5
- data/lib/funktor/job_activator.rb +52 -26
- data/lib/funktor/shard_utils.rb +6 -0
- data/lib/funktor/testing.rb +1 -0
- data/lib/funktor/version.rb +1 -1
- data/lib/funktor/web/application.rb +139 -0
- data/lib/funktor/web/views/index.erb +3 -0
- data/lib/funktor/web/views/layout.erb +58 -0
- data/lib/funktor/web/views/processing.erb +29 -0
- data/lib/funktor/web/views/queued.erb +29 -0
- data/lib/funktor/web/views/retries.erb +35 -0
- data/lib/funktor/web/views/scheduled.erb +26 -0
- data/lib/funktor/web/views/stats.erb +9 -0
- data/lib/funktor/web/views/table_stats_with_buttons.erb +11 -0
- data/lib/funktor/web.rb +1 -0
- data/lib/funktor/work_queue_handler.rb +41 -0
- data/lib/funktor/worker/funktor_options.rb +3 -1
- data/lib/funktor/worker.rb +8 -11
- data/lib/funktor.rb +16 -16
- metadata +46 -6
- data/funktor-testapp/funktor_config/function_definitions/single_thread_queue_handler.yml +0 -11
- data/funktor-testapp/funktor_config/resources/single_thread_queue.yml +0 -22
    
        checksums.yaml
    CHANGED
    
    | @@ -1,7 +1,7 @@ | |
| 1 1 | 
             
            ---
         | 
| 2 2 | 
             
            SHA256:
         | 
| 3 | 
            -
              metadata.gz:  | 
| 4 | 
            -
              data.tar.gz:  | 
| 3 | 
            +
              metadata.gz: 9d35f7173579d1ba71a003f80641e48fb27fa8e55227401d28d24d064207f805
         | 
| 4 | 
            +
              data.tar.gz: df984a68f3753231077c9a3014690e5b22a25e31fdfa8ef4a78e6f3eb4ef3317
         | 
| 5 5 | 
             
            SHA512:
         | 
| 6 | 
            -
              metadata.gz:  | 
| 7 | 
            -
              data.tar.gz:  | 
| 6 | 
            +
              metadata.gz: 13eb0dfc1050f99eb8358d34a8666a8fd90a45e931b22fff0dc932cb9e9df3e63534f4d7d660126fa977931d46f3e572232cdf2f0bdc17468a3893c99f2959ef
         | 
| 7 | 
            +
              data.tar.gz: a0ce634530cb573099c989b64fc2fd33a85fd04480d1bffa6b3af6c3bebea01cc93fea9f2c4da5cbd62ff29180cac82f5b939aedbd30e25955b80ba7edf0ba37
         | 
    
        data/Gemfile.lock
    CHANGED
    
    | @@ -1,7 +1,7 @@ | |
| 1 1 | 
             
            PATH
         | 
| 2 2 | 
             
              remote: .
         | 
| 3 3 | 
             
              specs:
         | 
| 4 | 
            -
                funktor (0. | 
| 4 | 
            +
                funktor (0.6.3)
         | 
| 5 5 | 
             
                  activesupport
         | 
| 6 6 | 
             
                  aws-sdk-dynamodb (~> 1.62)
         | 
| 7 7 | 
             
                  aws-sdk-sqs (~> 1.37)
         | 
| @@ -10,10 +10,10 @@ PATH | |
| 10 10 | 
             
            GEM
         | 
| 11 11 | 
             
              remote: https://rubygems.org/
         | 
| 12 12 | 
             
              specs:
         | 
| 13 | 
            -
                activejob (6.1.4)
         | 
| 14 | 
            -
                  activesupport (= 6.1.4)
         | 
| 13 | 
            +
                activejob (6.1.4.1)
         | 
| 14 | 
            +
                  activesupport (= 6.1.4.1)
         | 
| 15 15 | 
             
                  globalid (>= 0.3.6)
         | 
| 16 | 
            -
                activesupport (6.1.4)
         | 
| 16 | 
            +
                activesupport (6.1.4.1)
         | 
| 17 17 | 
             
                  concurrent-ruby (~> 1.0, >= 1.0.2)
         | 
| 18 18 | 
             
                  i18n (>= 1.6, < 2)
         | 
| 19 19 | 
             
                  minitest (>= 5.1)
         | 
| @@ -22,8 +22,8 @@ GEM | |
| 22 22 | 
             
                addressable (2.8.0)
         | 
| 23 23 | 
             
                  public_suffix (>= 2.0.2, < 5.0)
         | 
| 24 24 | 
             
                aws-eventstream (1.1.1)
         | 
| 25 | 
            -
                aws-partitions (1. | 
| 26 | 
            -
                aws-sdk-core (3.119. | 
| 25 | 
            +
                aws-partitions (1.491.0)
         | 
| 26 | 
            +
                aws-sdk-core (3.119.1)
         | 
| 27 27 | 
             
                  aws-eventstream (~> 1, >= 1.0.2)
         | 
| 28 28 | 
             
                  aws-partitions (~> 1, >= 1.239.0)
         | 
| 29 29 | 
             
                  aws-sigv4 (~> 1.1)
         | 
| @@ -42,8 +42,8 @@ GEM | |
| 42 42 | 
             
                crack (0.4.5)
         | 
| 43 43 | 
             
                  rexml
         | 
| 44 44 | 
             
                diff-lcs (1.4.4)
         | 
| 45 | 
            -
                docile (1. | 
| 46 | 
            -
                globalid (0.5. | 
| 45 | 
            +
                docile (1.4.0)
         | 
| 46 | 
            +
                globalid (0.5.2)
         | 
| 47 47 | 
             
                  activesupport (>= 5.0)
         | 
| 48 48 | 
             
                hashdiff (1.0.1)
         | 
| 49 49 | 
             
                i18n (1.8.10)
         | 
| @@ -51,6 +51,8 @@ GEM | |
| 51 51 | 
             
                jmespath (1.4.0)
         | 
| 52 52 | 
             
                method_source (1.0.0)
         | 
| 53 53 | 
             
                minitest (5.14.4)
         | 
| 54 | 
            +
                mustermann (1.1.1)
         | 
| 55 | 
            +
                  ruby2_keywords (~> 0.0.1)
         | 
| 54 56 | 
             
                pry (0.13.1)
         | 
| 55 57 | 
             
                  coderay (~> 1.1)
         | 
| 56 58 | 
             
                  method_source (~> 1.0)
         | 
| @@ -58,6 +60,9 @@ GEM | |
| 58 60 | 
             
                  byebug (~> 11.0)
         | 
| 59 61 | 
             
                  pry (~> 0.13.0)
         | 
| 60 62 | 
             
                public_suffix (4.0.6)
         | 
| 63 | 
            +
                rack (2.2.3)
         | 
| 64 | 
            +
                rack-protection (2.1.0)
         | 
| 65 | 
            +
                  rack
         | 
| 61 66 | 
             
                rake (12.3.3)
         | 
| 62 67 | 
             
                rexml (3.2.5)
         | 
| 63 68 | 
             
                rspec (3.10.0)
         | 
| @@ -73,17 +78,25 @@ GEM | |
| 73 78 | 
             
                  diff-lcs (>= 1.2.0, < 2.0)
         | 
| 74 79 | 
             
                  rspec-support (~> 3.10.0)
         | 
| 75 80 | 
             
                rspec-support (3.10.2)
         | 
| 81 | 
            +
                ruby2_keywords (0.0.5)
         | 
| 76 82 | 
             
                simplecov (0.21.2)
         | 
| 77 83 | 
             
                  docile (~> 1.1)
         | 
| 78 84 | 
             
                  simplecov-html (~> 0.11)
         | 
| 79 85 | 
             
                  simplecov_json_formatter (~> 0.1)
         | 
| 80 86 | 
             
                simplecov-html (0.12.3)
         | 
| 81 | 
            -
                simplecov_json_formatter (0.1. | 
| 87 | 
            +
                simplecov_json_formatter (0.1.3)
         | 
| 88 | 
            +
                sinatra (2.1.0)
         | 
| 89 | 
            +
                  mustermann (~> 1.0)
         | 
| 90 | 
            +
                  rack (~> 2.2)
         | 
| 91 | 
            +
                  rack-protection (= 2.1.0)
         | 
| 92 | 
            +
                  tilt (~> 2.0)
         | 
| 82 93 | 
             
                thor (1.1.0)
         | 
| 94 | 
            +
                tilt (2.0.10)
         | 
| 95 | 
            +
                timecop (0.9.4)
         | 
| 83 96 | 
             
                tzinfo (2.0.4)
         | 
| 84 97 | 
             
                  concurrent-ruby (~> 1.0)
         | 
| 85 | 
            -
                webmock (3. | 
| 86 | 
            -
                  addressable (>= 2. | 
| 98 | 
            +
                webmock (3.14.0)
         | 
| 99 | 
            +
                  addressable (>= 2.8.0)
         | 
| 87 100 | 
             
                  crack (>= 0.3.2)
         | 
| 88 101 | 
             
                  hashdiff (>= 0.4.0, < 2.0.0)
         | 
| 89 102 | 
             
                zeitwerk (2.4.2)
         | 
| @@ -98,6 +111,8 @@ DEPENDENCIES | |
| 98 111 | 
             
              rake (~> 12.0)
         | 
| 99 112 | 
             
              rspec (~> 3.0)
         | 
| 100 113 | 
             
              simplecov
         | 
| 114 | 
            +
              sinatra
         | 
| 115 | 
            +
              timecop
         | 
| 101 116 | 
             
              webmock
         | 
| 102 117 |  | 
| 103 118 | 
             
            BUNDLED WITH
         | 
| @@ -1,14 +1,14 @@ | |
| 1 1 | 
             
            GEM
         | 
| 2 2 | 
             
              remote: https://rubygems.org/
         | 
| 3 3 | 
             
              specs:
         | 
| 4 | 
            -
                activesupport (6.1.4)
         | 
| 4 | 
            +
                activesupport (6.1.4.1)
         | 
| 5 5 | 
             
                  concurrent-ruby (~> 1.0, >= 1.0.2)
         | 
| 6 6 | 
             
                  i18n (>= 1.6, < 2)
         | 
| 7 7 | 
             
                  minitest (>= 5.1)
         | 
| 8 8 | 
             
                  tzinfo (~> 2.0)
         | 
| 9 9 | 
             
                  zeitwerk (~> 2.3)
         | 
| 10 10 | 
             
                aws-eventstream (1.1.1)
         | 
| 11 | 
            -
                aws-partitions (1. | 
| 11 | 
            +
                aws-partitions (1.488.0)
         | 
| 12 12 | 
             
                aws-sdk-core (3.119.0)
         | 
| 13 13 | 
             
                  aws-eventstream (~> 1, >= 1.0.2)
         | 
| 14 14 | 
             
                  aws-partitions (~> 1, >= 1.239.0)
         | 
| @@ -23,7 +23,7 @@ class JobFlood | |
| 23 23 | 
             
                    job_target_time = Time.now + (minute * 60) + rand(60)
         | 
| 24 24 | 
             
                    job_sleep = rand(0.0..max_job_length_in_seconds.to_f)
         | 
| 25 25 | 
             
                    puts job_target_time
         | 
| 26 | 
            -
                    AuditWorker.perform_at(job_target_time, {
         | 
| 26 | 
            +
                    [AuditWorker, AuditWorker, SingleThreadAuditWorker].sample.perform_at(job_target_time, {
         | 
| 27 27 | 
             
                      mode: 'later',
         | 
| 28 28 | 
             
                      message: 'msg: from random JobFlood - ' + SecureRandom.hex,
         | 
| 29 29 | 
             
                      target_time: job_target_time,
         | 
| @@ -7,8 +7,8 @@ FUNKTOR_INCOMING_JOB_QUEUE: | |
| 7 7 | 
             
              Ref: IncomingJobQueue
         | 
| 8 8 | 
             
            FUNKTOR_DEFAULT_QUEUE:
         | 
| 9 9 | 
             
              Ref: DefaultQueue
         | 
| 10 | 
            -
             | 
| 11 | 
            -
              Ref:  | 
| 10 | 
            +
            FUNKTOR_LOW_CONCURRENCY_QUEUE:
         | 
| 11 | 
            +
              Ref: LowConcurrencyQueue
         | 
| 12 12 | 
             
            FUNKTOR_ACTIVITY_TABLE:
         | 
| 13 13 | 
             
              Ref: ActivityTable
         | 
| 14 14 | 
             
            FUNKTOR_JOBS_TABLE:
         | 
| @@ -1,10 +1,12 @@ | |
| 1 1 | 
             
            handler: lambda_event_handlers/default_queue_handler.call
         | 
| 2 | 
            -
            timeout: ${self:custom.funktor.DefaultQueueHandler. | 
| 2 | 
            +
            timeout: ${self:custom.funktor.DefaultQueueHandler.functionTimeout, 900}
         | 
| 3 3 | 
             
            reservedConcurrency: ${self:custom.funktor.DefaultQueueHandler.reservedConcurrency, null}
         | 
| 4 4 | 
             
            provisionedConcurrency: ${self:custom.funktor.DefaultQueueHandler.provisionedConcurrency, null}
         | 
| 5 5 | 
             
            memorySize: ${self:custom.funktor.DefaultQueueHandler.memorySize, 256}
         | 
| 6 6 | 
             
            events:
         | 
| 7 7 | 
             
              - sqs:
         | 
| 8 | 
            +
                  batchSize: ${self:custom.funktor.DefaultQueueHandler.batchSize, 1}
         | 
| 9 | 
            +
                  maximumBatchingWindow: ${self:custom.funktor.DefaultQueueHandler.maximumBatchingWindow, 0}
         | 
| 8 10 | 
             
                  arn:
         | 
| 9 11 | 
             
                    Fn::GetAtt:
         | 
| 10 12 | 
             
                      - DefaultQueue
         | 
| @@ -1,10 +1,12 @@ | |
| 1 1 | 
             
            handler: lambda_event_handlers/incoming_job_handler.call
         | 
| 2 | 
            -
            timeout: ${self:custom.funktor.IncomingJobHandler. | 
| 2 | 
            +
            timeout: ${self:custom.funktor.IncomingJobHandler.functionTimeout, 30}
         | 
| 3 3 | 
             
            reservedConcurrency: ${self:custom.funktor.IncomingJobHandler.reservedConcurrency, null}
         | 
| 4 4 | 
             
            provisionedConcurrency: ${self:custom.funktor.IncomingJobHandler.provisionedConcurrency, null}
         | 
| 5 5 | 
             
            memorySize: ${self:custom.funktor.IncomingJobHandler.memorySize, 256}
         | 
| 6 6 | 
             
            events:
         | 
| 7 7 | 
             
              - sqs:
         | 
| 8 | 
            +
                  batchSize: ${self:custom.funktor.IncomingJobHandler.batchSize, 1}
         | 
| 9 | 
            +
                  maximumBatchingWindow: ${self:custom.funktor.IncomingJobHandler.maximumBatchingWindow, 0}
         | 
| 8 10 | 
             
                  arn:
         | 
| 9 11 | 
             
                    Fn::GetAtt:
         | 
| 10 12 | 
             
                      - IncomingJobQueue
         | 
| @@ -1,8 +1,7 @@ | |
| 1 1 | 
             
            handler: lambda_event_handlers/job_activator.call
         | 
| 2 | 
            -
            timeout: ${self:custom.funktor.JobActivator. | 
| 2 | 
            +
            timeout: ${self:custom.funktor.JobActivator.functionTimeout, 30}
         | 
| 3 3 | 
             
            reservedConcurrency: ${self:custom.funktor.JobActivator.reservedConcurrency, null}
         | 
| 4 4 | 
             
            provisionedConcurrency: ${self:custom.funktor.JobActivator.provisionedConcurrency, null}
         | 
| 5 5 | 
             
            memorySize: ${self:custom.funktor.JobActivator.memorySize, 256}
         | 
| 6 | 
            -
            #reservedConcurrency: 1
         | 
| 7 6 | 
             
            events:
         | 
| 8 7 | 
             
              - schedule: rate(1 minute)
         | 
| @@ -0,0 +1,13 @@ | |
| 1 | 
            +
            handler: lambda_event_handlers/low_concurrency_queue_handler.call
         | 
| 2 | 
            +
            timeout: ${self:custom.funktor.LowConcurrencyQueueHandler.functionTimeout, 900}
         | 
| 3 | 
            +
            reservedConcurrency: ${self:custom.funktor.LowConcurrencyQueueHandler.reservedConcurrency, null}
         | 
| 4 | 
            +
            provisionedConcurrency: ${self:custom.funktor.LowConcurrencyQueueHandler.provisionedConcurrency, null}
         | 
| 5 | 
            +
            memorySize: ${self:custom.funktor.LowConcurrencyQueueHandler.memorySize, 256}
         | 
| 6 | 
            +
            events:
         | 
| 7 | 
            +
              - sqs:
         | 
| 8 | 
            +
                  batchSize: ${self:custom.funktor.LowConcurrencyQueueHandler.batchSize, 1}
         | 
| 9 | 
            +
                  maximumBatchingWindow: ${self:custom.funktor.LowConcurrencyQueueHandler.maximumBatchingWindow, 0}
         | 
| 10 | 
            +
                  arn:
         | 
| 11 | 
            +
                    Fn::GetAtt:
         | 
| 12 | 
            +
                      - LowConcurrencyQueue
         | 
| 13 | 
            +
                      - Arn
         | 
| @@ -3,7 +3,7 @@ IncomingJobHandler: | |
| 3 3 | 
             
              # to a handler at one time, so you'll want this to be at least 10x the maximum time you
         | 
| 4 4 | 
             
              # expect to spend for one message. The incoming job handler usually will be pretty fast,
         | 
| 5 5 | 
             
              # but we default to a high number here to allow for the times when things go weird.
         | 
| 6 | 
            -
               | 
| 6 | 
            +
              functionTimeout: 300
         | 
| 7 7 | 
             
              # reservedConcurrency represents the maximum number of concurrent executions.
         | 
| 8 8 | 
             
              # For the incoming job handler you probably don't want to limit it because you
         | 
| 9 9 | 
             
              # want to get things onto work queues as quickly as possible.
         | 
| @@ -16,17 +16,17 @@ IncomingJobHandler: | |
| 16 16 | 
             
              # For the incoming jobs handler you probably don't need this to be too large,
         | 
| 17 17 | 
             
              # but if you're seeing large delays at this stage it might help to bump it up.
         | 
| 18 18 | 
             
              memorySize: 256
         | 
| 19 | 
            -
              batchSize:  | 
| 20 | 
            -
              maximumBatchingWindow:  | 
| 21 | 
            -
              visibilityTimeout:  | 
| 22 | 
            -
               | 
| 19 | 
            +
              batchSize: 1
         | 
| 20 | 
            +
              maximumBatchingWindow: 0
         | 
| 21 | 
            +
              visibilityTimeout: 1800
         | 
| 22 | 
            +
              maxReceiveCount: 20
         | 
| 23 23 |  | 
| 24 24 | 
             
            JobActivator:
         | 
| 25 25 | 
             
              # timeout is how long the handler can possibly run. Up to 10 messages may be delivered
         | 
| 26 26 | 
             
              # to a handler at one time, so you'll want this to be at least 10x the maximum time you
         | 
| 27 27 | 
             
              # expect to spend for one message. The job activator usually will be pretty fast,
         | 
| 28 28 | 
             
              # but we default to a high number here to allow for the times when things go weird.
         | 
| 29 | 
            -
               | 
| 29 | 
            +
              functionTimeout: 300
         | 
| 30 30 | 
             
              # reservedConcurrency represents the maximum number of concurrent executions.
         | 
| 31 31 | 
             
              # For the job activator you probably don't want to limit it because you
         | 
| 32 32 | 
             
              # want to get things onto work queues as quickly as possible when they're ready.
         | 
| @@ -39,10 +39,10 @@ JobActivator: | |
| 39 39 | 
             
              # For the job activator you probably don't need this to be too large,
         | 
| 40 40 | 
             
              # but if you're seeing large delays at this stage it might help to bump it up.
         | 
| 41 41 | 
             
              memorySize: 256
         | 
| 42 | 
            -
              batchSize:  | 
| 43 | 
            -
              maximumBatchingWindow:  | 
| 44 | 
            -
              visibilityTimeout:  | 
| 45 | 
            -
               | 
| 42 | 
            +
              batchSize: 1
         | 
| 43 | 
            +
              maximumBatchingWindow: 0
         | 
| 44 | 
            +
              visibilityTimeout: 1800
         | 
| 45 | 
            +
              maxReceiveCount: 20
         | 
| 46 46 |  | 
| 47 47 |  | 
| 48 48 |  | 
| @@ -51,7 +51,7 @@ DefaultQueueHandler: | |
| 51 51 | 
             
              # to a handler at one time, so you'll want this to be at least 10x the maximum time you
         | 
| 52 52 | 
             
              # expect to spend for one message. The active job handler may be slow if your jobs are
         | 
| 53 53 | 
             
              # doing a lot of work, so we default to the maximum here.
         | 
| 54 | 
            -
               | 
| 54 | 
            +
              functionTimeout: 300
         | 
| 55 55 | 
             
              # reservedConcurrency represents the maximum number of concurrent executions.
         | 
| 56 56 | 
             
              # For the active job handler you may want to limit it if you have resource limitations
         | 
| 57 57 | 
             
              # like database connections that you need to avoid exhausting.
         | 
| @@ -65,21 +65,21 @@ DefaultQueueHandler: | |
| 65 65 | 
             
              # required to actually do your jobs. You can choose an even higher number to increase
         | 
| 66 66 | 
             
              # the available CPU to make the jobs run faster.
         | 
| 67 67 | 
             
              memorySize: 256
         | 
| 68 | 
            -
              batchSize:  | 
| 69 | 
            -
              maximumBatchingWindow:  | 
| 70 | 
            -
              visibilityTimeout:  | 
| 71 | 
            -
               | 
| 68 | 
            +
              batchSize: 1
         | 
| 69 | 
            +
              maximumBatchingWindow: 0
         | 
| 70 | 
            +
              visibilityTimeout: 1800
         | 
| 71 | 
            +
              maxReceiveCount: 20
         | 
| 72 72 |  | 
| 73 | 
            -
             | 
| 73 | 
            +
            LowConcurrencyQueueHandler:
         | 
| 74 74 | 
             
              # timeout is how long the handler can possibly run. Up to 10 messages may be delivered
         | 
| 75 75 | 
             
              # to a handler at one time, so you'll want this to be at least 10x the maximum time you
         | 
| 76 76 | 
             
              # expect to spend for one message. The active job handler may be slow if your jobs are
         | 
| 77 77 | 
             
              # doing a lot of work, so we default to the maximum here.
         | 
| 78 | 
            -
               | 
| 78 | 
            +
              functionTimeout: 300
         | 
| 79 79 | 
             
              # reservedConcurrency represents the maximum number of concurrent executions.
         | 
| 80 80 | 
             
              # For the active job handler you may want to limit it if you have resource limitations
         | 
| 81 81 | 
             
              # like database connections that you need to avoid exhausting.
         | 
| 82 | 
            -
              reservedConcurrency:  | 
| 82 | 
            +
              reservedConcurrency: 10
         | 
| 83 83 | 
             
              # provisionedConcurrency represents the number of lambda functions that will always
         | 
| 84 84 | 
             
              # be available. For the active job handler you probably don't need to set this
         | 
| 85 85 | 
             
              # unless your jobs are very bursty AND very time sensitive.
         | 
| @@ -89,10 +89,10 @@ SingleThreadQueueHandler: | |
| 89 89 | 
             
              # required to actually do your jobs. You can choose an even higher number to increase
         | 
| 90 90 | 
             
              # the available CPU to make the jobs run faster.
         | 
| 91 91 | 
             
              memorySize: 256
         | 
| 92 | 
            -
              batchSize:  | 
| 93 | 
            -
              maximumBatchingWindow:  | 
| 94 | 
            -
              visibilityTimeout:  | 
| 95 | 
            -
               | 
| 92 | 
            +
              batchSize: 1
         | 
| 93 | 
            +
              maximumBatchingWindow: 0
         | 
| 94 | 
            +
              visibilityTimeout: 1800
         | 
| 95 | 
            +
              maxReceiveCount: 20
         | 
| 96 96 |  | 
| 97 97 |  | 
| 98 98 |  | 
| @@ -106,9 +106,9 @@ DashboardName: ${self:service}-${self:custom.stage}-dashboard | |
| 106 106 | 
             
            DefaultQueueName: ${self:service}-${self:custom.stage}-default
         | 
| 107 107 | 
             
            DefaultDeadJobQueueName: ${self:service}-${self:custom.stage}-default-dead
         | 
| 108 108 | 
             
            DefaultQueueHandlerName: ${self:service}-${self:custom.stage}-DefaultQueueHandler
         | 
| 109 | 
            -
             | 
| 110 | 
            -
             | 
| 111 | 
            -
             | 
| 109 | 
            +
            LowConcurrencyQueueName: ${self:service}-${self:custom.stage}-low-concurrency
         | 
| 110 | 
            +
            LowConcurrencyDeadJobQueueName: ${self:service}-${self:custom.stage}-low-concurrency-dead
         | 
| 111 | 
            +
            LowConcurrencyQueueHandlerName: ${self:service}-${self:custom.stage}-LowConcurrencyQueueHandler
         | 
| 112 112 | 
             
            JobsTableName: ${self:service}-${self:custom.stage}-jobs
         | 
| 113 113 | 
             
            ActivityTableName: ${self:service}-${self:custom.stage}-activity
         | 
| 114 114 | 
             
            JobActivatorName: ${self:service}-${self:custom.stage}-JobActivator
         | 
| @@ -34,6 +34,8 @@ Resources: | |
| 34 34 | 
             
                                        [ "funktor-testapp", "Duration", "WorkerClassName", "GreetingsWorker" ],
         | 
| 35 35 | 
             
                                        [ "...", { "stat": "p99" } ],
         | 
| 36 36 | 
             
                                        [ "funktor-testapp", "Duration", "WorkerClassName", "HelloWorker" ],
         | 
| 37 | 
            +
                                        [ "...", { "stat": "p99" } ],
         | 
| 38 | 
            +
                                        [ "funktor-testapp", "Duration", "WorkerClassName", "SingleThreadAuditWorker" ],
         | 
| 37 39 | 
             
                                        [ "...", { "stat": "p99" } ]
         | 
| 38 40 | 
             
                                    ],
         | 
| 39 41 | 
             
                                    "view": "timeSeries",
         | 
| @@ -58,6 +60,8 @@ Resources: | |
| 58 60 | 
             
                                        [ "funktor-testapp", "processed", "WorkerClassName", "GreetingsWorker" ],
         | 
| 59 61 | 
             
                                        [ ".", "failed", ".", "." ],
         | 
| 60 62 | 
             
                                        [ "funktor-testapp", "processed", "WorkerClassName", "HelloWorker" ],
         | 
| 63 | 
            +
                                        [ ".", "failed", ".", "." ],
         | 
| 64 | 
            +
                                        [ "funktor-testapp", "processed", "WorkerClassName", "SingleThreadAuditWorker" ],
         | 
| 61 65 | 
             
                                        [ ".", "failed", ".", "." ]
         | 
| 62 66 | 
             
                                    ],
         | 
| 63 67 | 
             
                                    "view": "timeSeries",
         | 
| @@ -79,7 +83,7 @@ Resources: | |
| 79 83 | 
             
                                    "metrics": [
         | 
| 80 84 | 
             
                                        [ "funktor-testapp", "Duration", "Queue", "default" ],
         | 
| 81 85 | 
             
                                        [ "...", { "stat": "p99" } ],
         | 
| 82 | 
            -
                                        [ "funktor-testapp", "Duration", "Queue", " | 
| 86 | 
            +
                                        [ "funktor-testapp", "Duration", "Queue", "low_concurrency" ],
         | 
| 83 87 | 
             
                                        [ "...", { "stat": "p99" } ]
         | 
| 84 88 | 
             
                                    ],
         | 
| 85 89 | 
             
                                    "view": "timeSeries",
         | 
| @@ -100,7 +104,7 @@ Resources: | |
| 100 104 | 
             
                                    "metrics": [
         | 
| 101 105 | 
             
                                        [ "funktor-testapp", "processed", "Queue", "default" ],
         | 
| 102 106 | 
             
                                        [ ".", "failed", ".", "." ],
         | 
| 103 | 
            -
                                        [ "funktor-testapp", "processed", "Queue", " | 
| 107 | 
            +
                                        [ "funktor-testapp", "processed", "Queue", "low_concurrency" ],
         | 
| 104 108 | 
             
                                        [ ".", "failed", ".", "." ]
         | 
| 105 109 | 
             
                                    ],
         | 
| 106 110 | 
             
                                    "view": "timeSeries",
         | 
| @@ -426,7 +430,7 @@ Resources: | |
| 426 430 | 
             
                                "x": 0,
         | 
| 427 431 | 
             
                                "type": "text",
         | 
| 428 432 | 
             
                                "properties": {
         | 
| 429 | 
            -
                                    "markdown": "\n#  | 
| 433 | 
            +
                                    "markdown": "\n# LowConcurrency Queue\n"
         | 
| 430 434 | 
             
                                }
         | 
| 431 435 | 
             
                            },
         | 
| 432 436 | 
             
                            { 
         | 
| @@ -438,7 +442,7 @@ Resources: | |
| 438 442 | 
             
                                "properties": {
         | 
| 439 443 | 
             
                                    "metrics": [
         | 
| 440 444 | 
             
                                        [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName",
         | 
| 441 | 
            -
                                          "${self:custom.funktor. | 
| 445 | 
            +
                                          "${self:custom.funktor.LowConcurrencyQueueName}", { "label": "Messages Per Minute" } ]
         | 
| 442 446 | 
             
                                    ],
         | 
| 443 447 | 
             
                                    "view": "singleValue",
         | 
| 444 448 | 
             
                                    "region": "us-east-1",
         | 
| @@ -456,8 +460,8 @@ Resources: | |
| 456 460 | 
             
                                "properties": {
         | 
| 457 461 | 
             
                                    "metrics": [
         | 
| 458 462 | 
             
                                        [ "AWS/Lambda", "Duration", "FunctionName",
         | 
| 459 | 
            -
                                          "${self:custom.funktor. | 
| 460 | 
            -
                                          "${self:custom.funktor. | 
| 463 | 
            +
                                          "${self:custom.funktor.LowConcurrencyQueueHandlerName}", "Resource",
         | 
| 464 | 
            +
                                          "${self:custom.funktor.LowConcurrencyQueueHandlerName}", { "label": "p10" } ],
         | 
| 461 465 | 
             
                                        [ "...", { "label": "p50", "stat": "p50" } ],
         | 
| 462 466 | 
             
                                        [ "...", { "label": "p99", "stat": "p99" } ],
         | 
| 463 467 | 
             
                                        [ "...", { "label": "Average", "stat": "Average" } ]
         | 
| @@ -478,14 +482,14 @@ Resources: | |
| 478 482 | 
             
                                "properties": {
         | 
| 479 483 | 
             
                                    "metrics": [
         | 
| 480 484 | 
             
                                        [ "AWS/SQS", "NumberOfMessagesReceived", "QueueName",
         | 
| 481 | 
            -
                                          "${self:custom.funktor. | 
| 485 | 
            +
                                          "${self:custom.funktor.LowConcurrencyQueueName}",
         | 
| 482 486 | 
             
                                          { "label": "Received" } ],
         | 
| 483 487 | 
             
                                        [ ".", "NumberOfMessagesDeleted", ".", ".", { "label": "Handled" } ],
         | 
| 484 488 | 
             
                                        [ "AWS/Lambda", "Invocations", "FunctionName",
         | 
| 485 | 
            -
                                          "${self:custom.funktor. | 
| 486 | 
            -
                                          "${self:custom.funktor. | 
| 489 | 
            +
                                          "${self:custom.funktor.LowConcurrencyQueueHandlerName}", "Resource",
         | 
| 490 | 
            +
                                          "${self:custom.funktor.LowConcurrencyQueueHandlerName}", { "label": "Handler Invocations" } ],
         | 
| 487 491 | 
             
                                        [ "AWS/SQS", "ApproximateNumberOfMessagesVisible", "QueueName",
         | 
| 488 | 
            -
                                          "${self:custom.funktor. | 
| 492 | 
            +
                                          "${self:custom.funktor.LowConcurrencyQueueName}", { "label": "Pending?" } ],
         | 
| 489 493 | 
             
                                        [ ".", "ApproximateNumberOfMessagesNotVisible", ".", ".", { "label": "Backlog?" } ],
         | 
| 490 494 | 
             
                                        [ ".", "NumberOfMessagesSent", ".", ".", { "label": "Sent" } ],
         | 
| 491 495 | 
             
                                        [ ".", "ApproximateNumberOfMessagesDelayed", ".", ".", { "label": "Delayed" } ]
         | 
| @@ -493,7 +497,7 @@ Resources: | |
| 493 497 | 
             
                                    "view": "timeSeries",
         | 
| 494 498 | 
             
                                    "stacked": false,
         | 
| 495 499 | 
             
                                    "region": "us-east-1",
         | 
| 496 | 
            -
                                    "title": " | 
| 500 | 
            +
                                    "title": "LowConcurrency Queue (Async jobs go here immediately, scheduled jobs land here in the minute before they're scheduled)",
         | 
| 497 501 | 
             
                                    "period": 60,
         | 
| 498 502 | 
             
                                    "stat": "Sum",
         | 
| 499 503 | 
             
                                    "liveData": true
         | 
| @@ -508,12 +512,12 @@ Resources: | |
| 508 512 | 
             
                                "properties": {
         | 
| 509 513 | 
             
                                    "period": 60,
         | 
| 510 514 | 
             
                                    "metrics": [
         | 
| 511 | 
            -
                                        [ "AWS/Lambda", "Duration", "FunctionName", "${self:custom.funktor. | 
| 515 | 
            +
                                        [ "AWS/Lambda", "Duration", "FunctionName", "${self:custom.funktor.LowConcurrencyQueueHandlerName}", { "stat": "Minimum" } ],
         | 
| 512 516 | 
             
                                        [ "...", { "stat": "Average" } ],
         | 
| 513 517 | 
             
                                        [ "...", { "stat": "Maximum" } ]
         | 
| 514 518 | 
             
                                    ],
         | 
| 515 519 | 
             
                                    "region": "us-east-1",
         | 
| 516 | 
            -
                                    "title": " | 
| 520 | 
            +
                                    "title": "LowConcurrency Queue Handler Duration in Milliseconds",
         | 
| 517 521 | 
             
                                    "view": "timeSeries",
         | 
| 518 522 | 
             
                                    "stacked": false,
         | 
| 519 523 | 
             
                                    "liveData": true
         | 
| @@ -529,12 +533,12 @@ Resources: | |
| 529 533 | 
             
                                    "period": 60,
         | 
| 530 534 | 
             
                                    "metrics": [
         | 
| 531 535 | 
             
                                        [ "AWS/Lambda", "Errors", "FunctionName",
         | 
| 532 | 
            -
                                          "${self:custom.funktor. | 
| 536 | 
            +
                                          "${self:custom.funktor.LowConcurrencyQueueHandlerName}", { "id": "errors", "stat": "Sum", "color": "#d13212" } ],
         | 
| 533 537 | 
             
                                        [ ".", "Invocations", ".", ".", { "id": "invocations", "stat": "Sum", "visible": false } ],
         | 
| 534 538 | 
             
                                        [ { "expression": "100 - 100 * errors / MAX([errors, invocations])", "label": "Success rate (%)", "id": "availability", "yAxis": "right", "region": "us-east-1" } ]
         | 
| 535 539 | 
             
                                    ],
         | 
| 536 540 | 
             
                                    "region": "us-east-1",
         | 
| 537 | 
            -
                                    "title": " | 
| 541 | 
            +
                                    "title": "LowConcurrency Queue Handler Error count and success rate (%)",
         | 
| 538 542 | 
             
                                    "yAxis": {
         | 
| 539 543 | 
             
                                        "right": {
         | 
| 540 544 | 
             
                                            "max": 100
         | 
| @@ -555,10 +559,10 @@ Resources: | |
| 555 559 | 
             
                                    "period": 60,
         | 
| 556 560 | 
             
                                    "metrics": [
         | 
| 557 561 | 
             
                                        [ "AWS/Lambda", "ConcurrentExecutions", "FunctionName",
         | 
| 558 | 
            -
                                          "${self:custom.funktor. | 
| 562 | 
            +
                                          "${self:custom.funktor.LowConcurrencyQueueHandlerName}", { "stat": "Maximum" } ]
         | 
| 559 563 | 
             
                                    ],
         | 
| 560 564 | 
             
                                    "region": "us-east-1",
         | 
| 561 | 
            -
                                    "title": " | 
| 565 | 
            +
                                    "title": "LowConcurrency Queue Handler Concurrent executions",
         | 
| 562 566 | 
             
                                    "view": "timeSeries",
         | 
| 563 567 | 
             
                                    "stacked": false,
         | 
| 564 568 | 
             
                                    "liveData": true
         | 
| @@ -765,6 +769,7 @@ Resources: | |
| 765 769 | 
             
                                        [ "...", "Query" ],
         | 
| 766 770 | 
             
                                        [ ".", "ThrottledRequests", ".", ".", ".", "PutItem", { "yAxis": "right", "visible": false } ],
         | 
| 767 771 | 
             
                                        [ ".", "SuccessfulRequestLatency", ".", ".", ".", "DeleteItem" ],
         | 
| 772 | 
            +
                                        [ ".", "SuccessfulRequestLatency", ".", ".", ".", "UpdateItem" ],
         | 
| 768 773 | 
             
                                        [ ".", "ThrottledRequests", ".", ".", ".", ".", { "yAxis": "right", "visible": false } ]
         | 
| 769 774 | 
             
                                    ],
         | 
| 770 775 | 
             
                                    "view": "timeSeries",
         |