funktor 0.4.7 → 0.5.0

data/funktor-testapp/funktor_config/resources/default_queue.yml

@@ -0,0 +1,22 @@
+Resources:
+  DefaultQueue:
+    Type: AWS::SQS::Queue
+    Properties:
+      QueueName: ${self:custom.funktor.DefaultQueueName}
+      VisibilityTimeout: 300
+      RedrivePolicy:
+        deadLetterTargetArn:
+          "Fn::GetAtt": [ DefaultDeadLetterQueue, Arn ]
+        maxReceiveCount: 5
+  DefaultDeadLetterQueue:
+    Type: AWS::SQS::Queue
+    Properties:
+      QueueName: ${self:custom.funktor.DefaultDeadJobQueueName}
+
+Outputs:
+  DefaultQueueUrl:
+    Value:
+      Ref: DefaultQueue
+  DefaultDeadLetterQueueUrl:
+    Value:
+      Ref: DefaultDeadLetterQueue

data/funktor-testapp/funktor_config/resources/incoming_job_queue.yml

@@ -0,0 +1,22 @@
+Resources:
+  IncomingJobQueue:
+    Type: AWS::SQS::Queue
+    Properties:
+      QueueName: ${self:custom.funktor.IncomingJobQueueName}
+      VisibilityTimeout: 300
+      RedrivePolicy:
+        deadLetterTargetArn:
+          "Fn::GetAtt": [ IncomingJobDeadLetterQueue, Arn ]
+        maxReceiveCount: 5
+  IncomingJobDeadLetterQueue:
+    Type: AWS::SQS::Queue
+    Properties:
+      QueueName: ${self:custom.funktor.IncomingDeadJobQueueName}
+
+Outputs:
+  IncomingJobQueueUrl:
+    Value:
+      Ref: IncomingJobQueue
+  IncomingJobDeadLetterQueueUrl:
+    Value:
+      Ref: IncomingJobDeadLetterQueue

data/funktor-testapp/funktor_config/resources/incoming_job_queue_user.yml

@@ -0,0 +1,26 @@
+Resources:
+  IncomingJobQueueUser:
+    Type: AWS::IAM::User
+    Properties:
+      Policies:
+        - PolicyName: incoming-job-queue-access
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+            - Effect: Allow
+              Action:
+              - sqs:*
+              Resource:
+                - "Fn::GetAtt": [ IncomingJobQueue, Arn ]
+
+  IncomingJobQueueUserAccessKey:
+    Type: AWS::IAM::AccessKey
+    Properties:
+      UserName: !Ref IncomingJobQueueUser
+
+
+Outputs:
+  AccessKeyID:
+    Value: !Ref IncomingJobQueueUserAccessKey
+  SecretAccessKey:
+    Value: !GetAtt IncomingJobQueueUserAccessKey.SecretAccessKey

data/funktor-testapp/funktor_config/resources/jobs_table.yml

@@ -0,0 +1,44 @@
+Resources:
+  JobsTable:
+    Type: AWS::DynamoDB::Table
+    Properties:
+      TableName: ${self:custom.funktor.JobsTableName}
+      AttributeDefinitions:
+        - AttributeName: jobShard
+          AttributeType: N
+        - AttributeName: jobId
+          AttributeType: S
+        #- AttributeName: category
+          #AttributeType: S
+        - AttributeName: performAt
+          AttributeType: S
+        - AttributeName: dummy
+          AttributeType: S
+      KeySchema:
+        - AttributeName: jobShard
+          KeyType: HASH
+        - AttributeName: jobId
+          KeyType: RANGE
+      GlobalSecondaryIndexes:
+        - IndexName: performAtIndex
+          KeySchema:
+          - AttributeName: dummy
+            KeyType: HASH
+          - AttributeName: performAt
+            KeyType: RANGE
+          Projection:
+            NonKeyAttributes:
+            - jobId
+            - payload
+            - category
+            ProjectionType: INCLUDE
+      BillingMode: PAY_PER_REQUEST
+      # Set the capacity based on the stage
+      #ProvisionedThroughput:
+        #ReadCapacityUnits: ${self:custom.tableThroughput}
+        #WriteCapacityUnits: ${self:custom.tableThroughput}
+
+Outputs:
+  JobsTable:
+    Value:
+      Ref: JobsTable

data/funktor-testapp/funktor_config/resources/single_thread_queue.yml

@@ -0,0 +1,22 @@
+Resources:
+  SingleThreadQueue:
+    Type: AWS::SQS::Queue
+    Properties:
+      QueueName: ${self:custom.funktor.SingleThreadQueueName}
+      VisibilityTimeout: 300
+      RedrivePolicy:
+        deadLetterTargetArn:
+          "Fn::GetAtt": [ SingleThreadDeadLetterQueue, Arn ]
+        maxReceiveCount: 5
+  SingleThreadDeadLetterQueue:
+    Type: AWS::SQS::Queue
+    Properties:
+      QueueName: ${self:custom.funktor.SingleThreadDeadJobQueueName}
+
+Outputs:
+  SingleThreadQueueUrl:
+    Value:
+      Ref: SingleThreadQueue
+  SingleThreadDeadLetterQueueUrl:
+    Value:
+      Ref: SingleThreadDeadLetterQueue

data/funktor-testapp/funktor_config/ruby_layer.yml

@@ -0,0 +1,11 @@
+use_docker: false
+#docker_yums:
+  #- postgresql-devel
+#native_libs:
+  #- /usr/lib64/libpq.so.5
+  #- /usr/lib64/libldap_r-2.4.so.2
+  #- /usr/lib64/liblber-2.4.so.2
+  #- /usr/lib64/libsasl2.so.3
+  #- /usr/lib64/libssl3.so
+  #- /usr/lib64/libsmime3.so
+  #- /usr/lib64/libnss3.so

data/funktor-testapp/funktor_init.yml

@@ -0,0 +1,61 @@
+# ℹ️  WARNING: This file doesn't control anything directly. It's used to allow you to set/maintain config options in a central place.
+# When you make changes to this file you should run `funktor init` again to propagate the changes to the 'active' configuration files.
+appName: funktor-testapp
+runtime: ruby2.7
+
+# These values will be applied to any funktor handlers that don't specify custom values.
+# TODO - These still need to be wired up to do anything.
+handlerDefaults:
+  # timeout is how long the handler can possibly run. Up to 10 messages may be delivered
+  # to a handler at one time, so you'll want this to be at least 10x the maximum time you
+  # expect to spend for one message. We default to a high number here to allow for the
+  # times when things go weird.
+  timeout: 300
+
+  # reservedConcurrency represents the maximum number of concurrent executions.
+  # Usually you'll want to leave this as null so that handlers can scale infinitely
+  # (within your account limits). If you need to restrict concurrency to prevent
+  # resource exhaustion you should do it at the queue level.
+  reservedConcurrency: null
+
+  # provisionedConcurrency represents the number of lambda functions that will always
+  # be available. You probably want to set this on individual queues if you're going
+  # to use it.
+  provisionedConcurrency: null
+
+  # Use memory_size to adjust the reousrces (both memory and CPU) available.
+  # We default to 256 as a reasonable trade off on saving money but getting
+  # reasonable performance.
+  memorySize: 256
+
+  # You can set the batch size. Max of 10_000 for normal queues, 10 for FIFO.
+  batchSize: 10
+
+  # How many seconds should AWS wait for a batch to fill up before executing lambda?
+  # For immediate execution set the batch size to 1.
+  maximumBatchingWindow : 1
+
+  # Visibility timeout should only come into play in the case of Funktor errors.
+  # Application level errors should be handled by Funktor retry mechanisms.
+  # The visibility timeout should be at least as long as the function timeout, and up to 6 times larger.
+  visibilityTimeout: 900
+
+  # Set log rentention to save money
+  logRetentionInDays: 30
+
+# Incoming Job Handler
+incomingJobHandler:
+  # If your jobs are bursty AND time-sensitive you might want to have some lambdas pre-provisioned
+  # to quickly handle jobs at the beginning of a burst. Uncomment the line below if so.
+  # provisionedConcurrency: 4
+
+
+queues:
+  - default:
+      # Set queue specific config options here
+      # memorySize: 512
+      # TODO - Is it advisable to use FIFO queuues with Funktor? Maybe this isn't really even supported by CloudFormation?
+      # fifo: false
+  - singleThread:
+      reservedConcurrency: 1
+

data/funktor-testapp/lambda_event_handlers/default_queue_handler.rb

@@ -0,0 +1,8 @@
+require_relative '../funktor_config/boot'
+
+$handler = Funktor::WorkQueueHandler.new
+
+def call(event:, context:)
+  $handler.call(event: event, context: context)
+end
+

data/funktor-testapp/lambda_event_handlers/incoming_job_handler.rb

@@ -0,0 +1,8 @@
+require_relative '../funktor_config/boot'
+
+$handler = Funktor::IncomingJobHandler.new
+
+def call(event:, context:)
+  $handler.call(event: event, context: context)
+end
+

data/funktor-testapp/lambda_event_handlers/job_activator.rb

@@ -0,0 +1,8 @@
+require_relative '../funktor_config/boot'
+
+$handler = Funktor::JobActivator.new
+
+def call(event:, context:)
+  $handler.call(event: event, context: context)
+end
+

data/funktor-testapp/lambda_event_handlers/random_job_generator.rb

@@ -0,0 +1,35 @@
+require_relative '../funktor_config/boot'
+
+
+# This class runs on a schedule (every minute, by default) and generates a series of jobs to be
+# run over the next few minutes.
+class RandomJobGenerator
+
+  def self.call(event:, context:)
+    job_count = 0
+    jobs_to_generate = rand(ENV['MIN_RANDOM_JOBS_PER_MINUTE'].to_i..ENV['MAX_RANDOM_JOBS_PER_MINUTE'].to_i)
+    sleep_delay = 53.0/jobs_to_generate # Calculate our sleep based on the target rates
+    puts "jobs_to_generate = #{jobs_to_generate} and sleep_delay = #{sleep_delay}"
+    while context.get_remaining_time_in_millis > 5_000 do # This lets us exit gracefully and resume on the next round instead of getting forcibly killed.
+      delay = rand(0..ENV['MAX_JOB_DELAY_IN_SECONDS'].to_f) # Schedule the job for sometime between now and MAX_JOB_DELAY
+      job_sleep = rand(0..ENV['MAX_JOB_LENGTH_IN_SECONDS'].to_f) # Then sleep for up to MAX_SLEEP_DELAY
+      error_percentage = ENV['ERROR_PERCENTAGE'].to_i
+
+      # Schedule a job to be performed after our randomly generated delay
+      [HelloWorker, GreetingsWorker].sample.perform_in(delay, {
+        mode: 'later',
+        message: 'msg: from random job generator - ' + SecureRandom.hex,
+        delay: delay,
+        error_percentage: error_percentage,
+        job_sleep: job_sleep
+      })
+      job_count += 1
+      puts "message delay = #{delay} and sleep delay = #{sleep_delay} - job_count = #{job_count}"
+
+      # Now sleep before generating the next job
+      sleep(sleep_delay)
+    end
+  end
+
+end
+

data/funktor-testapp/lambda_event_handlers/single_thread_queue_handler.rb

@@ -0,0 +1,8 @@
+require_relative '../funktor_config/boot'
+
+$handler = Funktor::WorkQueueHandler.new
+
+def call(event:, context:)
+  $handler.call(event: event, context: context)
+end
+

data/funktor-testapp/package-lock.json

@@ -0,0 +1,248 @@
+{
+  "name": "funktor-testapp",
+  "version": "0.1.0",
+  "lockfileVersion": 1,
+  "requires": true,
+  "dependencies": {
+    "balanced-match": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
+      "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
+      "dev": true
+    },
+    "bluebird": {
+      "version": "3.7.2",
+      "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz",
+      "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==",
+      "dev": true
+    },
+    "brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+      "dev": true,
+      "requires": {
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
+      }
+    },
+    "concat-map": {
+      "version": "0.0.1",
+      "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
+      "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
+      "dev": true
+    },
+    "core-util-is": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
+      "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=",
+      "dev": true
+    },
+    "fs-extra": {
+      "version": "8.1.0",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz",
+      "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==",
+      "dev": true,
+      "requires": {
+        "graceful-fs": "^4.2.0",
+        "jsonfile": "^4.0.0",
+        "universalify": "^0.1.0"
+      }
+    },
+    "fs.realpath": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
+      "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=",
+      "dev": true
+    },
+    "glob": {
+      "version": "7.1.7",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz",
+      "integrity": "sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==",
+      "dev": true,
+      "requires": {
+        "fs.realpath": "^1.0.0",
+        "inflight": "^1.0.4",
+        "inherits": "2",
+        "minimatch": "^3.0.4",
+        "once": "^1.3.0",
+        "path-is-absolute": "^1.0.0"
+      }
+    },
+    "graceful-fs": {
+      "version": "4.2.6",
+      "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.6.tgz",
+      "integrity": "sha512-nTnJ528pbqxYanhpDYsi4Rd8MAeaBA67+RZ10CM1m3bTAVFEDcd5AuA4a6W5YkGZ1iNXHzZz8T6TBKLeBuNriQ==",
+      "dev": true
+    },
+    "immediate": {
+      "version": "3.0.6",
+      "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz",
+      "integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=",
+      "dev": true
+    },
+    "inflight": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
+      "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
+      "dev": true,
+      "requires": {
+        "once": "^1.3.0",
+        "wrappy": "1"
+      }
+    },
+    "inherits": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
+      "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
+      "dev": true
+    },
+    "isarray": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
+      "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=",
+      "dev": true
+    },
+    "jsonfile": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz",
+      "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=",
+      "dev": true,
+      "requires": {
+        "graceful-fs": "^4.1.6"
+      }
+    },
+    "jszip": {
+      "version": "3.7.0",
+      "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.7.0.tgz",
+      "integrity": "sha512-Y2OlFIzrDOPWUnpU0LORIcDn2xN7rC9yKffFM/7pGhQuhO+SUhfm2trkJ/S5amjFvem0Y+1EALz/MEPkvHXVNw==",
+      "dev": true,
+      "requires": {
+        "lie": "~3.3.0",
+        "pako": "~1.0.2",
+        "readable-stream": "~2.3.6",
+        "set-immediate-shim": "~1.0.1"
+      }
+    },
+    "lie": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz",
+      "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==",
+      "dev": true,
+      "requires": {
+        "immediate": "~3.0.5"
+      }
+    },
+    "minimatch": {
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
+      "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
+      "dev": true,
+      "requires": {
+        "brace-expansion": "^1.1.7"
+      }
+    },
+    "once": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
+      "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
+      "dev": true,
+      "requires": {
+        "wrappy": "1"
+      }
+    },
+    "pako": {
+      "version": "1.0.11",
+      "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz",
+      "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==",
+      "dev": true
+    },
+    "path-is-absolute": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
+      "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=",
+      "dev": true
+    },
+    "process-nextick-args": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
+      "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==",
+      "dev": true
+    },
+    "readable-stream": {
+      "version": "2.3.7",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
+      "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
+      "dev": true,
+      "requires": {
+        "core-util-is": "~1.0.0",
+        "inherits": "~2.0.3",
+        "isarray": "~1.0.0",
+        "process-nextick-args": "~2.0.0",
+        "safe-buffer": "~5.1.1",
+        "string_decoder": "~1.1.1",
+        "util-deprecate": "~1.0.1"
+      }
+    },
+    "rimraf": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
+      "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
+      "dev": true,
+      "requires": {
+        "glob": "^7.1.3"
+      }
+    },
+    "safe-buffer": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
+      "dev": true
+    },
+    "serverless-ruby-layer": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/serverless-ruby-layer/-/serverless-ruby-layer-1.5.0.tgz",
+      "integrity": "sha512-hyhHmqHEPErGhPHH6QPb64ZziiFvX3PcwxKiaMQk2h30mYAvXaLEzFPpEImvvR39LOb/x+PIrjENa0iZUcXD3A==",
+      "dev": true,
+      "requires": {
+        "bluebird": "^3.5.5",
+        "fs-extra": "^8.1.0",
+        "jszip": "^3.1.0",
+        "rimraf": "^3.0.2"
+      }
+    },
+    "set-immediate-shim": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz",
+      "integrity": "sha1-SysbJ+uAip+NzEgaWOXlb1mfP2E=",
+      "dev": true
+    },
+    "string_decoder": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+      "dev": true,
+      "requires": {
+        "safe-buffer": "~5.1.0"
+      }
+    },
+    "universalify": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz",
+      "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==",
+      "dev": true
+    },
+    "util-deprecate": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
+      "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=",
+      "dev": true
+    },
+    "wrappy": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
+      "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=",
+      "dev": true
+    }
+  }
+}