funktor 0.6.3 → 0.7.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (23) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile.lock +3 -3
  3. data/funktor-testapp/funktor_config/environment.yml +0 -2
  4. data/funktor-testapp/funktor_config/resources/incoming_job_queue_user.yml +5 -0
  5. data/funktor-testapp/funktor_config/resources/jobs_table.yml +1 -1
  6. data/funktor-testapp/serverless.yml +1 -2
  7. data/lib/active_job/queue_adapters/funktor_adapter.rb +3 -6
  8. data/lib/funktor/activity_tracker.rb +4 -4
  9. data/lib/funktor/cli/init.rb +0 -2
  10. data/lib/funktor/cli/templates/funktor_config/environment.yml +0 -2
  11. data/lib/funktor/cli/templates/funktor_config/resources/incoming_job_queue_user.yml +5 -0
  12. data/lib/funktor/cli/templates/funktor_config/resources/jobs_table.yml +1 -1
  13. data/lib/funktor/cli/templates/serverless.yml +1 -0
  14. data/lib/funktor/shard_utils.rb +1 -1
  15. data/lib/funktor/version.rb +1 -1
  16. data/lib/funktor/web/application.rb +5 -5
  17. data/lib/funktor/web/views/retries.erb +1 -1
  18. data/lib/funktor/web/views/scheduled.erb +1 -1
  19. metadata +2 -6
  20. data/funktor-testapp/funktor_config/iam_permissions/activity_table.yml +0 -5
  21. data/funktor-testapp/funktor_config/resources/activity_table.yml +0 -22
  22. data/lib/funktor/cli/templates/funktor_config/iam_permissions/activity_table.yml +0 -5
  23. data/lib/funktor/cli/templates/funktor_config/resources/activity_table.yml +0 -22
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 9d35f7173579d1ba71a003f80641e48fb27fa8e55227401d28d24d064207f805
4
- data.tar.gz: df984a68f3753231077c9a3014690e5b22a25e31fdfa8ef4a78e6f3eb4ef3317
3
+ metadata.gz: 7f680156fa395a72f19c164e93862d822f6f5dca0cc18f297d3dd087d59abdb6
4
+ data.tar.gz: a2f28da51595a5de1605f40e4107ac0dd9bbf0063ab27bbbce29213d9638b0c7
5
5
  SHA512:
6
- metadata.gz: 13eb0dfc1050f99eb8358d34a8666a8fd90a45e931b22fff0dc932cb9e9df3e63534f4d7d660126fa977931d46f3e572232cdf2f0bdc17468a3893c99f2959ef
7
- data.tar.gz: a0ce634530cb573099c989b64fc2fd33a85fd04480d1bffa6b3af6c3bebea01cc93fea9f2c4da5cbd62ff29180cac82f5b939aedbd30e25955b80ba7edf0ba37
6
+ metadata.gz: 38d87e8cab761848a1d815b2efdbf8964448db3801ee51dcf11c4c618580da74a05297d83ed324929c4085e4be9a94840cb9181d7fa76559a6fc6dbd46faa4f0
7
+ data.tar.gz: c2e3a7907f10f21afd3dd2f7ae8228645d1e8368545f66baf4d10e641cd5729a1f2026a031998779fe928860c4acbc1786b322a740c02c53bb436cbfc9a4b4b8
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- funktor (0.6.3)
4
+ funktor (0.7.1)
5
5
  activesupport
6
6
  aws-sdk-dynamodb (~> 1.62)
7
7
  aws-sdk-sqs (~> 1.37)
@@ -22,7 +22,7 @@ GEM
22
22
  addressable (2.8.0)
23
23
  public_suffix (>= 2.0.2, < 5.0)
24
24
  aws-eventstream (1.1.1)
25
- aws-partitions (1.491.0)
25
+ aws-partitions (1.493.0)
26
26
  aws-sdk-core (3.119.1)
27
27
  aws-eventstream (~> 1, >= 1.0.2)
28
28
  aws-partitions (~> 1, >= 1.239.0)
@@ -31,7 +31,7 @@ GEM
31
31
  aws-sdk-dynamodb (1.62.0)
32
32
  aws-sdk-core (~> 3, >= 3.119.0)
33
33
  aws-sigv4 (~> 1.1)
34
- aws-sdk-sqs (1.42.0)
34
+ aws-sdk-sqs (1.43.0)
35
35
  aws-sdk-core (~> 3, >= 3.119.0)
36
36
  aws-sigv4 (~> 1.1)
37
37
  aws-sigv4 (1.2.4)
data/funktor-testapp/funktor_config/environment.yml CHANGED
@@ -9,7 +9,5 @@ FUNKTOR_DEFAULT_QUEUE:
9
9
  Ref: DefaultQueue
10
10
  FUNKTOR_LOW_CONCURRENCY_QUEUE:
11
11
  Ref: LowConcurrencyQueue
12
- FUNKTOR_ACTIVITY_TABLE:
13
- Ref: ActivityTable
14
12
  FUNKTOR_JOBS_TABLE:
15
13
  Ref: JobsTable
data/funktor-testapp/funktor_config/resources/incoming_job_queue_user.yml CHANGED
@@ -12,6 +12,11 @@ Resources:
12
12
  - sqs:*
13
13
  Resource:
14
14
  - "Fn::GetAtt": [ IncomingJobQueue, Arn ]
15
+ - Effect: Allow
16
+ Action:
17
+ - dynamodb:*
18
+ Resource:
19
+ - "Fn::GetAtt": [ JobsTable, Arn ]
15
20
 
16
21
  IncomingJobQueueUserAccessKey:
17
22
  Type: AWS::IAM::AccessKey
data/funktor-testapp/funktor_config/resources/jobs_table.yml CHANGED
@@ -5,7 +5,7 @@ Resources:
5
5
  TableName: ${self:custom.funktor.JobsTableName}
6
6
  AttributeDefinitions:
7
7
  - AttributeName: jobShard
8
- AttributeType: N
8
+ AttributeType: S
9
9
  - AttributeName: jobId
10
10
  AttributeType: S
11
11
  - AttributeName: category
data/funktor-testapp/serverless.yml CHANGED
@@ -23,12 +23,12 @@ frameworkVersion: '2'
23
23
  provider:
24
24
  name: aws
25
25
  runtime: ruby2.7
26
+ stage: ${opt:stage, 'dev'}
26
27
  lambdaHashingVersion: 20201221
27
28
  environment: ${file(funktor_config/environment.yml)}
28
29
  versionFunctions: false # Reduces the amount of storage used since all Lambdas together are limited to 75GB
29
30
  logRetentionInDays: 7
30
31
  iamRoleStatements:
31
- - ${file(funktor_config/iam_permissions/activity_table.yml)}
32
32
  - ${file(funktor_config/iam_permissions/default_queue.yml)}
33
33
  - ${file(funktor_config/iam_permissions/incoming_job_queue.yml)}
34
34
  - ${file(funktor_config/iam_permissions/ssm.yml)}
@@ -54,7 +54,6 @@ functions:
54
54
  RandomJobGenerator: ${file(funktor_config/function_definitions/random_job_generator.yml)}
55
55
 
56
56
  resources:
57
- - ${file(funktor_config/resources/activity_table.yml)}
58
57
  - ${file(funktor_config/resources/incoming_job_queue_user.yml)}
59
58
  - ${file(funktor_config/resources/cloudwatch_dashboard.yml)}
60
59
  - ${file(funktor_config/resources/default_queue.yml)}
data/lib/active_job/queue_adapters/funktor_adapter.rb CHANGED
@@ -16,21 +16,18 @@ module ActiveJob
16
16
  "worker" => JobWrapper.to_s,
17
17
  "wrapped" => job.class,
18
18
  "queue" => job.class.work_queue,
19
- "delay" => 0,
19
+ "perform_at" => Time.now.utc,
20
20
  "worker_params" => [ job.serialize ]
21
21
  })
22
22
  end
23
23
 
24
24
  def enqueue_at(job, timestamp) # :nodoc:
25
- delay = (Time.at(timestamp).utc - Time.now.utc).round
26
- if delay < 0
27
- delay = 0
28
- end
25
+ time = Time.at(timestamp).utc
29
26
  job.provider_job_id = Funktor.job_pusher.push({
30
27
  "worker" => JobWrapper.to_s,
31
28
  "wrapped" => job.class,
32
29
  "queue" => job.class.work_queue,
33
- "delay" => delay,
30
+ "perform_at" => time,
34
31
  "worker_params" => [ job.serialize ]
35
32
  })
36
33
  end
data/lib/funktor/activity_tracker.rb CHANGED
@@ -56,8 +56,8 @@ module Funktor
56
56
  def increment_key(key)
57
57
  #put_metric_to_stdout(key, 1)
58
58
  dynamodb_client.update_item({
59
- table_name: ENV['FUNKTOR_ACTIVITY_TABLE'],
60
- key: { category: 'stat', statName: key },
59
+ table_name: ENV['FUNKTOR_JOBS_TABLE'],
60
+ key: { jobShard: 'stat', jobId: key },
61
61
  expression_attribute_values:{ ":start": 0, ":inc": 1 },
62
62
  update_expression: "SET stat_value = if_not_exists(stat_value, :start) + :inc",
63
63
  })
@@ -66,8 +66,8 @@ module Funktor
66
66
  def decrement_key(key)
67
67
  #put_metric_to_stdout(key, -1)
68
68
  dynamodb_client.update_item({
69
- table_name: ENV['FUNKTOR_ACTIVITY_TABLE'],
70
- key: { category: 'stat', statName: key },
69
+ table_name: ENV['FUNKTOR_JOBS_TABLE'],
70
+ key: { jobShard: 'stat', jobId: key },
71
71
  expression_attribute_values:{ ":start": 0, ":inc": 1 },
72
72
  update_expression: "SET stat_value = if_not_exists(stat_value, :start) - :inc",
73
73
  })
data/lib/funktor/cli/init.rb CHANGED
@@ -66,7 +66,6 @@ module Funktor
66
66
  template File.join("funktor_config", "resources", "work_queue.yml"), File.join("funktor_config", "resources", "#{work_queue_name.underscore}_queue.yml")
67
67
  end
68
68
  template File.join("funktor_config", "resources", "jobs_table.yml"), File.join("funktor_config", "resources", "jobs_table.yml")
69
- template File.join("funktor_config", "resources", "activity_table.yml"), File.join("funktor_config", "resources", "activity_table.yml")
70
69
  end
71
70
 
72
71
  def iam_permissions
@@ -79,7 +78,6 @@ module Funktor
79
78
  end
80
79
  template File.join("funktor_config", "iam_permissions", "jobs_table.yml"), File.join("funktor_config", "iam_permissions", "jobs_table.yml")
81
80
  template File.join("funktor_config", "iam_permissions", "jobs_table_secondary_index.yml"), File.join("funktor_config", "iam_permissions", "jobs_table_secondary_index.yml")
82
- template File.join("funktor_config", "iam_permissions", "activity_table.yml"), File.join("funktor_config", "iam_permissions", "activity_table.yml")
83
81
  end
84
82
 
85
83
  def function_definitions
data/lib/funktor/cli/templates/funktor_config/environment.yml CHANGED
@@ -9,7 +9,5 @@ FUNKTOR_INCOMING_JOB_QUEUE:
9
9
  FUNKTOR_<%= queue_name.underscore.upcase %>_QUEUE:
10
10
  Ref: <%= queue_name.camelize %>Queue
11
11
  <%- end -%>
12
- FUNKTOR_ACTIVITY_TABLE:
13
- Ref: ActivityTable
14
12
  FUNKTOR_JOBS_TABLE:
15
13
  Ref: JobsTable
data/lib/funktor/cli/templates/funktor_config/resources/incoming_job_queue_user.yml CHANGED
@@ -12,6 +12,11 @@ Resources:
12
12
  - sqs:*
13
13
  Resource:
14
14
  - "Fn::GetAtt": [ IncomingJobQueue, Arn ]
15
+ - Effect: Allow
16
+ Action:
17
+ - dynamodb:*
18
+ Resource:
19
+ - "Fn::GetAtt": [ JobsTable, Arn ]
15
20
 
16
21
  IncomingJobQueueUserAccessKey:
17
22
  Type: AWS::IAM::AccessKey
data/lib/funktor/cli/templates/funktor_config/resources/jobs_table.yml CHANGED
@@ -5,7 +5,7 @@ Resources:
5
5
  TableName: ${self:custom.funktor.JobsTableName}
6
6
  AttributeDefinitions:
7
7
  - AttributeName: jobShard
8
- AttributeType: N
8
+ AttributeType: S
9
9
  - AttributeName: jobId
10
10
  AttributeType: S
11
11
  - AttributeName: category
data/lib/funktor/cli/templates/serverless.yml CHANGED
@@ -23,6 +23,7 @@ frameworkVersion: '2'
23
23
  provider:
24
24
  name: aws
25
25
  runtime: <%= runtime %>
26
+ stage: ${opt:stage, 'dev'}
26
27
  lambdaHashingVersion: 20201221
27
28
  environment: ${file(funktor_config/environment.yml)}
28
29
  versionFunctions: false # Reduces the amount of storage used since all Lambdas together are limited to 75GB
data/lib/funktor/shard_utils.rb CHANGED
@@ -1,6 +1,6 @@
1
1
  module ShardUtils
2
2
  def calculate_shard(job_id)
3
3
  # TODO - Should the number of shards be configurable?
4
- job_id.sum % 64
4
+ (job_id.sum % 64).to_s
5
5
  end
6
6
  end
data/lib/funktor/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Funktor
2
- VERSION = "0.6.3"
2
+ VERSION = "0.7.1"
3
3
  end
data/lib/funktor/web/application.rb CHANGED
@@ -80,16 +80,16 @@ module Funktor
80
80
  def get_activity_data
81
81
  query_params = {
82
82
  expression_attribute_values: {
83
- ":category" => "stat"
83
+ ":jobShard" => "stat"
84
84
  },
85
- key_condition_expression: "category = :category",
86
- projection_expression: "statName, stat_value",
87
- table_name: ENV['FUNKTOR_ACTIVITY_TABLE']
85
+ key_condition_expression: "jobShard = :jobShard",
86
+ projection_expression: "jobId, stat_value",
87
+ table_name: ENV['FUNKTOR_JOBS_TABLE']
88
88
  }
89
89
  resp = dynamodb_client.query(query_params)
90
90
  @activity_stats = {}
91
91
  resp.items.each do |item|
92
- @activity_stats[item["statName"]] = item["stat_value"].to_i
92
+ @activity_stats[item["jobId"]] = item["stat_value"].to_i
93
93
  end
94
94
  return @activity_stats
95
95
  end
data/lib/funktor/web/views/retries.erb CHANGED
@@ -1,5 +1,5 @@
1
1
  <h1>Retries</h1>
2
- <form action="/update_jobs" method="post" accept-charset="utf-8">
2
+ <form action="./update_jobs" method="post" accept-charset="utf-8">
3
3
  <input type="hidden" name="source" value="retry" />
4
4
  <%= erb :table_stats_with_buttons, locals: { jobs: jobs, stat_category: 'retries' } %>
5
5
  <table role="grid">
data/lib/funktor/web/views/scheduled.erb CHANGED
@@ -1,5 +1,5 @@
1
1
  <h1>Scheduled</h1>
2
- <form action="/update_jobs" method="post" accept-charset="utf-8">
2
+ <form action="./update_jobs" method="post" accept-charset="utf-8">
3
3
  <input type="hidden" name="source" value="scheduled" />
4
4
  <%= erb :table_stats_with_buttons, locals: { jobs: jobs, stat_category: 'scheduled' } %>
5
5
  <table role="grid">
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: funktor
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.3
4
+ version: 0.7.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jeremy Green
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2021-08-25 00:00:00.000000000 Z
11
+ date: 2021-08-31 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: aws-sdk-sqs
@@ -192,7 +192,6 @@ files:
192
192
  - funktor-testapp/funktor_config/function_definitions/low_concurrency_queue_handler.yml
193
193
  - funktor-testapp/funktor_config/function_definitions/random_job_generator.yml
194
194
  - funktor-testapp/funktor_config/funktor.yml
195
- - funktor-testapp/funktor_config/iam_permissions/activity_table.yml
196
195
  - funktor-testapp/funktor_config/iam_permissions/default_queue.yml
197
196
  - funktor-testapp/funktor_config/iam_permissions/incoming_job_queue.yml
198
197
  - funktor-testapp/funktor_config/iam_permissions/jobs_table.yml
@@ -200,7 +199,6 @@ files:
200
199
  - funktor-testapp/funktor_config/iam_permissions/low_concurrency_queue.yml
201
200
  - funktor-testapp/funktor_config/iam_permissions/ssm.yml
202
201
  - funktor-testapp/funktor_config/package.yml
203
- - funktor-testapp/funktor_config/resources/activity_table.yml
204
202
  - funktor-testapp/funktor_config/resources/cloudwatch_dashboard.yml
205
203
  - funktor-testapp/funktor_config/resources/default_queue.yml
206
204
  - funktor-testapp/funktor_config/resources/incoming_job_queue.yml
@@ -237,14 +235,12 @@ files:
237
235
  - lib/funktor/cli/templates/funktor_config/function_definitions/job_activator.yml
238
236
  - lib/funktor/cli/templates/funktor_config/function_definitions/work_queue_handler.yml
239
237
  - lib/funktor/cli/templates/funktor_config/funktor.yml
240
- - lib/funktor/cli/templates/funktor_config/iam_permissions/activity_table.yml
241
238
  - lib/funktor/cli/templates/funktor_config/iam_permissions/incoming_job_queue.yml
242
239
  - lib/funktor/cli/templates/funktor_config/iam_permissions/jobs_table.yml
243
240
  - lib/funktor/cli/templates/funktor_config/iam_permissions/jobs_table_secondary_index.yml
244
241
  - lib/funktor/cli/templates/funktor_config/iam_permissions/ssm.yml
245
242
  - lib/funktor/cli/templates/funktor_config/iam_permissions/work_queue.yml
246
243
  - lib/funktor/cli/templates/funktor_config/package.yml
247
- - lib/funktor/cli/templates/funktor_config/resources/activity_table.yml
248
244
  - lib/funktor/cli/templates/funktor_config/resources/cloudwatch_dashboard.yml
249
245
  - lib/funktor/cli/templates/funktor_config/resources/incoming_job_queue.yml
250
246
  - lib/funktor/cli/templates/funktor_config/resources/incoming_job_queue_user.yml
data/funktor-testapp/funktor_config/iam_permissions/activity_table.yml DELETED
@@ -1,5 +0,0 @@
1
- Effect: Allow
2
- Action:
3
- - dynamodb:*
4
- Resource:
5
- - "Fn::GetAtt": [ ActivityTable, Arn ]
data/funktor-testapp/funktor_config/resources/activity_table.yml DELETED
@@ -1,22 +0,0 @@
1
- Resources:
2
- ActivityTable:
3
- Type: AWS::DynamoDB::Table
4
- Properties:
5
- # Generate a name based on the stage
6
- TableName: ${self:custom.funktor.ActivityTableName}
7
- BillingMode: PAY_PER_REQUEST
8
- AttributeDefinitions:
9
- - AttributeName: category
10
- AttributeType: S
11
- - AttributeName: statName
12
- AttributeType: S
13
- KeySchema:
14
- - AttributeName: category
15
- KeyType: HASH
16
- - AttributeName: statName
17
- KeyType: RANGE
18
-
19
- Outputs:
20
- ActivityTable:
21
- Value:
22
- Ref: ActivityTable
data/lib/funktor/cli/templates/funktor_config/iam_permissions/activity_table.yml DELETED
@@ -1,5 +0,0 @@
1
- Effect: Allow
2
- Action:
3
- - dynamodb:*
4
- Resource:
5
- - "Fn::GetAtt": [ ActivityTable, Arn ]
data/lib/funktor/cli/templates/funktor_config/resources/activity_table.yml DELETED
@@ -1,22 +0,0 @@
1
- Resources:
2
- ActivityTable:
3
- Type: AWS::DynamoDB::Table
4
- Properties:
5
- # Generate a name based on the stage
6
- TableName: ${self:custom.funktor.ActivityTableName}
7
- BillingMode: PAY_PER_REQUEST
8
- AttributeDefinitions:
9
- - AttributeName: category
10
- AttributeType: S
11
- - AttributeName: statName
12
- AttributeType: S
13
- KeySchema:
14
- - AttributeName: category
15
- KeyType: HASH
16
- - AttributeName: statName
17
- KeyType: RANGE
18
-
19
- Outputs:
20
- ActivityTable:
21
- Value:
22
- Ref: ActivityTable