funktor 0.6.4 → 0.7.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (21) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile.lock +3 -3
  3. data/funktor-testapp/funktor_config/environment.yml +0 -2
  4. data/funktor-testapp/funktor_config/resources/incoming_job_queue_user.yml +13 -0
  5. data/funktor-testapp/funktor_config/resources/jobs_table.yml +1 -1
  6. data/funktor-testapp/serverless.yml +1 -2
  7. data/lib/active_job/queue_adapters/funktor_adapter.rb +3 -6
  8. data/lib/funktor/activity_tracker.rb +4 -4
  9. data/lib/funktor/cli/init.rb +0 -2
  10. data/lib/funktor/cli/templates/funktor_config/environment.yml +0 -2
  11. data/lib/funktor/cli/templates/funktor_config/resources/incoming_job_queue_user.yml +13 -0
  12. data/lib/funktor/cli/templates/funktor_config/resources/jobs_table.yml +1 -1
  13. data/lib/funktor/cli/templates/serverless.yml +1 -0
  14. data/lib/funktor/shard_utils.rb +1 -1
  15. data/lib/funktor/version.rb +1 -1
  16. data/lib/funktor/web/application.rb +5 -5
  17. metadata +2 -6
  18. data/funktor-testapp/funktor_config/iam_permissions/activity_table.yml +0 -5
  19. data/funktor-testapp/funktor_config/resources/activity_table.yml +0 -22
  20. data/lib/funktor/cli/templates/funktor_config/iam_permissions/activity_table.yml +0 -5
  21. data/lib/funktor/cli/templates/funktor_config/resources/activity_table.yml +0 -22
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: b3416a7e487adad405490efe9897d7ad50049a6d8cf29731668b32f16ab59927
4
- data.tar.gz: 9b03a1acd26827521079f978557eebe391f40d1d91731146ee950209075db6c8
3
+ metadata.gz: 666dbb222916dd61aa30c450c8dba4c0ba72104694c59f73311509c655e7e314
4
+ data.tar.gz: c14a2b2bfedd970aa8278d7d04e3f17612528317a0ca5ad3ef0747da09147a79
5
5
  SHA512:
6
- metadata.gz: 835212e6c8ac820bfac1cae4a3ef7162f4bacbb734b64675eaf81171dc0ffab3dcbf66ffdef0bbab215393a8e01cf60763cbf60b9437c5400e1cf6360b0d60dd
7
- data.tar.gz: 39b04ad6472adc2e96b40fa1ada27f6386eba359eb810f209f8880c03f3099f8cb78132f0835c233f3a99fe050b3072cbf49e906b7af70495ec5c7de084392e4
6
+ metadata.gz: 9c1d2f538b1e725d94459d79ddbcc68e42654e8188f4072e197e641b0e094a05961c6e49caa3cb273fb5ef19a8f09e8587d13746bc2593b89188d76eac6d811d
7
+ data.tar.gz: dc8b60572bd5a130274e60c7ceecc3094ff93804c29fd179457129ae3a789dd0d956ea24aaf88e8d70ce9096756280718d40a476620a17fabee71a47fa184cc8
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- funktor (0.6.4)
4
+ funktor (0.7.2)
5
5
  activesupport
6
6
  aws-sdk-dynamodb (~> 1.62)
7
7
  aws-sdk-sqs (~> 1.37)
@@ -22,7 +22,7 @@ GEM
22
22
  addressable (2.8.0)
23
23
  public_suffix (>= 2.0.2, < 5.0)
24
24
  aws-eventstream (1.1.1)
25
- aws-partitions (1.491.0)
25
+ aws-partitions (1.493.0)
26
26
  aws-sdk-core (3.119.1)
27
27
  aws-eventstream (~> 1, >= 1.0.2)
28
28
  aws-partitions (~> 1, >= 1.239.0)
@@ -31,7 +31,7 @@ GEM
31
31
  aws-sdk-dynamodb (1.62.0)
32
32
  aws-sdk-core (~> 3, >= 3.119.0)
33
33
  aws-sigv4 (~> 1.1)
34
- aws-sdk-sqs (1.42.0)
34
+ aws-sdk-sqs (1.43.0)
35
35
  aws-sdk-core (~> 3, >= 3.119.0)
36
36
  aws-sigv4 (~> 1.1)
37
37
  aws-sigv4 (1.2.4)
data/funktor-testapp/funktor_config/environment.yml CHANGED
@@ -9,7 +9,5 @@ FUNKTOR_DEFAULT_QUEUE:
9
9
  Ref: DefaultQueue
10
10
  FUNKTOR_LOW_CONCURRENCY_QUEUE:
11
11
  Ref: LowConcurrencyQueue
12
- FUNKTOR_ACTIVITY_TABLE:
13
- Ref: ActivityTable
14
12
  FUNKTOR_JOBS_TABLE:
15
13
  Ref: JobsTable
data/funktor-testapp/funktor_config/resources/incoming_job_queue_user.yml CHANGED
@@ -12,6 +12,19 @@ Resources:
12
12
  - sqs:*
13
13
  Resource:
14
14
  - "Fn::GetAtt": [ IncomingJobQueue, Arn ]
15
+ - Effect: Allow
16
+ Action:
17
+ - dynamodb:*
18
+ Resource:
19
+ - "Fn::GetAtt": [ JobsTable, Arn ]
20
+ - Effect: Allow
21
+ Action:
22
+ - dynamodb:Query
23
+ Resource:
24
+ Fn::Join:
25
+ - ""
26
+ - - "Fn::GetAtt": [ JobsTable, Arn ]
27
+ - "/index/categoryIndex"
15
28
 
16
29
  IncomingJobQueueUserAccessKey:
17
30
  Type: AWS::IAM::AccessKey
data/funktor-testapp/funktor_config/resources/jobs_table.yml CHANGED
@@ -5,7 +5,7 @@ Resources:
5
5
  TableName: ${self:custom.funktor.JobsTableName}
6
6
  AttributeDefinitions:
7
7
  - AttributeName: jobShard
8
- AttributeType: N
8
+ AttributeType: S
9
9
  - AttributeName: jobId
10
10
  AttributeType: S
11
11
  - AttributeName: category
data/funktor-testapp/serverless.yml CHANGED
@@ -23,12 +23,12 @@ frameworkVersion: '2'
23
23
  provider:
24
24
  name: aws
25
25
  runtime: ruby2.7
26
+ stage: ${opt:stage, 'dev'}
26
27
  lambdaHashingVersion: 20201221
27
28
  environment: ${file(funktor_config/environment.yml)}
28
29
  versionFunctions: false # Reduces the amount of storage used since all Lambdas together are limited to 75GB
29
30
  logRetentionInDays: 7
30
31
  iamRoleStatements:
31
- - ${file(funktor_config/iam_permissions/activity_table.yml)}
32
32
  - ${file(funktor_config/iam_permissions/default_queue.yml)}
33
33
  - ${file(funktor_config/iam_permissions/incoming_job_queue.yml)}
34
34
  - ${file(funktor_config/iam_permissions/ssm.yml)}
@@ -54,7 +54,6 @@ functions:
54
54
  RandomJobGenerator: ${file(funktor_config/function_definitions/random_job_generator.yml)}
55
55
 
56
56
  resources:
57
- - ${file(funktor_config/resources/activity_table.yml)}
58
57
  - ${file(funktor_config/resources/incoming_job_queue_user.yml)}
59
58
  - ${file(funktor_config/resources/cloudwatch_dashboard.yml)}
60
59
  - ${file(funktor_config/resources/default_queue.yml)}
data/lib/active_job/queue_adapters/funktor_adapter.rb CHANGED
@@ -16,21 +16,18 @@ module ActiveJob
16
16
  "worker" => JobWrapper.to_s,
17
17
  "wrapped" => job.class,
18
18
  "queue" => job.class.work_queue,
19
- "delay" => 0,
19
+ "perform_at" => Time.now.utc,
20
20
  "worker_params" => [ job.serialize ]
21
21
  })
22
22
  end
23
23
 
24
24
  def enqueue_at(job, timestamp) # :nodoc:
25
- delay = (Time.at(timestamp).utc - Time.now.utc).round
26
- if delay < 0
27
- delay = 0
28
- end
25
+ time = Time.at(timestamp).utc
29
26
  job.provider_job_id = Funktor.job_pusher.push({
30
27
  "worker" => JobWrapper.to_s,
31
28
  "wrapped" => job.class,
32
29
  "queue" => job.class.work_queue,
33
- "delay" => delay,
30
+ "perform_at" => time,
34
31
  "worker_params" => [ job.serialize ]
35
32
  })
36
33
  end
data/lib/funktor/activity_tracker.rb CHANGED
@@ -56,8 +56,8 @@ module Funktor
56
56
  def increment_key(key)
57
57
  #put_metric_to_stdout(key, 1)
58
58
  dynamodb_client.update_item({
59
- table_name: ENV['FUNKTOR_ACTIVITY_TABLE'],
60
- key: { category: 'stat', statName: key },
59
+ table_name: ENV['FUNKTOR_JOBS_TABLE'],
60
+ key: { jobShard: 'stat', jobId: key },
61
61
  expression_attribute_values:{ ":start": 0, ":inc": 1 },
62
62
  update_expression: "SET stat_value = if_not_exists(stat_value, :start) + :inc",
63
63
  })
@@ -66,8 +66,8 @@ module Funktor
66
66
  def decrement_key(key)
67
67
  #put_metric_to_stdout(key, -1)
68
68
  dynamodb_client.update_item({
69
- table_name: ENV['FUNKTOR_ACTIVITY_TABLE'],
70
- key: { category: 'stat', statName: key },
69
+ table_name: ENV['FUNKTOR_JOBS_TABLE'],
70
+ key: { jobShard: 'stat', jobId: key },
71
71
  expression_attribute_values:{ ":start": 0, ":inc": 1 },
72
72
  update_expression: "SET stat_value = if_not_exists(stat_value, :start) - :inc",
73
73
  })
data/lib/funktor/cli/init.rb CHANGED
@@ -66,7 +66,6 @@ module Funktor
66
66
  template File.join("funktor_config", "resources", "work_queue.yml"), File.join("funktor_config", "resources", "#{work_queue_name.underscore}_queue.yml")
67
67
  end
68
68
  template File.join("funktor_config", "resources", "jobs_table.yml"), File.join("funktor_config", "resources", "jobs_table.yml")
69
- template File.join("funktor_config", "resources", "activity_table.yml"), File.join("funktor_config", "resources", "activity_table.yml")
70
69
  end
71
70
 
72
71
  def iam_permissions
@@ -79,7 +78,6 @@ module Funktor
79
78
  end
80
79
  template File.join("funktor_config", "iam_permissions", "jobs_table.yml"), File.join("funktor_config", "iam_permissions", "jobs_table.yml")
81
80
  template File.join("funktor_config", "iam_permissions", "jobs_table_secondary_index.yml"), File.join("funktor_config", "iam_permissions", "jobs_table_secondary_index.yml")
82
- template File.join("funktor_config", "iam_permissions", "activity_table.yml"), File.join("funktor_config", "iam_permissions", "activity_table.yml")
83
81
  end
84
82
 
85
83
  def function_definitions
data/lib/funktor/cli/templates/funktor_config/environment.yml CHANGED
@@ -9,7 +9,5 @@ FUNKTOR_INCOMING_JOB_QUEUE:
9
9
  FUNKTOR_<%= queue_name.underscore.upcase %>_QUEUE:
10
10
  Ref: <%= queue_name.camelize %>Queue
11
11
  <%- end -%>
12
- FUNKTOR_ACTIVITY_TABLE:
13
- Ref: ActivityTable
14
12
  FUNKTOR_JOBS_TABLE:
15
13
  Ref: JobsTable
data/lib/funktor/cli/templates/funktor_config/resources/incoming_job_queue_user.yml CHANGED
@@ -12,6 +12,19 @@ Resources:
12
12
  - sqs:*
13
13
  Resource:
14
14
  - "Fn::GetAtt": [ IncomingJobQueue, Arn ]
15
+ - Effect: Allow
16
+ Action:
17
+ - dynamodb:*
18
+ Resource:
19
+ - "Fn::GetAtt": [ JobsTable, Arn ]
20
+ - Effect: Allow
21
+ Action:
22
+ - dynamodb:Query
23
+ Resource:
24
+ Fn::Join:
25
+ - ""
26
+ - - "Fn::GetAtt": [ JobsTable, Arn ]
27
+ - "/index/categoryIndex"
15
28
 
16
29
  IncomingJobQueueUserAccessKey:
17
30
  Type: AWS::IAM::AccessKey
data/lib/funktor/cli/templates/funktor_config/resources/jobs_table.yml CHANGED
@@ -5,7 +5,7 @@ Resources:
5
5
  TableName: ${self:custom.funktor.JobsTableName}
6
6
  AttributeDefinitions:
7
7
  - AttributeName: jobShard
8
- AttributeType: N
8
+ AttributeType: S
9
9
  - AttributeName: jobId
10
10
  AttributeType: S
11
11
  - AttributeName: category
data/lib/funktor/cli/templates/serverless.yml CHANGED
@@ -23,6 +23,7 @@ frameworkVersion: '2'
23
23
  provider:
24
24
  name: aws
25
25
  runtime: <%= runtime %>
26
+ stage: ${opt:stage, 'dev'}
26
27
  lambdaHashingVersion: 20201221
27
28
  environment: ${file(funktor_config/environment.yml)}
28
29
  versionFunctions: false # Reduces the amount of storage used since all Lambdas together are limited to 75GB
data/lib/funktor/shard_utils.rb CHANGED
@@ -1,6 +1,6 @@
1
1
  module ShardUtils
2
2
  def calculate_shard(job_id)
3
3
  # TODO - Should the number of shards be configurable?
4
- job_id.sum % 64
4
+ (job_id.sum % 64).to_s
5
5
  end
6
6
  end
data/lib/funktor/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Funktor
2
- VERSION = "0.6.4"
2
+ VERSION = "0.7.2"
3
3
  end
data/lib/funktor/web/application.rb CHANGED
@@ -80,16 +80,16 @@ module Funktor
80
80
  def get_activity_data
81
81
  query_params = {
82
82
  expression_attribute_values: {
83
- ":category" => "stat"
83
+ ":jobShard" => "stat"
84
84
  },
85
- key_condition_expression: "category = :category",
86
- projection_expression: "statName, stat_value",
87
- table_name: ENV['FUNKTOR_ACTIVITY_TABLE']
85
+ key_condition_expression: "jobShard = :jobShard",
86
+ projection_expression: "jobId, stat_value",
87
+ table_name: ENV['FUNKTOR_JOBS_TABLE']
88
88
  }
89
89
  resp = dynamodb_client.query(query_params)
90
90
  @activity_stats = {}
91
91
  resp.items.each do |item|
92
- @activity_stats[item["statName"]] = item["stat_value"].to_i
92
+ @activity_stats[item["jobId"]] = item["stat_value"].to_i
93
93
  end
94
94
  return @activity_stats
95
95
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: funktor
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.4
4
+ version: 0.7.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jeremy Green
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2021-08-25 00:00:00.000000000 Z
11
+ date: 2021-08-31 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: aws-sdk-sqs
@@ -192,7 +192,6 @@ files:
192
192
  - funktor-testapp/funktor_config/function_definitions/low_concurrency_queue_handler.yml
193
193
  - funktor-testapp/funktor_config/function_definitions/random_job_generator.yml
194
194
  - funktor-testapp/funktor_config/funktor.yml
195
- - funktor-testapp/funktor_config/iam_permissions/activity_table.yml
196
195
  - funktor-testapp/funktor_config/iam_permissions/default_queue.yml
197
196
  - funktor-testapp/funktor_config/iam_permissions/incoming_job_queue.yml
198
197
  - funktor-testapp/funktor_config/iam_permissions/jobs_table.yml
@@ -200,7 +199,6 @@ files:
200
199
  - funktor-testapp/funktor_config/iam_permissions/low_concurrency_queue.yml
201
200
  - funktor-testapp/funktor_config/iam_permissions/ssm.yml
202
201
  - funktor-testapp/funktor_config/package.yml
203
- - funktor-testapp/funktor_config/resources/activity_table.yml
204
202
  - funktor-testapp/funktor_config/resources/cloudwatch_dashboard.yml
205
203
  - funktor-testapp/funktor_config/resources/default_queue.yml
206
204
  - funktor-testapp/funktor_config/resources/incoming_job_queue.yml
@@ -237,14 +235,12 @@ files:
237
235
  - lib/funktor/cli/templates/funktor_config/function_definitions/job_activator.yml
238
236
  - lib/funktor/cli/templates/funktor_config/function_definitions/work_queue_handler.yml
239
237
  - lib/funktor/cli/templates/funktor_config/funktor.yml
240
- - lib/funktor/cli/templates/funktor_config/iam_permissions/activity_table.yml
241
238
  - lib/funktor/cli/templates/funktor_config/iam_permissions/incoming_job_queue.yml
242
239
  - lib/funktor/cli/templates/funktor_config/iam_permissions/jobs_table.yml
243
240
  - lib/funktor/cli/templates/funktor_config/iam_permissions/jobs_table_secondary_index.yml
244
241
  - lib/funktor/cli/templates/funktor_config/iam_permissions/ssm.yml
245
242
  - lib/funktor/cli/templates/funktor_config/iam_permissions/work_queue.yml
246
243
  - lib/funktor/cli/templates/funktor_config/package.yml
247
- - lib/funktor/cli/templates/funktor_config/resources/activity_table.yml
248
244
  - lib/funktor/cli/templates/funktor_config/resources/cloudwatch_dashboard.yml
249
245
  - lib/funktor/cli/templates/funktor_config/resources/incoming_job_queue.yml
250
246
  - lib/funktor/cli/templates/funktor_config/resources/incoming_job_queue_user.yml
data/funktor-testapp/funktor_config/iam_permissions/activity_table.yml DELETED
@@ -1,5 +0,0 @@
1
- Effect: Allow
2
- Action:
3
- - dynamodb:*
4
- Resource:
5
- - "Fn::GetAtt": [ ActivityTable, Arn ]
data/funktor-testapp/funktor_config/resources/activity_table.yml DELETED
@@ -1,22 +0,0 @@
1
- Resources:
2
- ActivityTable:
3
- Type: AWS::DynamoDB::Table
4
- Properties:
5
- # Generate a name based on the stage
6
- TableName: ${self:custom.funktor.ActivityTableName}
7
- BillingMode: PAY_PER_REQUEST
8
- AttributeDefinitions:
9
- - AttributeName: category
10
- AttributeType: S
11
- - AttributeName: statName
12
- AttributeType: S
13
- KeySchema:
14
- - AttributeName: category
15
- KeyType: HASH
16
- - AttributeName: statName
17
- KeyType: RANGE
18
-
19
- Outputs:
20
- ActivityTable:
21
- Value:
22
- Ref: ActivityTable
data/lib/funktor/cli/templates/funktor_config/iam_permissions/activity_table.yml DELETED
@@ -1,5 +0,0 @@
1
- Effect: Allow
2
- Action:
3
- - dynamodb:*
4
- Resource:
5
- - "Fn::GetAtt": [ ActivityTable, Arn ]
data/lib/funktor/cli/templates/funktor_config/resources/activity_table.yml DELETED
@@ -1,22 +0,0 @@
1
- Resources:
2
- ActivityTable:
3
- Type: AWS::DynamoDB::Table
4
- Properties:
5
- # Generate a name based on the stage
6
- TableName: ${self:custom.funktor.ActivityTableName}
7
- BillingMode: PAY_PER_REQUEST
8
- AttributeDefinitions:
9
- - AttributeName: category
10
- AttributeType: S
11
- - AttributeName: statName
12
- AttributeType: S
13
- KeySchema:
14
- - AttributeName: category
15
- KeyType: HASH
16
- - AttributeName: statName
17
- KeyType: RANGE
18
-
19
- Outputs:
20
- ActivityTable:
21
- Value:
22
- Ref: ActivityTable