funktor 0.6.5 → 0.7.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (18) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile.lock +3 -3
  3. data/funktor-testapp/funktor_config/environment.yml +0 -2
  4. data/funktor-testapp/funktor_config/resources/jobs_table.yml +1 -1
  5. data/funktor-testapp/serverless.yml +1 -2
  6. data/lib/funktor/activity_tracker.rb +4 -4
  7. data/lib/funktor/cli/init.rb +0 -2
  8. data/lib/funktor/cli/templates/funktor_config/environment.yml +0 -2
  9. data/lib/funktor/cli/templates/funktor_config/resources/jobs_table.yml +1 -1
  10. data/lib/funktor/cli/templates/serverless.yml +1 -0
  11. data/lib/funktor/shard_utils.rb +1 -1
  12. data/lib/funktor/version.rb +1 -1
  13. data/lib/funktor/web/application.rb +5 -5
  14. metadata +2 -6
  15. data/funktor-testapp/funktor_config/iam_permissions/activity_table.yml +0 -5
  16. data/funktor-testapp/funktor_config/resources/activity_table.yml +0 -22
  17. data/lib/funktor/cli/templates/funktor_config/iam_permissions/activity_table.yml +0 -5
  18. data/lib/funktor/cli/templates/funktor_config/resources/activity_table.yml +0 -22
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: c92c5b6465ea858b39776c939ca3ac42e4680b19bae5071660e212b80ad5fa23
4
- data.tar.gz: 54b4a61d1ae01a7e06a1ce3bd027f2aaf55faccda40e4e1789d698b60ff4a57d
3
+ metadata.gz: a5bf167646edb446db33542dd6d92d87a50829299a9f385634b3f4c8039e51fb
4
+ data.tar.gz: 27de5e56d319fabe8f1570337b0bc2f88a4589d84613c716591f7f805a509a1d
5
5
  SHA512:
6
- metadata.gz: fb515d5ed863cf279cee8ebcafa1ee352b0a470146c0795ed7d51d8b11d83cf0bdb577d90f0abdc5155d81a3d2446058401f2d138cf2a92ac283c54f3236bf34
7
- data.tar.gz: 92add3f0a131296bd1e11d3cc9c87b97e257647d6ef2ac54e00c28bbc2aaa98961672128318cab465d7d5553094b931bf8f6078bb985530d6f41cf1c73fc6299
6
+ metadata.gz: 6ed734ccf309256291fb01dfa3f30c575a20dc33bf7092c109ea3e23ab94ebeb436ae5ddfd21d597957a45ec5f8db14030f572fb5e7de1d9252d175b8b78ced2
7
+ data.tar.gz: '0023679e9e6189b5c6ca6c3547a76171e3a8b5e7e4df404aa91ff9cc7b3f889620838bde43c2bdd73b59af97221f7b913eccde3fef7279abc7defbd4fb9eaf0d'
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- funktor (0.6.5)
4
+ funktor (0.7.0)
5
5
  activesupport
6
6
  aws-sdk-dynamodb (~> 1.62)
7
7
  aws-sdk-sqs (~> 1.37)
@@ -22,7 +22,7 @@ GEM
22
22
  addressable (2.8.0)
23
23
  public_suffix (>= 2.0.2, < 5.0)
24
24
  aws-eventstream (1.1.1)
25
- aws-partitions (1.491.0)
25
+ aws-partitions (1.493.0)
26
26
  aws-sdk-core (3.119.1)
27
27
  aws-eventstream (~> 1, >= 1.0.2)
28
28
  aws-partitions (~> 1, >= 1.239.0)
@@ -31,7 +31,7 @@ GEM
31
31
  aws-sdk-dynamodb (1.62.0)
32
32
  aws-sdk-core (~> 3, >= 3.119.0)
33
33
  aws-sigv4 (~> 1.1)
34
- aws-sdk-sqs (1.42.0)
34
+ aws-sdk-sqs (1.43.0)
35
35
  aws-sdk-core (~> 3, >= 3.119.0)
36
36
  aws-sigv4 (~> 1.1)
37
37
  aws-sigv4 (1.2.4)
data/funktor-testapp/funktor_config/environment.yml CHANGED
@@ -9,7 +9,5 @@ FUNKTOR_DEFAULT_QUEUE:
9
9
  Ref: DefaultQueue
10
10
  FUNKTOR_LOW_CONCURRENCY_QUEUE:
11
11
  Ref: LowConcurrencyQueue
12
- FUNKTOR_ACTIVITY_TABLE:
13
- Ref: ActivityTable
14
12
  FUNKTOR_JOBS_TABLE:
15
13
  Ref: JobsTable
data/funktor-testapp/funktor_config/resources/jobs_table.yml CHANGED
@@ -5,7 +5,7 @@ Resources:
5
5
  TableName: ${self:custom.funktor.JobsTableName}
6
6
  AttributeDefinitions:
7
7
  - AttributeName: jobShard
8
- AttributeType: N
8
+ AttributeType: S
9
9
  - AttributeName: jobId
10
10
  AttributeType: S
11
11
  - AttributeName: category
data/funktor-testapp/serverless.yml CHANGED
@@ -23,12 +23,12 @@ frameworkVersion: '2'
23
23
  provider:
24
24
  name: aws
25
25
  runtime: ruby2.7
26
+ stage: ${opt:stage, 'dev'}
26
27
  lambdaHashingVersion: 20201221
27
28
  environment: ${file(funktor_config/environment.yml)}
28
29
  versionFunctions: false # Reduces the amount of storage used since all Lambdas together are limited to 75GB
29
30
  logRetentionInDays: 7
30
31
  iamRoleStatements:
31
- - ${file(funktor_config/iam_permissions/activity_table.yml)}
32
32
  - ${file(funktor_config/iam_permissions/default_queue.yml)}
33
33
  - ${file(funktor_config/iam_permissions/incoming_job_queue.yml)}
34
34
  - ${file(funktor_config/iam_permissions/ssm.yml)}
@@ -54,7 +54,6 @@ functions:
54
54
  RandomJobGenerator: ${file(funktor_config/function_definitions/random_job_generator.yml)}
55
55
 
56
56
  resources:
57
- - ${file(funktor_config/resources/activity_table.yml)}
58
57
  - ${file(funktor_config/resources/incoming_job_queue_user.yml)}
59
58
  - ${file(funktor_config/resources/cloudwatch_dashboard.yml)}
60
59
  - ${file(funktor_config/resources/default_queue.yml)}
data/lib/funktor/activity_tracker.rb CHANGED
@@ -56,8 +56,8 @@ module Funktor
56
56
  def increment_key(key)
57
57
  #put_metric_to_stdout(key, 1)
58
58
  dynamodb_client.update_item({
59
- table_name: ENV['FUNKTOR_ACTIVITY_TABLE'],
60
- key: { category: 'stat', statName: key },
59
+ table_name: ENV['FUNKTOR_JOBS_TABLE'],
60
+ key: { jobShard: 'stat', jobId: key },
61
61
  expression_attribute_values:{ ":start": 0, ":inc": 1 },
62
62
  update_expression: "SET stat_value = if_not_exists(stat_value, :start) + :inc",
63
63
  })
@@ -66,8 +66,8 @@ module Funktor
66
66
  def decrement_key(key)
67
67
  #put_metric_to_stdout(key, -1)
68
68
  dynamodb_client.update_item({
69
- table_name: ENV['FUNKTOR_ACTIVITY_TABLE'],
70
- key: { category: 'stat', statName: key },
69
+ table_name: ENV['FUNKTOR_JOBS_TABLE'],
70
+ key: { jobShard: 'stat', jobId: key },
71
71
  expression_attribute_values:{ ":start": 0, ":inc": 1 },
72
72
  update_expression: "SET stat_value = if_not_exists(stat_value, :start) - :inc",
73
73
  })
data/lib/funktor/cli/init.rb CHANGED
@@ -66,7 +66,6 @@ module Funktor
66
66
  template File.join("funktor_config", "resources", "work_queue.yml"), File.join("funktor_config", "resources", "#{work_queue_name.underscore}_queue.yml")
67
67
  end
68
68
  template File.join("funktor_config", "resources", "jobs_table.yml"), File.join("funktor_config", "resources", "jobs_table.yml")
69
- template File.join("funktor_config", "resources", "activity_table.yml"), File.join("funktor_config", "resources", "activity_table.yml")
70
69
  end
71
70
 
72
71
  def iam_permissions
@@ -79,7 +78,6 @@ module Funktor
79
78
  end
80
79
  template File.join("funktor_config", "iam_permissions", "jobs_table.yml"), File.join("funktor_config", "iam_permissions", "jobs_table.yml")
81
80
  template File.join("funktor_config", "iam_permissions", "jobs_table_secondary_index.yml"), File.join("funktor_config", "iam_permissions", "jobs_table_secondary_index.yml")
82
- template File.join("funktor_config", "iam_permissions", "activity_table.yml"), File.join("funktor_config", "iam_permissions", "activity_table.yml")
83
81
  end
84
82
 
85
83
  def function_definitions
data/lib/funktor/cli/templates/funktor_config/environment.yml CHANGED
@@ -9,7 +9,5 @@ FUNKTOR_INCOMING_JOB_QUEUE:
9
9
  FUNKTOR_<%= queue_name.underscore.upcase %>_QUEUE:
10
10
  Ref: <%= queue_name.camelize %>Queue
11
11
  <%- end -%>
12
- FUNKTOR_ACTIVITY_TABLE:
13
- Ref: ActivityTable
14
12
  FUNKTOR_JOBS_TABLE:
15
13
  Ref: JobsTable
data/lib/funktor/cli/templates/funktor_config/resources/jobs_table.yml CHANGED
@@ -5,7 +5,7 @@ Resources:
5
5
  TableName: ${self:custom.funktor.JobsTableName}
6
6
  AttributeDefinitions:
7
7
  - AttributeName: jobShard
8
- AttributeType: N
8
+ AttributeType: S
9
9
  - AttributeName: jobId
10
10
  AttributeType: S
11
11
  - AttributeName: category
data/lib/funktor/cli/templates/serverless.yml CHANGED
@@ -23,6 +23,7 @@ frameworkVersion: '2'
23
23
  provider:
24
24
  name: aws
25
25
  runtime: <%= runtime %>
26
+ stage: ${opt:stage, 'dev'}
26
27
  lambdaHashingVersion: 20201221
27
28
  environment: ${file(funktor_config/environment.yml)}
28
29
  versionFunctions: false # Reduces the amount of storage used since all Lambdas together are limited to 75GB
data/lib/funktor/shard_utils.rb CHANGED
@@ -1,6 +1,6 @@
1
1
  module ShardUtils
2
2
  def calculate_shard(job_id)
3
3
  # TODO - Should the number of shards be configurable?
4
- job_id.sum % 64
4
+ (job_id.sum % 64).to_s
5
5
  end
6
6
  end
data/lib/funktor/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Funktor
2
- VERSION = "0.6.5"
2
+ VERSION = "0.7.0"
3
3
  end
data/lib/funktor/web/application.rb CHANGED
@@ -80,16 +80,16 @@ module Funktor
80
80
  def get_activity_data
81
81
  query_params = {
82
82
  expression_attribute_values: {
83
- ":category" => "stat"
83
+ ":jobShard" => "stat"
84
84
  },
85
- key_condition_expression: "category = :category",
86
- projection_expression: "statName, stat_value",
87
- table_name: ENV['FUNKTOR_ACTIVITY_TABLE']
85
+ key_condition_expression: "jobShard = :jobShard",
86
+ projection_expression: "jobId, stat_value",
87
+ table_name: ENV['FUNKTOR_JOBS_TABLE']
88
88
  }
89
89
  resp = dynamodb_client.query(query_params)
90
90
  @activity_stats = {}
91
91
  resp.items.each do |item|
92
- @activity_stats[item["statName"]] = item["stat_value"].to_i
92
+ @activity_stats[item["jobId"]] = item["stat_value"].to_i
93
93
  end
94
94
  return @activity_stats
95
95
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: funktor
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.5
4
+ version: 0.7.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jeremy Green
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2021-08-25 00:00:00.000000000 Z
11
+ date: 2021-08-31 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: aws-sdk-sqs
@@ -192,7 +192,6 @@ files:
192
192
  - funktor-testapp/funktor_config/function_definitions/low_concurrency_queue_handler.yml
193
193
  - funktor-testapp/funktor_config/function_definitions/random_job_generator.yml
194
194
  - funktor-testapp/funktor_config/funktor.yml
195
- - funktor-testapp/funktor_config/iam_permissions/activity_table.yml
196
195
  - funktor-testapp/funktor_config/iam_permissions/default_queue.yml
197
196
  - funktor-testapp/funktor_config/iam_permissions/incoming_job_queue.yml
198
197
  - funktor-testapp/funktor_config/iam_permissions/jobs_table.yml
@@ -200,7 +199,6 @@ files:
200
199
  - funktor-testapp/funktor_config/iam_permissions/low_concurrency_queue.yml
201
200
  - funktor-testapp/funktor_config/iam_permissions/ssm.yml
202
201
  - funktor-testapp/funktor_config/package.yml
203
- - funktor-testapp/funktor_config/resources/activity_table.yml
204
202
  - funktor-testapp/funktor_config/resources/cloudwatch_dashboard.yml
205
203
  - funktor-testapp/funktor_config/resources/default_queue.yml
206
204
  - funktor-testapp/funktor_config/resources/incoming_job_queue.yml
@@ -237,14 +235,12 @@ files:
237
235
  - lib/funktor/cli/templates/funktor_config/function_definitions/job_activator.yml
238
236
  - lib/funktor/cli/templates/funktor_config/function_definitions/work_queue_handler.yml
239
237
  - lib/funktor/cli/templates/funktor_config/funktor.yml
240
- - lib/funktor/cli/templates/funktor_config/iam_permissions/activity_table.yml
241
238
  - lib/funktor/cli/templates/funktor_config/iam_permissions/incoming_job_queue.yml
242
239
  - lib/funktor/cli/templates/funktor_config/iam_permissions/jobs_table.yml
243
240
  - lib/funktor/cli/templates/funktor_config/iam_permissions/jobs_table_secondary_index.yml
244
241
  - lib/funktor/cli/templates/funktor_config/iam_permissions/ssm.yml
245
242
  - lib/funktor/cli/templates/funktor_config/iam_permissions/work_queue.yml
246
243
  - lib/funktor/cli/templates/funktor_config/package.yml
247
- - lib/funktor/cli/templates/funktor_config/resources/activity_table.yml
248
244
  - lib/funktor/cli/templates/funktor_config/resources/cloudwatch_dashboard.yml
249
245
  - lib/funktor/cli/templates/funktor_config/resources/incoming_job_queue.yml
250
246
  - lib/funktor/cli/templates/funktor_config/resources/incoming_job_queue_user.yml
data/funktor-testapp/funktor_config/iam_permissions/activity_table.yml DELETED
@@ -1,5 +0,0 @@
1
- Effect: Allow
2
- Action:
3
- - dynamodb:*
4
- Resource:
5
- - "Fn::GetAtt": [ ActivityTable, Arn ]
data/funktor-testapp/funktor_config/resources/activity_table.yml DELETED
@@ -1,22 +0,0 @@
1
- Resources:
2
- ActivityTable:
3
- Type: AWS::DynamoDB::Table
4
- Properties:
5
- # Generate a name based on the stage
6
- TableName: ${self:custom.funktor.ActivityTableName}
7
- BillingMode: PAY_PER_REQUEST
8
- AttributeDefinitions:
9
- - AttributeName: category
10
- AttributeType: S
11
- - AttributeName: statName
12
- AttributeType: S
13
- KeySchema:
14
- - AttributeName: category
15
- KeyType: HASH
16
- - AttributeName: statName
17
- KeyType: RANGE
18
-
19
- Outputs:
20
- ActivityTable:
21
- Value:
22
- Ref: ActivityTable
data/lib/funktor/cli/templates/funktor_config/iam_permissions/activity_table.yml DELETED
@@ -1,5 +0,0 @@
1
- Effect: Allow
2
- Action:
3
- - dynamodb:*
4
- Resource:
5
- - "Fn::GetAtt": [ ActivityTable, Arn ]
data/lib/funktor/cli/templates/funktor_config/resources/activity_table.yml DELETED
@@ -1,22 +0,0 @@
1
- Resources:
2
- ActivityTable:
3
- Type: AWS::DynamoDB::Table
4
- Properties:
5
- # Generate a name based on the stage
6
- TableName: ${self:custom.funktor.ActivityTableName}
7
- BillingMode: PAY_PER_REQUEST
8
- AttributeDefinitions:
9
- - AttributeName: category
10
- AttributeType: S
11
- - AttributeName: statName
12
- AttributeType: S
13
- KeySchema:
14
- - AttributeName: category
15
- KeyType: HASH
16
- - AttributeName: statName
17
- KeyType: RANGE
18
-
19
- Outputs:
20
- ActivityTable:
21
- Value:
22
- Ref: ActivityTable