funktor 0.6.5 → 0.7.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: c92c5b6465ea858b39776c939ca3ac42e4680b19bae5071660e212b80ad5fa23
4
- data.tar.gz: 54b4a61d1ae01a7e06a1ce3bd027f2aaf55faccda40e4e1789d698b60ff4a57d
3
+ metadata.gz: 58967787594138d8c3e24feb2cb0cf5c742899d494099e972a505de9a49e8bc2
4
+ data.tar.gz: bae55701018e66b63cacbd5d7911cdd5fa1f063969716f6726be423ffaee896b
5
5
  SHA512:
6
- metadata.gz: fb515d5ed863cf279cee8ebcafa1ee352b0a470146c0795ed7d51d8b11d83cf0bdb577d90f0abdc5155d81a3d2446058401f2d138cf2a92ac283c54f3236bf34
7
- data.tar.gz: 92add3f0a131296bd1e11d3cc9c87b97e257647d6ef2ac54e00c28bbc2aaa98961672128318cab465d7d5553094b931bf8f6078bb985530d6f41cf1c73fc6299
6
+ metadata.gz: 2251b6acccc727761a109cee8cf847000c4857726441cc01dc04cb6649f53b272eb0d0f335494d31a1089cfd335988073a872dd660295dc55e6e4c07e9bf813a
7
+ data.tar.gz: b785cc6ba1db1c2e4e3af54b0a1c40045bb0f1d9cf2456edd82be1c1b370fa23edd71bc36edc5f8dfd9cf08990dcf490b5ec20ebd855d2da5610e48d7737e174
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- funktor (0.6.5)
4
+ funktor (0.7.3)
5
5
  activesupport
6
6
  aws-sdk-dynamodb (~> 1.62)
7
7
  aws-sdk-sqs (~> 1.37)
@@ -21,20 +21,20 @@ GEM
21
21
  zeitwerk (~> 2.3)
22
22
  addressable (2.8.0)
23
23
  public_suffix (>= 2.0.2, < 5.0)
24
- aws-eventstream (1.1.1)
25
- aws-partitions (1.491.0)
26
- aws-sdk-core (3.119.1)
24
+ aws-eventstream (1.2.0)
25
+ aws-partitions (1.494.0)
26
+ aws-sdk-core (3.120.0)
27
27
  aws-eventstream (~> 1, >= 1.0.2)
28
28
  aws-partitions (~> 1, >= 1.239.0)
29
29
  aws-sigv4 (~> 1.1)
30
30
  jmespath (~> 1.0)
31
- aws-sdk-dynamodb (1.62.0)
32
- aws-sdk-core (~> 3, >= 3.119.0)
31
+ aws-sdk-dynamodb (1.63.0)
32
+ aws-sdk-core (~> 3, >= 3.120.0)
33
33
  aws-sigv4 (~> 1.1)
34
- aws-sdk-sqs (1.42.0)
35
- aws-sdk-core (~> 3, >= 3.119.0)
34
+ aws-sdk-sqs (1.44.0)
35
+ aws-sdk-core (~> 3, >= 3.120.0)
36
36
  aws-sigv4 (~> 1.1)
37
- aws-sigv4 (1.2.4)
37
+ aws-sigv4 (1.3.0)
38
38
  aws-eventstream (~> 1, >= 1.0.2)
39
39
  byebug (11.1.3)
40
40
  coderay (1.1.3)
@@ -9,7 +9,5 @@ FUNKTOR_DEFAULT_QUEUE:
9
9
  Ref: DefaultQueue
10
10
  FUNKTOR_LOW_CONCURRENCY_QUEUE:
11
11
  Ref: LowConcurrencyQueue
12
- FUNKTOR_ACTIVITY_TABLE:
13
- Ref: ActivityTable
14
12
  FUNKTOR_JOBS_TABLE:
15
13
  Ref: JobsTable
@@ -12,6 +12,19 @@ Resources:
12
12
  - sqs:*
13
13
  Resource:
14
14
  - "Fn::GetAtt": [ IncomingJobQueue, Arn ]
15
+ - Effect: Allow
16
+ Action:
17
+ - dynamodb:*
18
+ Resource:
19
+ - "Fn::GetAtt": [ JobsTable, Arn ]
20
+ - Effect: Allow
21
+ Action:
22
+ - dynamodb:Query
23
+ Resource:
24
+ Fn::Join:
25
+ - ""
26
+ - - "Fn::GetAtt": [ JobsTable, Arn ]
27
+ - "/index/categoryIndex"
15
28
 
16
29
  IncomingJobQueueUserAccessKey:
17
30
  Type: AWS::IAM::AccessKey
@@ -5,7 +5,7 @@ Resources:
5
5
  TableName: ${self:custom.funktor.JobsTableName}
6
6
  AttributeDefinitions:
7
7
  - AttributeName: jobShard
8
- AttributeType: N
8
+ AttributeType: S
9
9
  - AttributeName: jobId
10
10
  AttributeType: S
11
11
  - AttributeName: category
@@ -23,12 +23,12 @@ frameworkVersion: '2'
23
23
  provider:
24
24
  name: aws
25
25
  runtime: ruby2.7
26
+ stage: ${opt:stage, 'dev'}
26
27
  lambdaHashingVersion: 20201221
27
28
  environment: ${file(funktor_config/environment.yml)}
28
29
  versionFunctions: false # Reduces the amount of storage used since all Lambdas together are limited to 75GB
29
30
  logRetentionInDays: 7
30
31
  iamRoleStatements:
31
- - ${file(funktor_config/iam_permissions/activity_table.yml)}
32
32
  - ${file(funktor_config/iam_permissions/default_queue.yml)}
33
33
  - ${file(funktor_config/iam_permissions/incoming_job_queue.yml)}
34
34
  - ${file(funktor_config/iam_permissions/ssm.yml)}
@@ -54,7 +54,6 @@ functions:
54
54
  RandomJobGenerator: ${file(funktor_config/function_definitions/random_job_generator.yml)}
55
55
 
56
56
  resources:
57
- - ${file(funktor_config/resources/activity_table.yml)}
58
57
  - ${file(funktor_config/resources/incoming_job_queue_user.yml)}
59
58
  - ${file(funktor_config/resources/cloudwatch_dashboard.yml)}
60
59
  - ${file(funktor_config/resources/default_queue.yml)}
@@ -56,8 +56,8 @@ module Funktor
56
56
  def increment_key(key)
57
57
  #put_metric_to_stdout(key, 1)
58
58
  dynamodb_client.update_item({
59
- table_name: ENV['FUNKTOR_ACTIVITY_TABLE'],
60
- key: { category: 'stat', statName: key },
59
+ table_name: ENV['FUNKTOR_JOBS_TABLE'],
60
+ key: { jobShard: 'stat', jobId: key },
61
61
  expression_attribute_values:{ ":start": 0, ":inc": 1 },
62
62
  update_expression: "SET stat_value = if_not_exists(stat_value, :start) + :inc",
63
63
  })
@@ -66,8 +66,8 @@ module Funktor
66
66
  def decrement_key(key)
67
67
  #put_metric_to_stdout(key, -1)
68
68
  dynamodb_client.update_item({
69
- table_name: ENV['FUNKTOR_ACTIVITY_TABLE'],
70
- key: { category: 'stat', statName: key },
69
+ table_name: ENV['FUNKTOR_JOBS_TABLE'],
70
+ key: { jobShard: 'stat', jobId: key },
71
71
  expression_attribute_values:{ ":start": 0, ":inc": 1 },
72
72
  update_expression: "SET stat_value = if_not_exists(stat_value, :start) - :inc",
73
73
  })
@@ -40,15 +40,44 @@ module Funktor
40
40
  end
41
41
 
42
42
  def package_json
43
- template "package.json", File.join("package.json")
43
+ if File.exist?("package.json")
44
+ package_data = File.open("package.json").read
45
+ if package_data =~ /serverless-ruby-layer/
46
+ say "serverless-ruby-layer is already installed in package.json"
47
+ else
48
+ if File.exist?("package-lock.json")
49
+ run "npm install serverless-ruby-layer@1.4.0"
50
+ # TODO - Add handers for yarn and what not
51
+ else
52
+ say "You should install serverless-ruby-layer version 1.4.0 using yor package manager of choice."
53
+ end
54
+ end
55
+ else
56
+ template "package.json", File.join("package.json")
57
+ end
44
58
  end
45
59
 
46
60
  def gemfile
47
- template "Gemfile", File.join("Gemfile")
61
+ if File.exist?("Gemfile")
62
+ gem_data = File.open("Gemfile").read
63
+ if gem_data =~ /funktor/
64
+ say "funktor is already installed in Gemfile"
65
+ else
66
+ run "bundle add funktor"
67
+ end
68
+ else
69
+ template "Gemfile", File.join("Gemfile")
70
+ end
48
71
  end
49
72
 
50
73
  def gitignore
51
- template "gitignore", File.join(".gitignore")
74
+ # TODO clean this up so that we have a single source of truth for .gitignor stuff
75
+ if File.exist?(".gitignore")
76
+ append_to_file ".gitignore", ".serverless"
77
+ append_to_file ".gitignore", "node_modules"
78
+ else
79
+ template "gitignore", File.join(".gitignore")
80
+ end
52
81
  end
53
82
 
54
83
  def workers
@@ -66,7 +95,6 @@ module Funktor
66
95
  template File.join("funktor_config", "resources", "work_queue.yml"), File.join("funktor_config", "resources", "#{work_queue_name.underscore}_queue.yml")
67
96
  end
68
97
  template File.join("funktor_config", "resources", "jobs_table.yml"), File.join("funktor_config", "resources", "jobs_table.yml")
69
- template File.join("funktor_config", "resources", "activity_table.yml"), File.join("funktor_config", "resources", "activity_table.yml")
70
98
  end
71
99
 
72
100
  def iam_permissions
@@ -79,7 +107,6 @@ module Funktor
79
107
  end
80
108
  template File.join("funktor_config", "iam_permissions", "jobs_table.yml"), File.join("funktor_config", "iam_permissions", "jobs_table.yml")
81
109
  template File.join("funktor_config", "iam_permissions", "jobs_table_secondary_index.yml"), File.join("funktor_config", "iam_permissions", "jobs_table_secondary_index.yml")
82
- template File.join("funktor_config", "iam_permissions", "activity_table.yml"), File.join("funktor_config", "iam_permissions", "activity_table.yml")
83
110
  end
84
111
 
85
112
  def function_definitions
@@ -9,7 +9,5 @@ FUNKTOR_INCOMING_JOB_QUEUE:
9
9
  FUNKTOR_<%= queue_name.underscore.upcase %>_QUEUE:
10
10
  Ref: <%= queue_name.camelize %>Queue
11
11
  <%- end -%>
12
- FUNKTOR_ACTIVITY_TABLE:
13
- Ref: ActivityTable
14
12
  FUNKTOR_JOBS_TABLE:
15
13
  Ref: JobsTable
@@ -12,6 +12,19 @@ Resources:
12
12
  - sqs:*
13
13
  Resource:
14
14
  - "Fn::GetAtt": [ IncomingJobQueue, Arn ]
15
+ - Effect: Allow
16
+ Action:
17
+ - dynamodb:*
18
+ Resource:
19
+ - "Fn::GetAtt": [ JobsTable, Arn ]
20
+ - Effect: Allow
21
+ Action:
22
+ - dynamodb:Query
23
+ Resource:
24
+ Fn::Join:
25
+ - ""
26
+ - - "Fn::GetAtt": [ JobsTable, Arn ]
27
+ - "/index/categoryIndex"
15
28
 
16
29
  IncomingJobQueueUserAccessKey:
17
30
  Type: AWS::IAM::AccessKey
@@ -5,7 +5,7 @@ Resources:
5
5
  TableName: ${self:custom.funktor.JobsTableName}
6
6
  AttributeDefinitions:
7
7
  - AttributeName: jobShard
8
- AttributeType: N
8
+ AttributeType: S
9
9
  - AttributeName: jobId
10
10
  AttributeType: S
11
11
  - AttributeName: category
@@ -1,8 +1,8 @@
1
1
  {
2
- "name":"<%= name %>",
3
- "description":"",
4
- "version":"0.1.0",
5
- "devDependencies":{
6
- "serverless-ruby-layer":"^1.4.0"
2
+ "name": "<%= name %>",
3
+ "description": "",
4
+ "version": "0.1.0",
5
+ "devDependencies": {
6
+ "serverless-ruby-layer": "^1.4.0"
7
7
  }
8
8
  }
@@ -23,6 +23,7 @@ frameworkVersion: '2'
23
23
  provider:
24
24
  name: aws
25
25
  runtime: <%= runtime %>
26
+ stage: ${opt:stage, 'dev'}
26
27
  lambdaHashingVersion: 20201221
27
28
  environment: ${file(funktor_config/environment.yml)}
28
29
  versionFunctions: false # Reduces the amount of storage used since all Lambdas together are limited to 75GB
@@ -1,6 +1,6 @@
1
1
  module ShardUtils
2
2
  def calculate_shard(job_id)
3
3
  # TODO - Should the number of shards be configurable?
4
- job_id.sum % 64
4
+ (job_id.sum % 64).to_s
5
5
  end
6
6
  end
@@ -1,3 +1,3 @@
1
1
  module Funktor
2
- VERSION = "0.6.5"
2
+ VERSION = "0.7.3"
3
3
  end
@@ -80,16 +80,16 @@ module Funktor
80
80
  def get_activity_data
81
81
  query_params = {
82
82
  expression_attribute_values: {
83
- ":category" => "stat"
83
+ ":jobShard" => "stat"
84
84
  },
85
- key_condition_expression: "category = :category",
86
- projection_expression: "statName, stat_value",
87
- table_name: ENV['FUNKTOR_ACTIVITY_TABLE']
85
+ key_condition_expression: "jobShard = :jobShard",
86
+ projection_expression: "jobId, stat_value",
87
+ table_name: ENV['FUNKTOR_JOBS_TABLE']
88
88
  }
89
89
  resp = dynamodb_client.query(query_params)
90
90
  @activity_stats = {}
91
91
  resp.items.each do |item|
92
- @activity_stats[item["statName"]] = item["stat_value"].to_i
92
+ @activity_stats[item["jobId"]] = item["stat_value"].to_i
93
93
  end
94
94
  return @activity_stats
95
95
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: funktor
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.5
4
+ version: 0.7.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jeremy Green
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2021-08-25 00:00:00.000000000 Z
11
+ date: 2021-09-02 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: aws-sdk-sqs
@@ -192,7 +192,6 @@ files:
192
192
  - funktor-testapp/funktor_config/function_definitions/low_concurrency_queue_handler.yml
193
193
  - funktor-testapp/funktor_config/function_definitions/random_job_generator.yml
194
194
  - funktor-testapp/funktor_config/funktor.yml
195
- - funktor-testapp/funktor_config/iam_permissions/activity_table.yml
196
195
  - funktor-testapp/funktor_config/iam_permissions/default_queue.yml
197
196
  - funktor-testapp/funktor_config/iam_permissions/incoming_job_queue.yml
198
197
  - funktor-testapp/funktor_config/iam_permissions/jobs_table.yml
@@ -200,7 +199,6 @@ files:
200
199
  - funktor-testapp/funktor_config/iam_permissions/low_concurrency_queue.yml
201
200
  - funktor-testapp/funktor_config/iam_permissions/ssm.yml
202
201
  - funktor-testapp/funktor_config/package.yml
203
- - funktor-testapp/funktor_config/resources/activity_table.yml
204
202
  - funktor-testapp/funktor_config/resources/cloudwatch_dashboard.yml
205
203
  - funktor-testapp/funktor_config/resources/default_queue.yml
206
204
  - funktor-testapp/funktor_config/resources/incoming_job_queue.yml
@@ -237,14 +235,12 @@ files:
237
235
  - lib/funktor/cli/templates/funktor_config/function_definitions/job_activator.yml
238
236
  - lib/funktor/cli/templates/funktor_config/function_definitions/work_queue_handler.yml
239
237
  - lib/funktor/cli/templates/funktor_config/funktor.yml
240
- - lib/funktor/cli/templates/funktor_config/iam_permissions/activity_table.yml
241
238
  - lib/funktor/cli/templates/funktor_config/iam_permissions/incoming_job_queue.yml
242
239
  - lib/funktor/cli/templates/funktor_config/iam_permissions/jobs_table.yml
243
240
  - lib/funktor/cli/templates/funktor_config/iam_permissions/jobs_table_secondary_index.yml
244
241
  - lib/funktor/cli/templates/funktor_config/iam_permissions/ssm.yml
245
242
  - lib/funktor/cli/templates/funktor_config/iam_permissions/work_queue.yml
246
243
  - lib/funktor/cli/templates/funktor_config/package.yml
247
- - lib/funktor/cli/templates/funktor_config/resources/activity_table.yml
248
244
  - lib/funktor/cli/templates/funktor_config/resources/cloudwatch_dashboard.yml
249
245
  - lib/funktor/cli/templates/funktor_config/resources/incoming_job_queue.yml
250
246
  - lib/funktor/cli/templates/funktor_config/resources/incoming_job_queue_user.yml
@@ -1,5 +0,0 @@
1
- Effect: Allow
2
- Action:
3
- - dynamodb:*
4
- Resource:
5
- - "Fn::GetAtt": [ ActivityTable, Arn ]
@@ -1,22 +0,0 @@
1
- Resources:
2
- ActivityTable:
3
- Type: AWS::DynamoDB::Table
4
- Properties:
5
- # Generate a name based on the stage
6
- TableName: ${self:custom.funktor.ActivityTableName}
7
- BillingMode: PAY_PER_REQUEST
8
- AttributeDefinitions:
9
- - AttributeName: category
10
- AttributeType: S
11
- - AttributeName: statName
12
- AttributeType: S
13
- KeySchema:
14
- - AttributeName: category
15
- KeyType: HASH
16
- - AttributeName: statName
17
- KeyType: RANGE
18
-
19
- Outputs:
20
- ActivityTable:
21
- Value:
22
- Ref: ActivityTable
@@ -1,5 +0,0 @@
1
- Effect: Allow
2
- Action:
3
- - dynamodb:*
4
- Resource:
5
- - "Fn::GetAtt": [ ActivityTable, Arn ]
@@ -1,22 +0,0 @@
1
- Resources:
2
- ActivityTable:
3
- Type: AWS::DynamoDB::Table
4
- Properties:
5
- # Generate a name based on the stage
6
- TableName: ${self:custom.funktor.ActivityTableName}
7
- BillingMode: PAY_PER_REQUEST
8
- AttributeDefinitions:
9
- - AttributeName: category
10
- AttributeType: S
11
- - AttributeName: statName
12
- AttributeType: S
13
- KeySchema:
14
- - AttributeName: category
15
- KeyType: HASH
16
- - AttributeName: statName
17
- KeyType: RANGE
18
-
19
- Outputs:
20
- ActivityTable:
21
- Value:
22
- Ref: ActivityTable