exercism-config 0.63.0 → 0.64.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/rubocop.yml +2 -2
- data/.github/workflows/tests.yml +13 -2
- data/bin/setup_exercism_local_aws +0 -113
- data/exercism_config.gemspec +1 -0
- data/lib/exercism-config.rb +5 -0
- data/lib/exercism/tooling_job.rb +147 -0
- data/lib/exercism_config/version.rb +1 -1
- data/settings/ci.yml +3 -0
- data/settings/docker.yml +3 -0
- data/settings/local.yml +3 -0
- metadata +17 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4a0fe18eb17f6624dc4464ae5e56e9196d7d6acbf8c230ca80546431508a875f
|
4
|
+
data.tar.gz: 10af19438e32b6730ea3aa209f1de089519b467ab47e55c231c7af87808614b0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 32cb6508a29b336952bd3758bc4547efe489256353d618b8089901d6905c9124d82649fb08c3d0b79bae7e31d884fda087aab831d2675c16003d13e688c003a4
|
7
|
+
data.tar.gz: 1e2d16f13a96ee928766fa3950f57dba0f6100b29adc329de7a0a367898da36a8984005ab457f9ed77010721d1334fe8c0441ee1ed150286e76c8dde2e551316
|
data/.github/workflows/tests.yml
CHANGED
@@ -2,9 +2,9 @@ name: Tests
|
|
2
2
|
|
3
3
|
on:
|
4
4
|
push:
|
5
|
-
branches: [
|
5
|
+
branches: [main]
|
6
6
|
pull_request:
|
7
|
-
branches: [
|
7
|
+
branches: [main]
|
8
8
|
|
9
9
|
jobs:
|
10
10
|
test:
|
@@ -16,6 +16,16 @@ jobs:
|
|
16
16
|
ports:
|
17
17
|
- 4566
|
18
18
|
|
19
|
+
redis:
|
20
|
+
image: redis
|
21
|
+
ports:
|
22
|
+
- 6379:6379
|
23
|
+
options: >-
|
24
|
+
--health-cmd "redis-cli ping"
|
25
|
+
--health-interval 1s
|
26
|
+
--health-timeout 2s
|
27
|
+
--health-retries 10
|
28
|
+
|
19
29
|
steps:
|
20
30
|
###
|
21
31
|
# Checkout using GitHub's checkout action
|
@@ -38,6 +48,7 @@ jobs:
|
|
38
48
|
AWS_PORT: ${{ job.services.aws.ports['4566'] }}
|
39
49
|
run: |
|
40
50
|
bundle exec setup_exercism_config
|
51
|
+
bundle exec setup_exercism_local_aws
|
41
52
|
|
42
53
|
###
|
43
54
|
# Setup code climate
|
@@ -27,119 +27,6 @@ unless ENV["EXERCISM_SKIP_S3"]
|
|
27
27
|
end
|
28
28
|
end
|
29
29
|
|
30
|
-
###########################################
|
31
|
-
# Setup local dynamodb tooling jobs table #
|
32
|
-
###########################################
|
33
|
-
[
|
34
|
-
Exercism.config.dynamodb_tooling_jobs_table,
|
35
|
-
Exercism.config.dynamodb_tooling_jobs_table + "-test"
|
36
|
-
].each do |table_name|
|
37
|
-
begin
|
38
|
-
Exercism.dynamodb_client.delete_table(
|
39
|
-
table_name: table_name
|
40
|
-
)
|
41
|
-
rescue Aws::DynamoDB::Errors::ResourceNotFoundException
|
42
|
-
end
|
43
|
-
puts "[x] #{table_name}"
|
44
|
-
|
45
|
-
Exercism.dynamodb_client.create_table(
|
46
|
-
table_name: table_name,
|
47
|
-
attribute_definitions: [
|
48
|
-
{
|
49
|
-
attribute_name: "id",
|
50
|
-
attribute_type: "S"
|
51
|
-
}
|
52
|
-
],
|
53
|
-
key_schema: [
|
54
|
-
{
|
55
|
-
attribute_name: "id",
|
56
|
-
key_type: "HASH"
|
57
|
-
}
|
58
|
-
],
|
59
|
-
provisioned_throughput: {
|
60
|
-
read_capacity_units: 1,
|
61
|
-
write_capacity_units: 1
|
62
|
-
}
|
63
|
-
)
|
64
|
-
|
65
|
-
Exercism.dynamodb_client.update_table(
|
66
|
-
table_name: table_name,
|
67
|
-
attribute_definitions: [
|
68
|
-
{
|
69
|
-
attribute_name: "job_status",
|
70
|
-
attribute_type: "S"
|
71
|
-
},
|
72
|
-
{
|
73
|
-
attribute_name: "created_at",
|
74
|
-
attribute_type: "N"
|
75
|
-
}
|
76
|
-
],
|
77
|
-
global_secondary_index_updates: [
|
78
|
-
{
|
79
|
-
create: {
|
80
|
-
index_name: "job_status", # required
|
81
|
-
key_schema: [ # required
|
82
|
-
{
|
83
|
-
attribute_name: "job_status", # required
|
84
|
-
key_type: "HASH" # required, accepts HASH, RANGE
|
85
|
-
},
|
86
|
-
{
|
87
|
-
attribute_name: "created_at", # required
|
88
|
-
key_type: "RANGE" # required, accepts HASH, RANGE
|
89
|
-
}
|
90
|
-
],
|
91
|
-
projection: { # required
|
92
|
-
projection_type: "KEYS_ONLY"
|
93
|
-
},
|
94
|
-
provisioned_throughput: {
|
95
|
-
read_capacity_units: 1, # required
|
96
|
-
write_capacity_units: 1 # required
|
97
|
-
}
|
98
|
-
}
|
99
|
-
}
|
100
|
-
]
|
101
|
-
)
|
102
|
-
|
103
|
-
Exercism.dynamodb_client.update_table(
|
104
|
-
table_name: table_name,
|
105
|
-
attribute_definitions: [
|
106
|
-
{
|
107
|
-
attribute_name: "submission_uuid",
|
108
|
-
attribute_type: "S"
|
109
|
-
},
|
110
|
-
{
|
111
|
-
attribute_name: "type",
|
112
|
-
attribute_type: "S"
|
113
|
-
}
|
114
|
-
],
|
115
|
-
global_secondary_index_updates: [
|
116
|
-
{
|
117
|
-
create: {
|
118
|
-
index_name: "submission_type", # required
|
119
|
-
key_schema: [ # required
|
120
|
-
{
|
121
|
-
attribute_name: "submission_uuid", # required
|
122
|
-
key_type: "HASH" # required, accepts HASH, RANGE
|
123
|
-
},
|
124
|
-
{
|
125
|
-
attribute_name: "type", # required
|
126
|
-
key_type: "RANGE" # required, accepts HASH, RANGE
|
127
|
-
}
|
128
|
-
],
|
129
|
-
projection: { # required
|
130
|
-
projection_type: "INCLUDE",
|
131
|
-
non_key_attributes: ["id", "job_status"]
|
132
|
-
},
|
133
|
-
provisioned_throughput: {
|
134
|
-
read_capacity_units: 1, # required
|
135
|
-
write_capacity_units: 1 # required
|
136
|
-
}
|
137
|
-
}
|
138
|
-
}
|
139
|
-
]
|
140
|
-
)
|
141
|
-
end
|
142
|
-
|
143
30
|
##############################################
|
144
31
|
# Setup local dynamodb language groups table #
|
145
32
|
##############################################
|
data/exercism_config.gemspec
CHANGED
data/lib/exercism-config.rb
CHANGED
@@ -16,6 +16,7 @@ require_relative 'exercism_config/retrieve_secrets'
|
|
16
16
|
require_relative 'exercism_config/version'
|
17
17
|
require_relative 'exercism/config'
|
18
18
|
require_relative 'exercism/secrets'
|
19
|
+
require_relative 'exercism/tooling_job'
|
19
20
|
|
20
21
|
module Exercism
|
21
22
|
class ConfigError < RuntimeError; end
|
@@ -32,6 +33,10 @@ module Exercism
|
|
32
33
|
@secrets ||= ExercismConfig::RetrieveSecrets.()
|
33
34
|
end
|
34
35
|
|
36
|
+
def self.redis_tooling_client
|
37
|
+
Redis.new(url: config.tooling_redis_url)
|
38
|
+
end
|
39
|
+
|
35
40
|
def self.dynamodb_client
|
36
41
|
Aws::DynamoDB::Client.new(ExercismConfig::GenerateAwsSettings.())
|
37
42
|
end
|
@@ -0,0 +1,147 @@
|
|
1
|
+
module Exercism
|
2
|
+
class ToolingJob
|
3
|
+
require 'aws-sdk-s3'
|
4
|
+
require 'redis'
|
5
|
+
|
6
|
+
extend Mandate::Memoize
|
7
|
+
|
8
|
+
def self.create!(type, submission_uuid, language, exercise, attributes = {})
|
9
|
+
job_id = SecureRandom.uuid.tr('-', '')
|
10
|
+
redis = Exercism.redis_tooling_client
|
11
|
+
redis.multi do
|
12
|
+
redis.set(
|
13
|
+
"job:#{job_id}",
|
14
|
+
attributes.merge(
|
15
|
+
id: job_id,
|
16
|
+
submission_uuid: submission_uuid,
|
17
|
+
type: type,
|
18
|
+
language: language,
|
19
|
+
exercise: exercise,
|
20
|
+
created_at: Time.now.utc.to_i
|
21
|
+
).to_json
|
22
|
+
)
|
23
|
+
redis.rpush(key_for_queued, job_id)
|
24
|
+
redis.set("submission:#{submission_uuid}:#{type}", job_id)
|
25
|
+
end
|
26
|
+
job_id
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.find(id)
|
30
|
+
json = Exercism.redis_tooling_client.get("job:#{id}")
|
31
|
+
new(id, JSON.parse(json))
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.find_for_submission_uuid_and_type(submission_uuid, type)
|
35
|
+
redis = Exercism.redis_tooling_client
|
36
|
+
job_id = redis.get("submission:#{submission_uuid}:#{type}")
|
37
|
+
json = redis.get("job:#{job_id}")
|
38
|
+
new(job_id, JSON.parse(json))
|
39
|
+
end
|
40
|
+
|
41
|
+
attr_reader :id, :data
|
42
|
+
|
43
|
+
def initialize(id, data)
|
44
|
+
@id = id
|
45
|
+
@data = data.freeze
|
46
|
+
end
|
47
|
+
|
48
|
+
def to_h
|
49
|
+
data.to_h
|
50
|
+
end
|
51
|
+
|
52
|
+
def respond_to_missing?(meth)
|
53
|
+
data.key?(meth.to_s) || super
|
54
|
+
end
|
55
|
+
|
56
|
+
def method_missing(meth)
|
57
|
+
super unless respond_to_missing?(meth)
|
58
|
+
|
59
|
+
data[meth.to_s]
|
60
|
+
end
|
61
|
+
|
62
|
+
def locked!
|
63
|
+
redis = Exercism.redis_tooling_client
|
64
|
+
redis.multi do
|
65
|
+
redis.lrem(key_for_queued, 1, id)
|
66
|
+
redis.rpush(key_for_locked, id)
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def executed!(status, output)
|
71
|
+
redis = Exercism.redis_tooling_client
|
72
|
+
redis.multi do
|
73
|
+
redis.lrem(key_for_queued, 1, id)
|
74
|
+
redis.lrem(key_for_locked, 1, id)
|
75
|
+
redis.rpush(key_for_executed, id)
|
76
|
+
|
77
|
+
redis.set(
|
78
|
+
"job:#{id}",
|
79
|
+
data.merge(
|
80
|
+
execution_status: status,
|
81
|
+
execution_output: output
|
82
|
+
).to_json
|
83
|
+
)
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
def processed!
|
88
|
+
redis = Exercism.redis_tooling_client
|
89
|
+
redis.multi do
|
90
|
+
redis.lrem(key_for_executed, 1, id)
|
91
|
+
redis.rpush(key_for_processed, id)
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
def cancelled!
|
96
|
+
redis = Exercism.redis_tooling_client
|
97
|
+
redis.multi do
|
98
|
+
redis.lrem(key_for_queued, 1, id)
|
99
|
+
redis.rpush(key_for_cancelled, id)
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
def ==(other)
|
104
|
+
id == other.id
|
105
|
+
end
|
106
|
+
|
107
|
+
def stderr
|
108
|
+
read_s3_file('stderr')
|
109
|
+
end
|
110
|
+
|
111
|
+
def stdout
|
112
|
+
read_s3_file('stdout')
|
113
|
+
end
|
114
|
+
|
115
|
+
private
|
116
|
+
def read_s3_file(name)
|
117
|
+
Exercism.s3_client.get_object(
|
118
|
+
bucket: s3_bucket_name,
|
119
|
+
key: "#{s3_folder}/#{name}"
|
120
|
+
).body.read
|
121
|
+
rescue StandardError
|
122
|
+
""
|
123
|
+
end
|
124
|
+
|
125
|
+
memoize
|
126
|
+
def s3_folder
|
127
|
+
"#{Exercism.env}/#{id}"
|
128
|
+
end
|
129
|
+
|
130
|
+
memoize
|
131
|
+
def s3_bucket_name
|
132
|
+
Exercism.config.aws_tooling_jobs_bucket
|
133
|
+
end
|
134
|
+
|
135
|
+
%w[queued locked executed processed cancelled].each do |key|
|
136
|
+
ToolingJob.singleton_class.class_eval do
|
137
|
+
define_method "key_for_#{key}" do
|
138
|
+
Exercism.env.production? ? key : "#{Exercism.env}:#{key}"
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
define_method "key_for_#{key}" do
|
143
|
+
self.class.send("key_for_#{key}")
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
147
|
+
end
|
data/settings/ci.yml
CHANGED
@@ -2,6 +2,9 @@
|
|
2
2
|
anycable_redis_url: redis://127.0.0.1:6379/1
|
3
3
|
anycable_rpc_host: 127.0.0.1:50051
|
4
4
|
|
5
|
+
# Tooling
|
6
|
+
tooling_redis_url: redis://127.0.0.1:6379/3
|
7
|
+
|
5
8
|
# DynamoDB config
|
6
9
|
dynamodb_tooling_jobs_table: tooling_jobs
|
7
10
|
dynamodb_tooling_language_groups_table: tooling_language_groups
|
data/settings/docker.yml
CHANGED
@@ -2,6 +2,9 @@
|
|
2
2
|
anycable_redis_url: redis://redis:6379/1
|
3
3
|
anycable_rpc_host: 0.0.0.0:50051
|
4
4
|
|
5
|
+
# Tooling
|
6
|
+
tooling_redis_url: redis://redis:6379/3
|
7
|
+
|
5
8
|
# DynamoDB config
|
6
9
|
dynamodb_tooling_jobs_table: tooling_jobs
|
7
10
|
dynamodb_tooling_language_groups_table: tooling_language_groups
|
data/settings/local.yml
CHANGED
@@ -2,6 +2,9 @@
|
|
2
2
|
anycable_redis_url: redis://127.0.0.1:6379/1
|
3
3
|
anycable_rpc_host: 127.0.0.1:50051
|
4
4
|
|
5
|
+
# Tooling
|
6
|
+
tooling_redis_url: redis://127.0.0.1:6379/3
|
7
|
+
|
5
8
|
# DynamoDB config
|
6
9
|
dynamodb_tooling_jobs_table: tooling_jobs
|
7
10
|
dynamodb_tooling_language_groups_table: tooling_language_groups
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: exercism-config
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.64.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jeremy Walker
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-05-
|
11
|
+
date: 2021-05-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: aws-sdk-dynamodb
|
@@ -150,6 +150,20 @@ dependencies:
|
|
150
150
|
- - ">="
|
151
151
|
- !ruby/object:Gem::Version
|
152
152
|
version: '0'
|
153
|
+
- !ruby/object:Gem::Dependency
|
154
|
+
name: redis
|
155
|
+
requirement: !ruby/object:Gem::Requirement
|
156
|
+
requirements:
|
157
|
+
- - ">="
|
158
|
+
- !ruby/object:Gem::Version
|
159
|
+
version: '0'
|
160
|
+
type: :development
|
161
|
+
prerelease: false
|
162
|
+
version_requirements: !ruby/object:Gem::Requirement
|
163
|
+
requirements:
|
164
|
+
- - ">="
|
165
|
+
- !ruby/object:Gem::Version
|
166
|
+
version: '0'
|
153
167
|
description:
|
154
168
|
email:
|
155
169
|
- jez.walker@gmail.com
|
@@ -177,6 +191,7 @@ files:
|
|
177
191
|
- lib/exercism-config.rb
|
178
192
|
- lib/exercism/config.rb
|
179
193
|
- lib/exercism/secrets.rb
|
194
|
+
- lib/exercism/tooling_job.rb
|
180
195
|
- lib/exercism_config/determine_environment.rb
|
181
196
|
- lib/exercism_config/environment.rb
|
182
197
|
- lib/exercism_config/generate_aws_settings.rb
|