exercism-config 0.63.0 → 0.64.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2885e2be67c5b8c19538402917bf67b22ffa283e54291ba2244f286ffc273332
-  data.tar.gz: 9b2489153000d0552976ec9fb9bd60657488ac63e20212d1fc89ea9f2924859d
+  metadata.gz: 4a0fe18eb17f6624dc4464ae5e56e9196d7d6acbf8c230ca80546431508a875f
+  data.tar.gz: 10af19438e32b6730ea3aa209f1de089519b467ab47e55c231c7af87808614b0
 SHA512:
-  metadata.gz: 96f62b62af1d81c90be6245335f82ef4323ceedfe5bbc495e552427744f03b7607372cc0cbe2155936c42935be8347d3b5b76b2132a3441d279b3bb1c6bfb9e9
-  data.tar.gz: 8caaf0d56a6be6fd139c4dfe075fd17e5d5929368482ae466f5b4c81c05f6d50bd29d3759f81111969d8e700dc83c701b77216c5a0d469e0b9811c136552ef66
+  metadata.gz: 32cb6508a29b336952bd3758bc4547efe489256353d618b8089901d6905c9124d82649fb08c3d0b79bae7e31d884fda087aab831d2675c16003d13e688c003a4
+  data.tar.gz: 1e2d16f13a96ee928766fa3950f57dba0f6100b29adc329de7a0a367898da36a8984005ab457f9ed77010721d1334fe8c0441ee1ed150286e76c8dde2e551316

data/.github/workflows/rubocop.yml

@@ -2,9 +2,9 @@ name: Rubocop
 
 on:
   push:
-    branches: [master]
+    branches: [main]
   pull_request:
-    branches: [master]
+    branches: [main]
 
 jobs:
   rubocop:

data/.github/workflows/tests.yml

@@ -2,9 +2,9 @@ name: Tests
 
 on:
   push:
-    branches: [master]
+    branches: [main]
   pull_request:
-    branches: [master]
+    branches: [main]
 
 jobs:
   test:
@@ -16,6 +16,16 @@ jobs:
         ports:
           - 4566
 
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 1s
+          --health-timeout 2s
+          --health-retries 10
+
     steps:
       ###
       # Checkout using GitHub's checkout action
@@ -38,6 +48,7 @@ jobs:
           AWS_PORT: ${{ job.services.aws.ports['4566'] }}
         run: |
           bundle exec setup_exercism_config
+          bundle exec setup_exercism_local_aws
 
       ###
       # Setup code climate

data/bin/setup_exercism_local_aws

@@ -27,119 +27,6 @@ unless ENV["EXERCISM_SKIP_S3"]
   end
 end
 
-###########################################
-# Setup local dynamodb tooling jobs table #
-###########################################
-[
-  Exercism.config.dynamodb_tooling_jobs_table,
-  Exercism.config.dynamodb_tooling_jobs_table + "-test"
-].each do |table_name|
-  begin
-    Exercism.dynamodb_client.delete_table(
-      table_name: table_name
-    )
-  rescue Aws::DynamoDB::Errors::ResourceNotFoundException
-  end
-  puts "[x] #{table_name}"
-
-  Exercism.dynamodb_client.create_table(
-    table_name: table_name,
-    attribute_definitions: [
-      {
-        attribute_name: "id",
-        attribute_type: "S"
-      }
-    ],
-    key_schema: [
-      {
-        attribute_name: "id",
-        key_type: "HASH"
-      }
-    ],
-    provisioned_throughput: {
-      read_capacity_units: 1,
-      write_capacity_units: 1
-    }
-  )
-
-  Exercism.dynamodb_client.update_table(
-    table_name: table_name,
-    attribute_definitions: [
-      {
-        attribute_name: "job_status",
-        attribute_type: "S"
-      },
-      {
-        attribute_name: "created_at",
-        attribute_type: "N"
-      }
-    ],
-    global_secondary_index_updates: [
-      {
-        create: {
-          index_name: "job_status", # required
-          key_schema: [ # required
-            {
-              attribute_name: "job_status", # required
-              key_type: "HASH" # required, accepts HASH, RANGE
-            },
-            {
-              attribute_name: "created_at", # required
-              key_type: "RANGE" # required, accepts HASH, RANGE
-            }
-          ],
-          projection: { # required
-            projection_type: "KEYS_ONLY"
-          },
-          provisioned_throughput: {
-            read_capacity_units: 1, # required
-            write_capacity_units: 1 # required
-          }
-        }
-      }
-    ]
-  )
-
-  Exercism.dynamodb_client.update_table(
-    table_name: table_name,
-    attribute_definitions: [
-      {
-        attribute_name: "submission_uuid",
-        attribute_type: "S"
-      },
-      {
-        attribute_name: "type",
-        attribute_type: "S"
-      }
-    ],
-    global_secondary_index_updates: [
-      {
-        create: {
-          index_name: "submission_type", # required
-          key_schema: [ # required
-            {
-              attribute_name: "submission_uuid", # required
-              key_type: "HASH" # required, accepts HASH, RANGE
-            },
-            {
-              attribute_name: "type", # required
-              key_type: "RANGE" # required, accepts HASH, RANGE
-            }
-          ],
-          projection: { # required
-            projection_type: "INCLUDE",
-            non_key_attributes: ["id", "job_status"]
-          },
-          provisioned_throughput: {
-            read_capacity_units: 1, # required
-            write_capacity_units: 1 # required
-          }
-        }
-      }
-    ]
-  )
-end
-
 ##############################################
 # Setup local dynamodb language groups table #
 ##############################################

data/exercism_config.gemspec

@@ -40,4 +40,5 @@ Gem::Specification.new do |spec|
   # own Gemfile when using this.
   spec.add_development_dependency 'aws-sdk-ecr'
   spec.add_development_dependency 'aws-sdk-s3'
+  spec.add_development_dependency 'redis'
 end

data/lib/exercism-config.rb

@@ -16,6 +16,7 @@ require_relative 'exercism_config/retrieve_secrets'
 require_relative 'exercism_config/version'
 require_relative 'exercism/config'
 require_relative 'exercism/secrets'
+require_relative 'exercism/tooling_job'
 
 module Exercism
   class ConfigError < RuntimeError; end
@@ -32,6 +33,10 @@ module Exercism
     @secrets ||= ExercismConfig::RetrieveSecrets.()
   end
 
+  def self.redis_tooling_client
+    Redis.new(url: config.tooling_redis_url)
+  end
+
   def self.dynamodb_client
     Aws::DynamoDB::Client.new(ExercismConfig::GenerateAwsSettings.())
   end

data/lib/exercism/tooling_job.rb

@@ -0,0 +1,147 @@
+module Exercism
+  class ToolingJob
+    require 'aws-sdk-s3'
+    require 'redis'
+
+    extend Mandate::Memoize
+
+    def self.create!(type, submission_uuid, language, exercise, attributes = {})
+      job_id = SecureRandom.uuid.tr('-', '')
+      redis = Exercism.redis_tooling_client
+      redis.multi do
+        redis.set(
+          "job:#{job_id}",
+          attributes.merge(
+            id: job_id,
+            submission_uuid: submission_uuid,
+            type: type,
+            language: language,
+            exercise: exercise,
+            created_at: Time.now.utc.to_i
+          ).to_json
+        )
+        redis.rpush(key_for_queued, job_id)
+        redis.set("submission:#{submission_uuid}:#{type}", job_id)
+      end
+      job_id
+    end
+
+    def self.find(id)
+      json = Exercism.redis_tooling_client.get("job:#{id}")
+      new(id, JSON.parse(json))
+    end
+
+    def self.find_for_submission_uuid_and_type(submission_uuid, type)
+      redis = Exercism.redis_tooling_client
+      job_id = redis.get("submission:#{submission_uuid}:#{type}")
+      json = redis.get("job:#{job_id}")
+      new(job_id, JSON.parse(json))
+    end
+
+    attr_reader :id, :data
+
+    def initialize(id, data)
+      @id = id
+      @data = data.freeze
+    end
+
+    def to_h
+      data.to_h
+    end
+
+    def respond_to_missing?(meth)
+      data.key?(meth.to_s) || super
+    end
+
+    def method_missing(meth)
+      super unless respond_to_missing?(meth)
+
+      data[meth.to_s]
+    end
+
+    def locked!
+      redis = Exercism.redis_tooling_client
+      redis.multi do
+        redis.lrem(key_for_queued, 1, id)
+        redis.rpush(key_for_locked, id)
+      end
+    end
+
+    def executed!(status, output)
+      redis = Exercism.redis_tooling_client
+      redis.multi do
+        redis.lrem(key_for_queued, 1, id)
+        redis.lrem(key_for_locked, 1, id)
+        redis.rpush(key_for_executed, id)
+
+        redis.set(
+          "job:#{id}",
+          data.merge(
+            execution_status: status,
+            execution_output: output
+          ).to_json
+        )
+      end
+    end
+
+    def processed!
+      redis = Exercism.redis_tooling_client
+      redis.multi do
+        redis.lrem(key_for_executed, 1, id)
+        redis.rpush(key_for_processed, id)
+      end
+    end
+
+    def cancelled!
+      redis = Exercism.redis_tooling_client
+      redis.multi do
+        redis.lrem(key_for_queued, 1, id)
+        redis.rpush(key_for_cancelled, id)
+      end
+    end
+
+    def ==(other)
+      id == other.id
+    end
+
+    def stderr
+      read_s3_file('stderr')
+    end
+
+    def stdout
+      read_s3_file('stdout')
+    end
+
+    private
+    def read_s3_file(name)
+      Exercism.s3_client.get_object(
+        bucket: s3_bucket_name,
+        key: "#{s3_folder}/#{name}"
+      ).body.read
+    rescue StandardError
+      ""
+    end
+
+    memoize
+    def s3_folder
+      "#{Exercism.env}/#{id}"
+    end
+
+    memoize
+    def s3_bucket_name
+      Exercism.config.aws_tooling_jobs_bucket
+    end
+
+    %w[queued locked executed processed cancelled].each do |key|
+      ToolingJob.singleton_class.class_eval do
+        define_method "key_for_#{key}" do
+          Exercism.env.production? ? key : "#{Exercism.env}:#{key}"
+        end
+      end
+
+      define_method "key_for_#{key}" do
+        self.class.send("key_for_#{key}")
+      end
+    end
+  end
+end

data/lib/exercism_config/version.rb

@@ -1,3 +1,3 @@
 module ExercismConfig
-  VERSION = '0.63.0'.freeze
+  VERSION = '0.64.0'.freeze
 end

data/settings/ci.yml

@@ -2,6 +2,9 @@
 anycable_redis_url: redis://127.0.0.1:6379/1
 anycable_rpc_host: 127.0.0.1:50051
 
+# Tooling
+tooling_redis_url: redis://127.0.0.1:6379/3
+
 # DynamoDB config
 dynamodb_tooling_jobs_table: tooling_jobs
 dynamodb_tooling_language_groups_table: tooling_language_groups

data/settings/docker.yml

@@ -2,6 +2,9 @@
 anycable_redis_url: redis://redis:6379/1
 anycable_rpc_host: 0.0.0.0:50051
 
+# Tooling
+tooling_redis_url: redis://redis:6379/3
+
 # DynamoDB config
 dynamodb_tooling_jobs_table: tooling_jobs
 dynamodb_tooling_language_groups_table: tooling_language_groups

data/settings/local.yml

@@ -2,6 +2,9 @@
 anycable_redis_url: redis://127.0.0.1:6379/1
 anycable_rpc_host: 127.0.0.1:50051
 
+# Tooling
+tooling_redis_url: redis://127.0.0.1:6379/3
+
 # DynamoDB config
 dynamodb_tooling_jobs_table: tooling_jobs
 dynamodb_tooling_language_groups_table: tooling_language_groups

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: exercism-config
 version: !ruby/object:Gem::Version
-  version: 0.63.0
+  version: 0.64.0
 platform: ruby
 authors:
 - Jeremy Walker
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-05-07 00:00:00.000000000 Z
+date: 2021-05-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-dynamodb
@@ -150,6 +150,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description:
 email:
 - jez.walker@gmail.com
@@ -177,6 +191,7 @@ files:
 - lib/exercism-config.rb
 - lib/exercism/config.rb
 - lib/exercism/secrets.rb
+- lib/exercism/tooling_job.rb
 - lib/exercism_config/determine_environment.rb
 - lib/exercism_config/environment.rb
 - lib/exercism_config/generate_aws_settings.rb