exercism-config 0.60.0 → 0.65.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ee93a1ec8c95f19849197185dbd41303681608f69f5f93f29bec61932c27662f
-  data.tar.gz: 6f3dee077a249d39bdb9a0c14628d150a293702106fcf4c64ffb16d75ace34f6
+  metadata.gz: f4ee8bd41a5f762b518a5510028a4b913c87390a8fa74bb10f7f0d295d3efff5
+  data.tar.gz: 48d18aa707cd2f1562412d57763427d08c8a2bd58d2da428a23e346dcf9fcb83
 SHA512:
-  metadata.gz: 181e84801c3b08f210b0af8aa9bdfaf212a5e214f77ea4658d5a4b20ae30520e3e704669396b061681bafbc3e8ab76e52753567f6952150f31c7b83a8f77fb37
-  data.tar.gz: aee501f90b327b368a110eeb32c173561752923850c0bf0d75da88b7a86dfc5985c8185c0b2413c7059ad4f7398d8ecded0d9780277651489e4a896a5c042ef6
+  metadata.gz: 005bc9e61b363021dcd154791a623d99c4bcac9732211ab64f9c2851a75b660c5973c7d224d80f7258fd01a29e6f4119ce846f1f8ccbdc71e6bd482f4ae88617
+  data.tar.gz: 63b50fb4d9e4b0faffab1926ad244bc975af3361136525dc0a3e521f02e3fad4b2df03f86f242d794136b4e0ea043f82a846e91a2a49bac107d38fc20cff6aa4

data/.github/workflows/rubocop.yml

@@ -2,9 +2,9 @@ name: Rubocop
 
 on:
   push:
-    branches: [master]
+    branches: [main]
   pull_request:
-    branches: [master]
+    branches: [main]
 
 jobs:
   rubocop:

data/.github/workflows/tests.yml

@@ -2,9 +2,9 @@ name: Tests
 
 on:
   push:
-    branches: [master]
+    branches: [main]
   pull_request:
-    branches: [master]
+    branches: [main]
 
 jobs:
   test:
@@ -16,6 +16,16 @@ jobs:
         ports:
           - 4566
 
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 1s
+          --health-timeout 2s
+          --health-retries 10
+
     steps:
       ###
       # Checkout using GitHub's checkout action
@@ -38,6 +48,7 @@ jobs:
           AWS_PORT: ${{ job.services.aws.ports['4566'] }}
         run: |
           bundle exec setup_exercism_config
+          bundle exec setup_exercism_local_aws
 
       ###
       # Setup code climate

data/Gemfile

@@ -10,3 +10,5 @@ gem 'gem-release'
 gem 'rubocop'
 gem 'rubocop-minitest'
 gem 'rubocop-performance'
+
+gem 'octokit'

data/README.md

@@ -36,9 +36,10 @@ Exercism.secrets.hcaptcha_secret
 Exercism.dynamodb_client
 Exercism.s3_client
 Exercism.ecr_client
+Exercism.octokit_client
 ```
 
-## Explaination
+## Explanation
 
 When terraform creates Exercism's infrastructure, it writes endpoints and DNS entries to DynamoDB.
 This gem allows you to trivially retrieve that data.

data/bin/setup_exercism_local_aws

@@ -27,119 +27,6 @@ unless ENV["EXERCISM_SKIP_S3"]
   end
 end
 
-###########################################
-# Setup local dynamodb tooling jobs table #
-###########################################
-[
-  Exercism.config.dynamodb_tooling_jobs_table,
-  Exercism.config.dynamodb_tooling_jobs_table + "-test"
-].each do |table_name|
-  begin
-    Exercism.dynamodb_client.delete_table(
-      table_name: table_name
-    )
-  rescue Aws::DynamoDB::Errors::ResourceNotFoundException
-  end
-  puts "[x] #{table_name}"
-
-  Exercism.dynamodb_client.create_table(
-    table_name: table_name,
-    attribute_definitions: [
-      {
-        attribute_name: "id",
-        attribute_type: "S"
-      }
-    ],
-    key_schema: [
-      {
-        attribute_name: "id",
-        key_type: "HASH"
-      }
-    ],
-    provisioned_throughput: {
-      read_capacity_units: 1,
-      write_capacity_units: 1
-    }
-  )
-
-  Exercism.dynamodb_client.update_table(
-    table_name: table_name,
-    attribute_definitions: [
-      {
-        attribute_name: "job_status",
-        attribute_type: "S"
-      },
-      {
-        attribute_name: "created_at",
-        attribute_type: "N"
-      }
-    ],
-    global_secondary_index_updates: [
-      {
-        create: {
-          index_name: "job_status", # required
-          key_schema: [ # required
-            {
-              attribute_name: "job_status", # required
-              key_type: "HASH" # required, accepts HASH, RANGE
-            },
-            {
-              attribute_name: "created_at", # required
-              key_type: "RANGE" # required, accepts HASH, RANGE
-            }
-          ],
-          projection: { # required
-            projection_type: "KEYS_ONLY"
-          },
-          provisioned_throughput: {
-            read_capacity_units: 1, # required
-            write_capacity_units: 1 # required
-          }
-        }
-      }
-    ]
-  )
-
-  Exercism.dynamodb_client.update_table(
-    table_name: table_name,
-    attribute_definitions: [
-      {
-        attribute_name: "submission_uuid",
-        attribute_type: "S"
-      },
-      {
-        attribute_name: "type",
-        attribute_type: "S"
-      }
-    ],
-    global_secondary_index_updates: [
-      {
-        create: {
-          index_name: "submission_type", # required
-          key_schema: [ # required
-            {
-              attribute_name: "submission_uuid", # required
-              key_type: "HASH" # required, accepts HASH, RANGE
-            },
-            {
-              attribute_name: "type", # required
-              key_type: "RANGE" # required, accepts HASH, RANGE
-            }
-          ],
-          projection: { # required
-            projection_type: "INCLUDE",
-            non_key_attributes: ["id", "job_status"]
-          },
-          provisioned_throughput: {
-            read_capacity_units: 1, # required
-            write_capacity_units: 1 # required
-          }
-        }
-      }
-    ]
-  )
-end
-
 ##############################################
 # Setup local dynamodb language groups table #
 ##############################################

data/exercism_config.gemspec

@@ -40,4 +40,5 @@ Gem::Specification.new do |spec|
   # own Gemfile when using this.
   spec.add_development_dependency 'aws-sdk-ecr'
   spec.add_development_dependency 'aws-sdk-s3'
+  spec.add_development_dependency 'redis'
 end

data/lib/exercism-config.rb

@@ -16,6 +16,7 @@ require_relative 'exercism_config/retrieve_secrets'
 require_relative 'exercism_config/version'
 require_relative 'exercism/config'
 require_relative 'exercism/secrets'
+require_relative 'exercism/tooling_job'
 
 module Exercism
   class ConfigError < RuntimeError; end
@@ -32,6 +33,10 @@ module Exercism
     @secrets ||= ExercismConfig::RetrieveSecrets.()
   end
 
+  def self.redis_tooling_client
+    Redis.new(url: config.tooling_redis_url)
+  end
+
   def self.dynamodb_client
     Aws::DynamoDB::Client.new(ExercismConfig::GenerateAwsSettings.())
   end
@@ -49,4 +54,19 @@ module Exercism
     require 'aws-sdk-ecr'
     Aws::ECR::Client.new(ExercismConfig::GenerateAwsSettings.())
   end
+
+  def self.octokit_client
+    require 'octokit'
+
+    access_token = ENV.fetch(
+      "GITHUB_ACCESS_TOKEN",
+      self.secrets.github_access_token
+    )
+
+    @octokit_client ||= Octokit::Client.new(
+      access_token: access_token
+    ).tap do |c|
+      c.auto_paginate = true
+    end
+  end
 end

data/lib/exercism/config.rb

@@ -15,7 +15,7 @@ module Exercism
         next unless val.nil?
 
         keys = to_h.keys
-        raise NoMethodError unless keys.include?(name.to_s) || keys.include?(name.to_sym)
+        raise NoMethodError, name unless keys.include?(name.to_s) || keys.include?(name.to_sym)
       end
     end
 

data/lib/exercism/tooling_job.rb

@@ -0,0 +1,151 @@
+module Exercism
+  class ToolingJob
+    require 'aws-sdk-s3'
+    require 'redis'
+
+    extend Mandate::Memoize
+
+    def self.create!(type, submission_uuid, language, exercise, extra = {})
+      job_id = SecureRandom.uuid.tr('-', '')
+      data = extra.merge(
+        id: job_id,
+        submission_uuid: submission_uuid,
+        type: type,
+        language: language,
+        exercise: exercise,
+        created_at: Time.now.utc.to_i
+      )
+
+      redis = Exercism.redis_tooling_client
+      redis.multi do
+        redis.set(
+          "job:#{job_id}",
+          data.to_json
+        )
+        redis.rpush(key_for_queued, job_id)
+        redis.set("submission:#{submission_uuid}:#{type}", job_id)
+      end
+      new(job_id, data)
+    end
+
+    def self.find(id)
+      json = Exercism.redis_tooling_client.get("job:#{id}")
+      new(id, JSON.parse(json))
+    end
+
+    def self.find_for_submission_uuid_and_type(submission_uuid, type)
+      redis = Exercism.redis_tooling_client
+      job_id = redis.get("submission:#{submission_uuid}:#{type}")
+      json = redis.get("job:#{job_id}")
+      new(job_id, JSON.parse(json))
+    end
+
+    attr_reader :id
+
+    def initialize(id, data)
+      @id = id
+      @data = data.freeze
+    end
+
+    def to_h
+      data.to_h
+    end
+
+    def respond_to_missing?(meth)
+      data.key?(meth.to_s) || super
+    end
+
+    def method_missing(meth)
+      super unless respond_to_missing?(meth)
+
+      data[meth.to_s]
+    end
+
+    def locked!
+      redis = Exercism.redis_tooling_client
+      redis.multi do
+        redis.lrem(key_for_queued, 1, id)
+        redis.rpush(key_for_locked, id)
+      end
+    end
+
+    def executed!(status, output)
+      redis = Exercism.redis_tooling_client
+      redis.multi do
+        redis.lrem(key_for_queued, 1, id)
+        redis.lrem(key_for_locked, 1, id)
+        redis.rpush(key_for_executed, id)
+
+        redis.set(
+          "job:#{id}",
+          data.merge(
+            execution_status: status,
+            execution_output: output
+          ).to_json
+        )
+      end
+    end
+
+    def processed!
+      redis = Exercism.redis_tooling_client
+      redis.multi do
+        redis.lrem(key_for_executed, 1, id)
+        redis.rpush(key_for_processed, id)
+      end
+    end
+
+    def cancelled!
+      redis = Exercism.redis_tooling_client
+      redis.multi do
+        redis.lrem(key_for_queued, 1, id)
+        redis.rpush(key_for_cancelled, id)
+      end
+    end
+
+    def ==(other)
+      id == other.id
+    end
+
+    def stderr
+      read_s3_file('stderr')
+    end
+
+    def stdout
+      read_s3_file('stdout')
+    end
+
+    private
+    attr_reader :data
+
+    def read_s3_file(name)
+      Exercism.s3_client.get_object(
+        bucket: s3_bucket_name,
+        key: "#{s3_folder}/#{name}"
+      ).body.read
+    rescue StandardError
+      ""
+    end
+
+    memoize
+    def s3_folder
+      "#{Exercism.env}/#{id}"
+    end
+
+    memoize
+    def s3_bucket_name
+      Exercism.config.aws_tooling_jobs_bucket
+    end
+
+    %w[queued locked executed processed cancelled].each do |key|
+      ToolingJob.singleton_class.class_eval do
+        define_method "key_for_#{key}" do
+          Exercism.env.production? ? key : "#{Exercism.env}:#{key}"
+        end
+      end
+
+      define_method "key_for_#{key}" do
+        self.class.send("key_for_#{key}")
+      end
+    end
+  end
+end

data/lib/exercism_config/version.rb

@@ -1,3 +1,3 @@
 module ExercismConfig
-  VERSION = '0.60.0'.freeze
+  VERSION = '0.65.0'.freeze
 end

data/settings/ci.yml

@@ -2,6 +2,9 @@
 anycable_redis_url: redis://127.0.0.1:6379/1
 anycable_rpc_host: 127.0.0.1:50051
 
+# Tooling
+tooling_redis_url: redis://127.0.0.1:6379/3
+
 # DynamoDB config
 dynamodb_tooling_jobs_table: tooling_jobs
 dynamodb_tooling_language_groups_table: tooling_language_groups
@@ -20,5 +23,12 @@ mysql_port: <%= ENV["MYSQL_PORT"] %>
 aws_submissions_bucket: exercism-staging-submissions
 aws_tooling_jobs_bucket: exercism-staging-tooling-jobs
 
+# Hosts
+website_icons_host: https://exercism-icons-staging.s3.eu-west-2.amazonaws.com
+
 # Sidekiq Config
 sidekiq_redis_url: redis://127.0.0.1:6379/2
+
+# EFS Config
+efs_submissions_mount_point: "/tmp/exercism/efs/submissions"
+efs_repositories_mount_point: "/tmp/exercism/efs/repos"

data/settings/docker.yml

@@ -2,6 +2,9 @@
 anycable_redis_url: redis://redis:6379/1
 anycable_rpc_host: 0.0.0.0:50051
 
+# Tooling
+tooling_redis_url: redis://redis:6379/3
+
 # DynamoDB config
 dynamodb_tooling_jobs_table: tooling_jobs
 dynamodb_tooling_language_groups_table: tooling_language_groups
@@ -20,5 +23,12 @@ mysql_port: 3306
 aws_submissions_bucket: exercism-staging-submissions
 aws_tooling_jobs_bucket: exercism-staging-tooling-jobs
 
+# Hosts
+website_icons_host: https://exercism-icons-staging.s3.eu-west-2.amazonaws.com
+
 # Sidekiq Config
 sidekiq_redis_url: redis://redis:6379/2
+
+# EFS Config
+efs_submissions_mount_point: "/tmp/exercism/efs/submissions"
+efs_repositories_mount_point: "/tmp/exercism/efs/repos"

data/settings/local.yml

@@ -2,6 +2,9 @@
 anycable_redis_url: redis://127.0.0.1:6379/1
 anycable_rpc_host: 127.0.0.1:50051
 
+# Tooling
+tooling_redis_url: redis://127.0.0.1:6379/3
+
 # DynamoDB config
 dynamodb_tooling_jobs_table: tooling_jobs
 dynamodb_tooling_language_groups_table: tooling_language_groups
@@ -20,5 +23,18 @@ mysql_port: null
 aws_submissions_bucket: exercism-staging-submissions
 aws_tooling_jobs_bucket: exercism-staging-tooling-jobs
 
+# Hosts
+website_icons_host: https://exercism-icons-staging.s3.eu-west-2.amazonaws.com
+
 # Sidekiq Config
 sidekiq_redis_url: redis://127.0.0.1:6379/2
+
+# EFS Config
+efs_submissions_mount_point: "/tmp/exercism/efs/submissions"
+efs_repositories_mount_point: "/tmp/exercism/efs/repos"
+
+# Extra things not used in development, but here
+# so that this file can provide a reference
+website_assets_host:
+aws_attachments_bucket:
+aws_attachments_region:

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: exercism-config
 version: !ruby/object:Gem::Version
-  version: 0.60.0
+  version: 0.65.0
 platform: ruby
 authors:
 - Jeremy Walker
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-12-16 00:00:00.000000000 Z
+date: 2021-05-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-dynamodb
@@ -150,6 +150,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description:
 email:
 - jez.walker@gmail.com
@@ -177,6 +191,7 @@ files:
 - lib/exercism-config.rb
 - lib/exercism/config.rb
 - lib/exercism/secrets.rb
+- lib/exercism/tooling_job.rb
 - lib/exercism_config/determine_environment.rb
 - lib/exercism_config/environment.rb
 - lib/exercism_config/generate_aws_settings.rb