ecsutil 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: aeb2f9770b7183831199b073f64629a5c7d804bc3cbe3b17c95dda700d57d93c
+  data.tar.gz: 83f7201fe909f6379c3ca0b5851925f269bd992a320906154f27e5db758c61f2
+SHA512:
+  metadata.gz: aeda8bbb5d7a008583aa6edd4a814081312acbd0929f62d7f0358065b853a05feca66fda7deef85eae03d4e43ff8e8aea9909d2417363933e3a5f77f9d5323bd
+  data.tar.gz: '0093c78233a8467aa795e9ef72cfcc866fdaff876de97caeaa39d2c09cb1de00ac0c5ad1090a185e8890d25d805ed2336d64c7ef856a7bcc5e3f26ebcaf3f17e'

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.DS_Store

data/README.md

@@ -0,0 +1,118 @@
+# ecsutil
+
+Tool to simplify deployments to ECS/Fargate
+
+## Overview
+
+- You bring your own infrastructure resources using Terraform (optional)
+- `ecsutil` will manage ECS task definitions, scheduled tasks, services and secrets
+- Deployment config is YAML-based with ability to reference Terraform outputs
+- Cloud secrets are stored in AWS Parameter Store, encrypted by KMS
+- Local secrets are encrypted via Ansible Vault (optional)
+
+## Requirements
+
+- AWS CLI
+- Terraform (optional)
+
+## Usage
+
+```
+Usage: escutil <stage> <command>
+
+Available commands:
+* deploy  - Perform a deployment
+* run     - Run a task
+* scale   - Change service quantities
+* status  - Show current status
+* secrets - Manage secrets
+* destroy - Delete all cloud resources
+```
+
+## Config
+
+Example deployment configuration:
+
+```yaml
+app: myapp
+env: staging
+
+cluster: staging
+repository: your-ecr-repo-url
+subnets:
+  - a
+  - b
+  - c
+
+roles:
+  task: role ARN
+  execution: role ARN
+  schedule: role ARN
+
+tasks:
+  web:
+    command: bundle exec ruby app.rb
+    env:
+      PORT: 4567
+    security_groups:
+      - sg1
+      - sg2
+    ports:
+      - 4567
+    awslogs:
+      region: us-east-1
+      group: myapp-staging
+      prefix: web
+
+scheduled_tasks:
+  hourly:
+    task: web
+    command: bundle exec rake worker
+    expression: rate(1 hour)
+
+services:
+  web:
+    task: web
+    desired_count: 3
+    max_percent: 200
+    min_healthy_percent: 100
+    lb:
+      target_group: load balancer target group ARN
+      container_name: web
+      container_port: 4567
+```
+
+### Reference Terraform outputs
+
+Given you have `./terraform/(staging/production)`that contains all stage-specific
+configuration and resources, you can add an output file `outputs.tf` that might be
+referenced in the deployment config. Here's an example:
+
+```tf
+// Output for subnets
+// You can use regular terraform resources here
+output "subnets" {
+  value = [
+    "subnet-a",
+    "subnet-b",
+    "subnet-c"
+  ]
+}
+
+// Output for "web" security group
+output "sg_web" {
+  value = aws_security_group.web.id
+}
+```
+
+Once `terraform apply` is executed your state file (or remote state) will include 
+the `sg_web` output. We can reference it in the config:
+
+```yaml
+# ...
+subnets: $tf.subnets
+# ....
+tasks:
+  web:
+    security_groups: $tf.sg_web
+```

data/Rakefile

@@ -0,0 +1,9 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:test) do |t|
+  t.pattern = "spec/**/*_spec.rb"
+  t.verbose = false
+end
+
+task default: :test

data/bin/ecsutil

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+lib = File.expand_path(File.dirname(__FILE__) + '/../lib')
+$LOAD_PATH.unshift(lib) if File.directory?(lib) && !$LOAD_PATH.include?(lib)
+
+require "rubygems"
+require "ecsutil"
+
+ECSUtil::Runner.new(Dir.pwd, ARGV).run

data/ecsutil.gemspec

@@ -0,0 +1,22 @@
+require File.expand_path("../lib/ecsutil/version", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "ecsutil"
+  s.version     = ECSUtil::VERSION
+  s.summary     = "TBD"
+  s.description = "TBD"
+  s.homepage    = ""
+  s.authors     = ["Dan Sosedoff"]
+  s.email       = ["dan.sosedoff@gmail.com"]
+  s.license     = "MIT"
+
+  s.add_development_dependency "rake", "~> 10"
+  s.add_dependency "json", "~> 2"
+  s.add_dependency "ansible-vault", "~> 0.2"
+  s.add_dependency "hashie", "~> 4"
+  
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{|f| File.basename(f)}
+  s.require_paths = ["lib"]
+end

data/lib/ecsutil.rb

@@ -0,0 +1,6 @@
+require "ecsutil/version"
+require "ecsutil/command"
+require "ecsutil/runner"
+
+module ECSUtil
+end

data/lib/ecsutil/aws.rb

@@ -0,0 +1,258 @@
+module ECSUtil
+  module AWS
+    def aws_call(service, method, data)
+      input = data.is_a?(String) ? data : "--cli-input-json file://#{json_file(data)}" 
+
+      result = `aws #{service} #{method} #{input}`.strip
+      unless $?.success?
+        fail "#{service} #{method} failed!"
+      end
+      JSON.load(result)
+    end
+
+    def generate_event_rule(config)
+      {
+        Name:               config[:name],
+        ScheduleExpression: config[:expression],
+        State:              config[:enabled] ? "ENABLED" : "DISABLED",
+        Tags:               array_hash(config[:tags] || {}, "Key", "Value")
+      }
+    end
+
+    def generate_event_target(config, task_name, schedule_name)
+      task     = config["tasks"][task_name]
+      schedule = config["scheduled_tasks"][schedule_name]
+      input    = {}
+
+      if schedule["command"]
+        input = {
+          "containerOverrides": [
+            {
+              "name": task_name,
+              "command": schedule["command"].split(" ")
+            }
+          ]
+        }
+      end
+
+      {
+        Rule: schedule["rule_name"],
+        Targets: [
+          {
+            Id: "default",
+            Arn: config["cluster"],
+            RoleArn: config["roles"]["schedule"],
+            Input: JSON.dump(input),
+            EcsParameters: {
+              TaskDefinitionArn: task["arn"],
+              TaskCount: 1,
+              LaunchType: "FARGATE",
+              PlatformVersion: "LATEST",
+              NetworkConfiguration: {
+                awsvpcConfiguration: {
+                  Subnets: [config["subnets"]].flatten,
+                  SecurityGroups: [task["security_groups"]].flatten,
+                  AssignPublicIp: "ENABLED"
+                }
+              }
+            }
+          }
+        ]
+      }
+    end
+
+    def generate_task_definition(config, task_name)
+      task         = config["tasks"][task_name]
+      service_name = config["app"]
+      service_env  = config["env"]
+      env          = array_hash(task["env"] || {}, :name)
+
+      secrets = (config["secrets_data"] || []).map do |item|
+        {
+          name:      item[:key],
+          valueFrom: item[:name]
+        }
+      end
+
+      log_config = nil
+
+      if awslogs = task["awslogs"]
+        log_config = {
+          logDriver: "awslogs",
+          options: {
+            "awslogs-group": awslogs["group"],
+            "awslogs-region": awslogs["region"],
+            "awslogs-stream-prefix": awslogs["prefix"] || task_name
+          }
+        }
+      end
+
+      if sumo = task["sumologs"]
+        log_config = {
+          logDriver: "splunk",
+          options: {
+            "splunk-url":        sumo["url"],
+            "splunk-token":      sumo["token"],
+            "splunk-source":     sumo["source"] || "",
+            "splunk-sourcetype": sumo["sourcetype"] || "",
+          }
+        }
+      end
+
+      port_mappings = nil
+      if ports = [task["ports"]].flatten.compact.uniq
+        port_mappings = ports.map do |p|
+          {
+            containerPort: p,
+            hostPort: p,
+            protocol: "tcp"
+          }
+        end
+      end
+
+      {
+        family: "#{service_name}-#{service_env}-#{task_name}",
+        taskRoleArn: config["roles"]["task"],
+        executionRoleArn: config["roles"]["execution"],
+        networkMode: "awsvpc",
+        requiresCompatibilities: ["FARGATE"],
+        cpu: (task["cpu"] || "256").to_s,
+        memory: (task["memory"] || "512").to_s,
+        containerDefinitions: [
+          {
+            name: task_name,
+            command: task["command"] ? task["command"].split(" ") : nil,
+            image: "#{config["repository"]}:#{config["git_commit"]}",
+            environment: env,
+            secrets: secrets,
+            logConfiguration: log_config,
+            portMappings: port_mappings
+          }.compact
+        ]
+      }
+    end
+
+    def degerister_task_definition(arn)
+      aws_call("ecs", "deregister-task-definition", "--task-definition #{arn}")
+    end
+
+    def register_task_definition(data)
+      aws_call("ecs", "register-task-definition", data)["taskDefinition"]
+    end
+
+    def put_rule(data)
+      aws_call("events", "put-rule", data)
+    end
+
+    def put_targets(data)
+      aws_call("events", "put-targets", data)
+    end
+
+    def delete_rule(name)
+      aws_call("events", "remove-targets", "--rule=#{name} --ids=default")
+      aws_call("events", "delete-rule", "--name=#{name}")
+    end
+
+    def list_active_task_definitions
+      aws_call("ecs", "list-task-definitions", "--status=ACTIVE --max-items=100")["taskDefinitionArns"]
+    end
+
+    def list_services(cluster)
+      aws_call("ecs", "list-services", "--cluster=#{cluster}")["serviceArns"].map do |s|
+        s.split("/", 3).last
+      end
+    end
+
+    def list_rules
+      aws_call("events", "list-rules", "")["Rules"]
+    end
+
+    def fetch_parameter_store_keys(prefix, process = true)
+      result = aws_call("ssm", "get-parameters-by-path", "--path=#{prefix} --with-decryption")
+      result["Parameters"].map do |p|
+        {
+          name: p["Name"],
+          key: p["Name"].split("/").last,
+          value: p["Value"]
+        }
+      end
+    end
+
+    def generate_service(config, service_name)
+      service           = config["services"][service_name]
+      task              = config["tasks"][service["task"]]
+      full_service_name = sprintf("%s-%s-%s", config["app"], config["env"], service_name)
+      exists            = service["exists"] == true
+
+      data = {
+        cluster: config["cluster"],
+        taskDefinition: task["arn"],
+        desiredCount: service["desired_count"] || 0,
+        deploymentConfiguration: {
+          maximumPercent: service["max_percent"] || 100,
+          minimumHealthyPercent: service["min_healthy_percent"] || 50
+        },
+        networkConfiguration: {
+          awsvpcConfiguration: {
+            subnets: [config["subnets"]].flatten,
+            securityGroups: [task["security_groups"]].flatten,
+            assignPublicIp: "ENABLED"
+          }
+        }
+      }
+
+      if exists
+        data.merge!(
+          service: full_service_name,
+          forceNewDeployment: service["force_deployment"] == true
+        )
+      else
+        data.merge!(
+          serviceName: full_service_name,
+          propagateTags: "SERVICE",
+          enableECSManagedTags: true,
+          schedulingStrategy: "REPLICA",
+          launchType: "FARGATE",
+        )
+
+        if lb = service["lb"]
+          data.merge!(
+            loadBalancers: [
+              {
+                targetGroupArn:   lb["target_group"],
+                loadBalancerName: lb["name"],
+                containerName:    lb["container_name"],
+                containerPort:    lb["container_port"]
+              }.compact
+            ]
+          )
+        end
+      end
+      
+      data
+    end
+
+    def describe_service(config, service_name)
+      full_service_name = sprintf("%s-%s-%s", config["app"], config["env"], service_name)
+      result = aws_call("ecs", "describe-services", "--cluster=#{config["cluster"]} --services=#{full_service_name}")
+      result["services"].first
+    end
+
+    def describe_services(config, names)
+      aws_call("ecs", "describe-services", "--cluster=#{config.cluster} --services=#{names.join(",")}")["services"]
+    end
+
+    def create_service(config, service_name)
+      aws_call("ecs", "create-service", generate_service(config, service_name))
+    end
+
+    def update_service(config, service_name)
+      aws_call("ecs", "update-service", generate_service(config, service_name))
+    end
+
+    def delete_service(config, service_name)
+      aws_call("ecs", "update-service", "--cluster=#{config["cluster"]} --service=#{service_name} --desired-count 0")
+      aws_call("ecs", "delete-service", "--cluster=#{config["cluster"]} --service=#{service_name}")
+    end
+  end
+end