ecs-kingpin 0.1.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2077aa518e8ab8b6bf3d6cc7eda665944f220e02
+  data.tar.gz: 0262ffb5562024ad8e1dec80dd5caba407ac6a6f
+SHA512:
+  metadata.gz: b2dd4f86e5ece59903b30c6bad4033badbc6edcf60dd5d84b42e1e88bbda717d240b3bbf8f1c8ce5c17362ed503bad62f6c8902dd5cedf43c7f30247ec2620d7
+  data.tar.gz: 251ef01a846eb465ce27aa2e0888d89a1ec59fe9505044ebb8b26f1dcd841293d6db767d16f20a3da3e211f7e675064861d17b82e2bcf426fdcf9e73ed5ec27c

data/.gitignore

@@ -0,0 +1,58 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+# Used by dotenv library to load environment variables.
+# .env
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+*.bridgesupport
+build-iPhoneOS/
+build-iPhoneSimulator/
+
+## Specific to RubyMotion (use of CocoaPods):
+#
+# We recommend against adding the Pods directory to your .gitignore. However
+# you should judge for yourself, the pros and cons are mentioned at:
+# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
+#
+# vendor/Pods/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+
+# Specific
+*.yml
+*.yaml
+task-definition.json
+*.tfvars
+log/
+.terraform*

data/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+gem 'aws-sdk', '2.6.2'
+gem 'trollop', '2.1.2'
+gem 'rake'
+gem 'hashie', '3.4.6'

data/Gemfile.lock

@@ -0,0 +1,23 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    aws-sdk (2.6.2)
+      aws-sdk-resources (= 2.6.2)
+    aws-sdk-core (2.6.2)
+      jmespath (~> 1.0)
+    aws-sdk-resources (2.6.2)
+      aws-sdk-core (= 2.6.2)
+    jmespath (1.3.1)
+    rake (10.4.2)
+    trollop (2.1.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  aws-sdk (= 2.6.2)
+  rake
+  trollop (= 2.1.2)
+
+BUNDLED WITH
+   1.13.6

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016 Jens Skott
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,2 @@
+# ecs-kingpin
+Quick rubyscript to test ecs api and create and update tasks and services

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/kingpin

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require_relative '../lib/kingpin.rb'
+
+Kingpin.new

data/def.yml

@@ -0,0 +1,83 @@
+# Type of a service to maintain: PublicService creates public ELB,  PrivateService creates internal ELB
+type: PublicService
+version: 1
+
+metadata:
+  name: my-frontend-ui
+  labels:
+    - name: "datadog"
+      value: "true"
+    - name: "ecs"
+      value: "false"
+spec:
+
+  # Container autoscaling
+  scaling:
+    min: 1
+    desired: 1
+    max: 2
+
+  # Roles required
+  iamRoles:
+    - "ecsBasicRole" # Always added
+    - "s3ReadAccess"
+    - "ec2Defaults"
+
+  # Shared volumes on ECS cluster
+  volumes:
+  - name: "webdata"
+    source: "/ecs/webdata"
+
+  # Container definition
+  containers:
+  # Container for Nginx
+  - name: nginx
+    image: ~/nginx:1.7.9 # If image starts with "~/" - auto-detect service-specific registry URL
+    essential: true # main container - if down, then restart all
+
+    env:
+    - name: "SERVER_PORT"
+      value: 8080
+
+    ports:
+    - containerPort: 8080 # Port EXPOSE
+      port: 80            # ELB & ELB Target Group port (listener)
+      protocol: "http"    # http or https
+      name: "public"      # "_default_" if omited (may only be one unnamed port)
+
+# Secondary ports are currently not supported. But if they were...
+#    - containerPort: 8081 # Port EXPOSE
+#      port: 81
+#      protocol: "https"
+#      name: "management"
+    logs:
+      logdriver: "gelf"
+      options:
+        gelf-address: "udp://crap.com:5212"
+        labels: "hej, då"
+
+    health:
+      portName: "public" # defaults to "_default_"
+      path: "/health"
+      timeout: 10s
+      interval: 5s
+      healthyTh: 3
+      unhealthyTh: 2
+
+    volumes:  # Volume mounting
+    - sourceVolume:
+      containerPath:
+
+    resources:  # Resource limits
+      cpu: ""
+      memory: ""
+
+    links:  #Container links
+    - "redis:redis"
+
+  # Container for Redis
+  - name: redis
+    image: redis
+    essential: false
+    resources:
+      memory: 128M # Units are parsed to be user friendly, no unit = MB

data/ecs-kingpin.gemspec

@@ -0,0 +1,21 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require 'bundler'
+require 'rake'
+
+Gem::Specification.new do |spec|
+  spec.name          = "ecs-kingpin"
+  spec.authors       = ["jens.skott"]
+  spec.email         = ["jens.skott@gmail.com"]
+  spec.summary       = %q{Create and manage ecs resources}
+  spec.description   = %q{Manage ecs trough aws api with commandline flags}
+  spec.homepage      = "http://github.com/jensskott/kingpin"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($\)
+  spec.executables   = ["kingpin"]
+  spec.require_paths = [lib]
+  spec.version       = '0.1.2'
+end

data/lib/kingpin/aws.rb

@@ -0,0 +1,21 @@
+def connectEcs(region)
+    Aws::ECS::Client.new(region: region)
+end
+
+def awsBucket(region, bucketName)
+    # connect to s3
+    s3 = Aws::S3::Client.new(
+        region: region
+    )
+    buckets = s3.list_buckets.to_h
+    buckets.extend(Hashie::Extensions::DeepLocate)
+    bucket = buckets.deep_locate -> (key, value, _object) { key == :name && value.include?(bucketName) }
+    if bucket.empty?
+        Kinglog.log.info "Creating bucket #{bucketName}!"
+        s3.create_bucket(bucket: bucketName)
+        Kinglog.log.info "Enabling bucket versioning!"
+        s3.put_bucket_versioning(bucket: bucketName, versioning_configuration: { mfa_delete: 'Disabled', status: 'Enabled' })
+    else
+        Kinglog.log.info "Bucket allready exists"
+    end
+end

data/lib/kingpin/log.rb

@@ -0,0 +1,15 @@
+class Kinglog
+    def self.log
+        # Create logdir
+        logdir = Dir.pwd + '/log'
+        Dir.mkdir logdir unless Dir.exist?(logdir)
+        # Create log
+        if @logger.nil?
+            @logger = Logger.new STDOUT
+            # @logger = Logger.new "#{logdir}/kingpin.log"
+            @logger.level = Logger::DEBUG
+            @logger.datetime_format = '%Y-%m-%d %H:%M:%S '
+        end
+        @logger
+    end
+end

data/lib/kingpin/options.rb

@@ -0,0 +1,58 @@
+def fileOpts(file)
+    yaml = YAML.load_file(file.to_s)
+    yaml
+end
+
+def cliOpts
+    opts = Trollop.options do
+        version 'Kingpin 0.1.0'
+        banner <<-EOS
+    Manage your ECS resources.
+
+    Usage:
+           kinpin [options]
+    where [options] are:
+    EOS
+
+        opt :region, 'Put your aws region here', type: :string
+        opt :env, 'Application Environment', type: :string
+        opt :product, 'The product where your services are related', type: :string
+        opt :command, 'Use aws cli or terraform to create service', type: :string, default: 'aws'
+        opt :file, 'Yaml file to read data from', type: :string
+        opt :local, 'Use local config files', type: :bool, default: false
+    end
+
+    # Check for options needed
+    Trollop.die :region, 'Define a region' if opts[:region].nil?
+    Trollop.die :env, 'Define an environment' if opts[:env].nil?
+    Trollop.die :product, 'Define a product' if opts[:product].nil?
+    Trollop.die :file, 'Define a file' if opts[:file].nil?
+
+    # Merge filepath + filename to opts
+    opts[:file] = File.expand_path(opts[:file])
+
+    # Check if file exists
+    unless File.exist? (opts[:file])
+        Kinglog.log.error 'File does not exist'
+        abort
+    end
+
+    # Check valid extention
+    accepted_formats = ['.yml', '.yaml']
+    unless accepted_formats.include? File.extname(opts[:file])
+        Kinglog.log.error 'Not a valid yaml extention'
+        abort
+    end
+
+    # Check valid yaml
+    Kinglog.log.info "Validating yaml file : #{opts[:file]}"
+    if YAML.load_file(opts[:file]) == false
+        Kinglog.log.error "failed to read #{opts[:file]}: #{$ERROR_INFO}"
+        abort
+    end
+
+    # Puts all file options into opts[:yaml]
+    # Accessable trough opts[:yaml]['metadata'] ....
+    opts[:yaml] = fileOpts(opts[:file])
+    opts
+end

data/lib/kingpin/parse.rb

@@ -0,0 +1,36 @@
+def parseLabels(array)
+    hash = {}
+    array.each do |l|
+        key = l['name']
+        value = l['value']
+        hash[key] = value
+    end
+    hash
+end
+
+def parseContainers(array, labels)
+    arr = []
+    array.each do |l|
+        c = {}
+        c['name'] = l['name']
+        c['essential'] = l['essential']
+        c['image'] = l['image']
+        c['environment'] = l['env']
+        c['memory'] = l['resources']['memory']
+        c['cpu'] = l['resources']['cpu']
+        c['mountPoints'] = l['volumes']
+        c['docker_labels'] = labels
+        c['links'] = l['links']
+        c['logConfiguration'] = l['logs']
+        arr << c
+    end
+    arr
+end
+
+def parseService(array)
+    arr = []
+    array.each do |l|
+        arr << l['ports'] unless l['ports'].nil?
+    end
+    arr
+end

data/lib/kingpin/service.rb

@@ -0,0 +1,28 @@
+# Create service.tfvars for terraform
+def createServiceVars(array, ports)
+    product = array[:product]
+    service = array[:yaml]['metadata']['name']
+    env = array[:env]
+    region = array[:region]
+    port = ports[0][0]['containerPort']
+    containerPort = ports[0][0]['port']
+    type = array[:yaml]['type']
+    serviceVars = "product: <%= product %>
+  service: <%= service %>
+  env: <%= env %>
+  region: <%= region %>
+  elb_port: <%= port %>
+  container_port: <%= containerPort %>
+  <% if type == \"PublicService\" %>service_type: public<% else %>service_type: private<% end %>"
+
+    # Create a service.tfvars
+    renderer = ERB.new(serviceVars)
+    File.open('service.tfvars', 'w') do |f|
+        f.write renderer.result(binding)
+    end
+    Kinglog.log.info 'Wrote service.tfvars'
+end
+
+# Create service options for aws api
+def serviceOpts
+end

data/lib/kingpin/task.rb

@@ -0,0 +1,23 @@
+# Create task-definition for terraform
+def createTaskDefinition(array, service)
+    json_opts = array.to_json
+    parsed = JSON.parse(json_opts)
+    last = parsed.last
+
+    task_definition = "{\"container_definitions\": [
+      <% parsed.each do |array| %><%= array.to_json %><% unless last[\"name\"] == array[\"name\"] %>,<% end %><% end %>
+      ],
+      \"family\": \"<%= service %>\"
+    }"
+
+    # Create a task-definition.json
+    renderer = ERB.new(task_definition)
+    File.open('task-definition.json', 'w') do |f|
+        f.write renderer.result(binding)
+    end
+    Kinglog.log.info 'Wrote task-defintion.json'
+end
+
+# Create task options for aws api
+def taskOpts
+end

data/lib/kingpin/terraform.rb

@@ -0,0 +1,47 @@
+# Check if terraform is installed on the system
+def terrformExecCheck
+    `which terraform`
+    if $? == false
+        Kinglog.log.error 'Terrform executable not found, please install terraform'
+        abort
+    end
+end
+
+# Configure remote s3 bucket for terraform state file
+def terrformConfig(array,bucketName)
+    product = array[:product]
+    service = array[:yaml]['metadata']['name']
+    env = array[:env]
+    region = array[:region]
+    serviceState = "#{service}-#{product}-#{env}.tfstate"
+    tfconfig = "terraform remote config -backend=s3 -backend-config=\"bucket=#{bucketName}\" \
+    -backend-config=\"key=#{serviceState}\" -backend-config=\"region=#{region}\""
+    system tfconfig
+    if $? != 0
+        Kinglog.log.error "Could not apply terraform config. Errorcode #{$?}"
+        abort
+    end
+end
+
+# Run terraform to apply config
+def terraformRun
+    tf = "terraform apply"
+    tfget = "terraform get"
+    system tfget
+    system tf
+    if $? != 0
+        Kinglog.log.error "Can't run terraform config. Error: #{$?}"
+        abort
+    end
+end
+
+# Clean up all temporary files from terraform
+def terraformCleanup
+    FileUtils.rm_rf('terraform.tfstate')
+    FileUtils.rm_rf('terraform.tfstate.backup')
+    FileUtils.rm_rf('mkmf.log')
+    FileUtils.rm_rf('.terraform')
+    FileUtils.rm_rf('task-definition.json')
+    FileUtils.rm_rf('service.tfvars')
+    Kinglog.log.info 'Removed all temporary files'
+end

data/lib/kingpin.rb

@@ -0,0 +1,47 @@
+# Require Gems
+require 'aws-sdk'
+require 'yaml'
+require 'trollop'
+require 'logger'
+require 'json'
+require 'erb'
+require 'fileutils'
+require 'mkmf'
+require 'hashie'
+
+# Require local libs
+require_relative 'kingpin/aws'
+require_relative 'kingpin/parse'
+require_relative 'kingpin/options'
+require_relative 'kingpin/log'
+require_relative 'kingpin/terraform'
+require_relative 'kingpin/service'
+require_relative 'kingpin/task'
+
+# Get options from CLI and file
+class Kingpin
+    # Get options from cli
+    opts = cliOpts
+
+    # Define varaibles for service and tasks
+    labels = parseLabels(opts[:yaml]['metadata']['labels'])
+    servicePort = parseService(opts[:yaml]['spec']['containers'])
+    containers = parseContainers(opts[:yaml]['spec']['containers'], labels)
+
+    case opts[:command]
+    when 'aws'
+        puts 'use aws cli'
+    when 'terraform'
+        Kinglog.log.info 'Running terraform to configure ECS environment'
+        terrformExecCheck
+        createTaskDefinition(containers, opts[:yaml]['metadata']['name'])
+        createServiceVars(opts, servicePort)
+        if opts[:local] == false
+            bucketName = "terraformstate-ecs-#{opts[:product]}-#{opts[:env]}-#{opts[:region]}"
+            awsBucket(opts[:region], bucketName)
+            terrformConfig(opts,bucketName)
+        end
+        terraformRun
+        terraformCleanup if opts[:local] == false
+    end
+end

metadata

@@ -0,0 +1,62 @@
+--- !ruby/object:Gem::Specification
+name: ecs-kingpin
+version: !ruby/object:Gem::Version
+  version: 0.1.2
+platform: ruby
+authors:
+- jens.skott
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-11-08 00:00:00.000000000 Z
+dependencies: []
+description: Manage ecs trough aws api with commandline flags
+email:
+- jens.skott@gmail.com
+executables:
+- kingpin
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- bin/kingpin
+- def.yml
+- ecs-kingpin.gemspec
+- lib/kingpin.rb
+- lib/kingpin/aws.rb
+- lib/kingpin/log.rb
+- lib/kingpin/options.rb
+- lib/kingpin/parse.rb
+- lib/kingpin/service.rb
+- lib/kingpin/task.rb
+- lib/kingpin/terraform.rb
+homepage: http://github.com/jensskott/kingpin
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- "/home/jskott/dev/kingpin-ruby/lib"
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Create and manage ecs resources
+test_files: []