orch 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2f67ca3214ddbc6302dde59fc294da08fd0dbd2b
-  data.tar.gz: 572ce735ddf78eebbd1ace4c6d0b33a05892d37f
+  metadata.gz: 2505f295b3b9d24ec8cf03009803c9efa3afb0ef
+  data.tar.gz: 85462611db90a1799169b89ee601076d2bc0d7b3
 SHA512:
-  metadata.gz: e890d242ed98f9986b2a2eced37f4798aa85baba10192ac3bc5baeae214611c84043fef017163388267fdd5f8807241eb2efda216e284849546acb205154af43
-  data.tar.gz: 876005978425fba1769c54227c1473dddae4e83e5f210d98fcb5e5aa5e49f10d2d56454f31ff763841f7d83e0e771432405a81c1b15d45ee608f29e249fdeef1
+  metadata.gz: 496fb704230bc3d7410e46b15c4d7e67b3a604e0781461f38b505d862f0e5ac6694883deac23ee29c68510560de71049efd58c543fa1d94cb420a5b7b005fc8b
+  data.tar.gz: 6e5d5fcdbfbe50becbd7227dab42ce6b86932ed9b130127b923d7aff7ad9eb54d25092bd376820608e57f952d2515e851c49b5948b89e8e313f9a46e07a7ec3c

data/README.md

@@ -14,9 +14,9 @@ Once the gem is installed, the command orch will be in your $PATH
 
 ## Usage
 
-Type "orch help" to get the list of commands and "orch help <command>" to get list of options for a given command.
+Type "orch help" to get the list of commands and "orch help _command_" to get list of options for a given command.
 
-The general usecase is to run "orch deploy <job_spec>" to deploy a configuration to to Marathon or Chronos.  A given job_spec can contain specifications for many jobs and for multiple environments.  Various options to the deploy command gives control to exactly what is deployed. 
+The general usecase is to run "orch deploy _job_spec_" to deploy a configuration to to Marathon or Chronos.  A given job_spec can contain specifications for many jobs and for multiple environments.  Various options to the deploy command gives control to exactly what is deployed. 
 
 ## Job spec format
 
@@ -33,7 +33,8 @@ The file ~/.orch/config.yaml would contain values for "chronos_url" and "maratho
 
 ## Examples
 
-TODO: show some examples
+The examples directory contains various examples of the features of orch.  
+View [documentation there](examples/Examples.md) for more details.
 
 ## Contributing
 

data/examples/Examples.md

@@ -0,0 +1,10 @@
+
+# Examples
+
+This directory contains several examples to show various orch features.
+
+| File                  | Description
+|-----------------------|------------------------------------------------------------------------|
+| shimple-chronos.yml   | A very simple chronos spec for a daily recurring job
+| env-chronos.yml       | Shows how to set up a chronos job for multiple deployment environments.
+| simple-marathon.yml   | A simple example of a Marathon job using orch

data/examples/env-chronos.yml

@@ -3,7 +3,7 @@ default_job: &DEFAULT_JOB
   kind: Chronos
   environment: override me
   cronos_spec: &DEFAULT_SPEC
-    name: "ray-test-{{ENV}}"
+    name: "ray-test-{{DEPLOY_ENV}}"
     schedule: "R/2015-01-25T21:00/PT24H"
     scheduleTimeZone: "PST"
     owner: "rjohnson@yp.com"
@@ -17,30 +17,46 @@ default_job: &DEFAULT_JOB
     environmentVariables: 
       - 
         name: "LOG_LOCATION"
-        value: "/var/{{ENV}}"
+        value: "/var/{{DEPLOY_ENV}}"
     forcePullImage: false
     command: "echo \"log here: $LOG_LOCATION\"; echo \"ENV: $DEPLOY_ENV\";sleep 10; echo bye"
 
 version: alpha1
-environments_var: DEPLOY_ENV
-environments:
-  - dev
-  - test
-  - prod
+environment_vars:
+  DEPLOY_ENV:
+    - dev
+    - test
+    - prod
+  COLO:
+    - west
+    - east
 applications:
   - <<: *DEFAULT_JOB
     environment: dev
+    DEPLOY_ENV: dev
+    COLO: east
     cronos_spec:
       <<: *DEFAULT_SPEC
       schedule: "R/2015-08-01T21:00/PT24H"
       forcePullImage: true
   - <<: *DEFAULT_JOB
     environment: test
+    DEPLOY_ENV: test
+    COLO: east
     cronos_spec:
       <<: *DEFAULT_SPEC
       schedule: "R/2015-08-01T21:10/PT24H"
   - <<: *DEFAULT_JOB
     environment: prod
+    DEPLOY_ENV: prod
+    COLO: east
+    cronos_spec:
+      <<: *DEFAULT_SPEC
+      schedule: "R/2015-08-01T21:30/PT24H"
+  - <<: *DEFAULT_JOB
+    environment: prod
+    DEPLOY_ENV: prod
+    COLO: west
     cronos_spec:
       <<: *DEFAULT_SPEC
       schedule: "R/2015-08-01T21:30/PT24H"

data/examples/simple-marathon.yml

@@ -1,14 +1,14 @@
 version: alpha1
-environments_var: DEPLOY_ENV
-environments:
-  - dev
-  - test
-  - prod
+environment_vars:
+  DEPLOY_ENV:
+    - dev
+    - test
+    - prod
 applications:
   - kind: Marathon
-    environment: dev
+    DEPLOY_ENV: dev
     marathon_spec:
-      id: "ray-web-{{ENV}}"
+      id: "ray-web-{{DEPLOY_ENV}}"
       cpus: 0.1
       mem: 300
       instances: 10

data/lib/config.rb

@@ -16,6 +16,22 @@ module Orch
       end
     end
 
+    def check_for_chronos_url
+      if (!  @options.has_key?("chronos_url")) && (@APP_CONFIG.nil? || @APP_CONFIG["chronos_url"].nil?)
+        return false
+      else
+        return true
+      end
+    end
+
+    def check_for_marathon_url
+      if (!  @options.has_key?("marathon_url")) && (@APP_CONFIG.nil? || @APP_CONFIG["marathon_url"].nil?)
+        return false
+      else
+        return true
+      end
+    end
+
     def chronos_url
       if @options.has_key?("chronos_url")
         # If passed in on command line override what is in config file

data/lib/deploy.rb

@@ -36,6 +36,11 @@ module Orch
     end
 
     def verify_chronos(json_payload)
+      if @config.check_for_chronos_url == false
+        puts "no chronos_url - can not verify with server"
+        return
+      end
+
       spec = Hashie::Mash.new(JSON.parse(json_payload))
 
       uri = URI(@config.chronos_url)
@@ -79,6 +84,7 @@ module Orch
           end
           next
         end
+        # TODO: not sure how to compare arrays
         specVal = spec[key]
         jobVal = job[key]
         if spec[key].to_s.numeric?
@@ -89,7 +95,7 @@ module Orch
           jobVal = job[key]
         end
         if specVal != jobVal
-          puts "#{key}= spec:#{spec[key]}, server:#{job[key]}"
+          puts "#{key}= spec:#{specVal}, server:#{jobVal}"
           foundDiff = true
         end
       end
@@ -101,9 +107,8 @@ module Orch
       uri = URI(@config.marathon_url)
       json_headers = {"Content-Type" => "application/json",
                 "Accept" => "application/json"}
-
-      # curl -L -H 'Content-Type: application/json' -X POST -d @$schedule_file $CHRONOS_URL/scheduler/iso8601
       puts uri
+
       http = Net::HTTP.new(uri.host, uri.port)
       response = http.put("/v2/apps/#{json_payload["id"]}", json_payload, json_headers)
 
@@ -117,5 +122,38 @@ module Orch
 
       return response
     end
+
+    def verify_marathon(json_payload)
+      if @config.check_for_marathon_url == false
+        puts "no marathon_url - can not verify with server"
+        return
+      end
+
+      spec = Hashie::Mash.new(JSON.parse(json_payload))
+
+      uri = URI(@config.marathon_url)
+      json_headers = {"Content-Type" => "application/json",
+                "Accept" => "application/json"}
+
+      http = Net::HTTP.new(uri.host, uri.port)
+      response = http.get("/v2/apps/#{spec.id}", json_headers)
+
+      # puts "Response #{response.code} #{response.message}: #{response.body}"
+
+      if response.code == 404.to_s
+        puts "job: #{spec.id} - is not deployed"
+        return
+      end
+
+      if response.code == 200.to_s
+        job = Hashie::Mash.new(JSON.parse(response.body))
+        foundDiffs = find_diffs(spec, job.app)
+      end
+
+      # TODO: handle error codes
+
+      return response
+    end
+
   end
 end

data/lib/orch/version.rb

@@ -1,3 +1,3 @@
 module Orch
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/lib/orch.rb

@@ -27,14 +27,16 @@ module Orch
       config.setup_config(marathon_url, chronos_url)
     end
 
-    option :deploy_type, :default => 'all',
-           :desc => 'chronos, marathon, all'
-    option :env_type, :default => 'all',
-           :desc => 'a valid environment defined in your config or all to deploy to all environments'
+    option :deploy_kind, :default => 'all',
+           :desc => 'deploys only the given application kind: chronos, marathon, all'
+    option :deploy_env, :default => 'all',
+           :desc => 'ENV_VAR=VALUE deploys only if an environment_var matches the given value'
     option :subst,
            :desc => 'KEY=VALUE substitute KEY with VALUE globaly in your config'
     option :show_json, :default => false,
            :desc => 'show the json result that would be sent to Chronos or Marathon'
+    option :server_verify, :default => true,
+           :desc => 'verify the configuration against the server'
     desc 'verify PATH', 'Checks basic syntax and does not deploy'
     def verify(file_name)
       parser = Orch::Parse.new(file_name, options)
@@ -42,25 +44,28 @@ module Orch
       puts "Number of configs found: #{result.length}"
       deploy = Orch::Deploy.new(options)
       result.each do |app|
-        if parser.hasEnvironments
-          puts "Name: #{app[:name]}, Type: #{app[:type]}, Would deploy: #{app[:deploy]}, Environment: #{app[:environment]}"
-        else
-          puts "Name: #{app[:name]}, Type: #{app[:type]}, Would deploy: #{app[:deploy]}"
+        printf "Name: %s, Type: %s, Deploy?: %s", app[:name], app[:type], app[:deploy]
+        app[:env_vars].each do |key, value|
+          printf ", %s: %s", key, value
         end
+        printf "\n"
         if options[:show_json]
           pretty = JSON.pretty_generate(app[:json])
           puts "JSON: #{pretty}"
         end
-        if app[:type] == "Chronos"
+        if (app[:type] == "Chronos") && (options[:server_verify] == true)
           deploy.verify_chronos(app[:json].to_json)
         end
+        if (app[:type] == "Marathon") && (options[:server_verify] == true)
+          deploy.verify_marathon(app[:json].to_json)
+        end
       end
     end
 
-    option :deploy_type, :default => 'all',
-           :desc => 'chronos, marathon, all'
-    option :env_type, :default => 'all',
-           :desc => 'a valid environment defined in your config or all to deploy to all environments'
+    option :deploy_kind, :default => 'all',
+           :desc => 'deploys only the given application kind: chronos, marathon, all'
+    option :deploy_env, :default => 'all',
+           :desc => 'ENV_VAR=VALUE deploys only if an environment_var matches the given value'
     option :chronos_url,
            :desc => 'url to chronos master'
     option :marathon_url,

data/lib/parse.rb

@@ -3,7 +3,6 @@ require 'json'
 
 module Orch
   class Parse
-    attr_reader :hasEnvironments
 
     def initialize(path, options)
       if ! File.file?(path)
@@ -11,7 +10,12 @@ module Orch
         exit 1
       end
       # TODO: try resecue for any syntax issues in yaml file
-      yaml = ::YAML.load_file(path)
+      begin
+        yaml = ::YAML.load_file(path)
+      rescue Psych::SyntaxError => e
+        puts "error parsing yaml file #{e}"
+        exit 1
+      end
       #puts yaml.to_json
 
       @options = options
@@ -35,19 +39,8 @@ module Orch
         exit 1
       end
 
-      # Does this config support environments
-      @hasEnvironments = false
-      defined_environments = {}
-      if (! spec.environments.nil?)
-        @hasEnvironments = true
-        spec.environments.each do |e|
-          defined_environments[e] = e
-        end
-      end
-
-      if (! spec.environments_var.nil?)
-        @environments_var = spec.environments_var
-      end
+      # Check for vault vars
+      env_var_values = parse_vault_vars(@spec)
 
       if spec.applications.nil?
         puts "required section applications: must have at least one application defined"
@@ -58,40 +51,30 @@ module Orch
       spec.applications.each do |app|
         # Check for valid kind paramter
         if app.kind.nil?
-          puts "required field kind was not found"
+          puts "required field 'kind:' was not found"
           exit 1
         end
 
-
         if !(app.kind == "Chronos" || app.kind == "Marathon")
           puts "unsupported kind specified: #{app.kind} - must be: Chronos | Marathon"
           exit 1
         end
 
-        if @hasEnvironments
-          if !defined_environments.has_key?(app.environment)
-            puts "environment \"#{app.environment}\" not defined in environments"
-            exit 1
-          end
-        end
+        # Generate any environment variables that need to be merged in
+        spec_env_vars = parse_env_vars(app)
+        env_var_values = env_var_values.merge(spec_env_vars)
 
         if (app.kind == "Chronos")
-          chronos_spec = parse_chronos(app)
+          chronos_spec = parse_chronos(app, env_var_values)
 
-          result = {:name => chronos_spec["name"], :type => app.kind, :deploy => should_deploy?(app), :json => chronos_spec}
-          if @hasEnvironments
-            result[:environment] = app.environment
-          end
+          result = {:name => chronos_spec["name"], :type => app.kind, :deploy => should_deploy?(app), :env_vars => env_var_values, :json => chronos_spec}
           results << result
         end
 
         if (app.kind == "Marathon")
-          marathon_spec = parse_marathon(app)
+          marathon_spec = parse_marathon(app, env_var_values)
 
-          result = {:name => marathon_spec["id"], :type => app.kind, :deploy => should_deploy?(app), :json => marathon_spec}
-          if @hasEnvironments
-            result[:environment] = app.environment
-          end
+          result = {:name => marathon_spec["id"], :type => app.kind, :deploy => should_deploy?(app), :env_vars => env_var_values, :json => marathon_spec}
           results << result
         end
       end
@@ -99,31 +82,46 @@ module Orch
       return results
     end
 
-    def parse_chronos(app)
-      # TODO: check if it exists
-      cronos_spec = app.cronos_spec
-
-      # This adds environment vaiables to the spec that conform to our docker-wrapper security hack
-      if ! app.vault.nil?
-        # TODO: check that an environment was defined
-        if (! @hasEnvironments) || app.environment.nil?
-          puts "the vault feature requires an environment to be set"
-          exit 1
+    def parse_env_vars(app)
+      result = {}
+      if (! @spec.environment_vars.nil?)
+        @spec.environment_vars.each do |key, value|
+          if app[key].nil?
+            puts "environments_var #{key} specified - but not included in app"
+            # TODO: would be nice to put the app name...
+            exit 1
+          end
+          if ! @spec.environment_vars[key].include? app[key]
+            puts "#{key} value \"#{app[key]}\" not in #{@spec.environment_vars[key].to_s}"
+            exit 1
+          end
+          result[key] = app[key]
         end
+      end
 
+      return result
+    end
+
+    def parse_vault_vars(spec)
+      result = {}
+      if ! spec.vault.nil?
         count = 1
-        cronos_spec.environmentVariables ||= []
-        app.vault.each do |vault_key|
-          # Add an environment
-          pair = {"name" => "VAULT_KEY_#{count}", "value" => vault_key}
+        spec.vault.each do |vault_key|
+          result["VAULT_KEY_#{count}"] = vault_key
           count += 1
-          cronos_spec.environmentVariables << pair
         end
       end
+      return result
+    end
+
+    def parse_chronos(app, env_var_values)
+      # TODO: check if it exists
+      cronos_spec = app.cronos_spec
 
-      if (! @environments_var.nil?)
-        pair = {"name" => @environments_var, "value" => app.environment}
-        cronos_spec.environmentVariables << pair
+      # Augment any spec environment variables with meta values
+      env_var_values.each do |key, value|
+        pair = {"name" => key, "value" => value}
+        cronos_spec.environmentVariables << pair        
       end
 
       # Do subst processing
@@ -133,29 +131,13 @@ module Orch
       return cronos_spec
     end
 
-    def parse_marathon(app)
+    def parse_marathon(app, env_var_values)
       # TODO: check if it exists
       marathon_spec = app.marathon_spec
 
-      # This adds environment vaiables to the spec that conform to our docker-wrapper security hack
-      if ! app.vault.nil?
-        # TODO: check that an environment was defined
-        if (! @hasEnvironments) || app.environment.nil?
-          puts "the vault feature requires an environment to be set"
-          exit 1
-        end
-
-        count = 1
-        marathon_spec.env ||= {}
-        app.vault.each do |vault_key|
-          # Add an environment
-          marathon_spec.env["VAULT_KEY_#{count}"] = vault_key
-          count += 1
-        end
-      end
-
-      if (! @environments_var.nil?)
-        marathon_spec.env[@environments_var] = app.environment
+      # Augment any spec environment variables with meta values
+      env_var_values.each do |key, value|
+        marathon_spec.env[key] = value     
       end
 
       spec_str = do_subst(marathon_spec, app)
@@ -166,15 +148,25 @@ module Orch
 
     def should_deploy?(app)
       result = true
-      if (app.kind == "Marathon") && @options[:deploy_type] == 'chronos'
+      if (app.kind == "Marathon") && @options[:deploy_kind] == 'chronos'
         result = false
       end
-      if (app.kind == "Chronos") && @options[:deploy_type] == 'marathon'
+      if (app.kind == "Chronos") && @options[:deploy_kind] == 'marathon'
         result = false
       end
-      if @options[:env_type] != 'all'
-        if @options[:env_type] != app.environment
-         result = false
+
+      if @options[:deploy_env] != 'all'
+        @options[:deploy_env].split(",").each do |x|
+          pair = x.split("=")
+          envVar = pair[0]
+          envVal = pair[1]
+          if app[envVar].nil?
+            puts "environment var of '#{envVar}' not found in app"
+            exit 1
+          end
+          if app[envVar] != envVal
+            result = false
+          end
         end
       end
 
@@ -182,9 +174,16 @@ module Orch
     end
 
     def do_subst(spec, app)
-      # {{ENV}} is a special value
-      spec_str = spec.to_json.to_s.gsub(/{{ENV}}/, app.environment)
+      spec_str = spec.to_json.to_s
+
+      # Subst any of the environment_vars values
+      if (! @spec.environment_vars.nil?)
+        @spec.environment_vars.each do |key, value|
+          spec_str = spec_str.gsub(/{{#{key}}}/, app[key])
+        end
+      end
 
+      # Subst any values that were passed in via command line
       if ! @options[:subst].nil?
         @options[:subst].split(",").each do |x|
           pair = x.split("=")
@@ -200,8 +199,8 @@ module Orch
         return
       end
 
-      if ! ['chronos', 'marathon', 'all'].include?(@options[:deploy_type])
-        puts "value of --deploy-type was #{@options[:deploy_type]}, must be chronos, marathon or all"
+      if ! ['chronos', 'marathon', 'all'].include?(@options[:deploy_kind])
+        puts "value of --deploy-type was #{@options[:deploy_kind]}, must be chronos, marathon or all"
         exit 1
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: orch
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Ray Johnson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-21 00:00:00.000000000 Z
+date: 2015-08-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -80,6 +80,7 @@ files:
 - README.md
 - Rakefile
 - bin/orch
+- examples/Examples.md
 - examples/env-chronos.yml
 - examples/simple-chronos.yml
 - examples/simple-marathon.yml