monkey_king 0.1.5 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: beb82abaa898573f203304c24f512196d2188232
-  data.tar.gz: b303c14f5e6000d5287456d75db0ca2c39570be9
+  metadata.gz: c24e35b55ba57f034926c86a82c3a095b46addeb
+  data.tar.gz: 254ebb7b6fcff6acc074a2dbcd5b88ea49668d2f
 SHA512:
-  metadata.gz: 4a5aabc89c3e9404a0df74936f165f37c3db1c5fcc8dc917a48e18969d6f1b8d163ea3ec249ba43349ed994123a05a3dfadfec84b340ebdd50b1d366d98b7588
-  data.tar.gz: 20c4198faa50e4a02a4ec7949ff32f9f19570b1da17ab3fb24f8b17e83cba75014e5011a8e734539e38b1d5b2188084610cf2d04c076eea6403f644cfa350944
+  metadata.gz: 626b160499b2067a239196a0351b38fb63b4d377e0c85b54092d1ae466128a118800098ff860a7a10b7128b78e8ced7b9ff999dd700cc9f395f69dcc00c72516
+  data.tar.gz: d4a51f2ee54a8d9010397c49a61b05ec6bd3ad597291b54109f79cdbb15c860e2a80d7acb031ce5aeb5d14905031e3f3310846ecee73c5edc3058d8d4f49cac2

data/Gemfile

@@ -8,4 +8,5 @@ gem 'climate_control'
 gem 'pry-byebug'
 gem 'mothership'
 gem 'highline'
+gem 'sexpistol'
 

data/README.md

@@ -2,30 +2,32 @@
 
 Monkey king is a tool which is initially designed for generating deployment manifests for a bosh deployment based on an existing deployment, and it could also be used for other purposes like performing functions on keys/values in yaml files based on yaml tags.
 
+Here is some scenarios about how it works, some of these examples are in the `./fixtures/` directory:
 
-Here is some scenarios about how it works:
 ## Using secret generation
-The `!MK:secret` directive generates a random secret of the same length as the value it replaces.
-BEFORE
+The `!MK:secret(<password_length>)` directive generates a random secret using [a-Z,0-9] with the length specified in the parameter.
+
+Before:
+
 ```
 ---
 meta:
-  secret: !MK:secret old_secret
-  another_secret: !MK:secret old_secret
+  secret: !MK:secret(12) replace_me
+  another_secret: !MK:secret replace_me
   not_secret: not_secret
 ```
 
-AFTER
+After:
 ```
 ---
 meta:
-  secret: !MK:secret new_secret
-  another_secret: !MK:secret new_secret
+  secret: !MK:secret(12) SomeRandomPass
+  another_secret: !MK:secret() AnotherRandomPass
   not_secret: not_secret
 ```
 
 ## Using environment variables
-The `!MK:env:<variable_name>` directive pulls values from environment variables and replaces the tagged keys/values
+The `!MK:env(<variable_name>)` directive pulls values from environment variables and replaces the tagged keys/values
 
 Given:
 
@@ -40,10 +42,10 @@ Before:
 meta1:
   not_secret: not_secret
   layer1:
-  - id1: !MK:env:id1 id1_before
+  - id1: !MK:env(id1) id1_before
   - layer2:
-    - id2: !MK:env:id2 id2_before
-  - !MK:env:id1 id3: !MK:env:id1 id1_before
+    - id2: !MK:env(id2) id2_before
+  - !MK:env(id1) id3: !MK:env:id1 id1_before
 ```
 
 After:
@@ -52,13 +54,110 @@ After:
 meta1:
   not_secret: not_secret
   layer1:
-  - id1: !MK:env:id1 id1_from_env
+  - id1: !MK:env(id1) id1_from_env
   - layer2:
-    - id2: !MK:env:id2 id2_from_env
-  - !MK:env:id1 id1_from_env: !MK:env:id1 id1_from_env
+    - id2: !MK:env(id2) id2_from_env
+  - !MK:env(id1) id1_from_env: !MK:env:id1 id1_from_env    
+```
+
+## Using Read and Write
+The `!MK:read(<variable_name>)` and `!MK:write(<variable_name>,<value>)` directive is used to save the generated value and use it later.
+
+**NOTE: at this time, reads must be ordered after writes in the YAML document until we implement a dependency graph.**
+
+Before:
+
+```
+---
+meta:
+  secret: !MK:write(nat_secret,secret(12)) replace_me
+  same_secret_again: !MK:read(nat_secret) replace_me
+```
+
+After:
+
+```
+---
+meta:
+  secret: !MK:write(nat_secret,secret(12)) SAME_PASSWORD_HERE
+  same_secret_again: !MK:read(nat_secret) SAME_PASSWORD_HERE
+```
+
+# Using string format
+The `!MK:format(<variable_1>,<variable_2>,...,<string>)` directive can be used to format the string given in the yaml field. This is usally used with `write_value` directive which store the template to a variable.
+
+**NOTE: The string literal must be defined outside of the YAML tag, as there is a limited set of allowed characters in YAML tags. See usage of `nat_template` in example below**
+
+Given:
+
+```
+export NAT_HOST=10.10.0.6
+```
+
+Before:
+
+```
+---
+nat_template: write_value(TEMPLATE) https://%s
+meta:
+  nat_url: !MK:format(env(NAT_HOST),read(TEMPLATE)) replaceme
+```
+
+After:
+```
+---
+nat_template: write_value(TEMPLATE) https://%s
+meta:
+  nat_url: !MK:format(env(NAT_HOST),read(TEMPLATE)) https://10.10.0.6
+```
+
+## Combine them all
+You can combine the directive in a LISP-like syntax to create more poweful usages:
+
+####Example:
+
+Given:
+
+```
+export NATS_USER=nats_user
+export NATS_HOST=10.10.0.6
+```
+
+Before:
+
+```
+nat_template_1: !MK:write_value(NAT_TEMPLATE_1) https://%s:%s@%s
+nat_template_2: !MK:write_value(NAT_TEMPLATE_2) '%s/info'
+
+meta1:
+  not_secret: not_secret
+  layer1:
+  - nat_user: !MK:write(NATS_USER,env(NATS_USER)) replaceme
+  - nat_host: !MK:write(NATS_HOST,env(NATS_HOST)) replaceme
+  - nat_password: !MK:write(NATS_PASSWORD,secret(12)) replaceme
+  layer2:
+  - nat_connection: !MK:write(NATS_STRING,format(read(NATS_USER),read(NATS_PASSWORD),read(NATS_HOST),read(NAT_TEMPLATE_1))) 
+  - info_endpoint: !MK:format(read(NATS_STRING),read(NAT_TEMPLATE_2)) 
 ```
 
 
+After:
+
+```
+---
+nat_template_1: !MK:write_value(NAT_TEMPLATE_1) https://%s:%s@%s
+nat_template_2: !MK:write_value(NAT_TEMPLATE_2) '%s/info'
+meta1:
+  not_secret: not_secret
+  layer1:
+  - nat_user: !MK:write(NATS_USER,env(NATS_USER)) nats_user
+  - nat_host: !MK:write(NATS_HOST,env(NATS_HOST)) 10.10.0.6
+  - nat_password: !MK:write(NATS_PASSWORD,secret(12)) WxhUE4RoJXYF
+  layer2:
+  - nat_connection: !MK:write(NATS_STRING,format(read(NATS_USER),read(NATS_PASSWORD),read(NATS_HOST),read(NAT_TEMPLATE_1))) https://nats_user:WxhUE4RoJXYF@10.10.0.6
+  - info_endpoint: !MK:format(read(NATS_STRING),read(NAT_TEMPLATE_2)) https://nats_user:WxhUE4RoJXYF@10.10.0.6/info
+```
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -74,14 +173,35 @@ And then execute:
 Or install it yourself as:
 
     $ gem install monkey_king
+    
+**Or if you want the cutting edge**
+
+```
+git clone https://github.com/pivotal-cloudops/monkey_king.git
+cd monkey_king
+bundle exec mk
+```
+
+## Try it Out:
+
+You can create a yaml file (example: demo.yml in ~/tmp) with the 'MK' yaml tags as described earlier.
+
+Then run:
+
+```
+cd monkey_king
+bundle exec mk demo ~/tmp/demo.yml
+``` 
+
 
-## Usage
+## Full Usage
 ```
-$ mk help
+$ mk
 Commands:
-  help [COMMAND]   	Help!
-  clone REPO DIR...	Clone the repo and replace secret and env annotation
-  replace GLOBS... 	Replace secret and env annotation for existing directory
+  help [COMMAND]  	Help!
+  clone REPO DIR  	use MK to clone github repo and transform
+  replace GLOBS...	Do MK transform for existing directory(ies)
+  demo FILE       	Demo MK transform for one file```
 ```
 ```
 $ mk help clone

data/lib/monkey_king/clone_command.rb

@@ -4,7 +4,7 @@ require 'highline/import'
 module MonkeyKing
   class CloneCommand < Mothership
 
-    desc "Clone the repo and replace secret and env annotation"
+    desc "use MK to clone github repo and transform"
     input :repo, :argument => true
     input :dir, :argument => [:splat, true]
     def clone
@@ -23,6 +23,7 @@ module MonkeyKing
 
       deployment_yaml_files.each do |file|
         puts "Transforming #{file}..."
+        MonkeyKing.variables = {}
         transformed_content = parser.transform(file)
         File.open(file, "w") do |overwrite_file|
           overwrite_file.write transformed_content
@@ -31,7 +32,7 @@ module MonkeyKing
       puts "Done."
     end
 
-    desc "Replace secret and env annotation for existing directory"
+    desc "Do MK transform for existing directory(ies)"
     input :globs, :argument => :splat
     def replace
       globs = input[:globs]
@@ -47,6 +48,7 @@ module MonkeyKing
         extension = file.split('.').last
         if extension == 'yml'
           puts "Transforming #{file}..."
+          MonkeyKing.variables = {}
           transformed_content = parser.transform(file)
           File.open(file, "w") do |overwrite_file|
             overwrite_file.write transformed_content
@@ -58,5 +60,13 @@ module MonkeyKing
       puts "Done."
     end
 
+    desc "Demo MK transform for one file"
+    input :file, :argument => true
+    def demo
+      file = input[:file]
+      parser = MonkeyKing::Parser.new
+      puts parser.transform(file)
+    end
+
   end
 end

data/lib/monkey_king/parser.rb

@@ -1,84 +1,150 @@
 require 'yaml'
+require 'securerandom'
+require 'sexpistol'
+require 'pry'
 
 module MonkeyKing
-  class SecretTag
-    yaml_tag '!MK:secret'
+  @@variables = {}
 
-    attr_reader :secret
+  def self.variables
+    @@variables
+  end
+
+  def self.variables=(value)
+    @@variables = value
+  end
+
+  def self.set_variable(name, value)
+    @@variables[name] = value
+    return value
+  end
+
+  class FunctionTag
+    attr_accessor :scalar
 
-    def initialize( *secret )
-      self.secret = *secret
+    def register(tag)
+      self.class.send(:yaml_tag, tag)
     end
 
-    def init_with( coder )
-      case coder.type
-      when :scalar
-        self.secret = coder.scalar
-      else
+    def init_with(coder)
+      unless coder.type == :scalar
         raise "Dunno how to handle #{coder.type} for #{coder.inspect}"
       end
+      self.scalar = coder.scalar
     end
 
-    def encode_with( coder )
+    def encode_with(coder)
       coder.style = Psych::Nodes::Mapping::FLOW
-      coder.scalar = gen_secret(@secret.length)
+      s_expression = coder.tag.sub(/^!MK:/, '')
+      s_expression.gsub!(/,/, ' ')
+      expression_tree = Sexpistol.new.parse_string(s_expression)
+      coder.scalar = expand(expression_tree).first
     end
 
-    protected def secret=( str )
-      @secret= str
-    end
 
-    def gen_secret(length)
-      [*('a'..'z'),*('0'..'9'),*('A'..'Z')].shuffle[0,length].join
+    def argment_count_checking(params_count, legit_count, function_name)
+      if params_count > legit_count
+        raise "too many arguments for #{function_name} function (#{params_count} of #{legit_count})"
+      elsif params_count < legit_count
+        raise "not enough arguments for #{function_name} function (#{params_count} of #{legit_count})"
+      end
     end
-  end
 
-  class EnvTag
-    attr_reader :env_tag
+    def expand(expression)
+      if expression.is_a? Array
+        function_array = []
+        expression.each do |ex|
+          function_array << expand(ex)
+        end
 
-    def register(tag)
-      if tag.split(':')[1] == 'env'
-        self.class.send(:yaml_tag, tag)
+        process_array = []
+
+        while !function_array.empty? do
+          key_word = function_array.shift
+          case key_word
+          when :write
+            params = function_array.shift
+            argment_count_checking(params.size, 2, key_word)
+            key = params.first
+            value = params.last
+            raise "attempting to redefine immutable variable #{key}, exiting" unless MonkeyKing.variables[key].nil?
+            process_array << MonkeyKing.set_variable(key, value.to_s)
+          when :read
+            params = function_array.shift
+            argment_count_checking(params.size, 1, key_word)
+            key = params.first
+            raise "unresolved variables #{key}" if MonkeyKing.variables[key].nil?
+            process_array << MonkeyKing.variables[key]
+          when :secret
+            params = function_array.shift
+            argment_count_checking(params.size, 1, key_word)
+            raise "argument error for secret function: got #{params.first.class} instead of Fixnum" if !params.first.is_a? Fixnum
+            length = params.first
+            process_array << gen_secret(length)
+          when :env
+            params = function_array.shift
+            argment_count_checking(params.size, 1, key_word)
+            key = params.first.to_s
+            process_array << get_env(key)
+          when :write_value
+            params = function_array.shift
+            argment_count_checking(params.size, 1, key_word)
+            key = params.first
+            process_array << MonkeyKing.set_variable(key, self.scalar)
+          when :format
+            params = function_array.shift
+            formating_string = params.pop
+            raise('format template not found') if formating_string.nil?
+            replace_count =  formating_string.scan(/%s/).count
+            argment_count_checking(params.size, replace_count, key_word)
+            params.each do |param|
+              formating_string = formating_string.sub(/%s/, param.to_s)
+            end
+            process_array << formating_string
+          else
+            process_array << key_word
+          end
+        end
+        return process_array
+
+      elsif expression.is_a? Symbol
+        return expression
+      elsif expression.is_a? Numeric
+        return expression
+      elsif expression.is_a? String
+        return expression
+      else
+        raise "unknown expression #{expression}"
       end
     end
 
-    def init_with( coder )
-      unless coder.type == :scalar
-        raise "Dunno how to handle #{coder.type} for #{coder.inspect}"
-      end
+    def get_env(key)
+      raise "#{key} not found in env" if ENV[key].nil?
+      return ENV[key]
     end
 
-    def encode_with(coder)
-      coder.style = Psych::Nodes::Mapping::FLOW
-      tag=coder.tag.split(':')[2]
-      if ENV[tag].nil?
-        raise "#{tag} not found in env"
-      end
-      coder.scalar = ENV[tag]
+    def gen_secret(length)
+      return [*('a'..'z'),*('0'..'9'),*('A'..'Z')].shuffle[0,length].join
     end
   end
 
   class Parser
+
     def transform(yaml_file)
+      function_tag_instances = {}
       tags = get_tags(yaml_file)
-      env_tag_instances={}
+
       tags.each do |tag|
-        command = tag.split(':')[1]
-        unless command.nil? or command != 'env'
-          class_name = tag.split(':')[2]
-          unless class_name.nil?
-            tag_class = Class.new(EnvTag)
-
-            # Hacky way to give each class a global uniq name
-            random_string = [*('a'..'z')].shuffle[0,32].join
-            Object.const_set("EnvTag#{class_name}#{random_string}", tag_class)
-
-            tag_instance = tag_class.new
-            tag_instance.register(tag)
-            env_tag_instances[tag] = tag_instance
-          end
+        if tag =~ /!MK:/
+          tag_class = Class.new(FunctionTag)
+          random_string = SecureRandom.uuid.gsub(/-/, '')
+          Object.const_set("FunctionTag#{random_string}", tag_class)
+          tag_instance = tag_class.new
+          tag_instance.register(tag)
+          function_tag_instances[tag] = tag_instance
         end
       end
+
       yaml = YAML.load_file(yaml_file)
       yaml.to_yaml
     end
@@ -97,5 +163,4 @@ module MonkeyKing
       tags.uniq
     end
   end
-
 end

data/lib/monkey_king/version.rb

@@ -1,3 +1,3 @@
 module MonkeyKing
-  VERSION = "0.1.5"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: monkey_king
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.2.0
 platform: ruby
 authors:
 - cloudops_hosted
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-15 00:00:00.000000000 Z
+date: 2016-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mothership