cfhighlander 0.8.2 → 0.9.0.alpha.1556659966

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 02fe854d713ab142174a9116c6d6710f9669eaf0256195b48eb1da86d191e0d8
-  data.tar.gz: f0aed4b99a952f156b95cd68670e9720173948e7532441dd989738b91fae94de
+  metadata.gz: 001df982a303f15386c95edefe223bb6d18e67adc357f0164b08f7b11b6735df
+  data.tar.gz: 287a98c600c8697d05b0439bfe44779022a32c48a48504d46d07f8fde61706ef
 SHA512:
-  metadata.gz: 4021a5058a31c3bb410d221184dabf8770659fad62047af43525ec599d03fe543d6d583e607839b375da37bfcb654267114dc273d8352c9d8dd321442da2612d
-  data.tar.gz: b3ca6ee6a1945654fff3155c739f093a075300ccd784e8bc525a8936f3dd768bab9220fb1ffbd07fa4caa8fcec6ea9e07caa1d8eb40841c3588bd41efed79dc7
+  metadata.gz: 988727c8d0185cdd2f7b33c790156374e4e01aae50b5424bcb1072ae1f3f1bf53ff5b0c399c8daf4c610babe6fdd7872bd68ed48e93268b91e468b06dc33f0ab
+  data.tar.gz: 6f30346bb04ebdb05ddcd93ecdd618a00b959f72d35c5ced2aaddd39b937ccecc0e3462adffe749e04427f300ad3839c3ea9ce7b730a53460952cdd974e18870

data/README.md

@@ -653,12 +653,140 @@ so extension methods can be consumed within cfndsl template.
 
 ### LambdaFunctions
 
+
+Cfhighlander supports following in terms of lambda source code management
+
+- Package and deploy lambda that has source code published to s3 as zip archive
+- Package and deploy lambda that has source code published to http(s) url
+- Package and deploy lambda with absolute source code location, or relative to component
+  root directory
+- When extending certain highlander component, of 
+- Execute arbitrary 'package command' before creating final archive. This allows for downloading
+code dependencies
+
+#### Configuration and rendering
+
+Lambda functions are defined in cfhighlander templates using `LambdaFunctions` DSL statement:
+
+```ruby
+CfhighlanderTemplate do
+    Name 'my_app'
+    LambdaFunctions 'highlanderdocoexample'
+end
+```
+
+In example above, `lambdas` - value passed to `LambdaFunctions` dsl statement is actually
+cfhighlander component configuration key, under which lambda functions, and their respective
+IAM roles are defined. Consider configuration below - all keys are commented with explanation
+
+```yaml
+highlanderdocoexample:
+
+  # custom policies can be referenced in roles
+  custom_policies:
+    cognito:
+      action:
+        - cognito-idp:*
+      resource: '*'
+
+  # at least one role must be defined
+  roles:
+    default:
+      # using one of the default policies, or custom policies defined above
+      # defined at https://github.com/theonestack/cfhighlander/blob/develop/cfndsl_ext/config/managed_policies.yaml
+      policies_inline:
+        - cloudwatch-logs
+        - cognito
+
+      # managed IAM policies are supported as well
+      policies_managed:
+        - arn:aws:iam::aws:policy/IAMReadOnlyAccess
+        - Fn::Sub: 'arn:aws:iam::${AWS::AccountId}:policy/my_app_policy'
+
+  # you can have multiple functions defined, each as key under 'functions'
+  functions:
+    myapp:
+
+      # link to a role key above defined - mandatory
+      role: default
+
+      # code location. Can be file, archive, s3://url or http(s)://url.
+      # mandatory configuration option
+      code: src/app.py
+
+      # lambda runtime
+      runtime: python3.6
+
+      # functions that are not named are having their name auto generated via cloudformation
+      # this key defaults to false if not given
+      named: true
+
+      # if function is named, either top level key (myapp) will be used, or explicit name
+      # this key is optional, and used only if named: is set to true
+      function_name:
+        Fn::Sub: '${EnvironmentName}-myapp-${EnvironmentVersion}'
+
+      # function timeout (defaults to 30)
+      timeout: 30
+
+      # lambda function entrypoint
+      handler: app.index
+
+      # command to install any dependencies (optional)
+      # if you don't want to get prompted for every command execution use -q (quiet) option
+      package_cmd: 'pip3 install -r requirements.txt -t .'
+
+      # (optional) allowed source. e.g. invocation using SNS
+      # for every allowed source, source_arn can be provided optionally
+      allowed_sources:
+       -
+         principal: sns.amazonaws.com
+       -
+         principal: sns.amazonaws.com
+         source_arn: arn:aws:sns:us-east-2:123456789012:my-topic
+
+      # (optional) invoke function on a schedule, with optional payload
+      schedules:
+       - cronExpression: 'rate(1 minute)'
+         payload: '{ "message": "ping" }'
+```
+
+
 #### Packaging and publishing
 
-#### Rendering
+During cfhighlander compilation process, every defined lambda functio goes through process
+of packaging:
+
+- If s3 URI or http(s) uri is given as function code, it is being downloaded 
+- Temporary packaging directory is created
+- If `package_cmd` key is given, this command is being executed in temporary directory
+- Whole temporary directory is compressed and moved to `out/lambdas/$function.$timestamp.zip` 
+- Sha256 hash is calculated for given file and rendered into cloudformation as function 
+  version
+- Packaging information is rendered into `out/lambdas/$function.$timestamp.zip.info.yaml` 
+  and added to final archive
+  
+Any archive with `*.zip` extension will be uploaded to s3 with `cfpublish` command
+
 
 #### Referencing
 
+As all of the lambda functions are rendered as cloudformation resources, they can be 
+referenced in other blocks. E.g. with example above, application component could have 
+following output defined in component's cfndsl file
+
+```ruby
+# myapp.cfndsl.rb
+CloudFormation do
+
+    Output('MyAppFunctionName') do
+        Value(Ref('myapp'))
+    end
+
+end
+
+
+```
 
 ## Finding templates and creating components
 

data/bin/cfhighlander.rb

@@ -39,6 +39,7 @@ class HighlanderCli < Thor
 
     # compile cfndsl template
     component_compiler = Cfhighlander::Compiler::ComponentCompiler.new(component)
+    component_compiler.clear_out_dir
     component_compiler.writeConfig(true)
   end
 
@@ -59,6 +60,7 @@ class HighlanderCli < Thor
 
     # compile cfndsl template
     component_compiler = Cfhighlander::Compiler::ComponentCompiler.new(component)
+    component_compiler.clear_out_dir
     component_compiler.silent_mode = options[:quiet]
     out_format = options[:format]
     component_compiler.compileCfnDsl out_format
@@ -102,6 +104,7 @@ class HighlanderCli < Thor
 
     # compile cloud formation
     component_compiler = Cfhighlander::Compiler::ComponentCompiler.new(component)
+    component_compiler.clear_out_dir
     component_compiler.silent_mode = options[:quiet]
     out_format = options[:format]
     component_compiler.compileCloudFormation out_format

data/cfndsl_ext/lambda_helper.rb

@@ -43,7 +43,11 @@ def render_lambda_functions(cfndsl, lambdas, lambda_metadata, distribution)
       end
 
       if !lambda_config['named'].nil? && lambda_config['named']
-        FunctionName(name)
+        if lambda_config['function_name'].nil?
+          FunctionName(name)
+        else
+          FunctionName(lambda_config['function_name'])
+        end
       end
     end
 
@@ -53,6 +57,33 @@ def render_lambda_functions(cfndsl, lambdas, lambda_metadata, distribution)
       CodeSha256(lambda_metadata['sha256'][key])
     end
 
+    # Scheduled triggering of lambda function
+    if lambda_config.key?('schedules')
+      lambda_config['schedules'].each_with_index do |schedule, index|
+        Events_Rule("Lambda#{name}Schedule#{index}") do
+          if schedule['cronExpression'].include?('rate')
+          then
+            expression = schedule['cronExpression']
+          else
+            expression = "cron(#{schedule['cronExpression']})"
+          end
+          ScheduleExpression(expression)
+          State('ENABLED')
+          target = {
+              'Arn' => FnGetAtt(name, 'Arn'), 'Id' => "lambda#{name}",
+          }
+          target['Input'] = schedule['payload'] if schedule.key?('payload')
+          Targets([target])
+        end
+        Lambda_Permission("Lambda#{name}Schedule#{index}Permission") do
+          FunctionName(Ref(name))
+          Action('lambda:InvokeFunction')
+          Principal('events.amazonaws.com')
+          SourceArn FnGetAtt("Lambda#{name}Schedule#{index}", 'Arn')
+        end
+      end
+    end
+
     # Generate lambda function Policy
     unless lambda_config['allowed_sources'].nil?
       i = 1
@@ -69,21 +100,6 @@ def render_lambda_functions(cfndsl, lambdas, lambda_metadata, distribution)
       end
     end
 
-    # Scheduled triggering of lambda function
-    if lambda_config.key?('schedules')
-      lambda_config['schedules'].each_with_index do |schedule, index|
-        Events_Rule("Lambda#{name}Schedule#{index}") do
-          ScheduleExpression("cron(#{schedule['cronExpression']})")
-          State('ENABLED')
-          target = {
-              'Arn' => FnGetAtt(name, 'Arn'), 'Id' => "lambda#{name}",
-          }
-          target['Input'] = schedule['payload'] if schedule.key?('payload')
-          Targets([target])
-        end
-      end
-    end
-
 
   end
 end

data/lib/cfhighlander.compiler.rb

@@ -51,7 +51,7 @@ module Cfhighlander
 
         if @@global_extensions_paths.empty?
           global_extensions_folder = "#{File.dirname(__FILE__)}/../cfndsl_ext"
-          Dir["#{global_extensions_folder}/*.rb"].each {|f| @@global_extensions_paths << f}
+          Dir["#{global_extensions_folder}/*.rb"].each { |f| @@global_extensions_paths << f }
         end
 
         @component.highlander_dsl.subcomponents.each do |sub_component|
@@ -61,14 +61,19 @@ module Cfhighlander
         end
       end
 
+      def clear_out_dir
+        # Clear previous packages
+        FileUtils.rmtree "#{@workdir}/out/"
+      end
+
       def process_lambdas=(value)
         @process_lambdas = value
-        @sub_components.each {|scc| scc.process_lambdas = value}
+        @sub_components.each { |scc| scc.process_lambdas = value }
       end
 
       def silent_mode=(value)
         @silent_mode = value
-        @sub_components.each {|scc| scc.silent_mode = value}
+        @sub_components.each { |scc| scc.silent_mode = value }
       end
 
       def compileCfnDsl(out_format)
@@ -77,7 +82,7 @@ module Cfhighlander
         dsl = @component.highlander_dsl
         component_cfndsl = @component.cfndsl_content
 
-        @component.highlander_dsl.subcomponents.each {|sc|
+        @component.highlander_dsl.subcomponents.each { |sc|
           sc.distribution_format = out_format
         }
 
@@ -91,7 +96,7 @@ module Cfhighlander
             'component_cfndsl' => component_cfndsl,
             'component_requires' => (@@global_extensions_paths + @component.cfndsl_ext_files),
             'distribution_format' => out_format
-        }).instance_eval {binding})
+        }).instance_eval { binding })
 
         # write to output file
         output_dir = "#{@workdir}/out/cfndsl"
@@ -102,7 +107,7 @@ module Cfhighlander
         puts "cfndsl template for #{dsl.name} written to #{output_path}"
         @cfndsl_compiled_path = output_path
 
-        @sub_components.each {|subcomponent_compiler|
+        @sub_components.each { |subcomponent_compiler|
           puts "Rendering sub-component cfndsl: #{subcomponent_compiler.component_name}"
           subcomponent_compiler.compileCfnDsl out_format
         }
@@ -230,9 +235,6 @@ module Cfhighlander
 
       def generateSourceArchives
 
-        # Clear previous packages
-        FileUtils.rmtree "#{@workdir}/output/lambdas"
-
         archive_paths = []
 
         # Cached downloads map
@@ -299,13 +301,27 @@ module Cfhighlander
               # zip local code
               component = @component
               component_dir = component.template.template_location
-              full_path = "#{component_dir}/lambdas/#{lambda_config['code']}"
-
+              full_path_candidate_1 = "#{component_dir}/lambdas/#{lambda_config['code']}"
+              full_path_candidate_2 = "#{component_dir}/#{lambda_config['code']}"
+              full_path_candidate_3 = lambda_config['code']
+              full_path = full_path_candidate_1
+              full_path = full_path_candidate_2 if (File.exist? full_path_candidate_2 and (not File.exist? full_path))
+              full_path = full_path_candidate_3 if (File.exist? full_path_candidate_3 and (not File.exist? full_path))
+
+              # if component extends another component, both parent and child paths should be taken in
+              # consideration
               until (File.exist? full_path or component_dir.nil?)
                 parent_exists = (not component.extended_component.nil?)
                 component = component.extended_component if parent_exists
                 component_dir = component.template.template_location if parent_exists
-                full_path = "#{component_dir}/lambdas/#{lambda_config['code']}" if parent_exists
+                full_path_candidate_1 = "#{component_dir}/lambdas/#{lambda_config['code']}" if parent_exists
+                full_path_candidate_2 = "#{component_dir}/#{lambda_config['code']}" if parent_exists
+                full_path_candidate_3 = "#{lambda_config['code']}" if parent_exists
+
+                full_path = full_path_candidate_1
+                full_path = full_path_candidate_2 if (File.exist? full_path_candidate_2 and (not File.exist? full_path))
+                full_path = full_path_candidate_3 if (File.exist? full_path_candidate_3 and (not File.exist? full_path))
+
                 component_dir = nil unless parent_exists
               end
               if component_dir.nil?
@@ -343,9 +359,14 @@ module Cfhighlander
                 end
               end
               File.delete full_destination_path if File.exist? full_destination_path
-              zip_generator = Cfhighlander::Util::ZipFileGenerator.new(lambda_source_dir, full_destination_path)
-              zip_generator.write
 
+              # if source is already zip file, just add manifest to it
+              if full_path.end_with? '.zip'
+                FileUtils.copy full_path, full_destination_path
+              else
+                zip_generator = Cfhighlander::Util::ZipFileGenerator.new(lambda_source_dir, full_destination_path)
+                zip_generator.write
+              end
             end
           end
           # add version information to avoid same package ever deployed 2 times
@@ -357,13 +378,19 @@ module Cfhighlander
           puts "INFO | Lambda #{name} | Created zip package #{full_destination_path} with digest #{sha256}"
           @metadata['sha256'][name] = sha256
           @metadata['version'][name] = timestamp
-        end
+        end if ((not @lambda_config.nil?) and @lambda_config.key? 'functions')
 
         return archive_paths
       end
 
       def mergeComponentConfig
-        @component.config['lambda_metadata'] = @metadata
+        if @component.config.key? 'lambda_metadata'
+          @metadata.each do |mk,mh|
+            mh.each do |k,v| @component.config['lambda_metadata'][mk][k] = v end
+          end
+        else
+          @component.config['lambda_metadata'] = @metadata
+        end
       end
 
     end

data/lib/cfhighlander.version.rb

@@ -1,3 +1,3 @@
 module Cfhighlander
-  VERSION="0.8.2".freeze
+  VERSION="0.9.0".freeze
 end

data/lib/util/zip.util.rb

@@ -1,6 +1,5 @@
 require 'zip'
 
-
 module Cfhighlander
   module Util
     ###
@@ -18,7 +17,7 @@ module Cfhighlander
       def write
         entries = Dir.entries(@input_dir) - %w(. ..)
         puts "DEBUG: Compressing #{@input_dir} to #{@output_file}"
-        ::Zip::File.open(@output_file, ::Zip::File::CREATE) do |zipfile|
+        Zip::File.open(@output_file, ::Zip::File::CREATE) do |zipfile|
           write_entries entries, '', zipfile
         end
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cfhighlander
 version: !ruby/object:Gem::Version
-  version: 0.8.2
+  version: 0.9.0.alpha.1556659966
 platform: ruby
 authors:
 - Nikola Tosic
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-02-27 00:00:00.000000000 Z
+date: 2019-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: highline
@@ -297,11 +297,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubygems_version: 3.0.2
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: DSL on top of cfndsl. Manage libraries of cloudformation components