cloudformation-tool 0.9.1 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 3e948781a997b22548ec5adf8ff16d7eb0bbc3db
-  data.tar.gz: dfef8541f9c310b4a69b064ee7206b3827e6d0be
+SHA256:
+  metadata.gz: ef4e953cd8194f197b68c1e110dae6f3a81f5c4e2730fe63950d5ac1392f1b02
+  data.tar.gz: 9d5ea95468d689707ec79534281ea14ae51521d3491787e523a1ef0c11c295a8
 SHA512:
-  metadata.gz: e2e5f5a9026d6fc920c232b0fad618beb951c5b79d8e1507b715ea690fb7c9597e25e6ec1361c19b706ae78203cd5ee411a6c445e7841c2cbe7a0371f354d44a
-  data.tar.gz: 2ab9fb5203d7fa729a70640c215dc0db4721e61b2a8b9ba7010a1390425893e87c1d30c3bedb22168ae20543968ded5001b42b561f47151e5d07b7978323eefe
+  metadata.gz: d43ffe6a9b74696f7a96c36a4f25bcc51322bd61f6d72000133ba6ef13720bd9412132c693597d40018b9bfd771038e576a1739e3c0b2e1cf895ecbdfaaaeee1
+  data.tar.gz: 1b6226952862d335bfa9970b8c394e9201548284af96363a39c54c00aac1a6ba946207c2076676c983fbe17c07fef566c23047aec842cc4dd71db6a3f4b7c31d

data/README.md

@@ -192,8 +192,10 @@ specifying the S3 bucket and object key, either of the following fields may be u
   - The field `URL` may be used to specify an HTTP URL	from which the code is to be uploaded to AWS Lambda. The tool 
     will download the code file from the specified URL, upload it to S3 and specify the correct S3 location for 
     CloudFormation.
-  - The field `Path` may be used to specify a local file or directory containing the code to be uploaded. 
-    If the path specifies a directory, it will be compressed as a Zip file. 
+  - The field `Path` may be used to specify a local file or directory containing the code to be uploaded.
+    If the path specifies a directory, it will be compressed and uploaded to S3 as a Zip file. If the path is a
+    single file, it will be converted to a `ZipFile`, allowing implicit use of the CloudFormation `cfn-response` module
+    and the AWS SDK, but the file is also subject to all `ZipFile` restrictions - such as limited to 4KB size.
 
 #### Example:
 

data/lib/cloud_formation_tool/cli/compile.rb

@@ -14,7 +14,7 @@ module CloudFormationTool
         else
           tpl = CloudFormation.parse(file)
           params = get_params
-          data = tpl.compile;
+          data = tpl.compile(params);
           data['Parameters'].each do |name,param|
             param['Default'] = params[name] if params.has_key? name
           end

data/lib/cloud_formation_tool/cloud_formation/lambda_code.rb

@@ -7,47 +7,92 @@ module CloudFormationTool
     class LambdaCode
       include Storable
       
-      def initialize(url: nil, path: nil)
-        log "Downloading Lambda code from #{url}#{path}"
-        case url
-          when nil
-            @s3_url = if File.directory?(path)
-                        URI(upload(make_filename(path.split('/').last), fetch_from_folder(path), mime_type: 'application/zip',  gzip: false))
-                      else
-                        URI(upload(make_filename(path.split('/').last), File.open(path, "rb").read, gzip: false))
-                      end
-          else
+      def initialize(code, tpl)
+        @data = code
+        @data['Url'] = @data.delete 'URL' if @data.key? 'URL' # normalize to CF convention if seeing old key
+        if @data.key? 'Url'
+          log "Trying Lambda code from #{@data['Url']}"
+          @data['Url'] = url = tpl.resolveVal(@data['Url'])
+          return unless url.is_a? String
+          log "Downloading Lambda code from #{url}"
+          unless already_in_cache(url)
             res = fetch_from_url(url)
             @s3_url = URI(upload(make_filename(url.split('.').last), res.body, mime_type: res['content-type'], gzip: false))
+            log "uploaded Lambda function to #{@s3_url}"
           end
-        log "uploaded Lambda function to #{@s3_url}"
+        elsif @data.key? 'Path'
+          @data['Path'] = path = tpl.resolveVal(@data['Path'])
+          return unless path.is_a? String
+          log "Reading Lambda code from #{path}"
+          path = if path.start_with? "/" then path else "#{tpl.basedir}/#{path}" end
+          if File.directory?(path)
+            @s3_url = URI(upload(make_filename('zip'), zip_path(path), mime_type: 'application/zip', gzip: false))
+            log "uploaded Lambda function to #{@s3_url}"
+          else # Convert files to ZipFile
+            @data.delete 'Path'
+            @data['ZipFile'] = File.read(path)
+          end
+        end
       end
       
-      def fetch_from_folder(path_str)
+      def zip_path(path)
+        temp_file = Tempfile.new
+        temp_path = temp_file.path + '.zip'
         begin
-          temp_file = Tempfile.new("#{path_str.split('/').last}.zip")
-          Zip::ZipOutputStream.open(temp_file) { |zos| }
-          Zip::ZipFile.open(temp_file.path, Zip::ZipFile::CREATE) do |zipfile|
-            Dir[File.join(path_str, '*')].each do |file|
-              zipfile.add(file.sub("#{path_str}/", ''), file)
+          Zip::ZipFile.open(temp_path, true) do |zipfile|
+            Dir[File.join(path, '**','*')].each do |file|
+              zipfile.add(file.sub("#{path}/", ''), file)
             end
           end
-          zip_data = File.read(temp_file.path)
+          File.read(temp_path)
         ensure
-          temp_file.close
-          temp_file.unlink
+          temp_file.close!
+          File.unlink temp_path
         end
-        zip_data
       end
       
-    def fetch_from_url(uri_str)
-      $__fetch_cache ||= Hash.new do |h, url|
-        h[url] = fetch_from_url_real(url)
+      def already_in_cache(uri_str, limit = 10)
+        raise ArgumentError, 'too many HTTP redirects' if limit == 0
+        url = URI(uri_str)
+        begin
+          Net::HTTP.start(url.host, url.port) do |http|
+            request = Net::HTTP::Get.new(url)
+            http.request(request) do |response|
+              # handle redirects like Github likes to do
+              case response
+                when Net::HTTPSuccess then
+                  check_cached(response['ETag'])
+                when Net::HTTPRedirection then
+                  location = response['location']
+                  log "redirected to #{location}"
+                  already_in_cache(location, limit - 1)
+                else
+                  raise ArgumentError, "Error getting response: #{response}"
+              end
+            end
+          end
+        rescue EOFError
+        end
+        !@s3_url.nil?
+      end
+      
+      def check_cached(etag)
+        etag.gsub!(/"/,'') unless etag.nil?
+        o = cached_object(etag)
+        unless o.nil?
+          log 'reusing cached object'
+          @s3_url = o.public_url
+        end
+      end
+      
+      def fetch_from_url(uri_str)
+        $__fetch_cache ||= Hash.new do |h, url|
+          h[url] = fetch_from_url_real(url)
+        end
+        $__fetch_cache[uri_str]
       end
-      $__fetch_cache[uri_str]
-    end
     
-    def fetch_from_url_real(uri_str, limit = 10)
+      def fetch_from_url_real(uri_str, limit = 10)
         raise ArgumentError, 'too many HTTP redirects' if limit == 0
         response = Net::HTTP.get_response(URI(uri_str))
         case response
@@ -63,10 +108,14 @@ module CloudFormationTool
       end
       
       def to_cloudformation
-        {
-          'S3Bucket' => @s3_url.hostname.split('.').first,
-          'S3Key' => @s3_url.path[1..-1]
-        }
+        if @s3_url.nil?
+          @data
+        else
+          {
+            'S3Bucket' => @s3_url.hostname.split('.').first,
+            'S3Key' => @s3_url.path[1..-1]
+          }
+        end
       end
     end
 

data/lib/cloud_formation_tool/cloud_formation.rb

@@ -10,6 +10,8 @@ module CloudFormationTool
       CloudFormation.new(path)
     end
     
+    attr_reader :basedir
+    
     def initialize(path)
       log "Loading #{path}"
       @path = path
@@ -35,9 +37,8 @@ module CloudFormationTool
       end
     end
     
-    def compile
-      return @data if @compiled
-      @compiled = true
+    def compile(parameters = nil)
+      @params = parameters unless parameters.nil?
       embed_includes
       @data = load_files(@data)
     end
@@ -127,7 +128,7 @@ module CloudFormationTool
     def resolveVal(value)
       case value
       when Hash
-        if value['Ref']
+        if value.key? 'Ref'
           if @params.nil?
             # no parameters, we are probably in a sub template, just return the ref and hope
             # a parent template has what it takes to resolve the ref
@@ -143,41 +144,37 @@ module CloudFormationTool
           raise CloudFormationTool::Errors::AppError, "Value #{value} is not a valid value or reference"
         end
       else
-        value;
+        value
       end
     end
     
-    def load_files(data)
+    def load_files(data, restype = nil)
       case data
       when Array
-        data.collect { |data| load_files(data) }
+        data.collect { |data| load_files(data, restype) }
       when Hash
+        # remember the current resource type
+        restype = data['Type'] if restype.nil? and data.key?('Type')
         data.inject({}) do |dict, (key, val)|
-          dict[key] = if (key == "UserData") and (val["File"]) 
-            # Support LaunchConfiguration UserData from file
-            CloudInit.new("#{@basedir}/#{val["File"]}").to_base64
-          elsif (key == "UserData") and (val["FileTemplate"]) 
-            # Support LaunchConfiguration UserData from file with substitutions
-            { "Fn::Base64" => { "Fn::Sub" => CloudInit.new("#{@basedir}/#{val["FileTemplate"]}").compile } }
-          elsif (key == "Code") and (val["URL"])
-            # Support Lambda Code from arbitrary URLs
-            url = resolveVal(val["URL"])
-            log "Resolving lambda code URL: #{url}"
-            if url.is_a? String # resolving works
-              LambdaCode.new(url: url).to_cloudformation
-            else # resolving didn't work - we probably don't have parameters
-              # push it upstream and hope a parent template can resolve it
-              val
-            end
-          elsif (key == "Code") and (val["Path"])
-            path = resolveVal(val["Path"])
-            if path.is_a? String # resolving works
-              LambdaCode.new(path: if path.start_with? "/" then path else "#{@basedir}/#{path}" end).to_cloudformation
-            else # resolving didn't work - we probably don't have parameters
-              val
-            end
-          else
-            load_files(val)
+          dict[key] = case restype
+            when 'AWS::AutoScaling::LaunchConfiguration'
+              if (key == "UserData") and (val["File"]) 
+                # Support LaunchConfiguration UserData from file
+                CloudInit.new("#{@basedir}/#{val["File"]}").to_base64
+              elsif (key == "UserData") and (val["FileTemplate"]) 
+                # Support LaunchConfiguration UserData from file with substitutions
+                { "Fn::Base64" => { "Fn::Sub" => CloudInit.new("#{@basedir}/#{val["FileTemplate"]}").compile } }
+              else
+                load_files(val, restype)
+              end
+            when 'AWS::Lambda::Function'
+              if key == 'Code'
+                LambdaCode.new(val, self).to_cloudformation
+              else
+                load_files(val, restype)
+              end
+            else
+              load_files(val, restype)
           end
           dict
         end

data/lib/cloud_formation_tool/storable.rb

@@ -14,6 +14,10 @@ module CloudFormationTool
       end
     end
     
+    def cached_object(md5)
+      Aws::S3::Bucket.new(s3_bucket_name(region), client: awss3(region)).objects(prefix: "cf-compiled/#{md5[0]}/#{md5[1..2]}/#{md5}/").first
+    end
+    
     def upload(path, content, mime_type: 'text/yaml', gzip: true)
       md5 = Digest::MD5.hexdigest content
       prefix = "#{md5[0]}/#{md5[1..2]}/#{md5}"
@@ -23,7 +27,7 @@ module CloudFormationTool
       # do a local copy to the requested path) because this way cloudformation can see
       # that the updated template is exactly the same as the old one and will not force
       # an unneeded update.
-      o = b.objects(prefix: "cf-compiled/#{prefix}/").first
+      o = cached_object(md5)
       if o.nil?
         # no such luck, we need to actually upload the file
         o = b.object("cf-compiled/#{prefix}/#{path}")

data/lib/cloud_formation_tool/version.rb

@@ -1,3 +1,3 @@
 module CloudFormationTool
-  VERSION = '0.9.1'
+  VERSION = '1.0.1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cloudformation-tool
 version: !ruby/object:Gem::Version
-  version: 0.9.1
+  version: 1.0.1
 platform: ruby
 authors:
 - Oded Arbel
@@ -176,7 +176,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.14
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: A pre-compiler tool for CloudFormation YAML templates