ec2_amitools 1.0.2

data/lib/ec2/common/s3support.rb

@@ -0,0 +1,231 @@
+# Copyright 2008-2014 Amazon.com, Inc. or its affiliates.  All Rights
+# Reserved.  Licensed under the Amazon Software License (the
+# "License").  You may not use this file except in compliance with the
+# License. A copy of the License is located at
+# http://aws.amazon.com/asl or in the "license" file accompanying this
+# file.  This file is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
+# the License for the specific language governing permissions and
+# limitations under the License.
+
+# ---------------------------------------------------------------------------
+# Module that provides higher-level S3 functionality
+# ---------------------------------------------------------------------------
+require 'uri'
+require 'cgi'
+require 'ec2/common/constants'
+require 'ec2/common/http'
+
+module EC2
+  module Common
+    class S3Support
+
+      attr_accessor :s3_url,
+                    :user,
+                    :pass
+
+      DEFAULT_SIGV = EC2::Common::SIGV4
+      DEFAULT_REGION = 'us-east-1'
+
+      # Return true if the bucket name is S3 safe.
+      #
+      # Per the S3 dev guide @ http://docs.amazonwebservices.com/AmazonS3/2006-03-01/index.html?BucketRestrictions.html
+      # - Be between 3 and 255 characters long
+      # - Start with a number or letter
+      # - Contain lowercase letters, numbers, periods (.), underscores (_), and dashes (-)
+      # - Not be in an IP address style (e.g., "192.168.5.4")
+      #
+      # * Notes:
+      # - !!(....) silliness to force a boolean to be returned
+      def self.bucket_name_s3_safe?(bucket_name)
+        # Can most probably fold this all into 1 grand regexp but 
+        # for now opt for semi-clarity.
+        !!((3..255).include?(bucket_name.length) and
+           (/^[a-z0-9][a-z0-9\._-]+$/ =~ bucket_name) and
+           (/^(\d{1,3}\.){3}\d{1,3}$/ !~ bucket_name))
+      end
+
+
+      # Return true if the bucket name is S3 (v2) safe.
+      #
+      # Per the S3 dev guide @ http://docs.amazonwebservices.com/AmazonS3/2006-03-01/index.html?BucketRestrictions.html
+      # - Bucket names should not contain underscores (_)
+      # - Bucket names should be between 3 and 63 characters long
+      # - Bucket names should not end with a dash
+      # - Bucket names cannot contain dashes next to periods (e.g., "my-.bucket.com" and "my.-bucket" are invalid)
+      # - Bucket names must only contain lower case letters
+      #
+      # and naturally also fulfills bucket_name_s3_safe?() requirements
+      #
+      # * Notes:
+      # - !!(....) silliness to force a boolean to be returned
+      def self.bucket_name_s3_v2_safe?(bucket_name)
+        # Can most probably fold this all into 1 grand regexp but 
+        # for now opt for semi-clarity.
+        !!(self.bucket_name_s3_safe?(bucket_name) and
+           bucket_name.length <= 63 and
+           not bucket_name.include?('_') and
+           bucket_name[-1] != ?- and
+           /(-\.)|(\.-)/ !~ bucket_name)
+      end
+
+      def initialize(s3_url, user, pass, format=nil, debug=nil, sigv=DEFAULT_SIGV, region=DEFAULT_REGION)
+        @user = user
+        @pass = pass
+        @s3_url = fix_s3_url(s3_url)
+        @format = format
+        @debug = debug
+        @sigv = sigv
+        @region = region
+      end
+
+      def fix_s3_url(s3_url)
+        if s3_url !~ %r{://}
+          s3_url = "https://#{s3_url}"
+        end
+        if s3_url[-1..-1] != "/"
+          s3_url << "/"
+        end
+        s3_url
+      end
+
+      def get_bucket_url(bucket)
+        # The returned variable should not end with '/', makes constructing the path for sigv4 easier
+        if not @format
+          if bucket.include? "."
+            @format = :path
+          else
+            @format = :subdomain
+          end
+        end
+
+        case @format
+        when :subdomain
+          uri = URI.parse(@s3_url)
+          return ["#{uri.scheme}://#{bucket}.#{uri.host}", bucket]
+        when :path
+          return ["#{@s3_url}#{bucket}", bucket]
+        end
+      end
+
+      def get_acl(bucket, key, options={})
+        begin
+          url, bkt = get_bucket_url(bucket)
+          url << "/#{CGI::escape(key)}?acl"
+          http_fallback(url, @sigv) do |url, signature|
+            EC2::Common::HTTP::get(url, bkt, nil, options, @user, @pass, nil, nil, @debug, signature, @region)
+          end
+        end
+      end
+
+      def check_bucket_exists(bucket, options={})
+        url, bkt = get_bucket_url(bucket)
+        http_fallback(url, @sigv) do |url, signature|
+          EC2::Common::HTTP::head(url, bkt, options, @user, @pass, @debug, signature, @region)
+        end
+      end
+
+      def get_bucket_location(bucket, options={})
+        url, bkt = get_bucket_url(bucket)
+        url << "?location"
+        http_fallback(url, @sigv) do |url, signature|
+          EC2::Common::HTTP::get(url, bkt, nil, options, @user, @pass, nil, nil, @debug, signature, @region)
+        end
+      end
+
+      def create_bucket(bucket, location, options={})
+        url, bkt = get_bucket_url(bucket)
+        binary_data = ""
+        if (location != nil)
+          binary_data = "<CreateBucketConstraint><LocationConstraint>#{location}</LocationConstraint></CreateBucketConstraint>"
+        end
+        http_fallback(url, @sigv) do |url, signature|
+          EC2::Common::HTTP::putdir(url, bkt, binary_data, options, @user, @pass, @debug, signature, @region)
+        end
+      end
+
+      def list_bucket(bucket, prefix=nil, max_keys=nil, path=nil, options={})
+        url, bkt = get_bucket_url(bucket)
+        params = []
+        params << "prefix=#{CGI::escape(prefix)}" if prefix
+        params << "max-keys=#{CGI::escape(max_keys)}" if max_keys
+        url << "?" + params.join("&") unless params.empty?
+        http_fallback(url, @sigv) do |url, signature|
+          EC2::Common::HTTP::get(url, bkt, path, options, @user, @pass, nil, nil, @debug, signature, @region)
+        end
+      end
+
+      def get(bucket, key, path=nil, options={})
+        url, bkt = get_bucket_url(bucket)
+        url << "/#{CGI::escape(key)}"
+        http_fallback(url, @sigv) do |url, signature|
+          EC2::Common::HTTP::get(url, bkt, path, options, @user, @pass, nil, nil, @debug, signature, @region)
+        end
+      end
+
+      def put(bucket, key, file, options={})
+        url, bkt = get_bucket_url(bucket)
+        url << "/#{CGI::escape(key)}"
+        http_fallback(url, @sigv) do |url, signature|
+          EC2::Common::HTTP::put(url, bkt, file, options, @user, @pass, @debug, signature, @region)
+        end
+      end
+
+      def copy(bucket, key, source, options={})
+        url, bkt = get_bucket_url(bucket)
+        url << "/#{CGI::escape(key)}"
+        options['x-amz-copy-source'] = CGI::escape(source)
+        http_fallback(url, @sigv) do |url, signature|
+          EC2::Common::HTTP::put(url, bkt, nil, options, @user, @pass, @debug, signature, @region)
+        end
+      end
+
+      def delete(bucket, key="", options={})
+        url, bkt = get_bucket_url(bucket)
+        url << "/#{CGI::escape(key)}"
+        http_fallback(url, @sigv) do |url, signature|
+          EC2::Common::HTTP::delete(url, bkt, options, @user, @pass, @debug, signature, @region)
+        end
+      end
+
+      private
+      # This method will cycle through all possible signatures versions, starting
+      # with the given the one as first choice
+      def http_fallback url, sigv, &block
+        all_sigv = [EC2::Common::SIGV2, EC2::Common::SIGV4]
+        all_sigv.delete(sigv)
+        ordered_sigv = [sigv].concat all_sigv
+       
+        resp = nil
+        access_denied = false
+        ordered_sigv.each do |o_sigv|
+          redirected = true
+          while redirected
+            redirected = false
+            begin
+              resp = block.call(url, o_sigv)
+            rescue EC2::Common::HTTP::Error::Redirect => e
+              redirected = true
+              url = e.endpoint
+              STDERR.puts "Redirecting to #{url}, resigning request"
+            rescue EC2::Common::HTTP::Error => e
+              if e.code == 403
+                access_denied = true 
+              else
+                raise e
+              end
+            end
+          end
+          # If the response was 403, try other sigv
+          if access_denied == true
+            access_denied = false
+            STDERR.puts "Signature version #{o_sigv} authentication failed, trying different signature version"
+          else
+            break
+          end
+        end
+        resp
+      end
+    end
+  end
+end

data/lib/ec2/common/signature.rb

@@ -0,0 +1,68 @@
+# Copyright 2008-2014 Amazon.com, Inc. or its affiliates.  All Rights
+# Reserved.  Licensed under the Amazon Software License (the
+# "License").  You may not use this file except in compliance with the
+# License. A copy of the License is located at
+# http://aws.amazon.com/asl or in the "license" file accompanying this
+# file.  This file is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
+# the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'uri'
+
+require 'ec2/common/headers'
+require 'ec2/common/headersv4'
+
+module EC2
+  module Common
+    module Signature
+      def self.curl_args_sigv2(url, bucket, http_method, options, user, pass)
+          headers = EC2::Common::Headers.new(http_method)
+          options.each do |name, value| headers.add(name, value) end
+          headers.sign(user, pass, url, bucket) if user and pass
+          headers.get
+      end
+ 
+      def self.curl_args_sigv4(url, region, bucket, http_method, optional_headers, user, pass, data=nil)
+        aws_secret_access_key = pass
+        aws_access_key_id = user
+        if (user.is_a?(Hash))
+            aws_access_key_id = user['aws_access_key_id']
+            aws_secret_access_key = user['aws_secret_access_key']
+            delegation_token = user['aws_delegation_token']
+        end
+        host, path, query = parseURL(url, bucket)
+        hexdigest = if data
+          HeadersV4::hexdigest data
+        else
+          HeadersV4::hexdigest ""
+        end
+        optional_headers ||= {}
+        unless delegation_token.nil?
+            optional_headers[EC2::Common::Headers::X_AMZ_SECURITY_TOKEN] = delegation_token
+        end
+        headers_obj = HeadersV4.new({:host => host,
+                               :hexdigest_body => hexdigest,
+                               :region => region,
+                               :service => "s3",
+                               :http_method => http_method,
+                               :path => path,
+                               :querystring => query,
+                               :access_key_id => aws_access_key_id,
+                               :secret_access_key => aws_secret_access_key},
+                               optional_headers)
+        headers = headers_obj.add_authorization!
+        headers
+      end
+
+      def self.parseURL(url, bucket)
+        uri = URI.parse(url)
+        host = uri.host
+        path = uri.path
+        path = "/" if path == ""
+
+        [host, path, "#{uri.query}"]
+      end
+    end
+  end
+end

data/lib/ec2/oem/LICENSE.txt

@@ -0,0 +1,58 @@
+Ruby is copyrighted free software by Yukihiro Matsumoto <matz@netlab.co.jp>.
+You can redistribute it and/or modify it under either the terms of the GPL
+(see COPYING.txt file), or the conditions below:
+
+  1. You may make and give away verbatim copies of the source form of the
+     software without restriction, provided that you duplicate all of the
+     original copyright notices and associated disclaimers.
+
+  2. You may modify your copy of the software in any way, provided that
+     you do at least ONE of the following:
+
+       a) place your modifications in the Public Domain or otherwise
+          make them Freely Available, such as by posting said
+	  modifications to Usenet or an equivalent medium, or by allowing
+	  the author to include your modifications in the software.
+
+       b) use the modified software only within your corporation or
+          organization.
+
+       c) rename any non-standard executables so the names do not conflict
+	  with standard executables, which must also be provided.
+
+       d) make other distribution arrangements with the author.
+
+  3. You may distribute the software in object code or executable
+     form, provided that you do at least ONE of the following:
+
+       a) distribute the executables and library files of the software,
+	  together with instructions (in the manual page or equivalent)
+	  on where to get the original distribution.
+
+       b) accompany the distribution with the machine-readable source of
+	  the software.
+
+       c) give non-standard executables non-standard names, with
+          instructions on where to get the original software distribution.
+
+       d) make other distribution arrangements with the author.
+
+  4. You may modify and include the part of the software into any other
+     software (possibly commercial).  But some files in the distribution
+     are not written by the author, so that they are not under this terms.
+
+     They are gc.c(partly), utils.c(partly), regex.[ch], st.[ch] and some
+     files under the ./missing directory.  See each file for the copying
+     condition.
+
+  5. The scripts and library files supplied as input to or produced as 
+     output from the software do not automatically fall under the
+     copyright of the software, but belong to whomever generated them, 
+     and may be sold commercially, and may be aggregated with this
+     software.
+
+  6. THIS SOFTWARE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR
+     IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
+     WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+     PURPOSE.
+

data/lib/ec2/oem/open4.rb

@@ -0,0 +1,399 @@
+# ------------------------------------------------------------------------
+# Name: Open4
+# Version: 4-0.9.6 <http://codeforpeople.com/lib/ruby/open4/open4-0.9.6.tgz>
+# Author:  Ara T. Howard <ara[dot]t[dot]howard[at]noaa[dot]gov>
+# License: Ruby License <http://www.ruby-lang.org/en/LICENSE.txt>
+# ------------------------------------------------------------------------
+# vim: ts=2:sw=2:sts=2:et:fdm=marker
+require 'fcntl'
+require 'timeout'
+require 'thread'
+
+module Open4
+#--{{{
+  VERSION = '0.9.6'
+  def self.version() VERSION end
+
+  class Error < ::StandardError; end
+
+  def popen4(*cmd, &b)
+#--{{{
+    pw, pr, pe, ps = IO.pipe, IO.pipe, IO.pipe, IO.pipe
+
+    verbose = $VERBOSE
+    begin
+      $VERBOSE = nil
+      ps.last.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC)
+
+      cid = fork {
+        pw.last.close
+        STDIN.reopen pw.first
+        pw.first.close
+
+        pr.first.close
+        STDOUT.reopen pr.last
+        pr.last.close
+
+        pe.first.close
+        STDERR.reopen pe.last
+        pe.last.close
+
+        STDOUT.sync = STDERR.sync = true
+
+        begin
+          exec(*cmd)
+          raise 'forty-two' 
+        rescue Exception => e
+          Marshal.dump(e, ps.last)
+          ps.last.flush
+        end
+        ps.last.close unless (ps.last.closed?)
+        exit!
+      }
+    ensure
+      $VERBOSE = verbose
+    end
+
+    [pw.first, pr.last, pe.last, ps.last].each{|fd| fd.close}
+
+    begin
+      e = Marshal.load ps.first
+      raise(Exception === e ? e : "unknown failure!")
+    rescue EOFError # If we get an EOF error, then the exec was successful
+      42
+    ensure
+      ps.first.close
+    end
+
+    pw.last.sync = true
+
+    pi = [pw.last, pr.first, pe.first]
+
+    if b 
+      begin
+        b[cid, *pi]
+        Process.waitpid2(cid).last
+      ensure
+        pi.each{|fd| fd.close unless fd.closed?}
+      end
+    else
+      [cid, pw.last, pr.first, pe.first]
+    end
+#--}}}
+  end
+  alias open4 popen4
+  module_function :popen4
+  module_function :open4
+
+  class SpawnError < Error
+#--{{{
+    attr 'cmd'
+    attr 'status'
+    attr 'signals'
+    def exitstatus
+      @status.exitstatus
+    end
+    def initialize cmd, status
+      @cmd, @status = cmd, status
+      @signals = {} 
+      if status.signaled?
+        @signals['termsig'] = status.termsig
+        @signals['stopsig'] = status.stopsig
+      end
+      sigs = @signals.map{|k,v| "#{ k }:#{ v.inspect }"}.join(' ')
+      super "cmd <#{ cmd }> failed with status <#{ exitstatus.inspect }> signals <#{ sigs }>"
+    end
+#--}}}
+  end
+
+  class ThreadEnsemble
+#--{{{
+    attr 'threads'
+
+    def initialize cid
+      @cid, @threads, @argv, @done, @running = cid, [], [], Queue.new, false
+      @killed = false
+    end
+
+    def add_thread *a, &b
+      @running ? raise : (@argv << [a, b])
+    end
+
+#
+# take down process more nicely
+#
+    def killall
+      c = Thread.critical
+      return nil if @killed
+      Thread.critical = true
+      (@threads - [Thread.current]).each{|t| t.kill rescue nil}
+      @killed = true
+    ensure
+      Thread.critical = c
+    end
+
+    def run
+      @running = true
+
+      begin
+        @argv.each do |a, b|
+          @threads << Thread.new(*a) do |*a|
+            begin
+              b[*a]
+            ensure
+              killall rescue nil if $!
+              @done.push Thread.current
+            end
+          end
+        end
+      rescue
+        killall
+        raise
+      ensure
+        all_done
+      end
+
+      @threads.map{|t| t.value}
+    end
+
+    def all_done
+      @threads.size.times{ @done.pop }
+    end
+#--}}}
+  end
+
+  def to timeout = nil
+#--{{{
+    Timeout.timeout(timeout){ yield }
+#--}}}
+  end
+  module_function :to
+
+  def new_thread *a, &b
+#--{{{
+    cur = Thread.current
+    Thread.new(*a) do |*a|
+      begin
+        b[*a]
+      rescue Exception => e
+        cur.raise e
+      end
+    end
+#--}}}
+  end
+  module_function :new_thread
+
+  def getopts opts = {}
+#--{{{
+    lambda do |*args|
+      keys, default, ignored = args
+      catch('opt') do
+        [keys].flatten.each do |key|
+          [key, key.to_s, key.to_s.intern].each do |key|
+            throw 'opt', opts[key] if opts.has_key?(key)
+          end
+        end
+        default
+      end
+    end
+#--}}}
+  end
+  module_function :getopts
+
+  def relay src, dst = nil, t = nil
+#--{{{
+    unless src.nil?
+      if src.respond_to? :gets
+        while buf = to(t){ src.gets }
+          dst << buf if dst
+        end
+
+      elsif src.respond_to? :each
+        q = Queue.new
+        th = nil
+
+        timer_set = lambda do |t|
+          th = new_thread{ to(t){ q.pop } }
+        end
+
+        timer_cancel = lambda do |t|
+          th.kill if th rescue nil
+        end
+
+        timer_set[t]
+        begin
+          src.each do |buf|
+            timer_cancel[t]
+            dst << buf if dst
+            timer_set[t]
+          end
+        ensure
+          timer_cancel[t]
+        end
+
+      elsif src.respond_to? :read
+        buf = to(t){ src.read }
+        dst << buf if dst 
+
+      else
+        buf = to(t){ src.to_s }
+        dst << buf if dst 
+      end
+    end
+#--}}}
+  end
+  module_function :relay
+
+  def spawn arg, *argv 
+#--{{{
+    argv.unshift(arg)
+    opts = ((argv.size > 1 and Hash === argv.last) ? argv.pop : {})
+    argv.flatten!
+    cmd = argv.join(' ')
+
+
+    getopt = getopts opts
+
+    ignore_exit_failure = getopt[ 'ignore_exit_failure', getopt['quiet', false] ]
+    ignore_exec_failure = getopt[ 'ignore_exec_failure', !getopt['raise', true] ]
+    exitstatus = getopt[ %w( exitstatus exit_status status ) ]
+    stdin = getopt[ %w( stdin in i 0 ) << 0 ]
+    stdout = getopt[ %w( stdout out o 1 ) << 1 ]
+    stderr = getopt[ %w( stderr err e 2 ) << 2 ]
+    pid = getopt[ 'pid' ]
+    timeout = getopt[ %w( timeout spawn_timeout ) ]
+    stdin_timeout = getopt[ %w( stdin_timeout ) ]
+    stdout_timeout = getopt[ %w( stdout_timeout io_timeout ) ]
+    stderr_timeout = getopt[ %w( stderr_timeout ) ]
+    status = getopt[ %w( status ) ]
+    cwd = getopt[ %w( cwd dir ) ]
+
+    exitstatus =
+      case exitstatus
+        when TrueClass, FalseClass
+          ignore_exit_failure = true if exitstatus
+          [0]
+        else
+          [*(exitstatus || 0)].map{|i| Integer i}
+      end
+
+    stdin ||= '' if stdin_timeout
+    stdout ||= '' if stdout_timeout
+    stderr ||= '' if stderr_timeout
+
+    started = false
+
+    status =
+      begin
+        chdir(cwd) do
+          Timeout::timeout(timeout) do
+            popen4(*argv) do |c, i, o, e|
+              started = true
+
+              %w( replace pid= << push update ).each do |msg|
+                break(pid.send(msg, c)) if pid.respond_to? msg 
+              end
+
+              te = ThreadEnsemble.new c
+
+              te.add_thread(i, stdin) do |i, stdin|
+                relay stdin, i, stdin_timeout
+                i.close rescue nil
+              end
+
+              te.add_thread(o, stdout) do |o, stdout|
+                relay o, stdout, stdout_timeout
+              end
+
+              te.add_thread(e, stderr) do |o, stderr|
+                relay e, stderr, stderr_timeout
+              end
+
+              te.run
+            end
+          end
+        end
+      rescue
+        raise unless(not started and ignore_exec_failure)
+      end
+
+    raise SpawnError.new(cmd, status) unless
+      (ignore_exit_failure or (status.nil? and ignore_exec_failure) or exitstatus.include?(status.exitstatus))
+
+    status
+#--}}}
+  end
+  module_function :spawn
+
+  def chdir cwd, &block
+    return(block.call Dir.pwd) unless cwd
+    Dir.chdir cwd, &block
+  end
+  module_function :chdir
+
+  def background arg, *argv 
+#--{{{
+    require 'thread'
+    q = Queue.new
+    opts = { 'pid' => q, :pid => q }
+    case argv.last
+      when Hash
+        argv.last.update opts
+      else
+        argv.push opts
+    end
+    thread = Thread.new(arg, argv){|arg, argv| spawn arg, *argv}
+    sc = class << thread; self; end
+    sc.module_eval {
+      define_method(:pid){ @pid ||= q.pop }
+      define_method(:spawn_status){ @spawn_status ||= value }
+      define_method(:exitstatus){ @exitstatus ||= spawn_status.exitstatus }
+    }
+    thread
+#--}}}
+  end
+  alias bg background
+  module_function :background
+  module_function :bg
+
+  def maim pid, opts = {}
+#--{{{
+    getopt = getopts opts
+    sigs = getopt[ 'signals', %w(SIGTERM SIGQUIT SIGKILL) ]
+    suspend = getopt[ 'suspend', 4 ]
+    pid = Integer pid
+    existed = false
+    sigs.each do |sig|
+      begin
+        Process.kill sig, pid
+        existed = true 
+      rescue Errno::ESRCH
+        return(existed ? nil : true)
+      end
+      return true unless alive? pid
+      sleep suspend
+      return true unless alive? pid
+    end
+    return(not alive?(pid)) 
+#--}}}
+  end
+  module_function :maim
+
+  def alive pid
+#--{{{
+    pid = Integer pid
+    begin
+      Process.kill 0, pid
+      true
+    rescue Errno::ESRCH
+      false
+    end
+#--}}}
+  end
+  alias alive? alive
+  module_function :alive
+  module_function :'alive?'
+#--}}}
+end
+
+def open4(*cmd, &b) cmd.size == 0 ? Open4 : Open4::popen4(*cmd, &b) end