cknife 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6a8aeeb2e92a76ac079da5b63fe8e9403b451d6e
+  data.tar.gz: a74e4274ccc2b39ab451c06d3069ef649884d80f
+SHA512:
+  metadata.gz: edd89d9f659ef18a4c88b4b91b57f13252c6caa48d1443ac0a99ddbf5e8b5e183d5c8bfd66b7a3d7edcd60548d8c268996760da19638c63462faae89ac5febe0
+  data.tar.gz: 626d66f0e47119dae79626e6819196b75356d4b49a893edb04a2a0bc9867011ba32f4f2062223acdee678dc9c440c0f5e1ce8e5de8406a879854de5d76c8331a

data/Gemfile

@@ -0,0 +1,17 @@
+source "http://rubygems.org"
+
+gem "rest-client"
+gem "nokogiri"
+gem "i18n"
+gem "activesupport"
+gem "thor"
+gem "builder"
+gem "fog", ">= 1.15.0"
+gem "unf"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "bundler", "~> 1.0"
+  gem "jeweler", "~> 2.0.1"
+end

data/Gemfile.lock

@@ -0,0 +1,86 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    activesupport (3.0.7)
+    addressable (2.3.6)
+    builder (3.0.0)
+    descendants_tracker (0.0.4)
+      thread_safe (~> 0.3, >= 0.3.1)
+    excon (0.27.6)
+    faraday (0.9.0)
+      multipart-post (>= 1.2, < 3)
+    fog (1.17.0)
+      builder
+      excon (~> 0.27.0)
+      formatador (~> 0.2.0)
+      mime-types
+      multi_json (~> 1.0)
+      net-scp (~> 1.1)
+      net-ssh (>= 2.1.3)
+      nokogiri (~> 1.5)
+      ruby-hmac
+    formatador (0.2.4)
+    git (1.2.6)
+    github_api (0.11.3)
+      addressable (~> 2.3)
+      descendants_tracker (~> 0.0.1)
+      faraday (~> 0.8, < 0.10)
+      hashie (>= 1.2)
+      multi_json (>= 1.7.5, < 2.0)
+      nokogiri (~> 1.6.0)
+      oauth2
+    hashie (3.0.0)
+    highline (1.6.21)
+    i18n (0.6.0)
+    jeweler (2.0.1)
+      builder
+      bundler (>= 1.0)
+      git (>= 1.2.5)
+      github_api
+      highline (>= 1.6.15)
+      nokogiri (>= 1.5.10)
+      rake
+      rdoc
+    jwt (1.0.0)
+    mime-types (1.16)
+    mini_portile (0.5.2)
+    multi_json (1.8.2)
+    multi_xml (0.5.5)
+    multipart-post (2.0.0)
+    net-scp (1.1.2)
+      net-ssh (>= 2.6.5)
+    net-ssh (2.7.0)
+    nokogiri (1.6.0)
+      mini_portile (~> 0.5.0)
+    oauth2 (0.9.4)
+      faraday (>= 0.8, < 0.10)
+      jwt (~> 1.0)
+      multi_json (~> 1.3)
+      multi_xml (~> 0.5)
+      rack (~> 1.2)
+    rack (1.5.2)
+    rake (0.9.6)
+    rdoc (4.0.0)
+    rest-client (1.6.3)
+      mime-types (>= 1.16)
+    ruby-hmac (0.4.0)
+    thor (0.14.6)
+    thread_safe (0.3.4)
+    unf (0.1.3)
+      unf_ext
+    unf_ext (0.0.6)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport
+  builder
+  bundler (~> 1.0)
+  fog (>= 1.15.0)
+  i18n
+  jeweler (~> 2.0.1)
+  nokogiri
+  rest-client
+  thor
+  unf

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 The Mike De La Loza Company
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,169 @@
+Cali Army Knife
+==============
+
+An Amazon Web Services S3 command line tool, and a few other command
+line tools. Written in Ruby with Thor. It depends on the Fog gem for
+all of its S3 operations.
+
+Uses multipart uploads with a chunksize of 10 megabytes to keep RAM
+usage down.
+
+The premier feature of the tool is the "upsync" command, which can be
+used to run a backups schedule with multiple classes of files
+(partitioned by a glob pattern). **It is your responsibility to
+generate one uniquely-named backup file per day**, as this tool does
+not do that part for you.
+
+If you *don't* use the `backups-retain` option, then its like a very
+weak **rsync** that can upload from a local filesystem into a bucket.
+Which is also pretty useful.
+
+[Github Link](https://github.com/mikedll/cali-army-knife)
+
+Examples:
+
+    # download entire my-photos bucke to CWD
+    > aws.rb download my-photos 
+    
+    # upload and sync /tmp/*.sql into my-frog-app-backups
+    # bucket. Treat the files as backup files, and keep one backup
+    # file for each of the last 5 months, 10 weeks, and 30 days.    
+    > aws.rb upsync my-frog-app-backups ./tmp --glob "*.sql" --noprompt --backups-retain true --months-retain 5 --weeks-retain 10 --days-retain 30
+
+    # as above, but now do redis backup files (./tmp/*.rdb). these will not produce
+    # namespace collisions with the sql files, and thus the same bucket
+    # can be used to store backups for both.    
+    > aws.rb upsync my-frog-app-backups ./tmp --glob "*.rdb" --noprompt --backups-retain true --months-retain 5 --weeks-retain 10 --days-retain 30
+
+    # DO NOT DO THIS INSTEAD OF THE ABOVE 2 COMMANDS, THINKING IT WILL
+    # TREAT .SQL AND .RDB FILES SEPARATELY. INSTEAD, YOU WILL LOSE
+    # SOME OF YOUR BACKUP FILES.    
+    > aws.rb upsync my-frog-app-backups ./tmp --glob "*" --noprompt --backups-retain true --months-retain 5 --weeks-retain 10 --days-retain 30
+
+    # Dry run mode. Try one of the prior backups retain commands, but
+    # let's see what will happen, first.
+    > aws.rb upsync my-frog-app-backups ./tmp --glob "*.sql" --noprompt --backups-retain true --months-retain 5 --weeks-retain 10 --days-retain 30 --dry-run
+
+
+# Getting Started / Installation
+
+  Clone this repo, `bundle` the Gemfile, and make the aws.rb invokable
+  either by adding it to your PATH or making a bash alias to it, using
+  an absolute path to it, or something. I dunno, I'm working on that.
+
+# aws
+
+    Tasks:
+      aws.rb afew [BUCKET_NAME]                # Show first 5 files in bucket
+      aws.rb create [BUCKET_NAME]              # Create a bucket
+      aws.rb create_cloudfront [BUCKET_NAME]   # Create a cloudfront distribution (a CDN)
+      aws.rb delete [BUCKET_NAME]              # Destroy a bucket
+      aws.rb download [BUCKET_NAME]            # Download all files in a bucket to CWD. Or one file.
+      aws.rb help [TASK]                       # Describe available tasks or one specific task
+      aws.rb list                              # Show all buckets
+      aws.rb list_cloudfront                   # List cloudfront distributions (CDNs)
+      aws.rb list_servers                      # Show all servers
+      aws.rb show [BUCKET_NAME]                # Show info about bucket
+      aws.rb start_server [SERVER_ID]          # Start a given EC2 server
+      aws.rb stop_server [SERVER_ID]           # Stop a given EC2 server (does not terminate it)
+      aws.rb upsync [BUCKET_NAME] [DIRECTORY]  # Push local files matching glob PATTERN into bucket. Ignore unchanged files.      
+
+## Synchronizing a local directory's files with an Amazon S3 Bucket
+
+    Usage:
+      aws.rb upsync [BUCKET_NAME] [DIRECTORY]
+
+    Options:
+      [--public]             
+      [--region=REGION]      
+                             # Default: us-east-1
+      [--noprompt=NOPROMPT]  
+      [--glob=GLOB]          
+                             # Default: **/*
+      [--backups-retain]     
+      [--days-retain=N]      
+                             # Default: 30
+      [--months-retain=N]    
+                             # Default: 3
+      [--weeks-retain=N]     
+                             # Default: 5
+      [--dry-run]            
+
+The glob allows you to determine whether you want to recursively
+upload an entire directory, or just a set of *.dat or *.sql files,
+ignoring whatever else may be in the specified directory. This glob
+pattern is appended to the directory you specify.
+
+For determining whether to upload a file, first the mod time is used,
+and if that match fails, an md5 checksum comparison is used.
+
+The file's local modification time, from the file system from which it
+was uploaded, is used to determined whether it qualifies for retention
+in the backup program you specify.
+
+This info is uploaded with the file to Amazon's S3 servers when the
+file is uploaded, in the S3 file metadata. Without this, S3 uses a
+modtime that is equal to when the file was last uploaded, which is not
+comparable to the file's local mod time.
+
+## Dumping an Amazon S3 bucket
+
+Sometimes you want to download an entire S3 bucket to your local
+directory - a set of photos, for example.
+
+    > aws.rb help download 
+    Usage:
+      aws.rb download [BUCKET_NAME]
+
+    Options:
+      [--region=REGION]  
+                         # Default: us-east-1
+      [--one=ONE]        
+
+    Download all files in a bucket to CWD. Or one file.
+
+
+## Key and Secret Configuration
+
+In order of priority, Setup your key and secret with:
+
+  - $CWD/cknife.yml
+  - $CWD/tmp/cknife.yml
+  - environment variables: `KEY`, `SECRET`
+  - environment variablse: `AMAZON_ACCESS_KEY_ID`, `AMAZON_SECRET_ACCESS_KEY` 
+
+The format of your cknife.yml must be like so:
+
+    ---
+    key: AKIAblahblahb...
+    secret: 8xILhOsecretsecretsecretsecret...
+
+
+# zerigo 
+
+These tasks can be used to manage your DNS via Zerigo.  They changed
+their rates drastically with little notice in January of 2014, so I
+switched to DNS Simple and don't use this much anymore.
+
+    > zerigo 
+    Tasks:
+      zerigo.rb create [HOST_NAME]  # Create a host
+      zerigo.rb delete [ID]         # Delete an entry by id
+      zerigo.rb help [TASK]         # Describe available tasks or one specific task
+      zerigo.rb list                # List available host names.
+
+# dub
+
+Like du, but sorts your output by size.  This helps you determine
+which directories are taking up the most space:
+
+    > dub
+          37.0G .
+          23.0G ./Personal
+          14.0G ./Library
+         673.0M ./Work
+           0.0B ./Colloquy Transcripts
+
+## options      
+
+    -c Enable colorized output. 

data/Rakefile

@@ -0,0 +1,38 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://guides.rubygems.org/specification-reference/ for more options
+  gem.name = "cknife"
+  gem.homepage = "http://github.com/mikedll/cknife"
+  gem.license = "MIT"
+  gem.summary = "Cali Army Knife"
+  gem.description = "An Amazon Web Services S3 command line tool, and a few other command line tools."
+  gem.email = "mikedll@mikedll.com"
+  gem.authors = ["Mike De La Loza"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+task :default => :spec
+
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "cknife #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/bin/cknifeaws

@@ -0,0 +1,508 @@
+#!/usr/bin/env ruby
+
+ENV['BUNDLE_GEMFILE'] = File.join( File.dirname( File.expand_path( __FILE__ ) ),  "../Gemfile" )
+
+require 'rubygems'
+require 'bundler'
+Bundler.require
+
+require 'active_support/all'
+require 'zlib'
+require 'digest/md5'
+
+class Aws < Thor
+
+  FILE_BUFFER_SIZE = 10.megabytes
+  LOCAL_MOD_KEY = "x-amz-meta-mtime"
+  EPSILON = 1.second
+
+  no_tasks do
+
+    def config
+      return @config if @config
+
+      @config = {
+        :key => ENV["KEY"] || ENV['AMAZON_ACCESS_KEY_ID'], 
+        :secret => ENV["SECRET"] || ENV['AMAZON_SECRET_ACCESS_KEY']
+      }
+      
+      config_file = nil
+      Pathname.new(Dir.getwd).tap do |here|
+        config_file = [["cknife.yml"], ["tmp", "cknife.yml"]].map { |args|
+          here.join(*args)
+        }.select { |path|
+          File.exists?(path) 
+        }.first
+      end
+
+      if config_file
+        begin
+          @config.merge!(YAML.load(config_file.read).symbolize_keys!)
+        rescue
+          say ("Found, but could not parse config: #{config_file}")
+        end        
+      end
+
+      @config
+    end
+
+    def fog_opts
+      opts = {
+        :provider => 'AWS',
+        :aws_access_key_id => config[:key],
+        :aws_secret_access_key => config[:secret]
+      }      
+      opts.merge!({ :region => options[:region] }) if !options[:region].blank?
+      opts
+    end
+
+    def fog_storage
+      return @storage if @storage 
+      @storage = Fog::Storage.new(fog_opts)
+      begin
+        @storage.directories.count # test login
+      rescue Excon::Errors::Forbidden => e
+        say("Received Forbidden error while accessing account info. Is your key/secret correct?")
+        raise SystemExit
+      end
+      @storage
+    end
+
+    def fog_compute
+      @compute ||= Fog::Compute.new(fog_opts)
+    end
+
+    def fog_cdn
+      @cdn ||= Fog::CDN.new(fog_opts)
+    end
+
+    def show_buckets
+      fog_storage.directories.sort { |a,b| a.key <=> b.key }.each { |b| puts "#{b.key}" }
+    end
+
+    def show_servers
+      fog_compute.servers.sort { |a,b| a.key_name <=> b.key_name }.each do |s|
+        puts "#{s.tags['Name']} (state: #{s.state}): id=#{s.id} keyname=#{s.key_name} dns=#{s.dns_name} flavor=#{s.flavor_id}"
+      end
+    end
+    
+    def show_cdns
+      puts fog_cdn.get_distribution_list.body['DistributionSummary'].to_yaml
+    end
+
+    def with_bucket(bucket_name)
+      d = fog_storage.directories.select { |d| d.key == bucket_name }.first
+      if d.nil?
+        say ("Could not find bucket with name #{bucket_name}")
+        return
+      end
+
+      say ("Found bucket named #{bucket_name}")
+      yield d
+    end
+
+    def fog_key_for(target_root, file_path)
+      target_root_path_length ||= target_root.to_s.length + "/".length
+      relative = file_path[ target_root_path_length, file_path.length]
+      relative
+    end
+
+    def s3_download(s3_file)
+      dir_path = Pathname.new(s3_file.key).dirname
+      dir_path.mkpath
+      File.open(s3_file.key, "w") do |f|
+        f.write s3_file.body
+      end      
+    end
+
+
+    def content_hash(file)
+      md5 = Digest::MD5.new
+
+      while !file.eof?
+        md5.update(file.read(FILE_BUFFER_SIZE))        
+      end
+
+      md5.hexdigest
+    end
+
+  end
+
+  desc "list_servers", "Show all servers"
+  def list_servers
+    show_servers
+  end
+
+  desc "start_server [SERVER_ID]", "Start a given EC2 server"
+  def start_server(server_id)
+    s = fog_compute.servers.select { |s| s.id == server_id}.first
+    if s
+      say("found server. starting/resuming. #{s.id}")
+      s.start
+      show_servers
+    else
+      say("no server with that id found. nothing done.")
+    end
+  end
+
+  desc "stop_server [SERVER_ID]", "Stop a given EC2 server (does not terminate it)"
+  def stop_server(server_id)
+    s = fog_compute.servers.select { |s| s.id == server_id}.first
+    if s
+      say("found server. stopping. #{s.id}")
+      s.stop
+    else
+      say("no server with that id found. nothing done.")
+    end
+  end
+
+  desc "list_cloudfront", "List cloudfront distributions (CDNs)"
+  def list_cloudfront
+    show_cdns
+  end
+
+  desc "create_cloudfront [BUCKET_NAME]", "Create a cloudfront distribution (a CDN)"
+  def create_cloudfront(bucket_id)
+    fog_cdn.post_distribution({
+                            'S3Origin' => {
+                              'DNSName' => "#{bucket_id}.s3.amazonaws.com"
+                            },
+                            'Enabled' => true
+                          })
+
+    show_cdns
+  end
+
+  desc "list", "Show all buckets"
+  method_options :region => "us-east-1"
+  def list
+    show_buckets
+  end
+
+  desc "afew [BUCKET_NAME]", "Show first 5 files in bucket"
+  method_options :count => "5"
+  method_options :glob => "**/*"
+  def afew(bucket_name)
+    d = fog_storage.directories.select { |d| d.key == bucket_name }.first
+    if d.nil?
+      say ("Found no bucket by name #{bucket_name}")
+      return
+    end
+
+    found = []
+
+    i = 0
+    d.files.each do |f|
+      if File.fnmatch(options[:glob], f.key)
+        found.push(d.files.head(f.key))
+        break if i >= options[:count].to_i
+        i += 1
+      end
+    end
+
+    unit_to_mult = {
+      'B' => 1,
+      'K' => 2**10,
+      'M' => 2**20,
+      'G' => 2**30
+    }
+
+    found.map { |f|
+      matching = unit_to_mult.keys.select { |k| 
+        f.content_length >= unit_to_mult[k]
+      }.last
+
+      [f.key,
+       "#{(f.content_length.to_f / unit_to_mult[matching]).round(2)}#{matching}",
+       f.content_type,
+       f.last_modified
+      ] 
+    }.tap do |tabular|
+      print_table(tabular, :ident => 2)
+    end
+
+  end
+  
+  desc "download [BUCKET_NAME]", "Download all files in a bucket to CWD. Or one file."
+  method_options :region => "us-east-1"
+  method_options :one => nil
+  def download(bucket_name)
+    with_bucket bucket_name do |d|
+      if options[:one].nil?
+        if yes?("Are you sure you want to download all files into the CWD?", :red)
+          d.files.each do |s3_file|
+            say("Creating path for and downloading #{s3_file.key}")
+            s3_download(s3_file)
+          end
+        else
+          say("No action taken.")
+        end
+      else
+        s3_file = d.files.get(options[:one])
+        if !s3_file.nil?
+          s3_download(s3_file)
+        else
+          say("Could not find #{options[:one]}. No action taken.")
+        end
+      end
+    end
+  end
+
+  desc "upsync [BUCKET_NAME] [DIRECTORY]", "Push local files matching glob PATTERN into bucket. Ignore unchanged files."
+  method_options :public => false
+  method_options :region => "us-east-1"
+  method_options :noprompt => nil
+  method_options :glob => "**/*"
+  method_options :backups_retain => false
+  method_options :days_retain => 30
+  method_options :months_retain => 3
+  method_options :weeks_retain => 5
+  method_options :dry_run => false
+  def upsync(bucket_name, directory)
+
+    say("This is a dry run.") if options[:dry_run]
+
+    if !File.exists?(directory) || !File.directory?(directory)
+      say("'#{directory} does not exist or is not a directory.")
+      return
+    end
+
+    target_root = Pathname.new(directory)
+
+    files = Dir.glob(target_root.join(options[:glob])).select { |f| !File.directory?(f) }.map(&:to_s)
+    if !options[:backups_retain] && files.count == 0
+      say("No files to upload and no backups retain requested.")
+      return
+    end
+
+    say("Found #{files.count} candidate file upload(s).")
+
+    spn = dn = sn = un = cn = 0
+    with_bucket bucket_name do |d|
+
+      # having a brain fart and cant get this to simplify
+      go = false
+      if options[:noprompt] != nil
+        go = true
+      else
+        go = yes?("Proceed?", :red)
+      end
+
+      if go
+        time_marks = []
+        immediate_successors = {}
+        if options[:backups_retain]
+          # inclusive lower bound, exclusive upper bound
+          time_marks = []
+          Time.now.beginning_of_day.tap do |start|
+            options[:days_retain].times do |i|
+              time_marks.push(start - i.days)
+            end
+          end
+
+          Time.now.beginning_of_week.tap do |start|
+            options[:weeks_retain].times do |i|
+              time_marks.push(start - i.weeks)
+            end
+          end
+
+          Time.now.beginning_of_month.tap do |start|
+            options[:months_retain].times do |i|
+              time_marks.push(start - i.months)
+            end
+          end
+
+          time_marks.each do |tm|
+            files.each do |to_upload|
+              File.open(to_upload) do |localfile|
+                if localfile.mtime >= tm && (immediate_successors[tm].nil? || localfile.mtime < immediate_successors[tm][:last_modified])
+                  immediate_successors[tm] = { :local_path => to_upload, :last_modified => localfile.mtime }
+                end
+              end
+            end
+          end
+        end
+
+        # don't pointlessly upload large files if we already know we're going to delete them!
+        if options[:backups_retain] 
+          immediate_successors.values.map { |h| h[:local_path] }.tap do |kept_files| 
+            before_reject = files.count # blah...lame
+            files.reject! { |to_upload| !kept_files.include?(to_upload) } 
+            sn += before_reject - files.count 
+
+            say("Found #{files.count} file(s) that meet backups retention criteria for upload. Comparing against bucket...")
+
+          end
+        end
+
+        files.each do |to_upload|
+          say("#{to_upload} (no output if skipped)...")
+          k = fog_key_for(target_root, to_upload)
+
+          existing_head = d.files.head(k)
+
+          time_mismatch = false
+          content_hash_mistmatched = false
+          File.open(to_upload) do |localfile|
+            time_mismatch = !existing_head.nil? && (existing_head.metadata[LOCAL_MOD_KEY].nil? || (Time.parse(existing_head.metadata[LOCAL_MOD_KEY]) - localfile.mtime).abs > EPSILON)
+            if time_mismatch
+              content_hash_mistmatched = existing_head.etag != content_hash(localfile)
+            end
+          end
+
+          if existing_head && time_mismatch && content_hash_mistmatched
+            if !options[:dry_run]
+              File.open(to_upload) do |localfile|
+                existing_head.metadata = { LOCAL_MOD_KEY => localfile.mtime.to_s }
+                existing_head.body = localfile
+                existing_head.multipart_chunk_size = FILE_BUFFER_SIZE # creates multipart_save
+                existing_head.save
+              end
+            end
+            say("updated.")
+            un += 1
+          elsif existing_head && time_mismatch
+            if !options[:dry_run]
+              File.open(to_upload) do |localfile|
+                existing_head.metadata = { LOCAL_MOD_KEY => localfile.mtime.to_s }
+                existing_head.save
+              end
+            end
+            say("updated.")
+            un += 1            
+          elsif existing_head.nil?
+            if !options[:dry_run]
+              File.open(to_upload) do |localfile|
+                file = d.files.create(
+                                      :key    => k,
+                                      :public => options[:public],
+                                      :body => ""
+                                      )
+                file.metadata = { LOCAL_MOD_KEY => localfile.mtime.to_s }
+                file.multipart_chunk_size = FILE_BUFFER_SIZE # creates multipart_save
+                file.body = localfile
+                file.save
+              end
+            end
+            say("created.")
+            cn += 1
+          else
+            sn += 1
+            # skipped
+          end
+        end
+
+
+        if options[:backups_retain]
+
+          # This array of hashes is computed because we need to do
+          # nested for loops of M*N complexity, where M=time_marks
+          # and N=files.  We also need to do an remote get call to
+          # fetch the metadata of all N remote files (d.files.each
+          # will not do this). so, for performance sanity, we cache
+          # all the meta data for all the N files.
+
+          file_keys_modtimes = []
+          d.files.each { |f| 
+            if File.fnmatch(options[:glob], f.key) 
+              existing_head = d.files.head(f.key)
+              md = existing_head.metadata
+              file_keys_modtimes.push({
+                                        :key => f.key,
+                                        :last_modified => md[LOCAL_MOD_KEY] ? Time.parse(md[LOCAL_MOD_KEY]) : f.last_modified,
+                                        :existing_head => existing_head
+                                      })
+            end
+          }
+
+          say("#{file_keys_modtimes.length} file(s) found to consider for remote retention or remote deletion.")
+
+          # this generates as many 'kept files' as there are time marks...which seems wrong.
+          immediate_successors = {}
+          time_marks.each do |tm|
+            file_keys_modtimes.each do |fkm|
+              if fkm[:last_modified] >= tm && (immediate_successors[tm].nil? || fkm[:last_modified] < immediate_successors[tm][:last_modified])
+                immediate_successors[tm] = fkm
+              end
+            end
+          end
+
+          immediate_successors.values.map { |v| v[:key] }.tap do |kept_keys|
+            file_keys_modtimes.each do |fkm|
+              if kept_keys.include?(fkm[:key])
+                say("Remote retained #{fkm[:key]}.")
+                spn += 1
+              else
+                fkm[:existing_head].destroy if !options[:dry_run]
+                say("Remote deleted #{fkm[:key]}.")
+                dn += 1
+              end
+            end
+          end
+        end
+      else
+        say ("No action taken.")
+      end
+    end
+    say("Done. #{cn} created. #{un} updated. #{sn} local skipped. #{dn} deleted remotely. #{spn} retained remotely.")
+  end
+
+  desc "delete [BUCKET_NAME]", "Destroy a bucket"
+  method_options :region => "us-east-1"
+  def delete(bucket_name)
+    d = fog_storage.directories.select { |d| d.key == bucket_name }.first
+
+    if d.nil?
+      say ("Found no bucket by name #{bucket_name}")
+      return
+    end
+
+    if d.files.length > 0
+      say "Bucket has #{d.files.length} files. Please empty before destroying."
+      return
+    end
+
+    if yes?("Are you sure you want to delete this bucket #{d.key}?", :red)
+      d.destroy
+      say "Destroyed bucket named #{bucket_name}."
+      show_buckets
+    else
+      say "No action taken."
+    end
+
+  end
+
+  desc "create [BUCKET_NAME]", "Create a bucket"
+  method_options :region => "us-east-1"
+  def create(bucket_name = nil)
+    if !bucket_name
+      puts "No bucket name given." 
+      return
+    end
+
+    fog_storage.directories.create(
+                                   :key => bucket_name,
+                                   :location => options[:region]
+                                   )
+
+    puts "Created bucket #{bucket_name}."
+    show_buckets
+  end
+
+  desc "show [BUCKET_NAME]", "Show info about bucket"
+  method_options :region => "us-east-1"
+  def show(bucket_name = nil)
+    if !bucket_name
+      puts "No bucket name given." 
+      return
+    end
+
+    with_bucket(bucket_name) do |d|
+      say "#{d}: "
+      say d.location
+    end
+  end
+
+end
+
+Aws.start

data/bin/cknifedub

@@ -0,0 +1,66 @@
+#!/usr/bin/env ruby
+
+color_highlighting = false
+
+ARGV.each do |arg|
+  if arg == '-c'
+    color_highlighting = true
+  end
+end
+
+duout = `du -h -d 1`
+
+comps = []
+
+duout.split(/\n/).each do |l|
+  l =~ /\s*(\d+(\.\d+)?)(\w)\s+(.*)$/
+  size = $1
+  unit = $3
+  path = $4
+  comps << [size.to_f, unit, path]
+end
+
+unit_to_mult = {
+  'B' => 1,
+  'K' => 2**10,
+  'M' => 2**20,
+  'G' => 2**30
+}
+
+comps.sort! do |a, b|
+  (b[0] * unit_to_mult[b[1]]) <=> (a[0] * unit_to_mult[a[1]])
+end
+
+BLACK = "\033[30m"
+RED = "\033[31m"
+RED_BLINK = "\033[5;31m"
+GREEN = "\033[32m"
+YELLOW = "\033[33m"
+BLUE = "\033[34m"
+MAGENTA = "\033[35m"
+CYAN = "\033[36m"
+WHITE = "\033[37m"
+RESET_COLOR = "\033[0m"
+
+if color_highlighting
+  comps.each do |(size,unit,path)|
+    case unit
+    when 'G'
+      output_color = RED 
+    when 'M'
+      output_color = YELLOW
+    when 'K'
+      output_color = CYAN 
+    when 'B'
+      output_color = GREEN 
+    else 
+      output_color = BLACK 
+    end
+
+    printf( "%s%10.1f%s %s%s\n",output_color, size, unit, path,RESET_COLOR)
+  end
+else
+  comps.each do |(size,unit,path)|
+    printf( "%10.1f%s %s\n",size, unit, path)
+  end
+end

data/bin/cknifenowtimestamp

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+
+puts Time.now.strftime("%Y%m%d%H%M%S")

data/bin/cknifewcdir

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+
+def run
+  wccmd = `find . -ipath #{ARGV[0]} -exec wc -l {} \\;`
+
+  comps = []
+
+  total = 0
+  wccmd.split(/\n/).each do |l|
+    l =~ /\s*(\d+(\.\d+)?)\s+(.*)$/
+    size = $1
+    path = $3
+    comps << [size.to_i, path]
+    total += size.to_i
+  end
+
+  comps.sort! do |a, b|
+    b[0] <=> a[0]
+  end
+
+  comps.each do |(size,path)|
+    printf( "%d %s\n", size, path)
+  end
+
+  puts "Total: #{total}"
+end
+
+run

data/bin/cknifezerigo

@@ -0,0 +1,101 @@
+#!/usr/bin/env ruby
+
+ENV['BUNDLE_GEMFILE'] = File.join( File.dirname( File.expand_path( __FILE__ ) ),  "Gemfile" )
+
+require 'rubygems'
+require 'bundler'
+Bundler.require
+
+require 'active_support/all'
+
+class Zerigo < Thor
+
+  ENDPOINT = "https://ns.zerigo.com/api/1.1"
+  USERNAME = ENV['ZERIGO_USERNAME']
+  PASSWORD = ENV['ZERIGO_PASSWORD']
+  DOMAIN = ENV['ZERIGO_DEFAULT_DOMAIN']
+
+
+  no_tasks do
+    def zerigo
+      @zerigo ||= RestClient::Resource.new ENDPOINT, USERNAME, PASSWORD
+    end
+
+    def show_hosts(response)
+      fmt = "%6.5s %30.29s %40.39s %12.11s"
+
+      puts sprintf(fmt, 
+                   "Type",
+                   "Hostname",
+                   "Data",
+                   "ID")
+
+      Nokogiri::XML( response.body ).css('host').each do |host|
+        puts sprintf(fmt,
+                     host.at_css('host-type').text,
+                     host.at_css('hostname').text,
+                     host.at_css('data').text,
+                     host.at_css('id').text)
+      end
+    end
+
+    def zone
+      if @zone.nil?
+        zones = []
+        response = zerigo['/zones.xml'].get 
+        unless response.code.to_s !~ /2\d{2}/
+          Nokogiri::XML( response.body ).css('zone').each do |zone|
+            zones << {
+              :id => zone.css('id').first.text,
+              :domain => zone.css('domain').first.text,
+            }
+          end
+        end
+        found = zones.select { |z| z[:domain] == DOMAIN }.first
+        raise "Unable to find zone matching domain #{DOMAIN}" unless found
+
+        @zone = found[:id]
+      end
+      @zone
+    end
+  end
+
+  module HostTypes
+    CNAME = "CNAME"
+    ARECORD = "A"
+    MX = "MX"
+  end
+
+  desc "list", "List available host names."
+  def list
+    response = zerigo["/zones/#{zone}/hosts.xml"].get
+    unless response.code.to_s !~ /2\d{2}/
+      show_hosts( response )
+    end
+  end
+
+  desc "create [HOST_NAME]", "Create a host"
+  method_options :data => "proxy.heroku.com"
+  method_options :host_type => HostTypes::CNAME
+  def create(hostname = "")
+    new_host = { :data => options[:data], :host_type => options[:host_type], :hostname => hostname }
+
+    response = zerigo["/zones/#{zone}/hosts.xml"].post new_host.to_xml(:root => "host"), :content_type => "application/xml"
+    if response.code.to_s =~ /201/
+      puts "Created a host."
+      show_hosts( response )
+    end    
+  end
+
+
+  desc "delete [ID]", "Delete an entry by id"
+  def delete(id)
+    response = zerigo["/hosts/#{id}.xml"].delete
+    if response.code.to_s =~ /200/
+      puts "Delete a host with id #{id}"
+    end    
+  end
+
+end
+
+Zerigo.start

data/cknife.gemspec

@@ -0,0 +1,80 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+# stub: cknife 0.1.0 ruby lib
+
+Gem::Specification.new do |s|
+  s.name = "cknife"
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.require_paths = ["lib"]
+  s.authors = ["Mike De La Loza"]
+  s.date = "2014-06-06"
+  s.description = "An Amazon Web Services S3 command line tool, and a few other command line tools."
+  s.email = "mikedll@mikedll.com"
+  s.executables = ["cknifeaws", "cknifedub", "cknifenowtimestamp", "cknifewcdir", "cknifezerigo"]
+  s.extra_rdoc_files = [
+    "LICENSE",
+    "README.md"
+  ]
+  s.files = [
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE",
+    "README.md",
+    "Rakefile",
+    "VERSION",
+    "bin/cknifeaws",
+    "bin/cknifedub",
+    "bin/cknifenowtimestamp",
+    "bin/cknifewcdir",
+    "bin/cknifezerigo",
+    "cknife.gemspec"
+  ]
+  s.homepage = "http://github.com/mikedll/cknife"
+  s.licenses = ["MIT"]
+  s.rubygems_version = "2.2.2"
+  s.summary = "Cali Army Knife"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 4
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<rest-client>, [">= 0"])
+      s.add_runtime_dependency(%q<nokogiri>, [">= 0"])
+      s.add_runtime_dependency(%q<i18n>, [">= 0"])
+      s.add_runtime_dependency(%q<activesupport>, [">= 0"])
+      s.add_runtime_dependency(%q<thor>, [">= 0"])
+      s.add_runtime_dependency(%q<builder>, [">= 0"])
+      s.add_runtime_dependency(%q<fog>, [">= 1.15.0"])
+      s.add_runtime_dependency(%q<unf>, [">= 0"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 2.0.1"])
+    else
+      s.add_dependency(%q<rest-client>, [">= 0"])
+      s.add_dependency(%q<nokogiri>, [">= 0"])
+      s.add_dependency(%q<i18n>, [">= 0"])
+      s.add_dependency(%q<activesupport>, [">= 0"])
+      s.add_dependency(%q<thor>, [">= 0"])
+      s.add_dependency(%q<builder>, [">= 0"])
+      s.add_dependency(%q<fog>, [">= 1.15.0"])
+      s.add_dependency(%q<unf>, [">= 0"])
+      s.add_dependency(%q<bundler>, ["~> 1.0"])
+      s.add_dependency(%q<jeweler>, ["~> 2.0.1"])
+    end
+  else
+    s.add_dependency(%q<rest-client>, [">= 0"])
+    s.add_dependency(%q<nokogiri>, [">= 0"])
+    s.add_dependency(%q<i18n>, [">= 0"])
+    s.add_dependency(%q<activesupport>, [">= 0"])
+    s.add_dependency(%q<thor>, [">= 0"])
+    s.add_dependency(%q<builder>, [">= 0"])
+    s.add_dependency(%q<fog>, [">= 1.15.0"])
+    s.add_dependency(%q<unf>, [">= 0"])
+    s.add_dependency(%q<bundler>, ["~> 1.0"])
+    s.add_dependency(%q<jeweler>, ["~> 2.0.1"])
+  end
+end
+

metadata

@@ -0,0 +1,203 @@
+--- !ruby/object:Gem::Specification
+name: cknife
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Mike De La Loza
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-06-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: i18n
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: builder
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fog
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.15.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.15.0
+- !ruby/object:Gem::Dependency
+  name: unf
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.0.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.0.1
+description: An Amazon Web Services S3 command line tool, and a few other command
+  line tools.
+email: mikedll@mikedll.com
+executables:
+- cknifeaws
+- cknifedub
+- cknifenowtimestamp
+- cknifewcdir
+- cknifezerigo
+extensions: []
+extra_rdoc_files:
+- LICENSE
+- README.md
+files:
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- VERSION
+- bin/cknifeaws
+- bin/cknifedub
+- bin/cknifenowtimestamp
+- bin/cknifewcdir
+- bin/cknifezerigo
+- cknife.gemspec
+homepage: http://github.com/mikedll/cknife
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Cali Army Knife
+test_files: []