tina 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5341f9de0800c6433ed74730845512e4bad3e21d
+  data.tar.gz: c2614fed632aceb92ee0fd0ea4312716c90415bd
+SHA512:
+  metadata.gz: b35c9014deb0bb60fe01b035b82936fdf7d623eba2b3de3d8d630fb9e0b4d7baf5d15d9a9038afcac46e8b726b7f858e8ce8dbd6a75ed1b35cf29e66daa0e643
+  data.tar.gz: 4d4d88719c30b2e7bbf1feae0ce8d3a705edb136497e16ad669ee568b948c87b13ac7a3fcd6fc1c9d0074e437ce851e53d60f56afe0748307db385afb662f93c

data/README.md

@@ -0,0 +1,91 @@
+# Tina
+
+Tina is a tool for restoring objects from Amazon Glacier into Amazon
+S3, while maintaining control over costs.
+
+Amazon Glacier allows for a certain amount of the total storage to be
+restored for free. The pricing model is however very complicated when
+this threshold is exceeded, and it is not trivial to calculate when it
+will be. Tina was written in order to solve this by estimating a price
+for a restore given the total storage, the duration of the restore,
+and what objects to restore.
+
+## Install
+
+    $ gem install tina
+
+## Usage
+
+What you need:
+
+* The `total-storage` number, which is the amount of data stored in
+  Glacier, in bytes. You can find a good enough estimate for this
+  number by looking at the "Amazon Simple Storage Service
+  EU-TimedStorage-GlacierByteHrs" line item on your bill for last
+  month ("Amazon Simple Storage Service TimedStorage-GlacierByteHrs"
+  for US regions).
+* A `PREFIX_FILE` with lines on the format `s3://<BUCKET>/<PREFIX>` with
+  all the prefixes you want to restore
+
+An example of how this tool can be used follows.
+
+Caroline stores 227 TiB of data in Glacier, which is 249589139505152
+bytes. She wants to restore all photos from June 2014 from the bucket
+`my-photos` and all her horror movies starting with the letter A and B
+from `my-movies`. She prepares a file called `my-restore.txt` with the
+following contents:
+
+    s3://my-photos/2014/06/
+    s3://my-movies/horror/A
+    s3://my-movies/horror/B
+
+She can now run tina like this;
+
+    $ tina --total-storage=249589139505152 --duration=20h --keep-days=14 my-restore.txt
+
+This will instruct tina to prepare a restore over __20 hours__ for all
+objects matching the prefixes in `my-restore.txt` and keep the objects
+on S3 for __14 days__. Using the total storage amount, tina can
+estimate a price for the restore.
+
+After printing information about the restore and an estimated price,
+tina will ask Caroline whether to proceed.
+
+Please note that tina is a long running process, which means it is a
+good idea to run it under `screen` or `tmux`, and on a machine that is
+constantly connected, e.g. an EC2 instance.
+
+## Notes
+
+* The estimated cost does not include the cost for the restore
+  requests or the temporary storage on S3.
+* The estimated cost is based on the assumption that no other restores
+  are running in parallel, since that would incur a higher peak
+  restore rate and consequently a higher cost.
+* The parameter for specifying the number of days to keep objects on
+  S3 is passed directly to the restore request. This means that
+  objects restored in one of the first chunks may expire sooner from
+  S3 than objects restored in one of the last chunks.
+
+## Future improvements
+
+* Speed up initial object listing by parallelizing requests
+* Implement a mode where tina figures out the required restore time to
+  restore given a specific budget (that might be $0)
+* Implement resume and failure handling. Currently, if tina fails (for
+  example due to a restore request failing) the prefix file would have
+  to be updated manually in order to resume at the same place later.
+* Use a first-fit algorithm to spread objects into chunks, instead of
+  the current naïve ordered chunking
+
+## Disclaimer
+
+The authors make no guarantees that the costs calculated using this
+script are correct and will not take any responsibility for any costs
+caused by running this script. Please beware that restoring objects is
+a potentially costly operation, that Amazon's pricing model may change
+at any time and that this script may contain nasty bugs.
+
+## Copyright
+
+© 2014 Burt AB, see LICENSE.txt (BSD 3-Clause).

data/bin/tina

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+$: << File.expand_path('../../lib', __FILE__)
+require 'tina'
+
+Tina::CLI.start

data/lib/tina.rb

@@ -0,0 +1,6 @@
+module Tina
+  S3Object = Struct.new(:bucket, :key, :size)
+end
+require 'tina/s3_client'
+require 'tina/cli'
+require 'tina/restore_plan'

data/lib/tina/cli.rb

@@ -0,0 +1,92 @@
+require 'thor'
+require 'aws-sdk-core'
+
+module Tina
+  class CLI < Thor
+    desc "restore PREFIX_FILE", "Restore files from Glacier into S3"
+    option :total_storage, type: :numeric, required: true, desc: 'the total amount stored in Glacier, in bytes'
+    option :duration, type: :string, required: true, desc: 'duration of the restore, in the format <N>h or <N>d'
+    option :keep_days, type: :numeric, required: true, desc: 'The number of days to keep objects on S3'
+    option :aws_region, type: :string, required: true, default: 'eu-west-1', desc: 'The Amazon region to operate against'
+    def restore(prefix_file)
+      total_storage = options[:total_storage]
+      duration = options[:duration]
+      keep_days = options[:keep_days]
+      duration_in_seconds = parse_duration(duration)
+
+      prefixes = File.readlines(prefix_file).map(&:chomp)
+      objects = RestorePlan::ObjectCollection.new(s3_client.list_bucket_prefixes(prefixes))
+      restore_plan = RestorePlan.new(total_storage.to_i, objects)
+      price = restore_plan.price(duration_in_seconds)
+      chunks = objects.chunk(duration_in_seconds)
+      say
+      say "Restores will be performed in the following chunks:"
+      say "-" * 60
+      chunks.each_with_index do |chunk, index|
+        chunk_size = chunk.map(&:size).reduce(&:+)
+        say "#{index+1}) #{chunk.size} objects of total size %.2f GiB / %.2f TiB" % [chunk_size / 1024 ** 3, chunk_size.to_f / 1024 ** 4]
+      end
+      say "-" * 60
+      say "Actual restore time: %i days, %i hours" % [(4 * chunks.size) / 24, (4 * chunks.size) % 24]
+      say "Number of objects to restore: #{objects.size}"
+      say "Total restore size: %.2f MiB / %.2f GiB / %.2f TiB" % [objects.total_size.to_f / 1024 ** 2, objects.total_size.to_f / 1024 ** 3, objects.total_size.to_f / 1024 ** 4]
+      say "Estimated cost: $#{price}"
+      say "Days to keep objects on S3: #{keep_days} days"
+      say "-" * 60
+      say "* Please beware that these costs are not included in estimated cost:"
+      say "*   - Cost for %i restore requests" % [objects.size]
+      say "*   - Storage on S3 of %.2f GiB during %i days" % [objects.total_size.to_f / 1024 ** 3, keep_days]
+      say "-" * 60
+      return unless yes?("Do you feel rich? [y/n]", :yellow)
+      restore_chunks(chunks, keep_days)
+    end
+
+    private
+
+    UNIT_FACTORS = { 'd' => 24 * 3600, 'h' => 1 * 3600 }
+    CHUNK_INTERVAL = 4 * 3600
+
+    def parse_duration(duration)
+      duration.match(/^(\d+)(h|d)$/)
+      raise "DURATION not in required format, [0-9]+(h|d)" unless (count = $1.to_i rescue nil) && (unit = $2)
+      count * UNIT_FACTORS[unit]
+    end
+
+    def restore_chunks(chunks, keep_days)
+      chunks.each_with_index do |chunk, index|
+        start = Time.now
+        say "Restoring #{chunk.size} objects, chunk #{index+1} of #{chunks.size}"
+
+        chunk.each do |object|
+          begin
+            s3_client.restore_object(object, keep_days)
+          rescue Aws::S3::Errors::RestoreAlreadyInProgress, Aws::S3::Errors::InvalidObjectState => e
+            say "Error restoring #{object.bucket} / #{object.key} was ignored: #{e}", :yellow
+          else
+            say "Restore issued for #{object.bucket} / #{object.key}", :green
+          end
+        end
+
+        say "Restore for all objects in chunk %i requested. Took %.1f seconds." % [index+1, Time.now - start]
+
+        if index + 1 < chunks.size
+          next_start = start + CHUNK_INTERVAL
+          sleep_time = (next_start - Time.now)
+          if sleep_time < 0
+            say "Warning! Issuing restores took more than 4 hours, so the end time will be delayed. Proceeding immediately with next chunk.", :yellow
+          else
+            say "Sleeping for %.1f seconds, until #{next_start}" % sleep_time, :green
+            sleep sleep_time
+          end
+        end
+      end
+    end
+
+    def s3_client
+      @s3_client ||= begin
+        s3 = Aws::S3::Client.new(region: options[:aws_region])
+        s3_client = S3Client.new(s3, shell)
+      end
+    end
+  end
+end

data/lib/tina/restore_plan.rb

@@ -0,0 +1,62 @@
+module Tina
+  class RestorePlan
+
+    MONTHLY_FREE_TIER_ALLOWANCE_FACTOR = 0.05
+    DAYS_PER_MONTH = 30
+    PRICE_PER_GB_PER_HOUR = 0.011
+
+    def initialize(total_storage_size, objects, options = {})
+      @total_storage_size = total_storage_size
+      @objects = objects
+      @price_per_gb_per_hour = options[:price_per_gb_per_hour] || PRICE_PER_GB_PER_HOUR
+
+      @daily_allowance = @total_storage_size * MONTHLY_FREE_TIER_ALLOWANCE_FACTOR / DAYS_PER_MONTH
+    end
+
+    def price(total_time)
+      total_time = [total_time, 4 * 3600].max
+      chunk_size = quadhourly_restore_rate(total_time)
+      chunks = @objects.chunk(chunk_size)
+      largest_chunk_object_size = chunks.map { |chunk| chunk.map(&:size).reduce(&:+) }.max
+      quadhours = chunks.size
+      quadhourly_allowance = @daily_allowance / ( [(24 / 4), quadhours].min * 4)
+
+      peak_retrieval_rate = largest_chunk_object_size / 4
+      peak_billable_retrieval_rate = [0, peak_retrieval_rate - quadhourly_allowance].max
+
+      peak_billable_retrieval_rate * (@price_per_gb_per_hour / 1024 ** 3) * 720
+    end
+    private
+
+    def quadhourly_restore_rate(total_time)
+      @objects.total_size / (total_time / (4 * 3600))
+    end
+
+    class ObjectCollection
+      attr_reader :total_size
+
+      def initialize(objects)
+        @objects = objects
+        @total_size = objects.map(&:size).reduce(&:+)
+      end
+
+      def size
+        @objects.size
+      end
+
+      def chunk(max_chunk_size)
+        @chunks ||= begin
+          chunks = @objects.chunk(sum: 0, index: 0) do |object, state|
+            state[:sum] += object.size
+            if state[:sum] > max_chunk_size
+              state[:sum] = object.size
+              state[:index] += 1
+            end
+            state[:index]
+          end
+          chunks.map(&:last)
+        end
+      end
+    end
+  end
+end

data/lib/tina/s3_client.rb

@@ -0,0 +1,38 @@
+require 'uri'
+
+module Tina
+  class S3Client
+    ClientError = Class.new(StandardError)
+
+    def initialize(s3, shell)
+      @s3 = s3
+      @shell = shell
+    end
+
+    def list_bucket_prefixes(prefix_uris)
+      bucket_prefixes = prefix_uris.map do |prefix_uri|
+        uri = URI.parse(prefix_uri)
+        raise ClientError, "Invalid S3 URI: #{uri}" unless uri.scheme == 's3'
+        [uri.host, uri.path.sub(%r[^/], '')]
+      end
+      bucket_prefixes.flat_map do |(bucket,prefix)|
+        @shell.say "Listing prefix #{bucket}/#{prefix}..."
+        objects = []
+        marker = nil
+        loop do
+          listing = @s3.list_objects(bucket: bucket, prefix: prefix, marker: marker)
+          listing.contents.each do |object|
+            objects << S3Object.new(bucket, object.key, object.size)
+            marker = object.key
+          end
+          break unless listing.is_truncated
+        end
+        objects
+      end
+    end
+
+    def restore_object(object, keep_days)
+      @s3.restore_object(bucket: object.bucket, key: object.key, restore_request: { days: keep_days })
+    end
+  end
+end

data/lib/tina/version.rb

@@ -0,0 +1,3 @@
+module Tina
+  VERSION = '1.0.0'
+end

data/spec/spec_helper.rb

@@ -0,0 +1 @@
+require 'tina'

data/spec/tina/restore_plan_spec.rb

@@ -0,0 +1,107 @@
+require 'spec_helper'
+
+module Tina
+  describe RestorePlan do
+    subject do
+      described_class.new(total_storage_size, object_collection, options)
+    end
+
+    let :total_storage_size do
+      75 * (1024 ** 4)
+    end
+
+    let :total_restore_size do
+      140 * (1024 ** 3)
+    end
+
+    let :object_collection do
+      SpecHelpers::ObjectCollection.new(total_restore_size)
+    end
+
+    let :options do
+      {
+        price_per_gb_per_hour: 0.01
+      }
+    end
+
+    describe '#price' do
+      context 'with perfectly aligned chunks' do
+        # http://aws.amazon.com/glacier/faqs/
+        context 'with the examples given on the Amazon Glacier pricing FAQ' do
+          it 'matches the the price for a restore with everything at once' do
+            expect(subject.price(4 * 3600)).to be_within(0.05).of(21.6)
+          end
+
+          it 'matches the the price for a restore over 8 hours' do
+            expect(subject.price(8 * 3600)).to be_within(0.05).of(10.8)
+          end
+
+          it 'matches the the price for a restore over 28 hours' do
+            expect(subject.price(28 * 3600)).to eq 0
+          end
+        end
+
+        # http://calculator.s3.amazonaws.com/index.html
+        context 'with arbitrary examples taken from the Amazon calculator' do
+          let :options do
+            {
+              price_per_gb_per_hour: 0.011
+            }
+          end
+
+          let :total_storage_size do
+            227 * 1024 ** 4
+          end
+
+          let :total_restore_size do
+            12_000 * 1024 ** 3
+          end
+
+          it 'matches the price for a restore over a month' do
+            expect(subject.price(30 * 24 * 3600)).to be_within(0.05).of(4.16)
+          end
+
+          it 'matches the price for a restore over a week' do
+            expect(subject.price(7 * 24 * 3600)).to be_within(0.05).of(437.87)
+          end
+
+          it 'matches the price for a restore over a day' do
+            expect(subject.price(1 * 24 * 3600)).to be_within(0.05).of(3832.16)
+          end
+
+          it 'matches the price for a restore over a 4 hour period' do
+            expect(subject.price(4 * 3600)).to be_within(0.05).of(22992.93)
+          end
+        end
+
+        context 'with the examples Amazon supplied in an e-mail' do
+          let :total_storage_size do
+            227 * 1024 ** 4
+          end
+
+          let :total_restore_size do
+            12_000 * 1024 ** 3
+          end
+
+          it 'matches the price for a restore over 4 days' do
+            expect(subject.price(4 * 24 * 3600)).to be_within(20).of(768)
+          end
+        end
+      end
+    end
+  end
+end
+
+module SpecHelpers
+  class ObjectCollection
+    attr_reader :total_size
+
+    def initialize(total_restore_size)
+      @total_size = total_restore_size
+    end
+
+    def chunk(max_chunk_size)
+      [[Tina::S3Object.new('bucket', 'key', max_chunk_size)]] * (total_size / max_chunk_size)
+    end
+  end
+end

data/spec/tina/s3_client_spec.rb

@@ -0,0 +1,61 @@
+module Tina
+  describe S3Client do
+    let :s3 do
+      double('s3', list_objects: object_list)
+    end
+
+    let :shell do
+      double('shell', say: nil)
+    end
+
+    let :object_list do
+      double('object list', contents: [double('object', key: 'first', size: 123)], is_truncated: false)
+    end
+
+    subject do
+      described_class.new(s3, shell)
+    end
+
+    describe '#list_bucket_prefixes' do
+      it 'raises a client error when one of the input URIs does not have the correct scheme' do
+        expect { subject.list_bucket_prefixes(%w(http://foo)) }.to raise_error(described_class::ClientError, /Invalid S3 URI/)
+      end
+
+      it 'retrieves s3 objects' do
+        allow(object_list).to receive(:contents).and_return([double('object', key: 'foo', size: 123)])
+        expect(subject.list_bucket_prefixes(['s3://bucket/prefix'])).to eq [S3Object.new('bucket', 'foo', 123)]
+        expect(s3).to have_received(:list_objects).with(hash_including(bucket: 'bucket', prefix: 'prefix'))
+      end
+
+      context 'for truncated response listings' do
+        let :object_list2 do
+          double('object list 2', contents: [double('object', key: 'second', size: 123)], is_truncated: false)
+        end
+
+        before do
+          allow(object_list).to receive(:is_truncated).and_return(true)
+          allow(s3).to receive(:list_objects).and_return(object_list, object_list2)
+        end
+
+        it 'specifies the last key of the first request as the marker for the next request when truncated' do
+          markers = []
+          first = true
+          allow(s3).to receive(:list_objects) do |options|
+            markers << options[:marker]
+            ret = first ? object_list : object_list2
+            first = false
+            ret
+          end
+          subject.list_bucket_prefixes(['s3://bucket/prefix'])
+          expect(markers).to eq [nil, "first"]
+        end
+
+        it 'returns the complete list of objects' do
+          actual_objects = subject.list_bucket_prefixes(['s3://bucket/prefix'])
+          expected_objects = object_list.contents + object_list2.contents
+          expect(actual_objects.map(&:key)).to eq(expected_objects.map(&:key))
+        end
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,87 @@
+--- !ruby/object:Gem::Specification
+name: tina
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Björn Ramberg
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-09-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.19.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.19.1
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 2.0.0.rc15
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 2.0.0.rc15
+description: CLI tool for restoring objects from Amazon Glacier over time in order
+  to control costs
+email:
+- bjorn@burtcorp.com
+executables:
+- tina
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- bin/tina
+- lib/tina.rb
+- lib/tina/cli.rb
+- lib/tina/restore_plan.rb
+- lib/tina/s3_client.rb
+- lib/tina/version.rb
+- spec/spec_helper.rb
+- spec/tina/restore_plan_spec.rb
+- spec/tina/s3_client_spec.rb
+homepage: http://github.com/burtcorp/tina
+licenses:
+- BSD-3-Clause
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: CLI tool for restoring objects from Amazon Glacier
+test_files:
+- spec/spec_helper.rb
+- spec/tina/restore_plan_spec.rb
+- spec/tina/s3_client_spec.rb