fireworks 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0fe5cd9e7bcef7e37ce48b908cc335e79ca4a140
+  data.tar.gz: 1efda50386a57ba7c90043984464130bd7213e8f
+SHA512:
+  metadata.gz: 2dccce2cdb849a06ae0b322e5917a3f5aeab40fc7b0e4bc990dd7c592b41ea1b0ff14ac79e7871c207da1b98808a242972113c1cfe7be9421ecb389215156735
+  data.tar.gz: 3a4313d7012f1a3b7bc2ca4d3609401e1b1cc98e6bd3e439c46eb36d2fc35e68d4a7f2fa3f866ce548eeeb70bad3baf57205f6b1298089a360d264c8647a2fd9

data/.gitignore

@@ -0,0 +1,2 @@
+Gemfile.lock
+pkg

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://www.rubygems.org'
+
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Kongregate
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/bin/fireworks

@@ -0,0 +1,40 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '/../lib'))
+require 'fireworks'
+
+require 'optparse'
+
+options = {}
+optparse = OptionParser.new do |opts|
+  opts.banner = 'Usage: ebs-firweorks.rb [options]'
+
+  opts.on('-d', '--device DEVICE', 'Device to prewarm') do |device|
+    options[:device] = device
+  end
+
+  opts.on('-t', '--threads THREADS', Integer, 'Number of threads to run') do |threads|
+    options[:threads] = threads
+  end
+
+  opts.on('-i', '--interval INTERVAL', Integer, 'Interval, in seconds, for updates') do |interval|
+    options[:interval] = interval
+  end
+end
+
+begin
+  optparse.parse!
+  mandatory = [:device]
+  missing = mandatory.select { |param| options[param].nil? }
+  unless missing.empty?
+    puts "Missing options: #{missing.join(', ')}"
+    puts optparse
+    exit
+  end
+rescue OptionParser::InvalidOption, OptionParser::MissingArgument => e
+  puts e
+  puts optparse
+  exit
+end
+
+Fireworks::BlockDevice.new(options).prewarm

data/fireworks.gemspec

@@ -0,0 +1,16 @@
+Gem::Specification.new do |s|
+  s.name        = 'fireworks'
+  s.version     = '0.0.1'
+  s.date        = '2015-08-06'
+  s.summary     = 'Faster EBS prewarming'
+  s.description = 'Faster EBS prewarming via running multiple dds in parallel'
+  s.authors     = ['Andrew Grim']
+  s.email       = 'stopdropandrew@gmail.com'
+  s.files       = `git ls-files -z`.split("\x0")
+  s.executables = ['fireworks']
+  s.homepage    = 'https://github.com/kongregate/fireworks'
+  s.license     = 'MIT'
+
+  s.add_runtime_dependency 'filesize'
+  s.add_runtime_dependency 'chronic_duration'
+end

data/lib/fireworks.rb

@@ -0,0 +1 @@
+require_relative 'fireworks/block_device'

data/lib/fireworks/block_device.rb

@@ -0,0 +1,133 @@
+require 'open3'
+require 'filesize'
+require 'chronic_duration'
+
+module Fireworks
+  class BlockDevice
+    attr_reader :device, :threads, :interval
+
+    def initialize(device:, threads: 1, interval: 5)
+      @threadpool = []
+
+      @device = device
+      @threads = threads
+      @interval = interval
+      raise ArgumentError, "Device #{device} does not exist" unless File.exist?(device)
+      raise ArgumentError, "Device #{device} is empty" unless device_size > 0
+      raise ArgumentError, "Device #{device} is already mounted" if already_mounted?
+    end
+
+    def device_size
+      @device_size ||= `blockdev --getsize64 #{device}`.to_i
+    end
+
+    def already_mounted?
+      !`mount | grep #{device}`.empty?
+    end
+
+    def prewarm
+      start_threads
+
+      until @threadpool.all?(&:complete?)
+        @threadpool.each(&:update_status)
+
+        output_stats
+        sleep interval
+      end
+    end
+
+    def current_stats
+      {
+        bytes_completed: @threadpool.map(&:bytes_completed).inject(:+),
+        total_rate: @threadpool.map(&:rate).inject(:+),
+        num_up_to_date: @threadpool.select(&:up_to_date?).size,
+        num_complete: @threadpool.select(&:complete?).size
+      }
+    end
+
+    def output_stats
+      stats = current_stats
+      bytes_completed = Filesize.new(stats[:bytes_completed])
+      total_rate = Filesize.new(stats[:total_rate])
+      device_file_size = Filesize.new(device_size)
+      time_left = ((device_file_size - bytes_completed) / total_rate)
+      puts format(
+        '%s / %s (%.2f%%) [%s/s] %d UpToDate - %d Complete - ETA %s',
+        bytes_completed.pretty,
+        device_file_size.pretty,
+        100.0 * bytes_completed / device_file_size,
+        total_rate.pretty,
+        stats[:num_up_to_date],
+        stats[:num_complete],
+        total_rate > 0 ? ChronicDuration.output(time_left, format: :short) : 'Infinity'
+      )
+    end
+
+  private
+
+    def start_threads
+      total_chunks = Filesize.new(device_size) / Filesize.from('1MiB')
+      chunks_per_thread = (total_chunks / threads).to_i
+
+      1.upto(threads).each do |thread_num|
+        start_point = chunks_per_thread * (thread_num - 1)
+
+        stdin, stdout, stderr, thread = Open3.popen3("dd if=#{device} of=#{device} bs=1M skip=#{start_point} seek=#{start_point} count=#{chunks_per_thread} conv=notrunc")
+        stdin.close
+        stdout.close
+
+        @threadpool.push(Fireworks::DDThread.new(thread: thread, stderr: stderr))
+      end
+    end
+  end
+
+  class DDThread
+    attr_reader :bytes_completed, :rate
+
+    def initialize(thread:, stderr:)
+      @thread = thread
+      @stderr = stderr
+      @eof = false
+      @bytes_completed = 0
+      @rate = 0
+      @updated_at = nil
+    end
+
+    def complete?
+      !@thread.alive? && @eof
+    end
+
+    def update_status
+      usr1! # send signal to dump stderr
+
+      sleep 0.1 # need a short sleep to let IO threads run
+
+      buffer = attempt_read
+      most_recent_status = buffer.to_s.split("\n").last
+      return unless most_recent_status && (match = most_recent_status.match(/(\d+) bytes.*, (\d+(?:\.\d+)? \w+)\/s/))
+
+      @bytes_completed = match[1].to_i
+      @rate = Filesize.from(match[2]).to_i
+      @updated_at = Time.now
+    end
+
+    def up_to_date?
+      return false if @updated_at.nil?
+      Time.now - @updated_at < 120
+    end
+
+  private
+
+    def attempt_read
+      @stderr.read_nonblock(8196)
+    rescue Errno::EAGAIN, Errno::EWOULDBLOCK
+      return # No data!
+    rescue EOFError
+      @eof = true
+    end
+
+    def usr1!
+      Process.kill('USR1', @thread.pid) if @thread.alive?
+    end
+  end
+end

metadata

@@ -0,0 +1,80 @@
+--- !ruby/object:Gem::Specification
+name: fireworks
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Andrew Grim
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-08-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: filesize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: chronic_duration
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Faster EBS prewarming via running multiple dds in parallel
+email: stopdropandrew@gmail.com
+executables:
+- fireworks
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE
+- Rakefile
+- bin/fireworks
+- fireworks.gemspec
+- lib/fireworks.rb
+- lib/fireworks/block_device.rb
+homepage: https://github.com/kongregate/fireworks
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Faster EBS prewarming
+test_files: []