uricp 0.0.1

data/features/segmented_upload_options.feature

@@ -0,0 +1,65 @@
+Feature: Documented Help for Segmented Upload
+  In order to work out what segment_upload does
+  As a command line user
+  I want to get detailed usage information
+
+  Scenario: Basic UI
+    When I get help for "segment_upload"
+    Then the exit status should be 0
+    And the banner should include the version
+    And the banner should document that this app takes options
+    And the following options should be documented:
+      |--version|
+      |--auth-token|
+      |--auth-user|
+      |--auth-key|
+      |--from|
+      |--segment-size|
+    And the option "dry-run" should be documented which is negatable
+    And the banner should document that this app's arguments are:
+      |to_uri|which is required|
+
+  Scenario: auth token and auth-user are mutually exclusive
+    When I run `segment_upload --auth-token abcdef --auth-user xyz --auth-key abc http://source file:///target`
+    Then the exit status should not be 0
+    And the stderr should contain "needless argument"
+
+  Scenario: auth user needs auth key
+    When I run `segment_upload --auth-user xyz http://source`
+    Then the exit status should not be 0
+    And the stderr should contain "missing argument"
+
+  Scenario: auth key needs auth user
+    When I run `segment_upload --auth-user xyz http://source`
+    Then the exit status should not be 0
+    And the stderr should contain "missing argument"
+
+  Scenario: needs some sort of authentication
+    When I run `segment_upload http://source`
+    Then the exit status should not be 0
+    And the stderr should contain "missing argument"
+
+  Scenario: no http should fail
+    When I run `segment_upload ftp://source`
+    Then the exit status should not be 0
+    And the stderr should contain "unsupported url"
+
+  Scenario: authentication with normal http should fail
+    When I run `segment_upload --auth-user cli-xxxxx --auth-key fred https://foss-lab-manual.googlecode.com/files`
+    Then the exit status should not be 0
+    And the stderr should contain "Cannot authenticate"
+
+  Scenario: bad authentication should fail against orbit
+    When I run `segment_upload --auth-user cli-xxxxx --auth-key fred https://orbit.brightbox.com/v1/acc-xxxxx/test/test.img`
+    Then the exit status should not be 0
+    And the stderr should contain "Cannot authenticate"
+
+  Scenario: should not accept plain number for segment size
+    When I run `segment_upload --segment-size 100 --auth-token abcdef https:///orbit.brightbox.com/v1/acc-xxxxx/test/test.img`
+    Then the exit status should not be 0
+    And the stderr should contain "Unparseable filesize"
+
+  Scenario: should trap attempts to open a command
+    When I run `segment_upload --segment-size 100MB --auth-token abcdef --from ' |fred' https:///orbit.brightbox.com/v1/acc-xxxxx/test/test.img`
+    Then the exit status should not be 0
+    And the stderr should contain "invalid argument"

data/features/step_definitions/orbit_steps.rb

@@ -0,0 +1,194 @@
+require 'inifile'
+
+def obtain_credentials
+  config = IniFile.load(ENV['HOME']+ '/.brightbox/config') ||
+    raise(RuntimeError, "No brightbox config in home directory. Can't obtain auth token for tests")
+  section = config.sections.first
+  [config[section]['client_id'], config[section]['secret']]
+end
+
+def fetch_orbit_token
+  @clientid, @key = obtain_credentials
+  cmd = "curl -I https://orbit.brightbox.com/v1 -H 'X-Auth-User: #{@clientid}' -H 'X-Auth-Key: #{@key}'"
+  run_simple(unescape(cmd))
+  stdout_from(cmd).each_line do |line|
+    key, value = line.strip.split(/\s*:\s*/,2)
+    @current_auth_token = value if key == 'X-Auth-Token'
+    @current_storage_url = value if key == 'X-Storage-Url'
+  end
+end
+
+def create_download_file(filetype, name, container)
+  upload_url = File.join(@current_storage_url, container, name)
+  tempfile = '/tmp/fred55322'
+  FileUtils.rm_f(Dir.glob(tempfile+'*'))
+  format = 'raw'
+  format = 'qcow2' if filetype == 'qcow2'
+  cmd = "qemu-img create -q -f #{format} #{tempfile} 1G"
+  system(unescape(cmd))
+  cmd = "cat #{tempfile}"
+  if filetype == 'lz4'
+    cmd += " | lz4c "
+  end
+  cmd += "| curl --silent --fail -T - -H 'X-Auth-Token: #{@current_auth_token}' #{upload_url} "
+  system(unescape(cmd))
+end
+
+def create_container(name)
+  url = File.join(@current_storage_url, name)
+  cmd = "curl --silent --fail -I -H 'X-Auth-Token: #{@current_auth_token}' #{url}>/dev/null || curl --silent --fail -I -H 'X-Auth-Token: #{@current_auth_token}' #{url} -X PUT"
+  system(unescape(cmd))
+end
+
+Given(/^a container called "([^"]*)"$/) do |container|
+  create_container(container)
+end
+
+Given(/^a cache of "(.*?)" from container "(.*?)" at "(.*?)"$/) do |name, container, cache|
+  url = File.join(@current_storage_url, container, name)
+  target = File.join(cache, 'cache', name)
+  steps %{
+    When I successfully run `curl --fail --silent -o #{target} -H 'X-Auth-Token: #{@current_auth_token}' #{url}`
+  }
+end
+
+Given /^a (\d+) byte lz4 file named "([^"]*)"$/ do |file_size, file_name|
+  write_fixed_size_file(file_name, file_size.to_i)
+  system("lz4c #{file_name}")
+  File.rename file_name+'.lz4', file_name
+end
+
+When(/^I store "([^"]*)" into container "([^"]*)" from "([^"]*)"$/) do |name, container, source|
+  steps %{
+    When I store "#{name}" with options "" into container "#{container}" from "#{source}"
+  }
+end
+
+When(/^I retrieve "([^"]*)" with options "([^"]*)" from container "([^"]*)" into "([^"]*)"$/) do |name, runoptions, container, target|
+  url = File.join(@current_storage_url, container, name)
+  steps %{
+    When I successfully run `uricp #{runoptions} --auth-token #{@current_auth_token} #{url} #{target}`
+  }
+end
+
+When(/^I retrieve "([^"]*)" from container "([^"]*)" into "([^"]*)" with a userid$/) do |name, container, target|
+  url = File.join(@current_storage_url, container, name)
+  steps %{
+    When I successfully run `uricp --auth-user '#{@clientid}' --auth-key '#{@key}' #{url} #{target}`
+  }
+end
+
+When(/^I store "([^"]*)" into container "([^"]*)" from "([^"]*)" with a userid$/) do |name, container, source|
+  @current_container = container
+  @current_name = name
+  url = File.join(@current_storage_url, container, name)
+  steps %{
+    When I successfully run `uricp --auth-user '#{@clientid}' --auth-key '#{@key}' #{source} #{url}`
+  }
+end
+
+When(/^I retrieve "([^"]*)" from container "([^"]*)" into "([^"]*)"$/) do |name, container, target|
+  steps %{
+    When I retrieve "#{name}" with options "" from container "#{container}" into "#{target}"
+  }
+end
+
+When(/^I store "([^"]*)" with options "([^"]*)" into container "([^"]*)" from "([^"]*)"$/) do |name, runoptions, container, source|
+  @current_container = container
+  @current_name = name
+  url = File.join(@current_storage_url, container, name)
+  steps %{
+    When I successfully run `uricp #{runoptions} --auth-token #{@current_auth_token} #{source} #{url}`
+  }
+end
+
+When(/^I store "([^"]*)" with segment size "([^"]*)" into container "([^"]*)" from "([^"]*)" with a userid$/) do |name, segment_size, container, source|
+  @current_container = container
+  @current_name = name
+  url = File.join(@current_storage_url, container, name)
+  steps %{
+    When I successfully run `uricp --auth-user '#{@clientid}' --auth-key '#{@key}' --segment-size '#{segment_size}' #{source} #{url}`
+  }
+end
+
+When(/^I store "([^"]*)" with segment size "([^"]*)" and options "([^"]*)" into container "([^"]*)" from "([^"]*)" with a userid$/) do |name, segment_size, runoptions, container, source|
+  @current_container = container
+  @current_name = name
+  url = File.join(@current_storage_url, container, name)
+  steps %{
+    When I successfully run `uricp #{runoptions} --auth-user '#{@clientid}' --auth-key '#{@key}' --segment-size '#{segment_size}' #{source} #{url}`
+  }
+end
+
+When(/^I upload "([^"]*)" with segment size "([^"]*)" into container "([^"]*)" from "([^"]*)"$/) do |name, segment_size, container, source|
+  @current_container = container
+  @current_name = name
+  url = File.join(@current_storage_url, container, name)
+  steps %{
+    When I successfully run `segment_upload --from #{source} --auth-token #{@current_auth_token} --segment-size #{segment_size} #{url}`
+  }
+end
+
+When(/^I upload "([^"]*)" with segment size "([^"]*)" into container "([^"]*)" from "([^"]*)" as a stream$/) do |name, segment_size, container, source|
+  @current_container = container
+  @current_name = name
+  url = File.join(@current_storage_url, container, name)
+  steps %{
+    When I successfully run `sh -c 'cat #{source} | segment_upload --auth-token #{@current_auth_token} --segment-size #{segment_size} #{url}'`
+  }
+end
+
+Then(/^a (\d+) byte entry should exist in container "(.*?)" called "(.*?)"$/) do |size, container, name|
+  url = File.join(@current_storage_url, container, name)
+  command = "curl -I --fail --silent -H 'X-Auth-Token:#{@current_auth_token}' #{url}"
+  steps %{
+    When I successfully run `#{command}`
+    Then the output should match /\\sContent-Length: #{size}\\s/
+  }
+end
+
+Then(/^the container "(.*?)" should contain (\d+) entries$/) do |container, quantity|
+  url = File.join(@current_storage_url, container)
+  command = "curl -I --fail --silent -H 'X-Auth-Token:#{@current_auth_token}' #{url}"
+  steps %{
+    When I successfully run `#{command}`
+    Then the output should match /\\sX-Container-Object-Count: #{quantity}\\s/
+  }
+end
+
+Then(/^an lz4 compressed entry should exist in container "(.*?)" called "(.*?)"$/) do |container, name|
+  url = File.join(@current_storage_url, container, name)
+  cmd="curl -r 0-3 --fail --silent -H X-Auth-Token:#{@current_auth_token} #{url}"
+  run_simple(unescape(cmd))
+  assert_exact_output([0x184D2204].pack('V'), stdout_from(cmd))
+end
+
+Then(/^a qcow2 entry should exist in container "(.*?)" called "(.*?)"$/) do |container, name|
+  url = File.join(@current_storage_url, container, name)
+  cmd="curl -r 0-3 --fail --silent -H X-Auth-Token:#{@current_auth_token} #{url}"
+  run_simple(unescape(cmd))
+  assert_exact_output(['QFI',0xfb].pack('a3C'), stdout_from(cmd))
+end
+
+Before('@orbit') do
+  fetch_orbit_token
+end
+
+Before('@orbitdownloads') do
+  $orbit_setup ||= false
+  unless $orbit_setup
+    create_container('test')
+    create_download_file('qcow2', 'img-qcow2', 'test')
+    create_download_file('lz4', 'img-lz4cy', 'test')
+    $orbit_setup = true
+  end
+end
+
+After('@orbit') do
+  if @current_auth_token && @current_container
+    steps %{
+      When I successfully run `swift --os-storage-url=#{@current_storage_url} --os-auth-token=#{@current_auth_token} delete #{@current_container} #{@current_name}` 
+    }
+    @current_container = @current_name = nil
+  end
+end

data/features/step_definitions/uricp_steps.rb

@@ -0,0 +1,34 @@
+Then(/^the file named "(.*?)" should have a file format of "(.*?)"$/) do |filename, format|
+  case format
+  when 'lz4'
+    assert_exact_output([0x184D2204].pack('V'),
+      File.open(filename, 'rb') {|f| f.read(4) })
+  when 'qcow2v3', 'qcow3'
+    steps %{
+      When I successfully run `qemu-img info #{filename}`
+      Then the output from "qemu-img info #{filename}" should contain "file format: qcow2"
+      And the output from "qemu-img info #{filename}" should contain "compat: 1.1"
+    }
+  else
+    assert_no_partial_output([0x184D2204].pack('V'),
+      File.open(filename, 'rb') {|f| f.read(4) })
+    steps(%{
+      When I successfully run `qemu-img info #{filename}`
+      Then the output from "qemu-img info #{filename}" should contain "file format: #{format}"
+      And the output from "qemu-img info #{filename}" should not contain "compat: 1.1"
+    })
+  end
+end
+
+Given(/^a correctly initialised cache at "(.*?)"$/) do |basedir|
+  steps %{
+    Given a directory named "#{basedir}/cache"
+    And a directory named "#{basedir}/temp"
+  }
+end
+
+Given(/^an empty directory named "([^"]*)"$/) do |dir_name|
+  create_dir(dir_name)
+  FileUtils.rm_rf(File.join(dir_name, '.'), :secure => true)
+end
+

data/features/support/env.rb

@@ -0,0 +1,20 @@
+require 'aruba/cucumber'
+require 'methadone/cucumber'
+require 'fileutils'
+
+ENV['PATH'] = "#{File.expand_path(File.dirname(__FILE__) + '/../../bin')}#{File::PATH_SEPARATOR}#{ENV['PATH']}"
+LIB_DIR = File.join(File.expand_path(File.dirname(__FILE__)),'..','..','lib')
+
+Before do
+  # Using "announce" causes massive warnings on 1.9.2
+  @puts = true
+  @original_rubylib = ENV['RUBYLIB']
+  ENV['RUBYLIB'] = LIB_DIR + File::PATH_SEPARATOR + ENV['RUBYLIB'].to_s
+end
+
+After do
+  ENV['RUBYLIB'] = @original_rubylib
+  
+  FileUtils.rm_rf(Dir.glob('/tmp/uricp/*'))
+
+end

data/features/userid_download.feature

@@ -0,0 +1,21 @@
+@orbit
+@orbitdownloads
+Feature: User authenticated download of images from orbit
+  In order to download images from orbit
+  As a command line user
+  I want to retrieve the URI via an optional cache and copy correctly to target in the right format using a user id directly
+
+  Background: 
+    Given an empty directory named "/tmp/uricp"
+    And the default aruba timeout is 15 seconds
+    And a container called "test"
+
+  Scenario: qcow download no conversion, no cache with userid
+    When I retrieve "img-qcow2" from container "test" into "file:///tmp/uricp/srv-test1" with a userid
+    Then a file named "/tmp/uricp/srv-test1" should exist
+    And the file named "/tmp/uricp/srv-test1" should have a file format of "qcow2v3"
+
+  Scenario: direct upload with userid
+    Given a 102400 byte file named "/tmp/uricp/srv-testy"
+    When I store "img-usrid" into container "test" from "file:///tmp/uricp/srv-testy" with a userid
+    Then a 102400 byte entry should exist in container "test" called "img-usrid"

data/lib/segment_upload.rb

@@ -0,0 +1,38 @@
+require "uricp/version"
+require "uricp/curl_primitives"
+require "uricp/orbit_auth"
+require "uricp/segmenter"
+
+module Uricp
+
+  UnsupportedURLtype = Class.new(ArgumentError)
+
+end
+
+#Monkey patch a copy_stream facility in using 'sendfile'
+unless IO.respond_to? :copy_stream
+  require 'sendfile'
+
+  def IO.copy_stream(src, dst, copy_length=nil, offset=nil)
+    unless src.stat.pipe?
+      current_pos = src.pos
+      count = dst.sendfile(src, offset || current_pos, copy_length)
+      src.seek(count, IO::SEEK_CUR)
+      return count
+    else
+      amount = copy_length.to_i
+      buf_size = 2**16
+      buffer=""
+      while amount > 0 do
+	src.read(buf_size, buffer)
+	amount_read = buffer.length
+	dst.write(buffer)
+	amount -= amount_read
+	break if src.eof?
+      end
+      return copy_length.to_i - amount
+    end
+  rescue EOFError
+    0
+  end
+end

data/lib/uricp/curl_primitives.rb

@@ -0,0 +1,45 @@
+module Uricp::CurlPrimitives
+
+    attr_reader :options
+
+    def from
+      options['from_uri']
+    end
+
+    def from=(target)
+      options['from_uri'] = target
+    end
+
+    def to
+      options['to_uri']
+    end
+
+    def to=(target)
+      options['to_uri'] = target
+    end
+  
+    def curl_command
+      "curl --fail --silent"
+    end
+
+    def authentication
+      "-H X-Auth-Token:#{options['auth-token']}" if http_authentication?
+    end
+
+    def http_authentication?
+      options['auth-token']
+    end
+
+    def curl_upload_from(source, destination = to)
+      "#{curl_command} #{authentication} -T #{source} #{destination.to_s};"
+    end
+
+    def curl_download_to_pipe
+      "#{curl_command} #{authentication} #{from.to_s} |"
+    end
+
+    def curl_manifest(object_manifest, destination = to)
+      "#{curl_command} #{authentication} -X PUT -H 'X-Object-Manifest: #{object_manifest}' #{destination.to_s} --data-binary ''"
+    end
+  
+end

data/lib/uricp/orbit_auth.rb

@@ -0,0 +1,59 @@
+require 'open-uri'
+module Uricp
+
+  class OrbitAuth
+    
+    AuthenticationFailure = Class.new(ArgumentError)
+
+    def initialize(auth_uri, auth_id, auth_key)
+      auth_uri.open(
+        'X-Auth-User' => auth_id,
+	'X-Auth-Key' => auth_key,
+	'Range' => 'bytes=0-0'
+      ) do |uri|
+        @storage_url = uri.meta['x-storage-url']
+	@token = uri.meta['x-auth-token']
+      end
+    rescue OpenURI::HTTPError => e
+      raise AuthenticationFailure, "Cannot authenticate against #{auth_uri}"
+    end
+
+    attr_reader :storage_url, :token
+
+    def self.validate_options(options)
+      if options['auth-token'] && (options['auth-key'] || options['auth-user'])
+	raise ::OptionParser::NeedlessArgument,
+	  "use either key based or token based authentication"
+      end
+      if options['auth-key'].nil? ^ options['auth-user'].nil?
+	raise ::OptionParser::MissingArgument,
+	  "'auth-user' requires 'auth-key'"
+      end
+      if (options['auth-token'] || options['auth-user']) && options['auth_uri'].nil?
+	raise ::OptionParser::NeedlessArgument,
+	  "authentication is for http uris only"
+      end
+    end
+
+    def self.add_auth_token(options)
+      if options['auth-user']
+	orbit_credentials = self.new(options['auth_uri'],
+			      options['auth-user'], options['auth-key'])
+	options['auth-token'] = orbit_credentials.token
+	options.delete('auth-key')
+	options.delete('auth-user')
+      end
+    end
+
+    def self.add_auth_to_optionparser(app)
+      app.on("--auth-token AUTH_TOKEN",
+	"Use AUTH_TOKEN for non-local requests")
+      app.on("--auth-user AUTH_USER",
+	"Use AUTH_USER for authentication")
+      app.on("--auth-key AUTH_KEY",
+	"Use AUTH_KEY for authentication")
+    end
+
+  end
+
+end

data/lib/uricp/segmenter.rb

@@ -0,0 +1,88 @@
+require 'pathname'
+
+module Uricp
+
+  class Segmenter
+
+    include Methadone::CLILogging
+    include Methadone::SH
+    include Uricp::CurlPrimitives
+
+    def initialize(options)
+      @options = options
+      source = options['from']
+      if @source = source && open(source)
+        @stream = File.pipe?(source) || File.chardev?(source)
+      else
+        @source = STDIN
+	@stream = true
+      end
+      split_path
+    end
+
+    def split_path
+      elements = Pathname.new(to.path).enum_for(:each_filename).to_a
+      elements.shift
+      @account=elements.shift
+      @container=elements.shift
+      @object_path=File.join(elements)
+    end
+
+    def upload
+      if large_upload?
+        segmented_upload
+      else
+        sh! curl_upload_from(options['from']),
+	  :on_fail => "Upload to #{to} failed"
+      end
+    end
+
+    def segment_size
+      options['segment-size'].to_i
+    end
+
+    def stream?
+      @stream
+    end
+
+    def large_upload?
+      stream? || @source.stat.size > segment_size
+    end
+
+    def manifest_suffix
+      @manifest_suffix ||= [Time.now.to_f, @source.stat.size, segment_size].join('/')
+    end
+
+    def object_manifest
+      @object_manifest ||= File.join(@container, @object_path, manifest_suffix)+'/'
+    end
+
+    def segmented_upload
+      debug "#{self.class.name}: Large upload detected - segmenting into #{segment_size} byte chunks."
+      suffix = 0
+      until @source.eof?
+	debug "#{self.class.name}: Uploading segment #{suffix}"
+	upload_segment(suffix)
+	suffix = suffix.next
+      end
+      add_manifest
+    end
+
+    def upload_segment(segment_number)
+      segment_name = File.join(to.to_s, manifest_suffix, '%08d' % segment_number)
+      debug "Uploading with #{curl_upload_from('-', segment_name)}"
+      open('|'+curl_upload_from('-', segment_name), 'w') do |destination|
+        copy_length = IO.copy_stream(@source, destination, segment_size)
+	debug "#{self.class.name}: Uploaded #{copy_length} bytes to #{segment_name}"
+      end
+    end
+
+    def add_manifest
+      debug "Adding DLO object_manifest #{object_manifest}"
+      sh! curl_manifest(object_manifest),
+	:on_fail => "Upload to #{to} failed"
+    end
+
+  end
+
+end

data/lib/uricp/strategy/cache_common.rb

@@ -0,0 +1,48 @@
+require 'fileutils'
+
+module Uricp::Strategy
+
+  module CacheCommon
+    
+    def validate_cache!
+      raise Uricp::MissingCache,
+	"No cache found at #{cache_root}. Expected a 'cache' and 'temp' directory" unless cache_exists?
+      raise Uricp::MissingCache,
+	"No cache name found" unless cache_name
+    end
+      
+    def in_cache?
+      File.readable? cache_file
+    end
+
+    def cache_root
+      options['cache']
+    end
+
+    def cache_name
+      options['cache_name']
+    end
+
+    def cache_exists?
+      cache_root && %w{temp cache}.all? do |d|
+        File.directory?(File.join(cache_root, d))
+      end
+    end
+
+    def temp_cache_file
+      @temp_cache_file ||= File.join(cache_root, 'temp', cache_name)
+    end
+
+    def temp_cache_uri
+      URI.join('file:///', temp_cache_file)
+    end
+
+    def cache_file
+      @cache_file ||= File.join(cache_root, 'cache', cache_name)
+    end
+
+  end
+
+end
+
+

data/lib/uricp/strategy/cached_get.rb

@@ -0,0 +1,48 @@
+require 'uri'
+
+module Uricp::Strategy
+  
+  class CachedGet
+
+    include Uricp::Strategy::Common
+    include Uricp::Strategy::CacheCommon
+
+    def appropriate?
+      if cache_root
+        validate_cache!
+	if in_cache? || file_source?
+	  return proposal
+	else
+	  debug "#{self.class.name}: no cache entry for #{options['from_uri']}"
+	end
+      else
+	debug "#{self.class.name}: not appropriate"
+      end
+      false
+    end
+
+    def command
+      ":;" 
+    end
+
+    def proposal
+      @proposed_options = options.dup
+      unless file_source?
+	@proposed_options['from_uri'] = URI.join('file:///', cache_file)
+      end
+      @proposed_options.delete('cache')
+      @proposed_options.delete('cache_name')
+      if conversion_required?
+	@proposed_options['source-format'] =
+	  File.open(@proposed_options['from_uri'].path) {|f| encoding(f)}
+	if @proposed_options['source-format'] == @proposed_options['target-format']
+	  @proposed_options.delete('source-format')
+	  @proposed_options.delete('target-format')
+	end
+      end
+      self
+    end
+
+  end
+
+end