uricp 0.0.11 → 0.0.16

Sign up to get free protection for your applications and to get access to all the features.
Files changed (48) hide show
  1. checksums.yaml +5 -5
  2. data/.gitignore +2 -1
  3. data/.rubocop.yml +3 -0
  4. data/Gemfile.lock +15 -19
  5. data/Jenkinsfile +113 -0
  6. data/README.md +2 -2
  7. data/Rakefile +26 -31
  8. data/bionic/Dockerfile +20 -0
  9. data/centos7/Dockerfile +20 -0
  10. data/features/auth_download.feature +17 -0
  11. data/features/step_definitions/orbit_steps.rb +19 -13
  12. data/lib/segment_upload.rb +20 -22
  13. data/lib/uricp.rb +3 -29
  14. data/lib/uricp/curl_primitives.rb +31 -33
  15. data/lib/uricp/orbit_auth.rb +23 -27
  16. data/lib/uricp/segmenter.rb +12 -16
  17. data/lib/uricp/strategy/cache_common.rb +11 -11
  18. data/lib/uricp/strategy/cached_get.rb +14 -20
  19. data/lib/uricp/strategy/cleaner.rb +2 -8
  20. data/lib/uricp/strategy/common.rb +27 -19
  21. data/lib/uricp/strategy/local_convert.rb +9 -16
  22. data/lib/uricp/strategy/local_link.rb +10 -8
  23. data/lib/uricp/strategy/piped_cache.rb +7 -13
  24. data/lib/uricp/strategy/piped_cache_convert.rb +14 -18
  25. data/lib/uricp/strategy/piped_compress.rb +3 -8
  26. data/lib/uricp/strategy/piped_decompress.rb +7 -11
  27. data/lib/uricp/strategy/piped_local_compress.rb +0 -4
  28. data/lib/uricp/strategy/piped_local_decompress.rb +11 -17
  29. data/lib/uricp/strategy/piped_local_get.rb +0 -5
  30. data/lib/uricp/strategy/piped_local_put.rb +3 -9
  31. data/lib/uricp/strategy/piped_rbd_get.rb +44 -0
  32. data/lib/uricp/strategy/piped_remote_get.rb +28 -19
  33. data/lib/uricp/strategy/rbd_remote_put.rb +36 -0
  34. data/lib/uricp/strategy/rbd_sweeper.rb +22 -0
  35. data/lib/uricp/strategy/remote_put.rb +11 -17
  36. data/lib/uricp/strategy/segmented_remote_put.rb +16 -28
  37. data/lib/uricp/strategy/sweeper.rb +2 -8
  38. data/lib/uricp/uri_strategy.rb +14 -13
  39. data/lib/uricp/version.rb +2 -2
  40. data/trusty/Dockerfile +20 -0
  41. data/uricp.gemspec +3 -4
  42. data/xenial/Dockerfile +20 -0
  43. metadata +39 -51
  44. data/Dockerfile_centos6.6 +0 -39
  45. data/Dockerfile_centos7 +0 -39
  46. data/Dockerfile_trusty-ruby193 +0 -32
  47. data/README.rdoc +0 -23
  48. data/spec/something_spec.rb +0 -5
@@ -1,36 +1,34 @@
1
- require "uricp/version"
2
- require "uricp/curl_primitives"
3
- require "uricp/orbit_auth"
4
- require "uricp/segmenter"
1
+ require 'uricp/version'
2
+ require 'uricp/curl_primitives'
3
+ require 'uricp/orbit_auth'
4
+ require 'uricp/segmenter'
5
5
 
6
6
  module Uricp
7
-
8
7
  UnsupportedURLtype = Class.new(ArgumentError)
9
-
10
8
  end
11
9
 
12
- #Monkey patch a copy_stream facility in using 'sendfile'
10
+ # Monkey patch a copy_stream facility in using 'sendfile'
13
11
  unless IO.respond_to? :copy_stream
14
12
  require 'sendfile'
15
13
 
16
- def IO.copy_stream(src, dst, copy_length=nil, offset=nil)
17
- unless src.stat.pipe?
18
- current_pos = src.pos
19
- count = dst.sendfile(src, offset || current_pos, copy_length)
20
- src.seek(count, IO::SEEK_CUR)
21
- return count
22
- else
14
+ def IO.copy_stream(src, dst, copy_length = nil, offset = nil)
15
+ if src.stat.pipe?
23
16
  amount = copy_length.to_i
24
17
  buf_size = [amount, 2**16].min
25
- buffer=""
26
- while amount > 0 do
27
- src.read(buf_size, buffer)
28
- amount_read = buffer.length
29
- dst.write(buffer)
30
- amount -= amount_read
31
- break if src.eof?
18
+ buffer = ''
19
+ while amount > 0
20
+ src.read(buf_size, buffer)
21
+ amount_read = buffer.length
22
+ dst.write(buffer)
23
+ amount -= amount_read
24
+ break if src.eof?
32
25
  end
33
- return copy_length.to_i - amount
26
+ copy_length.to_i - amount
27
+ else
28
+ current_pos = src.pos
29
+ count = dst.sendfile(src, offset || current_pos, copy_length)
30
+ src.seek(count, IO::SEEK_CUR)
31
+ count
34
32
  end
35
33
  rescue EOFError
36
34
  0
@@ -7,52 +7,26 @@ require 'uricp/strategy/local_link'
7
7
  require 'uricp/strategy/piped_cache'
8
8
  require 'uricp/strategy/piped_cache_convert'
9
9
  require 'uricp/strategy/piped_local_get'
10
+ require 'uricp/strategy/piped_rbd_get'
10
11
  require 'uricp/strategy/piped_local_put'
11
12
  require 'uricp/strategy/piped_remote_get'
12
13
  require 'uricp/strategy/piped_compress'
13
14
  require 'uricp/strategy/piped_decompress'
14
15
  require 'uricp/strategy/piped_local_decompress'
15
16
  require 'uricp/strategy/piped_local_compress'
17
+ require 'uricp/strategy/rbd_remote_put'
16
18
  require 'uricp/strategy/segmented_remote_put'
17
19
  require 'uricp/strategy/remote_put'
18
20
  require 'uricp/strategy/cached_get'
21
+ require 'uricp/strategy/rbd_sweeper'
19
22
  require 'uricp/strategy/sweeper'
20
23
  require 'uricp/strategy/cleaner'
21
24
  require 'uricp/uri_strategy'
22
25
  require 'uricp/orbit_auth'
23
26
 
24
27
  module Uricp
25
-
26
28
  UnsupportedURLtype = Class.new(ArgumentError)
27
29
  MissingCache = Class.new(ArgumentError)
28
30
  UnsupportedConversion = Class.new(ArgumentError)
29
31
  ConversionRequired = Class.new(ArgumentError)
30
-
31
- end
32
-
33
- require 'tempfile'
34
- class Dir
35
- module Tmpname
36
- unless respond_to? :make_tmpname
37
-
38
- module_function
39
- def make_tmpname(prefix_suffix, n)
40
- case prefix_suffix
41
- when String
42
- prefix = prefix_suffix
43
- suffix = ""
44
- when Array
45
- prefix = prefix_suffix[0]
46
- suffix = prefix_suffix[1]
47
- else
48
- raise ArgumentError, "unexpected prefix_suffix: #{prefix_suffix.inspect}"
49
- end
50
- t = Time.now.strftime("%Y%m%d")
51
- path = "#{prefix}#{t}-#{$$}-#{rand(0x100000000).to_s(36)}"
52
- path << "-#{n}" if n
53
- path << suffix
54
- end
55
-
56
- end
57
- end
58
32
  end
@@ -1,45 +1,43 @@
1
1
  module Uricp::CurlPrimitives
2
+ attr_reader :options
2
3
 
3
- attr_reader :options
4
+ def from
5
+ options['from_uri']
6
+ end
4
7
 
5
- def from
6
- options['from_uri']
7
- end
8
+ def from=(target)
9
+ options['from_uri'] = target
10
+ end
8
11
 
9
- def from=(target)
10
- options['from_uri'] = target
11
- end
12
+ def to
13
+ options['to_uri']
14
+ end
12
15
 
13
- def to
14
- options['to_uri']
15
- end
16
+ def to=(target)
17
+ options['to_uri'] = target
18
+ end
16
19
 
17
- def to=(target)
18
- options['to_uri'] = target
19
- end
20
-
21
- def curl_command
22
- "curl --fail --silent"
23
- end
20
+ def curl_command
21
+ 'curl --fail --silent'
22
+ end
24
23
 
25
- def authentication
26
- "-H X-Auth-Token:#{options['authenticator'].call}" if http_authentication?
27
- end
24
+ def authentication
25
+ "-H X-Auth-Token:#{options['authenticator'].call}" if http_authentication?
26
+ end
28
27
 
29
- def http_authentication?
30
- options['authenticator']
31
- end
28
+ def http_authentication?
29
+ options['authenticator']
30
+ end
32
31
 
33
- def curl_upload_from(source, destination = to)
34
- "#{curl_command} #{authentication} -T #{source} #{destination.to_s};"
35
- end
32
+ def curl_upload_from(source, destination = to)
33
+ "#{curl_command} #{authentication} -T #{source} #{destination};"
34
+ end
36
35
 
37
- def curl_download_to_pipe
38
- "#{curl_command} #{authentication} #{from.to_s} |"
39
- end
36
+ def curl_download_to_pipe
37
+ "#{curl_command} #{authentication} #{from} |"
38
+ end
40
39
 
41
- def curl_manifest(object_manifest, destination = to)
42
- "#{curl_command} #{authentication} -X PUT -H 'X-Object-Manifest: #{object_manifest}' #{destination.to_s} --data-binary ''"
43
- end
44
-
40
+ def curl_manifest(object_manifest, destination = to)
41
+ "#{curl_command} #{authentication} -X PUT -H 'X-Object-Manifest: #{object_manifest}' #{destination} --data-binary ''"
42
+ end
45
43
  end
@@ -1,8 +1,6 @@
1
1
  require 'open-uri'
2
2
  module Uricp
3
-
4
3
  class OrbitAuth
5
-
6
4
  AuthenticationFailure = Class.new(ArgumentError)
7
5
 
8
6
  def initialize(auth_uri, auth_id, auth_key)
@@ -14,9 +12,9 @@ module Uricp
14
12
  def storage_url
15
13
  @storage_url ||
16
14
  begin
17
- authenticate
18
- @storage_url
19
- end
15
+ authenticate
16
+ @storage_url
17
+ end
20
18
  end
21
19
 
22
20
  def token
@@ -26,27 +24,27 @@ module Uricp
26
24
 
27
25
  def self.validate_options(options)
28
26
  if options['auth-token'] && (options['auth-key'] || options['auth-user'])
29
- raise ::OptionParser::NeedlessArgument,
30
- "use either key based or token based authentication"
27
+ raise ::OptionParser::NeedlessArgument,
28
+ 'use either key based or token based authentication'
31
29
  end
32
30
  if options['auth-key'].nil? ^ options['auth-user'].nil?
33
- raise ::OptionParser::MissingArgument,
34
- "'auth-user' requires 'auth-key'"
31
+ raise ::OptionParser::MissingArgument,
32
+ "'auth-user' requires 'auth-key'"
35
33
  end
36
34
  if (options['auth-token'] || options['auth-user']) && options['auth_uri'].nil?
37
- raise ::OptionParser::NeedlessArgument,
38
- "authentication is for http uris only"
35
+ raise ::OptionParser::NeedlessArgument,
36
+ 'authentication is for http uris only'
39
37
  end
40
38
  end
41
39
 
42
40
  def self.add_auth_token(options)
43
41
  if options['auth-user']
44
- orbit_credentials = self.new(options['auth_uri'],
45
- options['auth-user'], options['auth-key'])
46
- options['authenticator'] = orbit_credentials.method(:token)
42
+ orbit_credentials = new(options['auth_uri'],
43
+ options['auth-user'], options['auth-key'])
44
+ options['authenticator'] = orbit_credentials.method(:token)
47
45
  elsif options['auth-token']
48
46
  orbit_token = options['auth-token']
49
- options['authenticator'] = lambda { orbit_token }
47
+ options['authenticator'] = -> { orbit_token }
50
48
  end
51
49
  options.delete('auth-key')
52
50
  options.delete('auth-user')
@@ -54,29 +52,27 @@ module Uricp
54
52
  end
55
53
 
56
54
  def self.add_auth_to_optionparser(app)
57
- app.on("--auth-token AUTH_TOKEN",
58
- "Use AUTH_TOKEN for non-local requests")
59
- app.on("--auth-user AUTH_USER",
60
- "Use AUTH_USER for authentication")
61
- app.on("--auth-key AUTH_KEY",
62
- "Use AUTH_KEY for authentication")
55
+ app.on('--auth-token AUTH_TOKEN',
56
+ 'Use AUTH_TOKEN for non-local requests')
57
+ app.on('--auth-user AUTH_USER',
58
+ 'Use AUTH_USER for authentication')
59
+ app.on('--auth-key AUTH_KEY',
60
+ 'Use AUTH_KEY for authentication')
63
61
  end
64
62
 
65
- private
63
+ private
66
64
 
67
65
  def authenticate
68
66
  @auth_uri.open(
69
67
  'X-Auth-User' => @auth_id,
70
- 'X-Auth-Key' => @auth_key,
71
- 'Range' => 'bytes=0-0'
68
+ 'X-Auth-Key' => @auth_key,
69
+ 'Range' => 'bytes=0-0'
72
70
  ) do |uri|
73
71
  @storage_url = uri.meta['x-storage-url']
74
- @token = uri.meta['x-auth-token']
72
+ @token = uri.meta['x-auth-token']
75
73
  end
76
74
  rescue OpenURI::HTTPError => e
77
75
  raise AuthenticationFailure, "Cannot authenticate against #{@auth_uri}"
78
76
  end
79
-
80
77
  end
81
-
82
78
  end
@@ -1,9 +1,7 @@
1
1
  require 'pathname'
2
2
 
3
3
  module Uricp
4
-
5
4
  class Segmenter
6
-
7
5
  include Methadone::CLILogging
8
6
  include Methadone::SH
9
7
  include Uricp::CurlPrimitives
@@ -15,7 +13,7 @@ module Uricp
15
13
  @stream = File.pipe?(source) || File.chardev?(source)
16
14
  else
17
15
  @source = STDIN
18
- @stream = true
16
+ @stream = true
19
17
  end
20
18
  split_path
21
19
  end
@@ -23,9 +21,9 @@ module Uricp
23
21
  def split_path
24
22
  elements = Pathname.new(to.path).enum_for(:each_filename).to_a
25
23
  elements.shift
26
- @account=elements.shift
27
- @container=elements.shift
28
- @object_path=File.join(elements)
24
+ @account = elements.shift
25
+ @container = elements.shift
26
+ @object_path = File.join(elements)
29
27
  end
30
28
 
31
29
  def upload
@@ -33,7 +31,7 @@ module Uricp
33
31
  segmented_upload
34
32
  else
35
33
  sh! curl_upload_from(options['from']),
36
- :on_fail => "Upload to #{to} failed"
34
+ on_fail: "Upload to #{to} failed"
37
35
  end
38
36
  end
39
37
 
@@ -54,16 +52,16 @@ module Uricp
54
52
  end
55
53
 
56
54
  def object_manifest
57
- @object_manifest ||= File.join(@container, @object_path, manifest_suffix)+'/'
55
+ @object_manifest ||= File.join(@container, @object_path, manifest_suffix) + '/'
58
56
  end
59
57
 
60
58
  def segmented_upload
61
59
  debug "#{self.class.name}: Large upload detected - segmenting into #{segment_size} byte chunks."
62
60
  suffix = 0
63
61
  until @source.eof?
64
- debug "#{self.class.name}: Uploading segment #{suffix}"
65
- upload_segment(suffix)
66
- suffix = suffix.next
62
+ debug "#{self.class.name}: Uploading segment #{suffix}"
63
+ upload_segment(suffix)
64
+ suffix = suffix.next
67
65
  end
68
66
  add_manifest
69
67
  end
@@ -71,18 +69,16 @@ module Uricp
71
69
  def upload_segment(segment_number)
72
70
  segment_name = File.join(to.to_s, manifest_suffix, '%08d' % segment_number)
73
71
  debug "Uploading with #{curl_upload_from('-', segment_name)}"
74
- open('|'+curl_upload_from('-', segment_name), 'w') do |destination|
72
+ open('|' + curl_upload_from('-', segment_name), 'w') do |destination|
75
73
  copy_length = IO.copy_stream(@source, destination, segment_size)
76
- debug "#{self.class.name}: Uploaded #{copy_length} bytes to #{segment_name}"
74
+ debug "#{self.class.name}: Uploaded #{copy_length} bytes to #{segment_name}"
77
75
  end
78
76
  end
79
77
 
80
78
  def add_manifest
81
79
  debug "Adding DLO object_manifest #{object_manifest}"
82
80
  sh! curl_manifest(object_manifest),
83
- :on_fail => "Upload to #{to} failed"
81
+ on_fail: "Upload to #{to} failed"
84
82
  end
85
-
86
83
  end
87
-
88
84
  end
@@ -1,16 +1,20 @@
1
1
  require 'fileutils'
2
2
 
3
3
  module Uricp::Strategy
4
-
5
4
  module CacheCommon
6
-
7
5
  def validate_cache!
6
+ return if dry_run?
7
+
8
+ unless cache_exists?
9
+ raise Uricp::MissingCache,
10
+ "no cache found at #{cache_root}. Expected a 'cache' and 'temp' directory"
11
+ end
12
+ return if cache_name
13
+
8
14
  raise Uricp::MissingCache,
9
- "no cache found at #{cache_root}. Expected a 'cache' and 'temp' directory" unless cache_exists?
10
- raise Uricp::MissingCache,
11
- "no cache name found" unless cache_name
15
+ 'no cache name found'
12
16
  end
13
-
17
+
14
18
  def in_cache?
15
19
  File.readable? cache_file
16
20
  end
@@ -24,7 +28,7 @@ module Uricp::Strategy
24
28
  end
25
29
 
26
30
  def cache_exists?
27
- cache_root && %w{temp cache}.all? do |d|
31
+ cache_root && %w[temp cache].all? do |d|
28
32
  File.directory?(File.join(cache_root, d))
29
33
  end
30
34
  end
@@ -40,9 +44,5 @@ module Uricp::Strategy
40
44
  def cache_file
41
45
  @cache_file ||= File.join(cache_root, 'cache', cache_name)
42
46
  end
43
-
44
47
  end
45
-
46
48
  end
47
-
48
-
@@ -1,48 +1,42 @@
1
1
  require 'uri'
2
2
 
3
3
  module Uricp::Strategy
4
-
5
4
  class CachedGet
6
-
7
5
  include Uricp::Strategy::Common
8
6
  include Uricp::Strategy::CacheCommon
9
7
 
10
8
  def appropriate?
11
9
  if cache_root
12
10
  validate_cache!
13
- if in_cache? || file_source?
14
- return proposal
15
- else
16
- debug "#{self.class.name}: no cache entry for #{options['from_uri']}"
17
- end
11
+ return proposal if in_cache? || file_source?
12
+
13
+ debug "#{self.class.name}: no cache entry for #{options['from_uri']}"
18
14
  else
19
- debug "#{self.class.name}: not appropriate"
15
+ debug "#{self.class.name}: not appropriate"
20
16
  end
21
17
  false
22
18
  end
23
19
 
24
20
  def command
25
- ":;"
21
+ ':;'
26
22
  end
27
23
 
28
24
  def proposal
29
25
  @proposed_options = options.dup
30
- unless file_source?
31
- @proposed_options['from_uri'] = URI.join('file:///', cache_file)
32
- end
26
+ @proposed_options['from_uri'] = URI.join('file:///', cache_file) unless file_source?
33
27
  @proposed_options.delete('cache')
34
28
  @proposed_options.delete('cache_name')
35
29
  if conversion_required?
36
- @proposed_options['source-format'] =
37
- File.open(@proposed_options['from_uri'].path) {|f| encoding(f)}
38
- if @proposed_options['source-format'] == @proposed_options['target-format']
39
- @proposed_options.delete('source-format')
40
- @proposed_options.delete('target-format')
41
- end
30
+ unless dry_run?
31
+ @proposed_options['source-format'] =
32
+ File.open(@proposed_options['from_uri'].path) { |f| encoding(f) }
33
+ end
34
+ if @proposed_options['source-format'] == @proposed_options['target-format']
35
+ @proposed_options.delete('source-format')
36
+ @proposed_options.delete('target-format')
37
+ end
42
38
  end
43
39
  self
44
40
  end
45
-
46
41
  end
47
-
48
42
  end