oci 2.0.6 → 2.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +23 -1
- data/lib/oci.rb +4 -7
- data/lib/oci/api_client.rb +51 -6
- data/lib/oci/auth/auth.rb +5 -2
- data/lib/oci/auth/federation_client.rb +2 -2
- data/lib/oci/auth/internal/auth_token_request_signer.rb +2 -2
- data/lib/oci/auth/security_token_container.rb +2 -2
- data/lib/oci/auth/session_key_supplier.rb +3 -4
- data/lib/oci/auth/signers/instance_principals_security_token_signer.rb +30 -17
- data/lib/oci/auth/signers/security_token_signer.rb +13 -6
- data/lib/oci/auth/signers/x509_federation_client_based_security_token_signer.rb +8 -4
- data/lib/oci/auth/url_based_certificate_retriever.rb +3 -4
- data/lib/oci/auth/util.rb +6 -4
- data/lib/oci/base_signer.rb +54 -50
- data/lib/oci/config_file_loader.rb +3 -11
- data/lib/oci/core/blockstorage_client.rb +211 -0
- data/lib/oci/core/compute_client.rb +4 -2
- data/lib/oci/core/core.rb +9 -0
- data/lib/oci/core/models/attach_i_scsi_volume_details.rb +5 -1
- data/lib/oci/core/models/attach_volume_details.rb +13 -1
- data/lib/oci/core/models/create_public_ip_details.rb +186 -0
- data/lib/oci/core/models/create_vnic_details.rb +8 -2
- data/lib/oci/core/models/create_volume_backup_details.rb +27 -1
- data/lib/oci/core/models/create_volume_backup_policy_assignment_details.rb +133 -0
- data/lib/oci/core/models/create_volume_details.rb +15 -1
- data/lib/oci/core/models/get_public_ip_by_ip_address_details.rb +125 -0
- data/lib/oci/core/models/get_public_ip_by_private_ip_id_details.rb +124 -0
- data/lib/oci/core/models/i_scsi_volume_attachment.rb +5 -1
- data/lib/oci/core/models/public_ip.rb +328 -0
- data/lib/oci/core/models/update_public_ip_details.rb +140 -0
- data/lib/oci/core/models/volume_attachment.rb +13 -1
- data/lib/oci/core/models/volume_backup.rb +72 -1
- data/lib/oci/core/models/volume_backup_policy.rb +161 -0
- data/lib/oci/core/models/volume_backup_policy_assignment.rb +159 -0
- data/lib/oci/core/models/volume_backup_schedule.rb +191 -0
- data/lib/oci/core/virtual_network_client.rb +374 -4
- data/lib/oci/database/database_client.rb +3 -1
- data/lib/oci/dns/dns.rb +34 -0
- data/lib/oci/dns/dns_client.rb +985 -0
- data/lib/oci/dns/models/create_zone_details.rb +174 -0
- data/lib/oci/dns/models/external_master.rb +145 -0
- data/lib/oci/dns/models/patch_domain_records_details.rb +120 -0
- data/lib/oci/dns/models/patch_rr_set_details.rb +120 -0
- data/lib/oci/dns/models/patch_zone_records_details.rb +120 -0
- data/lib/oci/dns/models/record.rb +204 -0
- data/lib/oci/dns/models/record_collection.rb +121 -0
- data/lib/oci/dns/models/record_details.rb +204 -0
- data/lib/oci/dns/models/record_operation.rb +253 -0
- data/lib/oci/dns/models/rr_set.rb +123 -0
- data/lib/oci/dns/models/sort_order.rb +12 -0
- data/lib/oci/dns/models/tsig.rb +149 -0
- data/lib/oci/dns/models/update_domain_records_details.rb +120 -0
- data/lib/oci/dns/models/update_rr_set_details.rb +120 -0
- data/lib/oci/dns/models/update_zone_details.rb +122 -0
- data/lib/oci/dns/models/update_zone_records_details.rb +120 -0
- data/lib/oci/dns/models/zone.rb +272 -0
- data/lib/oci/dns/models/zone_summary.rb +230 -0
- data/lib/oci/dns/util.rb +2 -0
- data/lib/oci/errors.rb +20 -0
- data/lib/oci/identity/identity_client.rb +1 -0
- data/lib/oci/identity/models/create_dynamic_group_details.rb +4 -2
- data/lib/oci/identity/models/dynamic_group.rb +9 -6
- data/lib/oci/identity/models/update_dynamic_group_details.rb +4 -2
- data/lib/oci/load_balancer/load_balancer.rb +6 -0
- data/lib/oci/load_balancer/load_balancer_client.rb +219 -0
- data/lib/oci/load_balancer/models/create_listener_details.rb +17 -1
- data/lib/oci/load_balancer/models/create_load_balancer_details.rb +12 -1
- data/lib/oci/load_balancer/models/create_path_route_set_details.rb +138 -0
- data/lib/oci/load_balancer/models/listener.rb +17 -1
- data/lib/oci/load_balancer/models/listener_details.rb +17 -1
- data/lib/oci/load_balancer/models/load_balancer.rb +12 -1
- data/lib/oci/load_balancer/models/path_match_type.rb +153 -0
- data/lib/oci/load_balancer/models/path_route.rb +161 -0
- data/lib/oci/load_balancer/models/path_route_set.rb +139 -0
- data/lib/oci/load_balancer/models/path_route_set_details.rb +122 -0
- data/lib/oci/load_balancer/models/update_listener_details.rb +17 -1
- data/lib/oci/load_balancer/models/update_path_route_set_details.rb +122 -0
- data/lib/oci/load_balancer/util.rb +1 -3
- data/lib/oci/object_storage/transfer/multipart/internal/file_part_io_wrapper.rb +111 -0
- data/lib/oci/object_storage/transfer/multipart/internal/multipart_upload_parts_collection.rb +41 -0
- data/lib/oci/object_storage/transfer/multipart/internal/seekable_non_file_part_io_wrapper.rb +48 -0
- data/lib/oci/object_storage/transfer/multipart/internal/stdin_part_io_wrapper.rb +54 -0
- data/lib/oci/object_storage/transfer/multipart/multipart_object_assembler.rb +545 -0
- data/lib/oci/object_storage/transfer/transfer.rb +31 -0
- data/lib/oci/object_storage/transfer/upload_manager.rb +212 -0
- data/lib/oci/object_storage/transfer/upload_manager_config.rb +60 -0
- data/lib/oci/regions.rb +3 -1
- data/lib/oci/response.rb +1 -3
- data/lib/oci/version.rb +1 -1
- data/lib/oci/waiter.rb +16 -11
- data/lib/oraclebmc.rb +1 -1
- metadata +46 -2
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
# Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
|
|
2
|
+
|
|
3
|
+
require 'date'
|
|
4
|
+
|
|
5
|
+
module OCI
|
|
6
|
+
# An updated set of path route rules that overwrites the existing set of rules.
|
|
7
|
+
class LoadBalancer::Models::UpdatePathRouteSetDetails
|
|
8
|
+
# **[Required]** The set of path route rules.
|
|
9
|
+
# @return [Array<OCI::LoadBalancer::Models::PathRoute>]
|
|
10
|
+
attr_accessor :path_routes
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
# Initializes the object
|
|
14
|
+
# @param [Hash] attributes Model attributes in the form of hash
|
|
15
|
+
# @option attributes [Array<OCI::LoadBalancer::Models::PathRoute>] :pathRoutes The value to assign to the {#path_routes} property
|
|
16
|
+
def initialize(attributes = {})
|
|
17
|
+
return unless attributes.is_a?(Hash)
|
|
18
|
+
|
|
19
|
+
# convert string to symbol for hash key
|
|
20
|
+
attributes = attributes.each_with_object({}){|(k,v), h| h[k.to_sym] = v}
|
|
21
|
+
|
|
22
|
+
if attributes[:'pathRoutes']
|
|
23
|
+
self.path_routes = attributes[:'pathRoutes']
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Checks equality by comparing each attribute.
|
|
29
|
+
# @param [Object] other_object to be compared
|
|
30
|
+
def ==(other_object)
|
|
31
|
+
return true if self.equal?(other_object)
|
|
32
|
+
self.class == other_object.class &&
|
|
33
|
+
path_routes == other_object.path_routes
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# @see the `==` method
|
|
37
|
+
# @param [Object] other_object to be compared
|
|
38
|
+
def eql?(other_object)
|
|
39
|
+
self == other_object
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Calculates hash code according to all attributes.
|
|
43
|
+
# @return [Fixnum] Hash code
|
|
44
|
+
def hash
|
|
45
|
+
[path_routes].hash
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Builds the object from hash
|
|
49
|
+
# @param [Hash] attributes Model attributes in the form of hash
|
|
50
|
+
# @return [Object] Returns the model itself
|
|
51
|
+
def build_from_hash(attributes)
|
|
52
|
+
return nil unless attributes.is_a?(Hash)
|
|
53
|
+
self.class.swagger_types.each_pair do |key, type|
|
|
54
|
+
if type =~ /^Array<(.*)>/i
|
|
55
|
+
# check to ensure the input is an array given that the the attribute
|
|
56
|
+
# is documented as an array but the input is not
|
|
57
|
+
if attributes[self.class.attribute_map[key]].is_a?(Array)
|
|
58
|
+
self.public_method("#{key}=").call(attributes[self.class.attribute_map[key]].map{ |v| OCI::Internal::Util.convert_to_type($1, v) } )
|
|
59
|
+
end
|
|
60
|
+
elsif !attributes[self.class.attribute_map[key]].nil?
|
|
61
|
+
self.public_method("#{key}=").call(OCI::Internal::Util.convert_to_type(type, attributes[self.class.attribute_map[key]]))
|
|
62
|
+
end # or else data not found in attributes(hash), not an issue as the data can be optional
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
self
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Returns the string representation of the object
|
|
69
|
+
# @return [String] String presentation of the object
|
|
70
|
+
def to_s
|
|
71
|
+
to_hash.to_s
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# Returns the object in the form of hash
|
|
75
|
+
# @return [Hash] Returns the object in the form of hash
|
|
76
|
+
def to_hash
|
|
77
|
+
hash = {}
|
|
78
|
+
self.class.attribute_map.each_pair do |attr, param|
|
|
79
|
+
value = public_method(attr).call
|
|
80
|
+
next if value.nil? && !instance_variable_defined?("@#{attr}")
|
|
81
|
+
hash[param] = _to_hash(value)
|
|
82
|
+
end
|
|
83
|
+
hash
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
private
|
|
87
|
+
|
|
88
|
+
# Outputs non-array value in the form of hash
|
|
89
|
+
# For object, use to_hash. Otherwise, just return the value
|
|
90
|
+
# @param [Object] value Any valid value
|
|
91
|
+
# @return [Hash] Returns the value in the form of hash
|
|
92
|
+
def _to_hash(value)
|
|
93
|
+
if value.is_a?(Array)
|
|
94
|
+
value.compact.map{ |v| _to_hash(v) }
|
|
95
|
+
elsif value.is_a?(Hash)
|
|
96
|
+
{}.tap do |hash|
|
|
97
|
+
value.each { |k, v| hash[k] = _to_hash(v) }
|
|
98
|
+
end
|
|
99
|
+
elsif value.respond_to? :to_hash
|
|
100
|
+
value.to_hash
|
|
101
|
+
else
|
|
102
|
+
value
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
# Attribute mapping from ruby-style variable name to JSON key.
|
|
109
|
+
def self.attribute_map
|
|
110
|
+
{
|
|
111
|
+
:'path_routes' => :'pathRoutes'
|
|
112
|
+
}
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# Attribute type mapping.
|
|
116
|
+
def self.swagger_types
|
|
117
|
+
{
|
|
118
|
+
:'path_routes' => :'Array<OCI::LoadBalancer::Models::PathRoute>'
|
|
119
|
+
}
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
end
|
|
@@ -48,9 +48,7 @@ module OCI
|
|
|
48
48
|
sleep(interval_seconds)
|
|
49
49
|
|
|
50
50
|
interval_seconds *= 2
|
|
51
|
-
if interval_seconds > max_interval_seconds
|
|
52
|
-
interval_seconds = max_interval_seconds
|
|
53
|
-
end
|
|
51
|
+
interval_seconds = max_interval_seconds if interval_seconds > max_interval_seconds
|
|
54
52
|
end
|
|
55
53
|
end
|
|
56
54
|
end
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
# Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
|
|
2
|
+
|
|
3
|
+
module OCI
|
|
4
|
+
module ObjectStorage
|
|
5
|
+
module Transfer
|
|
6
|
+
module Multipart
|
|
7
|
+
module Internal
|
|
8
|
+
# An IO-like interface over a part of a file which will participate in a multipart upload. This allows us
|
|
9
|
+
# to upload a given segment of the file as an "upload part" of a multipart upload.
|
|
10
|
+
class FilePartIOWrapper
|
|
11
|
+
# The source file, identified by either a File/Tempfile object or a path to the file
|
|
12
|
+
# @return [String, File, Tempfile]
|
|
13
|
+
attr_reader :source
|
|
14
|
+
|
|
15
|
+
# The zero-based position in the file to start reading from
|
|
16
|
+
# @return [Integer]
|
|
17
|
+
attr_reader :offset
|
|
18
|
+
|
|
19
|
+
# The number of bytes to read from the file
|
|
20
|
+
# @return [Integer]
|
|
21
|
+
attr_reader :part_size
|
|
22
|
+
|
|
23
|
+
# The position of the first byte to start reading from. Synonym for offset
|
|
24
|
+
# @return [Integer]
|
|
25
|
+
attr_reader :first_byte
|
|
26
|
+
|
|
27
|
+
# The position of the last byte we'll read to (exclusive). Calculated as offset + part_size
|
|
28
|
+
# @return [Integer]
|
|
29
|
+
attr_reader :last_byte
|
|
30
|
+
|
|
31
|
+
# Creates a new FilePartIOWrapper
|
|
32
|
+
#
|
|
33
|
+
# @param [String, File, Tempfile] source The source file, or a path to it
|
|
34
|
+
# @param [Integer] offset The zero-based position in the file to start reading from
|
|
35
|
+
# @param [Integer] part_size The number of bytes to read from the file
|
|
36
|
+
def initialize(source:, offset:, part_size:)
|
|
37
|
+
@source = source
|
|
38
|
+
@offset = offset
|
|
39
|
+
@part_size = part_size
|
|
40
|
+
|
|
41
|
+
@first_byte = offset
|
|
42
|
+
@last_byte = offset + part_size
|
|
43
|
+
|
|
44
|
+
@file_handle = nil
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def close
|
|
48
|
+
@file_handle.close if @file_handle
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def read(bytes = nil, output_buffer = nil)
|
|
52
|
+
open_file unless @file_handle
|
|
53
|
+
read_internal(bytes, output_buffer)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def write(_content)
|
|
57
|
+
raise 'FilePartIOWrapper does not support writing'
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def size
|
|
61
|
+
@part_size
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def rewind
|
|
65
|
+
if @file_handle
|
|
66
|
+
@file_handle.seek(@first_byte)
|
|
67
|
+
@position = @first_byte
|
|
68
|
+
end
|
|
69
|
+
0
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
private
|
|
73
|
+
|
|
74
|
+
def open_file
|
|
75
|
+
@file_handle = File.open(@source)
|
|
76
|
+
rewind
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def read_internal(bytes, output_buffer)
|
|
80
|
+
if bytes
|
|
81
|
+
data = @file_handle.read([remaining_bytes, bytes].min)
|
|
82
|
+
data = nil if data == ''
|
|
83
|
+
end
|
|
84
|
+
data = @file_handle.read(remaining_bytes) unless bytes
|
|
85
|
+
|
|
86
|
+
@position += data.bytesize if data
|
|
87
|
+
|
|
88
|
+
# From IO docs (https://ruby-doc.org/core-2.3.1/IO.html#method-i-read):
|
|
89
|
+
#
|
|
90
|
+
# "If the optional outbuf argument is present, it must reference a String, which will receive the data.
|
|
91
|
+
# The outbuf will contain only the received data after the method call even if it is not empty at the
|
|
92
|
+
# beginning."
|
|
93
|
+
#
|
|
94
|
+
# So we use String's replace() method to sub in the data we read, or an empty string if there was
|
|
95
|
+
# no data read
|
|
96
|
+
if output_buffer
|
|
97
|
+
output_buffer.replace(data || '')
|
|
98
|
+
else
|
|
99
|
+
data
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def remaining_bytes
|
|
104
|
+
@last_byte - @position
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
end
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
|
|
2
|
+
|
|
3
|
+
module OCI
|
|
4
|
+
module ObjectStorage
|
|
5
|
+
module Transfer
|
|
6
|
+
module Multipart
|
|
7
|
+
module Internal
|
|
8
|
+
# Wraps a collection of parts to be uploaded to Object Storage in order to provide
|
|
9
|
+
# thread-safe access to the collection
|
|
10
|
+
class MultipartUploadPartsCollection
|
|
11
|
+
def initialize(parts = [])
|
|
12
|
+
@parts = parts
|
|
13
|
+
@lock = Mutex.new
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def push(part)
|
|
17
|
+
@lock.synchronize { @parts.push(part) }
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def shift
|
|
21
|
+
@lock.synchronize { @parts.shift }
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def clear!
|
|
25
|
+
@lock.synchronize { @parts.clear }
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def to_a
|
|
29
|
+
@lock.synchronize { @parts.dup }
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def length
|
|
33
|
+
@lock.synchronize { @parts.length }
|
|
34
|
+
end
|
|
35
|
+
alias size length
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
|
|
2
|
+
|
|
3
|
+
module OCI
|
|
4
|
+
module ObjectStorage
|
|
5
|
+
module Transfer
|
|
6
|
+
module Multipart
|
|
7
|
+
module Internal
|
|
8
|
+
# A class which wraps a seekable (but non-file) IO-like object (for example, a StringIO) so that
|
|
9
|
+
# we can extract a part of it to upload as an "upload part" of a multipart upload. This works by
|
|
10
|
+
# seeking to a given position in the IO-like object and then reading a certain number of bytes
|
|
11
|
+
# from that position onwards.
|
|
12
|
+
#
|
|
13
|
+
# Reading a part of the IO-like object is synchronised so that only a single thread can access the
|
|
14
|
+
# IO at any one time. This prevents issues where we would do, for example, concurrent seeks and so
|
|
15
|
+
# potentially end up reading content we didn't intend to read.
|
|
16
|
+
class SeekableNonFilePartIOWrapper
|
|
17
|
+
# The underlying IO-like object which we'll extract content from
|
|
18
|
+
# @return [IO]
|
|
19
|
+
attr_reader :source
|
|
20
|
+
|
|
21
|
+
# Creates a new SeekableNonFilePartIOWrapper
|
|
22
|
+
# @param [IO] source the IO-like object which will back this wrapper
|
|
23
|
+
def initialize(source:)
|
|
24
|
+
raise 'The provided source is not seekable' unless source.respond_to?(:seek)
|
|
25
|
+
@source = source
|
|
26
|
+
@lock = Mutex.new
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Reads a part/segment from the IO-like object by seeking to a given location and then
|
|
30
|
+
# reading a given number of bytes. This is synchronised so that only a single thread can
|
|
31
|
+
# perform this operation at any one time.
|
|
32
|
+
#
|
|
33
|
+
# @param [Integer] offset the zero-based position in the IO-like object where we'll start reading from
|
|
34
|
+
# @param [Integer] part_size the number of bytes to read
|
|
35
|
+
#
|
|
36
|
+
# @return [String] The content read from the IO-like object
|
|
37
|
+
def read(offset, part_size)
|
|
38
|
+
@lock.synchronize do
|
|
39
|
+
@source.seek(offset)
|
|
40
|
+
@source.read(part_size)
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
|
|
2
|
+
|
|
3
|
+
module OCI
|
|
4
|
+
module ObjectStorage
|
|
5
|
+
module Transfer
|
|
6
|
+
module Multipart
|
|
7
|
+
module Internal
|
|
8
|
+
# A class which wraps standard input ($stdin) so that, for example, piped input can participate
|
|
9
|
+
# in multipart uploads. Standard input is not seekable so we need a special class to handle it
|
|
10
|
+
# rather than using {OCI::ObjectStorage::Transfer::Multipart::Internal::SeekableNonFilePartIOWrapper}.
|
|
11
|
+
#
|
|
12
|
+
# This class can also vend data for each part to upload as part of a multipart upload. However, since
|
|
13
|
+
# we may not know the full size of $stdin beforehand, in addition to vending data this wrapper will
|
|
14
|
+
# also vend the part number to assign to the uploaded part.
|
|
15
|
+
#
|
|
16
|
+
# Reading data out of stdin is synchronised so that only a single thread can read at any one time. This
|
|
17
|
+
# prevents issues where we could write the same part multiple times or write inconsistent data.
|
|
18
|
+
class StdinPartIOWrapper
|
|
19
|
+
# A reference to $stdin
|
|
20
|
+
# @return [IO]
|
|
21
|
+
attr_reader :source
|
|
22
|
+
|
|
23
|
+
# Creates a new StdinPartIOWrapper
|
|
24
|
+
#
|
|
25
|
+
# @param [IO] source A reference to $stdin
|
|
26
|
+
def initialize(source:)
|
|
27
|
+
@source = source
|
|
28
|
+
@lock = Mutex.new
|
|
29
|
+
@part_number = 0
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# Reads bytes from $stdin. This is synchronised so that only a single thread can
|
|
33
|
+
# perform this operation at any one time.
|
|
34
|
+
#
|
|
35
|
+
# @param [Integer] part_size The number of bytes to read from $stdin
|
|
36
|
+
#
|
|
37
|
+
# @return [Hash] A hash with two keys. The :content key will contain the content we read from $stdin
|
|
38
|
+
# and the :part_number key will contain the part number we will assign to the part when we upload it as
|
|
39
|
+
# part of a multipart upload. If there is no more data from $stdin then this method will return nil.
|
|
40
|
+
def read(part_size)
|
|
41
|
+
@lock.synchronize do
|
|
42
|
+
read_content = @source.read(part_size)
|
|
43
|
+
if read_content
|
|
44
|
+
@part_number += 1
|
|
45
|
+
{ content: read_content, part_number: @part_number }
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
@@ -0,0 +1,545 @@
|
|
|
1
|
+
# Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
|
|
2
|
+
|
|
3
|
+
require 'base64'
|
|
4
|
+
require 'digest'
|
|
5
|
+
require 'tempfile'
|
|
6
|
+
|
|
7
|
+
require_relative 'internal/file_part_io_wrapper'
|
|
8
|
+
require_relative 'internal/stdin_part_io_wrapper'
|
|
9
|
+
require_relative 'internal/seekable_non_file_part_io_wrapper'
|
|
10
|
+
require_relative 'internal/multipart_upload_parts_collection'
|
|
11
|
+
|
|
12
|
+
module OCI
|
|
13
|
+
module ObjectStorage
|
|
14
|
+
module Transfer
|
|
15
|
+
module Multipart
|
|
16
|
+
# MultipartObjectAssembler provides a simplified interaction when uploading large
|
|
17
|
+
# objects using multi-part uploads.
|
|
18
|
+
#
|
|
19
|
+
# An assembler can be used to begin a new upload, or resume a previous one. A new assembler
|
|
20
|
+
# should be created per new upload or resumed upload to be performed. The same assembler is not
|
|
21
|
+
# resubale across multiple new uploads/resumes.
|
|
22
|
+
class MultipartObjectAssembler
|
|
23
|
+
MD5_CALC_PART_READ_BYTES = 8 * OCI::ObjectStorage::Transfer::MEBIBYTE
|
|
24
|
+
|
|
25
|
+
# Settings for the exponential backoff and retry (with jitter) which the assembler does
|
|
26
|
+
DEFAULT_MAX_ATTEMPTS = 3
|
|
27
|
+
DEFAULT_BASE_SLEEP_MILLIS = 1000
|
|
28
|
+
DEFAULT_MAX_SLEEP_TIME_MILLIS = 8000
|
|
29
|
+
DEFAULT_EXPONENTIAL_GROWTH_FACTOR = 2
|
|
30
|
+
|
|
31
|
+
# The client used to interact with the Object Storage service
|
|
32
|
+
# @return [OCI::ObjectStorage::ObjectStorageClient]
|
|
33
|
+
attr_accessor :object_storage_client
|
|
34
|
+
|
|
35
|
+
# The namespace containing the bucket in which to store the object
|
|
36
|
+
# @return [String]
|
|
37
|
+
attr_accessor :namespace
|
|
38
|
+
|
|
39
|
+
# The bucket where we'll upload the object
|
|
40
|
+
# @return [String]
|
|
41
|
+
attr_accessor :bucket_name
|
|
42
|
+
|
|
43
|
+
# The name of the object in Object Storage
|
|
44
|
+
# @return [String]
|
|
45
|
+
attr_accessor :object_name
|
|
46
|
+
|
|
47
|
+
# The size, in bytes, of each part of a multipart upload. This applies when we are uploading files from disk
|
|
48
|
+
# and defaults to 128 MiB
|
|
49
|
+
# @return [Integer]
|
|
50
|
+
attr_accessor :multipart_part_size
|
|
51
|
+
|
|
52
|
+
# The size, in bytes, of each part of a multipart upload when we are reading from stdin or a non-file IO-like
|
|
53
|
+
# source (e.g. a StringIO). Defaults to 10 MiB
|
|
54
|
+
# @return [Integer]
|
|
55
|
+
attr_accessor :non_file_io_multipart_part_size
|
|
56
|
+
|
|
57
|
+
# How many parts we can upload in parallel. Defaults to 3. If this is set to 1, this is the
|
|
58
|
+
# equivalent of not allowing parts to be uploaded in parallel.
|
|
59
|
+
# @return [Integer]
|
|
60
|
+
attr_accessor :parallel_process_count
|
|
61
|
+
|
|
62
|
+
# A bag of optional parameter (e.g. the client request ID, metadata) which we can use when making calls to the Object Storage service.
|
|
63
|
+
# @return [Hash]
|
|
64
|
+
attr_accessor :multipart_upload_opts
|
|
65
|
+
|
|
66
|
+
# If we encounter a failure when performing an operation and need to retry, the maximum number of attempts we
|
|
67
|
+
# can make before declaring failure. Attempts are 1-based, i.e. the first call we make is considered attempt 1.
|
|
68
|
+
# @return [Integer]
|
|
69
|
+
attr_accessor :max_attempts
|
|
70
|
+
|
|
71
|
+
# For exponential backoff and retry, the base time to use in our retry calculation in milliseconds. Defaults to 1000ms
|
|
72
|
+
# @return [Integer]
|
|
73
|
+
attr_accessor :base_sleep_millis
|
|
74
|
+
|
|
75
|
+
# The maximum amount of time to wait between retries. Defaults to 8000ms
|
|
76
|
+
# @return [Integer]
|
|
77
|
+
attr_accessor :max_sleep_time_millis
|
|
78
|
+
|
|
79
|
+
# For exponential backoff and retry, the exponent which we will raise to the power of the number of attempts. Defaults to 2
|
|
80
|
+
# @return [Integer]
|
|
81
|
+
attr_accessor :exponential_growth_factor
|
|
82
|
+
|
|
83
|
+
# Contains the upload ID for the multipart upload and other upload-specific information (e.g. the destination namespace, bucket and object,
|
|
84
|
+
# and the parts which have been uploaded)
|
|
85
|
+
# @return [Hash]
|
|
86
|
+
attr_reader :manifest
|
|
87
|
+
|
|
88
|
+
# @param [OCI::ObjectStorage::ObjectStorageClient] object_storage_client The client used to interact with the Object Storage service
|
|
89
|
+
# @param [String] namespace The namespace containing the bucket in which to store the object
|
|
90
|
+
# @param [String] bucket_name The bucket where we'll upload the object
|
|
91
|
+
# @param [String] object_name The name of the object in Object Storage
|
|
92
|
+
# @param [Integer] multipart_part_size The size, in bytes, of each part of a multipart upload. This applies when we are uploading files from disk and defaults to 128 MiB
|
|
93
|
+
# @param [Integer] non_file_io_multipart_part_size The size, in bytes, of each part of a multipart upload when we are reading from stdin or a non-file IO-like source (e.g. a StringIO). Defaults to 10 MiB
|
|
94
|
+
# @param [Integer] parallel_process_count How many parts we can upload in parallel. Defaults to 3
|
|
95
|
+
# @param [Hash] multipart_upload_opts
|
|
96
|
+
# @option multipart_upload_opts [String] :if_match The entity tag to match. Only used for new multipart uploads and ignored otherwise.
|
|
97
|
+
# @option multipart_upload_opts [String] :if_none_match The entity tag to avoid matching. The only valid value is *, which indicates that the request should fail if the object already exists. Only used for new multipart uploads and ignored otherwise.
|
|
98
|
+
# @option multipart_upload_opts [String] :opc_client_request_id The client request ID for tracing. Will be applied to all requests made by this assembler.
|
|
99
|
+
# @option multipart_upload_opts [String] :content_type The content type of the object. Defaults to 'application/octet-stream' if not overridden. Only used for new multipart uploads and ignored otherwise.
|
|
100
|
+
# @option multipart_upload_opts [String] :content_language The content language of the object. Only used for new multipart uploads and ignored otherwise.
|
|
101
|
+
# @option multipart_upload_opts [String] :content_encoding The content encoding of the object. Only used for new multipart uploads and ignored otherwise.
|
|
102
|
+
# @option multipart_upload_opts [Hash<String, String>] :metadata A hash of string keys to string values representing any custom metadata to be applied to the object. Only used for new multipart uploads and ignored otherwise.
|
|
103
|
+
def initialize(object_storage_client:,
|
|
104
|
+
namespace:,
|
|
105
|
+
bucket_name:,
|
|
106
|
+
object_name:,
|
|
107
|
+
multipart_part_size: OCI::ObjectStorage::Transfer::MULTIPART_PART_SIZE,
|
|
108
|
+
non_file_io_multipart_part_size: OCI::ObjectStorage::Transfer::NON_FILE_IO_MULTIPART_PART_SIZE,
|
|
109
|
+
parallel_process_count: OCI::ObjectStorage::Transfer::Multipart::DEFAULT_PARALLEL_PROCESS_COUNT,
|
|
110
|
+
multipart_upload_opts: {})
|
|
111
|
+
@object_storage_client = object_storage_client
|
|
112
|
+
@namespace = namespace
|
|
113
|
+
@bucket_name = bucket_name
|
|
114
|
+
@object_name = object_name
|
|
115
|
+
@multipart_part_size = multipart_part_size
|
|
116
|
+
@non_file_io_multipart_part_size = non_file_io_multipart_part_size
|
|
117
|
+
@parallel_process_count = parallel_process_count
|
|
118
|
+
@object_io = nil
|
|
119
|
+
|
|
120
|
+
@manifest = {
|
|
121
|
+
upload_id: nil,
|
|
122
|
+
namespace: @namespace,
|
|
123
|
+
bucket_name: @bucket_name,
|
|
124
|
+
object_name: @object_name,
|
|
125
|
+
object_io_or_file_path: nil,
|
|
126
|
+
parts: OCI::ObjectStorage::Transfer::Multipart::Internal::MultipartUploadPartsCollection.new([])
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
@multipart_upload_opts = multipart_upload_opts
|
|
130
|
+
@multipart_upload_opts.delete('content_md5') if @multipart_upload_opts.key?('content_md5')
|
|
131
|
+
|
|
132
|
+
@max_attempts = DEFAULT_MAX_ATTEMPTS
|
|
133
|
+
@base_sleep_millis = DEFAULT_BASE_SLEEP_MILLIS
|
|
134
|
+
@exponential_growth_factor = DEFAULT_EXPONENTIAL_GROWTH_FACTOR
|
|
135
|
+
@max_sleep_time_millis = DEFAULT_MAX_SLEEP_TIME_MILLIS
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
# Initializes a new multipart upload.
|
|
139
|
+
#
|
|
140
|
+
# @return [Response] A Response object with data of type OCI::ObjectStorage::Models::MultipartUpload
|
|
141
|
+
def new_upload
|
|
142
|
+
raise 'This MultipartObjectAssembler has already been initialized with an upload' if @manifest[:upload_id]
|
|
143
|
+
|
|
144
|
+
create_multipart_upload_opts = {}
|
|
145
|
+
create_multipart_upload_opts[:if_match] = @multipart_upload_opts[:if_match] if @multipart_upload_opts[:if_match]
|
|
146
|
+
create_multipart_upload_opts[:if_none_match] = @multipart_upload_opts[:if_none_match] \
|
|
147
|
+
if @multipart_upload_opts[:if_none_match]
|
|
148
|
+
|
|
149
|
+
create_multipart_upload_opts[:opc_client_request_id] = @multipart_upload_opts[:opc_client_request_id] \
|
|
150
|
+
if @multipart_upload_opts[:opc_client_request_id]
|
|
151
|
+
|
|
152
|
+
create_multipart_upload_details =
|
|
153
|
+
OCI::ObjectStorage::Models::CreateMultipartUploadDetails.new(object: @object_name)
|
|
154
|
+
|
|
155
|
+
create_multipart_upload_details.content_type = @multipart_upload_opts[:content_type] \
|
|
156
|
+
if @multipart_upload_opts[:content_type]
|
|
157
|
+
|
|
158
|
+
create_multipart_upload_details.content_language = @multipart_upload_opts[:content_language] \
|
|
159
|
+
if @multipart_upload_opts[:content_language]
|
|
160
|
+
|
|
161
|
+
create_multipart_upload_details.content_encoding = @multipart_upload_opts[:content_encoding] \
|
|
162
|
+
if @multipart_upload_opts[:content_encoding]
|
|
163
|
+
|
|
164
|
+
if @multipart_upload_opts[:metadata] && !@multipart_upload_opts[:metadata].empty?
|
|
165
|
+
processed_metadata = {}
|
|
166
|
+
@multipart_upload_opts[:metadata].each do |key, value|
|
|
167
|
+
processed_metadata[key] = value if key.to_s.start_with?('opc-meta-')
|
|
168
|
+
processed_metadata["opc-meta-#{key}"] = value unless key.to_s.start_with?('opc-meta-')
|
|
169
|
+
end
|
|
170
|
+
create_multipart_upload_details.metadata = processed_metadata
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
response = make_retrying_call do
|
|
174
|
+
@object_storage_client.create_multipart_upload(
|
|
175
|
+
@namespace, @bucket_name,
|
|
176
|
+
create_multipart_upload_details,
|
|
177
|
+
create_multipart_upload_opts
|
|
178
|
+
)
|
|
179
|
+
end
|
|
180
|
+
@manifest[:upload_id] = response.data.upload_id
|
|
181
|
+
|
|
182
|
+
response
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
# Initializes the parts in the manifest based on the provided input. If the input is stdin then the list of parts in the manifest
|
|
186
|
+
# will be left empty (the parts in this case are figured out dynamically at upload time since we may not have all the information in advance).
|
|
187
|
+
# If the input is the path to a file (a String), a file or a IO-like object (e.g. a StringIO) then the parts in the manifest will
|
|
188
|
+
# be initialized so that the assembler can go through and upload them.
|
|
189
|
+
#
|
|
190
|
+
# No uploads will be performed until the upload method is called.
|
|
191
|
+
#
|
|
192
|
+
# @param [String, IO] object_io_or_file_path Either a path to the file to upload, an IO-like object containing the data to upload or $stdin.
|
|
193
|
+
def io_for_transfer=(object_io_or_file_path)
|
|
194
|
+
@manifest[:object_io_or_file_path] = object_io_or_file_path
|
|
195
|
+
|
|
196
|
+
return if stdin?(object_io_or_file_path)
|
|
197
|
+
|
|
198
|
+
opened_file = false
|
|
199
|
+
if object_io_or_file_path.is_a?(String)
|
|
200
|
+
object_io = File.open(object_io_or_file_path, 'rb')
|
|
201
|
+
opened_file = true
|
|
202
|
+
end
|
|
203
|
+
object_io = object_io_or_file_path if object_io_or_file_path.respond_to?(:seek)
|
|
204
|
+
|
|
205
|
+
total_size = object_io.size if object_io.respond_to?(:size)
|
|
206
|
+
total_size = object_io.stat.size unless object_io.respond_to?(:size)
|
|
207
|
+
|
|
208
|
+
part_size_to_use = @multipart_part_size if file_based_io?(object_io_or_file_path)
|
|
209
|
+
part_size_to_use = @non_file_io_multipart_part_size unless file_based_io?(object_io_or_file_path)
|
|
210
|
+
|
|
211
|
+
offset = 0
|
|
212
|
+
part_number = 1
|
|
213
|
+
while offset < total_size
|
|
214
|
+
part_info = {
|
|
215
|
+
offset: offset,
|
|
216
|
+
part_size: calculate_part_size(total_size, offset, part_size_to_use),
|
|
217
|
+
part_number: part_number,
|
|
218
|
+
part_md5_hash: nil
|
|
219
|
+
}
|
|
220
|
+
@manifest[:parts].push(part_info)
|
|
221
|
+
offset += part_size_to_use
|
|
222
|
+
part_number += 1
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
nil
|
|
226
|
+
ensure
|
|
227
|
+
object_io.close if opened_file && object_io
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
# Performs a multipart upload
|
|
231
|
+
#
|
|
232
|
+
# @return [Array] If the multipart upload was successful, an empty array. If there were errors then the array will contain
|
|
233
|
+
# one element per error which occurred.
|
|
234
|
+
def upload
|
|
235
|
+
all_parts = @manifest[:parts].to_a
|
|
236
|
+
pending_parts = OCI::ObjectStorage::Transfer::Multipart::Internal::MultipartUploadPartsCollection.new(
|
|
237
|
+
all_parts.select { |ap| ap[:opc_md5].nil? && ap[:etag].nil? }
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
unless file_based_io?(@manifest[:object_io_or_file_path]) || stdin?(@manifest[:object_io_or_file_path])
|
|
241
|
+
seekable_io_wrapper = OCI::ObjectStorage::Transfer::Multipart::Internal::SeekableNonFilePartIOWrapper.new(
|
|
242
|
+
source: @manifest[:object_io_or_file_path]
|
|
243
|
+
)
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
if stdin?(@manifest[:object_io_or_file_path])
|
|
247
|
+
stdin_io_wrapper = OCI::ObjectStorage::Transfer::Multipart::Internal::StdinPartIOWrapper.new(
|
|
248
|
+
source: @manifest[:object_io_or_file_path]
|
|
249
|
+
)
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
threads = []
|
|
253
|
+
@parallel_process_count.times do
|
|
254
|
+
thread = Thread.new do
|
|
255
|
+
begin
|
|
256
|
+
upload_non_stdin(pending_parts, seekable_io_wrapper) unless stdin?(@manifest[:object_io_or_file_path])
|
|
257
|
+
upload_stdin(stdin_io_wrapper) if stdin?(@manifest[:object_io_or_file_path])
|
|
258
|
+
|
|
259
|
+
nil
|
|
260
|
+
rescue => error
|
|
261
|
+
pending_parts.clear! # Stop any futher processing on error
|
|
262
|
+
error
|
|
263
|
+
end
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
thread.abort_on_exception = true # Do not continue if we encounter exceptions
|
|
267
|
+
threads << thread
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
threads.map(&:value).compact
|
|
271
|
+
end
|
|
272
|
+
|
|
273
|
+
# Resume uploading a multipart object to Object Storage. This assumes that the assembler already has knowledge
|
|
274
|
+
# of the parts which it could potentially need to upload (i.e. they have been prepared via the set_io_for_transfer
|
|
275
|
+
# method).
|
|
276
|
+
#
|
|
277
|
+
# Prior to resuming the upload, we'll attempt to reconcile with Object Storage any previously uploaded parts so that
|
|
278
|
+
# they are not uploaded again
|
|
279
|
+
#
|
|
280
|
+
# @param [String] upload_id The ID of the multipart upload to resume.
|
|
281
|
+
#
|
|
282
|
+
# @return [Array] If the multipart upload was successful, an empty array. If there were errors then the array will contain
|
|
283
|
+
# one element per error which occurred.
|
|
284
|
+
def resume(upload_id)
|
|
285
|
+
raise 'An upload ID must be provided' if upload_id.nil?
|
|
286
|
+
raise 'Parts must be initialized prior to resuming' if @manifest[:parts].length.zero?
|
|
287
|
+
|
|
288
|
+
manifest[:upload_id] = upload_id
|
|
289
|
+
upload_parts = list_uploaded_parts(upload_id)
|
|
290
|
+
known_parts = @manifest[:parts].to_a
|
|
291
|
+
|
|
292
|
+
raise 'There are more parts on the server than parts to resume, please check the upload ID.' \
|
|
293
|
+
if upload_parts.length > known_parts.length
|
|
294
|
+
|
|
295
|
+
upload_parts.each do |up|
|
|
296
|
+
index = up.part_number - 1
|
|
297
|
+
manifest_part = known_parts[index]
|
|
298
|
+
|
|
299
|
+
if manifest_part[:part_size] != up.size
|
|
300
|
+
raise 'Cannot resume upload with different part size. ' \
|
|
301
|
+
+ "Parts were uploaded with a part size of #{up.size / OCI::ObjectStorage::Transfer::MEBIBYTE} MiB"
|
|
302
|
+
end
|
|
303
|
+
|
|
304
|
+
manifest_part[:etag] = up.etag
|
|
305
|
+
manifest_part[:opc_md5] = up.md5
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
upload
|
|
309
|
+
end
|
|
310
|
+
|
|
311
|
+
# Commits the multipart upload.
|
|
312
|
+
#
|
|
313
|
+
# @return [Response] A Response object with data of type nil. For a multipart upload, the headers of the response will contain
|
|
314
|
+
# an opc-multipart-md5 key.
|
|
315
|
+
def commit
|
|
316
|
+
raise 'Cannot commit as this MultipartObjectAssembler has not been initialized with an upload' \
|
|
317
|
+
unless @manifest[:upload_id]
|
|
318
|
+
|
|
319
|
+
parts = @manifest[:parts].to_a
|
|
320
|
+
|
|
321
|
+
commit_upload_details = OCI::ObjectStorage::Models::CommitMultipartUploadDetails.new
|
|
322
|
+
commit_upload_details.parts_to_commit = []
|
|
323
|
+
commit_upload_details.parts_to_exclude = []
|
|
324
|
+
|
|
325
|
+
parts.each do |part|
|
|
326
|
+
if part[:etag]
|
|
327
|
+
commit_upload_details.parts_to_commit << OCI::ObjectStorage::Models::CommitMultipartUploadPartDetails.new(
|
|
328
|
+
partNum: part[:part_number],
|
|
329
|
+
etag: part[:etag]
|
|
330
|
+
)
|
|
331
|
+
end
|
|
332
|
+
|
|
333
|
+
commit_upload_details.parts_to_exclude << part[:part_number] unless part[:etag]
|
|
334
|
+
end
|
|
335
|
+
|
|
336
|
+
response = make_retrying_call do
|
|
337
|
+
@object_storage_client.commit_multipart_upload(
|
|
338
|
+
@namespace,
|
|
339
|
+
@bucket_name,
|
|
340
|
+
@object_name,
|
|
341
|
+
@manifest[:upload_id],
|
|
342
|
+
commit_upload_details
|
|
343
|
+
)
|
|
344
|
+
end
|
|
345
|
+
|
|
346
|
+
response
|
|
347
|
+
end
|
|
348
|
+
|
|
349
|
+
# Aborts a multipart upload.
|
|
350
|
+
#
|
|
351
|
+
# @param [String] upload_id The ID of the multipart upload to abort.
|
|
352
|
+
# @return [Response] A Response object with data of type nil
|
|
353
|
+
def abort(upload_id)
|
|
354
|
+
abort_opts = {}
|
|
355
|
+
abort_opts[:opc_client_request_id] = @multipart_upload_opts[:opc_client_request_id] \
|
|
356
|
+
if @multipart_upload_opts[:opc_client_request_id]
|
|
357
|
+
|
|
358
|
+
response = make_retrying_call do
|
|
359
|
+
@object_storage_client.abort_multipart_upload(
|
|
360
|
+
@namespace,
|
|
361
|
+
@bucket_name,
|
|
362
|
+
@object_name,
|
|
363
|
+
upload_id,
|
|
364
|
+
abort_opts
|
|
365
|
+
)
|
|
366
|
+
end
|
|
367
|
+
response
|
|
368
|
+
end
|
|
369
|
+
|
|
370
|
+
private
|
|
371
|
+
|
|
372
|
+
def calculate_part_size(total_size, offset, part_size)
|
|
373
|
+
if offset + part_size > total_size
|
|
374
|
+
total_size - offset
|
|
375
|
+
else
|
|
376
|
+
part_size
|
|
377
|
+
end
|
|
378
|
+
end
|
|
379
|
+
|
|
380
|
+
def calculate_md5(file_part: nil, raw_content: nil)
|
|
381
|
+
raise 'Cannot specify both a file_part and raw_content' if file_part && raw_content
|
|
382
|
+
md5 = Digest::MD5.new
|
|
383
|
+
|
|
384
|
+
if file_part
|
|
385
|
+
file_part.rewind
|
|
386
|
+
loop do
|
|
387
|
+
part_read = file_part.read(MD5_CALC_PART_READ_BYTES)
|
|
388
|
+
break unless part_read
|
|
389
|
+
md5.update(part_read)
|
|
390
|
+
end
|
|
391
|
+
file_part.rewind
|
|
392
|
+
end
|
|
393
|
+
|
|
394
|
+
md5.update(raw_content) if raw_content
|
|
395
|
+
|
|
396
|
+
md5.base64digest
|
|
397
|
+
end
|
|
398
|
+
|
|
399
|
+
def list_uploaded_parts(upload_id)
|
|
400
|
+
list_multipart_upload_parts_opts = {}
|
|
401
|
+
if @multipart_upload_opts[:opc_client_request_id]
|
|
402
|
+
list_multipart_upload_parts_opts[:opc_client_request_id] = @multipart_upload_opts[:opc_client_request_id]
|
|
403
|
+
end
|
|
404
|
+
|
|
405
|
+
upload_parts = []
|
|
406
|
+
@object_storage_client.list_multipart_upload_parts(
|
|
407
|
+
@namespace,
|
|
408
|
+
@bucket_name,
|
|
409
|
+
@object_name,
|
|
410
|
+
upload_id,
|
|
411
|
+
list_multipart_upload_parts_opts
|
|
412
|
+
).each do |response|
|
|
413
|
+
upload_parts += response.data
|
|
414
|
+
end
|
|
415
|
+
|
|
416
|
+
upload_parts
|
|
417
|
+
end
|
|
418
|
+
|
|
419
|
+
def file_based_io?(object_io_or_file_path)
|
|
420
|
+
object_io_or_file_path.is_a?(String) ||
|
|
421
|
+
object_io_or_file_path.is_a?(File) ||
|
|
422
|
+
object_io_or_file_path.is_a?(Tempfile)
|
|
423
|
+
end
|
|
424
|
+
|
|
425
|
+
def stdin?(object_io_or_file_path)
|
|
426
|
+
object_io_or_file_path.eql?($stdin)
|
|
427
|
+
end
|
|
428
|
+
|
|
429
|
+
def upload_non_stdin(pending_parts, seekable_io_wrapper = nil)
|
|
430
|
+
while (part = pending_parts.shift) # While we have parts left to process
|
|
431
|
+
if file_based_io?(@manifest[:object_io_or_file_path])
|
|
432
|
+
part_content = OCI::ObjectStorage::Transfer::Multipart::Internal::FilePartIOWrapper.new(
|
|
433
|
+
source: @manifest[:object_io_or_file_path],
|
|
434
|
+
offset: part[:offset],
|
|
435
|
+
part_size: part[:part_size]
|
|
436
|
+
)
|
|
437
|
+
part[:part_md5_hash] = calculate_md5(file_part: part_content)
|
|
438
|
+
else
|
|
439
|
+
part_content = seekable_io_wrapper.read(part[:offset], part[:part_size])
|
|
440
|
+
part[:part_md5_hash] = calculate_md5(raw_content: part_content)
|
|
441
|
+
end
|
|
442
|
+
|
|
443
|
+
upload_part_opts = { content_md5: part[:md5_hash] }
|
|
444
|
+
if @multipart_upload_opts[:opc_client_request_id]
|
|
445
|
+
upload_part_opts[:opc_client_request_id] = @multipart_upload_opts[:opc_client_request_id]
|
|
446
|
+
end
|
|
447
|
+
|
|
448
|
+
response = make_retrying_call do
|
|
449
|
+
@object_storage_client.upload_part(
|
|
450
|
+
@namespace,
|
|
451
|
+
@bucket_name,
|
|
452
|
+
@object_name,
|
|
453
|
+
@manifest[:upload_id],
|
|
454
|
+
part[:part_number],
|
|
455
|
+
part_content,
|
|
456
|
+
upload_part_opts
|
|
457
|
+
)
|
|
458
|
+
end
|
|
459
|
+
part_content.close if file_based_io?(@manifest[:object_io_or_file_path])
|
|
460
|
+
|
|
461
|
+
part[:etag] = response.headers['etag']
|
|
462
|
+
part[:opc_md5] = response.headers['opc-content-md5']
|
|
463
|
+
end
|
|
464
|
+
end
|
|
465
|
+
|
|
466
|
+
def upload_stdin(stdin_io_wrapper)
|
|
467
|
+
# While we can still pull content from stdin
|
|
468
|
+
while (stdin_part = stdin_io_wrapper.read(@non_file_io_multipart_part_size))
|
|
469
|
+
part = {
|
|
470
|
+
part_number: stdin_part[:part_number],
|
|
471
|
+
part_size: stdin_part[:content].size,
|
|
472
|
+
part_md5_hash: calculate_md5(raw_content: stdin_part[:content])
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
upload_part_opts = { content_md5: part[:md5_hash] }
|
|
476
|
+
if @multipart_upload_opts[:opc_client_request_id]
|
|
477
|
+
upload_part_opts[:opc_client_request_id] = @multipart_upload_opts[:opc_client_request_id]
|
|
478
|
+
end
|
|
479
|
+
|
|
480
|
+
response = make_retrying_call do
|
|
481
|
+
@object_storage_client.upload_part(
|
|
482
|
+
@namespace,
|
|
483
|
+
@bucket_name,
|
|
484
|
+
@object_name,
|
|
485
|
+
@manifest[:upload_id],
|
|
486
|
+
part[:part_number],
|
|
487
|
+
stdin_part[:content],
|
|
488
|
+
upload_part_opts
|
|
489
|
+
)
|
|
490
|
+
end
|
|
491
|
+
|
|
492
|
+
part[:etag] = response.headers['etag']
|
|
493
|
+
part[:opc_md5] = response.headers['opc-content-md5']
|
|
494
|
+
|
|
495
|
+
@manifest[:parts].push(part)
|
|
496
|
+
end
|
|
497
|
+
end
|
|
498
|
+
|
|
499
|
+
# Make a retrying call using a provided block (that we yield to) and return the result of the block
|
|
500
|
+
# on success.
|
|
501
|
+
def make_retrying_call
|
|
502
|
+
@max_attempts.times do |attempt|
|
|
503
|
+
try = attempt + 1
|
|
504
|
+
begin
|
|
505
|
+
return yield
|
|
506
|
+
rescue OCI::Errors::NetworkError # Network errors should be retriable
|
|
507
|
+
raise if try >= @max_attempts # Short-circuit if we're already on our last attempt
|
|
508
|
+
|
|
509
|
+
# Use full jitter on networking errors (as hopefully they are short-lived/intermittent)
|
|
510
|
+
sleep(get_full_jitter_sleep_time_millis(attempt) / 1000.0)
|
|
511
|
+
rescue OCI::Errors::ServiceError => svc_err
|
|
512
|
+
raise if try >= @max_attempts # Short-circuit if we're already on our last attempt
|
|
513
|
+
|
|
514
|
+
# For internal server errors, use full jitter as it's hopefully intermittent
|
|
515
|
+
sleep(get_full_jitter_sleep_time_millis(attempt) / 1000.0) if svc_err.status >= 500 || svc_err.status == -1
|
|
516
|
+
|
|
517
|
+
# For throttles and consistency errors, use equal jitter as this guarantees some sleep time
|
|
518
|
+
# between attempts (full jitter doesn't because we get a value across the range 0 to something, i.e.
|
|
519
|
+
# it is possible to get a very small sleep time)
|
|
520
|
+
if svc_err.status == 429 || (svc_err.status == 409 && svc_err.code == 'ConcurrentObjectUpdate')
|
|
521
|
+
sleep(get_equal_jitter_sleep_time_millis(attempt) / 1000.0)
|
|
522
|
+
end
|
|
523
|
+
|
|
524
|
+
# A non-retriable service error, so just throw it
|
|
525
|
+
raise
|
|
526
|
+
end
|
|
527
|
+
end
|
|
528
|
+
end
|
|
529
|
+
|
|
530
|
+
def get_full_jitter_sleep_time_millis(attempt)
|
|
531
|
+
random = Random.new
|
|
532
|
+
random.rand(0..[@base_sleep_millis * (@exponent_growth_factor**attempt), @max_sleep_time_millis].min)
|
|
533
|
+
end
|
|
534
|
+
|
|
535
|
+
def get_equal_jitter_sleep_time_millis(attempt)
|
|
536
|
+
random = Random.new
|
|
537
|
+
sleep_raw = [@base_sleep_millis * (@exponent_growth_factor**attempt), @max_sleep_time_millis].min
|
|
538
|
+
|
|
539
|
+
(sleep_raw / 2) + random.rand(0..(sleep_raw / 2))
|
|
540
|
+
end
|
|
541
|
+
end
|
|
542
|
+
end
|
|
543
|
+
end
|
|
544
|
+
end
|
|
545
|
+
end
|