oci 2.0.6 → 2.0.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +23 -1
  3. data/lib/oci.rb +4 -7
  4. data/lib/oci/api_client.rb +51 -6
  5. data/lib/oci/auth/auth.rb +5 -2
  6. data/lib/oci/auth/federation_client.rb +2 -2
  7. data/lib/oci/auth/internal/auth_token_request_signer.rb +2 -2
  8. data/lib/oci/auth/security_token_container.rb +2 -2
  9. data/lib/oci/auth/session_key_supplier.rb +3 -4
  10. data/lib/oci/auth/signers/instance_principals_security_token_signer.rb +30 -17
  11. data/lib/oci/auth/signers/security_token_signer.rb +13 -6
  12. data/lib/oci/auth/signers/x509_federation_client_based_security_token_signer.rb +8 -4
  13. data/lib/oci/auth/url_based_certificate_retriever.rb +3 -4
  14. data/lib/oci/auth/util.rb +6 -4
  15. data/lib/oci/base_signer.rb +54 -50
  16. data/lib/oci/config_file_loader.rb +3 -11
  17. data/lib/oci/core/blockstorage_client.rb +211 -0
  18. data/lib/oci/core/compute_client.rb +4 -2
  19. data/lib/oci/core/core.rb +9 -0
  20. data/lib/oci/core/models/attach_i_scsi_volume_details.rb +5 -1
  21. data/lib/oci/core/models/attach_volume_details.rb +13 -1
  22. data/lib/oci/core/models/create_public_ip_details.rb +186 -0
  23. data/lib/oci/core/models/create_vnic_details.rb +8 -2
  24. data/lib/oci/core/models/create_volume_backup_details.rb +27 -1
  25. data/lib/oci/core/models/create_volume_backup_policy_assignment_details.rb +133 -0
  26. data/lib/oci/core/models/create_volume_details.rb +15 -1
  27. data/lib/oci/core/models/get_public_ip_by_ip_address_details.rb +125 -0
  28. data/lib/oci/core/models/get_public_ip_by_private_ip_id_details.rb +124 -0
  29. data/lib/oci/core/models/i_scsi_volume_attachment.rb +5 -1
  30. data/lib/oci/core/models/public_ip.rb +328 -0
  31. data/lib/oci/core/models/update_public_ip_details.rb +140 -0
  32. data/lib/oci/core/models/volume_attachment.rb +13 -1
  33. data/lib/oci/core/models/volume_backup.rb +72 -1
  34. data/lib/oci/core/models/volume_backup_policy.rb +161 -0
  35. data/lib/oci/core/models/volume_backup_policy_assignment.rb +159 -0
  36. data/lib/oci/core/models/volume_backup_schedule.rb +191 -0
  37. data/lib/oci/core/virtual_network_client.rb +374 -4
  38. data/lib/oci/database/database_client.rb +3 -1
  39. data/lib/oci/dns/dns.rb +34 -0
  40. data/lib/oci/dns/dns_client.rb +985 -0
  41. data/lib/oci/dns/models/create_zone_details.rb +174 -0
  42. data/lib/oci/dns/models/external_master.rb +145 -0
  43. data/lib/oci/dns/models/patch_domain_records_details.rb +120 -0
  44. data/lib/oci/dns/models/patch_rr_set_details.rb +120 -0
  45. data/lib/oci/dns/models/patch_zone_records_details.rb +120 -0
  46. data/lib/oci/dns/models/record.rb +204 -0
  47. data/lib/oci/dns/models/record_collection.rb +121 -0
  48. data/lib/oci/dns/models/record_details.rb +204 -0
  49. data/lib/oci/dns/models/record_operation.rb +253 -0
  50. data/lib/oci/dns/models/rr_set.rb +123 -0
  51. data/lib/oci/dns/models/sort_order.rb +12 -0
  52. data/lib/oci/dns/models/tsig.rb +149 -0
  53. data/lib/oci/dns/models/update_domain_records_details.rb +120 -0
  54. data/lib/oci/dns/models/update_rr_set_details.rb +120 -0
  55. data/lib/oci/dns/models/update_zone_details.rb +122 -0
  56. data/lib/oci/dns/models/update_zone_records_details.rb +120 -0
  57. data/lib/oci/dns/models/zone.rb +272 -0
  58. data/lib/oci/dns/models/zone_summary.rb +230 -0
  59. data/lib/oci/dns/util.rb +2 -0
  60. data/lib/oci/errors.rb +20 -0
  61. data/lib/oci/identity/identity_client.rb +1 -0
  62. data/lib/oci/identity/models/create_dynamic_group_details.rb +4 -2
  63. data/lib/oci/identity/models/dynamic_group.rb +9 -6
  64. data/lib/oci/identity/models/update_dynamic_group_details.rb +4 -2
  65. data/lib/oci/load_balancer/load_balancer.rb +6 -0
  66. data/lib/oci/load_balancer/load_balancer_client.rb +219 -0
  67. data/lib/oci/load_balancer/models/create_listener_details.rb +17 -1
  68. data/lib/oci/load_balancer/models/create_load_balancer_details.rb +12 -1
  69. data/lib/oci/load_balancer/models/create_path_route_set_details.rb +138 -0
  70. data/lib/oci/load_balancer/models/listener.rb +17 -1
  71. data/lib/oci/load_balancer/models/listener_details.rb +17 -1
  72. data/lib/oci/load_balancer/models/load_balancer.rb +12 -1
  73. data/lib/oci/load_balancer/models/path_match_type.rb +153 -0
  74. data/lib/oci/load_balancer/models/path_route.rb +161 -0
  75. data/lib/oci/load_balancer/models/path_route_set.rb +139 -0
  76. data/lib/oci/load_balancer/models/path_route_set_details.rb +122 -0
  77. data/lib/oci/load_balancer/models/update_listener_details.rb +17 -1
  78. data/lib/oci/load_balancer/models/update_path_route_set_details.rb +122 -0
  79. data/lib/oci/load_balancer/util.rb +1 -3
  80. data/lib/oci/object_storage/transfer/multipart/internal/file_part_io_wrapper.rb +111 -0
  81. data/lib/oci/object_storage/transfer/multipart/internal/multipart_upload_parts_collection.rb +41 -0
  82. data/lib/oci/object_storage/transfer/multipart/internal/seekable_non_file_part_io_wrapper.rb +48 -0
  83. data/lib/oci/object_storage/transfer/multipart/internal/stdin_part_io_wrapper.rb +54 -0
  84. data/lib/oci/object_storage/transfer/multipart/multipart_object_assembler.rb +545 -0
  85. data/lib/oci/object_storage/transfer/transfer.rb +31 -0
  86. data/lib/oci/object_storage/transfer/upload_manager.rb +212 -0
  87. data/lib/oci/object_storage/transfer/upload_manager_config.rb +60 -0
  88. data/lib/oci/regions.rb +3 -1
  89. data/lib/oci/response.rb +1 -3
  90. data/lib/oci/version.rb +1 -1
  91. data/lib/oci/waiter.rb +16 -11
  92. data/lib/oraclebmc.rb +1 -1
  93. metadata +46 -2
@@ -0,0 +1,122 @@
1
+ # Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
2
+
3
+ require 'date'
4
+
5
+ module OCI
6
+ # An updated set of path route rules that overwrites the existing set of rules.
7
+ class LoadBalancer::Models::UpdatePathRouteSetDetails
8
+ # **[Required]** The set of path route rules.
9
+ # @return [Array<OCI::LoadBalancer::Models::PathRoute>]
10
+ attr_accessor :path_routes
11
+
12
+
13
+ # Initializes the object
14
+ # @param [Hash] attributes Model attributes in the form of hash
15
+ # @option attributes [Array<OCI::LoadBalancer::Models::PathRoute>] :pathRoutes The value to assign to the {#path_routes} property
16
+ def initialize(attributes = {})
17
+ return unless attributes.is_a?(Hash)
18
+
19
+ # convert string to symbol for hash key
20
+ attributes = attributes.each_with_object({}){|(k,v), h| h[k.to_sym] = v}
21
+
22
+ if attributes[:'pathRoutes']
23
+ self.path_routes = attributes[:'pathRoutes']
24
+ end
25
+
26
+ end
27
+
28
+ # Checks equality by comparing each attribute.
29
+ # @param [Object] other_object to be compared
30
+ def ==(other_object)
31
+ return true if self.equal?(other_object)
32
+ self.class == other_object.class &&
33
+ path_routes == other_object.path_routes
34
+ end
35
+
36
+ # @see the `==` method
37
+ # @param [Object] other_object to be compared
38
+ def eql?(other_object)
39
+ self == other_object
40
+ end
41
+
42
+ # Calculates hash code according to all attributes.
43
+ # @return [Fixnum] Hash code
44
+ def hash
45
+ [path_routes].hash
46
+ end
47
+
48
+ # Builds the object from hash
49
+ # @param [Hash] attributes Model attributes in the form of hash
50
+ # @return [Object] Returns the model itself
51
+ def build_from_hash(attributes)
52
+ return nil unless attributes.is_a?(Hash)
53
+ self.class.swagger_types.each_pair do |key, type|
54
+ if type =~ /^Array<(.*)>/i
55
+ # check to ensure the input is an array given that the the attribute
56
+ # is documented as an array but the input is not
57
+ if attributes[self.class.attribute_map[key]].is_a?(Array)
58
+ self.public_method("#{key}=").call(attributes[self.class.attribute_map[key]].map{ |v| OCI::Internal::Util.convert_to_type($1, v) } )
59
+ end
60
+ elsif !attributes[self.class.attribute_map[key]].nil?
61
+ self.public_method("#{key}=").call(OCI::Internal::Util.convert_to_type(type, attributes[self.class.attribute_map[key]]))
62
+ end # or else data not found in attributes(hash), not an issue as the data can be optional
63
+ end
64
+
65
+ self
66
+ end
67
+
68
+ # Returns the string representation of the object
69
+ # @return [String] String presentation of the object
70
+ def to_s
71
+ to_hash.to_s
72
+ end
73
+
74
+ # Returns the object in the form of hash
75
+ # @return [Hash] Returns the object in the form of hash
76
+ def to_hash
77
+ hash = {}
78
+ self.class.attribute_map.each_pair do |attr, param|
79
+ value = public_method(attr).call
80
+ next if value.nil? && !instance_variable_defined?("@#{attr}")
81
+ hash[param] = _to_hash(value)
82
+ end
83
+ hash
84
+ end
85
+
86
+ private
87
+
88
+ # Outputs non-array value in the form of hash
89
+ # For object, use to_hash. Otherwise, just return the value
90
+ # @param [Object] value Any valid value
91
+ # @return [Hash] Returns the value in the form of hash
92
+ def _to_hash(value)
93
+ if value.is_a?(Array)
94
+ value.compact.map{ |v| _to_hash(v) }
95
+ elsif value.is_a?(Hash)
96
+ {}.tap do |hash|
97
+ value.each { |k, v| hash[k] = _to_hash(v) }
98
+ end
99
+ elsif value.respond_to? :to_hash
100
+ value.to_hash
101
+ else
102
+ value
103
+ end
104
+ end
105
+
106
+
107
+
108
+ # Attribute mapping from ruby-style variable name to JSON key.
109
+ def self.attribute_map
110
+ {
111
+ :'path_routes' => :'pathRoutes'
112
+ }
113
+ end
114
+
115
+ # Attribute type mapping.
116
+ def self.swagger_types
117
+ {
118
+ :'path_routes' => :'Array<OCI::LoadBalancer::Models::PathRoute>'
119
+ }
120
+ end
121
+ end
122
+ end
@@ -48,9 +48,7 @@ module OCI
48
48
  sleep(interval_seconds)
49
49
 
50
50
  interval_seconds *= 2
51
- if interval_seconds > max_interval_seconds
52
- interval_seconds = max_interval_seconds
53
- end
51
+ interval_seconds = max_interval_seconds if interval_seconds > max_interval_seconds
54
52
  end
55
53
  end
56
54
  end
@@ -0,0 +1,111 @@
1
+ # Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
2
+
3
+ module OCI
4
+ module ObjectStorage
5
+ module Transfer
6
+ module Multipart
7
+ module Internal
8
+ # An IO-like interface over a part of a file which will participate in a multipart upload. This allows us
9
+ # to upload a given segment of the file as an "upload part" of a multipart upload.
10
+ class FilePartIOWrapper
11
+ # The source file, identified by either a File/Tempfile object or a path to the file
12
+ # @return [String, File, Tempfile]
13
+ attr_reader :source
14
+
15
+ # The zero-based position in the file to start reading from
16
+ # @return [Integer]
17
+ attr_reader :offset
18
+
19
+ # The number of bytes to read from the file
20
+ # @return [Integer]
21
+ attr_reader :part_size
22
+
23
+ # The position of the first byte to start reading from. Synonym for offset
24
+ # @return [Integer]
25
+ attr_reader :first_byte
26
+
27
+ # The position of the last byte we'll read to (exclusive). Calculated as offset + part_size
28
+ # @return [Integer]
29
+ attr_reader :last_byte
30
+
31
+ # Creates a new FilePartIOWrapper
32
+ #
33
+ # @param [String, File, Tempfile] source The source file, or a path to it
34
+ # @param [Integer] offset The zero-based position in the file to start reading from
35
+ # @param [Integer] part_size The number of bytes to read from the file
36
+ def initialize(source:, offset:, part_size:)
37
+ @source = source
38
+ @offset = offset
39
+ @part_size = part_size
40
+
41
+ @first_byte = offset
42
+ @last_byte = offset + part_size
43
+
44
+ @file_handle = nil
45
+ end
46
+
47
+ def close
48
+ @file_handle.close if @file_handle
49
+ end
50
+
51
+ def read(bytes = nil, output_buffer = nil)
52
+ open_file unless @file_handle
53
+ read_internal(bytes, output_buffer)
54
+ end
55
+
56
+ def write(_content)
57
+ raise 'FilePartIOWrapper does not support writing'
58
+ end
59
+
60
+ def size
61
+ @part_size
62
+ end
63
+
64
+ def rewind
65
+ if @file_handle
66
+ @file_handle.seek(@first_byte)
67
+ @position = @first_byte
68
+ end
69
+ 0
70
+ end
71
+
72
+ private
73
+
74
+ def open_file
75
+ @file_handle = File.open(@source)
76
+ rewind
77
+ end
78
+
79
+ def read_internal(bytes, output_buffer)
80
+ if bytes
81
+ data = @file_handle.read([remaining_bytes, bytes].min)
82
+ data = nil if data == ''
83
+ end
84
+ data = @file_handle.read(remaining_bytes) unless bytes
85
+
86
+ @position += data.bytesize if data
87
+
88
+ # From IO docs (https://ruby-doc.org/core-2.3.1/IO.html#method-i-read):
89
+ #
90
+ # "If the optional outbuf argument is present, it must reference a String, which will receive the data.
91
+ # The outbuf will contain only the received data after the method call even if it is not empty at the
92
+ # beginning."
93
+ #
94
+ # So we use String's replace() method to sub in the data we read, or an empty string if there was
95
+ # no data read
96
+ if output_buffer
97
+ output_buffer.replace(data || '')
98
+ else
99
+ data
100
+ end
101
+ end
102
+
103
+ def remaining_bytes
104
+ @last_byte - @position
105
+ end
106
+ end
107
+ end
108
+ end
109
+ end
110
+ end
111
+ end
@@ -0,0 +1,41 @@
1
+ # Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
2
+
3
+ module OCI
4
+ module ObjectStorage
5
+ module Transfer
6
+ module Multipart
7
+ module Internal
8
+ # Wraps a collection of parts to be uploaded to Object Storage in order to provide
9
+ # thread-safe access to the collection
10
+ class MultipartUploadPartsCollection
11
+ def initialize(parts = [])
12
+ @parts = parts
13
+ @lock = Mutex.new
14
+ end
15
+
16
+ def push(part)
17
+ @lock.synchronize { @parts.push(part) }
18
+ end
19
+
20
+ def shift
21
+ @lock.synchronize { @parts.shift }
22
+ end
23
+
24
+ def clear!
25
+ @lock.synchronize { @parts.clear }
26
+ end
27
+
28
+ def to_a
29
+ @lock.synchronize { @parts.dup }
30
+ end
31
+
32
+ def length
33
+ @lock.synchronize { @parts.length }
34
+ end
35
+ alias size length
36
+ end
37
+ end
38
+ end
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,48 @@
1
+ # Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
2
+
3
+ module OCI
4
+ module ObjectStorage
5
+ module Transfer
6
+ module Multipart
7
+ module Internal
8
+ # A class which wraps a seekable (but non-file) IO-like object (for example, a StringIO) so that
9
+ # we can extract a part of it to upload as an "upload part" of a multipart upload. This works by
10
+ # seeking to a given position in the IO-like object and then reading a certain number of bytes
11
+ # from that position onwards.
12
+ #
13
+ # Reading a part of the IO-like object is synchronised so that only a single thread can access the
14
+ # IO at any one time. This prevents issues where we would do, for example, concurrent seeks and so
15
+ # potentially end up reading content we didn't intend to read.
16
+ class SeekableNonFilePartIOWrapper
17
+ # The underlying IO-like object which we'll extract content from
18
+ # @return [IO]
19
+ attr_reader :source
20
+
21
+ # Creates a new SeekableNonFilePartIOWrapper
22
+ # @param [IO] source the IO-like object which will back this wrapper
23
+ def initialize(source:)
24
+ raise 'The provided source is not seekable' unless source.respond_to?(:seek)
25
+ @source = source
26
+ @lock = Mutex.new
27
+ end
28
+
29
+ # Reads a part/segment from the IO-like object by seeking to a given location and then
30
+ # reading a given number of bytes. This is synchronised so that only a single thread can
31
+ # perform this operation at any one time.
32
+ #
33
+ # @param [Integer] offset the zero-based position in the IO-like object where we'll start reading from
34
+ # @param [Integer] part_size the number of bytes to read
35
+ #
36
+ # @return [String] The content read from the IO-like object
37
+ def read(offset, part_size)
38
+ @lock.synchronize do
39
+ @source.seek(offset)
40
+ @source.read(part_size)
41
+ end
42
+ end
43
+ end
44
+ end
45
+ end
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,54 @@
1
+ # Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
2
+
3
+ module OCI
4
+ module ObjectStorage
5
+ module Transfer
6
+ module Multipart
7
+ module Internal
8
+ # A class which wraps standard input ($stdin) so that, for example, piped input can participate
9
+ # in multipart uploads. Standard input is not seekable so we need a special class to handle it
10
+ # rather than using {OCI::ObjectStorage::Transfer::Multipart::Internal::SeekableNonFilePartIOWrapper}.
11
+ #
12
+ # This class can also vend data for each part to upload as part of a multipart upload. However, since
13
+ # we may not know the full size of $stdin beforehand, in addition to vending data this wrapper will
14
+ # also vend the part number to assign to the uploaded part.
15
+ #
16
+ # Reading data out of stdin is synchronised so that only a single thread can read at any one time. This
17
+ # prevents issues where we could write the same part multiple times or write inconsistent data.
18
+ class StdinPartIOWrapper
19
+ # A reference to $stdin
20
+ # @return [IO]
21
+ attr_reader :source
22
+
23
+ # Creates a new StdinPartIOWrapper
24
+ #
25
+ # @param [IO] source A reference to $stdin
26
+ def initialize(source:)
27
+ @source = source
28
+ @lock = Mutex.new
29
+ @part_number = 0
30
+ end
31
+
32
+ # Reads bytes from $stdin. This is synchronised so that only a single thread can
33
+ # perform this operation at any one time.
34
+ #
35
+ # @param [Integer] part_size The number of bytes to read from $stdin
36
+ #
37
+ # @return [Hash] A hash with two keys. The :content key will contain the content we read from $stdin
38
+ # and the :part_number key will contain the part number we will assign to the part when we upload it as
39
+ # part of a multipart upload. If there is no more data from $stdin then this method will return nil.
40
+ def read(part_size)
41
+ @lock.synchronize do
42
+ read_content = @source.read(part_size)
43
+ if read_content
44
+ @part_number += 1
45
+ { content: read_content, part_number: @part_number }
46
+ end
47
+ end
48
+ end
49
+ end
50
+ end
51
+ end
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,545 @@
1
+ # Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
2
+
3
+ require 'base64'
4
+ require 'digest'
5
+ require 'tempfile'
6
+
7
+ require_relative 'internal/file_part_io_wrapper'
8
+ require_relative 'internal/stdin_part_io_wrapper'
9
+ require_relative 'internal/seekable_non_file_part_io_wrapper'
10
+ require_relative 'internal/multipart_upload_parts_collection'
11
+
12
+ module OCI
13
+ module ObjectStorage
14
+ module Transfer
15
+ module Multipart
16
+ # MultipartObjectAssembler provides a simplified interaction when uploading large
17
+ # objects using multi-part uploads.
18
+ #
19
+ # An assembler can be used to begin a new upload, or resume a previous one. A new assembler
20
+ # should be created per new upload or resumed upload to be performed. The same assembler is not
21
+ # resubale across multiple new uploads/resumes.
22
+ class MultipartObjectAssembler
23
+ MD5_CALC_PART_READ_BYTES = 8 * OCI::ObjectStorage::Transfer::MEBIBYTE
24
+
25
+ # Settings for the exponential backoff and retry (with jitter) which the assembler does
26
+ DEFAULT_MAX_ATTEMPTS = 3
27
+ DEFAULT_BASE_SLEEP_MILLIS = 1000
28
+ DEFAULT_MAX_SLEEP_TIME_MILLIS = 8000
29
+ DEFAULT_EXPONENTIAL_GROWTH_FACTOR = 2
30
+
31
+ # The client used to interact with the Object Storage service
32
+ # @return [OCI::ObjectStorage::ObjectStorageClient]
33
+ attr_accessor :object_storage_client
34
+
35
+ # The namespace containing the bucket in which to store the object
36
+ # @return [String]
37
+ attr_accessor :namespace
38
+
39
+ # The bucket where we'll upload the object
40
+ # @return [String]
41
+ attr_accessor :bucket_name
42
+
43
+ # The name of the object in Object Storage
44
+ # @return [String]
45
+ attr_accessor :object_name
46
+
47
+ # The size, in bytes, of each part of a multipart upload. This applies when we are uploading files from disk
48
+ # and defaults to 128 MiB
49
+ # @return [Integer]
50
+ attr_accessor :multipart_part_size
51
+
52
+ # The size, in bytes, of each part of a multipart upload when we are reading from stdin or a non-file IO-like
53
+ # source (e.g. a StringIO). Defaults to 10 MiB
54
+ # @return [Integer]
55
+ attr_accessor :non_file_io_multipart_part_size
56
+
57
+ # How many parts we can upload in parallel. Defaults to 3. If this is set to 1, this is the
58
+ # equivalent of not allowing parts to be uploaded in parallel.
59
+ # @return [Integer]
60
+ attr_accessor :parallel_process_count
61
+
62
+ # A bag of optional parameter (e.g. the client request ID, metadata) which we can use when making calls to the Object Storage service.
63
+ # @return [Hash]
64
+ attr_accessor :multipart_upload_opts
65
+
66
+ # If we encounter a failure when performing an operation and need to retry, the maximum number of attempts we
67
+ # can make before declaring failure. Attempts are 1-based, i.e. the first call we make is considered attempt 1.
68
+ # @return [Integer]
69
+ attr_accessor :max_attempts
70
+
71
+ # For exponential backoff and retry, the base time to use in our retry calculation in milliseconds. Defaults to 1000ms
72
+ # @return [Integer]
73
+ attr_accessor :base_sleep_millis
74
+
75
+ # The maximum amount of time to wait between retries. Defaults to 8000ms
76
+ # @return [Integer]
77
+ attr_accessor :max_sleep_time_millis
78
+
79
+ # For exponential backoff and retry, the exponent which we will raise to the power of the number of attempts. Defaults to 2
80
+ # @return [Integer]
81
+ attr_accessor :exponential_growth_factor
82
+
83
+ # Contains the upload ID for the multipart upload and other upload-specific information (e.g. the destination namespace, bucket and object,
84
+ # and the parts which have been uploaded)
85
+ # @return [Hash]
86
+ attr_reader :manifest
87
+
88
+ # @param [OCI::ObjectStorage::ObjectStorageClient] object_storage_client The client used to interact with the Object Storage service
89
+ # @param [String] namespace The namespace containing the bucket in which to store the object
90
+ # @param [String] bucket_name The bucket where we'll upload the object
91
+ # @param [String] object_name The name of the object in Object Storage
92
+ # @param [Integer] multipart_part_size The size, in bytes, of each part of a multipart upload. This applies when we are uploading files from disk and defaults to 128 MiB
93
+ # @param [Integer] non_file_io_multipart_part_size The size, in bytes, of each part of a multipart upload when we are reading from stdin or a non-file IO-like source (e.g. a StringIO). Defaults to 10 MiB
94
+ # @param [Integer] parallel_process_count How many parts we can upload in parallel. Defaults to 3
95
+ # @param [Hash] multipart_upload_opts
96
+ # @option multipart_upload_opts [String] :if_match The entity tag to match. Only used for new multipart uploads and ignored otherwise.
97
+ # @option multipart_upload_opts [String] :if_none_match The entity tag to avoid matching. The only valid value is *, which indicates that the request should fail if the object already exists. Only used for new multipart uploads and ignored otherwise.
98
+ # @option multipart_upload_opts [String] :opc_client_request_id The client request ID for tracing. Will be applied to all requests made by this assembler.
99
+ # @option multipart_upload_opts [String] :content_type The content type of the object. Defaults to 'application/octet-stream' if not overridden. Only used for new multipart uploads and ignored otherwise.
100
+ # @option multipart_upload_opts [String] :content_language The content language of the object. Only used for new multipart uploads and ignored otherwise.
101
+ # @option multipart_upload_opts [String] :content_encoding The content encoding of the object. Only used for new multipart uploads and ignored otherwise.
102
+ # @option multipart_upload_opts [Hash<String, String>] :metadata A hash of string keys to string values representing any custom metadata to be applied to the object. Only used for new multipart uploads and ignored otherwise.
103
+ def initialize(object_storage_client:,
104
+ namespace:,
105
+ bucket_name:,
106
+ object_name:,
107
+ multipart_part_size: OCI::ObjectStorage::Transfer::MULTIPART_PART_SIZE,
108
+ non_file_io_multipart_part_size: OCI::ObjectStorage::Transfer::NON_FILE_IO_MULTIPART_PART_SIZE,
109
+ parallel_process_count: OCI::ObjectStorage::Transfer::Multipart::DEFAULT_PARALLEL_PROCESS_COUNT,
110
+ multipart_upload_opts: {})
111
+ @object_storage_client = object_storage_client
112
+ @namespace = namespace
113
+ @bucket_name = bucket_name
114
+ @object_name = object_name
115
+ @multipart_part_size = multipart_part_size
116
+ @non_file_io_multipart_part_size = non_file_io_multipart_part_size
117
+ @parallel_process_count = parallel_process_count
118
+ @object_io = nil
119
+
120
+ @manifest = {
121
+ upload_id: nil,
122
+ namespace: @namespace,
123
+ bucket_name: @bucket_name,
124
+ object_name: @object_name,
125
+ object_io_or_file_path: nil,
126
+ parts: OCI::ObjectStorage::Transfer::Multipart::Internal::MultipartUploadPartsCollection.new([])
127
+ }
128
+
129
+ @multipart_upload_opts = multipart_upload_opts
130
+ @multipart_upload_opts.delete('content_md5') if @multipart_upload_opts.key?('content_md5')
131
+
132
+ @max_attempts = DEFAULT_MAX_ATTEMPTS
133
+ @base_sleep_millis = DEFAULT_BASE_SLEEP_MILLIS
134
+ @exponential_growth_factor = DEFAULT_EXPONENTIAL_GROWTH_FACTOR
135
+ @max_sleep_time_millis = DEFAULT_MAX_SLEEP_TIME_MILLIS
136
+ end
137
+
138
+ # Initializes a new multipart upload.
139
+ #
140
+ # @return [Response] A Response object with data of type OCI::ObjectStorage::Models::MultipartUpload
141
+ def new_upload
142
+ raise 'This MultipartObjectAssembler has already been initialized with an upload' if @manifest[:upload_id]
143
+
144
+ create_multipart_upload_opts = {}
145
+ create_multipart_upload_opts[:if_match] = @multipart_upload_opts[:if_match] if @multipart_upload_opts[:if_match]
146
+ create_multipart_upload_opts[:if_none_match] = @multipart_upload_opts[:if_none_match] \
147
+ if @multipart_upload_opts[:if_none_match]
148
+
149
+ create_multipart_upload_opts[:opc_client_request_id] = @multipart_upload_opts[:opc_client_request_id] \
150
+ if @multipart_upload_opts[:opc_client_request_id]
151
+
152
+ create_multipart_upload_details =
153
+ OCI::ObjectStorage::Models::CreateMultipartUploadDetails.new(object: @object_name)
154
+
155
+ create_multipart_upload_details.content_type = @multipart_upload_opts[:content_type] \
156
+ if @multipart_upload_opts[:content_type]
157
+
158
+ create_multipart_upload_details.content_language = @multipart_upload_opts[:content_language] \
159
+ if @multipart_upload_opts[:content_language]
160
+
161
+ create_multipart_upload_details.content_encoding = @multipart_upload_opts[:content_encoding] \
162
+ if @multipart_upload_opts[:content_encoding]
163
+
164
+ if @multipart_upload_opts[:metadata] && !@multipart_upload_opts[:metadata].empty?
165
+ processed_metadata = {}
166
+ @multipart_upload_opts[:metadata].each do |key, value|
167
+ processed_metadata[key] = value if key.to_s.start_with?('opc-meta-')
168
+ processed_metadata["opc-meta-#{key}"] = value unless key.to_s.start_with?('opc-meta-')
169
+ end
170
+ create_multipart_upload_details.metadata = processed_metadata
171
+ end
172
+
173
+ response = make_retrying_call do
174
+ @object_storage_client.create_multipart_upload(
175
+ @namespace, @bucket_name,
176
+ create_multipart_upload_details,
177
+ create_multipart_upload_opts
178
+ )
179
+ end
180
+ @manifest[:upload_id] = response.data.upload_id
181
+
182
+ response
183
+ end
184
+
185
+ # Initializes the parts in the manifest based on the provided input. If the input is stdin then the list of parts in the manifest
186
+ # will be left empty (the parts in this case are figured out dynamically at upload time since we may not have all the information in advance).
187
+ # If the input is the path to a file (a String), a file or a IO-like object (e.g. a StringIO) then the parts in the manifest will
188
+ # be initialized so that the assembler can go through and upload them.
189
+ #
190
+ # No uploads will be performed until the upload method is called.
191
+ #
192
+ # @param [String, IO] object_io_or_file_path Either a path to the file to upload, an IO-like object containing the data to upload or $stdin.
193
+ def io_for_transfer=(object_io_or_file_path)
194
+ @manifest[:object_io_or_file_path] = object_io_or_file_path
195
+
196
+ return if stdin?(object_io_or_file_path)
197
+
198
+ opened_file = false
199
+ if object_io_or_file_path.is_a?(String)
200
+ object_io = File.open(object_io_or_file_path, 'rb')
201
+ opened_file = true
202
+ end
203
+ object_io = object_io_or_file_path if object_io_or_file_path.respond_to?(:seek)
204
+
205
+ total_size = object_io.size if object_io.respond_to?(:size)
206
+ total_size = object_io.stat.size unless object_io.respond_to?(:size)
207
+
208
+ part_size_to_use = @multipart_part_size if file_based_io?(object_io_or_file_path)
209
+ part_size_to_use = @non_file_io_multipart_part_size unless file_based_io?(object_io_or_file_path)
210
+
211
+ offset = 0
212
+ part_number = 1
213
+ while offset < total_size
214
+ part_info = {
215
+ offset: offset,
216
+ part_size: calculate_part_size(total_size, offset, part_size_to_use),
217
+ part_number: part_number,
218
+ part_md5_hash: nil
219
+ }
220
+ @manifest[:parts].push(part_info)
221
+ offset += part_size_to_use
222
+ part_number += 1
223
+ end
224
+
225
+ nil
226
+ ensure
227
+ object_io.close if opened_file && object_io
228
+ end
229
+
230
+ # Performs a multipart upload
231
+ #
232
+ # @return [Array] If the multipart upload was successful, an empty array. If there were errors then the array will contain
233
+ # one element per error which occurred.
234
+ def upload
235
+ all_parts = @manifest[:parts].to_a
236
+ pending_parts = OCI::ObjectStorage::Transfer::Multipart::Internal::MultipartUploadPartsCollection.new(
237
+ all_parts.select { |ap| ap[:opc_md5].nil? && ap[:etag].nil? }
238
+ )
239
+
240
+ unless file_based_io?(@manifest[:object_io_or_file_path]) || stdin?(@manifest[:object_io_or_file_path])
241
+ seekable_io_wrapper = OCI::ObjectStorage::Transfer::Multipart::Internal::SeekableNonFilePartIOWrapper.new(
242
+ source: @manifest[:object_io_or_file_path]
243
+ )
244
+ end
245
+
246
+ if stdin?(@manifest[:object_io_or_file_path])
247
+ stdin_io_wrapper = OCI::ObjectStorage::Transfer::Multipart::Internal::StdinPartIOWrapper.new(
248
+ source: @manifest[:object_io_or_file_path]
249
+ )
250
+ end
251
+
252
+ threads = []
253
+ @parallel_process_count.times do
254
+ thread = Thread.new do
255
+ begin
256
+ upload_non_stdin(pending_parts, seekable_io_wrapper) unless stdin?(@manifest[:object_io_or_file_path])
257
+ upload_stdin(stdin_io_wrapper) if stdin?(@manifest[:object_io_or_file_path])
258
+
259
+ nil
260
+ rescue => error
261
+ pending_parts.clear! # Stop any futher processing on error
262
+ error
263
+ end
264
+ end
265
+
266
+ thread.abort_on_exception = true # Do not continue if we encounter exceptions
267
+ threads << thread
268
+ end
269
+
270
+ threads.map(&:value).compact
271
+ end
272
+
273
+ # Resume uploading a multipart object to Object Storage. This assumes that the assembler already has knowledge
274
+ # of the parts which it could potentially need to upload (i.e. they have been prepared via the set_io_for_transfer
275
+ # method).
276
+ #
277
+ # Prior to resuming the upload, we'll attempt to reconcile with Object Storage any previously uploaded parts so that
278
+ # they are not uploaded again
279
+ #
280
+ # @param [String] upload_id The ID of the multipart upload to resume.
281
+ #
282
+ # @return [Array] If the multipart upload was successful, an empty array. If there were errors then the array will contain
283
+ # one element per error which occurred.
284
+ def resume(upload_id)
285
+ raise 'An upload ID must be provided' if upload_id.nil?
286
+ raise 'Parts must be initialized prior to resuming' if @manifest[:parts].length.zero?
287
+
288
+ manifest[:upload_id] = upload_id
289
+ upload_parts = list_uploaded_parts(upload_id)
290
+ known_parts = @manifest[:parts].to_a
291
+
292
+ raise 'There are more parts on the server than parts to resume, please check the upload ID.' \
293
+ if upload_parts.length > known_parts.length
294
+
295
+ upload_parts.each do |up|
296
+ index = up.part_number - 1
297
+ manifest_part = known_parts[index]
298
+
299
+ if manifest_part[:part_size] != up.size
300
+ raise 'Cannot resume upload with different part size. ' \
301
+ + "Parts were uploaded with a part size of #{up.size / OCI::ObjectStorage::Transfer::MEBIBYTE} MiB"
302
+ end
303
+
304
+ manifest_part[:etag] = up.etag
305
+ manifest_part[:opc_md5] = up.md5
306
+ end
307
+
308
+ upload
309
+ end
310
+
311
+ # Commits the multipart upload.
312
+ #
313
+ # @return [Response] A Response object with data of type nil. For a multipart upload, the headers of the response will contain
314
+ # an opc-multipart-md5 key.
315
+ def commit
316
+ raise 'Cannot commit as this MultipartObjectAssembler has not been initialized with an upload' \
317
+ unless @manifest[:upload_id]
318
+
319
+ parts = @manifest[:parts].to_a
320
+
321
+ commit_upload_details = OCI::ObjectStorage::Models::CommitMultipartUploadDetails.new
322
+ commit_upload_details.parts_to_commit = []
323
+ commit_upload_details.parts_to_exclude = []
324
+
325
+ parts.each do |part|
326
+ if part[:etag]
327
+ commit_upload_details.parts_to_commit << OCI::ObjectStorage::Models::CommitMultipartUploadPartDetails.new(
328
+ partNum: part[:part_number],
329
+ etag: part[:etag]
330
+ )
331
+ end
332
+
333
+ commit_upload_details.parts_to_exclude << part[:part_number] unless part[:etag]
334
+ end
335
+
336
+ response = make_retrying_call do
337
+ @object_storage_client.commit_multipart_upload(
338
+ @namespace,
339
+ @bucket_name,
340
+ @object_name,
341
+ @manifest[:upload_id],
342
+ commit_upload_details
343
+ )
344
+ end
345
+
346
+ response
347
+ end
348
+
349
+ # Aborts a multipart upload.
350
+ #
351
+ # @param [String] upload_id The ID of the multipart upload to abort.
352
+ # @return [Response] A Response object with data of type nil
353
+ def abort(upload_id)
354
+ abort_opts = {}
355
+ abort_opts[:opc_client_request_id] = @multipart_upload_opts[:opc_client_request_id] \
356
+ if @multipart_upload_opts[:opc_client_request_id]
357
+
358
+ response = make_retrying_call do
359
+ @object_storage_client.abort_multipart_upload(
360
+ @namespace,
361
+ @bucket_name,
362
+ @object_name,
363
+ upload_id,
364
+ abort_opts
365
+ )
366
+ end
367
+ response
368
+ end
369
+
370
+ private
371
+
372
+ def calculate_part_size(total_size, offset, part_size)
373
+ if offset + part_size > total_size
374
+ total_size - offset
375
+ else
376
+ part_size
377
+ end
378
+ end
379
+
380
+ def calculate_md5(file_part: nil, raw_content: nil)
381
+ raise 'Cannot specify both a file_part and raw_content' if file_part && raw_content
382
+ md5 = Digest::MD5.new
383
+
384
+ if file_part
385
+ file_part.rewind
386
+ loop do
387
+ part_read = file_part.read(MD5_CALC_PART_READ_BYTES)
388
+ break unless part_read
389
+ md5.update(part_read)
390
+ end
391
+ file_part.rewind
392
+ end
393
+
394
+ md5.update(raw_content) if raw_content
395
+
396
+ md5.base64digest
397
+ end
398
+
399
+ def list_uploaded_parts(upload_id)
400
+ list_multipart_upload_parts_opts = {}
401
+ if @multipart_upload_opts[:opc_client_request_id]
402
+ list_multipart_upload_parts_opts[:opc_client_request_id] = @multipart_upload_opts[:opc_client_request_id]
403
+ end
404
+
405
+ upload_parts = []
406
+ @object_storage_client.list_multipart_upload_parts(
407
+ @namespace,
408
+ @bucket_name,
409
+ @object_name,
410
+ upload_id,
411
+ list_multipart_upload_parts_opts
412
+ ).each do |response|
413
+ upload_parts += response.data
414
+ end
415
+
416
+ upload_parts
417
+ end
418
+
419
+ def file_based_io?(object_io_or_file_path)
420
+ object_io_or_file_path.is_a?(String) ||
421
+ object_io_or_file_path.is_a?(File) ||
422
+ object_io_or_file_path.is_a?(Tempfile)
423
+ end
424
+
425
+ def stdin?(object_io_or_file_path)
426
+ object_io_or_file_path.eql?($stdin)
427
+ end
428
+
429
+ def upload_non_stdin(pending_parts, seekable_io_wrapper = nil)
430
+ while (part = pending_parts.shift) # While we have parts left to process
431
+ if file_based_io?(@manifest[:object_io_or_file_path])
432
+ part_content = OCI::ObjectStorage::Transfer::Multipart::Internal::FilePartIOWrapper.new(
433
+ source: @manifest[:object_io_or_file_path],
434
+ offset: part[:offset],
435
+ part_size: part[:part_size]
436
+ )
437
+ part[:part_md5_hash] = calculate_md5(file_part: part_content)
438
+ else
439
+ part_content = seekable_io_wrapper.read(part[:offset], part[:part_size])
440
+ part[:part_md5_hash] = calculate_md5(raw_content: part_content)
441
+ end
442
+
443
+ upload_part_opts = { content_md5: part[:md5_hash] }
444
+ if @multipart_upload_opts[:opc_client_request_id]
445
+ upload_part_opts[:opc_client_request_id] = @multipart_upload_opts[:opc_client_request_id]
446
+ end
447
+
448
+ response = make_retrying_call do
449
+ @object_storage_client.upload_part(
450
+ @namespace,
451
+ @bucket_name,
452
+ @object_name,
453
+ @manifest[:upload_id],
454
+ part[:part_number],
455
+ part_content,
456
+ upload_part_opts
457
+ )
458
+ end
459
+ part_content.close if file_based_io?(@manifest[:object_io_or_file_path])
460
+
461
+ part[:etag] = response.headers['etag']
462
+ part[:opc_md5] = response.headers['opc-content-md5']
463
+ end
464
+ end
465
+
466
+ def upload_stdin(stdin_io_wrapper)
467
+ # While we can still pull content from stdin
468
+ while (stdin_part = stdin_io_wrapper.read(@non_file_io_multipart_part_size))
469
+ part = {
470
+ part_number: stdin_part[:part_number],
471
+ part_size: stdin_part[:content].size,
472
+ part_md5_hash: calculate_md5(raw_content: stdin_part[:content])
473
+ }
474
+
475
+ upload_part_opts = { content_md5: part[:md5_hash] }
476
+ if @multipart_upload_opts[:opc_client_request_id]
477
+ upload_part_opts[:opc_client_request_id] = @multipart_upload_opts[:opc_client_request_id]
478
+ end
479
+
480
+ response = make_retrying_call do
481
+ @object_storage_client.upload_part(
482
+ @namespace,
483
+ @bucket_name,
484
+ @object_name,
485
+ @manifest[:upload_id],
486
+ part[:part_number],
487
+ stdin_part[:content],
488
+ upload_part_opts
489
+ )
490
+ end
491
+
492
+ part[:etag] = response.headers['etag']
493
+ part[:opc_md5] = response.headers['opc-content-md5']
494
+
495
+ @manifest[:parts].push(part)
496
+ end
497
+ end
498
+
499
+ # Make a retrying call using a provided block (that we yield to) and return the result of the block
500
+ # on success.
501
+ def make_retrying_call
502
+ @max_attempts.times do |attempt|
503
+ try = attempt + 1
504
+ begin
505
+ return yield
506
+ rescue OCI::Errors::NetworkError # Network errors should be retriable
507
+ raise if try >= @max_attempts # Short-circuit if we're already on our last attempt
508
+
509
+ # Use full jitter on networking errors (as hopefully they are short-lived/intermittent)
510
+ sleep(get_full_jitter_sleep_time_millis(attempt) / 1000.0)
511
+ rescue OCI::Errors::ServiceError => svc_err
512
+ raise if try >= @max_attempts # Short-circuit if we're already on our last attempt
513
+
514
+ # For internal server errors, use full jitter as it's hopefully intermittent
515
+ sleep(get_full_jitter_sleep_time_millis(attempt) / 1000.0) if svc_err.status >= 500 || svc_err.status == -1
516
+
517
+ # For throttles and consistency errors, use equal jitter as this guarantees some sleep time
518
+ # between attempts (full jitter doesn't because we get a value across the range 0 to something, i.e.
519
+ # it is possible to get a very small sleep time)
520
+ if svc_err.status == 429 || (svc_err.status == 409 && svc_err.code == 'ConcurrentObjectUpdate')
521
+ sleep(get_equal_jitter_sleep_time_millis(attempt) / 1000.0)
522
+ end
523
+
524
+ # A non-retriable service error, so just throw it
525
+ raise
526
+ end
527
+ end
528
+ end
529
+
530
+ def get_full_jitter_sleep_time_millis(attempt)
531
+ random = Random.new
532
+ random.rand(0..[@base_sleep_millis * (@exponent_growth_factor**attempt), @max_sleep_time_millis].min)
533
+ end
534
+
535
+ def get_equal_jitter_sleep_time_millis(attempt)
536
+ random = Random.new
537
+ sleep_raw = [@base_sleep_millis * (@exponent_growth_factor**attempt), @max_sleep_time_millis].min
538
+
539
+ (sleep_raw / 2) + random.rand(0..(sleep_raw / 2))
540
+ end
541
+ end
542
+ end
543
+ end
544
+ end
545
+ end