google-cloud-bigquery 1.1.0 → 1.2.0

data/lib/google/cloud/bigquery/table.rb

@@ -18,6 +18,7 @@ require "google/cloud/bigquery/service"
 require "google/cloud/bigquery/data"
 require "google/cloud/bigquery/table/list"
 require "google/cloud/bigquery/schema"
+require "google/cloud/bigquery/encryption_configuration"
 require "google/cloud/bigquery/external"
 require "google/cloud/bigquery/insert_response"
 require "google/cloud/bigquery/table/async_inserter"
@@ -782,6 +783,53 @@ module Google
           schema.headers
         end
 
+        ##
+        # The {EncryptionConfiguration} object that represents the custom
+        # encryption method used to protect the table. If not set, default
+        # encryption is used.
+        #
+        # Present only if the table is using custom encryption.
+        #
+        # @see https://cloud.google.com/bigquery/docs/customer-managed-encryption
+        #   Protecting Data with Cloud KMS Keys
+        #
+        # @return [EncryptionConfiguration, nil] The encryption configuration.
+        #
+        #   @!group Attributes
+        #
+        def encryption
+          return nil if reference?
+          ensure_full_data!
+          return nil if @gapi.encryption_configuration.nil?
+          EncryptionConfiguration.from_gapi(@gapi.encryption_configuration)
+                                 .freeze
+        end
+
+        ##
+        # Set the {EncryptionConfiguration} object that represents the custom
+        # encryption method used to protect the table. If not set, default
+        # encryption is used.
+        #
+        # Present only if the table is using custom encryption.
+        #
+        # If the table is not a full resource representation (see
+        # {#resource_full?}), the full representation will be retrieved before
+        # the update to comply with ETag-based optimistic concurrency control.
+        #
+        #
+        # @see https://cloud.google.com/bigquery/docs/customer-managed-encryption
+        #   Protecting Data with Cloud KMS Keys
+        #
+        # @param [EncryptionConfiguration] value The new encryption config.
+        #
+        # @!group Attributes
+        #
+        def encryption= value
+          reload! unless resource_full?
+          @gapi.encryption_configuration = value.to_gapi
+          patch_gapi! :encryption_configuration
+        end
+
         ##
         # The {External::DataSource} (or subclass) object that represents the
         # external data source that the table represents. Data can be queried
@@ -1080,6 +1128,12 @@ module Google
         # {Job#done?} to detect when the job is done, or simply block until the
         # job is done by calling #{Job#wait_until_done!}. See also {#copy}.
         #
+        # The geographic location for the job ("US", "EU", etc.) can be set via
+        # {CopyJob::Updater#location=} in a block passed to this method. If the
+        # table is a full resource representation (see {#resource_full?}), the
+        # location of the job will be automatically set to the location of the
+        # table.
+        #
         # @param [Table, String] destination_table The destination for the
         #   copied data. This can also be a string identifier as specified by
         #   the [Query
@@ -1125,6 +1179,9 @@ module Google
         #   dashes. International characters are allowed. Label values are
         #   optional. Label keys must start with a letter and each label in the
         #   list must have a different key.
+        # @yield [job] a job configuration object
+        # @yieldparam [Google::Cloud::Bigquery::CopyJob::Updater] job a job
+        #   configuration object for setting additional options.
         #
         # @return [Google::Cloud::Bigquery::CopyJob]
         #
@@ -1153,10 +1210,19 @@ module Google
                      job_id: nil, prefix: nil, labels: nil
           ensure_service!
           options = { create: create, write: write, dryrun: dryrun,
-                      job_id: job_id, prefix: prefix, labels: labels }
-          gapi = service.copy_table table_ref,
-                                    get_table_ref(destination_table),
-                                    options
+                      labels: labels, job_id: job_id, prefix: prefix }
+          updater = CopyJob::Updater.from_options(
+            service,
+            table_ref,
+            get_table_ref(destination_table),
+            options
+          )
+          updater.location = location if location # may be table reference
+
+          yield updater if block_given?
+
+          job_gapi = updater.to_gapi
+          gapi = service.copy_table job_gapi
           Job.from_gapi gapi, service
         end
 
@@ -1166,6 +1232,12 @@ module Google
         # generally handled as needed to complete the job. See also
         # {#copy_job}.
         #
+        # The geographic location for the job ("US", "EU", etc.) can be set via
+        # {CopyJob::Updater#location=} in a block passed to this method. If the
+        # table is a full resource representation (see {#resource_full?}), the
+        # location of the job will be automatically set to the location of the
+        # table.
+        #
         # @param [Table, String] destination_table The destination for the
         #   copied data. This can also be a string identifier as specified by
         #   the [Query
@@ -1189,6 +1261,9 @@ module Google
         #   * `append` - BigQuery appends the data to the table.
         #   * `empty` - An error will be returned if the destination table
         #     already contains data.
+        # @yield [job] a job configuration object
+        # @yieldparam [Google::Cloud::Bigquery::CopyJob::Updater] job a job
+        #   configuration object for setting additional options.
         #
         # @return [Boolean] Returns `true` if the copy operation succeeded.
         #
@@ -1213,20 +1288,10 @@ module Google
         #
         # @!group Data
         #
-        def copy destination_table, create: nil, write: nil
-          job = copy_job destination_table, create: create, write: write
+        def copy destination_table, create: nil, write: nil, &block
+          job = copy_job destination_table, create: create, write: write, &block
           job.wait_until_done!
-
-          if job.failed?
-            begin
-              # raise to activate ruby exception cause handling
-              raise job.gapi_error
-            rescue StandardError => e
-              # wrap Google::Apis::Error with Google::Cloud::Error
-              raise Google::Cloud::Error.from_error(e)
-            end
-          end
-
+          ensure_job_succeeded! job
           true
         end
 
@@ -1238,6 +1303,12 @@ module Google
         # simply block until the job is done by calling #{Job#wait_until_done!}.
         # See also {#extract}.
         #
+        # The geographic location for the job ("US", "EU", etc.) can be set via
+        # {ExtractJob::Updater#location=} in a block passed to this method. If
+        # the table is a full resource representation (see {#resource_full?}),
+        # the location of the job will be automatically set to the location of
+        # the table.
+        #
         # @see https://cloud.google.com/bigquery/exporting-data-from-bigquery
         #   Exporting Data From BigQuery
         #
@@ -1281,7 +1352,9 @@ module Google
         #   dashes. International characters are allowed. Label values are
         #   optional. Label keys must start with a letter and each label in the
         #   list must have a different key.
-        #
+        # @yield [job] a job configuration object
+        # @yieldparam [Google::Cloud::Bigquery::ExtractJob::Updater] job a job
+        #   configuration object for setting additional options.
         #
         # @return [Google::Cloud::Bigquery::ExtractJob]
         #
@@ -1304,7 +1377,14 @@ module Google
           options = { format: format, compression: compression,
                       delimiter: delimiter, header: header, dryrun: dryrun,
                       job_id: job_id, prefix: prefix, labels: labels }
-          gapi = service.extract_table table_ref, extract_url, options
+          updater = ExtractJob::Updater.from_options service, table_ref,
+                                                     extract_url, options
+          updater.location = location if location # may be table reference
+
+          yield updater if block_given?
+
+          job_gapi = updater.to_gapi
+          gapi = service.extract_table job_gapi
           Job.from_gapi gapi, service
         end
 
@@ -1314,6 +1394,12 @@ module Google
         # transient errors are generally handled as needed to complete the job.
         # See also {#extract_job}.
         #
+        # The geographic location for the job ("US", "EU", etc.) can be set via
+        # {ExtractJob::Updater#location=} in a block passed to this method. If
+        # the table is a full resource representation (see {#resource_full?}),
+        # the location of the job will be automatically set to the location of
+        # the table.
+        #
         # @see https://cloud.google.com/bigquery/exporting-data-from-bigquery
         #   Exporting Data From BigQuery
         #
@@ -1335,11 +1421,13 @@ module Google
         #   exported data. Default is <code>,</code>.
         # @param [Boolean] header Whether to print out a header row in the
         #   results. Default is `true`.
-        #
+        # @yield [job] a job configuration object
+        # @yieldparam [Google::Cloud::Bigquery::ExtractJob::Updater] job a job
+        #   configuration object for setting additional options.
         #
         # @return [Boolean] Returns `true` if the extract operation succeeded.
         #
-        # @example
+        # @example Extract to a JSON file:
         #   require "google/cloud/bigquery"
         #
         #   bigquery = Google::Cloud::Bigquery.new
@@ -1348,25 +1436,29 @@ module Google
         #
         #   table.extract "gs://my-bucket/file-name.json", format: "json"
         #
+        # @example Extract to a CSV file, attaching labels to the job:
+        #   require "google/cloud/bigquery"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #   dataset = bigquery.dataset "my_dataset"
+        #   table = dataset.table "my_table"
+        #
+        #   table.extract "gs://my-bucket/file-name.csv" do |extract|
+        #     extract.labels = { "custom-label" => "custom-value" }
+        #   end
+        #
         # @!group Data
         #
         def extract extract_url, format: nil, compression: nil, delimiter: nil,
-                    header: nil
-          job = extract_job extract_url, format: format,
-                                         compression: compression,
-                                         delimiter: delimiter, header: header
+                    header: nil, &block
+          job = extract_job extract_url,
+                            format: format,
+                            compression: compression,
+                            delimiter: delimiter,
+                            header: header,
+                            &block
           job.wait_until_done!
-
-          if job.failed?
-            begin
-              # raise to activate ruby exception cause handling
-              raise job.gapi_error
-            rescue StandardError => e
-              # wrap Google::Apis::Error with Google::Cloud::Error
-              raise Google::Cloud::Error.from_error(e)
-            end
-          end
-
+          ensure_job_succeeded! job
           true
         end
 
@@ -1376,9 +1468,16 @@ module Google
         # file directly. See [Loading Data with a POST Request](
         # https://cloud.google.com/bigquery/loading-data-post-request#multipart).
         #
-        # @param [File, Google::Cloud::Storage::File, String, URI] file A file
-        #   or the URI of a Google Cloud Storage file containing data to load
-        #   into the table.
+        # The geographic location for the job ("US", "EU", etc.) can be set via
+        # {LoadJob::Updater#location=} in a block passed to this method. If the
+        # table is a full resource representation (see {#resource_full?}), the
+        # location of the job will be automatically set to the location of the
+        # table.
+        #
+        # @param [File, Google::Cloud::Storage::File, String, URI,
+        #   Array<Google::Cloud::Storage::File, String, URI>] files
+        #   A file or the URI of a Google Cloud Storage file, or an Array of
+        #   those, containing data to load into the table.
         # @param [String] format The exported file format. The default value is
         #   `csv`.
         #
@@ -1488,6 +1587,8 @@ module Google
         #   dashes. International characters are allowed. Label values are
         #   optional. Label keys must start with a letter and each label in the
         #   list must have a different key.
+        # @yield [load_job] a block for setting the load job
+        # @yieldparam [LoadJob] load_job the load job object to be updated
         #
         # @return [Google::Cloud::Bigquery::LoadJob]
         #
@@ -1513,6 +1614,19 @@ module Google
         #   file = bucket.file "file-name.csv"
         #   load_job = table.load_job file
         #
+        # @example Pass a list of google-cloud-storage files:
+        #   require "google/cloud/bigquery"
+        #   require "google/cloud/storage"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #   dataset = bigquery.dataset "my_dataset"
+        #   table = dataset.table "my_table"
+        #
+        #   storage = Google::Cloud::Storage.new
+        #   bucket = storage.bucket "my-bucket"
+        #   file = bucket.file "file-name.csv"
+        #   load_job = table.load_job [file, "gs://my-bucket/file-name2.csv"]
+        #
         # @example Upload a file directly:
         #   require "google/cloud/bigquery"
         #
@@ -1525,25 +1639,35 @@ module Google
         #
         # @!group Data
         #
-        def load_job file, format: nil, create: nil, write: nil,
+        def load_job files, format: nil, create: nil, write: nil,
                      projection_fields: nil, jagged_rows: nil,
                      quoted_newlines: nil, encoding: nil, delimiter: nil,
                      ignore_unknown: nil, max_bad_records: nil, quote: nil,
                      skip_leading: nil, dryrun: nil, job_id: nil, prefix: nil,
                      labels: nil, autodetect: nil, null_marker: nil
           ensure_service!
-          options = { format: format, create: create, write: write,
-                      projection_fields: projection_fields,
-                      jagged_rows: jagged_rows,
-                      quoted_newlines: quoted_newlines, encoding: encoding,
-                      delimiter: delimiter, ignore_unknown: ignore_unknown,
-                      max_bad_records: max_bad_records, quote: quote,
-                      skip_leading: skip_leading, dryrun: dryrun,
-                      job_id: job_id, prefix: prefix, labels: labels,
-                      autodetect: autodetect, null_marker: null_marker }
-          return load_storage(file, options) if storage_url? file
-          return load_local(file, options) if local_file? file
-          raise Google::Cloud::Error, "Don't know how to load #{file}"
+
+          updater = load_job_updater format: format, create: create,
+                                     write: write,
+                                     projection_fields: projection_fields,
+                                     jagged_rows: jagged_rows,
+                                     quoted_newlines: quoted_newlines,
+                                     encoding: encoding,
+                                     delimiter: delimiter,
+                                     ignore_unknown: ignore_unknown,
+                                     max_bad_records: max_bad_records,
+                                     quote: quote, skip_leading: skip_leading,
+                                     dryrun: dryrun, job_id: job_id,
+                                     prefix: prefix, schema: schema,
+                                     labels: labels, autodetect: autodetect,
+                                     null_marker: null_marker
+
+          yield updater if block_given?
+
+          job_gapi = updater.to_gapi
+
+          return load_local(files, job_gapi) if local_file? files
+          load_storage files, job_gapi
         end
 
         ##
@@ -1552,9 +1676,16 @@ module Google
         # file directly. See [Loading Data with a POST Request](
         # https://cloud.google.com/bigquery/loading-data-post-request#multipart).
         #
-        # @param [File, Google::Cloud::Storage::File, String, URI] file A file
-        #   or the URI of a Google Cloud Storage file containing data to load
-        #   into the table.
+        # The geographic location for the job ("US", "EU", etc.) can be set via
+        # {LoadJob::Updater#location=} in a block passed to this method. If the
+        # table is a full resource representation (see {#resource_full?}), the
+        # location of the job will be automatically set to the location of the
+        # table.
+        #
+        # @param [File, Google::Cloud::Storage::File, String, URI,
+        #   Array<Google::Cloud::Storage::File, String, URI>] files
+        #   A file or the URI of a Google Cloud Storage file, or an Array of
+        #   those, containing data to load into the table.
         # @param [String] format The exported file format. The default value is
         #   `csv`.
         #
@@ -1643,7 +1774,14 @@ module Google
         #   value is `0`. This property is useful if you have header rows in the
         #   file that should be skipped.
         #
-        # @return [Google::Cloud::Bigquery::LoadJob]
+        # @yield [updater] A block for setting the schema of the destination
+        #   table and other options for the load job. The schema can be omitted
+        #   if the destination table already exists, or if you're loading data
+        #   from a Google Cloud Datastore backup.
+        # @yieldparam [Google::Cloud::Bigquery::LoadJob::Updater] updater An
+        #   updater to modify the load job and its schema.
+        #
+        # @return [Boolean] Returns `true` if the load job was successful.
         #
         # @example
         #   require "google/cloud/bigquery"
@@ -1652,7 +1790,7 @@ module Google
         #   dataset = bigquery.dataset "my_dataset"
         #   table = dataset.table "my_table"
         #
-        #   load_job = table.load_job "gs://my-bucket/file-name.csv"
+        #   success = table.load "gs://my-bucket/file-name.csv"
         #
         # @example Pass a google-cloud-storage `File` instance:
         #   require "google/cloud/bigquery"
@@ -1665,7 +1803,20 @@ module Google
         #   storage = Google::Cloud::Storage.new
         #   bucket = storage.bucket "my-bucket"
         #   file = bucket.file "file-name.csv"
-        #   load_job = table.load_job file
+        #   success = table.load file
+        #
+        # @example Pass a list of google-cloud-storage files:
+        #   require "google/cloud/bigquery"
+        #   require "google/cloud/storage"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #   dataset = bigquery.dataset "my_dataset"
+        #   table = dataset.table "my_table"
+        #
+        #   storage = Google::Cloud::Storage.new
+        #   bucket = storage.bucket "my-bucket"
+        #   file = bucket.file "file-name.csv"
+        #   table.load [file, "gs://my-bucket/file-name2.csv"]
         #
         # @example Upload a file directly:
         #   require "google/cloud/bigquery"
@@ -1674,38 +1825,39 @@ module Google
         #   dataset = bigquery.dataset "my_dataset"
         #   table = dataset.table "my_table"
         #
-        #   file = File.open "my_data.csv"
-        #   load_job = table.load_job file
+        #   file = File.open "my_data.json"
+        #   success = table.load file do |j|
+        #     j.format = "newline_delimited_json"
+        #   end
         #
         # @!group Data
         #
-        def load file, format: nil, create: nil, write: nil,
+        def load files, format: nil, create: nil, write: nil,
                  projection_fields: nil, jagged_rows: nil, quoted_newlines: nil,
                  encoding: nil, delimiter: nil, ignore_unknown: nil,
                  max_bad_records: nil, quote: nil, skip_leading: nil,
                  autodetect: nil, null_marker: nil
-          job = load_job file, format: format, create: create, write: write,
-                               projection_fields: projection_fields,
-                               jagged_rows: jagged_rows,
-                               quoted_newlines: quoted_newlines,
-                               encoding: encoding, delimiter: delimiter,
-                               ignore_unknown: ignore_unknown,
-                               max_bad_records: max_bad_records, quote: quote,
-                               skip_leading: skip_leading,
-                               autodetect: autodetect, null_marker: null_marker
+          ensure_service!
 
+          updater = load_job_updater format: format, create: create,
+                                     write: write,
+                                     projection_fields: projection_fields,
+                                     jagged_rows: jagged_rows,
+                                     quoted_newlines: quoted_newlines,
+                                     encoding: encoding,
+                                     delimiter: delimiter,
+                                     ignore_unknown: ignore_unknown,
+                                     max_bad_records: max_bad_records,
+                                     quote: quote, skip_leading: skip_leading,
+                                     schema: schema,
+                                     autodetect: autodetect,
+                                     null_marker: null_marker
+
+          yield updater if block_given?
+
+          job = load_local_or_uri files, updater
           job.wait_until_done!
-
-          if job.failed?
-            begin
-              # raise to activate ruby exception cause handling
-              raise job.gapi_error
-            rescue StandardError => e
-              # wrap Google::Apis::Error with Google::Cloud::Error
-              raise Google::Cloud::Error.from_error(e)
-            end
-          end
-
+          ensure_job_succeeded! job
           true
         end
 
@@ -2049,29 +2201,157 @@ module Google
           reload!
         end
 
-        def load_storage url, options = {}
+        def ensure_job_succeeded! job
+          return unless job.failed?
+          begin
+            # raise to activate ruby exception cause handling
+            raise job.gapi_error
+          rescue StandardError => e
+            # wrap Google::Apis::Error with Google::Cloud::Error
+            raise Google::Cloud::Error.from_error(e)
+          end
+        end
+
+        def load_job_gapi table_id, dryrun, job_id: nil, prefix: nil
+          job_ref = service.job_ref_from job_id, prefix
+          Google::Apis::BigqueryV2::Job.new(
+            job_reference: job_ref,
+            configuration: Google::Apis::BigqueryV2::JobConfiguration.new(
+              load: Google::Apis::BigqueryV2::JobConfigurationLoad.new(
+                destination_table: Google::Apis::BigqueryV2::TableReference.new(
+                  project_id: @service.project,
+                  dataset_id: dataset_id,
+                  table_id: table_id
+                )
+              ),
+              dry_run: dryrun
+            )
+          )
+        end
+
+        def load_job_csv_options! job, jagged_rows: nil,
+                                  quoted_newlines: nil,
+                                  delimiter: nil,
+                                  quote: nil, skip_leading: nil,
+                                  null_marker: nil
+          job.jagged_rows = jagged_rows unless jagged_rows.nil?
+          job.quoted_newlines = quoted_newlines unless quoted_newlines.nil?
+          job.delimiter = delimiter unless delimiter.nil?
+          job.null_marker = null_marker unless null_marker.nil?
+          job.quote = quote unless quote.nil?
+          job.skip_leading = skip_leading unless skip_leading.nil?
+        end
+
+        def load_job_file_options! job, format: nil,
+                                   projection_fields: nil,
+                                   jagged_rows: nil, quoted_newlines: nil,
+                                   encoding: nil, delimiter: nil,
+                                   ignore_unknown: nil, max_bad_records: nil,
+                                   quote: nil, skip_leading: nil,
+                                   null_marker: nil
+          job.format = format unless format.nil?
+          unless projection_fields.nil?
+            job.projection_fields = projection_fields
+          end
+          job.encoding = encoding unless encoding.nil?
+          job.ignore_unknown = ignore_unknown unless ignore_unknown.nil?
+          job.max_bad_records = max_bad_records unless max_bad_records.nil?
+          load_job_csv_options! job, jagged_rows: jagged_rows,
+                                     quoted_newlines: quoted_newlines,
+                                     delimiter: delimiter,
+                                     quote: quote,
+                                     skip_leading: skip_leading,
+                                     null_marker: null_marker
+        end
+
+        def load_job_updater format: nil, create: nil,
+                             write: nil, projection_fields: nil,
+                             jagged_rows: nil, quoted_newlines: nil,
+                             encoding: nil, delimiter: nil,
+                             ignore_unknown: nil, max_bad_records: nil,
+                             quote: nil, skip_leading: nil, dryrun: nil,
+                             schema: nil, job_id: nil, prefix: nil, labels: nil,
+                             autodetect: nil, null_marker: nil
+          new_job = load_job_gapi table_id, dryrun, job_id: job_id,
+                                                    prefix: prefix
+          LoadJob::Updater.new(new_job).tap do |job|
+            job.location = location if location # may be table reference
+            job.create = create unless create.nil?
+            job.write = write unless write.nil?
+            job.schema = schema unless schema.nil?
+            job.autodetect = autodetect unless autodetect.nil?
+            job.labels = labels unless labels.nil?
+            load_job_file_options! job, format: format,
+                                        projection_fields: projection_fields,
+                                        jagged_rows: jagged_rows,
+                                        quoted_newlines: quoted_newlines,
+                                        encoding: encoding,
+                                        delimiter: delimiter,
+                                        ignore_unknown: ignore_unknown,
+                                        max_bad_records: max_bad_records,
+                                        quote: quote,
+                                        skip_leading: skip_leading,
+                                        null_marker: null_marker
+          end
+        end
+
+        def load_storage urls, job_gapi
           # Convert to storage URL
-          url = url.to_gs_url if url.respond_to? :to_gs_url
-          url = url.to_s if url.is_a? URI
+          urls = [urls].flatten.map do |url|
+            if url.respond_to? :to_gs_url
+              url.to_gs_url
+            elsif url.is_a? URI
+              url.to_s
+            else
+              url
+            end
+          end
+
+          unless urls.nil?
+            job_gapi.configuration.load.update! source_uris: urls
+            if job_gapi.configuration.load.source_format.nil?
+              source_format = Convert.derive_source_format_from_list urls
+              unless source_format.nil?
+                job_gapi.configuration.load.source_format = source_format
+              end
+            end
+          end
 
-          gapi = service.load_table_gs_url dataset_id, table_id, url, options
+          gapi = service.load_table_gs_url job_gapi
           Job.from_gapi gapi, service
         end
 
-        def load_local file, options = {}
-          # Convert to storage URL
-          file = file.to_gs_url if file.respond_to? :to_gs_url
+        def load_local file, job_gapi
+          path = Pathname(file).to_path
+          if job_gapi.configuration.load.source_format.nil?
+            source_format = Convert.derive_source_format path
+            unless source_format.nil?
+              job_gapi.configuration.load.source_format = source_format
+            end
+          end
 
-          gapi = service.load_table_file dataset_id, table_id, file, options
+          gapi = service.load_table_file file, job_gapi
           Job.from_gapi gapi, service
         end
 
-        def storage_url? file
-          file.respond_to?(:to_gs_url) ||
-            (file.respond_to?(:to_str) &&
-            file.to_str.downcase.start_with?("gs://")) ||
-            (file.is_a?(URI) &&
-            file.to_s.downcase.start_with?("gs://"))
+        def load_local_or_uri file, updater
+          job_gapi = updater.to_gapi
+          job = if local_file? file
+                  load_local file, job_gapi
+                else
+                  load_storage file, job_gapi
+                end
+          job
+        end
+
+        def storage_url? files
+          [files].flatten.all? do |file|
+            file.respond_to?(:to_gs_url) ||
+              (file.respond_to?(:to_str) &&
+                  file.to_str.downcase.start_with?("gs://")) ||
+              (file.is_a?(URI) &&
+                  file.to_s.downcase.start_with?("gs://"))
+          end
         end
 
         def local_file? file