google-cloud-bigquery 1.16.0 → 1.17.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/AUTHENTICATION.md +7 -27
- data/CHANGELOG.md +8 -0
- data/CONTRIBUTING.md +1 -1
- data/OVERVIEW.md +5 -5
- data/lib/google-cloud-bigquery.rb +5 -11
- data/lib/google/cloud/bigquery.rb +3 -5
- data/lib/google/cloud/bigquery/convert.rb +23 -46
- data/lib/google/cloud/bigquery/copy_job.rb +6 -16
- data/lib/google/cloud/bigquery/credentials.rb +5 -12
- data/lib/google/cloud/bigquery/data.rb +10 -16
- data/lib/google/cloud/bigquery/dataset.rb +58 -118
- data/lib/google/cloud/bigquery/dataset/access.rb +5 -13
- data/lib/google/cloud/bigquery/dataset/list.rb +4 -9
- data/lib/google/cloud/bigquery/external.rb +14 -35
- data/lib/google/cloud/bigquery/extract_job.rb +2 -5
- data/lib/google/cloud/bigquery/insert_response.rb +1 -3
- data/lib/google/cloud/bigquery/job.rb +5 -9
- data/lib/google/cloud/bigquery/job/list.rb +3 -7
- data/lib/google/cloud/bigquery/load_job.rb +18 -33
- data/lib/google/cloud/bigquery/model.rb +1 -4
- data/lib/google/cloud/bigquery/model/list.rb +3 -7
- data/lib/google/cloud/bigquery/project.rb +27 -56
- data/lib/google/cloud/bigquery/project/list.rb +3 -7
- data/lib/google/cloud/bigquery/query_job.rb +40 -79
- data/lib/google/cloud/bigquery/schema.rb +3 -8
- data/lib/google/cloud/bigquery/schema/field.rb +6 -11
- data/lib/google/cloud/bigquery/service.rb +26 -58
- data/lib/google/cloud/bigquery/table.rb +58 -112
- data/lib/google/cloud/bigquery/table/async_inserter.rb +10 -18
- data/lib/google/cloud/bigquery/table/list.rb +3 -7
- data/lib/google/cloud/bigquery/version.rb +1 -1
- metadata +36 -42
@@ -366,9 +366,7 @@ module Google
|
|
366
366
|
return nil if reference?
|
367
367
|
ensure_full_data!
|
368
368
|
return nil if @gapi.default_encryption_configuration.nil?
|
369
|
-
EncryptionConfiguration.from_gapi(
|
370
|
-
@gapi.default_encryption_configuration
|
371
|
-
).freeze
|
369
|
+
EncryptionConfiguration.from_gapi(@gapi.default_encryption_configuration).freeze
|
372
370
|
end
|
373
371
|
|
374
372
|
##
|
@@ -690,9 +688,7 @@ module Google
|
|
690
688
|
#
|
691
689
|
def table table_id, skip_lookup: nil
|
692
690
|
ensure_service!
|
693
|
-
if skip_lookup
|
694
|
-
return Table.new_reference project_id, dataset_id, table_id, service
|
695
|
-
end
|
691
|
+
return Table.new_reference project_id, dataset_id, table_id, service if skip_lookup
|
696
692
|
gapi = service.get_table dataset_id, table_id
|
697
693
|
Table.from_gapi gapi, service
|
698
694
|
rescue Google::Cloud::NotFoundError
|
@@ -774,9 +770,7 @@ module Google
|
|
774
770
|
#
|
775
771
|
def model model_id, skip_lookup: nil
|
776
772
|
ensure_service!
|
777
|
-
if skip_lookup
|
778
|
-
return Model.new_reference project_id, dataset_id, model_id, service
|
779
|
-
end
|
773
|
+
return Model.new_reference project_id, dataset_id, model_id, service if skip_lookup
|
780
774
|
gapi = service.get_model dataset_id, model_id
|
781
775
|
Model.from_gapi_json gapi, service
|
782
776
|
rescue Google::Cloud::NotFoundError
|
@@ -1083,20 +1077,15 @@ module Google
|
|
1083
1077
|
#
|
1084
1078
|
# @!group Data
|
1085
1079
|
#
|
1086
|
-
def query_job query, params: nil, external: nil,
|
1087
|
-
|
1088
|
-
|
1089
|
-
|
1090
|
-
maximum_billing_tier: nil, maximum_bytes_billed: nil,
|
1091
|
-
job_id: nil, prefix: nil, labels: nil, udfs: nil
|
1080
|
+
def query_job query, params: nil, external: nil, priority: "INTERACTIVE", cache: true, table: nil, create: nil,
|
1081
|
+
write: nil, dryrun: nil, standard_sql: nil, legacy_sql: nil, large_results: nil, flatten: nil,
|
1082
|
+
maximum_billing_tier: nil, maximum_bytes_billed: nil, job_id: nil, prefix: nil, labels: nil,
|
1083
|
+
udfs: nil
|
1092
1084
|
ensure_service!
|
1093
|
-
options = { priority: priority, cache: cache, table: table,
|
1094
|
-
|
1095
|
-
|
1096
|
-
|
1097
|
-
maximum_billing_tier: maximum_billing_tier,
|
1098
|
-
maximum_bytes_billed: maximum_bytes_billed,
|
1099
|
-
job_id: job_id, prefix: prefix, params: params,
|
1085
|
+
options = { priority: priority, cache: cache, table: table, create: create, write: write, dryrun: dryrun,
|
1086
|
+
large_results: large_results, flatten: flatten, legacy_sql: legacy_sql,
|
1087
|
+
standard_sql: standard_sql, maximum_billing_tier: maximum_billing_tier,
|
1088
|
+
maximum_bytes_billed: maximum_bytes_billed, job_id: job_id, prefix: prefix, params: params,
|
1100
1089
|
external: external, labels: labels, udfs: udfs }
|
1101
1090
|
|
1102
1091
|
updater = QueryJob::Updater.from_options service, query, options
|
@@ -1293,10 +1282,8 @@ module Google
|
|
1293
1282
|
#
|
1294
1283
|
# @!group Data
|
1295
1284
|
#
|
1296
|
-
def query query, params: nil, external: nil, max: nil, cache: true,
|
1297
|
-
|
1298
|
-
job = query_job query, params: params, external: external,
|
1299
|
-
cache: cache, standard_sql: standard_sql,
|
1285
|
+
def query query, params: nil, external: nil, max: nil, cache: true, standard_sql: nil, legacy_sql: nil, &block
|
1286
|
+
job = query_job query, params: params, external: external, cache: cache, standard_sql: standard_sql,
|
1300
1287
|
legacy_sql: legacy_sql, &block
|
1301
1288
|
job.wait_until_done!
|
1302
1289
|
ensure_job_succeeded! job
|
@@ -1593,29 +1580,19 @@ module Google
|
|
1593
1580
|
#
|
1594
1581
|
# @!group Data
|
1595
1582
|
#
|
1596
|
-
def load_job table_id, files, format: nil, create: nil, write: nil,
|
1597
|
-
|
1598
|
-
|
1599
|
-
|
1600
|
-
skip_leading: nil, schema: nil, job_id: nil, prefix: nil,
|
1601
|
-
labels: nil, autodetect: nil, null_marker: nil, dryrun: nil
|
1583
|
+
def load_job table_id, files, format: nil, create: nil, write: nil, projection_fields: nil, jagged_rows: nil,
|
1584
|
+
quoted_newlines: nil, encoding: nil, delimiter: nil, ignore_unknown: nil, max_bad_records: nil,
|
1585
|
+
quote: nil, skip_leading: nil, schema: nil, job_id: nil, prefix: nil, labels: nil, autodetect: nil,
|
1586
|
+
null_marker: nil, dryrun: nil
|
1602
1587
|
ensure_service!
|
1603
1588
|
|
1604
1589
|
updater = load_job_updater table_id,
|
1605
|
-
format: format, create: create,
|
1606
|
-
|
1607
|
-
|
1608
|
-
|
1609
|
-
|
1610
|
-
|
1611
|
-
delimiter: delimiter,
|
1612
|
-
ignore_unknown: ignore_unknown,
|
1613
|
-
max_bad_records: max_bad_records,
|
1614
|
-
quote: quote, skip_leading: skip_leading,
|
1615
|
-
dryrun: dryrun, schema: schema,
|
1616
|
-
job_id: job_id, prefix: prefix,
|
1617
|
-
labels: labels, autodetect: autodetect,
|
1618
|
-
null_marker: null_marker
|
1590
|
+
format: format, create: create, write: write, projection_fields: projection_fields,
|
1591
|
+
jagged_rows: jagged_rows, quoted_newlines: quoted_newlines, encoding: encoding,
|
1592
|
+
delimiter: delimiter, ignore_unknown: ignore_unknown,
|
1593
|
+
max_bad_records: max_bad_records, quote: quote, skip_leading: skip_leading,
|
1594
|
+
dryrun: dryrun, schema: schema, job_id: job_id, prefix: prefix, labels: labels,
|
1595
|
+
autodetect: autodetect, null_marker: null_marker
|
1619
1596
|
|
1620
1597
|
yield updater if block_given?
|
1621
1598
|
|
@@ -1831,21 +1808,14 @@ module Google
|
|
1831
1808
|
#
|
1832
1809
|
# @!group Data
|
1833
1810
|
#
|
1834
|
-
def load table_id, files, format: nil, create: nil, write: nil,
|
1835
|
-
|
1836
|
-
|
1837
|
-
max_bad_records: nil, quote: nil, skip_leading: nil,
|
1838
|
-
schema: nil, autodetect: nil, null_marker: nil, &block
|
1811
|
+
def load table_id, files, format: nil, create: nil, write: nil, projection_fields: nil, jagged_rows: nil,
|
1812
|
+
quoted_newlines: nil, encoding: nil, delimiter: nil, ignore_unknown: nil, max_bad_records: nil,
|
1813
|
+
quote: nil, skip_leading: nil, schema: nil, autodetect: nil, null_marker: nil, &block
|
1839
1814
|
job = load_job table_id, files,
|
1840
|
-
format: format, create: create, write: write,
|
1841
|
-
|
1842
|
-
|
1843
|
-
|
1844
|
-
encoding: encoding, delimiter: delimiter,
|
1845
|
-
ignore_unknown: ignore_unknown,
|
1846
|
-
max_bad_records: max_bad_records,
|
1847
|
-
quote: quote, skip_leading: skip_leading,
|
1848
|
-
schema: schema, autodetect: autodetect,
|
1815
|
+
format: format, create: create, write: write, projection_fields: projection_fields,
|
1816
|
+
jagged_rows: jagged_rows, quoted_newlines: quoted_newlines, encoding: encoding,
|
1817
|
+
delimiter: delimiter, ignore_unknown: ignore_unknown, max_bad_records: max_bad_records,
|
1818
|
+
quote: quote, skip_leading: skip_leading, schema: schema, autodetect: autodetect,
|
1849
1819
|
null_marker: null_marker, &block
|
1850
1820
|
|
1851
1821
|
job.wait_until_done!
|
@@ -2010,10 +1980,8 @@ module Google
|
|
2010
1980
|
def self.new_reference project_id, dataset_id, service
|
2011
1981
|
raise ArgumentError, "dataset_id is required" unless dataset_id
|
2012
1982
|
new.tap do |b|
|
2013
|
-
reference_gapi = Google::Apis::BigqueryV2::DatasetReference.new
|
2014
|
-
project_id: project_id,
|
2015
|
-
dataset_id: dataset_id
|
2016
|
-
)
|
1983
|
+
reference_gapi = Google::Apis::BigqueryV2::DatasetReference.new \
|
1984
|
+
project_id: project_id, dataset_id: dataset_id
|
2017
1985
|
b.service = service
|
2018
1986
|
b.instance_variable_set :@reference, reference_gapi
|
2019
1987
|
end
|
@@ -2100,19 +2068,17 @@ module Google
|
|
2100
2068
|
#
|
2101
2069
|
# @!group Data
|
2102
2070
|
#
|
2103
|
-
def insert table_id, rows, insert_ids: nil, skip_invalid: nil,
|
2104
|
-
ignore_unknown: nil, autocreate: nil
|
2071
|
+
def insert table_id, rows, insert_ids: nil, skip_invalid: nil, ignore_unknown: nil, autocreate: nil
|
2105
2072
|
rows = [rows] if rows.is_a? Hash
|
2106
2073
|
insert_ids = Array insert_ids
|
2107
|
-
if insert_ids.count
|
2074
|
+
if insert_ids.count.positive? && insert_ids.count != rows.count
|
2108
2075
|
raise ArgumentError, "insert_ids must be the same size as rows"
|
2109
2076
|
end
|
2110
2077
|
|
2111
2078
|
if autocreate
|
2112
2079
|
begin
|
2113
|
-
insert_data table_id, rows,
|
2114
|
-
|
2115
|
-
insert_ids: insert_ids
|
2080
|
+
insert_data table_id, rows,
|
2081
|
+
skip_invalid: skip_invalid, ignore_unknown: ignore_unknown, insert_ids: insert_ids
|
2116
2082
|
rescue Google::Cloud::NotFoundError
|
2117
2083
|
sleep rand(1..60)
|
2118
2084
|
begin
|
@@ -2125,15 +2091,12 @@ module Google
|
|
2125
2091
|
# rubocop:enable Lint/HandleExceptions
|
2126
2092
|
|
2127
2093
|
sleep 60
|
2128
|
-
insert table_id, rows, skip_invalid:
|
2129
|
-
|
2130
|
-
autocreate: true,
|
2131
|
-
insert_ids: insert_ids
|
2094
|
+
insert table_id, rows, skip_invalid: skip_invalid, ignore_unknown: ignore_unknown, autocreate: true,
|
2095
|
+
insert_ids: insert_ids
|
2132
2096
|
end
|
2133
2097
|
else
|
2134
|
-
insert_data table_id, rows, skip_invalid:
|
2135
|
-
|
2136
|
-
insert_ids: insert_ids
|
2098
|
+
insert_data table_id, rows, skip_invalid: skip_invalid, ignore_unknown: ignore_unknown,
|
2099
|
+
insert_ids: insert_ids
|
2137
2100
|
end
|
2138
2101
|
end
|
2139
2102
|
|
@@ -2184,9 +2147,8 @@ module Google
|
|
2184
2147
|
#
|
2185
2148
|
# inserter.stop.wait!
|
2186
2149
|
#
|
2187
|
-
def insert_async table_id, skip_invalid: nil, ignore_unknown: nil,
|
2188
|
-
|
2189
|
-
threads: 4, &block
|
2150
|
+
def insert_async table_id, skip_invalid: nil, ignore_unknown: nil, max_bytes: 10_000_000, max_rows: 500,
|
2151
|
+
interval: 10, threads: 4, &block
|
2190
2152
|
ensure_service!
|
2191
2153
|
|
2192
2154
|
# Get table, don't use Dataset#table which handles NotFoundError
|
@@ -2240,9 +2202,7 @@ module Google
|
|
2240
2202
|
def patch_gapi! *attributes
|
2241
2203
|
return if attributes.empty?
|
2242
2204
|
ensure_service!
|
2243
|
-
patch_args = Hash[attributes.map
|
2244
|
-
[attr, @gapi.send(attr)]
|
2245
|
-
end]
|
2205
|
+
patch_args = Hash[attributes.map { |attr| [attr, @gapi.send(attr)] }]
|
2246
2206
|
patch_gapi = Google::Apis::BigqueryV2::Dataset.new patch_args
|
2247
2207
|
patch_gapi.etag = etag if etag
|
2248
2208
|
@gapi = service.patch_dataset dataset_id, patch_gapi
|
@@ -2283,11 +2243,8 @@ module Google
|
|
2283
2243
|
)
|
2284
2244
|
end
|
2285
2245
|
|
2286
|
-
def load_job_csv_options! job, jagged_rows: nil,
|
2287
|
-
|
2288
|
-
delimiter: nil,
|
2289
|
-
quote: nil, skip_leading: nil,
|
2290
|
-
null_marker: nil
|
2246
|
+
def load_job_csv_options! job, jagged_rows: nil, quoted_newlines: nil, delimiter: nil, quote: nil,
|
2247
|
+
skip_leading: nil, null_marker: nil
|
2291
2248
|
job.jagged_rows = jagged_rows unless jagged_rows.nil?
|
2292
2249
|
job.quoted_newlines = quoted_newlines unless quoted_newlines.nil?
|
2293
2250
|
job.delimiter = delimiter unless delimiter.nil?
|
@@ -2296,17 +2253,11 @@ module Google
|
|
2296
2253
|
job.skip_leading = skip_leading unless skip_leading.nil?
|
2297
2254
|
end
|
2298
2255
|
|
2299
|
-
def load_job_file_options! job, format: nil,
|
2300
|
-
|
2301
|
-
|
2302
|
-
encoding: nil, delimiter: nil,
|
2303
|
-
ignore_unknown: nil, max_bad_records: nil,
|
2304
|
-
quote: nil, skip_leading: nil,
|
2305
|
-
null_marker: nil
|
2256
|
+
def load_job_file_options! job, format: nil, projection_fields: nil, jagged_rows: nil, quoted_newlines: nil,
|
2257
|
+
encoding: nil, delimiter: nil, ignore_unknown: nil, max_bad_records: nil, quote: nil,
|
2258
|
+
skip_leading: nil, null_marker: nil
|
2306
2259
|
job.format = format unless format.nil?
|
2307
|
-
unless projection_fields.nil?
|
2308
|
-
job.projection_fields = projection_fields
|
2309
|
-
end
|
2260
|
+
job.projection_fields = projection_fields unless projection_fields.nil?
|
2310
2261
|
job.encoding = encoding unless encoding.nil?
|
2311
2262
|
job.ignore_unknown = ignore_unknown unless ignore_unknown.nil?
|
2312
2263
|
job.max_bad_records = max_bad_records unless max_bad_records.nil?
|
@@ -2318,16 +2269,11 @@ module Google
|
|
2318
2269
|
null_marker: null_marker
|
2319
2270
|
end
|
2320
2271
|
|
2321
|
-
def load_job_updater table_id, format: nil, create: nil,
|
2322
|
-
|
2323
|
-
|
2324
|
-
|
2325
|
-
|
2326
|
-
quote: nil, skip_leading: nil, dryrun: nil,
|
2327
|
-
schema: nil, job_id: nil, prefix: nil, labels: nil,
|
2328
|
-
autodetect: nil, null_marker: nil
|
2329
|
-
new_job = load_job_gapi table_id, dryrun, job_id: job_id,
|
2330
|
-
prefix: prefix
|
2272
|
+
def load_job_updater table_id, format: nil, create: nil, write: nil, projection_fields: nil, jagged_rows: nil,
|
2273
|
+
quoted_newlines: nil, encoding: nil, delimiter: nil, ignore_unknown: nil,
|
2274
|
+
max_bad_records: nil, quote: nil, skip_leading: nil, dryrun: nil, schema: nil, job_id: nil,
|
2275
|
+
prefix: nil, labels: nil, autodetect: nil, null_marker: nil
|
2276
|
+
new_job = load_job_gapi table_id, dryrun, job_id: job_id, prefix: prefix
|
2331
2277
|
LoadJob::Updater.new(new_job).tap do |job|
|
2332
2278
|
job.location = location if location # may be dataset reference
|
2333
2279
|
job.create = create unless create.nil?
|
@@ -2365,9 +2311,7 @@ module Google
|
|
2365
2311
|
job_gapi.configuration.load.update! source_uris: urls
|
2366
2312
|
if job_gapi.configuration.load.source_format.nil?
|
2367
2313
|
source_format = Convert.derive_source_format_from_list urls
|
2368
|
-
unless source_format.nil?
|
2369
|
-
job_gapi.configuration.load.source_format = source_format
|
2370
|
-
end
|
2314
|
+
job_gapi.configuration.load.source_format = source_format unless source_format.nil?
|
2371
2315
|
end
|
2372
2316
|
end
|
2373
2317
|
|
@@ -2379,9 +2323,7 @@ module Google
|
|
2379
2323
|
path = Pathname(file).to_path
|
2380
2324
|
if job_gapi.configuration.load.source_format.nil?
|
2381
2325
|
source_format = Convert.derive_source_format path
|
2382
|
-
unless source_format.nil?
|
2383
|
-
job_gapi.configuration.load.source_format = source_format
|
2384
|
-
end
|
2326
|
+
job_gapi.configuration.load.source_format = source_format unless source_format.nil?
|
2385
2327
|
end
|
2386
2328
|
|
2387
2329
|
gapi = service.load_table_file file, job_gapi
|
@@ -2401,10 +2343,8 @@ module Google
|
|
2401
2343
|
def storage_url? files
|
2402
2344
|
[files].flatten.all? do |file|
|
2403
2345
|
file.respond_to?(:to_gs_url) ||
|
2404
|
-
(file.respond_to?(:to_str) &&
|
2405
|
-
|
2406
|
-
(file.is_a?(URI) &&
|
2407
|
-
file.to_s.downcase.start_with?("gs://"))
|
2346
|
+
(file.respond_to?(:to_str) && file.to_str.downcase.start_with?("gs://")) ||
|
2347
|
+
(file.is_a?(URI) && file.to_s.downcase.start_with?("gs://"))
|
2408
2348
|
end
|
2409
2349
|
end
|
2410
2350
|
|
@@ -885,10 +885,8 @@ module Google
|
|
885
885
|
rules = Array gapi.access
|
886
886
|
new.tap do |s|
|
887
887
|
s.instance_variable_set :@rules, rules
|
888
|
-
s.instance_variable_set :@original_rules_hashes,
|
889
|
-
|
890
|
-
s.instance_variable_set :@dataset_reference,
|
891
|
-
gapi.dataset_reference
|
888
|
+
s.instance_variable_set :@original_rules_hashes, rules.map(&:to_h)
|
889
|
+
s.instance_variable_set :@dataset_reference, gapi.dataset_reference
|
892
890
|
end
|
893
891
|
end
|
894
892
|
|
@@ -902,18 +900,14 @@ module Google
|
|
902
900
|
# @private
|
903
901
|
def validate_role role
|
904
902
|
good_role = ROLES[role.to_s]
|
905
|
-
if good_role.nil?
|
906
|
-
raise ArgumentError "Unable to determine role for #{role}"
|
907
|
-
end
|
903
|
+
raise ArgumentError "Unable to determine role for #{role}" if good_role.nil?
|
908
904
|
good_role
|
909
905
|
end
|
910
906
|
|
911
907
|
# @private
|
912
908
|
def validate_scope scope
|
913
909
|
good_scope = SCOPES[scope.to_s]
|
914
|
-
if good_scope.nil?
|
915
|
-
raise ArgumentError "Unable to determine scope for #{scope}"
|
916
|
-
end
|
910
|
+
raise ArgumentError "Unable to determine scope for #{scope}" if good_scope.nil?
|
917
911
|
good_scope
|
918
912
|
end
|
919
913
|
|
@@ -984,9 +978,7 @@ module Google
|
|
984
978
|
# If scope is special group, make sure value is in the list
|
985
979
|
value = validate_special_group value if scope == :special_group
|
986
980
|
# Detect any rules of this role, scope, and value
|
987
|
-
!(!@rules.detect(
|
988
|
-
&find_by_role_and_scope_and_value(role, scope, value)
|
989
|
-
))
|
981
|
+
!(!@rules.detect(&find_by_role_and_scope_and_value(role, scope, value)))
|
990
982
|
end
|
991
983
|
|
992
984
|
# @private
|
@@ -123,15 +123,13 @@ module Google
|
|
123
123
|
#
|
124
124
|
def all request_limit: nil
|
125
125
|
request_limit = request_limit.to_i if request_limit
|
126
|
-
unless block_given?
|
127
|
-
return enum_for :all, request_limit: request_limit
|
128
|
-
end
|
126
|
+
return enum_for :all, request_limit: request_limit unless block_given?
|
129
127
|
results = self
|
130
128
|
loop do
|
131
129
|
results.each { |r| yield r }
|
132
130
|
if request_limit
|
133
131
|
request_limit -= 1
|
134
|
-
break if request_limit
|
132
|
+
break if request_limit.negative?
|
135
133
|
end
|
136
134
|
break unless results.next?
|
137
135
|
results = results.next
|
@@ -140,11 +138,8 @@ module Google
|
|
140
138
|
|
141
139
|
##
|
142
140
|
# @private New Dataset::List from a response object.
|
143
|
-
def self.from_gapi gapi_list, service, hidden = nil, filter = nil,
|
144
|
-
|
145
|
-
datasets = List.new(Array(gapi_list.datasets).map do |gapi_object|
|
146
|
-
Dataset.from_gapi gapi_object, service
|
147
|
-
end)
|
141
|
+
def self.from_gapi gapi_list, service, hidden = nil, filter = nil, max = nil
|
142
|
+
datasets = List.new(Array(gapi_list.datasets).map { |gapi_object| Dataset.from_gapi gapi_object, service })
|
148
143
|
datasets.instance_variable_set :@token, gapi_list.next_page_token
|
149
144
|
datasets.instance_variable_set :@etag, gapi_list.etag
|
150
145
|
datasets.instance_variable_set :@service, service
|
@@ -54,9 +54,7 @@ module Google
|
|
54
54
|
# @private New External from URLs and format
|
55
55
|
def self.from_urls urls, format = nil
|
56
56
|
external_format = source_format_for urls, format
|
57
|
-
if external_format.nil?
|
58
|
-
raise ArgumentError, "Unable to determine external table format"
|
59
|
-
end
|
57
|
+
raise ArgumentError, "Unable to determine external table format" if external_format.nil?
|
60
58
|
external_class = table_class_for external_format
|
61
59
|
external_class.new.tap do |e|
|
62
60
|
e.gapi.source_uris = Array(urls)
|
@@ -69,9 +67,7 @@ module Google
|
|
69
67
|
def self.from_gapi gapi
|
70
68
|
external_format = source_format_for gapi.source_uris,
|
71
69
|
gapi.source_format
|
72
|
-
if external_format.nil?
|
73
|
-
raise ArgumentError, "Unable to determine external table format"
|
74
|
-
end
|
70
|
+
raise ArgumentError, "Unable to determine external table format" if external_format.nil?
|
75
71
|
external_class = table_class_for external_format
|
76
72
|
external_class.from_gapi gapi
|
77
73
|
end
|
@@ -96,12 +92,8 @@ module Google
|
|
96
92
|
return "NEWLINE_DELIMITED_JSON" if url.end_with? ".json"
|
97
93
|
return "AVRO" if url.end_with? ".avro"
|
98
94
|
return "DATASTORE_BACKUP" if url.end_with? ".backup_info"
|
99
|
-
if url.start_with? "https://docs.google.com/spreadsheets/"
|
100
|
-
|
101
|
-
end
|
102
|
-
if url.start_with? "https://googleapis.com/bigtable/projects/"
|
103
|
-
return "BIGTABLE"
|
104
|
-
end
|
95
|
+
return "GOOGLE_SHEETS" if url.start_with? "https://docs.google.com/spreadsheets/"
|
96
|
+
return "BIGTABLE" if url.start_with? "https://googleapis.com/bigtable/projects/"
|
105
97
|
end
|
106
98
|
nil
|
107
99
|
end
|
@@ -556,8 +548,7 @@ module Google
|
|
556
548
|
|
557
549
|
def frozen_check!
|
558
550
|
return unless frozen?
|
559
|
-
raise ArgumentError,
|
560
|
-
"Cannot modify external data source when frozen"
|
551
|
+
raise ArgumentError, "Cannot modify external data source when frozen"
|
561
552
|
end
|
562
553
|
end
|
563
554
|
|
@@ -1163,8 +1154,7 @@ module Google
|
|
1163
1154
|
# @private Create an empty SheetsSource object.
|
1164
1155
|
def initialize
|
1165
1156
|
super
|
1166
|
-
@gapi.google_sheets_options =
|
1167
|
-
Google::Apis::BigqueryV2::GoogleSheetsOptions.new
|
1157
|
+
@gapi.google_sheets_options = Google::Apis::BigqueryV2::GoogleSheetsOptions.new
|
1168
1158
|
end
|
1169
1159
|
|
1170
1160
|
##
|
@@ -1309,8 +1299,7 @@ module Google
|
|
1309
1299
|
# @private Create an empty BigtableSource object.
|
1310
1300
|
def initialize
|
1311
1301
|
super
|
1312
|
-
@gapi.bigtable_options =
|
1313
|
-
Google::Apis::BigqueryV2::BigtableOptions.new
|
1302
|
+
@gapi.bigtable_options = Google::Apis::BigqueryV2::BigtableOptions.new
|
1314
1303
|
@families = []
|
1315
1304
|
end
|
1316
1305
|
|
@@ -1454,9 +1443,7 @@ module Google
|
|
1454
1443
|
def self.from_gapi gapi
|
1455
1444
|
new_table = super
|
1456
1445
|
families = Array gapi.bigtable_options.column_families
|
1457
|
-
families = families.map
|
1458
|
-
BigtableSource::ColumnFamily.from_gapi fam_gapi
|
1459
|
-
end
|
1446
|
+
families = families.map { |fam_gapi| BigtableSource::ColumnFamily.from_gapi fam_gapi }
|
1460
1447
|
new_table.instance_variable_set :@families, families
|
1461
1448
|
new_table
|
1462
1449
|
end
|
@@ -1473,8 +1460,7 @@ module Google
|
|
1473
1460
|
|
1474
1461
|
def frozen_check!
|
1475
1462
|
return unless frozen?
|
1476
|
-
raise ArgumentError,
|
1477
|
-
"Cannot modify external data source when frozen"
|
1463
|
+
raise ArgumentError, "Cannot modify external data source when frozen"
|
1478
1464
|
end
|
1479
1465
|
|
1480
1466
|
##
|
@@ -1993,9 +1979,7 @@ module Google
|
|
1993
1979
|
def self.from_gapi gapi
|
1994
1980
|
new_fam = new
|
1995
1981
|
new_fam.instance_variable_set :@gapi, gapi
|
1996
|
-
columns = Array(gapi.columns).map
|
1997
|
-
BigtableSource::Column.from_gapi col_gapi
|
1998
|
-
end
|
1982
|
+
columns = Array(gapi.columns).map { |col_gapi| BigtableSource::Column.from_gapi col_gapi }
|
1999
1983
|
new_fam.instance_variable_set :@columns, columns
|
2000
1984
|
new_fam
|
2001
1985
|
end
|
@@ -2012,8 +1996,7 @@ module Google
|
|
2012
1996
|
|
2013
1997
|
def frozen_check!
|
2014
1998
|
return unless frozen?
|
2015
|
-
raise ArgumentError,
|
2016
|
-
"Cannot modify external data source when frozen"
|
1999
|
+
raise ArgumentError, "Cannot modify external data source when frozen"
|
2017
2000
|
end
|
2018
2001
|
end
|
2019
2002
|
|
@@ -2082,8 +2065,7 @@ module Google
|
|
2082
2065
|
# end
|
2083
2066
|
#
|
2084
2067
|
def qualifier
|
2085
|
-
@gapi.qualifier_string ||
|
2086
|
-
Base64.strict_decode64(@gapi.qualifier_encoded.to_s)
|
2068
|
+
@gapi.qualifier_string || Base64.strict_decode64(@gapi.qualifier_encoded.to_s)
|
2087
2069
|
end
|
2088
2070
|
|
2089
2071
|
##
|
@@ -2128,9 +2110,7 @@ module Google
|
|
2128
2110
|
end
|
2129
2111
|
rescue EncodingError
|
2130
2112
|
@gapi.qualifier_encoded = Base64.strict_encode64 new_qualifier
|
2131
|
-
if @gapi.instance_variables.include? :@qualifier_string
|
2132
|
-
@gapi.remove_instance_variable :@qualifier_string
|
2133
|
-
end
|
2113
|
+
@gapi.remove_instance_variable :@qualifier_string if @gapi.instance_variables.include? :@qualifier_string
|
2134
2114
|
end
|
2135
2115
|
|
2136
2116
|
##
|
@@ -2390,8 +2370,7 @@ module Google
|
|
2390
2370
|
|
2391
2371
|
def frozen_check!
|
2392
2372
|
return unless frozen?
|
2393
|
-
raise ArgumentError,
|
2394
|
-
"Cannot modify external data source when frozen"
|
2373
|
+
raise ArgumentError, "Cannot modify external data source when frozen"
|
2395
2374
|
end
|
2396
2375
|
end
|
2397
2376
|
end
|