google-cloud-bigquery 1.27.0 → 1.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +58 -0
- data/CONTRIBUTING.md +3 -4
- data/LOGGING.md +1 -1
- data/OVERVIEW.md +15 -14
- data/lib/google/cloud/bigquery/convert.rb +72 -76
- data/lib/google/cloud/bigquery/copy_job.rb +1 -0
- data/lib/google/cloud/bigquery/data.rb +2 -2
- data/lib/google/cloud/bigquery/dataset.rb +181 -62
- data/lib/google/cloud/bigquery/dataset/access.rb +3 -3
- data/lib/google/cloud/bigquery/dataset/list.rb +2 -2
- data/lib/google/cloud/bigquery/external.rb +9 -2619
- data/lib/google/cloud/bigquery/external/bigtable_source.rb +230 -0
- data/lib/google/cloud/bigquery/external/bigtable_source/column.rb +404 -0
- data/lib/google/cloud/bigquery/external/bigtable_source/column_family.rb +945 -0
- data/lib/google/cloud/bigquery/external/csv_source.rb +481 -0
- data/lib/google/cloud/bigquery/external/data_source.rb +771 -0
- data/lib/google/cloud/bigquery/external/json_source.rb +170 -0
- data/lib/google/cloud/bigquery/external/parquet_source.rb +148 -0
- data/lib/google/cloud/bigquery/external/sheets_source.rb +166 -0
- data/lib/google/cloud/bigquery/extract_job.rb +4 -2
- data/lib/google/cloud/bigquery/job.rb +9 -3
- data/lib/google/cloud/bigquery/job/list.rb +4 -4
- data/lib/google/cloud/bigquery/load_job.rb +178 -19
- data/lib/google/cloud/bigquery/model/list.rb +2 -2
- data/lib/google/cloud/bigquery/policy.rb +2 -1
- data/lib/google/cloud/bigquery/project.rb +47 -43
- data/lib/google/cloud/bigquery/project/list.rb +2 -2
- data/lib/google/cloud/bigquery/query_job.rb +84 -62
- data/lib/google/cloud/bigquery/routine.rb +1 -4
- data/lib/google/cloud/bigquery/routine/list.rb +2 -2
- data/lib/google/cloud/bigquery/schema.rb +39 -3
- data/lib/google/cloud/bigquery/schema/field.rb +63 -13
- data/lib/google/cloud/bigquery/service.rb +11 -13
- data/lib/google/cloud/bigquery/standard_sql.rb +15 -3
- data/lib/google/cloud/bigquery/table.rb +312 -69
- data/lib/google/cloud/bigquery/table/async_inserter.rb +44 -17
- data/lib/google/cloud/bigquery/table/list.rb +2 -2
- data/lib/google/cloud/bigquery/version.rb +1 -1
- metadata +28 -14
|
@@ -483,7 +483,7 @@ module Google
|
|
|
483
483
|
table_id: m["tbl"]
|
|
484
484
|
}.delete_if { |_, v| v.nil? }
|
|
485
485
|
str_table_ref_hash = default_ref.to_h.merge str_table_ref_hash
|
|
486
|
-
ref = Google::Apis::BigqueryV2::TableReference.new
|
|
486
|
+
ref = Google::Apis::BigqueryV2::TableReference.new(**str_table_ref_hash)
|
|
487
487
|
validate_table_ref ref
|
|
488
488
|
ref
|
|
489
489
|
end
|
|
@@ -554,9 +554,9 @@ module Google
|
|
|
554
554
|
nil
|
|
555
555
|
end
|
|
556
556
|
|
|
557
|
-
def execute backoff: nil
|
|
557
|
+
def execute backoff: nil, &block
|
|
558
558
|
if backoff
|
|
559
|
-
Backoff.new(retries: retries).execute
|
|
559
|
+
Backoff.new(retries: retries).execute(&block)
|
|
560
560
|
else
|
|
561
561
|
yield
|
|
562
562
|
end
|
|
@@ -590,22 +590,20 @@ module Google
|
|
|
590
590
|
def execute
|
|
591
591
|
current_retries = 0
|
|
592
592
|
loop do
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
current_retries += 1
|
|
600
|
-
end
|
|
593
|
+
return yield
|
|
594
|
+
rescue Google::Apis::Error => e
|
|
595
|
+
raise e unless retry? e.body, current_retries
|
|
596
|
+
|
|
597
|
+
@backoff.call current_retries
|
|
598
|
+
current_retries += 1
|
|
601
599
|
end
|
|
602
600
|
end
|
|
603
601
|
|
|
604
602
|
protected
|
|
605
603
|
|
|
606
604
|
def retry? result, current_retries #:nodoc:
|
|
607
|
-
if current_retries < @retries
|
|
608
|
-
return true
|
|
605
|
+
if current_retries < @retries && retry_error_reason?(result)
|
|
606
|
+
return true
|
|
609
607
|
end
|
|
610
608
|
false
|
|
611
609
|
end
|
|
@@ -273,6 +273,17 @@ module Google
|
|
|
273
273
|
type_kind == "NUMERIC".freeze
|
|
274
274
|
end
|
|
275
275
|
|
|
276
|
+
##
|
|
277
|
+
# Checks if the {#type_kind} of the field is `BIGNUMERIC`.
|
|
278
|
+
#
|
|
279
|
+
# @return [Boolean] `true` when `BIGNUMERIC`, `false` otherwise.
|
|
280
|
+
#
|
|
281
|
+
# @!group Helpers
|
|
282
|
+
#
|
|
283
|
+
def bignumeric?
|
|
284
|
+
type_kind == "BIGNUMERIC".freeze
|
|
285
|
+
end
|
|
286
|
+
|
|
276
287
|
##
|
|
277
288
|
# Checks if the {#type_kind} of the field is `BOOL`.
|
|
278
289
|
#
|
|
@@ -401,11 +412,12 @@ module Google
|
|
|
401
412
|
# @private New Google::Apis::BigqueryV2::StandardSqlDataType from a String or StandardSql::DataType object.
|
|
402
413
|
def self.gapi_from_string_or_data_type data_type
|
|
403
414
|
return if data_type.nil?
|
|
404
|
-
|
|
415
|
+
case data_type
|
|
416
|
+
when StandardSql::DataType
|
|
405
417
|
data_type.to_gapi
|
|
406
|
-
|
|
418
|
+
when Hash
|
|
407
419
|
data_type
|
|
408
|
-
|
|
420
|
+
when String, Symbol
|
|
409
421
|
Google::Apis::BigqueryV2::StandardSqlDataType.new type_kind: data_type.to_s.upcase
|
|
410
422
|
else
|
|
411
423
|
raise ArgumentError, "Unable to convert #{data_type} to Google::Apis::BigqueryV2::StandardSqlDataType"
|
|
@@ -37,16 +37,16 @@ module Google
|
|
|
37
37
|
# repeated fields.
|
|
38
38
|
#
|
|
39
39
|
# The Table class can also represent a
|
|
40
|
-
# [view](https://cloud.google.com/bigquery/docs/views), which is a virtual
|
|
41
|
-
# table defined by a SQL query
|
|
42
|
-
# materialized views,
|
|
43
|
-
#
|
|
44
|
-
#
|
|
45
|
-
# indirectly by the top-level query. (See {#view?}, {#query}, {#query=},
|
|
46
|
-
# and {Dataset#create_view}.)
|
|
40
|
+
# [logical view](https://cloud.google.com/bigquery/docs/views), which is a virtual
|
|
41
|
+
# table defined by a SQL query (see {#view?} and {Dataset#create_view}); or a
|
|
42
|
+
# [materialized view](https://cloud.google.com/bigquery/docs/materialized-views-intro),
|
|
43
|
+
# which is a precomputed view that periodically caches results of a query for increased
|
|
44
|
+
# performance and efficiency (see {#materialized_view?} and {Dataset#create_materialized_view}).
|
|
47
45
|
#
|
|
48
46
|
# @see https://cloud.google.com/bigquery/docs/loading-data#loading_denormalized_nested_and_repeated_data
|
|
49
47
|
# Loading denormalized, nested, and repeated data
|
|
48
|
+
# @see https://cloud.google.com/bigquery/docs/views Creating views
|
|
49
|
+
# @see https://cloud.google.com/bigquery/docs/materialized-views-intro Introduction to materialized views
|
|
50
50
|
#
|
|
51
51
|
# @example
|
|
52
52
|
# require "google/cloud/bigquery"
|
|
@@ -77,7 +77,7 @@ module Google
|
|
|
77
77
|
# }
|
|
78
78
|
# table.insert row
|
|
79
79
|
#
|
|
80
|
-
# @example Creating a
|
|
80
|
+
# @example Creating a logical view:
|
|
81
81
|
# require "google/cloud/bigquery"
|
|
82
82
|
#
|
|
83
83
|
# bigquery = Google::Cloud::Bigquery.new
|
|
@@ -86,6 +86,15 @@ module Google
|
|
|
86
86
|
# "SELECT name, age FROM `my_project.my_dataset.my_table`"
|
|
87
87
|
# view.view? # true
|
|
88
88
|
#
|
|
89
|
+
# @example Creating a materialized view:
|
|
90
|
+
# require "google/cloud/bigquery"
|
|
91
|
+
#
|
|
92
|
+
# bigquery = Google::Cloud::Bigquery.new
|
|
93
|
+
# dataset = bigquery.dataset "my_dataset"
|
|
94
|
+
# view = dataset.create_materialized_view "my_materialized_view",
|
|
95
|
+
# "SELECT name, age FROM `my_project.my_dataset.my_table`"
|
|
96
|
+
# view.materialized_view? # true
|
|
97
|
+
#
|
|
89
98
|
class Table
|
|
90
99
|
##
|
|
91
100
|
# @private The Service object.
|
|
@@ -386,8 +395,8 @@ module Google
|
|
|
386
395
|
# {#resource_full?}), the full representation will be retrieved before
|
|
387
396
|
# the update to comply with ETag-based optimistic concurrency control.
|
|
388
397
|
#
|
|
389
|
-
# @param [Integer] expiration An expiration time, in seconds,
|
|
390
|
-
# for data in time partitions.
|
|
398
|
+
# @param [Integer, nil] expiration An expiration time, in seconds,
|
|
399
|
+
# for data in time partitions, , or `nil` to indicate no expiration time.
|
|
391
400
|
#
|
|
392
401
|
# @example
|
|
393
402
|
# require "google/cloud/bigquery"
|
|
@@ -407,8 +416,9 @@ module Google
|
|
|
407
416
|
#
|
|
408
417
|
def time_partitioning_expiration= expiration
|
|
409
418
|
reload! unless resource_full?
|
|
419
|
+
expiration_ms = expiration * 1000 if expiration
|
|
410
420
|
@gapi.time_partitioning ||= Google::Apis::BigqueryV2::TimePartitioning.new
|
|
411
|
-
@gapi.time_partitioning.expiration_ms =
|
|
421
|
+
@gapi.time_partitioning.expiration_ms = expiration_ms
|
|
412
422
|
patch_gapi! :time_partitioning
|
|
413
423
|
end
|
|
414
424
|
|
|
@@ -461,8 +471,13 @@ module Google
|
|
|
461
471
|
###
|
|
462
472
|
# Checks if the table is clustered.
|
|
463
473
|
#
|
|
474
|
+
# See {Table::Updater#clustering_fields=}, {Table#clustering_fields} and
|
|
475
|
+
# {Table#clustering_fields=}.
|
|
476
|
+
#
|
|
464
477
|
# @see https://cloud.google.com/bigquery/docs/clustered-tables
|
|
465
|
-
# Introduction to
|
|
478
|
+
# Introduction to clustered tables
|
|
479
|
+
# @see https://cloud.google.com/bigquery/docs/creating-clustered-tables
|
|
480
|
+
# Creating and using clustered tables
|
|
466
481
|
#
|
|
467
482
|
# @return [Boolean, nil] `true` when the table is clustered, or
|
|
468
483
|
# `false` otherwise, if the object is a resource (see {#resource?});
|
|
@@ -481,14 +496,16 @@ module Google
|
|
|
481
496
|
# first partitioned and subsequently clustered. The order of the
|
|
482
497
|
# returned fields determines the sort order of the data.
|
|
483
498
|
#
|
|
484
|
-
#
|
|
499
|
+
# BigQuery supports clustering for both partitioned and non-partitioned
|
|
500
|
+
# tables.
|
|
501
|
+
#
|
|
502
|
+
# See {Table::Updater#clustering_fields=}, {Table#clustering_fields=} and
|
|
503
|
+
# {Table#clustering?}.
|
|
485
504
|
#
|
|
486
|
-
# @see https://cloud.google.com/bigquery/docs/partitioned-tables
|
|
487
|
-
# Partitioned Tables
|
|
488
505
|
# @see https://cloud.google.com/bigquery/docs/clustered-tables
|
|
489
|
-
# Introduction to
|
|
506
|
+
# Introduction to clustered tables
|
|
490
507
|
# @see https://cloud.google.com/bigquery/docs/creating-clustered-tables
|
|
491
|
-
# Creating and
|
|
508
|
+
# Creating and using clustered tables
|
|
492
509
|
#
|
|
493
510
|
# @return [Array<String>, nil] The clustering fields, or `nil` if the
|
|
494
511
|
# table is not clustered or if the table is a reference (see
|
|
@@ -502,6 +519,53 @@ module Google
|
|
|
502
519
|
@gapi.clustering.fields if clustering?
|
|
503
520
|
end
|
|
504
521
|
|
|
522
|
+
##
|
|
523
|
+
# Updates the list of fields on which data should be clustered.
|
|
524
|
+
#
|
|
525
|
+
# Only top-level, non-repeated, simple-type fields are supported. When
|
|
526
|
+
# you cluster a table using multiple columns, the order of columns you
|
|
527
|
+
# specify is important. The order of the specified columns determines
|
|
528
|
+
# the sort order of the data.
|
|
529
|
+
#
|
|
530
|
+
# BigQuery supports clustering for both partitioned and non-partitioned
|
|
531
|
+
# tables.
|
|
532
|
+
#
|
|
533
|
+
# See {Table::Updater#clustering_fields=}, {Table#clustering_fields} and
|
|
534
|
+
# {Table#clustering?}.
|
|
535
|
+
#
|
|
536
|
+
# @see https://cloud.google.com/bigquery/docs/clustered-tables
|
|
537
|
+
# Introduction to clustered tables
|
|
538
|
+
# @see https://cloud.google.com/bigquery/docs/creating-clustered-tables
|
|
539
|
+
# Creating and using clustered tables
|
|
540
|
+
# @see https://cloud.google.com/bigquery/docs/creating-clustered-tables#modifying-cluster-spec
|
|
541
|
+
# Modifying clustering specification
|
|
542
|
+
#
|
|
543
|
+
# @param [Array<String>, nil] fields The clustering fields, or `nil` to
|
|
544
|
+
# remove the clustering configuration. Only top-level, non-repeated,
|
|
545
|
+
# simple-type fields are supported.
|
|
546
|
+
#
|
|
547
|
+
# @example
|
|
548
|
+
# require "google/cloud/bigquery"
|
|
549
|
+
#
|
|
550
|
+
# bigquery = Google::Cloud::Bigquery.new
|
|
551
|
+
# dataset = bigquery.dataset "my_dataset"
|
|
552
|
+
# table = dataset.table "my_table"
|
|
553
|
+
#
|
|
554
|
+
# table.clustering_fields = ["last_name", "first_name"]
|
|
555
|
+
#
|
|
556
|
+
# @!group Attributes
|
|
557
|
+
#
|
|
558
|
+
def clustering_fields= fields
|
|
559
|
+
reload! unless resource_full?
|
|
560
|
+
if fields
|
|
561
|
+
@gapi.clustering ||= Google::Apis::BigqueryV2::Clustering.new
|
|
562
|
+
@gapi.clustering.fields = fields
|
|
563
|
+
else
|
|
564
|
+
@gapi.clustering = nil
|
|
565
|
+
end
|
|
566
|
+
patch_gapi! :clustering
|
|
567
|
+
end
|
|
568
|
+
|
|
505
569
|
##
|
|
506
570
|
# The combined Project ID, Dataset ID, and Table ID for this table, in
|
|
507
571
|
# the format specified by the [Legacy SQL Query
|
|
@@ -726,7 +790,7 @@ module Google
|
|
|
726
790
|
end
|
|
727
791
|
|
|
728
792
|
##
|
|
729
|
-
# Checks if the table's type is
|
|
793
|
+
# Checks if the table's type is `TABLE`.
|
|
730
794
|
#
|
|
731
795
|
# @return [Boolean, nil] `true` when the type is `TABLE`, `false`
|
|
732
796
|
# otherwise, if the object is a resource (see {#resource?}); `nil` if
|
|
@@ -740,8 +804,10 @@ module Google
|
|
|
740
804
|
end
|
|
741
805
|
|
|
742
806
|
##
|
|
743
|
-
# Checks if the table's type is
|
|
744
|
-
# represents a BigQuery view. See {Dataset#create_view}.
|
|
807
|
+
# Checks if the table's type is `VIEW`, indicating that the table
|
|
808
|
+
# represents a BigQuery logical view. See {Dataset#create_view}.
|
|
809
|
+
#
|
|
810
|
+
# @see https://cloud.google.com/bigquery/docs/views Creating views
|
|
745
811
|
#
|
|
746
812
|
# @return [Boolean, nil] `true` when the type is `VIEW`, `false`
|
|
747
813
|
# otherwise, if the object is a resource (see {#resource?}); `nil` if
|
|
@@ -755,7 +821,25 @@ module Google
|
|
|
755
821
|
end
|
|
756
822
|
|
|
757
823
|
##
|
|
758
|
-
# Checks if the table's type is
|
|
824
|
+
# Checks if the table's type is `MATERIALIZED_VIEW`, indicating that
|
|
825
|
+
# the table represents a BigQuery materialized view.
|
|
826
|
+
# See {Dataset#create_materialized_view}.
|
|
827
|
+
#
|
|
828
|
+
# @see https://cloud.google.com/bigquery/docs/materialized-views-intro Introduction to materialized views
|
|
829
|
+
#
|
|
830
|
+
# @return [Boolean, nil] `true` when the type is `MATERIALIZED_VIEW`,
|
|
831
|
+
# `false` otherwise, if the object is a resource (see {#resource?});
|
|
832
|
+
# `nil` if the object is a reference (see {#reference?}).
|
|
833
|
+
#
|
|
834
|
+
# @!group Attributes
|
|
835
|
+
#
|
|
836
|
+
def materialized_view?
|
|
837
|
+
return nil if reference?
|
|
838
|
+
@gapi.type == "MATERIALIZED_VIEW"
|
|
839
|
+
end
|
|
840
|
+
|
|
841
|
+
##
|
|
842
|
+
# Checks if the table's type is `EXTERNAL`, indicating that the table
|
|
759
843
|
# represents an External Data Source. See {#external?} and
|
|
760
844
|
# {External::DataSource}.
|
|
761
845
|
#
|
|
@@ -1138,21 +1222,24 @@ module Google
|
|
|
1138
1222
|
end
|
|
1139
1223
|
|
|
1140
1224
|
##
|
|
1141
|
-
# The query that
|
|
1225
|
+
# The query that defines the view or materialized view. See {#view?} and
|
|
1226
|
+
# {#materialized_view?}.
|
|
1142
1227
|
#
|
|
1143
|
-
# @return [String] The query that defines the view
|
|
1228
|
+
# @return [String, nil] The query that defines the view or materialized_view;
|
|
1229
|
+
# or `nil` if not a view or materialized view.
|
|
1144
1230
|
#
|
|
1145
1231
|
# @!group Attributes
|
|
1146
1232
|
#
|
|
1147
1233
|
def query
|
|
1148
|
-
@gapi.view&.query
|
|
1234
|
+
view? ? @gapi.view&.query : @gapi.materialized_view&.query
|
|
1149
1235
|
end
|
|
1150
1236
|
|
|
1151
1237
|
##
|
|
1152
|
-
# Updates the query that
|
|
1238
|
+
# Updates the query that defines the view. (See {#view?}.) Not supported
|
|
1239
|
+
# for materialized views.
|
|
1153
1240
|
#
|
|
1154
|
-
# This sets the query using standard SQL. To specify legacy SQL or
|
|
1155
|
-
# use user-defined function resources use (#set_query) instead.
|
|
1241
|
+
# This method sets the query using standard SQL. To specify legacy SQL or
|
|
1242
|
+
# to use user-defined function resources for a view, use (#set_query) instead.
|
|
1156
1243
|
#
|
|
1157
1244
|
# @see https://cloud.google.com/bigquery/query-reference BigQuery Query
|
|
1158
1245
|
# Reference
|
|
@@ -1167,7 +1254,7 @@ module Google
|
|
|
1167
1254
|
# view = dataset.table "my_view"
|
|
1168
1255
|
#
|
|
1169
1256
|
# view.query = "SELECT first_name FROM " \
|
|
1170
|
-
#
|
|
1257
|
+
# "`my_project.my_dataset.my_table`"
|
|
1171
1258
|
#
|
|
1172
1259
|
# @!group Lifecycle
|
|
1173
1260
|
#
|
|
@@ -1176,12 +1263,12 @@ module Google
|
|
|
1176
1263
|
end
|
|
1177
1264
|
|
|
1178
1265
|
##
|
|
1179
|
-
# Updates the query that
|
|
1180
|
-
#
|
|
1181
|
-
# resources.
|
|
1266
|
+
# Updates the query that defines the view. (See {#view?}.) Not supported for
|
|
1267
|
+
# materialized views.
|
|
1182
1268
|
#
|
|
1183
|
-
#
|
|
1184
|
-
#
|
|
1269
|
+
# Allows setting of standard vs. legacy SQL and user-defined function resources.
|
|
1270
|
+
#
|
|
1271
|
+
# @see https://cloud.google.com/bigquery/query-reference BigQuery Query Reference
|
|
1185
1272
|
#
|
|
1186
1273
|
# @param [String] query The query that defines the view.
|
|
1187
1274
|
# @param [Boolean] standard_sql Specifies whether to use BigQuery's
|
|
@@ -1193,11 +1280,12 @@ module Google
|
|
|
1193
1280
|
# SQL](https://cloud.google.com/bigquery/docs/reference/legacy-sql)
|
|
1194
1281
|
# dialect. Optional. The default value is false.
|
|
1195
1282
|
# @param [Array<String>, String] udfs User-defined function resources
|
|
1196
|
-
# used in a legacy SQL query.
|
|
1197
|
-
#
|
|
1198
|
-
#
|
|
1199
|
-
#
|
|
1200
|
-
#
|
|
1283
|
+
# used in a legacy SQL query. Optional.
|
|
1284
|
+
#
|
|
1285
|
+
# May be either a code resource to load from a Google Cloud Storage URI
|
|
1286
|
+
# (`gs://bucket/path`), or an inline resource that contains code for a
|
|
1287
|
+
# user-defined function (UDF). Providing an inline code resource is equivalent
|
|
1288
|
+
# to providing a URI for a file containing the same code.
|
|
1201
1289
|
#
|
|
1202
1290
|
# This parameter is used for defining User Defined Function (UDF)
|
|
1203
1291
|
# resources only when using legacy SQL. Users of standard SQL should
|
|
@@ -1208,7 +1296,7 @@ module Google
|
|
|
1208
1296
|
# standard SQL - Differences in user-defined JavaScript
|
|
1209
1297
|
# functions](https://cloud.google.com/bigquery/docs/reference/standard-sql/migrating-from-legacy-sql#differences_in_user-defined_javascript_functions)
|
|
1210
1298
|
#
|
|
1211
|
-
# @example
|
|
1299
|
+
# @example Update a view:
|
|
1212
1300
|
# require "google/cloud/bigquery"
|
|
1213
1301
|
#
|
|
1214
1302
|
# bigquery = Google::Cloud::Bigquery.new
|
|
@@ -1216,12 +1304,13 @@ module Google
|
|
|
1216
1304
|
# view = dataset.table "my_view"
|
|
1217
1305
|
#
|
|
1218
1306
|
# view.set_query "SELECT first_name FROM " \
|
|
1219
|
-
#
|
|
1307
|
+
# "`my_project.my_dataset.my_table`",
|
|
1220
1308
|
# standard_sql: true
|
|
1221
1309
|
#
|
|
1222
1310
|
# @!group Lifecycle
|
|
1223
1311
|
#
|
|
1224
1312
|
def set_query query, standard_sql: nil, legacy_sql: nil, udfs: nil
|
|
1313
|
+
raise "Updating the query is not supported for Table type: #{@gapi.type}" unless view?
|
|
1225
1314
|
use_legacy_sql = Convert.resolve_legacy_sql standard_sql, legacy_sql
|
|
1226
1315
|
@gapi.view = Google::Apis::BigqueryV2::ViewDefinition.new(
|
|
1227
1316
|
query: query,
|
|
@@ -1232,26 +1321,28 @@ module Google
|
|
|
1232
1321
|
end
|
|
1233
1322
|
|
|
1234
1323
|
##
|
|
1235
|
-
# Checks if the view's query is using legacy sql.
|
|
1324
|
+
# Checks if the view's query is using legacy sql. See {#view?}.
|
|
1236
1325
|
#
|
|
1237
|
-
# @return [Boolean] `true` when legacy sql is used, `false` otherwise.
|
|
1326
|
+
# @return [Boolean] `true` when legacy sql is used, `false` otherwise; or `nil` if not a logical view.
|
|
1238
1327
|
#
|
|
1239
1328
|
# @!group Attributes
|
|
1240
1329
|
#
|
|
1241
1330
|
def query_legacy_sql?
|
|
1331
|
+
return nil unless @gapi.view
|
|
1242
1332
|
val = @gapi.view.use_legacy_sql
|
|
1243
1333
|
return true if val.nil?
|
|
1244
1334
|
val
|
|
1245
1335
|
end
|
|
1246
1336
|
|
|
1247
1337
|
##
|
|
1248
|
-
# Checks if the view's query is using standard sql.
|
|
1338
|
+
# Checks if the view's query is using standard sql. See {#view?}.
|
|
1249
1339
|
#
|
|
1250
1340
|
# @return [Boolean] `true` when standard sql is used, `false` otherwise.
|
|
1251
1341
|
#
|
|
1252
1342
|
# @!group Attributes
|
|
1253
1343
|
#
|
|
1254
1344
|
def query_standard_sql?
|
|
1345
|
+
return nil unless @gapi.view
|
|
1255
1346
|
!query_legacy_sql?
|
|
1256
1347
|
end
|
|
1257
1348
|
|
|
@@ -1263,18 +1354,92 @@ module Google
|
|
|
1263
1354
|
# equivalent to providing a URI for a file containing the same code. See
|
|
1264
1355
|
# [User-Defined
|
|
1265
1356
|
# Functions](https://cloud.google.com/bigquery/docs/reference/standard-sql/user-defined-functions).
|
|
1357
|
+
# See {#view?}.
|
|
1266
1358
|
#
|
|
1267
|
-
# @return [Array<String
|
|
1268
|
-
# and/or inline source code.
|
|
1359
|
+
# @return [Array<String>, nil] An array containing Google Cloud Storage URIs
|
|
1360
|
+
# and/or inline source code, or `nil` if not a logical view.
|
|
1269
1361
|
#
|
|
1270
1362
|
# @!group Attributes
|
|
1271
1363
|
#
|
|
1272
1364
|
def query_udfs
|
|
1365
|
+
return nil unless @gapi.view
|
|
1273
1366
|
udfs_gapi = @gapi.view.user_defined_function_resources
|
|
1274
1367
|
return [] if udfs_gapi.nil?
|
|
1275
1368
|
Array(udfs_gapi).map { |udf| udf.inline_code || udf.resource_uri }
|
|
1276
1369
|
end
|
|
1277
1370
|
|
|
1371
|
+
##
|
|
1372
|
+
# Whether automatic refresh of the materialized view is enabled. When true,
|
|
1373
|
+
# the materialized view is updated when the base table is updated. The default
|
|
1374
|
+
# value is true. See {#materialized_view?}.
|
|
1375
|
+
#
|
|
1376
|
+
# @return [Boolean, nil] `true` when automatic refresh is enabled, `false` otherwise;
|
|
1377
|
+
# or `nil` if not a materialized view.
|
|
1378
|
+
#
|
|
1379
|
+
# @!group Attributes
|
|
1380
|
+
#
|
|
1381
|
+
def enable_refresh?
|
|
1382
|
+
return nil unless @gapi.materialized_view
|
|
1383
|
+
val = @gapi.materialized_view.enable_refresh
|
|
1384
|
+
return true if val.nil?
|
|
1385
|
+
val
|
|
1386
|
+
end
|
|
1387
|
+
|
|
1388
|
+
##
|
|
1389
|
+
# Sets whether automatic refresh of the materialized view is enabled. When true,
|
|
1390
|
+
# the materialized view is updated when the base table is updated. See {#materialized_view?}.
|
|
1391
|
+
#
|
|
1392
|
+
# @param [Boolean] new_enable_refresh `true` when automatic refresh is enabled, `false` otherwise.
|
|
1393
|
+
#
|
|
1394
|
+
# @!group Attributes
|
|
1395
|
+
#
|
|
1396
|
+
def enable_refresh= new_enable_refresh
|
|
1397
|
+
@gapi.materialized_view = Google::Apis::BigqueryV2::MaterializedViewDefinition.new(
|
|
1398
|
+
enable_refresh: new_enable_refresh
|
|
1399
|
+
)
|
|
1400
|
+
patch_gapi! :materialized_view
|
|
1401
|
+
end
|
|
1402
|
+
|
|
1403
|
+
##
|
|
1404
|
+
# The time when the materialized view was last modified.
|
|
1405
|
+
# See {#materialized_view?}.
|
|
1406
|
+
#
|
|
1407
|
+
# @return [Time, nil] The time, or `nil` if not present or not a materialized view.
|
|
1408
|
+
#
|
|
1409
|
+
# @!group Attributes
|
|
1410
|
+
#
|
|
1411
|
+
def last_refresh_time
|
|
1412
|
+
Convert.millis_to_time @gapi.materialized_view&.last_refresh_time
|
|
1413
|
+
end
|
|
1414
|
+
|
|
1415
|
+
##
|
|
1416
|
+
# The maximum frequency in milliseconds at which the materialized view will be refreshed.
|
|
1417
|
+
# See {#materialized_view?}.
|
|
1418
|
+
#
|
|
1419
|
+
# @return [Integer, nil] The maximum frequency in milliseconds;
|
|
1420
|
+
# or `nil` if not a materialized view.
|
|
1421
|
+
#
|
|
1422
|
+
# @!group Attributes
|
|
1423
|
+
#
|
|
1424
|
+
def refresh_interval_ms
|
|
1425
|
+
@gapi.materialized_view&.refresh_interval_ms
|
|
1426
|
+
end
|
|
1427
|
+
|
|
1428
|
+
##
|
|
1429
|
+
# Sets the maximum frequency at which the materialized view will be refreshed.
|
|
1430
|
+
# See {#materialized_view?}.
|
|
1431
|
+
#
|
|
1432
|
+
# @param [Integer] new_refresh_interval_ms The maximum frequency in milliseconds.
|
|
1433
|
+
#
|
|
1434
|
+
# @!group Attributes
|
|
1435
|
+
#
|
|
1436
|
+
def refresh_interval_ms= new_refresh_interval_ms
|
|
1437
|
+
@gapi.materialized_view = Google::Apis::BigqueryV2::MaterializedViewDefinition.new(
|
|
1438
|
+
refresh_interval_ms: new_refresh_interval_ms
|
|
1439
|
+
)
|
|
1440
|
+
patch_gapi! :materialized_view
|
|
1441
|
+
end
|
|
1442
|
+
|
|
1278
1443
|
##
|
|
1279
1444
|
# Gets the Cloud IAM access control policy for the table. The latest policy will be read from the service. See
|
|
1280
1445
|
# also {#update_policy}.
|
|
@@ -2195,6 +2360,21 @@ module Google
|
|
|
2195
2360
|
# need to complete a load operation before the data can appear in query
|
|
2196
2361
|
# results.
|
|
2197
2362
|
#
|
|
2363
|
+
# Simple Ruby types are generally accepted per JSON rules, along with the following support for BigQuery's more
|
|
2364
|
+
# complex types:
|
|
2365
|
+
#
|
|
2366
|
+
# | BigQuery | Ruby | Notes |
|
|
2367
|
+
# |--------------|--------------------------------------|----------------------------------------------------|
|
|
2368
|
+
# | `NUMERIC` | `BigDecimal` | `BigDecimal` values will be rounded to scale 9. |
|
|
2369
|
+
# | `BIGNUMERIC` | `String` | Pass as `String` to avoid rounding to scale 9. |
|
|
2370
|
+
# | `DATETIME` | `DateTime` | `DATETIME` does not support time zone. |
|
|
2371
|
+
# | `DATE` | `Date` | |
|
|
2372
|
+
# | `TIMESTAMP` | `Time` | |
|
|
2373
|
+
# | `TIME` | `Google::Cloud::BigQuery::Time` | |
|
|
2374
|
+
# | `BYTES` | `File`, `IO`, `StringIO`, or similar | |
|
|
2375
|
+
# | `ARRAY` | `Array` | Nested arrays, `nil` values are not supported. |
|
|
2376
|
+
# | `STRUCT` | `Hash` | Hash keys may be strings or symbols. |
|
|
2377
|
+
#
|
|
2198
2378
|
# Because BigQuery's streaming API is designed for high insertion rates,
|
|
2199
2379
|
# modifications to the underlying table metadata are eventually
|
|
2200
2380
|
# consistent when interacting with the streaming system. In most cases
|
|
@@ -2208,7 +2388,10 @@ module Google
|
|
|
2208
2388
|
# BigQuery Troubleshooting: Metadata errors for streaming inserts
|
|
2209
2389
|
#
|
|
2210
2390
|
# @param [Hash, Array<Hash>] rows A hash object or array of hash objects
|
|
2211
|
-
# containing the data. Required.
|
|
2391
|
+
# containing the data. Required. `BigDecimal` values will be rounded to
|
|
2392
|
+
# scale 9 to conform with the BigQuery `NUMERIC` data type. To avoid
|
|
2393
|
+
# rounding `BIGNUMERIC` type values with scale greater than 9, use `String`
|
|
2394
|
+
# instead of `BigDecimal`.
|
|
2212
2395
|
# @param [Array<String|Symbol>, Symbol] insert_ids A unique ID for each row. BigQuery uses this property to
|
|
2213
2396
|
# detect duplicate insertion requests on a best-effort basis. For more information, see [data
|
|
2214
2397
|
# consistency](https://cloud.google.com/bigquery/streaming-data-into-bigquery#dataconsistency). Optional. If
|
|
@@ -2251,6 +2434,19 @@ module Google
|
|
|
2251
2434
|
# ]
|
|
2252
2435
|
# table.insert rows
|
|
2253
2436
|
#
|
|
2437
|
+
# @example Pass `BIGNUMERIC` value as a string to avoid rounding to scale 9 in the conversion from `BigDecimal`:
|
|
2438
|
+
# require "google/cloud/bigquery"
|
|
2439
|
+
#
|
|
2440
|
+
# bigquery = Google::Cloud::Bigquery.new
|
|
2441
|
+
# dataset = bigquery.dataset "my_dataset"
|
|
2442
|
+
# table = dataset.table "my_table"
|
|
2443
|
+
#
|
|
2444
|
+
# row = {
|
|
2445
|
+
# "my_numeric" => BigDecimal("123456798.987654321"),
|
|
2446
|
+
# "my_bignumeric" => "123456798.98765432100001" # BigDecimal would be rounded, use String instead!
|
|
2447
|
+
# }
|
|
2448
|
+
# table.insert row
|
|
2449
|
+
#
|
|
2254
2450
|
# @!group Data
|
|
2255
2451
|
#
|
|
2256
2452
|
def insert rows, insert_ids: nil, skip_invalid: nil, ignore_unknown: nil
|
|
@@ -2264,8 +2460,12 @@ module Google
|
|
|
2264
2460
|
end
|
|
2265
2461
|
|
|
2266
2462
|
ensure_service!
|
|
2267
|
-
|
|
2268
|
-
|
|
2463
|
+
gapi = service.insert_tabledata dataset_id,
|
|
2464
|
+
table_id,
|
|
2465
|
+
rows,
|
|
2466
|
+
skip_invalid: skip_invalid,
|
|
2467
|
+
ignore_unknown: ignore_unknown,
|
|
2468
|
+
insert_ids: insert_ids
|
|
2269
2469
|
InsertResponse.from_gapi rows, gapi
|
|
2270
2470
|
end
|
|
2271
2471
|
|
|
@@ -2564,7 +2764,7 @@ module Google
|
|
|
2564
2764
|
return if attributes.empty?
|
|
2565
2765
|
ensure_service!
|
|
2566
2766
|
patch_args = Hash[attributes.map { |attr| [attr, @gapi.send(attr)] }]
|
|
2567
|
-
patch_gapi = Google::Apis::BigqueryV2::Table.new
|
|
2767
|
+
patch_gapi = Google::Apis::BigqueryV2::Table.new(**patch_args)
|
|
2568
2768
|
patch_gapi.etag = etag if etag
|
|
2569
2769
|
@gapi = service.patch_table dataset_id, table_id, patch_gapi
|
|
2570
2770
|
|
|
@@ -2690,12 +2890,11 @@ module Google
|
|
|
2690
2890
|
|
|
2691
2891
|
def load_local_or_uri file, updater
|
|
2692
2892
|
job_gapi = updater.to_gapi
|
|
2693
|
-
|
|
2694
|
-
|
|
2695
|
-
|
|
2696
|
-
|
|
2697
|
-
|
|
2698
|
-
job
|
|
2893
|
+
if local_file? file
|
|
2894
|
+
load_local file, job_gapi
|
|
2895
|
+
else
|
|
2896
|
+
load_storage file, job_gapi
|
|
2897
|
+
end
|
|
2699
2898
|
end
|
|
2700
2899
|
|
|
2701
2900
|
def storage_url? files
|
|
@@ -2748,6 +2947,7 @@ module Google
|
|
|
2748
2947
|
##
|
|
2749
2948
|
# @private Create an Updater object.
|
|
2750
2949
|
def initialize gapi
|
|
2950
|
+
super()
|
|
2751
2951
|
@updates = []
|
|
2752
2952
|
@gapi = gapi
|
|
2753
2953
|
@schema = nil
|
|
@@ -2916,27 +3116,22 @@ module Google
|
|
|
2916
3116
|
end
|
|
2917
3117
|
|
|
2918
3118
|
##
|
|
2919
|
-
# Sets
|
|
2920
|
-
# specified with time-based partitioning, data in the table will be
|
|
2921
|
-
# first partitioned and subsequently clustered.
|
|
3119
|
+
# Sets the list of fields on which data should be clustered.
|
|
2922
3120
|
#
|
|
2923
3121
|
# Only top-level, non-repeated, simple-type fields are supported. When
|
|
2924
3122
|
# you cluster a table using multiple columns, the order of columns you
|
|
2925
3123
|
# specify is important. The order of the specified columns determines
|
|
2926
3124
|
# the sort order of the data.
|
|
2927
3125
|
#
|
|
2928
|
-
#
|
|
2929
|
-
#
|
|
2930
|
-
# on an existing table.
|
|
3126
|
+
# BigQuery supports clustering for both partitioned and non-partitioned
|
|
3127
|
+
# tables.
|
|
2931
3128
|
#
|
|
2932
|
-
# See {Table#clustering_fields}.
|
|
3129
|
+
# See {Table#clustering_fields} and {Table#clustering_fields=}.
|
|
2933
3130
|
#
|
|
2934
|
-
# @see https://cloud.google.com/bigquery/docs/partitioned-tables
|
|
2935
|
-
# Partitioned Tables
|
|
2936
3131
|
# @see https://cloud.google.com/bigquery/docs/clustered-tables
|
|
2937
|
-
# Introduction to
|
|
3132
|
+
# Introduction to clustered tables
|
|
2938
3133
|
# @see https://cloud.google.com/bigquery/docs/creating-clustered-tables
|
|
2939
|
-
# Creating and
|
|
3134
|
+
# Creating and using clustered tables
|
|
2940
3135
|
#
|
|
2941
3136
|
# @param [Array<String>] fields The clustering fields. Only top-level,
|
|
2942
3137
|
# non-repeated, simple-type fields are supported.
|
|
@@ -3115,9 +3310,18 @@ module Google
|
|
|
3115
3310
|
end
|
|
3116
3311
|
|
|
3117
3312
|
##
|
|
3118
|
-
# Adds a numeric number field to the schema.
|
|
3119
|
-
# fixed
|
|
3120
|
-
# the
|
|
3313
|
+
# Adds a numeric number field to the schema. `NUMERIC` is a decimal
|
|
3314
|
+
# type with fixed precision and scale. Precision is the number of
|
|
3315
|
+
# digits that the number contains. Scale is how many of these
|
|
3316
|
+
# digits appear after the decimal point. It supports:
|
|
3317
|
+
#
|
|
3318
|
+
# Precision: 38
|
|
3319
|
+
# Scale: 9
|
|
3320
|
+
# Min: -9.9999999999999999999999999999999999999E+28
|
|
3321
|
+
# Max: 9.9999999999999999999999999999999999999E+28
|
|
3322
|
+
#
|
|
3323
|
+
# This type can represent decimal fractions exactly, and is suitable
|
|
3324
|
+
# for financial calculations.
|
|
3121
3325
|
#
|
|
3122
3326
|
# See {Schema#numeric}
|
|
3123
3327
|
#
|
|
@@ -3144,6 +3348,45 @@ module Google
|
|
|
3144
3348
|
schema.numeric name, description: description, mode: mode
|
|
3145
3349
|
end
|
|
3146
3350
|
|
|
3351
|
+
##
|
|
3352
|
+
# Adds a bignumeric number field to the schema. `BIGNUMERIC` is a
|
|
3353
|
+
# decimal type with fixed precision and scale. Precision is the
|
|
3354
|
+
# number of digits that the number contains. Scale is how many of
|
|
3355
|
+
# these digits appear after the decimal point. It supports:
|
|
3356
|
+
#
|
|
3357
|
+
# Precision: 76.76 (the 77th digit is partial)
|
|
3358
|
+
# Scale: 38
|
|
3359
|
+
# Min: -5.7896044618658097711785492504343953926634992332820282019728792003956564819968E+38
|
|
3360
|
+
# Max: 5.7896044618658097711785492504343953926634992332820282019728792003956564819967E+38
|
|
3361
|
+
#
|
|
3362
|
+
# This type can represent decimal fractions exactly, and is suitable
|
|
3363
|
+
# for financial calculations.
|
|
3364
|
+
#
|
|
3365
|
+
# See {Schema#bignumeric}
|
|
3366
|
+
#
|
|
3367
|
+
# @param [String] name The field name. The name must contain only
|
|
3368
|
+
# letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
|
|
3369
|
+
# start with a letter or underscore. The maximum length is 128
|
|
3370
|
+
# characters.
|
|
3371
|
+
# @param [String] description A description of the field.
|
|
3372
|
+
# @param [Symbol] mode The field's mode. The possible values are
|
|
3373
|
+
# `:nullable`, `:required`, and `:repeated`. The default value is
|
|
3374
|
+
# `:nullable`.
|
|
3375
|
+
#
|
|
3376
|
+
# @example
|
|
3377
|
+
# require "google/cloud/bigquery"
|
|
3378
|
+
#
|
|
3379
|
+
# bigquery = Google::Cloud::Bigquery.new
|
|
3380
|
+
# dataset = bigquery.dataset "my_dataset"
|
|
3381
|
+
# table = dataset.create_table "my_table" do |schema|
|
|
3382
|
+
# schema.bignumeric "total_cost", mode: :required
|
|
3383
|
+
# end
|
|
3384
|
+
#
|
|
3385
|
+
# @!group Schema
|
|
3386
|
+
def bignumeric name, description: nil, mode: :nullable
|
|
3387
|
+
schema.bignumeric name, description: description, mode: mode
|
|
3388
|
+
end
|
|
3389
|
+
|
|
3147
3390
|
##
|
|
3148
3391
|
# Adds a boolean field to the schema.
|
|
3149
3392
|
#
|