google-cloud-bigquery 1.26.0 → 1.29.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +31 -0
- data/CONTRIBUTING.md +1 -1
- data/LOGGING.md +1 -1
- data/lib/google/cloud/bigquery/convert.rb +0 -4
- data/lib/google/cloud/bigquery/copy_job.rb +1 -0
- data/lib/google/cloud/bigquery/data.rb +2 -2
- data/lib/google/cloud/bigquery/dataset.rb +106 -21
- data/lib/google/cloud/bigquery/dataset/access.rb +3 -3
- data/lib/google/cloud/bigquery/dataset/list.rb +2 -2
- data/lib/google/cloud/bigquery/extract_job.rb +4 -2
- data/lib/google/cloud/bigquery/job.rb +43 -3
- data/lib/google/cloud/bigquery/job/list.rb +4 -4
- data/lib/google/cloud/bigquery/load_job.rb +1 -0
- data/lib/google/cloud/bigquery/model/list.rb +2 -2
- data/lib/google/cloud/bigquery/policy.rb +2 -1
- data/lib/google/cloud/bigquery/project.rb +3 -3
- data/lib/google/cloud/bigquery/project/list.rb +2 -2
- data/lib/google/cloud/bigquery/query_job.rb +24 -12
- data/lib/google/cloud/bigquery/routine.rb +128 -9
- data/lib/google/cloud/bigquery/routine/list.rb +2 -2
- data/lib/google/cloud/bigquery/service.rb +11 -13
- data/lib/google/cloud/bigquery/standard_sql.rb +4 -3
- data/lib/google/cloud/bigquery/table.rb +159 -45
- data/lib/google/cloud/bigquery/table/async_inserter.rb +24 -15
- data/lib/google/cloud/bigquery/table/list.rb +2 -2
- data/lib/google/cloud/bigquery/version.rb +1 -1
- metadata +12 -12
@@ -72,8 +72,8 @@ module Google
|
|
72
72
|
return nil unless next?
|
73
73
|
ensure_service!
|
74
74
|
next_kwargs = @kwargs.merge token: token
|
75
|
-
next_gapi = @service.list_jobs
|
76
|
-
self.class.from_gapi next_gapi, @service, next_kwargs
|
75
|
+
next_gapi = @service.list_jobs(**next_kwargs)
|
76
|
+
self.class.from_gapi next_gapi, @service, **next_kwargs
|
77
77
|
end
|
78
78
|
|
79
79
|
##
|
@@ -121,12 +121,12 @@ module Google
|
|
121
121
|
# puts job.state
|
122
122
|
# end
|
123
123
|
#
|
124
|
-
def all request_limit: nil
|
124
|
+
def all request_limit: nil, &block
|
125
125
|
request_limit = request_limit.to_i if request_limit
|
126
126
|
return enum_for :all, request_limit: request_limit unless block_given?
|
127
127
|
results = self
|
128
128
|
loop do
|
129
|
-
results.each
|
129
|
+
results.each(&block)
|
130
130
|
if request_limit
|
131
131
|
request_limit -= 1
|
132
132
|
break if request_limit.negative?
|
@@ -124,12 +124,12 @@ module Google
|
|
124
124
|
# puts model.model_id
|
125
125
|
# end
|
126
126
|
#
|
127
|
-
def all request_limit: nil
|
127
|
+
def all request_limit: nil, &block
|
128
128
|
request_limit = request_limit.to_i if request_limit
|
129
129
|
return enum_for :all, request_limit: request_limit unless block_given?
|
130
130
|
results = self
|
131
131
|
loop do
|
132
|
-
results.each
|
132
|
+
results.each(&block)
|
133
133
|
if request_limit
|
134
134
|
request_limit -= 1
|
135
135
|
break if request_limit.negative?
|
@@ -56,7 +56,8 @@ module Google
|
|
56
56
|
# @private The Service object.
|
57
57
|
attr_accessor :service
|
58
58
|
|
59
|
-
attr_reader :name
|
59
|
+
attr_reader :name
|
60
|
+
attr_reader :numeric_id
|
60
61
|
|
61
62
|
##
|
62
63
|
# Creates a new Service instance.
|
@@ -981,8 +982,7 @@ module Google
|
|
981
982
|
# @param [String] description A user-friendly description of the
|
982
983
|
# dataset.
|
983
984
|
# @param [Integer] expiration The default lifetime of all tables in the
|
984
|
-
# dataset, in milliseconds. The minimum value is
|
985
|
-
# (one hour).
|
985
|
+
# dataset, in milliseconds. The minimum value is `3_600_000` (one hour).
|
986
986
|
# @param [String] location The geographic location where the dataset
|
987
987
|
# should reside. Possible values include `EU` and `US`. The default
|
988
988
|
# value is `US`.
|
@@ -121,12 +121,12 @@ module Google
|
|
121
121
|
# puts project.name
|
122
122
|
# end
|
123
123
|
#
|
124
|
-
def all request_limit: nil
|
124
|
+
def all request_limit: nil, &block
|
125
125
|
request_limit = request_limit.to_i if request_limit
|
126
126
|
return enum_for :all, request_limit: request_limit unless block_given?
|
127
127
|
results = self
|
128
128
|
loop do
|
129
|
-
results.each
|
129
|
+
results.each(&block)
|
130
130
|
if request_limit
|
131
131
|
request_limit -= 1
|
132
132
|
break if request_limit.negative?
|
@@ -692,8 +692,11 @@ module Google
|
|
692
692
|
end
|
693
693
|
ensure_schema!
|
694
694
|
|
695
|
-
|
696
|
-
|
695
|
+
data_hash = service.list_tabledata destination_table_dataset_id,
|
696
|
+
destination_table_table_id,
|
697
|
+
token: token,
|
698
|
+
max: max,
|
699
|
+
start: start
|
697
700
|
Data.from_gapi_json data_hash, destination_table_gapi, @gapi, service
|
698
701
|
end
|
699
702
|
alias query_results data
|
@@ -704,12 +707,11 @@ module Google
|
|
704
707
|
##
|
705
708
|
# @private Create an Updater object.
|
706
709
|
def initialize service, gapi
|
710
|
+
super()
|
707
711
|
@service = service
|
708
712
|
@gapi = gapi
|
709
713
|
end
|
710
714
|
|
711
|
-
# rubocop:disable all
|
712
|
-
|
713
715
|
##
|
714
716
|
# @private Create an Updater from an options hash.
|
715
717
|
#
|
@@ -748,8 +750,6 @@ module Google
|
|
748
750
|
updater
|
749
751
|
end
|
750
752
|
|
751
|
-
# rubocop:enable all
|
752
|
-
|
753
753
|
##
|
754
754
|
# Sets the geographic location where the job should run. Required
|
755
755
|
# except for US and EU.
|
@@ -935,13 +935,13 @@ module Google
|
|
935
935
|
raise ArgumentError, "types must use the same format as params" if types.class != params.class
|
936
936
|
|
937
937
|
case params
|
938
|
-
when Array
|
938
|
+
when Array
|
939
939
|
@gapi.configuration.query.use_legacy_sql = false
|
940
940
|
@gapi.configuration.query.parameter_mode = "POSITIONAL"
|
941
941
|
@gapi.configuration.query.query_parameters = params.zip(types).map do |param, type|
|
942
942
|
Convert.to_query_param param, type
|
943
943
|
end
|
944
|
-
when Hash
|
944
|
+
when Hash
|
945
945
|
@gapi.configuration.query.use_legacy_sql = false
|
946
946
|
@gapi.configuration.query.parameter_mode = "NAMED"
|
947
947
|
@gapi.configuration.query.query_parameters = params.map do |name, param|
|
@@ -1592,9 +1592,20 @@ module Google
|
|
1592
1592
|
# end
|
1593
1593
|
#
|
1594
1594
|
class Stage
|
1595
|
-
attr_reader :compute_ratio_avg
|
1596
|
-
|
1597
|
-
|
1595
|
+
attr_reader :compute_ratio_avg
|
1596
|
+
attr_reader :compute_ratio_max
|
1597
|
+
attr_reader :id
|
1598
|
+
attr_reader :name
|
1599
|
+
attr_reader :read_ratio_avg
|
1600
|
+
attr_reader :read_ratio_max
|
1601
|
+
attr_reader :records_read
|
1602
|
+
attr_reader :records_written
|
1603
|
+
attr_reader :status
|
1604
|
+
attr_reader :steps
|
1605
|
+
attr_reader :wait_ratio_avg
|
1606
|
+
attr_reader :wait_ratio_max
|
1607
|
+
attr_reader :write_ratio_avg
|
1608
|
+
attr_reader :write_ratio_max
|
1598
1609
|
|
1599
1610
|
##
|
1600
1611
|
# @private Creates a new Stage instance.
|
@@ -1657,7 +1668,8 @@ module Google
|
|
1657
1668
|
# end
|
1658
1669
|
#
|
1659
1670
|
class Step
|
1660
|
-
attr_reader :kind
|
1671
|
+
attr_reader :kind
|
1672
|
+
attr_reader :substeps
|
1661
1673
|
|
1662
1674
|
##
|
1663
1675
|
# @private Creates a new Stage instance.
|
@@ -603,6 +603,93 @@ module Google
|
|
603
603
|
update_gapi!
|
604
604
|
end
|
605
605
|
|
606
|
+
###
|
607
|
+
# The JavaScript UDF determinism level. Optional.
|
608
|
+
#
|
609
|
+
# * `DETERMINISTIC` - Deterministic indicates that two calls with the same input to a UDF yield the same output.
|
610
|
+
# If all JavaScript UDFs are `DETERMINISTIC`, the query result is potentially cachable.
|
611
|
+
# * `NOT_DETERMINISTIC` - Not deterministic indicates that the output of the UDF is not guaranteed to yield the
|
612
|
+
# same output each time for a given set of inputs. If any JavaScript UDF is `NOT_DETERMINISTIC`, the query
|
613
|
+
# result is not cacheable.
|
614
|
+
#
|
615
|
+
# Even if a JavaScript UDF is deterministic, many other factors can prevent usage of cached query results.
|
616
|
+
# Example factors include but not limited to: DDL/DML, non-deterministic SQL function calls, update of
|
617
|
+
# referenced tables/views/UDFs or imported JavaScript libraries. SQL UDFs cannot have determinism specified.
|
618
|
+
# Their determinism is automatically determined.
|
619
|
+
#
|
620
|
+
# @return [String, nil] The routine determinism level in upper case, or `nil` if not set or the object is a
|
621
|
+
# reference (see {#reference?}).
|
622
|
+
#
|
623
|
+
# @example
|
624
|
+
# require "google/cloud/bigquery"
|
625
|
+
#
|
626
|
+
# bigquery = Google::Cloud::Bigquery.new
|
627
|
+
# dataset = bigquery.dataset "my_dataset"
|
628
|
+
# routine = dataset.routine "my_routine"
|
629
|
+
#
|
630
|
+
# routine.determinism_level #=> "NOT_DETERMINISTIC"
|
631
|
+
#
|
632
|
+
# @!group Attributes
|
633
|
+
#
|
634
|
+
def determinism_level
|
635
|
+
return nil if reference?
|
636
|
+
ensure_full_data!
|
637
|
+
@gapi.determinism_level
|
638
|
+
end
|
639
|
+
|
640
|
+
##
|
641
|
+
# Updates the JavaScript UDF determinism level. Optional.
|
642
|
+
#
|
643
|
+
# * `DETERMINISTIC` - Deterministic indicates that two calls with the same input to a UDF yield the same output.
|
644
|
+
# If all JavaScript UDFs are `DETERMINISTIC`, the query result is potentially cachable.
|
645
|
+
# * `NOT_DETERMINISTIC` - Not deterministic indicates that the output of the UDF is not guaranteed to yield the
|
646
|
+
# same output each time for a given set of inputs. If any JavaScript UDF is `NOT_DETERMINISTIC`, the query
|
647
|
+
# result is not cacheable.
|
648
|
+
#
|
649
|
+
# @param [String, nil] new_determinism_level The new routine determinism level in upper case.
|
650
|
+
#
|
651
|
+
# @example
|
652
|
+
# require "google/cloud/bigquery"
|
653
|
+
#
|
654
|
+
# bigquery = Google::Cloud::Bigquery.new
|
655
|
+
# dataset = bigquery.dataset "my_dataset"
|
656
|
+
# routine = dataset.routine "my_routine"
|
657
|
+
#
|
658
|
+
# routine.determinism_level #=> "NOT_DETERMINISTIC"
|
659
|
+
# routine.determinism_level = "DETERMINISTIC"
|
660
|
+
#
|
661
|
+
# @!group Attributes
|
662
|
+
#
|
663
|
+
def determinism_level= new_determinism_level
|
664
|
+
ensure_full_data!
|
665
|
+
@gapi.determinism_level = new_determinism_level
|
666
|
+
update_gapi!
|
667
|
+
end
|
668
|
+
|
669
|
+
##
|
670
|
+
# Checks if the value of {#determinism_level} is `DETERMINISTIC`. The default is `false`.
|
671
|
+
#
|
672
|
+
# @return [Boolean] `true` when `DETERMINISTIC` and the object is not a reference (see {#reference?}), `false`
|
673
|
+
# otherwise.
|
674
|
+
#
|
675
|
+
# @!group Attributes
|
676
|
+
#
|
677
|
+
def determinism_level_deterministic?
|
678
|
+
@gapi.determinism_level == "DETERMINISTIC"
|
679
|
+
end
|
680
|
+
|
681
|
+
##
|
682
|
+
# Checks if the value of {#determinism_level} is `NOT_DETERMINISTIC`. The default is `false`.
|
683
|
+
#
|
684
|
+
# @return [Boolean] `true` when `NOT_DETERMINISTIC` and the object is not a reference (see {#reference?}),
|
685
|
+
# `false` otherwise.
|
686
|
+
#
|
687
|
+
# @!group Attributes
|
688
|
+
#
|
689
|
+
def determinism_level_not_deterministic?
|
690
|
+
@gapi.determinism_level == "NOT_DETERMINISTIC"
|
691
|
+
end
|
692
|
+
|
606
693
|
##
|
607
694
|
# Updates the routine with changes made in the given block in a single update request. The following attributes
|
608
695
|
# may be set: {Updater#routine_type=}, {Updater#language=}, {Updater#arguments=}, {Updater#return_type=},
|
@@ -919,6 +1006,7 @@ module Google
|
|
919
1006
|
##
|
920
1007
|
# @private Create an Updater object.
|
921
1008
|
def initialize gapi
|
1009
|
+
super()
|
922
1010
|
@original_gapi = gapi
|
923
1011
|
@gapi = gapi.dup
|
924
1012
|
end
|
@@ -999,7 +1087,9 @@ module Google
|
|
999
1087
|
# routine = dataset.routine "my_routine"
|
1000
1088
|
#
|
1001
1089
|
# routine.return_type.type_kind #=> "INT64"
|
1002
|
-
# routine.
|
1090
|
+
# routine.update do |r|
|
1091
|
+
# r.return_type = "STRING"
|
1092
|
+
# end
|
1003
1093
|
#
|
1004
1094
|
def return_type= new_return_type
|
1005
1095
|
@gapi.return_type = StandardSql::DataType.gapi_from_string_or_data_type new_return_type
|
@@ -1019,9 +1109,11 @@ module Google
|
|
1019
1109
|
# dataset = bigquery.dataset "my_dataset"
|
1020
1110
|
# routine = dataset.routine "my_routine"
|
1021
1111
|
#
|
1022
|
-
# routine.
|
1023
|
-
#
|
1024
|
-
#
|
1112
|
+
# routine.update do |r|
|
1113
|
+
# r.imported_libraries = [
|
1114
|
+
# "gs://cloud-samples-data/bigquery/udfs/max-value.js"
|
1115
|
+
# ]
|
1116
|
+
# end
|
1025
1117
|
#
|
1026
1118
|
def imported_libraries= new_imported_libraries
|
1027
1119
|
@gapi.imported_libraries = new_imported_libraries
|
@@ -1069,12 +1161,43 @@ module Google
|
|
1069
1161
|
# routine = dataset.routine "my_routine"
|
1070
1162
|
#
|
1071
1163
|
# routine.description #=> "My routine description"
|
1072
|
-
# routine.
|
1164
|
+
# routine.update do |r|
|
1165
|
+
# r.description = "My updated routine description"
|
1166
|
+
# end
|
1073
1167
|
#
|
1074
1168
|
def description= new_description
|
1075
1169
|
@gapi.description = new_description
|
1076
1170
|
end
|
1077
1171
|
|
1172
|
+
##
|
1173
|
+
# Updates the JavaScript UDF determinism level. Optional.
|
1174
|
+
#
|
1175
|
+
# * `DETERMINISTIC` - Deterministic indicates that two calls with the same input to a UDF yield the same
|
1176
|
+
# output. If all JavaScript UDFs are `DETERMINISTIC`, the query result is potentially cachable.
|
1177
|
+
# * `NOT_DETERMINISTIC` - Not deterministic indicates that the output of the UDF is not guaranteed to yield
|
1178
|
+
# the same output each time for a given set of inputs. If any JavaScript UDF is `NOT_DETERMINISTIC`, the
|
1179
|
+
# query result is not cacheable.
|
1180
|
+
#
|
1181
|
+
# @param [String, nil] new_determinism_level The new routine determinism level in upper case.
|
1182
|
+
#
|
1183
|
+
# @example
|
1184
|
+
# require "google/cloud/bigquery"
|
1185
|
+
#
|
1186
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1187
|
+
# dataset = bigquery.dataset "my_dataset"
|
1188
|
+
# routine = dataset.routine "my_routine"
|
1189
|
+
#
|
1190
|
+
# routine.determinism_level #=> "NOT_DETERMINISTIC"
|
1191
|
+
# routine.update do |r|
|
1192
|
+
# r.determinism_level = "DETERMINISTIC"
|
1193
|
+
# end
|
1194
|
+
#
|
1195
|
+
# @!group Attributes
|
1196
|
+
#
|
1197
|
+
def determinism_level= new_determinism_level
|
1198
|
+
@gapi.determinism_level = new_determinism_level
|
1199
|
+
end
|
1200
|
+
|
1078
1201
|
def update
|
1079
1202
|
raise "not implemented in #{self.class}"
|
1080
1203
|
end
|
@@ -1088,15 +1211,11 @@ module Google
|
|
1088
1211
|
end
|
1089
1212
|
alias refresh! reload!
|
1090
1213
|
|
1091
|
-
# rubocop:disable Style/CaseEquality
|
1092
|
-
|
1093
1214
|
# @private
|
1094
1215
|
def updates?
|
1095
1216
|
!(@gapi === @original_gapi)
|
1096
1217
|
end
|
1097
1218
|
|
1098
|
-
# rubocop:enable Style/CaseEquality
|
1099
|
-
|
1100
1219
|
# @private
|
1101
1220
|
def to_gapi
|
1102
1221
|
@gapi
|
@@ -124,12 +124,12 @@ module Google
|
|
124
124
|
# puts routine.routine_id
|
125
125
|
# end
|
126
126
|
#
|
127
|
-
def all request_limit: nil
|
127
|
+
def all request_limit: nil, &block
|
128
128
|
request_limit = request_limit.to_i if request_limit
|
129
129
|
return enum_for :all, request_limit: request_limit unless block_given?
|
130
130
|
results = self
|
131
131
|
loop do
|
132
|
-
results.each
|
132
|
+
results.each(&block)
|
133
133
|
if request_limit
|
134
134
|
request_limit -= 1
|
135
135
|
break if request_limit.negative?
|
@@ -483,7 +483,7 @@ module Google
|
|
483
483
|
table_id: m["tbl"]
|
484
484
|
}.delete_if { |_, v| v.nil? }
|
485
485
|
str_table_ref_hash = default_ref.to_h.merge str_table_ref_hash
|
486
|
-
ref = Google::Apis::BigqueryV2::TableReference.new
|
486
|
+
ref = Google::Apis::BigqueryV2::TableReference.new(**str_table_ref_hash)
|
487
487
|
validate_table_ref ref
|
488
488
|
ref
|
489
489
|
end
|
@@ -554,9 +554,9 @@ module Google
|
|
554
554
|
nil
|
555
555
|
end
|
556
556
|
|
557
|
-
def execute backoff: nil
|
557
|
+
def execute backoff: nil, &block
|
558
558
|
if backoff
|
559
|
-
Backoff.new(retries: retries).execute
|
559
|
+
Backoff.new(retries: retries).execute(&block)
|
560
560
|
else
|
561
561
|
yield
|
562
562
|
end
|
@@ -590,22 +590,20 @@ module Google
|
|
590
590
|
def execute
|
591
591
|
current_retries = 0
|
592
592
|
loop do
|
593
|
-
|
594
|
-
|
595
|
-
|
596
|
-
|
597
|
-
|
598
|
-
|
599
|
-
current_retries += 1
|
600
|
-
end
|
593
|
+
return yield
|
594
|
+
rescue Google::Apis::Error => e
|
595
|
+
raise e unless retry? e.body, current_retries
|
596
|
+
|
597
|
+
@backoff.call current_retries
|
598
|
+
current_retries += 1
|
601
599
|
end
|
602
600
|
end
|
603
601
|
|
604
602
|
protected
|
605
603
|
|
606
604
|
def retry? result, current_retries #:nodoc:
|
607
|
-
if current_retries < @retries
|
608
|
-
return true
|
605
|
+
if current_retries < @retries && retry_error_reason?(result)
|
606
|
+
return true
|
609
607
|
end
|
610
608
|
false
|
611
609
|
end
|
@@ -401,11 +401,12 @@ module Google
|
|
401
401
|
# @private New Google::Apis::BigqueryV2::StandardSqlDataType from a String or StandardSql::DataType object.
|
402
402
|
def self.gapi_from_string_or_data_type data_type
|
403
403
|
return if data_type.nil?
|
404
|
-
|
404
|
+
case data_type
|
405
|
+
when StandardSql::DataType
|
405
406
|
data_type.to_gapi
|
406
|
-
|
407
|
+
when Hash
|
407
408
|
data_type
|
408
|
-
|
409
|
+
when String, Symbol
|
409
410
|
Google::Apis::BigqueryV2::StandardSqlDataType.new type_kind: data_type.to_s.upcase
|
410
411
|
else
|
411
412
|
raise ArgumentError, "Unable to convert #{data_type} to Google::Apis::BigqueryV2::StandardSqlDataType"
|