google-cloud-bigquery 1.26.0 → 1.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +31 -0
- data/CONTRIBUTING.md +1 -1
- data/LOGGING.md +1 -1
- data/lib/google/cloud/bigquery/convert.rb +0 -4
- data/lib/google/cloud/bigquery/copy_job.rb +1 -0
- data/lib/google/cloud/bigquery/data.rb +2 -2
- data/lib/google/cloud/bigquery/dataset.rb +106 -21
- data/lib/google/cloud/bigquery/dataset/access.rb +3 -3
- data/lib/google/cloud/bigquery/dataset/list.rb +2 -2
- data/lib/google/cloud/bigquery/extract_job.rb +4 -2
- data/lib/google/cloud/bigquery/job.rb +43 -3
- data/lib/google/cloud/bigquery/job/list.rb +4 -4
- data/lib/google/cloud/bigquery/load_job.rb +1 -0
- data/lib/google/cloud/bigquery/model/list.rb +2 -2
- data/lib/google/cloud/bigquery/policy.rb +2 -1
- data/lib/google/cloud/bigquery/project.rb +3 -3
- data/lib/google/cloud/bigquery/project/list.rb +2 -2
- data/lib/google/cloud/bigquery/query_job.rb +24 -12
- data/lib/google/cloud/bigquery/routine.rb +128 -9
- data/lib/google/cloud/bigquery/routine/list.rb +2 -2
- data/lib/google/cloud/bigquery/service.rb +11 -13
- data/lib/google/cloud/bigquery/standard_sql.rb +4 -3
- data/lib/google/cloud/bigquery/table.rb +159 -45
- data/lib/google/cloud/bigquery/table/async_inserter.rb +24 -15
- data/lib/google/cloud/bigquery/table/list.rb +2 -2
- data/lib/google/cloud/bigquery/version.rb +1 -1
- metadata +12 -12
@@ -72,8 +72,8 @@ module Google
|
|
72
72
|
return nil unless next?
|
73
73
|
ensure_service!
|
74
74
|
next_kwargs = @kwargs.merge token: token
|
75
|
-
next_gapi = @service.list_jobs
|
76
|
-
self.class.from_gapi next_gapi, @service, next_kwargs
|
75
|
+
next_gapi = @service.list_jobs(**next_kwargs)
|
76
|
+
self.class.from_gapi next_gapi, @service, **next_kwargs
|
77
77
|
end
|
78
78
|
|
79
79
|
##
|
@@ -121,12 +121,12 @@ module Google
|
|
121
121
|
# puts job.state
|
122
122
|
# end
|
123
123
|
#
|
124
|
-
def all request_limit: nil
|
124
|
+
def all request_limit: nil, &block
|
125
125
|
request_limit = request_limit.to_i if request_limit
|
126
126
|
return enum_for :all, request_limit: request_limit unless block_given?
|
127
127
|
results = self
|
128
128
|
loop do
|
129
|
-
results.each
|
129
|
+
results.each(&block)
|
130
130
|
if request_limit
|
131
131
|
request_limit -= 1
|
132
132
|
break if request_limit.negative?
|
@@ -124,12 +124,12 @@ module Google
|
|
124
124
|
# puts model.model_id
|
125
125
|
# end
|
126
126
|
#
|
127
|
-
def all request_limit: nil
|
127
|
+
def all request_limit: nil, &block
|
128
128
|
request_limit = request_limit.to_i if request_limit
|
129
129
|
return enum_for :all, request_limit: request_limit unless block_given?
|
130
130
|
results = self
|
131
131
|
loop do
|
132
|
-
results.each
|
132
|
+
results.each(&block)
|
133
133
|
if request_limit
|
134
134
|
request_limit -= 1
|
135
135
|
break if request_limit.negative?
|
@@ -56,7 +56,8 @@ module Google
|
|
56
56
|
# @private The Service object.
|
57
57
|
attr_accessor :service
|
58
58
|
|
59
|
-
attr_reader :name
|
59
|
+
attr_reader :name
|
60
|
+
attr_reader :numeric_id
|
60
61
|
|
61
62
|
##
|
62
63
|
# Creates a new Service instance.
|
@@ -981,8 +982,7 @@ module Google
|
|
981
982
|
# @param [String] description A user-friendly description of the
|
982
983
|
# dataset.
|
983
984
|
# @param [Integer] expiration The default lifetime of all tables in the
|
984
|
-
# dataset, in milliseconds. The minimum value is
|
985
|
-
# (one hour).
|
985
|
+
# dataset, in milliseconds. The minimum value is `3_600_000` (one hour).
|
986
986
|
# @param [String] location The geographic location where the dataset
|
987
987
|
# should reside. Possible values include `EU` and `US`. The default
|
988
988
|
# value is `US`.
|
@@ -121,12 +121,12 @@ module Google
|
|
121
121
|
# puts project.name
|
122
122
|
# end
|
123
123
|
#
|
124
|
-
def all request_limit: nil
|
124
|
+
def all request_limit: nil, &block
|
125
125
|
request_limit = request_limit.to_i if request_limit
|
126
126
|
return enum_for :all, request_limit: request_limit unless block_given?
|
127
127
|
results = self
|
128
128
|
loop do
|
129
|
-
results.each
|
129
|
+
results.each(&block)
|
130
130
|
if request_limit
|
131
131
|
request_limit -= 1
|
132
132
|
break if request_limit.negative?
|
@@ -692,8 +692,11 @@ module Google
|
|
692
692
|
end
|
693
693
|
ensure_schema!
|
694
694
|
|
695
|
-
|
696
|
-
|
695
|
+
data_hash = service.list_tabledata destination_table_dataset_id,
|
696
|
+
destination_table_table_id,
|
697
|
+
token: token,
|
698
|
+
max: max,
|
699
|
+
start: start
|
697
700
|
Data.from_gapi_json data_hash, destination_table_gapi, @gapi, service
|
698
701
|
end
|
699
702
|
alias query_results data
|
@@ -704,12 +707,11 @@ module Google
|
|
704
707
|
##
|
705
708
|
# @private Create an Updater object.
|
706
709
|
def initialize service, gapi
|
710
|
+
super()
|
707
711
|
@service = service
|
708
712
|
@gapi = gapi
|
709
713
|
end
|
710
714
|
|
711
|
-
# rubocop:disable all
|
712
|
-
|
713
715
|
##
|
714
716
|
# @private Create an Updater from an options hash.
|
715
717
|
#
|
@@ -748,8 +750,6 @@ module Google
|
|
748
750
|
updater
|
749
751
|
end
|
750
752
|
|
751
|
-
# rubocop:enable all
|
752
|
-
|
753
753
|
##
|
754
754
|
# Sets the geographic location where the job should run. Required
|
755
755
|
# except for US and EU.
|
@@ -935,13 +935,13 @@ module Google
|
|
935
935
|
raise ArgumentError, "types must use the same format as params" if types.class != params.class
|
936
936
|
|
937
937
|
case params
|
938
|
-
when Array
|
938
|
+
when Array
|
939
939
|
@gapi.configuration.query.use_legacy_sql = false
|
940
940
|
@gapi.configuration.query.parameter_mode = "POSITIONAL"
|
941
941
|
@gapi.configuration.query.query_parameters = params.zip(types).map do |param, type|
|
942
942
|
Convert.to_query_param param, type
|
943
943
|
end
|
944
|
-
when Hash
|
944
|
+
when Hash
|
945
945
|
@gapi.configuration.query.use_legacy_sql = false
|
946
946
|
@gapi.configuration.query.parameter_mode = "NAMED"
|
947
947
|
@gapi.configuration.query.query_parameters = params.map do |name, param|
|
@@ -1592,9 +1592,20 @@ module Google
|
|
1592
1592
|
# end
|
1593
1593
|
#
|
1594
1594
|
class Stage
|
1595
|
-
attr_reader :compute_ratio_avg
|
1596
|
-
|
1597
|
-
|
1595
|
+
attr_reader :compute_ratio_avg
|
1596
|
+
attr_reader :compute_ratio_max
|
1597
|
+
attr_reader :id
|
1598
|
+
attr_reader :name
|
1599
|
+
attr_reader :read_ratio_avg
|
1600
|
+
attr_reader :read_ratio_max
|
1601
|
+
attr_reader :records_read
|
1602
|
+
attr_reader :records_written
|
1603
|
+
attr_reader :status
|
1604
|
+
attr_reader :steps
|
1605
|
+
attr_reader :wait_ratio_avg
|
1606
|
+
attr_reader :wait_ratio_max
|
1607
|
+
attr_reader :write_ratio_avg
|
1608
|
+
attr_reader :write_ratio_max
|
1598
1609
|
|
1599
1610
|
##
|
1600
1611
|
# @private Creates a new Stage instance.
|
@@ -1657,7 +1668,8 @@ module Google
|
|
1657
1668
|
# end
|
1658
1669
|
#
|
1659
1670
|
class Step
|
1660
|
-
attr_reader :kind
|
1671
|
+
attr_reader :kind
|
1672
|
+
attr_reader :substeps
|
1661
1673
|
|
1662
1674
|
##
|
1663
1675
|
# @private Creates a new Stage instance.
|
@@ -603,6 +603,93 @@ module Google
|
|
603
603
|
update_gapi!
|
604
604
|
end
|
605
605
|
|
606
|
+
###
|
607
|
+
# The JavaScript UDF determinism level. Optional.
|
608
|
+
#
|
609
|
+
# * `DETERMINISTIC` - Deterministic indicates that two calls with the same input to a UDF yield the same output.
|
610
|
+
# If all JavaScript UDFs are `DETERMINISTIC`, the query result is potentially cachable.
|
611
|
+
# * `NOT_DETERMINISTIC` - Not deterministic indicates that the output of the UDF is not guaranteed to yield the
|
612
|
+
# same output each time for a given set of inputs. If any JavaScript UDF is `NOT_DETERMINISTIC`, the query
|
613
|
+
# result is not cacheable.
|
614
|
+
#
|
615
|
+
# Even if a JavaScript UDF is deterministic, many other factors can prevent usage of cached query results.
|
616
|
+
# Example factors include but not limited to: DDL/DML, non-deterministic SQL function calls, update of
|
617
|
+
# referenced tables/views/UDFs or imported JavaScript libraries. SQL UDFs cannot have determinism specified.
|
618
|
+
# Their determinism is automatically determined.
|
619
|
+
#
|
620
|
+
# @return [String, nil] The routine determinism level in upper case, or `nil` if not set or the object is a
|
621
|
+
# reference (see {#reference?}).
|
622
|
+
#
|
623
|
+
# @example
|
624
|
+
# require "google/cloud/bigquery"
|
625
|
+
#
|
626
|
+
# bigquery = Google::Cloud::Bigquery.new
|
627
|
+
# dataset = bigquery.dataset "my_dataset"
|
628
|
+
# routine = dataset.routine "my_routine"
|
629
|
+
#
|
630
|
+
# routine.determinism_level #=> "NOT_DETERMINISTIC"
|
631
|
+
#
|
632
|
+
# @!group Attributes
|
633
|
+
#
|
634
|
+
def determinism_level
|
635
|
+
return nil if reference?
|
636
|
+
ensure_full_data!
|
637
|
+
@gapi.determinism_level
|
638
|
+
end
|
639
|
+
|
640
|
+
##
|
641
|
+
# Updates the JavaScript UDF determinism level. Optional.
|
642
|
+
#
|
643
|
+
# * `DETERMINISTIC` - Deterministic indicates that two calls with the same input to a UDF yield the same output.
|
644
|
+
# If all JavaScript UDFs are `DETERMINISTIC`, the query result is potentially cachable.
|
645
|
+
# * `NOT_DETERMINISTIC` - Not deterministic indicates that the output of the UDF is not guaranteed to yield the
|
646
|
+
# same output each time for a given set of inputs. If any JavaScript UDF is `NOT_DETERMINISTIC`, the query
|
647
|
+
# result is not cacheable.
|
648
|
+
#
|
649
|
+
# @param [String, nil] new_determinism_level The new routine determinism level in upper case.
|
650
|
+
#
|
651
|
+
# @example
|
652
|
+
# require "google/cloud/bigquery"
|
653
|
+
#
|
654
|
+
# bigquery = Google::Cloud::Bigquery.new
|
655
|
+
# dataset = bigquery.dataset "my_dataset"
|
656
|
+
# routine = dataset.routine "my_routine"
|
657
|
+
#
|
658
|
+
# routine.determinism_level #=> "NOT_DETERMINISTIC"
|
659
|
+
# routine.determinism_level = "DETERMINISTIC"
|
660
|
+
#
|
661
|
+
# @!group Attributes
|
662
|
+
#
|
663
|
+
def determinism_level= new_determinism_level
|
664
|
+
ensure_full_data!
|
665
|
+
@gapi.determinism_level = new_determinism_level
|
666
|
+
update_gapi!
|
667
|
+
end
|
668
|
+
|
669
|
+
##
|
670
|
+
# Checks if the value of {#determinism_level} is `DETERMINISTIC`. The default is `false`.
|
671
|
+
#
|
672
|
+
# @return [Boolean] `true` when `DETERMINISTIC` and the object is not a reference (see {#reference?}), `false`
|
673
|
+
# otherwise.
|
674
|
+
#
|
675
|
+
# @!group Attributes
|
676
|
+
#
|
677
|
+
def determinism_level_deterministic?
|
678
|
+
@gapi.determinism_level == "DETERMINISTIC"
|
679
|
+
end
|
680
|
+
|
681
|
+
##
|
682
|
+
# Checks if the value of {#determinism_level} is `NOT_DETERMINISTIC`. The default is `false`.
|
683
|
+
#
|
684
|
+
# @return [Boolean] `true` when `NOT_DETERMINISTIC` and the object is not a reference (see {#reference?}),
|
685
|
+
# `false` otherwise.
|
686
|
+
#
|
687
|
+
# @!group Attributes
|
688
|
+
#
|
689
|
+
def determinism_level_not_deterministic?
|
690
|
+
@gapi.determinism_level == "NOT_DETERMINISTIC"
|
691
|
+
end
|
692
|
+
|
606
693
|
##
|
607
694
|
# Updates the routine with changes made in the given block in a single update request. The following attributes
|
608
695
|
# may be set: {Updater#routine_type=}, {Updater#language=}, {Updater#arguments=}, {Updater#return_type=},
|
@@ -919,6 +1006,7 @@ module Google
|
|
919
1006
|
##
|
920
1007
|
# @private Create an Updater object.
|
921
1008
|
def initialize gapi
|
1009
|
+
super()
|
922
1010
|
@original_gapi = gapi
|
923
1011
|
@gapi = gapi.dup
|
924
1012
|
end
|
@@ -999,7 +1087,9 @@ module Google
|
|
999
1087
|
# routine = dataset.routine "my_routine"
|
1000
1088
|
#
|
1001
1089
|
# routine.return_type.type_kind #=> "INT64"
|
1002
|
-
# routine.
|
1090
|
+
# routine.update do |r|
|
1091
|
+
# r.return_type = "STRING"
|
1092
|
+
# end
|
1003
1093
|
#
|
1004
1094
|
def return_type= new_return_type
|
1005
1095
|
@gapi.return_type = StandardSql::DataType.gapi_from_string_or_data_type new_return_type
|
@@ -1019,9 +1109,11 @@ module Google
|
|
1019
1109
|
# dataset = bigquery.dataset "my_dataset"
|
1020
1110
|
# routine = dataset.routine "my_routine"
|
1021
1111
|
#
|
1022
|
-
# routine.
|
1023
|
-
#
|
1024
|
-
#
|
1112
|
+
# routine.update do |r|
|
1113
|
+
# r.imported_libraries = [
|
1114
|
+
# "gs://cloud-samples-data/bigquery/udfs/max-value.js"
|
1115
|
+
# ]
|
1116
|
+
# end
|
1025
1117
|
#
|
1026
1118
|
def imported_libraries= new_imported_libraries
|
1027
1119
|
@gapi.imported_libraries = new_imported_libraries
|
@@ -1069,12 +1161,43 @@ module Google
|
|
1069
1161
|
# routine = dataset.routine "my_routine"
|
1070
1162
|
#
|
1071
1163
|
# routine.description #=> "My routine description"
|
1072
|
-
# routine.
|
1164
|
+
# routine.update do |r|
|
1165
|
+
# r.description = "My updated routine description"
|
1166
|
+
# end
|
1073
1167
|
#
|
1074
1168
|
def description= new_description
|
1075
1169
|
@gapi.description = new_description
|
1076
1170
|
end
|
1077
1171
|
|
1172
|
+
##
|
1173
|
+
# Updates the JavaScript UDF determinism level. Optional.
|
1174
|
+
#
|
1175
|
+
# * `DETERMINISTIC` - Deterministic indicates that two calls with the same input to a UDF yield the same
|
1176
|
+
# output. If all JavaScript UDFs are `DETERMINISTIC`, the query result is potentially cachable.
|
1177
|
+
# * `NOT_DETERMINISTIC` - Not deterministic indicates that the output of the UDF is not guaranteed to yield
|
1178
|
+
# the same output each time for a given set of inputs. If any JavaScript UDF is `NOT_DETERMINISTIC`, the
|
1179
|
+
# query result is not cacheable.
|
1180
|
+
#
|
1181
|
+
# @param [String, nil] new_determinism_level The new routine determinism level in upper case.
|
1182
|
+
#
|
1183
|
+
# @example
|
1184
|
+
# require "google/cloud/bigquery"
|
1185
|
+
#
|
1186
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1187
|
+
# dataset = bigquery.dataset "my_dataset"
|
1188
|
+
# routine = dataset.routine "my_routine"
|
1189
|
+
#
|
1190
|
+
# routine.determinism_level #=> "NOT_DETERMINISTIC"
|
1191
|
+
# routine.update do |r|
|
1192
|
+
# r.determinism_level = "DETERMINISTIC"
|
1193
|
+
# end
|
1194
|
+
#
|
1195
|
+
# @!group Attributes
|
1196
|
+
#
|
1197
|
+
def determinism_level= new_determinism_level
|
1198
|
+
@gapi.determinism_level = new_determinism_level
|
1199
|
+
end
|
1200
|
+
|
1078
1201
|
def update
|
1079
1202
|
raise "not implemented in #{self.class}"
|
1080
1203
|
end
|
@@ -1088,15 +1211,11 @@ module Google
|
|
1088
1211
|
end
|
1089
1212
|
alias refresh! reload!
|
1090
1213
|
|
1091
|
-
# rubocop:disable Style/CaseEquality
|
1092
|
-
|
1093
1214
|
# @private
|
1094
1215
|
def updates?
|
1095
1216
|
!(@gapi === @original_gapi)
|
1096
1217
|
end
|
1097
1218
|
|
1098
|
-
# rubocop:enable Style/CaseEquality
|
1099
|
-
|
1100
1219
|
# @private
|
1101
1220
|
def to_gapi
|
1102
1221
|
@gapi
|
@@ -124,12 +124,12 @@ module Google
|
|
124
124
|
# puts routine.routine_id
|
125
125
|
# end
|
126
126
|
#
|
127
|
-
def all request_limit: nil
|
127
|
+
def all request_limit: nil, &block
|
128
128
|
request_limit = request_limit.to_i if request_limit
|
129
129
|
return enum_for :all, request_limit: request_limit unless block_given?
|
130
130
|
results = self
|
131
131
|
loop do
|
132
|
-
results.each
|
132
|
+
results.each(&block)
|
133
133
|
if request_limit
|
134
134
|
request_limit -= 1
|
135
135
|
break if request_limit.negative?
|
@@ -483,7 +483,7 @@ module Google
|
|
483
483
|
table_id: m["tbl"]
|
484
484
|
}.delete_if { |_, v| v.nil? }
|
485
485
|
str_table_ref_hash = default_ref.to_h.merge str_table_ref_hash
|
486
|
-
ref = Google::Apis::BigqueryV2::TableReference.new
|
486
|
+
ref = Google::Apis::BigqueryV2::TableReference.new(**str_table_ref_hash)
|
487
487
|
validate_table_ref ref
|
488
488
|
ref
|
489
489
|
end
|
@@ -554,9 +554,9 @@ module Google
|
|
554
554
|
nil
|
555
555
|
end
|
556
556
|
|
557
|
-
def execute backoff: nil
|
557
|
+
def execute backoff: nil, &block
|
558
558
|
if backoff
|
559
|
-
Backoff.new(retries: retries).execute
|
559
|
+
Backoff.new(retries: retries).execute(&block)
|
560
560
|
else
|
561
561
|
yield
|
562
562
|
end
|
@@ -590,22 +590,20 @@ module Google
|
|
590
590
|
def execute
|
591
591
|
current_retries = 0
|
592
592
|
loop do
|
593
|
-
|
594
|
-
|
595
|
-
|
596
|
-
|
597
|
-
|
598
|
-
|
599
|
-
current_retries += 1
|
600
|
-
end
|
593
|
+
return yield
|
594
|
+
rescue Google::Apis::Error => e
|
595
|
+
raise e unless retry? e.body, current_retries
|
596
|
+
|
597
|
+
@backoff.call current_retries
|
598
|
+
current_retries += 1
|
601
599
|
end
|
602
600
|
end
|
603
601
|
|
604
602
|
protected
|
605
603
|
|
606
604
|
def retry? result, current_retries #:nodoc:
|
607
|
-
if current_retries < @retries
|
608
|
-
return true
|
605
|
+
if current_retries < @retries && retry_error_reason?(result)
|
606
|
+
return true
|
609
607
|
end
|
610
608
|
false
|
611
609
|
end
|
@@ -401,11 +401,12 @@ module Google
|
|
401
401
|
# @private New Google::Apis::BigqueryV2::StandardSqlDataType from a String or StandardSql::DataType object.
|
402
402
|
def self.gapi_from_string_or_data_type data_type
|
403
403
|
return if data_type.nil?
|
404
|
-
|
404
|
+
case data_type
|
405
|
+
when StandardSql::DataType
|
405
406
|
data_type.to_gapi
|
406
|
-
|
407
|
+
when Hash
|
407
408
|
data_type
|
408
|
-
|
409
|
+
when String, Symbol
|
409
410
|
Google::Apis::BigqueryV2::StandardSqlDataType.new type_kind: data_type.to_s.upcase
|
410
411
|
else
|
411
412
|
raise ArgumentError, "Unable to convert #{data_type} to Google::Apis::BigqueryV2::StandardSqlDataType"
|