google-cloud-bigquery 0.23.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -14,6 +14,7 @@
14
14
 
15
15
 
16
16
  require "google/cloud/bigquery/version"
17
+ require "google/cloud/bigquery/convert"
17
18
  require "google/cloud/errors"
18
19
  require "google/apis/bigquery_v2"
19
20
  require "pathname"
@@ -171,8 +172,7 @@ module Google
171
172
  def insert_tabledata dataset_id, table_id, rows, options = {}
172
173
  insert_rows = Array(rows).map do |row|
173
174
  Google::Apis::BigqueryV2::InsertAllTableDataRequest::Row.new(
174
- insert_id: Digest::MD5.base64digest(row.inspect),
175
- # Hash[row.map{|(k,v)| [k.to_s,v]}] for Hash<String,Object>
175
+ insert_id: Digest::MD5.base64digest(row.to_json),
176
176
  json: row
177
177
  )
178
178
  end
@@ -401,8 +401,9 @@ module Google
401
401
  allow_large_results: options[:large_results],
402
402
  flatten_results: options[:flatten],
403
403
  default_dataset: default_dataset,
404
- use_legacy_sql: resolve_legacy_sql(options[:legacy_sql],
405
- options[:standard_sql])
404
+ use_legacy_sql: Convert.resolve_legacy_sql(
405
+ options[:standard_sql], options[:legacy_sql])
406
+
406
407
  )
407
408
  )
408
409
  )
@@ -412,13 +413,13 @@ module Google
412
413
  req.configuration.query.use_legacy_sql = false
413
414
  req.configuration.query.parameter_mode = "POSITIONAL"
414
415
  req.configuration.query.query_parameters = options[:params].map do |param|
415
- to_query_param param
416
+ Convert.to_query_param param
416
417
  end
417
418
  elsif Hash === options[:params]
418
419
  req.configuration.query.use_legacy_sql = false
419
420
  req.configuration.query.parameter_mode = "NAMED"
420
421
  req.configuration.query.query_parameters = options[:params].map do |name, param|
421
- to_query_param(param).tap do |named_param|
422
+ Convert.to_query_param(param).tap do |named_param|
422
423
  named_param.name = String name
423
424
  end
424
425
  end
@@ -440,8 +441,8 @@ module Google
440
441
  timeout_ms: options[:timeout],
441
442
  dry_run: options[:dryrun],
442
443
  use_query_cache: options[:cache],
443
- use_legacy_sql: resolve_legacy_sql(options[:legacy_sql],
444
- options[:standard_sql])
444
+ use_legacy_sql: Convert.resolve_legacy_sql(
445
+ options[:standard_sql], options[:legacy_sql])
445
446
  )
446
447
 
447
448
  if options[:params]
@@ -449,13 +450,13 @@ module Google
449
450
  req.use_legacy_sql = false
450
451
  req.parameter_mode = "POSITIONAL"
451
452
  req.query_parameters = options[:params].map do |param|
452
- to_query_param param
453
+ Convert.to_query_param param
453
454
  end
454
455
  elsif Hash === options[:params]
455
456
  req.use_legacy_sql = false
456
457
  req.parameter_mode = "NAMED"
457
458
  req.query_parameters = options[:params].map do |name, param|
458
- to_query_param(param).tap do |named_param|
459
+ Convert.to_query_param(param).tap do |named_param|
459
460
  named_param.name = String name
460
461
  end
461
462
  end
@@ -467,103 +468,8 @@ module Google
467
468
  req
468
469
  end
469
470
 
470
- def to_query_param value
471
- if TrueClass === value
472
- return API::QueryParameter.new(
473
- parameter_type: API::QueryParameterType.new(type: "BOOL"),
474
- parameter_value: API::QueryParameterValue.new(value: true)
475
- )
476
- elsif FalseClass === value
477
- return API::QueryParameter.new(
478
- parameter_type: API::QueryParameterType.new(type: "BOOL"),
479
- parameter_value: API::QueryParameterValue.new(value: false)
480
- )
481
- elsif Integer === value
482
- return API::QueryParameter.new(
483
- parameter_type: API::QueryParameterType.new(type: "INT64"),
484
- parameter_value: API::QueryParameterValue.new(value: value)
485
- )
486
- elsif Float === value
487
- return API::QueryParameter.new(
488
- parameter_type: API::QueryParameterType.new(type: "FLOAT64"),
489
- parameter_value: API::QueryParameterValue.new(value: value)
490
- )
491
- elsif String === value
492
- return API::QueryParameter.new(
493
- parameter_type: API::QueryParameterType.new(type: "STRING"),
494
- parameter_value: API::QueryParameterValue.new(value: value)
495
- )
496
- elsif DateTime === value
497
- return API::QueryParameter.new(
498
- parameter_type: API::QueryParameterType.new(type: "DATETIME"),
499
- parameter_value: API::QueryParameterValue.new(
500
- value: value.strftime("%Y-%m-%d %H:%M:%S.%6N"))
501
- )
502
- elsif Date === value
503
- return API::QueryParameter.new(
504
- parameter_type: API::QueryParameterType.new(type: "DATE"),
505
- parameter_value: API::QueryParameterValue.new(value: value.to_s)
506
- )
507
- elsif ::Time === value
508
- return API::QueryParameter.new(
509
- parameter_type: API::QueryParameterType.new(type: "TIMESTAMP"),
510
- parameter_value: API::QueryParameterValue.new(
511
- value: value.strftime("%Y-%m-%d %H:%M:%S.%6N%:z"))
512
- )
513
- elsif Bigquery::Time === value
514
- return API::QueryParameter.new(
515
- parameter_type: API::QueryParameterType.new(type: "TIME"),
516
- parameter_value: API::QueryParameterValue.new(value: value.value)
517
- )
518
- elsif value.respond_to?(:read) && value.respond_to?(:rewind)
519
- value.rewind
520
- return API::QueryParameter.new(
521
- parameter_type: API::QueryParameterType.new(type: "BYTES"),
522
- parameter_value: API::QueryParameterValue.new(
523
- value: value.read.force_encoding("ASCII-8BIT"))
524
- )
525
- elsif Array === value
526
- array_params = value.map { |param| to_query_param param }
527
- return API::QueryParameter.new(
528
- parameter_type: API::QueryParameterType.new(
529
- type: "ARRAY",
530
- array_type: array_params.first.parameter_type
531
- ),
532
- parameter_value: API::QueryParameterValue.new(
533
- array_values: array_params.map(&:parameter_value)
534
- )
535
- )
536
- elsif Hash === value
537
- struct_pairs = value.map do |name, param|
538
- struct_param = to_query_param param
539
- [API::QueryParameterType::StructType.new(
540
- name: String(name),
541
- type: struct_param.parameter_type
542
- ), struct_param.parameter_value]
543
- end
544
-
545
- return API::QueryParameter.new(
546
- parameter_type: API::QueryParameterType.new(
547
- type: "STRUCT",
548
- struct_types: struct_pairs.map(&:first)
549
- ),
550
- parameter_value: API::QueryParameterValue.new(
551
- struct_values: struct_pairs.map(&:last)
552
- )
553
- )
554
- else
555
- fail "A query parameter of type #{value.class} is not supported."
556
- end
557
- v
558
- end
559
-
560
471
  # rubocop:enable all
561
472
 
562
- def resolve_legacy_sql legacy_sql, standard_sql
563
- return legacy_sql unless legacy_sql.nil?
564
- return !standard_sql unless standard_sql.nil?
565
- end
566
-
567
473
  ##
568
474
  # Job description for copy job
569
475
  def copy_table_config source, target, options = {}
@@ -120,7 +120,7 @@ module Google
120
120
 
121
121
  ##
122
122
  # The combined Project ID, Dataset ID, and Table ID for this table, in
123
- # the format specified by the [Query
123
+ # the format specified by the [Legacy SQL Query
124
124
  # Reference](https://cloud.google.com/bigquery/query-reference#from):
125
125
  # `project_name:datasetId.tableId`. To use this value in queries see
126
126
  # {#query_id}.
@@ -137,6 +137,15 @@ module Google
137
137
  # Reference](https://cloud.google.com/bigquery/query-reference#from).
138
138
  # Useful in queries.
139
139
  #
140
+ # @param [Boolean] standard_sql Specifies whether to use BigQuery's
141
+ # [standard
142
+ # SQL](https://cloud.google.com/bigquery/docs/reference/standard-sql/)
143
+ # dialect. Optional. The default value is true.
144
+ # @param [Boolean] legacy_sql Specifies whether to use BigQuery's
145
+ # [legacy
146
+ # SQL](https://cloud.google.com/bigquery/docs/reference/legacy-sql)
147
+ # dialect. Optional. The default value is false.
148
+ #
140
149
  # @example
141
150
  # require "google/cloud/bigquery"
142
151
  #
@@ -148,8 +157,12 @@ module Google
148
157
  #
149
158
  # @!group Attributes
150
159
  #
151
- def query_id
152
- project_id["-"] ? "[#{id}]" : id
160
+ def query_id standard_sql: nil, legacy_sql: nil
161
+ if Convert.resolve_legacy_sql standard_sql, legacy_sql
162
+ "[#{id}]"
163
+ else
164
+ "`#{project_id}.#{dataset_id}.#{table_id}`"
165
+ end
153
166
  end
154
167
 
155
168
  ##
@@ -354,7 +367,6 @@ module Google
354
367
  schema_builder = Schema.from_gapi empty_schema
355
368
  end
356
369
  yield schema_builder
357
- schema_builder.check_for_mutated_schema!
358
370
  if schema_builder.changed?
359
371
  @gapi.schema = schema_builder.to_gapi
360
372
  patch_gapi! :schema
@@ -378,7 +390,7 @@ module Google
378
390
  # @!group Attributes
379
391
  #
380
392
  def headers
381
- fields.map(&:name)
393
+ schema.headers
382
394
  end
383
395
 
384
396
  ##
@@ -713,6 +725,7 @@ module Google
713
725
  #
714
726
  def insert rows, skip_invalid: nil, ignore_unknown: nil
715
727
  rows = [rows] if rows.is_a? Hash
728
+ rows = Convert.to_json_rows rows
716
729
  ensure_service!
717
730
  options = { skip_invalid: skip_invalid,
718
731
  ignore_unknown: ignore_unknown }
@@ -1019,6 +1032,34 @@ module Google
1019
1032
  schema.boolean name, description: description, mode: mode
1020
1033
  end
1021
1034
 
1035
+ ##
1036
+ # Adds a bytes field to the schema.
1037
+ #
1038
+ # See {Schema#bytes}.
1039
+ #
1040
+ # @param [String] name The field name. The name must contain only
1041
+ # letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
1042
+ # start with a letter or underscore. The maximum length is 128
1043
+ # characters.
1044
+ # @param [String] description A description of the field.
1045
+ # @param [Symbol] mode The field's mode. The possible values are
1046
+ # `:nullable`, `:required`, and `:repeated`. The default value is
1047
+ # `:nullable`.
1048
+ #
1049
+ # @example
1050
+ # require "google/cloud/bigquery"
1051
+ #
1052
+ # bigquery = Google::Cloud::Bigquery.new
1053
+ # dataset = bigquery.dataset "my_dataset"
1054
+ # table = dataset.create_table "my_table" do |schema|
1055
+ # schema.bytes "avatar", mode: :required
1056
+ # end
1057
+ #
1058
+ # @!group Schema
1059
+ def bytes name, description: nil, mode: :nullable
1060
+ schema.bytes name, description: description, mode: mode
1061
+ end
1062
+
1022
1063
  ##
1023
1064
  # Adds a timestamp field to the schema.
1024
1065
  #
@@ -1047,6 +1088,90 @@ module Google
1047
1088
  schema.timestamp name, description: description, mode: mode
1048
1089
  end
1049
1090
 
1091
+ ##
1092
+ # Adds a time field to the schema.
1093
+ #
1094
+ # See {Schema#time}.
1095
+ #
1096
+ # @param [String] name The field name. The name must contain only
1097
+ # letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
1098
+ # start with a letter or underscore. The maximum length is 128
1099
+ # characters.
1100
+ # @param [String] description A description of the field.
1101
+ # @param [Symbol] mode The field's mode. The possible values are
1102
+ # `:nullable`, `:required`, and `:repeated`. The default value is
1103
+ # `:nullable`.
1104
+ #
1105
+ # @example
1106
+ # require "google/cloud/bigquery"
1107
+ #
1108
+ # bigquery = Google::Cloud::Bigquery.new
1109
+ # dataset = bigquery.dataset "my_dataset"
1110
+ # table = dataset.create_table "my_table" do |schema|
1111
+ # schema.time "duration", mode: :required
1112
+ # end
1113
+ #
1114
+ # @!group Schema
1115
+ def time name, description: nil, mode: :nullable
1116
+ schema.time name, description: description, mode: mode
1117
+ end
1118
+
1119
+ ##
1120
+ # Adds a datetime field to the schema.
1121
+ #
1122
+ # See {Schema#datetime}.
1123
+ #
1124
+ # @param [String] name The field name. The name must contain only
1125
+ # letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
1126
+ # start with a letter or underscore. The maximum length is 128
1127
+ # characters.
1128
+ # @param [String] description A description of the field.
1129
+ # @param [Symbol] mode The field's mode. The possible values are
1130
+ # `:nullable`, `:required`, and `:repeated`. The default value is
1131
+ # `:nullable`.
1132
+ #
1133
+ # @example
1134
+ # require "google/cloud/bigquery"
1135
+ #
1136
+ # bigquery = Google::Cloud::Bigquery.new
1137
+ # dataset = bigquery.dataset "my_dataset"
1138
+ # table = dataset.create_table "my_table" do |schema|
1139
+ # schema.datetime "target_end", mode: :required
1140
+ # end
1141
+ #
1142
+ # @!group Schema
1143
+ def datetime name, description: nil, mode: :nullable
1144
+ schema.datetime name, description: description, mode: mode
1145
+ end
1146
+
1147
+ ##
1148
+ # Adds a date field to the schema.
1149
+ #
1150
+ # See {Schema#date}.
1151
+ #
1152
+ # @param [String] name The field name. The name must contain only
1153
+ # letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
1154
+ # start with a letter or underscore. The maximum length is 128
1155
+ # characters.
1156
+ # @param [String] description A description of the field.
1157
+ # @param [Symbol] mode The field's mode. The possible values are
1158
+ # `:nullable`, `:required`, and `:repeated`. The default value is
1159
+ # `:nullable`.
1160
+ #
1161
+ # @example
1162
+ # require "google/cloud/bigquery"
1163
+ #
1164
+ # bigquery = Google::Cloud::Bigquery.new
1165
+ # dataset = bigquery.dataset "my_dataset"
1166
+ # table = dataset.create_table "my_table" do |schema|
1167
+ # schema.date "birthday", mode: :required
1168
+ # end
1169
+ #
1170
+ # @!group Schema
1171
+ def date name, description: nil, mode: :nullable
1172
+ schema.date name, description: description, mode: mode
1173
+ end
1174
+
1050
1175
  ##
1051
1176
  # Adds a record field to the schema. A block must be passed describing
1052
1177
  # the nested fields of the record. For more information about nested
@@ -1089,7 +1214,6 @@ module Google
1089
1214
  # Make sure any access changes are saved
1090
1215
  def check_for_mutated_schema!
1091
1216
  return if @schema.nil?
1092
- @schema.check_for_mutated_schema!
1093
1217
  return unless @schema.changed?
1094
1218
  @gapi.schema = @schema.to_gapi
1095
1219
  patch_gapi! :schema
@@ -30,7 +30,7 @@ module Google
30
30
  #
31
31
  # fourpm = Google::Cloud::Bigquery::Time.new "16:00:00"
32
32
  # data = bigquery.query "SELECT name " \
33
- # "FROM [my_proj:my_data.my_table]" \
33
+ # "FROM `my_proj.my_data.my_table`" \
34
34
  # "WHERE time_of_date = @time",
35
35
  # params: { time: fourpm }
36
36
  #
@@ -45,7 +45,7 @@ module Google
45
45
  #
46
46
  # precise_time = Google::Cloud::Bigquery::Time.new "16:35:15.376541"
47
47
  # data = bigquery.query "SELECT name " \
48
- # "FROM [my_proj:my_data.my_table]" \
48
+ # "FROM `my_proj.my_data.my_table`" \
49
49
  # "WHERE time_of_date >= @time",
50
50
  # params: { time: precise_time }
51
51
  #
@@ -16,7 +16,7 @@
16
16
  module Google
17
17
  module Cloud
18
18
  module Bigquery
19
- VERSION = "0.23.0"
19
+ VERSION = "0.24.0"
20
20
  end
21
21
  end
22
22
  end
@@ -40,7 +40,7 @@ module Google
40
40
  # bigquery = Google::Cloud::Bigquery.new
41
41
  # dataset = bigquery.dataset "my_dataset"
42
42
  # view = dataset.create_view "my_view",
43
- # "SELECT name, age FROM [proj:dataset.users]"
43
+ # "SELECT name, age FROM proj.dataset.users"
44
44
  #
45
45
  class View
46
46
  ##
@@ -98,7 +98,7 @@ module Google
98
98
 
99
99
  ##
100
100
  # The combined Project ID, Dataset ID, and Table ID for this table, in
101
- # the format specified by the [Query
101
+ # the format specified by the [Legacy SQL Query
102
102
  # Reference](https://cloud.google.com/bigquery/query-reference#from):
103
103
  # `project_name:datasetId.tableId`. To use this value in queries see
104
104
  # {#query_id}.
@@ -115,6 +115,15 @@ module Google
115
115
  # Reference](https://cloud.google.com/bigquery/query-reference#from).
116
116
  # Useful in queries.
117
117
  #
118
+ # @param [Boolean] standard_sql Specifies whether to use BigQuery's
119
+ # [standard
120
+ # SQL](https://cloud.google.com/bigquery/docs/reference/standard-sql/)
121
+ # dialect. Optional. The default value is true.
122
+ # @param [Boolean] legacy_sql Specifies whether to use BigQuery's
123
+ # [legacy
124
+ # SQL](https://cloud.google.com/bigquery/docs/reference/legacy-sql)
125
+ # dialect. Optional. The default value is false.
126
+ #
118
127
  # @example
119
128
  # require "google/cloud/bigquery"
120
129
  #
@@ -126,8 +135,12 @@ module Google
126
135
  #
127
136
  # @!group Attributes
128
137
  #
129
- def query_id
130
- project_id["-"] ? "[#{id}]" : id
138
+ def query_id standard_sql: nil, legacy_sql: nil
139
+ if Convert.resolve_legacy_sql standard_sql, legacy_sql
140
+ "[#{id}]"
141
+ else
142
+ "`#{project_id}.#{dataset_id}.#{table_id}`"
143
+ end
131
144
  end
132
145
 
133
146
  ##
@@ -287,7 +300,7 @@ module Google
287
300
  # @!group Attributes
288
301
  #
289
302
  def headers
290
- fields.map(&:name)
303
+ schema.headers
291
304
  end
292
305
 
293
306
  ##
@@ -306,6 +319,14 @@ module Google
306
319
  # Reference
307
320
  #
308
321
  # @param [String] new_query The query that defines the view.
322
+ # @param [Boolean] standard_sql Specifies whether to use BigQuery's
323
+ # [standard
324
+ # SQL](https://cloud.google.com/bigquery/docs/reference/standard-sql/)
325
+ # dialect. Optional. The default value is true.
326
+ # @param [Boolean] legacy_sql Specifies whether to use BigQuery's
327
+ # [legacy
328
+ # SQL](https://cloud.google.com/bigquery/docs/reference/legacy-sql)
329
+ # dialect. Optional. The default value is false.
309
330
  #
310
331
  # @example
311
332
  # require "google/cloud/bigquery"
@@ -315,13 +336,15 @@ module Google
315
336
  # view = dataset.table "my_view"
316
337
  #
317
338
  # view.query = "SELECT first_name FROM " \
318
- # "[my_project:my_dataset.my_table]"
339
+ # "`my_project.my_dataset.my_table`"
319
340
  #
320
341
  # @!group Lifecycle
321
342
  #
322
- def query= new_query
343
+ def query= new_query, standard_sql: nil, legacy_sql: nil
323
344
  @gapi.view ||= Google::Apis::BigqueryV2::ViewDefinition.new
324
345
  @gapi.view.update! query: new_query
346
+ @gapi.view.update! use_legacy_sql: \
347
+ Convert.resolve_legacy_sql(standard_sql, legacy_sql)
325
348
  patch_view_gapi! :query
326
349
  end
327
350
 
@@ -426,6 +449,12 @@ module Google
426
449
  fail "Must have active connection" unless service
427
450
  end
428
451
 
452
+ def resolve_legacy_sql legacy_sql, standard_sql
453
+ return legacy_sql unless legacy_sql.nil?
454
+ return !standard_sql unless standard_sql.nil?
455
+ false
456
+ end
457
+
429
458
  def patch_gapi! *attributes
430
459
  return if attributes.empty?
431
460
  patch_args = Hash[attributes.map do |attr|