google-cloud-bigquery 1.12.0 → 1.38.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (50) hide show
  1. checksums.yaml +4 -4
  2. data/AUTHENTICATION.md +9 -28
  3. data/CHANGELOG.md +372 -1
  4. data/CONTRIBUTING.md +328 -116
  5. data/LOGGING.md +2 -2
  6. data/OVERVIEW.md +21 -20
  7. data/TROUBLESHOOTING.md +2 -8
  8. data/lib/google/cloud/bigquery/argument.rb +197 -0
  9. data/lib/google/cloud/bigquery/convert.rb +154 -170
  10. data/lib/google/cloud/bigquery/copy_job.rb +40 -23
  11. data/lib/google/cloud/bigquery/credentials.rb +5 -12
  12. data/lib/google/cloud/bigquery/data.rb +109 -18
  13. data/lib/google/cloud/bigquery/dataset/access.rb +322 -51
  14. data/lib/google/cloud/bigquery/dataset/list.rb +7 -13
  15. data/lib/google/cloud/bigquery/dataset.rb +960 -279
  16. data/lib/google/cloud/bigquery/external/avro_source.rb +107 -0
  17. data/lib/google/cloud/bigquery/external/bigtable_source/column.rb +404 -0
  18. data/lib/google/cloud/bigquery/external/bigtable_source/column_family.rb +945 -0
  19. data/lib/google/cloud/bigquery/external/bigtable_source.rb +230 -0
  20. data/lib/google/cloud/bigquery/external/csv_source.rb +481 -0
  21. data/lib/google/cloud/bigquery/external/data_source.rb +771 -0
  22. data/lib/google/cloud/bigquery/external/json_source.rb +170 -0
  23. data/lib/google/cloud/bigquery/external/parquet_source.rb +148 -0
  24. data/lib/google/cloud/bigquery/external/sheets_source.rb +166 -0
  25. data/lib/google/cloud/bigquery/external.rb +50 -2256
  26. data/lib/google/cloud/bigquery/extract_job.rb +217 -58
  27. data/lib/google/cloud/bigquery/insert_response.rb +1 -3
  28. data/lib/google/cloud/bigquery/job/list.rb +13 -20
  29. data/lib/google/cloud/bigquery/job.rb +286 -11
  30. data/lib/google/cloud/bigquery/load_job.rb +801 -133
  31. data/lib/google/cloud/bigquery/model/list.rb +5 -9
  32. data/lib/google/cloud/bigquery/model.rb +247 -16
  33. data/lib/google/cloud/bigquery/policy.rb +432 -0
  34. data/lib/google/cloud/bigquery/project/list.rb +6 -11
  35. data/lib/google/cloud/bigquery/project.rb +526 -243
  36. data/lib/google/cloud/bigquery/query_job.rb +584 -125
  37. data/lib/google/cloud/bigquery/routine/list.rb +165 -0
  38. data/lib/google/cloud/bigquery/routine.rb +1227 -0
  39. data/lib/google/cloud/bigquery/schema/field.rb +413 -63
  40. data/lib/google/cloud/bigquery/schema.rb +221 -48
  41. data/lib/google/cloud/bigquery/service.rb +186 -109
  42. data/lib/google/cloud/bigquery/standard_sql.rb +269 -53
  43. data/lib/google/cloud/bigquery/table/async_inserter.rb +86 -42
  44. data/lib/google/cloud/bigquery/table/list.rb +6 -11
  45. data/lib/google/cloud/bigquery/table.rb +1188 -326
  46. data/lib/google/cloud/bigquery/time.rb +6 -0
  47. data/lib/google/cloud/bigquery/version.rb +1 -1
  48. data/lib/google/cloud/bigquery.rb +18 -8
  49. data/lib/google-cloud-bigquery.rb +15 -13
  50. metadata +67 -40
@@ -37,8 +37,8 @@ module Google
37
37
  # bigquery = Google::Cloud::Bigquery.new
38
38
  # dataset = bigquery.dataset "my_dataset"
39
39
  #
40
- # gs_url = "gs://my-bucket/file-name.csv"
41
- # load_job = dataset.load_job "my_new_table", gs_url do |schema|
40
+ # gcs_uri = "gs://my-bucket/file-name.csv"
41
+ # load_job = dataset.load_job "my_new_table", gcs_uri do |schema|
42
42
  # schema.string "first_name", mode: :required
43
43
  # schema.record "cities_lived", mode: :repeated do |nested_schema|
44
44
  # nested_schema.string "place", mode: :required
@@ -67,9 +67,7 @@ module Google
67
67
  def destination
68
68
  table = @gapi.configuration.load.destination_table
69
69
  return nil unless table
70
- retrieve_table table.project_id,
71
- table.dataset_id,
72
- table.table_id
70
+ retrieve_table table.project_id, table.dataset_id, table.table_id
73
71
  end
74
72
 
75
73
  ##
@@ -114,8 +112,7 @@ module Google
114
112
  # `false` otherwise.
115
113
  #
116
114
  def iso8859_1?
117
- val = @gapi.configuration.load.encoding
118
- val == "ISO-8859-1"
115
+ @gapi.configuration.load.encoding == "ISO-8859-1"
119
116
  end
120
117
 
121
118
  ##
@@ -197,8 +194,7 @@ module Google
197
194
  # `NEWLINE_DELIMITED_JSON`, `false` otherwise.
198
195
  #
199
196
  def json?
200
- val = @gapi.configuration.load.source_format
201
- val == "NEWLINE_DELIMITED_JSON"
197
+ @gapi.configuration.load.source_format == "NEWLINE_DELIMITED_JSON"
202
198
  end
203
199
 
204
200
  ##
@@ -220,8 +216,27 @@ module Google
220
216
  # `false` otherwise.
221
217
  #
222
218
  def backup?
223
- val = @gapi.configuration.load.source_format
224
- val == "DATASTORE_BACKUP"
219
+ @gapi.configuration.load.source_format == "DATASTORE_BACKUP"
220
+ end
221
+
222
+ ##
223
+ # Checks if the source format is ORC.
224
+ #
225
+ # @return [Boolean] `true` when the source format is `ORC`,
226
+ # `false` otherwise.
227
+ #
228
+ def orc?
229
+ @gapi.configuration.load.source_format == "ORC"
230
+ end
231
+
232
+ ##
233
+ # Checks if the source format is Parquet.
234
+ #
235
+ # @return [Boolean] `true` when the source format is `PARQUET`,
236
+ # `false` otherwise.
237
+ #
238
+ def parquet?
239
+ @gapi.configuration.load.source_format == "PARQUET"
225
240
  end
226
241
 
227
242
  ##
@@ -350,10 +365,169 @@ module Google
350
365
  end
351
366
 
352
367
  ###
353
- # Checks if the destination table will be time-partitioned. See
368
+ # Checks if hive partitioning options are set.
369
+ #
370
+ # @see https://cloud.google.com/bigquery/docs/hive-partitioned-loads-gcs Loading externally partitioned data
371
+ #
372
+ # @return [Boolean] `true` when hive partitioning options are set, or `false` otherwise.
373
+ #
374
+ # @!group Attributes
375
+ #
376
+ def hive_partitioning?
377
+ !@gapi.configuration.load.hive_partitioning_options.nil?
378
+ end
379
+
380
+ ###
381
+ # The mode of hive partitioning to use when reading data. The following modes are supported:
382
+ #
383
+ # 1. `AUTO`: automatically infer partition key name(s) and type(s).
384
+ # 2. `STRINGS`: automatically infer partition key name(s). All types are interpreted as strings.
385
+ # 3. `CUSTOM`: partition key schema is encoded in the source URI prefix.
386
+ #
387
+ # @see https://cloud.google.com/bigquery/docs/hive-partitioned-loads-gcs Loading externally partitioned data
388
+ #
389
+ # @return [String, nil] The mode of hive partitioning, or `nil` if not set.
390
+ #
391
+ # @!group Attributes
392
+ #
393
+ def hive_partitioning_mode
394
+ @gapi.configuration.load.hive_partitioning_options.mode if hive_partitioning?
395
+ end
396
+
397
+ ###
398
+ # The common prefix for all source uris when hive partition detection is requested. The prefix must end
399
+ # immediately before the partition key encoding begins. For example, consider files following this data layout:
400
+ #
401
+ # ```
402
+ # gs://bucket/path_to_table/dt=2019-01-01/country=BR/id=7/file.avro
403
+ # gs://bucket/path_to_table/dt=2018-12-31/country=CA/id=3/file.avro
404
+ # ```
405
+ #
406
+ # When hive partitioning is requested with either `AUTO` or `STRINGS` mode, the common prefix can be either of
407
+ # `gs://bucket/path_to_table` or `gs://bucket/path_to_table/` (trailing slash does not matter).
408
+ #
409
+ # @see https://cloud.google.com/bigquery/docs/hive-partitioned-loads-gcs Loading externally partitioned data
410
+ #
411
+ # @return [String, nil] The common prefix for all source uris, or `nil` if not set.
412
+ #
413
+ # @!group Attributes
414
+ #
415
+ def hive_partitioning_source_uri_prefix
416
+ @gapi.configuration.load.hive_partitioning_options.source_uri_prefix if hive_partitioning?
417
+ end
418
+
419
+ ###
420
+ # Checks if Parquet options are set.
421
+ #
422
+ # @see https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-parquet Loading Parquet data from Cloud
423
+ # Storage
424
+ #
425
+ # @return [Boolean] `true` when Parquet options are set, or `false` otherwise.
426
+ #
427
+ # @!group Attributes
428
+ #
429
+ def parquet_options?
430
+ !@gapi.configuration.load.parquet_options.nil?
431
+ end
432
+
433
+ ###
434
+ # Indicates whether to use schema inference specifically for Parquet `LIST` logical type.
435
+ #
436
+ # @see https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-parquet Loading Parquet data from Cloud
437
+ # Storage
438
+ #
439
+ # @return [Boolean, nil] The `enable_list_inference` value in Parquet options, or `nil` if Parquet options are
440
+ # not set.
441
+ #
442
+ # @!group Attributes
443
+ #
444
+ def parquet_enable_list_inference?
445
+ @gapi.configuration.load.parquet_options.enable_list_inference if parquet_options?
446
+ end
447
+
448
+ ###
449
+ # Indicates whether to infer Parquet `ENUM` logical type as `STRING` instead of `BYTES` by default.
450
+ #
451
+ # @see https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-parquet Loading Parquet data from Cloud
452
+ # Storage
453
+ #
454
+ # @return [Boolean, nil] The `enum_as_string` value in Parquet options, or `nil` if Parquet options are not set.
455
+ #
456
+ # @!group Attributes
457
+ #
458
+ def parquet_enum_as_string?
459
+ @gapi.configuration.load.parquet_options.enum_as_string if parquet_options?
460
+ end
461
+
462
+ ###
463
+ # Checks if the destination table will be range partitioned. See [Creating and using integer range partitioned
464
+ # tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
465
+ #
466
+ # @return [Boolean] `true` when the table is range partitioned, or `false` otherwise.
467
+ #
468
+ # @!group Attributes
469
+ #
470
+ def range_partitioning?
471
+ !@gapi.configuration.load.range_partitioning.nil?
472
+ end
473
+
474
+ ###
475
+ # The field on which the destination table will be range partitioned, if any. The field must be a
476
+ # top-level `NULLABLE/REQUIRED` field. The only supported type is `INTEGER/INT64`. See
477
+ # [Creating and using integer range partitioned
478
+ # tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
479
+ #
480
+ # @return [String, nil] The partition field, if a field was configured, or `nil` if not range partitioned.
481
+ #
482
+ # @!group Attributes
483
+ #
484
+ def range_partitioning_field
485
+ @gapi.configuration.load.range_partitioning.field if range_partitioning?
486
+ end
487
+
488
+ ###
489
+ # The start of range partitioning, inclusive. See [Creating and using integer range partitioned
490
+ # tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
491
+ #
492
+ # @return [Integer, nil] The start of range partitioning, inclusive, or `nil` if not range partitioned.
493
+ #
494
+ # @!group Attributes
495
+ #
496
+ def range_partitioning_start
497
+ @gapi.configuration.load.range_partitioning.range.start if range_partitioning?
498
+ end
499
+
500
+ ###
501
+ # The width of each interval. See [Creating and using integer range partitioned
502
+ # tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
503
+ #
504
+ # @return [Integer, nil] The width of each interval, for data in range partitions, or `nil` if not range
505
+ # partitioned.
506
+ #
507
+ # @!group Attributes
508
+ #
509
+ def range_partitioning_interval
510
+ return nil unless range_partitioning?
511
+ @gapi.configuration.load.range_partitioning.range.interval
512
+ end
513
+
514
+ ###
515
+ # The end of range partitioning, exclusive. See [Creating and using integer range partitioned
516
+ # tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
517
+ #
518
+ # @return [Integer, nil] The end of range partitioning, exclusive, or `nil` if not range partitioned.
519
+ #
520
+ # @!group Attributes
521
+ #
522
+ def range_partitioning_end
523
+ @gapi.configuration.load.range_partitioning.range.end if range_partitioning?
524
+ end
525
+
526
+ ###
527
+ # Checks if the destination table will be time partitioned. See
354
528
  # [Partitioned Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
355
529
  #
356
- # @return [Boolean, nil] `true` when the table will be time-partitioned,
530
+ # @return [Boolean] `true` when the table will be time-partitioned,
357
531
  # or `false` otherwise.
358
532
  #
359
533
  # @!group Attributes
@@ -363,11 +537,12 @@ module Google
363
537
  end
364
538
 
365
539
  ###
366
- # The period for which the destination table will be partitioned, if
540
+ # The period for which the destination table will be time partitioned, if
367
541
  # any. See [Partitioned Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
368
542
  #
369
- # @return [String, nil] The partition type. Currently the only supported
370
- # value is "DAY", or `nil` if not present.
543
+ # @return [String, nil] The time partition type. The supported types are `DAY`,
544
+ # `HOUR`, `MONTH`, and `YEAR`, which will generate one partition per day,
545
+ # hour, month, and year, respectively; or `nil` if not present.
371
546
  #
372
547
  # @!group Attributes
373
548
  #
@@ -376,13 +551,13 @@ module Google
376
551
  end
377
552
 
378
553
  ###
379
- # The field on which the destination table will be partitioned, if any.
380
- # If not set, the destination table will be partitioned by pseudo column
381
- # `_PARTITIONTIME`; if set, the table will be partitioned by this field.
554
+ # The field on which the destination table will be time partitioned, if any.
555
+ # If not set, the destination table will be time partitioned by pseudo column
556
+ # `_PARTITIONTIME`; if set, the table will be time partitioned by this field.
382
557
  # See [Partitioned Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
383
558
  #
384
- # @return [String, nil] The partition field, if a field was configured.
385
- # `nil` if not partitioned or not set (partitioned by pseudo column
559
+ # @return [String, nil] The time partition field, if a field was configured.
560
+ # `nil` if not time partitioned or not set (partitioned by pseudo column
386
561
  # '_PARTITIONTIME').
387
562
  #
388
563
  # @!group Attributes
@@ -392,28 +567,29 @@ module Google
392
567
  end
393
568
 
394
569
  ###
395
- # The expiration for the destination table partitions, if any, in
570
+ # The expiration for the destination table time partitions, if any, in
396
571
  # seconds. See [Partitioned
397
572
  # Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
398
573
  #
399
574
  # @return [Integer, nil] The expiration time, in seconds, for data in
400
- # partitions, or `nil` if not present.
575
+ # time partitions, or `nil` if not present.
401
576
  #
402
577
  # @!group Attributes
403
578
  #
404
579
  def time_partitioning_expiration
405
- @gapi.configuration.load.time_partitioning.expiration_ms / 1_000 if
406
- time_partitioning? &&
407
- !@gapi.configuration.load.time_partitioning.expiration_ms.nil?
580
+ return nil unless time_partitioning?
581
+ return nil if @gapi.configuration.load.time_partitioning.expiration_ms.nil?
582
+
583
+ @gapi.configuration.load.time_partitioning.expiration_ms / 1_000
408
584
  end
409
585
 
410
586
  ###
411
587
  # If set to true, queries over the destination table will require a
412
- # partition filter that can be used for partition elimination to be
588
+ # time partition filter that can be used for partition elimination to be
413
589
  # specified. See [Partitioned
414
590
  # Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
415
591
  #
416
- # @return [Boolean] `true` when a partition filter will be required,
592
+ # @return [Boolean] `true` when a time partition filter will be required,
417
593
  # or `false` otherwise.
418
594
  #
419
595
  # @!group Attributes
@@ -427,10 +603,15 @@ module Google
427
603
  ###
428
604
  # Checks if the destination table will be clustered.
429
605
  #
606
+ # See {LoadJob::Updater#clustering_fields=}, {Table#clustering_fields} and
607
+ # {Table#clustering_fields=}.
608
+ #
430
609
  # @see https://cloud.google.com/bigquery/docs/clustered-tables
431
- # Introduction to Clustered Tables
610
+ # Introduction to clustered tables
611
+ # @see https://cloud.google.com/bigquery/docs/creating-clustered-tables
612
+ # Creating and using clustered tables
432
613
  #
433
- # @return [Boolean, nil] `true` when the table will be clustered,
614
+ # @return [Boolean] `true` when the table will be clustered,
434
615
  # or `false` otherwise.
435
616
  #
436
617
  # @!group Attributes
@@ -445,14 +626,16 @@ module Google
445
626
  # be first partitioned and subsequently clustered. The order of the
446
627
  # returned fields determines the sort order of the data.
447
628
  #
448
- # See {LoadJob::Updater#clustering_fields=}.
629
+ # BigQuery supports clustering for both partitioned and non-partitioned
630
+ # tables.
631
+ #
632
+ # See {LoadJob::Updater#clustering_fields=}, {Table#clustering_fields} and
633
+ # {Table#clustering_fields=}.
449
634
  #
450
- # @see https://cloud.google.com/bigquery/docs/partitioned-tables
451
- # Partitioned Tables
452
635
  # @see https://cloud.google.com/bigquery/docs/clustered-tables
453
- # Introduction to Clustered Tables
636
+ # Introduction to clustered tables
454
637
  # @see https://cloud.google.com/bigquery/docs/creating-clustered-tables
455
- # Creating and Using Clustered Tables
638
+ # Creating and using clustered tables
456
639
  #
457
640
  # @return [Array<String>, nil] The clustering fields, or `nil` if the
458
641
  # destination table will not be clustered.
@@ -473,6 +656,7 @@ module Google
473
656
  ##
474
657
  # @private Create an Updater object.
475
658
  def initialize gapi
659
+ super()
476
660
  @updates = []
477
661
  @gapi = gapi
478
662
  @schema = nil
@@ -560,13 +744,19 @@ module Google
560
744
  # See {Schema#string}.
561
745
  #
562
746
  # @param [String] name The field name. The name must contain only
563
- # letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
747
+ # letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
564
748
  # start with a letter or underscore. The maximum length is 128
565
749
  # characters.
566
750
  # @param [String] description A description of the field.
567
751
  # @param [Symbol] mode The field's mode. The possible values are
568
752
  # `:nullable`, `:required`, and `:repeated`. The default value is
569
753
  # `:nullable`.
754
+ # @param [Array<String>, String] policy_tags The policy tag list or
755
+ # single policy tag for the field. Policy tag identifiers are of
756
+ # the form `projects/*/locations/*/taxonomies/*/policyTags/*`.
757
+ # At most 1 policy tag is currently allowed.
758
+ # @param [Integer] max_length The maximum UTF-8 length of strings
759
+ # allowed in the field.
570
760
  #
571
761
  # @example
572
762
  # require "google/cloud/bigquery"
@@ -578,8 +768,8 @@ module Google
578
768
  # end
579
769
  #
580
770
  # @!group Schema
581
- def string name, description: nil, mode: :nullable
582
- schema.string name, description: description, mode: mode
771
+ def string name, description: nil, mode: :nullable, policy_tags: nil, max_length: nil
772
+ schema.string name, description: description, mode: mode, policy_tags: policy_tags, max_length: max_length
583
773
  end
584
774
 
585
775
  ##
@@ -588,13 +778,17 @@ module Google
588
778
  # See {Schema#integer}.
589
779
  #
590
780
  # @param [String] name The field name. The name must contain only
591
- # letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
781
+ # letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
592
782
  # start with a letter or underscore. The maximum length is 128
593
783
  # characters.
594
784
  # @param [String] description A description of the field.
595
785
  # @param [Symbol] mode The field's mode. The possible values are
596
786
  # `:nullable`, `:required`, and `:repeated`. The default value is
597
787
  # `:nullable`.
788
+ # @param [Array<String>, String] policy_tags The policy tag list or
789
+ # single policy tag for the field. Policy tag identifiers are of
790
+ # the form `projects/*/locations/*/taxonomies/*/policyTags/*`.
791
+ # At most 1 policy tag is currently allowed.
598
792
  #
599
793
  # @example
600
794
  # require "google/cloud/bigquery"
@@ -606,8 +800,8 @@ module Google
606
800
  # end
607
801
  #
608
802
  # @!group Schema
609
- def integer name, description: nil, mode: :nullable
610
- schema.integer name, description: description, mode: mode
803
+ def integer name, description: nil, mode: :nullable, policy_tags: nil
804
+ schema.integer name, description: description, mode: mode, policy_tags: policy_tags
611
805
  end
612
806
 
613
807
  ##
@@ -616,13 +810,17 @@ module Google
616
810
  # See {Schema#float}.
617
811
  #
618
812
  # @param [String] name The field name. The name must contain only
619
- # letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
813
+ # letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
620
814
  # start with a letter or underscore. The maximum length is 128
621
815
  # characters.
622
816
  # @param [String] description A description of the field.
623
817
  # @param [Symbol] mode The field's mode. The possible values are
624
818
  # `:nullable`, `:required`, and `:repeated`. The default value is
625
819
  # `:nullable`.
820
+ # @param [Array<String>, String] policy_tags The policy tag list or
821
+ # single policy tag for the field. Policy tag identifiers are of
822
+ # the form `projects/*/locations/*/taxonomies/*/policyTags/*`.
823
+ # At most 1 policy tag is currently allowed.
626
824
  #
627
825
  # @example
628
826
  # require "google/cloud/bigquery"
@@ -634,25 +832,48 @@ module Google
634
832
  # end
635
833
  #
636
834
  # @!group Schema
637
- def float name, description: nil, mode: :nullable
638
- schema.float name, description: description, mode: mode
835
+ def float name, description: nil, mode: :nullable, policy_tags: nil
836
+ schema.float name, description: description, mode: mode, policy_tags: policy_tags
639
837
  end
640
838
 
641
839
  ##
642
- # Adds a numeric number field to the schema. Numeric is a
643
- # fixed-precision numeric type with 38 decimal digits, 9 that follow
644
- # the decimal point.
840
+ # Adds a numeric number field to the schema. `NUMERIC` is a decimal
841
+ # type with fixed precision and scale. Precision is the number of
842
+ # digits that the number contains. Scale is how many of these
843
+ # digits appear after the decimal point. It supports:
844
+ #
845
+ # Precision: 38
846
+ # Scale: 9
847
+ # Min: -9.9999999999999999999999999999999999999E+28
848
+ # Max: 9.9999999999999999999999999999999999999E+28
849
+ #
850
+ # This type can represent decimal fractions exactly, and is suitable
851
+ # for financial calculations.
645
852
  #
646
853
  # See {Schema#numeric}
647
854
  #
648
855
  # @param [String] name The field name. The name must contain only
649
- # letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
856
+ # letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
650
857
  # start with a letter or underscore. The maximum length is 128
651
858
  # characters.
652
859
  # @param [String] description A description of the field.
653
860
  # @param [Symbol] mode The field's mode. The possible values are
654
861
  # `:nullable`, `:required`, and `:repeated`. The default value is
655
862
  # `:nullable`.
863
+ # @param [Array<String>, String] policy_tags The policy tag list or
864
+ # single policy tag for the field. Policy tag identifiers are of
865
+ # the form `projects/*/locations/*/taxonomies/*/policyTags/*`.
866
+ # At most 1 policy tag is currently allowed.
867
+ # @param [Integer] precision The precision (maximum number of total
868
+ # digits) for the field. Acceptable values for precision must be:
869
+ # `1 ≤ (precision - scale) ≤ 29`. Values for scale must be:
870
+ # `0 ≤ scale ≤ 9`. If the scale value is set, the precision value
871
+ # must be set as well.
872
+ # @param [Integer] scale The scale (maximum number of digits in the
873
+ # fractional part) for the field. Acceptable values for precision
874
+ # must be: `1 ≤ (precision - scale) ≤ 29`. Values for scale must
875
+ # be: `0 ≤ scale ≤ 9`. If the scale value is set, the precision
876
+ # value must be set as well.
656
877
  #
657
878
  # @example
658
879
  # require "google/cloud/bigquery"
@@ -664,8 +885,71 @@ module Google
664
885
  # end
665
886
  #
666
887
  # @!group Schema
667
- def numeric name, description: nil, mode: :nullable
668
- schema.numeric name, description: description, mode: mode
888
+ def numeric name, description: nil, mode: :nullable, policy_tags: nil, precision: nil, scale: nil
889
+ schema.numeric name,
890
+ description: description,
891
+ mode: mode,
892
+ policy_tags: policy_tags,
893
+ precision: precision,
894
+ scale: scale
895
+ end
896
+
897
+ ##
898
+ # Adds a bignumeric number field to the schema. `BIGNUMERIC` is a
899
+ # decimal type with fixed precision and scale. Precision is the
900
+ # number of digits that the number contains. Scale is how many of
901
+ # these digits appear after the decimal point. It supports:
902
+ #
903
+ # Precision: 76.76 (the 77th digit is partial)
904
+ # Scale: 38
905
+ # Min: -5.7896044618658097711785492504343953926634992332820282019728792003956564819968E+38
906
+ # Max: 5.7896044618658097711785492504343953926634992332820282019728792003956564819967E+38
907
+ #
908
+ # This type can represent decimal fractions exactly, and is suitable
909
+ # for financial calculations.
910
+ #
911
+ # See {Schema#bignumeric}
912
+ #
913
+ # @param [String] name The field name. The name must contain only
914
+ # letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
915
+ # start with a letter or underscore. The maximum length is 128
916
+ # characters.
917
+ # @param [String] description A description of the field.
918
+ # @param [Symbol] mode The field's mode. The possible values are
919
+ # `:nullable`, `:required`, and `:repeated`. The default value is
920
+ # `:nullable`.
921
+ # @param [Array<String>, String] policy_tags The policy tag list or
922
+ # single policy tag for the field. Policy tag identifiers are of
923
+ # the form `projects/*/locations/*/taxonomies/*/policyTags/*`.
924
+ # At most 1 policy tag is currently allowed.
925
+ # @param [Integer] precision The precision (maximum number of total
926
+ # digits) for the field. Acceptable values for precision must be:
927
+ # `1 ≤ (precision - scale) ≤ 38`. Values for scale must be:
928
+ # `0 ≤ scale ≤ 38`. If the scale value is set, the precision value
929
+ # must be set as well.
930
+ # @param [Integer] scale The scale (maximum number of digits in the
931
+ # fractional part) for the field. Acceptable values for precision
932
+ # must be: `1 ≤ (precision - scale) ≤ 38`. Values for scale must
933
+ # be: `0 ≤ scale ≤ 38`. If the scale value is set, the precision
934
+ # value must be set as well.
935
+ #
936
+ # @example
937
+ # require "google/cloud/bigquery"
938
+ #
939
+ # bigquery = Google::Cloud::Bigquery.new
940
+ # dataset = bigquery.dataset "my_dataset"
941
+ # job = dataset.load_job "my_table", "gs://abc/file" do |schema|
942
+ # schema.bignumeric "total_cost", mode: :required
943
+ # end
944
+ #
945
+ # @!group Schema
946
+ def bignumeric name, description: nil, mode: :nullable, policy_tags: nil, precision: nil, scale: nil
947
+ schema.bignumeric name,
948
+ description: description,
949
+ mode: mode,
950
+ policy_tags: policy_tags,
951
+ precision: precision,
952
+ scale: scale
669
953
  end
670
954
 
671
955
  ##
@@ -674,13 +958,17 @@ module Google
674
958
  # See {Schema#boolean}.
675
959
  #
676
960
  # @param [String] name The field name. The name must contain only
677
- # letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
961
+ # letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
678
962
  # start with a letter or underscore. The maximum length is 128
679
963
  # characters.
680
964
  # @param [String] description A description of the field.
681
965
  # @param [Symbol] mode The field's mode. The possible values are
682
966
  # `:nullable`, `:required`, and `:repeated`. The default value is
683
967
  # `:nullable`.
968
+ # @param [Array<String>, String] policy_tags The policy tag list or
969
+ # single policy tag for the field. Policy tag identifiers are of
970
+ # the form `projects/*/locations/*/taxonomies/*/policyTags/*`.
971
+ # At most 1 policy tag is currently allowed.
684
972
  #
685
973
  # @example
686
974
  # require "google/cloud/bigquery"
@@ -692,8 +980,8 @@ module Google
692
980
  # end
693
981
  #
694
982
  # @!group Schema
695
- def boolean name, description: nil, mode: :nullable
696
- schema.boolean name, description: description, mode: mode
983
+ def boolean name, description: nil, mode: :nullable, policy_tags: nil
984
+ schema.boolean name, description: description, mode: mode, policy_tags: policy_tags
697
985
  end
698
986
 
699
987
  ##
@@ -702,13 +990,19 @@ module Google
702
990
  # See {Schema#bytes}.
703
991
  #
704
992
  # @param [String] name The field name. The name must contain only
705
- # letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
993
+ # letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
706
994
  # start with a letter or underscore. The maximum length is 128
707
995
  # characters.
708
996
  # @param [String] description A description of the field.
709
997
  # @param [Symbol] mode The field's mode. The possible values are
710
998
  # `:nullable`, `:required`, and `:repeated`. The default value is
711
999
  # `:nullable`.
1000
+ # @param [Array<String>, String] policy_tags The policy tag list or
1001
+ # single policy tag for the field. Policy tag identifiers are of
1002
+ # the form `projects/*/locations/*/taxonomies/*/policyTags/*`.
1003
+ # At most 1 policy tag is currently allowed.
1004
+ # @param [Integer] max_length The maximum the maximum number of
1005
+ # bytes in the field.
712
1006
  #
713
1007
  # @example
714
1008
  # require "google/cloud/bigquery"
@@ -720,8 +1014,8 @@ module Google
720
1014
  # end
721
1015
  #
722
1016
  # @!group Schema
723
- def bytes name, description: nil, mode: :nullable
724
- schema.bytes name, description: description, mode: mode
1017
+ def bytes name, description: nil, mode: :nullable, policy_tags: nil, max_length: nil
1018
+ schema.bytes name, description: description, mode: mode, policy_tags: policy_tags, max_length: max_length
725
1019
  end
726
1020
 
727
1021
  ##
@@ -730,13 +1024,17 @@ module Google
730
1024
  # See {Schema#timestamp}.
731
1025
  #
732
1026
  # @param [String] name The field name. The name must contain only
733
- # letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
1027
+ # letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
734
1028
  # start with a letter or underscore. The maximum length is 128
735
1029
  # characters.
736
1030
  # @param [String] description A description of the field.
737
1031
  # @param [Symbol] mode The field's mode. The possible values are
738
1032
  # `:nullable`, `:required`, and `:repeated`. The default value is
739
1033
  # `:nullable`.
1034
+ # @param [Array<String>, String] policy_tags The policy tag list or
1035
+ # single policy tag for the field. Policy tag identifiers are of
1036
+ # the form `projects/*/locations/*/taxonomies/*/policyTags/*`.
1037
+ # At most 1 policy tag is currently allowed.
740
1038
  #
741
1039
  # @example
742
1040
  # require "google/cloud/bigquery"
@@ -748,8 +1046,8 @@ module Google
748
1046
  # end
749
1047
  #
750
1048
  # @!group Schema
751
- def timestamp name, description: nil, mode: :nullable
752
- schema.timestamp name, description: description, mode: mode
1049
+ def timestamp name, description: nil, mode: :nullable, policy_tags: nil
1050
+ schema.timestamp name, description: description, mode: mode, policy_tags: policy_tags
753
1051
  end
754
1052
 
755
1053
  ##
@@ -758,13 +1056,17 @@ module Google
758
1056
  # See {Schema#time}.
759
1057
  #
760
1058
  # @param [String] name The field name. The name must contain only
761
- # letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
1059
+ # letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
762
1060
  # start with a letter or underscore. The maximum length is 128
763
1061
  # characters.
764
1062
  # @param [String] description A description of the field.
765
1063
  # @param [Symbol] mode The field's mode. The possible values are
766
1064
  # `:nullable`, `:required`, and `:repeated`. The default value is
767
1065
  # `:nullable`.
1066
+ # @param [Array<String>, String] policy_tags The policy tag list or
1067
+ # single policy tag for the field. Policy tag identifiers are of
1068
+ # the form `projects/*/locations/*/taxonomies/*/policyTags/*`.
1069
+ # At most 1 policy tag is currently allowed.
768
1070
  #
769
1071
  # @example
770
1072
  # require "google/cloud/bigquery"
@@ -776,8 +1078,8 @@ module Google
776
1078
  # end
777
1079
  #
778
1080
  # @!group Schema
779
- def time name, description: nil, mode: :nullable
780
- schema.time name, description: description, mode: mode
1081
+ def time name, description: nil, mode: :nullable, policy_tags: nil
1082
+ schema.time name, description: description, mode: mode, policy_tags: policy_tags
781
1083
  end
782
1084
 
783
1085
  ##
@@ -786,13 +1088,17 @@ module Google
786
1088
  # See {Schema#datetime}.
787
1089
  #
788
1090
  # @param [String] name The field name. The name must contain only
789
- # letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
1091
+ # letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
790
1092
  # start with a letter or underscore. The maximum length is 128
791
1093
  # characters.
792
1094
  # @param [String] description A description of the field.
793
1095
  # @param [Symbol] mode The field's mode. The possible values are
794
1096
  # `:nullable`, `:required`, and `:repeated`. The default value is
795
1097
  # `:nullable`.
1098
+ # @param [Array<String>, String] policy_tags The policy tag list or
1099
+ # single policy tag for the field. Policy tag identifiers are of
1100
+ # the form `projects/*/locations/*/taxonomies/*/policyTags/*`.
1101
+ # At most 1 policy tag is currently allowed.
796
1102
  #
797
1103
  # @example
798
1104
  # require "google/cloud/bigquery"
@@ -804,8 +1110,8 @@ module Google
804
1110
  # end
805
1111
  #
806
1112
  # @!group Schema
807
- def datetime name, description: nil, mode: :nullable
808
- schema.datetime name, description: description, mode: mode
1113
+ def datetime name, description: nil, mode: :nullable, policy_tags: nil
1114
+ schema.datetime name, description: description, mode: mode, policy_tags: policy_tags
809
1115
  end
810
1116
 
811
1117
  ##
@@ -814,13 +1120,17 @@ module Google
814
1120
  # See {Schema#date}.
815
1121
  #
816
1122
  # @param [String] name The field name. The name must contain only
817
- # letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
1123
+ # letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
818
1124
  # start with a letter or underscore. The maximum length is 128
819
1125
  # characters.
820
1126
  # @param [String] description A description of the field.
821
1127
  # @param [Symbol] mode The field's mode. The possible values are
822
1128
  # `:nullable`, `:required`, and `:repeated`. The default value is
823
1129
  # `:nullable`.
1130
+ # @param [Array<String>, String] policy_tags The policy tag list or
1131
+ # single policy tag for the field. Policy tag identifiers are of
1132
+ # the form `projects/*/locations/*/taxonomies/*/policyTags/*`.
1133
+ # At most 1 policy tag is currently allowed.
824
1134
  #
825
1135
  # @example
826
1136
  # require "google/cloud/bigquery"
@@ -832,8 +1142,44 @@ module Google
832
1142
  # end
833
1143
  #
834
1144
  # @!group Schema
835
- def date name, description: nil, mode: :nullable
836
- schema.date name, description: description, mode: mode
1145
+ def date name, description: nil, mode: :nullable, policy_tags: nil
1146
+ schema.date name, description: description, mode: mode, policy_tags: policy_tags
1147
+ end
1148
+
1149
+ ##
1150
+ # Adds a geography field to the schema.
1151
+ #
1152
+ # See {Schema#geography}.
1153
+ #
1154
+ # @see https://cloud.google.com/bigquery/docs/gis-data Working with BigQuery GIS data
1155
+ #
1156
+ # @param [String] name The field name. The name must contain only
1157
+ # letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
1158
+ # start with a letter or underscore. The maximum length is 128
1159
+ # characters.
1160
+ # @param [String] description A description of the field.
1161
+ # @param [Symbol] mode The field's mode. The possible values are
1162
+ # `:nullable`, `:required`, and `:repeated`. The default value is
1163
+ # `:nullable`.
1164
+ # @param [Array<String>, String] policy_tags The policy tag list or
1165
+ # single policy tag for the field. Policy tag identifiers are of
1166
+ # the form `projects/*/locations/*/taxonomies/*/policyTags/*`.
1167
+ # At most 1 policy tag is currently allowed.
1168
+ #
1169
+ # @example
1170
+ # require "google/cloud/bigquery"
1171
+ #
1172
+ # bigquery = Google::Cloud::Bigquery.new
1173
+ # dataset = bigquery.dataset "my_dataset"
1174
+ # job = dataset.load_job "my_table", "gs://abc/file" do |schema|
1175
+ # schema.record "cities_lived", mode: :repeated do |cities_lived|
1176
+ # cities_lived.geography "location", mode: :required
1177
+ # cities_lived.integer "number_of_years", mode: :required
1178
+ # end
1179
+ # end
1180
+ #
1181
+ def geography name, description: nil, mode: :nullable, policy_tags: nil
1182
+ schema.geography name, description: description, mode: mode, policy_tags: policy_tags
837
1183
  end
838
1184
 
839
1185
  ##
@@ -846,7 +1192,7 @@ module Google
846
1192
  # See {Schema#record}.
847
1193
  #
848
1194
  # @param [String] name The field name. The name must contain only
849
- # letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
1195
+ # letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
850
1196
  # start with a letter or underscore. The maximum length is 128
851
1197
  # characters.
852
1198
  # @param [String] description A description of the field.
@@ -934,8 +1280,7 @@ module Google
934
1280
  # @!group Attributes
935
1281
  #
936
1282
  def format= new_format
937
- @gapi.configuration.load.update! \
938
- source_format: Convert.source_format(new_format)
1283
+ @gapi.configuration.load.update! source_format: Convert.source_format(new_format)
939
1284
  end
940
1285
 
941
1286
  ##
@@ -955,8 +1300,7 @@ module Google
955
1300
  # @!group Attributes
956
1301
  #
957
1302
  def create= new_create
958
- @gapi.configuration.load.update! \
959
- create_disposition: Convert.create_disposition(new_create)
1303
+ @gapi.configuration.load.update! create_disposition: Convert.create_disposition(new_create)
960
1304
  end
961
1305
 
962
1306
  ##
@@ -977,8 +1321,7 @@ module Google
977
1321
  # @!group Attributes
978
1322
  #
979
1323
  def write= new_write
980
- @gapi.configuration.load.update! \
981
- write_disposition: Convert.write_disposition(new_write)
1324
+ @gapi.configuration.load.update! write_disposition: Convert.write_disposition(new_write)
982
1325
  end
983
1326
 
984
1327
  ##
@@ -999,8 +1342,7 @@ module Google
999
1342
  if new_fields.nil?
1000
1343
  @gapi.configuration.load.update! projection_fields: nil
1001
1344
  else
1002
- @gapi.configuration.load.update! \
1003
- projection_fields: Array(new_fields)
1345
+ @gapi.configuration.load.update! projection_fields: Array(new_fields)
1004
1346
  end
1005
1347
  end
1006
1348
 
@@ -1199,8 +1541,7 @@ module Google
1199
1541
  if new_options.nil?
1200
1542
  @gapi.configuration.load.update! schema_update_options: nil
1201
1543
  else
1202
- @gapi.configuration.load.update! \
1203
- schema_update_options: Array(new_options)
1544
+ @gapi.configuration.load.update! schema_update_options: Array(new_options)
1204
1545
  end
1205
1546
  end
1206
1547
 
@@ -1238,21 +1579,28 @@ module Google
1238
1579
  #
1239
1580
  # @!group Attributes
1240
1581
  def encryption= val
1241
- @gapi.configuration.load.update!(
1242
- destination_encryption_configuration: val.to_gapi
1243
- )
1582
+ @gapi.configuration.load.update! destination_encryption_configuration: val.to_gapi
1244
1583
  end
1245
1584
 
1246
1585
  ##
1247
1586
  # Sets the labels to use for the load job.
1248
1587
  #
1249
1588
  # @param [Hash] val A hash of user-provided labels associated with
1250
- # the job. You can use these to organize and group your jobs. Label
1251
- # keys and values can be no longer than 63 characters, can only
1252
- # contain lowercase letters, numeric characters, underscores and
1253
- # dashes. International characters are allowed. Label values are
1254
- # optional. Label keys must start with a letter and each label in
1255
- # the list must have a different key.
1589
+ # the job. You can use these to organize and group your jobs.
1590
+ #
1591
+ # The labels applied to a resource must meet the following requirements:
1592
+ #
1593
+ # * Each resource can have multiple labels, up to a maximum of 64.
1594
+ # * Each label must be a key-value pair.
1595
+ # * Keys have a minimum length of 1 character and a maximum length of
1596
+ # 63 characters, and cannot be empty. Values can be empty, and have
1597
+ # a maximum length of 63 characters.
1598
+ # * Keys and values can contain only lowercase letters, numeric characters,
1599
+ # underscores, and dashes. All characters must use UTF-8 encoding, and
1600
+ # international characters are allowed.
1601
+ # * The key portion of a label must be unique. However, you can use the
1602
+ # same key with multiple resources.
1603
+ # * Keys must start with a lowercase letter or international character.
1256
1604
  #
1257
1605
  # @!group Attributes
1258
1606
  #
@@ -1261,15 +1609,325 @@ module Google
1261
1609
  end
1262
1610
 
1263
1611
  ##
1264
- # Sets the partitioning for the destination table. See [Partitioned
1612
+ # Sets the mode of hive partitioning to use when reading data. The following modes are supported:
1613
+ #
1614
+ # 1. `auto`: automatically infer partition key name(s) and type(s).
1615
+ # 2. `strings`: automatically infer partition key name(s). All types are interpreted as strings.
1616
+ # 3. `custom`: partition key schema is encoded in the source URI prefix.
1617
+ #
1618
+ # Not all storage formats support hive partitioning. Requesting hive partitioning on an unsupported format
1619
+ # will lead to an error. Currently supported types include: `avro`, `csv`, `json`, `orc` and `parquet`.
1620
+ #
1621
+ # See {#format=} and {#hive_partitioning_source_uri_prefix=}.
1622
+ #
1623
+ # @see https://cloud.google.com/bigquery/docs/hive-partitioned-loads-gcs Loading externally partitioned data
1624
+ #
1625
+ # @param [String, Symbol] mode The mode of hive partitioning to use when reading data.
1626
+ #
1627
+ # @example
1628
+ # require "google/cloud/bigquery"
1629
+ #
1630
+ # bigquery = Google::Cloud::Bigquery.new
1631
+ # dataset = bigquery.dataset "my_dataset"
1632
+ #
1633
+ # gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
1634
+ # source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
1635
+ # load_job = dataset.load_job "my_new_table", gcs_uri do |job|
1636
+ # job.format = :parquet
1637
+ # job.hive_partitioning_mode = :auto
1638
+ # job.hive_partitioning_source_uri_prefix = source_uri_prefix
1639
+ # end
1640
+ #
1641
+ # load_job.wait_until_done!
1642
+ # load_job.done? #=> true
1643
+ #
1644
+ # @!group Attributes
1645
+ #
1646
+ def hive_partitioning_mode= mode
1647
+ @gapi.configuration.load.hive_partitioning_options ||= Google::Apis::BigqueryV2::HivePartitioningOptions.new
1648
+ @gapi.configuration.load.hive_partitioning_options.mode = mode.to_s.upcase
1649
+ end
1650
+
1651
+ ##
1652
+ # Sets the common prefix for all source uris when hive partition detection is requested. The prefix must end
1653
+ # immediately before the partition key encoding begins. For example, consider files following this data
1654
+ # layout:
1655
+ #
1656
+ # ```
1657
+ # gs://bucket/path_to_table/dt=2019-01-01/country=BR/id=7/file.avro
1658
+ # gs://bucket/path_to_table/dt=2018-12-31/country=CA/id=3/file.avro
1659
+ # ```
1660
+ #
1661
+ # When hive partitioning is requested with either `AUTO` or `STRINGS` mode, the common prefix can be either of
1662
+ # `gs://bucket/path_to_table` or `gs://bucket/path_to_table/` (trailing slash does not matter).
1663
+ #
1664
+ # See {#hive_partitioning_mode=}.
1665
+ #
1666
+ # @see https://cloud.google.com/bigquery/docs/hive-partitioned-loads-gcs Loading externally partitioned data
1667
+ #
1668
+ # @param [String] source_uri_prefix The common prefix for all source uris.
1669
+ #
1670
+ # @example
1671
+ # require "google/cloud/bigquery"
1672
+ #
1673
+ # bigquery = Google::Cloud::Bigquery.new
1674
+ # dataset = bigquery.dataset "my_dataset"
1675
+ #
1676
+ # gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
1677
+ # source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
1678
+ # load_job = dataset.load_job "my_new_table", gcs_uri do |job|
1679
+ # job.format = :parquet
1680
+ # job.hive_partitioning_mode = :auto
1681
+ # job.hive_partitioning_source_uri_prefix = source_uri_prefix
1682
+ # end
1683
+ #
1684
+ # load_job.wait_until_done!
1685
+ # load_job.done? #=> true
1686
+ #
1687
+ # @!group Attributes
1688
+ #
1689
+ def hive_partitioning_source_uri_prefix= source_uri_prefix
1690
+ @gapi.configuration.load.hive_partitioning_options ||= Google::Apis::BigqueryV2::HivePartitioningOptions.new
1691
+ @gapi.configuration.load.hive_partitioning_options.source_uri_prefix = source_uri_prefix
1692
+ end
1693
+
1694
+ ##
1695
+ # Sets whether to use schema inference specifically for Parquet `LIST` logical type.
1696
+ #
1697
+ # @see https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-parquet Loading Parquet data from
1698
+ # Cloud Storage
1699
+ #
1700
+ # @param [Boolean] enable_list_inference The `enable_list_inference` value to use in Parquet options.
1701
+ #
1702
+ # @example
1703
+ # require "google/cloud/bigquery"
1704
+ #
1705
+ # bigquery = Google::Cloud::Bigquery.new
1706
+ # dataset = bigquery.dataset "my_dataset"
1707
+ #
1708
+ # gcs_uris = ["gs://mybucket/00/*.parquet", "gs://mybucket/01/*.parquet"]
1709
+ # load_job = dataset.load_job "my_new_table", gcs_uris do |job|
1710
+ # job.format = :parquet
1711
+ # job.parquet_enable_list_inference = true
1712
+ # end
1713
+ #
1714
+ # load_job.wait_until_done!
1715
+ # load_job.done? #=> true
1716
+ #
1717
+ # @!group Attributes
1718
+ #
1719
+ def parquet_enable_list_inference= enable_list_inference
1720
+ @gapi.configuration.load.parquet_options ||= Google::Apis::BigqueryV2::ParquetOptions.new
1721
+ @gapi.configuration.load.parquet_options.enable_list_inference = enable_list_inference
1722
+ end
1723
+
1724
+ ##
1725
+ # Sets whether to infer Parquet `ENUM` logical type as `STRING` instead of `BYTES` by default.
1726
+ #
1727
+ # @see https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-parquet Loading Parquet data from
1728
+ # Cloud Storage
1729
+ #
1730
+ # @param [Boolean] enum_as_string The `enum_as_string` value to use in Parquet options.
1731
+ #
1732
+ # @example
1733
+ # require "google/cloud/bigquery"
1734
+ #
1735
+ # bigquery = Google::Cloud::Bigquery.new
1736
+ # dataset = bigquery.dataset "my_dataset"
1737
+ #
1738
+ # gcs_uris = ["gs://mybucket/00/*.parquet", "gs://mybucket/01/*.parquet"]
1739
+ # load_job = dataset.load_job "my_new_table", gcs_uris do |job|
1740
+ # job.format = :parquet
1741
+ # job.parquet_enum_as_string = true
1742
+ # end
1743
+ #
1744
+ # load_job.wait_until_done!
1745
+ # load_job.done? #=> true
1746
+ #
1747
+ # @!group Attributes
1748
+ #
1749
+ def parquet_enum_as_string= enum_as_string
1750
+ @gapi.configuration.load.parquet_options ||= Google::Apis::BigqueryV2::ParquetOptions.new
1751
+ @gapi.configuration.load.parquet_options.enum_as_string = enum_as_string
1752
+ end
1753
+
1754
+ ##
1755
+ # Sets the field on which to range partition the table. See [Creating and using integer range partitioned
1756
+ # tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
1757
+ #
1758
+ # See {#range_partitioning_start=}, {#range_partitioning_interval=} and {#range_partitioning_end=}.
1759
+ #
1760
+ # You can only set range partitioning when creating a table. BigQuery does not allow you to change
1761
+ # partitioning on an existing table.
1762
+ #
1763
+ # @param [String] field The range partition field. the destination table is partitioned by this
1764
+ # field. The field must be a top-level `NULLABLE/REQUIRED` field. The only supported
1765
+ # type is `INTEGER/INT64`.
1766
+ #
1767
+ # @example
1768
+ # require "google/cloud/bigquery"
1769
+ #
1770
+ # bigquery = Google::Cloud::Bigquery.new
1771
+ # dataset = bigquery.dataset "my_dataset"
1772
+ #
1773
+ # gcs_uri = "gs://my-bucket/file-name.csv"
1774
+ # load_job = dataset.load_job "my_new_table", gcs_uri do |job|
1775
+ # job.schema do |schema|
1776
+ # schema.integer "my_table_id", mode: :required
1777
+ # schema.string "my_table_data", mode: :required
1778
+ # end
1779
+ # job.range_partitioning_field = "my_table_id"
1780
+ # job.range_partitioning_start = 0
1781
+ # job.range_partitioning_interval = 10
1782
+ # job.range_partitioning_end = 100
1783
+ # end
1784
+ #
1785
+ # load_job.wait_until_done!
1786
+ # load_job.done? #=> true
1787
+ #
1788
+ # @!group Attributes
1789
+ #
1790
+ def range_partitioning_field= field
1791
+ @gapi.configuration.load.range_partitioning ||= Google::Apis::BigqueryV2::RangePartitioning.new(
1792
+ range: Google::Apis::BigqueryV2::RangePartitioning::Range.new
1793
+ )
1794
+ @gapi.configuration.load.range_partitioning.field = field
1795
+ end
1796
+
1797
+ ##
1798
+ # Sets the start of range partitioning, inclusive, for the destination table. See [Creating and using integer
1799
+ # range partitioned tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
1800
+ #
1801
+ # You can only set range partitioning when creating a table. BigQuery does not allow you to change
1802
+ # partitioning on an existing table.
1803
+ #
1804
+ # See {#range_partitioning_field=}, {#range_partitioning_interval=} and {#range_partitioning_end=}.
1805
+ #
1806
+ # @param [Integer] range_start The start of range partitioning, inclusive.
1807
+ #
1808
+ # @example
1809
+ # require "google/cloud/bigquery"
1810
+ #
1811
+ # bigquery = Google::Cloud::Bigquery.new
1812
+ # dataset = bigquery.dataset "my_dataset"
1813
+ #
1814
+ # gcs_uri = "gs://my-bucket/file-name.csv"
1815
+ # load_job = dataset.load_job "my_new_table", gcs_uri do |job|
1816
+ # job.schema do |schema|
1817
+ # schema.integer "my_table_id", mode: :required
1818
+ # schema.string "my_table_data", mode: :required
1819
+ # end
1820
+ # job.range_partitioning_field = "my_table_id"
1821
+ # job.range_partitioning_start = 0
1822
+ # job.range_partitioning_interval = 10
1823
+ # job.range_partitioning_end = 100
1824
+ # end
1825
+ #
1826
+ # load_job.wait_until_done!
1827
+ # load_job.done? #=> true
1828
+ #
1829
+ # @!group Attributes
1830
+ #
1831
+ def range_partitioning_start= range_start
1832
+ @gapi.configuration.load.range_partitioning ||= Google::Apis::BigqueryV2::RangePartitioning.new(
1833
+ range: Google::Apis::BigqueryV2::RangePartitioning::Range.new
1834
+ )
1835
+ @gapi.configuration.load.range_partitioning.range.start = range_start
1836
+ end
1837
+
1838
+ ##
1839
+ # Sets width of each interval for data in range partitions. See [Creating and using integer range partitioned
1840
+ # tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
1841
+ #
1842
+ # You can only set range partitioning when creating a table. BigQuery does not allow you to change
1843
+ # partitioning on an existing table.
1844
+ #
1845
+ # See {#range_partitioning_field=}, {#range_partitioning_start=} and {#range_partitioning_end=}.
1846
+ #
1847
+ # @param [Integer] range_interval The width of each interval, for data in partitions.
1848
+ #
1849
+ # @example
1850
+ # require "google/cloud/bigquery"
1851
+ #
1852
+ # bigquery = Google::Cloud::Bigquery.new
1853
+ # dataset = bigquery.dataset "my_dataset"
1854
+ #
1855
+ # gcs_uri = "gs://my-bucket/file-name.csv"
1856
+ # load_job = dataset.load_job "my_new_table", gcs_uri do |job|
1857
+ # job.schema do |schema|
1858
+ # schema.integer "my_table_id", mode: :required
1859
+ # schema.string "my_table_data", mode: :required
1860
+ # end
1861
+ # job.range_partitioning_field = "my_table_id"
1862
+ # job.range_partitioning_start = 0
1863
+ # job.range_partitioning_interval = 10
1864
+ # job.range_partitioning_end = 100
1865
+ # end
1866
+ #
1867
+ # load_job.wait_until_done!
1868
+ # load_job.done? #=> true
1869
+ #
1870
+ # @!group Attributes
1871
+ #
1872
+ def range_partitioning_interval= range_interval
1873
+ @gapi.configuration.load.range_partitioning ||= Google::Apis::BigqueryV2::RangePartitioning.new(
1874
+ range: Google::Apis::BigqueryV2::RangePartitioning::Range.new
1875
+ )
1876
+ @gapi.configuration.load.range_partitioning.range.interval = range_interval
1877
+ end
1878
+
1879
+ ##
1880
+ # Sets the end of range partitioning, exclusive, for the destination table. See [Creating and using integer
1881
+ # range partitioned tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
1882
+ #
1883
+ # You can only set range partitioning when creating a table. BigQuery does not allow you to change
1884
+ # partitioning on an existing table.
1885
+ #
1886
+ # See {#range_partitioning_start=}, {#range_partitioning_interval=} and {#range_partitioning_field=}.
1887
+ #
1888
+ # @param [Integer] range_end The end of range partitioning, exclusive.
1889
+ #
1890
+ # @example
1891
+ # require "google/cloud/bigquery"
1892
+ #
1893
+ # bigquery = Google::Cloud::Bigquery.new
1894
+ # dataset = bigquery.dataset "my_dataset"
1895
+ #
1896
+ # gcs_uri = "gs://my-bucket/file-name.csv"
1897
+ # load_job = dataset.load_job "my_new_table", gcs_uri do |job|
1898
+ # job.schema do |schema|
1899
+ # schema.integer "my_table_id", mode: :required
1900
+ # schema.string "my_table_data", mode: :required
1901
+ # end
1902
+ # job.range_partitioning_field = "my_table_id"
1903
+ # job.range_partitioning_start = 0
1904
+ # job.range_partitioning_interval = 10
1905
+ # job.range_partitioning_end = 100
1906
+ # end
1907
+ #
1908
+ # load_job.wait_until_done!
1909
+ # load_job.done? #=> true
1910
+ #
1911
+ # @!group Attributes
1912
+ #
1913
+ def range_partitioning_end= range_end
1914
+ @gapi.configuration.load.range_partitioning ||= Google::Apis::BigqueryV2::RangePartitioning.new(
1915
+ range: Google::Apis::BigqueryV2::RangePartitioning::Range.new
1916
+ )
1917
+ @gapi.configuration.load.range_partitioning.range.end = range_end
1918
+ end
1919
+
1920
+ ##
1921
+ # Sets the time partitioning for the destination table. See [Partitioned
1265
1922
  # Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
1266
1923
  #
1267
- # You can only set the partitioning field while creating a table.
1924
+ # You can only set the time partitioning field while creating a table.
1268
1925
  # BigQuery does not allow you to change partitioning on an existing
1269
1926
  # table.
1270
1927
  #
1271
- # @param [String] type The partition type. Currently the only
1272
- # supported value is "DAY".
1928
+ # @param [String] type The time partition type. The supported types are `DAY`,
1929
+ # `HOUR`, `MONTH`, and `YEAR`, which will generate one partition per day,
1930
+ # hour, month, and year, respectively.
1273
1931
  #
1274
1932
  # @example
1275
1933
  # require "google/cloud/bigquery"
@@ -1277,8 +1935,8 @@ module Google
1277
1935
  # bigquery = Google::Cloud::Bigquery.new
1278
1936
  # dataset = bigquery.dataset "my_dataset"
1279
1937
  #
1280
- # gs_url = "gs://my-bucket/file-name.csv"
1281
- # load_job = dataset.load_job "my_new_table", gs_url do |job|
1938
+ # gcs_uri = "gs://my-bucket/file-name.csv"
1939
+ # load_job = dataset.load_job "my_new_table", gcs_uri do |job|
1282
1940
  # job.time_partitioning_type = "DAY"
1283
1941
  # end
1284
1942
  #
@@ -1288,26 +1946,25 @@ module Google
1288
1946
  # @!group Attributes
1289
1947
  #
1290
1948
  def time_partitioning_type= type
1291
- @gapi.configuration.load.time_partitioning ||= \
1292
- Google::Apis::BigqueryV2::TimePartitioning.new
1949
+ @gapi.configuration.load.time_partitioning ||= Google::Apis::BigqueryV2::TimePartitioning.new
1293
1950
  @gapi.configuration.load.time_partitioning.update! type: type
1294
1951
  end
1295
1952
 
1296
1953
  ##
1297
- # Sets the field on which to partition the destination table. If not
1298
- # set, the destination table is partitioned by pseudo column
1299
- # `_PARTITIONTIME`; if set, the table is partitioned by this field.
1954
+ # Sets the field on which to time partition the destination table. If not
1955
+ # set, the destination table is time partitioned by pseudo column
1956
+ # `_PARTITIONTIME`; if set, the table is time partitioned by this field.
1300
1957
  # See [Partitioned
1301
1958
  # Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
1302
1959
  #
1303
- # The destination table must also be partitioned. See
1960
+ # The destination table must also be time partitioned. See
1304
1961
  # {#time_partitioning_type=}.
1305
1962
  #
1306
- # You can only set the partitioning field while creating a table.
1963
+ # You can only set the time partitioning field while creating a table.
1307
1964
  # BigQuery does not allow you to change partitioning on an existing
1308
1965
  # table.
1309
1966
  #
1310
- # @param [String] field The partition field. The field must be a
1967
+ # @param [String] field The time partition field. The field must be a
1311
1968
  # top-level TIMESTAMP or DATE field. Its mode must be NULLABLE or
1312
1969
  # REQUIRED.
1313
1970
  #
@@ -1317,8 +1974,8 @@ module Google
1317
1974
  # bigquery = Google::Cloud::Bigquery.new
1318
1975
  # dataset = bigquery.dataset "my_dataset"
1319
1976
  #
1320
- # gs_url = "gs://my-bucket/file-name.csv"
1321
- # load_job = dataset.load_job "my_new_table", gs_url do |job|
1977
+ # gcs_uri = "gs://my-bucket/file-name.csv"
1978
+ # load_job = dataset.load_job "my_new_table", gcs_uri do |job|
1322
1979
  # job.time_partitioning_type = "DAY"
1323
1980
  # job.time_partitioning_field = "dob"
1324
1981
  # job.schema do |schema|
@@ -1332,21 +1989,20 @@ module Google
1332
1989
  # @!group Attributes
1333
1990
  #
1334
1991
  def time_partitioning_field= field
1335
- @gapi.configuration.load.time_partitioning ||= \
1336
- Google::Apis::BigqueryV2::TimePartitioning.new
1992
+ @gapi.configuration.load.time_partitioning ||= Google::Apis::BigqueryV2::TimePartitioning.new
1337
1993
  @gapi.configuration.load.time_partitioning.update! field: field
1338
1994
  end
1339
1995
 
1340
1996
  ##
1341
- # Sets the partition expiration for the destination table. See
1997
+ # Sets the time partition expiration for the destination table. See
1342
1998
  # [Partitioned
1343
1999
  # Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
1344
2000
  #
1345
- # The destination table must also be partitioned. See
2001
+ # The destination table must also be time partitioned. See
1346
2002
  # {#time_partitioning_type=}.
1347
2003
  #
1348
2004
  # @param [Integer] expiration An expiration time, in seconds,
1349
- # for data in partitions.
2005
+ # for data in time partitions.
1350
2006
  #
1351
2007
  # @example
1352
2008
  # require "google/cloud/bigquery"
@@ -1354,8 +2010,8 @@ module Google
1354
2010
  # bigquery = Google::Cloud::Bigquery.new
1355
2011
  # dataset = bigquery.dataset "my_dataset"
1356
2012
  #
1357
- # gs_url = "gs://my-bucket/file-name.csv"
1358
- # load_job = dataset.load_job "my_new_table", gs_url do |job|
2013
+ # gcs_uri = "gs://my-bucket/file-name.csv"
2014
+ # load_job = dataset.load_job "my_new_table", gcs_uri do |job|
1359
2015
  # job.time_partitioning_type = "DAY"
1360
2016
  # job.time_partitioning_expiration = 86_400
1361
2017
  # end
@@ -1366,48 +2022,44 @@ module Google
1366
2022
  # @!group Attributes
1367
2023
  #
1368
2024
  def time_partitioning_expiration= expiration
1369
- @gapi.configuration.load.time_partitioning ||= \
1370
- Google::Apis::BigqueryV2::TimePartitioning.new
1371
- @gapi.configuration.load.time_partitioning.update! \
1372
- expiration_ms: expiration * 1000
2025
+ @gapi.configuration.load.time_partitioning ||= Google::Apis::BigqueryV2::TimePartitioning.new
2026
+ @gapi.configuration.load.time_partitioning.update! expiration_ms: expiration * 1000
1373
2027
  end
1374
2028
 
1375
2029
  ##
1376
2030
  # If set to true, queries over the destination table will require a
1377
- # partition filter that can be used for partition elimination to be
2031
+ # time partition filter that can be used for time partition elimination to be
1378
2032
  # specified. See [Partitioned
1379
2033
  # Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
1380
2034
  #
1381
2035
  # @param [Boolean] val Indicates if queries over the destination table
1382
- # will require a partition filter. The default value is `false`.
2036
+ # will require a time partition filter. The default value is `false`.
1383
2037
  #
1384
2038
  # @!group Attributes
1385
2039
  #
1386
2040
  def time_partitioning_require_filter= val
1387
- @gapi.configuration.load.time_partitioning ||= \
1388
- Google::Apis::BigqueryV2::TimePartitioning.new
1389
- @gapi.configuration.load.time_partitioning.update! \
1390
- require_partition_filter: val
2041
+ @gapi.configuration.load.time_partitioning ||= Google::Apis::BigqueryV2::TimePartitioning.new
2042
+ @gapi.configuration.load.time_partitioning.update! require_partition_filter: val
1391
2043
  end
1392
2044
 
1393
2045
  ##
1394
- # Sets one or more fields on which the destination table should be
1395
- # clustered. Must be specified with time-based partitioning, data in
1396
- # the table will be first partitioned and subsequently clustered.
2046
+ # Sets the list of fields on which data should be clustered.
1397
2047
  #
1398
2048
  # Only top-level, non-repeated, simple-type fields are supported. When
1399
2049
  # you cluster a table using multiple columns, the order of columns you
1400
2050
  # specify is important. The order of the specified columns determines
1401
2051
  # the sort order of the data.
1402
2052
  #
1403
- # See {LoadJob#clustering_fields}.
2053
+ # BigQuery supports clustering for both partitioned and non-partitioned
2054
+ # tables.
2055
+ #
2056
+ # See {LoadJob#clustering_fields}, {Table#clustering_fields} and
2057
+ # {Table#clustering_fields=}.
1404
2058
  #
1405
- # @see https://cloud.google.com/bigquery/docs/partitioned-tables
1406
- # Partitioned Tables
1407
2059
  # @see https://cloud.google.com/bigquery/docs/clustered-tables
1408
- # Introduction to Clustered Tables
2060
+ # Introduction to clustered tables
1409
2061
  # @see https://cloud.google.com/bigquery/docs/creating-clustered-tables
1410
- # Creating and Using Clustered Tables
2062
+ # Creating and using clustered tables
1411
2063
  #
1412
2064
  # @param [Array<String>] fields The clustering fields. Only top-level,
1413
2065
  # non-repeated, simple-type fields are supported.
@@ -1418,8 +2070,8 @@ module Google
1418
2070
  # bigquery = Google::Cloud::Bigquery.new
1419
2071
  # dataset = bigquery.dataset "my_dataset"
1420
2072
  #
1421
- # gs_url = "gs://my-bucket/file-name.csv"
1422
- # load_job = dataset.load_job "my_new_table", gs_url do |job|
2073
+ # gcs_uri = "gs://my-bucket/file-name.csv"
2074
+ # load_job = dataset.load_job "my_new_table", gcs_uri do |job|
1423
2075
  # job.time_partitioning_type = "DAY"
1424
2076
  # job.time_partitioning_field = "dob"
1425
2077
  # job.schema do |schema|
@@ -1436,11 +2088,27 @@ module Google
1436
2088
  # @!group Attributes
1437
2089
  #
1438
2090
  def clustering_fields= fields
1439
- @gapi.configuration.load.clustering ||= \
1440
- Google::Apis::BigqueryV2::Clustering.new
2091
+ @gapi.configuration.load.clustering ||= Google::Apis::BigqueryV2::Clustering.new
1441
2092
  @gapi.configuration.load.clustering.fields = fields
1442
2093
  end
1443
2094
 
2095
+ def cancel
2096
+ raise "not implemented in #{self.class}"
2097
+ end
2098
+
2099
+ def rerun!
2100
+ raise "not implemented in #{self.class}"
2101
+ end
2102
+
2103
+ def reload!
2104
+ raise "not implemented in #{self.class}"
2105
+ end
2106
+ alias refresh! reload!
2107
+
2108
+ def wait_until_done!
2109
+ raise "not implemented in #{self.class}"
2110
+ end
2111
+
1444
2112
  ##
1445
2113
  # @private Returns the Google API client library version of this job.
1446
2114
  #