google-cloud-bigquery 1.23.0 → 1.28.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 89f8e3d756fc514c769c82a9a7db9f342626761d65607b5b837c29a691e27c28
-  data.tar.gz: 93f69944889b1a3fd1ddf4a14e42b72c0d4842d905b6b055343928d0fec1a173
+  metadata.gz: 44443c8c7fdf80fdfc611bba7ffcad2469ef60ff3aef8899f56a07fb4ced8555
+  data.tar.gz: 3b3f3020b6c4eebfa1a2a974f6d047068bd5d7f114b5d8c843c31401108b15b1
 SHA512:
-  metadata.gz: c72ef2e170c0b2b0c897c0bab8c5e2e1e7694af1e0b1b3f19ce0e40acce6d07993bb39ecb5b90509bdcf8f8f1c66596ef4ea7801532bb25799cdd7b6edfa3bd0
-  data.tar.gz: b22e6d9c70f1428d970c8f9fed8076194f4cf58872f56449a214ff46b93561e06d71ebcf8ecd1af18f1899c50c1130454b200870ea40919acda00e6e0ffc5dcb
+  metadata.gz: 5fb27c2d28ebb83a3f28a11241b7b260373dceade40159df44e31d6ab5de016079a00ddeba6943a09a6c7f8d303d08a24ca665160cd480b95f314382c2e45943
+  data.tar.gz: cf98a8d90c55cf065265c094a74ccf9f421a89e1c459b05189d4a7b33c8d97c9304c0e7256729de190a9bc7f74195fd6cff9170748cfb3bc4732a297f902b320

data/CHANGELOG.md

@@ -1,5 +1,67 @@
 # Release History
 
+### 1.28.0 / 2021-03-09
+
+#### Features
+
+* Add Materialized View support
+  * Add Dataset#create_materialized_view
+  * Add Table#materialized_view?
+  * Add Table#enable_refresh?
+  * Add Table#enable_refresh=
+  * Add Table#last_refresh_time
+  * Add Table#refresh_interval_ms
+  * Add Table#refresh_interval_ms=
+
+### 1.27.0 / 2021-02-10
+
+#### Features
+
+* Add Job#reservation_usage
+* Add Routine#determinism_level
+  * Add Routine#determinism_level
+  * Add Routine#determinism_level=
+  * Add Routine#determinism_level_deterministic?
+  * Add Routine#determinism_level_not_deterministic?
+  * Add Routine::Updater#determinism_level=
+
+### 1.26.0 / 2021-01-13
+
+#### Features
+
+* Add support for Hive Partitioning
+  * Add hive partitioning options to External::DataSource
+  * Add hive partitioning options to LoadJob and LoadJob::Updater
+* Replace google-api-client with google-apis-bigquery_v2
+
+### 1.25.0 / 2020-11-16
+
+#### Features
+
+* Add routine (UDF) to Dataset::Access
+* Add support for Table ACLS (IAM Policy)
+  * feat(bigquery): Add support for Table ACLS
+  * Add Bigquery::Policy
+  * Add Table#policy
+  * Add Table#test_iam_permissions
+  * Add Table#update_policy
+
+### 1.24.0 / 2020-10-29
+
+#### Features
+
+* Add iamMember to Dataset::Access
+
+#### Bug Fixes
+
+* Ensure dense encoding of JSON responses
+  * Set query param prettyPrint=false for all requests.
+  * Upgrade google-api-client to ~> 0.47
+
+#### Documentation
+
+* Update supported types for time partition type
+
 ### 1.23.0 / 2020-09-17
 
 #### Features

data/CONTRIBUTING.md

@@ -45,7 +45,7 @@ there is a small amount of setup:
 
    ```sh
    $ cd google-cloud-bigquery/
-   $ bundle exec rake bundleupdate
+   $ bundle install
    ```
 
 ## Console

data/lib/google/cloud/bigquery/dataset.rb

@@ -618,15 +618,17 @@ module Google
         end
 
         ##
-        # Creates a new [view](https://cloud.google.com/bigquery/docs/views)
-        # table, which is a virtual table defined by the given SQL query.
+        # Creates a new view, which is a virtual table defined by the given SQL query.
         #
-        # BigQuery's views are logical views, not materialized views, which
-        # means that the query that defines the view is re-executed every time
-        # the view is queried. Queries are billed according to the total amount
+        # With BigQuery's logical views, the query that defines the view is re-executed
+        # every time the view is queried. Queries are billed according to the total amount
         # of data in all table fields referenced directly or indirectly by the
         # top-level query. (See {Table#view?} and {Table#query}.)
         #
+        # For materialized views, see {#create_materialized_view}.
+        #
+        # @see https://cloud.google.com/bigquery/docs/views Creating views
+        #
         # @param [String] table_id The ID of the view table. The ID must contain
         #   only letters (a-z, A-Z), numbers (0-9), or underscores (_). The
         #   maximum length is 1,024 characters.
@@ -667,7 +669,7 @@ module Google
         #   dataset = bigquery.dataset "my_dataset"
         #
         #   view = dataset.create_view "my_view",
-        #             "SELECT name, age FROM proj.dataset.users"
+        #                              "SELECT name, age FROM proj.dataset.users"
         #
         # @example A name and description can be provided:
         #   require "google/cloud/bigquery"
@@ -676,13 +678,18 @@ module Google
         #   dataset = bigquery.dataset "my_dataset"
         #
         #   view = dataset.create_view "my_view",
-        #             "SELECT name, age FROM proj.dataset.users",
-        #             name: "My View", description: "This is my view"
+        #                              "SELECT name, age FROM proj.dataset.users",
+        #                              name: "My View", description: "This is my view"
         #
         # @!group Table
         #
-        def create_view table_id, query, name: nil, description: nil,
-                        standard_sql: nil, legacy_sql: nil, udfs: nil
+        def create_view table_id,
+                        query,
+                        name: nil,
+                        description: nil,
+                        standard_sql: nil,
+                        legacy_sql: nil,
+                        udfs: nil
           use_legacy_sql = Convert.resolve_legacy_sql standard_sql, legacy_sql
           new_view_opts = {
             table_reference: Google::Apis::BigqueryV2::TableReference.new(
@@ -704,6 +711,80 @@ module Google
           Table.from_gapi gapi, service
         end
 
+        ##
+        # Creates a new materialized view.
+        #
+        # Materialized views are precomputed views that periodically cache results of a query for increased performance
+        # and efficiency. BigQuery leverages precomputed results from materialized views and whenever possible reads
+        # only delta changes from the base table to compute up-to-date results.
+        #
+        # Queries that use materialized views are generally faster and consume less resources than queries that retrieve
+        # the same data only from the base table. Materialized views are helpful to significantly boost performance of
+        # workloads that have the characteristic of common and repeated queries.
+        #
+        # For logical views, see {#create_view}.
+        #
+        # @see https://cloud.google.com/bigquery/docs/materialized-views-intro Introduction to materialized views
+        #
+        # @param [String] table_id The ID of the materialized view table. The ID must contain only letters (a-z, A-Z),
+        #   numbers (0-9), or underscores (_). The maximum length is 1,024 characters.
+        # @param [String] query The query that BigQuery executes when the materialized view is referenced.
+        # @param [String] name A descriptive name for the table.
+        # @param [String] description A user-friendly description of the table.
+        # @param [Boolean] enable_refresh Enable automatic refresh of the materialized view when the base table is
+        #   updated. Optional. The default value is true.
+        # @param [Integer] refresh_interval_ms The maximum frequency in milliseconds at which this materialized view
+        #   will be refreshed. Optional. The default value is `1_800_000` (30 minutes).
+        #
+        # @return [Google::Cloud::Bigquery::Table] A new table object.
+        #
+        # @example
+        #   require "google/cloud/bigquery"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #   dataset = bigquery.dataset "my_dataset"
+        #
+        #   materialized_view = dataset.create_materialized_view "my_materialized_view",
+        #                                                        "SELECT name, age FROM proj.dataset.users"
+        #
+        # @example Automatic refresh can be disabled:
+        #   require "google/cloud/bigquery"
+        #
+        #   bigquery = Google::Cloud::Bigquery.new
+        #   dataset = bigquery.dataset "my_dataset"
+        #
+        #   materialized_view = dataset.create_materialized_view "my_materialized_view",
+        #                                                        "SELECT name, age FROM proj.dataset.users",
+        #                                                        enable_refresh: false
+        #
+        # @!group Table
+        #
+        def create_materialized_view table_id,
+                                     query,
+                                     name: nil,
+                                     description: nil,
+                                     enable_refresh: nil,
+                                     refresh_interval_ms: nil
+          new_view_opts = {
+            table_reference:   Google::Apis::BigqueryV2::TableReference.new(
+              project_id: project_id,
+              dataset_id: dataset_id,
+              table_id:   table_id
+            ),
+            friendly_name:     name,
+            description:       description,
+            materialized_view: Google::Apis::BigqueryV2::MaterializedViewDefinition.new(
+              enable_refresh:      enable_refresh,
+              query:               query,
+              refresh_interval_ms: refresh_interval_ms
+            )
+          }.delete_if { |_, v| v.nil? }
+          new_view = Google::Apis::BigqueryV2::Table.new new_view_opts
+
+          gapi = service.insert_table dataset_id, new_view
+          Table.from_gapi gapi, service
+        end
+
         ##
         # Retrieves an existing table by ID.
         #
@@ -2756,6 +2837,12 @@ module Google
             raise "not implemented in #{self.class}"
           end
 
+          ##
+          # @raise [RuntimeError] not implemented
+          def create_materialized_view(*)
+            raise "not implemented in #{self.class}"
+          end
+
           ##
           # @raise [RuntimeError] not implemented
           def table(*)

data/lib/google/cloud/bigquery/dataset/access.rb

@@ -48,16 +48,19 @@ module Google
 
           # @private
           SCOPES = {
-            "user"           => :user_by_email,
-            "user_by_email"  => :user_by_email,
-            "userByEmail"    => :user_by_email,
+            "domain"         => :domain,
             "group"          => :group_by_email,
             "group_by_email" => :group_by_email,
             "groupByEmail"   => :group_by_email,
-            "domain"         => :domain,
+            "iam_member"     => :iam_member,
+            "iamMember"      => :iam_member,
+            "routine"        => :routine,
             "special"        => :special_group,
             "special_group"  => :special_group,
             "specialGroup"   => :special_group,
+            "user"           => :user_by_email,
+            "user_by_email"  => :user_by_email,
+            "userByEmail"    => :user_by_email,
             "view"           => :view
           }.freeze
 
@@ -150,6 +153,26 @@ module Google
             add_access_role_scope_value :reader, :group, email
           end
 
+          ##
+          # Add reader access to some other type of member that appears in the IAM
+          # Policy but isn't a user, group, domain, or special group.
+          #
+          # @param [String] identity The identity reference.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #   dataset = bigquery.dataset "my_dataset"
+          #
+          #   dataset.access do |access|
+          #     access.add_reader_iam_member "entity@example.com"
+          #   end
+          #
+          def add_reader_iam_member identity
+            add_access_role_scope_value :reader, :iam_member, identity
+          end
+
           ##
           # Add reader access to a domain.
           #
@@ -190,6 +213,33 @@ module Google
             add_access_role_scope_value :reader, :special, group
           end
 
+          ##
+          # Add access to a routine from a different dataset. Queries executed
+          # against that routine will have read access to views/tables/routines
+          # in this dataset. Only UDF is supported for now. The role field is
+          # not required when this field is set. If that routine is updated by
+          # any user, access to the routine needs to be granted again via an
+          # update operation.
+          #
+          # @param [Google::Cloud::Bigquery::Routine] routine A routine object.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #   dataset = bigquery.dataset "my_dataset"
+          #   other_dataset = bigquery.dataset "my_other_dataset", skip_lookup: true
+          #
+          #   routine = other_dataset.routine "my_routine"
+          #
+          #   dataset.access do |access|
+          #     access.add_reader_routine routine
+          #   end
+          #
+          def add_reader_routine routine
+            add_access_routine routine
+          end
+
           ##
           # Add reader access to a view.
           #
@@ -205,9 +255,9 @@ module Google
           #
           #   bigquery = Google::Cloud::Bigquery.new
           #   dataset = bigquery.dataset "my_dataset"
-          #   other_dataset = bigquery.dataset "my_other_dataset"
+          #   other_dataset = bigquery.dataset "my_other_dataset", skip_lookup: true
           #
-          #   view = other_dataset.table "my_view"
+          #   view = other_dataset.table "my_view", skip_lookup: true
           #
           #   dataset.access do |access|
           #     access.add_reader_view view
@@ -255,6 +305,26 @@ module Google
             add_access_role_scope_value :writer, :group, email
           end
 
+          ##
+          # Add writer access to some other type of member that appears in the IAM
+          # Policy but isn't a user, group, domain, or special group.
+          #
+          # @param [String] identity The identity reference.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #   dataset = bigquery.dataset "my_dataset"
+          #
+          #   dataset.access do |access|
+          #     access.add_writer_iam_member "entity@example.com"
+          #   end
+          #
+          def add_writer_iam_member identity
+            add_access_role_scope_value :writer, :iam_member, identity
+          end
+
           ##
           # Add writer access to a domain.
           #
@@ -333,6 +403,26 @@ module Google
             add_access_role_scope_value :owner, :group, email
           end
 
+          ##
+          # Add owner access to some other type of member that appears in the IAM
+          # Policy but isn't a user, group, domain, or special group.
+          #
+          # @param [String] identity The identity reference.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #   dataset = bigquery.dataset "my_dataset"
+          #
+          #   dataset.access do |access|
+          #     access.add_owner_iam_member "entity@example.com"
+          #   end
+          #
+          def add_owner_iam_member identity
+            add_access_role_scope_value :owner, :iam_member, identity
+          end
+
           ##
           # Add owner access to a domain.
           #
@@ -411,6 +501,26 @@ module Google
             remove_access_role_scope_value :reader, :group, email
           end
 
+          ##
+          # Remove reader access from some other type of member that appears in the IAM
+          # Policy but isn't a user, group, domain, or special group.
+          #
+          # @param [String] identity The identity reference.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #   dataset = bigquery.dataset "my_dataset"
+          #
+          #   dataset.access do |access|
+          #     access.remove_reader_iam_member "entity@example.com"
+          #   end
+          #
+          def remove_reader_iam_member identity
+            remove_access_role_scope_value :reader, :iam_member, identity
+          end
+
           ##
           # Remove reader access from a domain.
           #
@@ -451,6 +561,28 @@ module Google
             remove_access_role_scope_value :reader, :special, group
           end
 
+          ##
+          # Remove reader access from a routine from a different dataset.
+          #
+          # @param [Google::Cloud::Bigquery::Routine] routine A routine object.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #   dataset = bigquery.dataset "my_dataset"
+          #   other_dataset = bigquery.dataset "my_other_dataset", skip_lookup: true
+          #
+          #   routine = other_dataset.routine "my_routine", skip_lookup: true
+          #
+          #   dataset.access do |access|
+          #     access.remove_reader_routine routine
+          #   end
+          #
+          def remove_reader_routine routine
+            remove_access_routine routine
+          end
+
           ##
           # Remove reader access from a view.
           #
@@ -466,9 +598,9 @@ module Google
           #
           #   bigquery = Google::Cloud::Bigquery.new
           #   dataset = bigquery.dataset "my_dataset"
-          #   other_dataset = bigquery.dataset "my_other_dataset"
+          #   other_dataset = bigquery.dataset "my_other_dataset", skip_lookup: true
           #
-          #   view = other_dataset.table "my_view"
+          #   view = other_dataset.table "my_view", skip_lookup: true
           #
           #   dataset.access do |access|
           #     access.remove_reader_view view
@@ -516,6 +648,26 @@ module Google
             remove_access_role_scope_value :writer, :group, email
           end
 
+          ##
+          # Remove writer access from some other type of member that appears in the IAM
+          # Policy but isn't a user, group, domain, or special group.
+          #
+          # @param [String] identity The identity reference.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #   dataset = bigquery.dataset "my_dataset"
+          #
+          #   dataset.access do |access|
+          #     access.remove_writer_iam_member "entity@example.com"
+          #   end
+          #
+          def remove_writer_iam_member identity
+            remove_access_role_scope_value :writer, :iam_member, identity
+          end
+
           ##
           # Remove writer access from a domain.
           #
@@ -594,6 +746,26 @@ module Google
             remove_access_role_scope_value :owner, :group, email
           end
 
+          ##
+          # Remove owner access from some other type of member that appears in the IAM
+          # Policy but isn't a user, group, domain, or special group.
+          #
+          # @param [String] identity The identity reference.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #   dataset = bigquery.dataset "my_dataset"
+          #
+          #   dataset.access do |access|
+          #     access.remove_owner_iam_member "entity@example.com"
+          #   end
+          #
+          def remove_owner_iam_member identity
+            remove_access_role_scope_value :owner, :iam_member, identity
+          end
+
           ##
           # Remove owner access from a domain.
           #
@@ -670,6 +842,25 @@ module Google
             lookup_access_role_scope_value :reader, :group, email
           end
 
+          ##
+          # Checks reader access for some other type of member that appears in the IAM
+          # Policy but isn't a user, group, domain, or special group.
+          #
+          # @param [String] identity The identity reference.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #   dataset = bigquery.dataset "my_dataset"
+          #
+          #   access = dataset.access
+          #   access.reader_iam_member? "entity@example.com" #=> false
+          #
+          def reader_iam_member? identity
+            lookup_access_role_scope_value :reader, :iam_member, identity
+          end
+
           ##
           # Checks reader access for a domain.
           #
@@ -708,6 +899,32 @@ module Google
             lookup_access_role_scope_value :reader, :special, group
           end
 
+          ##
+          # Checks access for a routine from a different dataset. Queries executed
+          # against that routine will have read access to views/tables/routines
+          # in this dataset. Only UDF is supported for now. The role field is
+          # not required when this field is set. If that routine is updated by
+          # any user, access to the routine needs to be granted again via an
+          # update operation.
+          #
+          # @param [Google::Cloud::Bigquery::Routine] routine A routine object.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #   dataset = bigquery.dataset "my_dataset"
+          #   other_dataset = bigquery.dataset "my_other_dataset", skip_lookup: true
+          #
+          #   routine = other_dataset.routine "my_routine", skip_lookup: true
+          #
+          #   access = dataset.access
+          #   access.reader_routine? routine #=> false
+          #
+          def reader_routine? routine
+            lookup_access_routine routine
+          end
+
           ##
           # Checks reader access for a view.
           #
@@ -723,9 +940,9 @@ module Google
           #
           #   bigquery = Google::Cloud::Bigquery.new
           #   dataset = bigquery.dataset "my_dataset"
-          #   other_dataset = bigquery.dataset "my_other_dataset"
+          #   other_dataset = bigquery.dataset "my_other_dataset", skip_lookup: true
           #
-          #   view = other_dataset.table "my_view"
+          #   view = other_dataset.table "my_view", skip_lookup: true
           #
           #   access = dataset.access
           #   access.reader_view? view #=> false
@@ -770,6 +987,25 @@ module Google
             lookup_access_role_scope_value :writer, :group, email
           end
 
+          ##
+          # Checks writer access for some other type of member that appears in the IAM
+          # Policy but isn't a user, group, domain, or special group.
+          #
+          # @param [String] identity The identity reference.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #   dataset = bigquery.dataset "my_dataset"
+          #
+          #   access = dataset.access
+          #   access.writer_iam_member? "entity@example.com" #=> false
+          #
+          def writer_iam_member? identity
+            lookup_access_role_scope_value :writer, :iam_member, identity
+          end
+
           ##
           # Checks writer access for a domain.
           #
@@ -844,6 +1080,25 @@ module Google
             lookup_access_role_scope_value :owner, :group, email
           end
 
+          ##
+          # Checks owner access for some other type of member that appears in the IAM
+          # Policy but isn't a user, group, domain, or special group.
+          #
+          # @param [String] identity The identity reference.
+          #
+          # @example
+          #   require "google/cloud/bigquery"
+          #
+          #   bigquery = Google::Cloud::Bigquery.new
+          #   dataset = bigquery.dataset "my_dataset"
+          #
+          #   access = dataset.access
+          #   access.owner_iam_member? "entity@example.com" #=> false
+          #
+          def owner_iam_member? identity
+            lookup_access_role_scope_value :owner, :iam_member, identity
+          end
+
           ##
           # Checks owner access for a domain.
           #
@@ -942,12 +1197,22 @@ module Google
             @rules << Google::Apis::BigqueryV2::Dataset::Access.new(opts)
           end
 
+          # @private
+          def add_access_routine routine
+            value = routine.routine_ref
+            # Remove existing routine rule, if any
+            @rules.reject!(&find_by_scope_and_resource_ref(:routine, value))
+            # Add new rule for this role, scope, and value
+            opts = { routine: value }
+            @rules << Google::Apis::BigqueryV2::Dataset::Access.new(opts)
+          end
+
           # @private
           def add_access_view value
             # scope is view, make sure value is in the right format
             value = validate_view value
             # Remove existing view rule, if any
-            @rules.reject!(&find_view(value))
+            @rules.reject!(&find_by_scope_and_resource_ref(:view, value))
             # Add new rule for this role, scope, and value
             opts = { view: value }
             @rules << Google::Apis::BigqueryV2::Dataset::Access.new(opts)
@@ -965,12 +1230,18 @@ module Google
             )
           end
 
+          # @private
+          def remove_access_routine routine
+            # Remove existing routine rule, if any
+            @rules.reject!(&find_by_scope_and_resource_ref(:routine, routine.routine_ref))
+          end
+
           # @private
           def remove_access_view value
             # scope is view, make sure value is in the right format
             value = validate_view value
             # Remove existing view rule, if any
-            @rules.reject!(&find_view(value))
+            @rules.reject!(&find_by_scope_and_resource_ref(:view, value))
           end
 
           # @private
@@ -983,12 +1254,18 @@ module Google
             !(!@rules.detect(&find_by_role_and_scope_and_value(role, scope, value)))
           end
 
+          # @private
+          def lookup_access_routine routine
+            # Detect routine rule, if any
+            !(!@rules.detect(&find_by_scope_and_resource_ref(:routine, routine.routine_ref)))
+          end
+
           # @private
           def lookup_access_view value
             # scope is view, make sure value is in the right format
             value = validate_view value
             # Detect view rule, if any
-            !(!@rules.detect(&find_view(value)))
+            !(!@rules.detect(&find_by_scope_and_resource_ref(:view, value)))
           end
 
           # @private
@@ -1007,11 +1284,11 @@ module Google
             end
           end
 
-          # @private
-          def find_view value
+          # @private Compare hash representations to find table_ref, routine_ref.
+          def find_by_scope_and_resource_ref scope, value
             lambda do |a|
               h = a.to_h
-              h[:view].to_h == value.to_h
+              h[scope].to_h == value.to_h
             end
           end
         end