google-cloud-bigtable 2.2.0 → 2.5.0

data/lib/google/cloud/bigtable/instance.rb

@@ -509,8 +509,8 @@ module Google
         #
         #   entry = table.new_mutation_entry("user-1")
         #   entry.set_cell(
-        #     "cf-1",
-        #     "field-1",
+        #     "cf1",
+        #     "field1",
         #     "XYZ",
         #     timestamp: (Time.now.to_f * 1000000).round(-3) # microseconds
         #   ).delete_cells("cf2", "field02")
@@ -520,6 +520,7 @@ module Google
         def table table_id, view: nil, perform_lookup: nil, app_profile_id: nil
           ensure_service!
 
+          view ||= :SCHEMA_VIEW
           table = if perform_lookup
                     grpc = service.get_table instance_id, table_id, view: view
                     Table.from_grpc grpc, service, view: view

data/lib/google/cloud/bigtable/instance/cluster_map.rb

@@ -15,28 +15,31 @@
 # limitations under the License.
 
 
+require "google/cloud/bigtable/admin/v2"
+
 module Google
   module Cloud
     module Bigtable
       class Instance
         ##
-        # Instance::ClusterMap is a hash with cluster ID keys and cluster configuration values.
-        # It is used to create a cluster.
+        # Instance::ClusterMap is a hash with cluster ID keys and cluster configuration values. It is used to create a
+        # cluster.
         #
         # @example
+        #   require "google/cloud/bigtable"
         #
-        #  clusters = Google::Cloud::Bigtable::Instance::ClusterMap.new
+        #   bigtable = Google::Cloud::Bigtable.new
         #
-        #  clusters.add("cluster-1", "us-east1-b", nodes: 3, storage_type: :SSD)
+        #   job = bigtable.create_instance("my-instance") do |clusters|
+        #     clusters.add("test-cluster", "us-east1-b", nodes: 3, storage_type: :SSD)
+        #   end
         #
-        #  # Or
-        #  clusters.add("cluster-2", "us-east1-b")
+        #   job.wait_until_done!
         #
         class ClusterMap < DelegateClass(::Hash)
           # @private
           #
-          # Creates a new Instance::ClusterMap with an hash of Cluster name and
-          # cluster grpc instances.
+          # Creates a new Instance::ClusterMap with an hash of Cluster name and cluster grpc instances.
           def initialize value = {}
             super value
           end
@@ -45,24 +48,61 @@ module Google
           # Adds a cluster to the cluster map.
           #
           # @param name [String] The unique identifier for the cluster.
-          # @param location [String]
-          #   The location where this cluster's nodes and storage reside. For best
-          #   performance, clients should be located as close as possible to this
-          #   cluster. Currently only zones are supported.
-          # @param nodes [Integer] Number of nodes for the cluster. When creating
-          #   an instance of type `:DEVELOPMENT`, `nodes` must not be set.
-          # @param storage_type [Symbol]
-          #   Valid values are:
-          #   * `:SSD`(Flash (SSD) storage should be used),
-          #   *`:HDD`(Magnetic drive (HDD) storage should be used)
+          # @param location [String] The location where this cluster's nodes and storage reside. For best performance,
+          #   clients should be located as close as possible to this cluster. Currently only zones are supported.
+          # @param nodes [Integer] Number of nodes for the cluster. When creating an instance of type `:DEVELOPMENT`,
+          #   `nodes` must not be set.
+          # @param storage_type [Symbol] The type of storage used by this cluster to serve its parent instance's tables,
+          #   unless explicitly overridden. Valid values are:
+          #
+          #   * `:SSD` - Flash (SSD) storage should be used.
+          #   * `:HDD` - Magnetic drive (HDD) storage should be used.
+          #
+          #   If not set then default will set to `:STORAGE_TYPE_UNSPECIFIED`.
+          # @param kms_key [String] The full name of a Cloud KMS encryption key for a CMEK-protected cluster, in the
+          #   format `projects/{key_project_id}/locations/{location}/keyRings/{ring_name}/cryptoKeys/{key_name}`.
+          #
+          #   The requirements for this key are:
+          #
+          #   1. The Cloud Bigtable service account associated with the project that contains this cluster must be
+          #      granted the `cloudkms.cryptoKeyEncrypterDecrypter` role on the CMEK key.
+          #   2. Only regional keys can be used and the region of the CMEK key must match the region of the cluster.
+          #   3. All clusters within an instance must use the same CMEK key.
+          #
+          # @example
+          #   require "google/cloud/bigtable"
+          #
+          #   bigtable = Google::Cloud::Bigtable.new
+          #
+          #   job = bigtable.create_instance("my-instance") do |clusters|
+          #     clusters.add("test-cluster", "us-east1-b", nodes: 3, storage_type: :SSD)
+          #   end
+          #
+          #   job.wait_until_done!
+          #
+          # @example With a Cloud KMS encryption key name for a CMEK-protected cluster:
+          #   require "google/cloud/bigtable"
+          #
+          #   bigtable = Google::Cloud::Bigtable.new
+          #
+          #   kms_key_name = "projects/a/locations/b/keyRings/c/cryptoKeys/d"
+          #   job = bigtable.create_instance("my-instance") do |clusters|
+          #     clusters.add("test-cluster", "us-east1-b", kms_key: kms_key_name)
+          #   end
           #
-          #   If not set then default will set to `:STORAGE_TYPE_UNSPECIFIED`
+          #   job.wait_until_done!
           #
-          def add name, location, nodes: nil, storage_type: nil
+          def add name, location, nodes: nil, storage_type: nil, kms_key: nil
+            if kms_key
+              encryption_config = Google::Cloud::Bigtable::Admin::V2::Cluster::EncryptionConfig.new(
+                kms_key_name: kms_key
+              )
+            end
             attrs = {
               serve_nodes:          nodes,
               location:             location,
-              default_storage_type: storage_type
+              default_storage_type: storage_type,
+              encryption_config:    encryption_config
             }.delete_if { |_, v| v.nil? }
 
             self[name] = Google::Cloud::Bigtable::Admin::V2::Cluster.new attrs

data/lib/google/cloud/bigtable/instance/list.rb

@@ -126,12 +126,12 @@ module Google
           #     puts instance.instance_id
           #   end
           #
-          def all
+          def all &block
             return enum_for :all unless block_given?
 
             results = self
             loop do
-              results.each { |r| yield r }
+              results.each(&block)
               break unless results.next?
               results = results.next
             end

data/lib/google/cloud/bigtable/mutation_entry.rb

@@ -15,6 +15,8 @@
 # limitations under the License.
 
 
+require "google/cloud/bigtable/convert"
+
 module Google
   module Cloud
     module Bigtable
@@ -84,8 +86,9 @@ module Google
         # @param qualifier [String] Column qualifier name.
         #   The qualifier of the column into which new data should be written.
         #   Can be any byte string, including an empty string.
-        # @param value [String, Integer] Cell value data.
-        #   The value to be written into the specified cell.
+        # @param value [String, Integer] Cell value data. The value to be written
+        #   into the specified cell. If the argument is an Integer, it will be
+        #   encoded as a 64-bit signed big-endian integer.
         # @param timestamp [Integer] Timestamp value in microseconds.
         #   The timestamp of the cell into which new data should be written.
         #   Use -1 for current Bigtable server time.
@@ -104,15 +107,15 @@ module Google
         # @example With timestamp.
         #   entry = Google::Cloud::Bigtable::MutationEntry.new("user-1")
         #   entry.set_cell(
-        #     "cf-1",
-        #     "field-1",
+        #     "cf1",
+        #     "field1",
         #     "XYZ",
         #     timestamp: (Time.now.to_f * 1000000).round(-3) # microseconds
         #   )
         #
         def set_cell family, qualifier, value, timestamp: nil
           # If value is integer, covert it to a 64-bit signed big-endian integer.
-          value = [value].pack "q>" if value.is_a? Integer
+          value = Convert.integer_to_signed_be_64 value
           options = {
             family_name:      family,
             column_qualifier: qualifier,
@@ -156,14 +159,14 @@ module Google
         #
         # @example Without timestamp range.
         #   entry = Google::Cloud::Bigtable::MutationEntry.new("user-1")
-        #   entry.delete_cells("cf-1", "field-1")
+        #   entry.delete_cells("cf1", "field1")
         #
         # @example With timestamp range.
         #   entry = Google::Cloud::Bigtable::MutationEntry.new("user-1")
         #   timestamp_micros = (Time.now.to_f * 1000000).round(-3)
         #   entry.delete_cells(
         #     "cf1",
-        #     "field-1",
+        #     "field1",
         #     timestamp_from: timestamp_micros - 5000000,
         #     timestamp_to: timestamp_micros
         #   )
@@ -172,7 +175,7 @@ module Google
         #   timestamp_micros = (Time.now.to_f * 1000000).round(-3)
         #   entry.delete_cells(
         #     "cf1",
-        #     "field-1",
+        #     "field1",
         #     timestamp_from: timestamp_micros - 5000000
         #   )
         #
@@ -201,7 +204,7 @@ module Google
         #
         # @example
         #   entry = Google::Cloud::Bigtable::MutationEntry.new("user-1")
-        #   entry.delete_from_family("cf-1")
+        #   entry.delete_from_family("cf1")
         #
         def delete_from_family family
           @mutations << Google::Cloud::Bigtable::V2::Mutation.new(delete_from_family: { family_name: family })

data/lib/google/cloud/bigtable/mutation_operations.rb

@@ -65,8 +65,8 @@ module Google
         #
         #   entry = table.new_mutation_entry("user-1")
         #   entry.set_cell(
-        #     "cf-1",
-        #     "field-1",
+        #     "cf1",
+        #     "field1",
         #     "XYZ",
         #     timestamp: (Time.now.to_f * 1000000).round(-3) # microseconds
         #   ).delete_cells("cf2", "field02")
@@ -219,8 +219,8 @@ module Google
         #   predicate_filter = Google::Cloud::Bigtable::RowFilter.key("user-10")
         #   on_match_mutations = Google::Cloud::Bigtable::MutationEntry.new
         #   on_match_mutations.set_cell(
-        #     "cf-1",
-        #     "field-1",
+        #     "cf1",
+        #     "field1",
         #     "XYZ",
         #     timestamp: (Time.now.to_f * 1000000).round(-3) # microseconds
         #   ).delete_cells("cf2", "field02")
@@ -335,7 +335,8 @@ module Google
         #   end
         #
         class Response
-          attr_reader :index, :status
+          attr_reader :index
+          attr_reader :status
 
           ##
           # @private Creates a MutationEntry::Response object.

data/lib/google/cloud/bigtable/policy.rb

@@ -54,7 +54,8 @@ module Google
       #   policy.roles["roles/viewer"] = ["allUsers"]
       #
       class Policy
-        attr_reader :etag, :roles
+        attr_reader :etag
+        attr_reader :roles
 
         # Creates a Policy instance.
         # @param etag [String]

data/lib/google/cloud/bigtable/project.rb

@@ -316,18 +316,18 @@ module Google
         #
         #   table = bigtable.table("my-instance", "my-table")
         #
-        # @example Get a table with schema-only view.
+        # @example Retrieve a table with a schema-only view.
         #   require "google/cloud/bigtable"
         #
         #   bigtable = Google::Cloud::Bigtable.new
         #
-        #   table = bigtable.table("my-instance", "my-table", perform_lookup: true, view: :SCHEMA_VIEW)
+        #   table = bigtable.table("my-instance", "my-table", perform_lookup: true)
         #   if table
         #     puts table.name
         #     puts table.column_families
         #   end
         #
-        # @example Get a table with all fields, cluster states, and column families.
+        # @example Retrieve a table with all fields, cluster states, and column families.
         #   require "google/cloud/bigtable"
         #
         #   bigtable = Google::Cloud::Bigtable.new
@@ -342,6 +342,7 @@ module Google
         def table instance_id, table_id, view: nil, perform_lookup: nil, app_profile_id: nil
           ensure_service!
 
+          view ||= :SCHEMA_VIEW
           table = if perform_lookup
                     grpc = service.get_table instance_id, table_id, view: view
                     Table.from_grpc grpc, service, view: view

data/lib/google/cloud/bigtable/routing_policy.rb

@@ -141,7 +141,8 @@ module Google
       #     Default value is false.
       #
       class SingleClusterRouting < RoutingPolicy
-        attr_reader :cluster_id, :allow_transactional_writes
+        attr_reader :cluster_id
+        attr_reader :allow_transactional_writes
 
         ##
         # Creates a new single-cluster routing policy.
@@ -155,6 +156,7 @@ module Google
         #   Default value is false.
         #
         def initialize cluster_id, allow_transactional_writes
+          super()
           @cluster_id = cluster_id
           @allow_transactional_writes = allow_transactional_writes
         end

data/lib/google/cloud/bigtable/row.rb

@@ -30,7 +30,11 @@ module Google
         # Row cell built from data chunks.
         #
         class Cell
-          attr_reader :family, :qualifier, :value, :labels, :timestamp
+          attr_reader :family
+          attr_reader :qualifier
+          attr_reader :value
+          attr_reader :labels
+          attr_reader :timestamp
 
           ##
           # Creates a row cell instance.

data/lib/google/cloud/bigtable/row_filter/simple_filter.rb

@@ -15,6 +15,8 @@
 # limitations under the License.
 
 
+require "google/cloud/bigtable/convert"
+
 module Google
   module Cloud
     module Bigtable
@@ -147,10 +149,13 @@ module Google
           # will not match the new line character `\n`, which may be present in a
           # binary value.
           #
-          # @param regex [String] Regex to match cell value.
+          # @param regex [String, Integer] Regex to match cell value, or an Integer
+          #   value to be encoded as a 64-bit signed big-endian integer.
           # @return [Google::Cloud::Bigtable::RowFilter::SimpleFilter]
           #
           def value regex
+            # If regex is integer, covert it to a 64-bit signed big-endian integer.
+            regex = Convert.integer_to_signed_be_64 regex
             @grpc.value_regex_filter = regex
             self
           end

data/lib/google/cloud/bigtable/rows_reader.rb

@@ -170,10 +170,10 @@ module Google
         # @return [Boolean]
         #
         def start_key_read? range
-          start_key = if !range.start_key_closed.empty?
-                        range.start_key_closed
-                      else
+          start_key = if range.start_key_closed.empty?
                         range.start_key_open
+                      else
+                        range.start_key_closed
                       end
 
           start_key.empty? || last_key >= start_key
@@ -186,10 +186,10 @@ module Google
         # @return [Boolean]
         #
         def end_key_read? range
-          end_key = if !range.end_key_closed.empty?
-                      range.end_key_closed
-                    else
+          end_key = if range.end_key_closed.empty?
                       range.end_key_open
+                    else
+                      range.end_key_closed
                     end
 
           end_key && end_key <= last_key

data/lib/google/cloud/bigtable/status.rb

@@ -47,7 +47,10 @@ module Google
       #   end
       #
       class Status
-        attr_reader :code, :description, :message, :details
+        attr_reader :code
+        attr_reader :description
+        attr_reader :message
+        attr_reader :details
 
         ##
         # @private Creates a Status object.

data/lib/google/cloud/bigtable/table.rb

@@ -56,6 +56,14 @@ module Google
         # The gRPC Service object.
         attr_accessor :service
 
+        # @private
+        # The current gRPC resource, for testing only.
+        attr_accessor :grpc
+
+        # @private
+        # The current loaded_views, for testing only. See #check_view_and_load, below.
+        attr_reader :loaded_views
+
         ##
         # @return [String] App profile ID for request routing.
         #
@@ -64,10 +72,11 @@ module Google
         # @private
         #
         # Creates a new Table instance.
-        def initialize grpc, service, view: nil
+        def initialize grpc, service, view:
           @grpc = grpc
           @service = service
-          @view = view || :SCHEMA_VIEW
+          raise ArgumentError, "view must not be nil" if view.nil?
+          @loaded_views = Set[view]
         end
 
         ##
@@ -109,7 +118,8 @@ module Google
         end
 
         ##
-        # Reloads table data.
+        # Reloads table data with the provided `view`, or with `SCHEMA_VIEW`
+        # if none is provided. Previously loaded data is not retained.
         #
         # @param view [Symbol] Table view type.
         #   Default view type is `:SCHEMA_VIEW`.
@@ -123,22 +133,41 @@ module Google
         # @return [Google::Cloud::Bigtable::Table]
         #
         def reload! view: nil
-          @view = view || :SCHEMA_VIEW
+          view ||= :SCHEMA_VIEW
           @grpc = service.get_table instance_id, name, view: view
+          @loaded_views = Set[view]
           self
         end
 
         ##
-        # Map from cluster ID to per-cluster table state.
+        # Returns an array of {Table::ClusterState} objects that map cluster ID
+        # to per-cluster table state.
+        #
         # If it could not be determined whether or not the table has data in a
         # particular cluster (for example, if its zone is unavailable), then
-        # there will be an entry for the cluster with UNKNOWN `replication_status`.
-        # Views: `FULL`.
+        # the cluster state's `replication_state` will be `UNKNOWN`.
+        #
+        # Reloads the table with the `FULL` view type to retrieve the cluster states
+        # data, unless the table was previously loaded with view type `ENCRYPTION_VIEW`,
+        # `REPLICATION_VIEW` or `FULL`.
         #
         # @return [Array<Google::Cloud::Bigtable::Table::ClusterState>]
         #
+        # @example Retrieve a table with cluster states.
+        #   require "google/cloud/bigtable"
+        #
+        #   bigtable = Google::Cloud::Bigtable.new
+        #
+        #   table = bigtable.table("my-instance", "my-table", view: :FULL, perform_lookup: true)
+        #
+        #   table.cluster_states.each do |cs|
+        #     puts cs.cluster_name
+        #     puts cs.replication_state
+        #     puts cs.encryption_infos.first.encryption_type
+        #   end
+        #
         def cluster_states
-          check_view_and_load :REPLICATION_VIEW
+          check_view_and_load :FULL, skip_if: [:ENCRYPTION_VIEW, :REPLICATION_VIEW]
           @grpc.cluster_states.map do |name, state_grpc|
             ClusterState.from_grpc state_grpc, name
           end
@@ -146,15 +175,19 @@ module Google
 
         ##
         # Returns a frozen object containing the column families configured for
-        # the table, mapped by column family name. Reloads the table if
-        # necessary to retrieve the column families data, since it is only
-        # available in a table with view type `SCHEMA_VIEW` or `FULL`.
+        # the table, mapped by column family name.
+        #
+        # Reloads the table if necessary to retrieve the column families data,
+        # since it is only available in a table with view type `SCHEMA_VIEW`
+        # or `FULL`. Previously loaded data is retained.
         #
         # Also accepts a block for making modifications to the table's column
         # families. After the modifications are completed, the table will be
         # updated with the changes, and the updated column families will be
         # returned.
         #
+        # @see https://cloud.google.com/bigtable/docs/garbage-collection Garbage collection
+        #
         # @yield [column_families] A block for modifying the table's column
         #   families. Applies multiple column modifications. Performs a series
         #   of column family modifications on the specified table. Either all or
@@ -222,7 +255,10 @@ module Google
         # The granularity (e.g. `MILLIS`, `MICROS`) at which timestamps are stored in
         # this table. Timestamps not matching the granularity will be rejected.
         # If unspecified at creation time, the value will be set to `MILLIS`.
-        # Views: `SCHEMA_VIEW`, `FULL`.
+        #
+        # Reloads the table if necessary to retrieve the column families data,
+        # since it is only available in a table with view type `SCHEMA_VIEW`
+        # or `FULL`. Previously loaded data is retained.
         #
         # @return [Symbol]
         #
@@ -437,7 +473,7 @@ module Google
           }.delete_if { |_, v| v.nil? })
 
           grpc = service.create_table instance_id, table_id, table, initial_splits: initial_splits
-          from_grpc grpc, service
+          from_grpc grpc, service, view: :SCHEMA_VIEW
         end
 
         ##
@@ -623,7 +659,7 @@ module Google
         # @param view [Symbol] View type.
         # @return [Google::Cloud::Bigtable::Table]
         #
-        def self.from_grpc grpc, service, view: nil
+        def self.from_grpc grpc, service, view:
           new grpc, service, view: view
         end
 
@@ -653,20 +689,24 @@ module Google
 
         FIELDS_BY_VIEW = {
           SCHEMA_VIEW:      ["granularity", "column_families"],
+          ENCRYPTION_VIEW:  ["cluster_states"],
           REPLICATION_VIEW: ["cluster_states"],
           FULL:             ["granularity", "column_families", "cluster_states"]
         }.freeze
 
         # @private
         #
-        # Checks and reloads table with expected view and sets fields.
-        # @param view [Symbol] Expected view type.
+        # Checks and reloads table with expected view. Performs additive updates to fields specified by the given view.
+        # @param view [Symbol] The view type to load. If already loaded, no load is performed.
+        # @param skip_if [Symbol] Additional satisfying view types. If already loaded, no load is performed.
         #
-        def check_view_and_load view
+        def check_view_and_load view, skip_if: nil
           ensure_service!
-          @loaded_views ||= Set.new [@view]
 
-          return if @loaded_views.include?(view) || @loaded_views.include?(:FULL)
+          skip = Set.new skip_if
+          skip << view
+          skip << :FULL
+          return if (@loaded_views & skip).any?
 
           grpc = service.get_table instance_id, table_id, view: view
           @loaded_views << view
@@ -674,6 +714,9 @@ module Google
           FIELDS_BY_VIEW[view].each do |field|
             case grpc[field]
             when Google::Protobuf::Map
+              # Special handling for column_families:
+              # Replace contents of existing Map since setting the new Map won't work.
+              # See https://github.com/protocolbuffers/protobuf/issues/4969
               @grpc[field].clear
               grpc[field].each { |k, v| @grpc[field][k] = v }
             else