google-cloud-bigtable 0.1.0

data/lib/google/cloud/bigtable/row_range.rb

@@ -0,0 +1,174 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+module Google
+  module Cloud
+    module Bigtable
+      # # RowRange
+      #
+      # Specifies a contiguous range of rows.
+      #
+      # * from key bound : The row key at which to from the range.
+      #   If neither field is set, interpreted as the empty string, inclusive.
+      # * End key bound: The row key at which to end the range.
+      #   If neither field is set, interpreted as the infinite row key, exclusive.
+      #
+      # @example
+      #   require "google/cloud/bigtable"
+      #
+      #   bigtable = Google::Cloud::Bigtable.new
+      #   table = bigtable.table("my-instance", "my-table")
+      #
+      #   # Range that includes all row keys including "user-001" to "user-005"
+      #   table.new_row_range.from("user-001").to("user-005", inclusive: true)
+      #
+      #   # Range that includes all row keys including "user-001" up to exclusive "user-010".
+      #   table.new_row_range.from("user-001").to("user-010")
+      #
+      #   # Range that includes all row keys including "user-001" up until end of the row keys.
+      #   table.new_row_range.from("user-001")
+      #
+      #   # Range that includes all row keys exclusive "user-001" up until end of the row keys.
+      #   table.new_row_range.from("user-001", inclusive: false)
+      #
+      #   # Range with unbounded from and the exclusive end "user-010"
+      #   table.new_row_range.to("user-010")
+      #
+      #   # Range that includes all row keys including from and end row keys "user-001", "user-010"
+      #   table.new_row_range.between("user-001", "user-010")
+      #
+      #   # Range that includes all row keys including "user-001" up until "user-010"
+      #   table.new_row_range.of("user-001", "user-010")
+      #
+      class RowRange
+        # @private
+        # Create row range instance.
+        def initialize
+          @grpc = Google::Bigtable::V2::RowRange.new
+        end
+
+        # Set row range with the lower bound.
+        #
+        # @param key [String] Row key. Required
+        # @param inclusive [String] Inclusive/Exclusive lower bound.
+        #   Default it is an inclusive lower bound.
+        # @return [Google::Cloud::Bigtable::RowRange]
+        #
+        # @example Inclusive lower bound.
+        #   require "google/cloud/bigtable"
+        #
+        #   bigtable = Google::Cloud::Bigtable.new
+        #   table = bigtable.table("my-instance", "my-table")
+        #
+        #   range = table.new_row_range.from("key-001")
+        #
+        # @example Exclusive lower bound.
+        #   require "google/cloud/bigtable"
+        #
+        #   bigtable = Google::Cloud::Bigtable.new
+        #   table = bigtable.table("my-instance", "my-table")
+        #
+        #   range = table.new_row_range.from("key-001", inclusive: false)
+        #
+        def from key, inclusive: true
+          if inclusive
+            @grpc.start_key_closed = key
+          else
+            @grpc.start_key_open = key
+          end
+          self
+        end
+
+        # Set row range with the upper bound.
+        #
+        # @param key [String] Row key. Required
+        # @param inclusive [String] Inclusive/Exclusive upper bound.
+        #   Default it is an exclusive upper bound.
+        # @return [Google::Cloud::Bigtable::RowRange]
+        #
+        # @example Inclusive upper bound.
+        #   require "google/cloud/bigtable"
+        #
+        #   bigtable = Google::Cloud::Bigtable.new
+        #   table = bigtable.table("my-instance", "my-table")
+        #
+        #   range = table.new_row_range.to("key-001", inclusive: true)
+        #
+        # @example Exclusive upper bound.
+        #   require "google/cloud/bigtable"
+        #
+        #   bigtable = Google::Cloud::Bigtable.new
+        #   table = bigtable.table("my-instance", "my-table")
+        #
+        #   range = table.new_row_range.to("key-001")
+        #
+        def to key, inclusive: false
+          if inclusive
+            @grpc.end_key_closed = key
+          else
+            @grpc.end_key_open = key
+          end
+          self
+        end
+
+        # Set row range with the inclusive upper and lower bounds.
+        #
+        # @param from_key [String] Inclusive from row key. Required
+        # @param to_key [String] Inclusive end row key. Required
+        # @return [Google::Cloud::Bigtable::RowRange]
+        #   Range with inclusive from and end row key.
+        #
+        # @example
+        #   require "google/cloud/bigtable"
+        #
+        #   bigtable = Google::Cloud::Bigtable.new
+        #   table = bigtable.table("my-instance", "my-table")
+        #
+        #   range = table.new_row_range.between("key-001", "key-010")
+        #
+        def between from_key, to_key
+          from(from_key).to(to_key, inclusive: true)
+        end
+
+        # Set row range with the inclusive lower and the exclusive upper bound.
+        #
+        # @param from_key [String] Inclusive from row key
+        # @param to_key [String] Exclusive end row key
+        # @return [Google::Cloud::Bigtable::RowRange]
+        #   Range with inclusive from and exclusive end row key.
+        #
+        # @example
+        #   require "google/cloud/bigtable"
+        #
+        #   bigtable = Google::Cloud::Bigtable.new
+        #   table = bigtable.table("my-instance", "my-table")
+        #
+        #   range = table.new_row_range.of("key-001", "key-010")
+        #
+        def of from_key, to_key
+          from(from_key).to(to_key)
+        end
+
+        # @private
+        #
+        # @return [Google::Bigtable::V2::RowRange]
+        #
+        def to_grpc
+          @grpc
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/bigtable/rows_mutator.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "google/rpc/code_pb"
+require "google/cloud/bigtable/mutation_entry"
+
+module Google
+  module Cloud
+    module Bigtable
+      # @private
+      # # RowsMutator
+      #
+      # Retryable mutate rows helper
+      #
+      class RowsMutator
+        # @private
+        # Retryable status codes
+        RETRYABLE_CODES = {
+          Google::Rpc::Code::DEADLINE_EXCEEDED => true,
+          Google::Rpc::Code::ABORTED => true,
+          Google::Rpc::Code::UNAVAILABLE => true
+        }.freeze
+
+        # @private
+        RETRY_LIMIT = 3
+
+        # @private
+        #
+        # Create mutate rows instance
+        #
+        # @param table [Google::Cloud::Bigtable::TableDataOperations]
+        # @param entries [Array<Google::Cloud::Bigtable::MutationEntry>]
+        #
+        def initialize table, entries
+          @table = table
+          @entries = entries
+        end
+
+        # Apply mutations.
+        #
+        # @return [Array<Google::Bigtable::V2::MutateRowsResponse::Entry>]
+        #
+        def apply_mutations
+          @req_entries = @entries.map(&:to_grpc)
+          statuses = mutate_rows(@req_entries)
+
+          # Collect retryable mutations indices
+          indices = statuses.each_with_object([]) do |e, r|
+            if @entries[e.index].retryable? && RETRYABLE_CODES[e.status.code]
+              r << e.index
+            end
+          end
+
+          return statuses if indices.empty?
+
+          (RETRY_LIMIT - 1).times do
+            break if indices.empty?
+            indices = retry_entries(statuses, indices)
+          end
+
+          statuses
+        end
+
+        private
+
+        # Mutate rows
+        #
+        # @param entries [Array<Google::Cloud::Bigtable::MutationEntry>]
+        # @return [Array<Google::Bigtable::V2::MutateRowsResponse::Entry>]
+        #
+        def mutate_rows entries
+          response = @table.client.mutate_rows(
+            @table.path,
+            entries,
+            app_profile_id: @table.app_profile_id
+          )
+          response.each_with_object([]) do |res, statuses|
+            statuses.concat(res.entries)
+          end
+        rescue Google::Gax::GaxError => e
+          raise Google::Cloud::Error.from_error(e.cause)
+        rescue GRPC::BadStatus => e
+          raise Google::Cloud::Error.from_error(e)
+        end
+
+        # Collected failed entries, retry mutation and update status
+        #
+        # @param statuses [Array<Google::Bigtable::V2::MutateRowsResponse::Entry>]
+        # @param indices [Array<Integer>]
+        #   Retry entries position mapping list
+        # @return [Array<Integer>]
+        #   New list of failed entries positions
+        #
+        def retry_entries statuses, indices
+          entries = indices.map { |i| @req_entries[i] }
+          retry_statuses = mutate_rows(entries)
+
+          retry_statuses.each_with_object([]) do |e, next_indices|
+            next_indices << indices[e.index] if RETRYABLE_CODES[e.status.code]
+            statuses[indices[e.index]].status = e.status
+          end
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/bigtable/rows_reader.rb

@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "google/cloud/bigtable/chunk_processor"
+
+module Google
+  module Cloud
+    module Bigtable
+      # @private
+      # # RowsReader
+      #
+      # Retyable read rows helper
+      #
+      class RowsReader
+        # @private
+        # Retryable error list.
+        RETRYABLE_ERRORS = [
+          GRPC::DeadlineExceeded,
+          GRPC::Aborted,
+          GRPC::Unavailable,
+          GRPC::Core::CallError
+        ].freeze
+
+        # @private
+        # Default retry limit
+        RETRY_LIMIT = 3
+
+        # @private
+        # @return [Integer] Current retry count
+        attr_accessor :retry_count
+
+        # @private
+        #
+        # Create read rows instance
+        #
+        # @param table [Google::Cloud::Bigtable::TableDataOperations]
+        #
+        def initialize table
+          @table = table
+          @chunk_processor = ChunkProcessor.new
+          @rows_count = 0
+          @retry_count = 0
+        end
+
+        # Read rows
+        #
+        # @param rows [Google::Bigtable::V2::RowSet]
+        #   The row keys and/or ranges to read.
+        #   If not specified, reads from all rows.
+        #   A hash of the same form as `Google::Bigtable::V2::RowSet`
+        #   can also be provided.
+        # @param filter [Google::Bigtable::V2::RowFilter | Hash]
+        #   The filter to apply to the contents of the specified row(s). If unset,
+        #   reads the entirety of each row.
+        #   A hash of the same form as `Google::Bigtable::V2::RowFilter`
+        #   can also be provided.
+        # @param rows_limit [Integer]
+        #   The read will terminate after committing to N rows' worth of results.
+        #   The default (zero) is to return all results.
+        # @return [:yields: row]
+        #   Array of row or yield block for each processed row.
+
+        def read \
+            rows: nil,
+            filter: nil,
+            rows_limit: nil
+          response = @table.client.read_rows(
+            @table.path,
+            rows: rows,
+            filter: filter,
+            rows_limit: rows_limit,
+            app_profile_id: @table.app_profile_id
+          )
+          response.each do |res|
+            res.chunks.each do |chunk|
+              @retry_count = 0
+              row = @chunk_processor.process(chunk)
+              next if row.nil?
+              yield row
+              @rows_count += 1
+            end
+          end
+
+          @chunk_processor.validate_last_row_complete
+        end
+
+        # Last read row key
+        #
+        # @return [String]
+
+        def last_key
+          @chunk_processor.last_key
+        end
+
+        # Calucate and return read rows limit and row set based on last read key
+        #
+        # @param rows_limit [Integer]
+        #   The read will terminate after committing to N rows' worth of results.
+        #   The default (zero) is to return all results.
+        # @param row_set [Google::Bigtable::V2::RowSet]
+        #   The row keys and/or ranges to read.
+        #   If not specified, reads from all rows.
+        #   A hash of the same form as `Google::Bigtable::V2::RowSet`
+        #   can also be provided.
+        # @return [Integer, Google::Bigtable::V2::RowSet]
+
+        def retry_options rows_limit, row_set
+          return [rows_limit, row_set] unless last_key
+
+          # 1. Reduce the limit by the number of already returned responses.
+          rows_limit -= @rows_count if rows_limit
+
+          # 2. Remove ranges that have already been read, and reduce ranges that
+          # include the last read rows
+          if last_key
+            delete_indexes = []
+
+            row_set.row_ranges.each_with_index do |range, i|
+              if end_key_read?(range)
+                delete_indexes << i
+              elsif start_key_read?(range)
+                range.start_key_open = last_key
+              end
+            end
+
+            delete_indexes.each { |i| row_set.row_ranges.delete_at(i) }
+          end
+
+          if row_set.row_ranges.empty?
+            row_set.row_ranges <<
+              Google::Bigtable::V2::RowRange.new(start_key_open: last_key)
+          end
+
+          # 3. Remove all individual keys before and up to the last read key
+          row_set.row_keys.select! { |k| k > last_key }
+
+          @chunk_processor.reset_to_new_row
+          [rows_limit, row_set]
+        end
+
+        # Check if read operation is retryable.
+        #
+        # @return [Boolean]
+        def retryable?
+          @retry_count < RowsReader::RETRY_LIMIT
+        end
+
+        private
+
+        # Check start key already read for range
+        #
+        # @param range [Google::Bigtable::V2::RowRange]
+        # @return [Boolean]
+        #
+        def start_key_read? range
+          start_key = if !range.start_key_closed.empty?
+                        range.start_key_closed
+                      else
+                        range.start_key_open
+                      end
+
+          start_key.empty? || last_key >= start_key
+        end
+
+        # Check end key already read for range
+        #
+        # @param range [Google::Bigtable::V2::RowRange]
+        # @return [Boolean]
+        #
+        def end_key_read? range
+          end_key = if !range.end_key_closed.empty?
+                      range.end_key_closed
+                    else
+                      range.end_key_open
+                    end
+
+          end_key && end_key <= last_key
+        end
+      end
+    end
+  end
+end