google-cloud-bigtable 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.yardopts +10 -0
- data/LICENSE +201 -0
- data/README.md +65 -0
- data/lib/google/bigtable/admin/v2/bigtable_instance_admin_pb.rb +139 -0
- data/lib/google/bigtable/admin/v2/bigtable_instance_admin_services_pb.rb +85 -0
- data/lib/google/bigtable/admin/v2/bigtable_table_admin_pb.rb +137 -0
- data/lib/google/bigtable/admin/v2/bigtable_table_admin_services_pb.rb +117 -0
- data/lib/google/bigtable/admin/v2/common_pb.rb +24 -0
- data/lib/google/bigtable/admin/v2/instance_pb.rb +72 -0
- data/lib/google/bigtable/admin/v2/table_pb.rb +88 -0
- data/lib/google/bigtable/v2/bigtable_pb.rb +109 -0
- data/lib/google/bigtable/v2/bigtable_services_pb.rb +67 -0
- data/lib/google/bigtable/v2/data_pb.rb +155 -0
- data/lib/google/cloud/bigtable/admin/credentials.rb +26 -0
- data/lib/google/cloud/bigtable/admin/v2/bigtable_instance_admin_client.rb +1417 -0
- data/lib/google/cloud/bigtable/admin/v2/bigtable_instance_admin_client_config.json +123 -0
- data/lib/google/cloud/bigtable/admin/v2/bigtable_table_admin_client.rb +1079 -0
- data/lib/google/cloud/bigtable/admin/v2/bigtable_table_admin_client_config.json +109 -0
- data/lib/google/cloud/bigtable/admin/v2/credentials.rb +50 -0
- data/lib/google/cloud/bigtable/admin/v2/doc/google/bigtable/admin/v2/bigtable_instance_admin.rb +279 -0
- data/lib/google/cloud/bigtable/admin/v2/doc/google/bigtable/admin/v2/bigtable_table_admin.rb +353 -0
- data/lib/google/cloud/bigtable/admin/v2/doc/google/bigtable/admin/v2/instance.rb +194 -0
- data/lib/google/cloud/bigtable/admin/v2/doc/google/bigtable/admin/v2/table.rb +209 -0
- data/lib/google/cloud/bigtable/admin/v2/doc/google/iam/v1/iam_policy.rb +62 -0
- data/lib/google/cloud/bigtable/admin/v2/doc/google/iam/v1/policy.rb +127 -0
- data/lib/google/cloud/bigtable/admin/v2/doc/google/longrunning/operations.rb +92 -0
- data/lib/google/cloud/bigtable/admin/v2/doc/google/protobuf/any.rb +124 -0
- data/lib/google/cloud/bigtable/admin/v2/doc/google/protobuf/duration.rb +90 -0
- data/lib/google/cloud/bigtable/admin/v2/doc/google/protobuf/empty.rb +28 -0
- data/lib/google/cloud/bigtable/admin/v2/doc/google/protobuf/field_mask.rb +223 -0
- data/lib/google/cloud/bigtable/admin/v2/doc/google/protobuf/timestamp.rb +106 -0
- data/lib/google/cloud/bigtable/admin/v2/doc/google/rpc/status.rb +83 -0
- data/lib/google/cloud/bigtable/admin/v2.rb +200 -0
- data/lib/google/cloud/bigtable/admin.rb +196 -0
- data/lib/google/cloud/bigtable/app_profile/job.rb +102 -0
- data/lib/google/cloud/bigtable/app_profile/list.rb +159 -0
- data/lib/google/cloud/bigtable/app_profile.rb +373 -0
- data/lib/google/cloud/bigtable/chunk_processor.rb +253 -0
- data/lib/google/cloud/bigtable/cluster/job.rb +92 -0
- data/lib/google/cloud/bigtable/cluster/list.rb +169 -0
- data/lib/google/cloud/bigtable/cluster.rb +264 -0
- data/lib/google/cloud/bigtable/column_family.rb +280 -0
- data/lib/google/cloud/bigtable/column_range.rb +186 -0
- data/lib/google/cloud/bigtable/convert.rb +75 -0
- data/lib/google/cloud/bigtable/credentials.rb +24 -0
- data/lib/google/cloud/bigtable/errors.rb +35 -0
- data/lib/google/cloud/bigtable/gc_rule.rb +215 -0
- data/lib/google/cloud/bigtable/instance/cluster_map.rb +70 -0
- data/lib/google/cloud/bigtable/instance/job.rb +97 -0
- data/lib/google/cloud/bigtable/instance/list.rb +159 -0
- data/lib/google/cloud/bigtable/instance.rb +921 -0
- data/lib/google/cloud/bigtable/longrunning_job.rb +105 -0
- data/lib/google/cloud/bigtable/mutation_entry.rb +244 -0
- data/lib/google/cloud/bigtable/mutation_operations.rb +338 -0
- data/lib/google/cloud/bigtable/policy.rb +163 -0
- data/lib/google/cloud/bigtable/project.rb +580 -0
- data/lib/google/cloud/bigtable/read_modify_write_rule.rb +129 -0
- data/lib/google/cloud/bigtable/read_operations.rb +345 -0
- data/lib/google/cloud/bigtable/row.rb +125 -0
- data/lib/google/cloud/bigtable/row_filter/chain_filter.rb +539 -0
- data/lib/google/cloud/bigtable/row_filter/condition_filter.rb +108 -0
- data/lib/google/cloud/bigtable/row_filter/interleave_filter.rb +570 -0
- data/lib/google/cloud/bigtable/row_filter/simple_filter.rb +273 -0
- data/lib/google/cloud/bigtable/row_filter.rb +593 -0
- data/lib/google/cloud/bigtable/row_range.rb +174 -0
- data/lib/google/cloud/bigtable/rows_mutator.rb +120 -0
- data/lib/google/cloud/bigtable/rows_reader.rb +196 -0
- data/lib/google/cloud/bigtable/sample_row_key.rb +82 -0
- data/lib/google/cloud/bigtable/service.rb +817 -0
- data/lib/google/cloud/bigtable/table/cluster_state.rb +93 -0
- data/lib/google/cloud/bigtable/table/column_family_map.rb +68 -0
- data/lib/google/cloud/bigtable/table/list.rb +147 -0
- data/lib/google/cloud/bigtable/table.rb +676 -0
- data/lib/google/cloud/bigtable/v2/bigtable_client.rb +579 -0
- data/lib/google/cloud/bigtable/v2/bigtable_client_config.json +65 -0
- data/lib/google/cloud/bigtable/v2/credentials.rb +45 -0
- data/lib/google/cloud/bigtable/v2/doc/google/bigtable/v2/bigtable.rb +286 -0
- data/lib/google/cloud/bigtable/v2/doc/google/bigtable/v2/data.rb +492 -0
- data/lib/google/cloud/bigtable/v2/doc/google/protobuf/any.rb +124 -0
- data/lib/google/cloud/bigtable/v2/doc/google/protobuf/wrappers.rb +89 -0
- data/lib/google/cloud/bigtable/v2/doc/google/rpc/status.rb +83 -0
- data/lib/google/cloud/bigtable/v2.rb +132 -0
- data/lib/google/cloud/bigtable/value_range.rb +175 -0
- data/lib/google/cloud/bigtable/version.rb +22 -0
- data/lib/google/cloud/bigtable.rb +223 -0
- data/lib/google-cloud-bigtable.rb +167 -0
- metadata +283 -0
@@ -0,0 +1,174 @@
|
|
1
|
+
# Copyright 2018 Google LLC
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
|
16
|
+
module Google
|
17
|
+
module Cloud
|
18
|
+
module Bigtable
|
19
|
+
# # RowRange
|
20
|
+
#
|
21
|
+
# Specifies a contiguous range of rows.
|
22
|
+
#
|
23
|
+
# * from key bound : The row key at which to from the range.
|
24
|
+
# If neither field is set, interpreted as the empty string, inclusive.
|
25
|
+
# * End key bound: The row key at which to end the range.
|
26
|
+
# If neither field is set, interpreted as the infinite row key, exclusive.
|
27
|
+
#
|
28
|
+
# @example
|
29
|
+
# require "google/cloud/bigtable"
|
30
|
+
#
|
31
|
+
# bigtable = Google::Cloud::Bigtable.new
|
32
|
+
# table = bigtable.table("my-instance", "my-table")
|
33
|
+
#
|
34
|
+
# # Range that includes all row keys including "user-001" to "user-005"
|
35
|
+
# table.new_row_range.from("user-001").to("user-005", inclusive: true)
|
36
|
+
#
|
37
|
+
# # Range that includes all row keys including "user-001" up to exclusive "user-010".
|
38
|
+
# table.new_row_range.from("user-001").to("user-010")
|
39
|
+
#
|
40
|
+
# # Range that includes all row keys including "user-001" up until end of the row keys.
|
41
|
+
# table.new_row_range.from("user-001")
|
42
|
+
#
|
43
|
+
# # Range that includes all row keys exclusive "user-001" up until end of the row keys.
|
44
|
+
# table.new_row_range.from("user-001", inclusive: false)
|
45
|
+
#
|
46
|
+
# # Range with unbounded from and the exclusive end "user-010"
|
47
|
+
# table.new_row_range.to("user-010")
|
48
|
+
#
|
49
|
+
# # Range that includes all row keys including from and end row keys "user-001", "user-010"
|
50
|
+
# table.new_row_range.between("user-001", "user-010")
|
51
|
+
#
|
52
|
+
# # Range that includes all row keys including "user-001" up until "user-010"
|
53
|
+
# table.new_row_range.of("user-001", "user-010")
|
54
|
+
#
|
55
|
+
class RowRange
|
56
|
+
# @private
|
57
|
+
# Create row range instance.
|
58
|
+
def initialize
|
59
|
+
@grpc = Google::Bigtable::V2::RowRange.new
|
60
|
+
end
|
61
|
+
|
62
|
+
# Set row range with the lower bound.
|
63
|
+
#
|
64
|
+
# @param key [String] Row key. Required
|
65
|
+
# @param inclusive [String] Inclusive/Exclusive lower bound.
|
66
|
+
# Default it is an inclusive lower bound.
|
67
|
+
# @return [Google::Cloud::Bigtable::RowRange]
|
68
|
+
#
|
69
|
+
# @example Inclusive lower bound.
|
70
|
+
# require "google/cloud/bigtable"
|
71
|
+
#
|
72
|
+
# bigtable = Google::Cloud::Bigtable.new
|
73
|
+
# table = bigtable.table("my-instance", "my-table")
|
74
|
+
#
|
75
|
+
# range = table.new_row_range.from("key-001")
|
76
|
+
#
|
77
|
+
# @example Exclusive lower bound.
|
78
|
+
# require "google/cloud/bigtable"
|
79
|
+
#
|
80
|
+
# bigtable = Google::Cloud::Bigtable.new
|
81
|
+
# table = bigtable.table("my-instance", "my-table")
|
82
|
+
#
|
83
|
+
# range = table.new_row_range.from("key-001", inclusive: false)
|
84
|
+
#
|
85
|
+
def from key, inclusive: true
|
86
|
+
if inclusive
|
87
|
+
@grpc.start_key_closed = key
|
88
|
+
else
|
89
|
+
@grpc.start_key_open = key
|
90
|
+
end
|
91
|
+
self
|
92
|
+
end
|
93
|
+
|
94
|
+
# Set row range with the upper bound.
|
95
|
+
#
|
96
|
+
# @param key [String] Row key. Required
|
97
|
+
# @param inclusive [String] Inclusive/Exclusive upper bound.
|
98
|
+
# Default it is an exclusive upper bound.
|
99
|
+
# @return [Google::Cloud::Bigtable::RowRange]
|
100
|
+
#
|
101
|
+
# @example Inclusive upper bound.
|
102
|
+
# require "google/cloud/bigtable"
|
103
|
+
#
|
104
|
+
# bigtable = Google::Cloud::Bigtable.new
|
105
|
+
# table = bigtable.table("my-instance", "my-table")
|
106
|
+
#
|
107
|
+
# range = table.new_row_range.to("key-001", inclusive: true)
|
108
|
+
#
|
109
|
+
# @example Exclusive upper bound.
|
110
|
+
# require "google/cloud/bigtable"
|
111
|
+
#
|
112
|
+
# bigtable = Google::Cloud::Bigtable.new
|
113
|
+
# table = bigtable.table("my-instance", "my-table")
|
114
|
+
#
|
115
|
+
# range = table.new_row_range.to("key-001")
|
116
|
+
#
|
117
|
+
def to key, inclusive: false
|
118
|
+
if inclusive
|
119
|
+
@grpc.end_key_closed = key
|
120
|
+
else
|
121
|
+
@grpc.end_key_open = key
|
122
|
+
end
|
123
|
+
self
|
124
|
+
end
|
125
|
+
|
126
|
+
# Set row range with the inclusive upper and lower bounds.
|
127
|
+
#
|
128
|
+
# @param from_key [String] Inclusive from row key. Required
|
129
|
+
# @param to_key [String] Inclusive end row key. Required
|
130
|
+
# @return [Google::Cloud::Bigtable::RowRange]
|
131
|
+
# Range with inclusive from and end row key.
|
132
|
+
#
|
133
|
+
# @example
|
134
|
+
# require "google/cloud/bigtable"
|
135
|
+
#
|
136
|
+
# bigtable = Google::Cloud::Bigtable.new
|
137
|
+
# table = bigtable.table("my-instance", "my-table")
|
138
|
+
#
|
139
|
+
# range = table.new_row_range.between("key-001", "key-010")
|
140
|
+
#
|
141
|
+
def between from_key, to_key
|
142
|
+
from(from_key).to(to_key, inclusive: true)
|
143
|
+
end
|
144
|
+
|
145
|
+
# Set row range with the inclusive lower and the exclusive upper bound.
|
146
|
+
#
|
147
|
+
# @param from_key [String] Inclusive from row key
|
148
|
+
# @param to_key [String] Exclusive end row key
|
149
|
+
# @return [Google::Cloud::Bigtable::RowRange]
|
150
|
+
# Range with inclusive from and exclusive end row key.
|
151
|
+
#
|
152
|
+
# @example
|
153
|
+
# require "google/cloud/bigtable"
|
154
|
+
#
|
155
|
+
# bigtable = Google::Cloud::Bigtable.new
|
156
|
+
# table = bigtable.table("my-instance", "my-table")
|
157
|
+
#
|
158
|
+
# range = table.new_row_range.of("key-001", "key-010")
|
159
|
+
#
|
160
|
+
def of from_key, to_key
|
161
|
+
from(from_key).to(to_key)
|
162
|
+
end
|
163
|
+
|
164
|
+
# @private
|
165
|
+
#
|
166
|
+
# @return [Google::Bigtable::V2::RowRange]
|
167
|
+
#
|
168
|
+
def to_grpc
|
169
|
+
@grpc
|
170
|
+
end
|
171
|
+
end
|
172
|
+
end
|
173
|
+
end
|
174
|
+
end
|
@@ -0,0 +1,120 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# Copyright 2018 Google LLC
|
4
|
+
#
|
5
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
6
|
+
# you may not use this file except in compliance with the License.
|
7
|
+
# You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing, software
|
12
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
13
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
14
|
+
# See the License for the specific language governing permissions and
|
15
|
+
# limitations under the License.
|
16
|
+
|
17
|
+
|
18
|
+
require "google/rpc/code_pb"
|
19
|
+
require "google/cloud/bigtable/mutation_entry"
|
20
|
+
|
21
|
+
module Google
|
22
|
+
module Cloud
|
23
|
+
module Bigtable
|
24
|
+
# @private
|
25
|
+
# # RowsMutator
|
26
|
+
#
|
27
|
+
# Retryable mutate rows helper
|
28
|
+
#
|
29
|
+
class RowsMutator
|
30
|
+
# @private
|
31
|
+
# Retryable status codes
|
32
|
+
RETRYABLE_CODES = {
|
33
|
+
Google::Rpc::Code::DEADLINE_EXCEEDED => true,
|
34
|
+
Google::Rpc::Code::ABORTED => true,
|
35
|
+
Google::Rpc::Code::UNAVAILABLE => true
|
36
|
+
}.freeze
|
37
|
+
|
38
|
+
# @private
|
39
|
+
RETRY_LIMIT = 3
|
40
|
+
|
41
|
+
# @private
|
42
|
+
#
|
43
|
+
# Create mutate rows instance
|
44
|
+
#
|
45
|
+
# @param table [Google::Cloud::Bigtable::TableDataOperations]
|
46
|
+
# @param entries [Array<Google::Cloud::Bigtable::MutationEntry>]
|
47
|
+
#
|
48
|
+
def initialize table, entries
|
49
|
+
@table = table
|
50
|
+
@entries = entries
|
51
|
+
end
|
52
|
+
|
53
|
+
# Apply mutations.
|
54
|
+
#
|
55
|
+
# @return [Array<Google::Bigtable::V2::MutateRowsResponse::Entry>]
|
56
|
+
#
|
57
|
+
def apply_mutations
|
58
|
+
@req_entries = @entries.map(&:to_grpc)
|
59
|
+
statuses = mutate_rows(@req_entries)
|
60
|
+
|
61
|
+
# Collect retryable mutations indices
|
62
|
+
indices = statuses.each_with_object([]) do |e, r|
|
63
|
+
if @entries[e.index].retryable? && RETRYABLE_CODES[e.status.code]
|
64
|
+
r << e.index
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
return statuses if indices.empty?
|
69
|
+
|
70
|
+
(RETRY_LIMIT - 1).times do
|
71
|
+
break if indices.empty?
|
72
|
+
indices = retry_entries(statuses, indices)
|
73
|
+
end
|
74
|
+
|
75
|
+
statuses
|
76
|
+
end
|
77
|
+
|
78
|
+
private
|
79
|
+
|
80
|
+
# Mutate rows
|
81
|
+
#
|
82
|
+
# @param entries [Array<Google::Cloud::Bigtable::MutationEntry>]
|
83
|
+
# @return [Array<Google::Bigtable::V2::MutateRowsResponse::Entry>]
|
84
|
+
#
|
85
|
+
def mutate_rows entries
|
86
|
+
response = @table.client.mutate_rows(
|
87
|
+
@table.path,
|
88
|
+
entries,
|
89
|
+
app_profile_id: @table.app_profile_id
|
90
|
+
)
|
91
|
+
response.each_with_object([]) do |res, statuses|
|
92
|
+
statuses.concat(res.entries)
|
93
|
+
end
|
94
|
+
rescue Google::Gax::GaxError => e
|
95
|
+
raise Google::Cloud::Error.from_error(e.cause)
|
96
|
+
rescue GRPC::BadStatus => e
|
97
|
+
raise Google::Cloud::Error.from_error(e)
|
98
|
+
end
|
99
|
+
|
100
|
+
# Collected failed entries, retry mutation and update status
|
101
|
+
#
|
102
|
+
# @param statuses [Array<Google::Bigtable::V2::MutateRowsResponse::Entry>]
|
103
|
+
# @param indices [Array<Integer>]
|
104
|
+
# Retry entries position mapping list
|
105
|
+
# @return [Array<Integer>]
|
106
|
+
# New list of failed entries positions
|
107
|
+
#
|
108
|
+
def retry_entries statuses, indices
|
109
|
+
entries = indices.map { |i| @req_entries[i] }
|
110
|
+
retry_statuses = mutate_rows(entries)
|
111
|
+
|
112
|
+
retry_statuses.each_with_object([]) do |e, next_indices|
|
113
|
+
next_indices << indices[e.index] if RETRYABLE_CODES[e.status.code]
|
114
|
+
statuses[indices[e.index]].status = e.status
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
@@ -0,0 +1,196 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# Copyright 2018 Google LLC
|
4
|
+
#
|
5
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
6
|
+
# you may not use this file except in compliance with the License.
|
7
|
+
# You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing, software
|
12
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
13
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
14
|
+
# See the License for the specific language governing permissions and
|
15
|
+
# limitations under the License.
|
16
|
+
|
17
|
+
|
18
|
+
require "google/cloud/bigtable/chunk_processor"
|
19
|
+
|
20
|
+
module Google
|
21
|
+
module Cloud
|
22
|
+
module Bigtable
|
23
|
+
# @private
|
24
|
+
# # RowsReader
|
25
|
+
#
|
26
|
+
# Retyable read rows helper
|
27
|
+
#
|
28
|
+
class RowsReader
|
29
|
+
# @private
|
30
|
+
# Retryable error list.
|
31
|
+
RETRYABLE_ERRORS = [
|
32
|
+
GRPC::DeadlineExceeded,
|
33
|
+
GRPC::Aborted,
|
34
|
+
GRPC::Unavailable,
|
35
|
+
GRPC::Core::CallError
|
36
|
+
].freeze
|
37
|
+
|
38
|
+
# @private
|
39
|
+
# Default retry limit
|
40
|
+
RETRY_LIMIT = 3
|
41
|
+
|
42
|
+
# @private
|
43
|
+
# @return [Integer] Current retry count
|
44
|
+
attr_accessor :retry_count
|
45
|
+
|
46
|
+
# @private
|
47
|
+
#
|
48
|
+
# Create read rows instance
|
49
|
+
#
|
50
|
+
# @param table [Google::Cloud::Bigtable::TableDataOperations]
|
51
|
+
#
|
52
|
+
def initialize table
|
53
|
+
@table = table
|
54
|
+
@chunk_processor = ChunkProcessor.new
|
55
|
+
@rows_count = 0
|
56
|
+
@retry_count = 0
|
57
|
+
end
|
58
|
+
|
59
|
+
# Read rows
|
60
|
+
#
|
61
|
+
# @param rows [Google::Bigtable::V2::RowSet]
|
62
|
+
# The row keys and/or ranges to read.
|
63
|
+
# If not specified, reads from all rows.
|
64
|
+
# A hash of the same form as `Google::Bigtable::V2::RowSet`
|
65
|
+
# can also be provided.
|
66
|
+
# @param filter [Google::Bigtable::V2::RowFilter | Hash]
|
67
|
+
# The filter to apply to the contents of the specified row(s). If unset,
|
68
|
+
# reads the entirety of each row.
|
69
|
+
# A hash of the same form as `Google::Bigtable::V2::RowFilter`
|
70
|
+
# can also be provided.
|
71
|
+
# @param rows_limit [Integer]
|
72
|
+
# The read will terminate after committing to N rows' worth of results.
|
73
|
+
# The default (zero) is to return all results.
|
74
|
+
# @return [:yields: row]
|
75
|
+
# Array of row or yield block for each processed row.
|
76
|
+
|
77
|
+
def read \
|
78
|
+
rows: nil,
|
79
|
+
filter: nil,
|
80
|
+
rows_limit: nil
|
81
|
+
response = @table.client.read_rows(
|
82
|
+
@table.path,
|
83
|
+
rows: rows,
|
84
|
+
filter: filter,
|
85
|
+
rows_limit: rows_limit,
|
86
|
+
app_profile_id: @table.app_profile_id
|
87
|
+
)
|
88
|
+
response.each do |res|
|
89
|
+
res.chunks.each do |chunk|
|
90
|
+
@retry_count = 0
|
91
|
+
row = @chunk_processor.process(chunk)
|
92
|
+
next if row.nil?
|
93
|
+
yield row
|
94
|
+
@rows_count += 1
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
@chunk_processor.validate_last_row_complete
|
99
|
+
end
|
100
|
+
|
101
|
+
# Last read row key
|
102
|
+
#
|
103
|
+
# @return [String]
|
104
|
+
|
105
|
+
def last_key
|
106
|
+
@chunk_processor.last_key
|
107
|
+
end
|
108
|
+
|
109
|
+
# Calucate and return read rows limit and row set based on last read key
|
110
|
+
#
|
111
|
+
# @param rows_limit [Integer]
|
112
|
+
# The read will terminate after committing to N rows' worth of results.
|
113
|
+
# The default (zero) is to return all results.
|
114
|
+
# @param row_set [Google::Bigtable::V2::RowSet]
|
115
|
+
# The row keys and/or ranges to read.
|
116
|
+
# If not specified, reads from all rows.
|
117
|
+
# A hash of the same form as `Google::Bigtable::V2::RowSet`
|
118
|
+
# can also be provided.
|
119
|
+
# @return [Integer, Google::Bigtable::V2::RowSet]
|
120
|
+
|
121
|
+
def retry_options rows_limit, row_set
|
122
|
+
return [rows_limit, row_set] unless last_key
|
123
|
+
|
124
|
+
# 1. Reduce the limit by the number of already returned responses.
|
125
|
+
rows_limit -= @rows_count if rows_limit
|
126
|
+
|
127
|
+
# 2. Remove ranges that have already been read, and reduce ranges that
|
128
|
+
# include the last read rows
|
129
|
+
if last_key
|
130
|
+
delete_indexes = []
|
131
|
+
|
132
|
+
row_set.row_ranges.each_with_index do |range, i|
|
133
|
+
if end_key_read?(range)
|
134
|
+
delete_indexes << i
|
135
|
+
elsif start_key_read?(range)
|
136
|
+
range.start_key_open = last_key
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
delete_indexes.each { |i| row_set.row_ranges.delete_at(i) }
|
141
|
+
end
|
142
|
+
|
143
|
+
if row_set.row_ranges.empty?
|
144
|
+
row_set.row_ranges <<
|
145
|
+
Google::Bigtable::V2::RowRange.new(start_key_open: last_key)
|
146
|
+
end
|
147
|
+
|
148
|
+
# 3. Remove all individual keys before and up to the last read key
|
149
|
+
row_set.row_keys.select! { |k| k > last_key }
|
150
|
+
|
151
|
+
@chunk_processor.reset_to_new_row
|
152
|
+
[rows_limit, row_set]
|
153
|
+
end
|
154
|
+
|
155
|
+
# Check if read operation is retryable.
|
156
|
+
#
|
157
|
+
# @return [Boolean]
|
158
|
+
def retryable?
|
159
|
+
@retry_count < RowsReader::RETRY_LIMIT
|
160
|
+
end
|
161
|
+
|
162
|
+
private
|
163
|
+
|
164
|
+
# Check start key already read for range
|
165
|
+
#
|
166
|
+
# @param range [Google::Bigtable::V2::RowRange]
|
167
|
+
# @return [Boolean]
|
168
|
+
#
|
169
|
+
def start_key_read? range
|
170
|
+
start_key = if !range.start_key_closed.empty?
|
171
|
+
range.start_key_closed
|
172
|
+
else
|
173
|
+
range.start_key_open
|
174
|
+
end
|
175
|
+
|
176
|
+
start_key.empty? || last_key >= start_key
|
177
|
+
end
|
178
|
+
|
179
|
+
# Check end key already read for range
|
180
|
+
#
|
181
|
+
# @param range [Google::Bigtable::V2::RowRange]
|
182
|
+
# @return [Boolean]
|
183
|
+
#
|
184
|
+
def end_key_read? range
|
185
|
+
end_key = if !range.end_key_closed.empty?
|
186
|
+
range.end_key_closed
|
187
|
+
else
|
188
|
+
range.end_key_open
|
189
|
+
end
|
190
|
+
|
191
|
+
end_key && end_key <= last_key
|
192
|
+
end
|
193
|
+
end
|
194
|
+
end
|
195
|
+
end
|
196
|
+
end
|