google-cloud-bigtable 0.6.1 → 1.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (61) hide show
  1. checksums.yaml +4 -4
  2. data/AUTHENTICATION.md +4 -26
  3. data/CHANGELOG.md +85 -0
  4. data/CONTRIBUTING.md +1 -1
  5. data/OVERVIEW.md +388 -19
  6. data/lib/google-cloud-bigtable.rb +19 -22
  7. data/lib/google/bigtable/admin/v2/bigtable_table_admin_services_pb.rb +1 -1
  8. data/lib/google/bigtable/v2/bigtable_pb.rb +3 -0
  9. data/lib/google/bigtable/v2/bigtable_services_pb.rb +1 -1
  10. data/lib/google/cloud/bigtable.rb +11 -17
  11. data/lib/google/cloud/bigtable/admin.rb +1 -1
  12. data/lib/google/cloud/bigtable/admin/v2.rb +1 -1
  13. data/lib/google/cloud/bigtable/admin/v2/bigtable_table_admin_client.rb +1 -1
  14. data/lib/google/cloud/bigtable/admin/v2/doc/google/iam/v1/policy.rb +42 -21
  15. data/lib/google/cloud/bigtable/app_profile.rb +162 -96
  16. data/lib/google/cloud/bigtable/app_profile/job.rb +5 -8
  17. data/lib/google/cloud/bigtable/app_profile/list.rb +18 -12
  18. data/lib/google/cloud/bigtable/chunk_processor.rb +24 -36
  19. data/lib/google/cloud/bigtable/cluster.rb +45 -18
  20. data/lib/google/cloud/bigtable/cluster/job.rb +3 -7
  21. data/lib/google/cloud/bigtable/cluster/list.rb +22 -20
  22. data/lib/google/cloud/bigtable/column_family.rb +18 -231
  23. data/lib/google/cloud/bigtable/column_family_map.rb +426 -0
  24. data/lib/google/cloud/bigtable/column_range.rb +15 -7
  25. data/lib/google/cloud/bigtable/convert.rb +12 -4
  26. data/lib/google/cloud/bigtable/errors.rb +4 -1
  27. data/lib/google/cloud/bigtable/gc_rule.rb +188 -69
  28. data/lib/google/cloud/bigtable/instance.rb +209 -189
  29. data/lib/google/cloud/bigtable/instance/cluster_map.rb +17 -13
  30. data/lib/google/cloud/bigtable/instance/job.rb +6 -5
  31. data/lib/google/cloud/bigtable/instance/list.rb +18 -13
  32. data/lib/google/cloud/bigtable/longrunning_job.rb +7 -1
  33. data/lib/google/cloud/bigtable/mutation_entry.rb +36 -39
  34. data/lib/google/cloud/bigtable/mutation_operations.rb +90 -73
  35. data/lib/google/cloud/bigtable/policy.rb +9 -5
  36. data/lib/google/cloud/bigtable/project.rb +87 -196
  37. data/lib/google/cloud/bigtable/read_modify_write_rule.rb +15 -10
  38. data/lib/google/cloud/bigtable/read_operations.rb +42 -59
  39. data/lib/google/cloud/bigtable/routing_policy.rb +172 -0
  40. data/lib/google/cloud/bigtable/row.rb +32 -21
  41. data/lib/google/cloud/bigtable/row_filter.rb +80 -35
  42. data/lib/google/cloud/bigtable/row_filter/chain_filter.rb +119 -68
  43. data/lib/google/cloud/bigtable/row_filter/condition_filter.rb +8 -2
  44. data/lib/google/cloud/bigtable/row_filter/interleave_filter.rb +117 -66
  45. data/lib/google/cloud/bigtable/row_filter/simple_filter.rb +24 -9
  46. data/lib/google/cloud/bigtable/row_range.rb +5 -0
  47. data/lib/google/cloud/bigtable/rows_mutator.rb +14 -21
  48. data/lib/google/cloud/bigtable/rows_reader.rb +23 -18
  49. data/lib/google/cloud/bigtable/sample_row_key.rb +6 -3
  50. data/lib/google/cloud/bigtable/service.rb +200 -253
  51. data/lib/google/cloud/bigtable/status.rb +76 -0
  52. data/lib/google/cloud/bigtable/table.rb +158 -262
  53. data/lib/google/cloud/bigtable/table/cluster_state.rb +17 -6
  54. data/lib/google/cloud/bigtable/table/list.rb +16 -9
  55. data/lib/google/cloud/bigtable/v2.rb +1 -1
  56. data/lib/google/cloud/bigtable/v2/bigtable_client.rb +12 -12
  57. data/lib/google/cloud/bigtable/v2/doc/google/bigtable/v2/bigtable.rb +16 -13
  58. data/lib/google/cloud/bigtable/value_range.rb +19 -13
  59. data/lib/google/cloud/bigtable/version.rb +1 -1
  60. metadata +67 -25
  61. data/lib/google/cloud/bigtable/table/column_family_map.rb +0 -70
@@ -19,6 +19,7 @@ module Google
19
19
  module Cloud
20
20
  module Bigtable
21
21
  module RowFilter
22
+ ##
22
23
  #
23
24
  # # SimpleFilter
24
25
  #
@@ -30,6 +31,7 @@ module Google
30
31
  @grpc = Google::Bigtable::V2::RowFilter.new
31
32
  end
32
33
 
34
+ ##
33
35
  # Outputs all cells directly to the output of the read rather than to any parent filter.
34
36
  #
35
37
  # @return [Google::Cloud::Bigtable::RowFilter::SimpleFilter]
@@ -39,6 +41,7 @@ module Google
39
41
  self
40
42
  end
41
43
 
44
+ ##
42
45
  # Matches all cells, regardless of input. Functionally equivalent to
43
46
  # leaving `filter` unset, but included for completeness.
44
47
  #
@@ -49,6 +52,7 @@ module Google
49
52
  self
50
53
  end
51
54
 
55
+ ##
52
56
  # Does not match any cells, regardless of input. Useful for temporarily
53
57
  # disabling just part of a filter.
54
58
  #
@@ -59,6 +63,7 @@ module Google
59
63
  self
60
64
  end
61
65
 
66
+ ##
62
67
  # Replaces each cell's value with an empty string.
63
68
  #
64
69
  # @return [Google::Cloud::Bigtable::RowFilter::SimpleFilter]
@@ -68,6 +73,7 @@ module Google
68
73
  self
69
74
  end
70
75
 
76
+ ##
71
77
  # Matches only cells from rows whose keys satisfy the given RE2 regex. In
72
78
  # other words, passes through the entire row when the key matches, and
73
79
  # otherwise produces an empty row.
@@ -87,6 +93,7 @@ module Google
87
93
  self
88
94
  end
89
95
 
96
+ ##
90
97
  # Matches all cells from a row with probability p, and matches no cells
91
98
  # from the row with probability 1-p.
92
99
  #
@@ -95,15 +102,13 @@ module Google
95
102
  #
96
103
  def sample probability
97
104
  if probability >= 1 || probability <= 0
98
- raise(
99
- RowFilterError,
100
- "Probability must be greather then 0 and less then 1.0"
101
- )
105
+ raise RowFilterError, "Probability must be greater than 0 and less than 1.0."
102
106
  end
103
107
  @grpc.row_sample_filter = probability
104
108
  self
105
109
  end
106
110
 
111
+ ##
107
112
  # Matches only cells from columns whose families satisfy the given RE2
108
113
  # regex. For technical reasons, the regex must not contain the `:`
109
114
  # character, even if it is not being used as a literal.
@@ -119,6 +124,7 @@ module Google
119
124
  self
120
125
  end
121
126
 
127
+ ##
122
128
  # Matches only cells from columns whose qualifiers satisfy the given RE2
123
129
  # regex.
124
130
  # Note that, since column qualifiers can contain arbitrary bytes, the `\C`
@@ -134,6 +140,7 @@ module Google
134
140
  self
135
141
  end
136
142
 
143
+ ##
137
144
  # Matches only cells with values that satisfy the given regular expression.
138
145
  # Note that, since cell values can contain arbitrary bytes, the `\C` escape
139
146
  # sequence must be used if a true wildcard is desired. The `.` character
@@ -148,6 +155,7 @@ module Google
148
155
  self
149
156
  end
150
157
 
158
+ ##
151
159
  # Applies the given label to all cells in the output row. This allows
152
160
  # the client to determine which results were produced from which part of
153
161
  # the filter.
@@ -157,7 +165,7 @@ module Google
157
165
  #
158
166
  # Due to a technical limitation, it is not possible to apply
159
167
  # multiple labels to a cell. As a result, a chain may have no more than
160
- # one sub-filter whithatch contains an `apply_label_transformer`. It is okay for
168
+ # one sub-filter that contains an `apply_label_transformer`. It is okay for
161
169
  # an interleave to contain multiple `apply_label_transformers`, as they
162
170
  # will be applied to separate copies of the input.
163
171
  #
@@ -169,6 +177,7 @@ module Google
169
177
  self
170
178
  end
171
179
 
180
+ ##
172
181
  # Skips the first N cells of each row, matching all subsequent cells.
173
182
  # If duplicate cells are present, as is possible when using an interleave,
174
183
  # each copy of the cell is counted separately.
@@ -181,6 +190,7 @@ module Google
181
190
  self
182
191
  end
183
192
 
193
+ ##
184
194
  # Matches only the first N cells of each row.
185
195
  # If duplicate cells are present, as is possible when using an Interleave,
186
196
  # each copy of the cell is counted separately.
@@ -193,6 +203,7 @@ module Google
193
203
  self
194
204
  end
195
205
 
206
+ ##
196
207
  # Matches only the most recent N cells within each column. For example,
197
208
  # if N=2, this filter would match column `foo:bar` at timestamps 10 and 9,
198
209
  # skip all earlier cells in `foo:bar`, and then begin matching again in
@@ -208,7 +219,8 @@ module Google
208
219
  self
209
220
  end
210
221
 
211
- # Creates a timestamp-range filter instance.
222
+ ##
223
+ # Creates a timestamp-range filter.
212
224
  #
213
225
  # Matches only cells with timestamps within the given range.
214
226
  # Specifies a contiguous range of timestamps.
@@ -225,32 +237,35 @@ module Google
225
237
  self
226
238
  end
227
239
 
240
+ ##
228
241
  # Matches only cells with values that fall within the given range.
229
242
  #
230
243
  # See {Google::Cloud::Bigtable::ValueRange#from} and { Google::Cloud::Bigtable::ValueRange#to} for range
231
244
  # option inclusive/exclusive options
232
245
  #
233
246
  # * The value at which to start the range. If neither field is set, interpreted as an empty string, inclusive.
234
- # * The value at which to end the range. If neither field is set, interpreted as an infinite string, exclusive.
247
+ # * The value at which to end the range. If neither field is set, interpreted as an infinite string,
248
+ # exclusive.
235
249
  #
236
250
  # @param range [Google::Cloud::Bigtable::ValueRange]
237
251
  # @return [Google::Cloud::Bigtable::RowFilter::SimpleFilter]
238
252
  #
239
253
  def value_range range
240
- unless range.instance_of?(Google::Cloud::Bigtable::ValueRange)
254
+ unless range.instance_of? Google::Cloud::Bigtable::ValueRange
241
255
  raise RowFilterError, "Range type mustbe ValueRange"
242
256
  end
243
257
  @grpc.value_range_filter = range.to_grpc
244
258
  self
245
259
  end
246
260
 
261
+ ##
247
262
  # Matches only cells from columns within the given range.
248
263
  #
249
264
  # @param range [Google::Cloud::Bigtable::ColumnRange]
250
265
  # @return [Google::Cloud::Bigtable::RowFilter::SimpleFilter]
251
266
  #
252
267
  def column_range range
253
- unless range.instance_of?(Google::Cloud::Bigtable::ColumnRange)
268
+ unless range.instance_of? Google::Cloud::Bigtable::ColumnRange
254
269
  raise RowFilterError, "Range type mustbe ColumnRange"
255
270
  end
256
271
  @grpc.column_range_filter = range.to_grpc
@@ -16,6 +16,7 @@
16
16
  module Google
17
17
  module Cloud
18
18
  module Bigtable
19
+ ##
19
20
  # # RowRange
20
21
  #
21
22
  # Specifies a contiguous range of rows.
@@ -59,6 +60,7 @@ module Google
59
60
  @grpc = Google::Bigtable::V2::RowRange.new
60
61
  end
61
62
 
63
+ ##
62
64
  # Sets a row range with a lower bound.
63
65
  #
64
66
  # @param key [String] Row key. Required.
@@ -91,6 +93,7 @@ module Google
91
93
  self
92
94
  end
93
95
 
96
+ ##
94
97
  # Sets a row range with an upper bound.
95
98
  #
96
99
  # @param key [String] Row key. Required.
@@ -123,6 +126,7 @@ module Google
123
126
  self
124
127
  end
125
128
 
129
+ ##
126
130
  # Sets a row range with inclusive upper and lower bounds.
127
131
  #
128
132
  # @param from_key [String] Inclusive from row key. Required.
@@ -142,6 +146,7 @@ module Google
142
146
  from(from_key).to(to_key, inclusive: true)
143
147
  end
144
148
 
149
+ ##
145
150
  # Sets a row range with an inclusive lower bound and an exclusive upper bound.
146
151
  #
147
152
  # @param from_key [String] Inclusive from row key.
@@ -31,8 +31,8 @@ module Google
31
31
  # Retryable status codes
32
32
  RETRYABLE_CODES = {
33
33
  Google::Rpc::Code::DEADLINE_EXCEEDED => true,
34
- Google::Rpc::Code::ABORTED => true,
35
- Google::Rpc::Code::UNAVAILABLE => true
34
+ Google::Rpc::Code::ABORTED => true,
35
+ Google::Rpc::Code::UNAVAILABLE => true
36
36
  }.freeze
37
37
 
38
38
  # @private
@@ -50,26 +50,25 @@ module Google
50
50
  @entries = entries
51
51
  end
52
52
 
53
+ ##
53
54
  # Applies mutations.
54
55
  #
55
56
  # @return [Array<Google::Bigtable::V2::MutateRowsResponse::Entry>]
56
57
  #
57
58
  def apply_mutations
58
59
  @req_entries = @entries.map(&:to_grpc)
59
- statuses = mutate_rows(@req_entries)
60
+ statuses = mutate_rows @req_entries
60
61
 
61
62
  # Collects retryable mutations indices.
62
- indices = statuses.each_with_object([]) do |e, r|
63
- if @entries[e.index].retryable? && RETRYABLE_CODES[e.status.code]
64
- r << e.index
65
- end
63
+ indices = statuses.each_with_object [] do |e, r|
64
+ r << e.index if @entries[e.index].retryable? && RETRYABLE_CODES[e.status.code]
66
65
  end
67
66
 
68
67
  return statuses if indices.empty?
69
68
 
70
69
  (RETRY_LIMIT - 1).times do
71
70
  break if indices.empty?
72
- indices = retry_entries(statuses, indices)
71
+ indices = retry_entries statuses, indices
73
72
  end
74
73
 
75
74
  statuses
@@ -77,26 +76,20 @@ module Google
77
76
 
78
77
  private
79
78
 
79
+ ##
80
80
  # Mutates rows.
81
81
  #
82
82
  # @param entries [Array<Google::Cloud::Bigtable::MutationEntry>]
83
83
  # @return [Array<Google::Bigtable::V2::MutateRowsResponse::Entry>]
84
84
  #
85
85
  def mutate_rows entries
86
- response = @table.client.mutate_rows(
87
- @table.path,
88
- entries,
89
- app_profile_id: @table.app_profile_id
90
- )
91
- response.each_with_object([]) do |res, statuses|
92
- statuses.concat(res.entries)
86
+ response = @table.service.mutate_rows @table.path, entries, app_profile_id: @table.app_profile_id
87
+ response.each_with_object [] do |res, statuses|
88
+ statuses.concat res.entries
93
89
  end
94
- rescue Google::Gax::GaxError => e
95
- raise Google::Cloud::Error.from_error(e.cause)
96
- rescue GRPC::BadStatus => e
97
- raise Google::Cloud::Error.from_error(e)
98
90
  end
99
91
 
92
+ ##
100
93
  # Collects failed entries, retries mutation, and updates status.
101
94
  #
102
95
  # @param statuses [Array<Google::Bigtable::V2::MutateRowsResponse::Entry>]
@@ -107,9 +100,9 @@ module Google
107
100
  #
108
101
  def retry_entries statuses, indices
109
102
  entries = indices.map { |i| @req_entries[i] }
110
- retry_statuses = mutate_rows(entries)
103
+ retry_statuses = mutate_rows entries
111
104
 
112
- retry_statuses.each_with_object([]) do |e, next_indices|
105
+ retry_statuses.each_with_object [] do |e, next_indices|
113
106
  next_indices << indices[e.index] if RETRYABLE_CODES[e.status.code]
114
107
  statuses[indices[e.index]].status = e.status
115
108
  end
@@ -23,7 +23,7 @@ module Google
23
23
  # @private
24
24
  # # RowsReader
25
25
  #
26
- # Retyable read rows helper
26
+ # Retryable read rows helper
27
27
  #
28
28
  class RowsReader
29
29
  # @private
@@ -56,6 +56,7 @@ module Google
56
56
  @retry_count = 0
57
57
  end
58
58
 
59
+ ##
59
60
  # Read rows
60
61
  #
61
62
  # @param rows [Google::Bigtable::V2::RowSet]
@@ -72,22 +73,20 @@ module Google
72
73
  # The default (zero) is to return all results.
73
74
  # @return [:yields: row]
74
75
  # Array of row or yield block for each processed row.
75
-
76
- def read \
77
- rows: nil,
78
- filter: nil,
79
- rows_limit: nil
80
- response = @table.client.read_rows(
81
- @table.path,
82
- rows: rows,
83
- filter: filter,
84
- rows_limit: rows_limit,
76
+ #
77
+ def read rows: nil, filter: nil, rows_limit: nil
78
+ response = @table.service.read_rows(
79
+ @table.instance_id,
80
+ @table.table_id,
81
+ rows: rows,
82
+ filter: filter,
83
+ rows_limit: rows_limit,
85
84
  app_profile_id: @table.app_profile_id
86
85
  )
87
86
  response.each do |res|
88
87
  res.chunks.each do |chunk|
89
88
  @retry_count = 0
90
- row = @chunk_processor.process(chunk)
89
+ row = @chunk_processor.process chunk
91
90
  next if row.nil?
92
91
  yield row
93
92
  @rows_count += 1
@@ -97,15 +96,17 @@ module Google
97
96
  @chunk_processor.validate_last_row_complete
98
97
  end
99
98
 
99
+ ##
100
100
  # Last read row key.
101
101
  #
102
102
  # @return [String]
103
-
103
+ #
104
104
  def last_key
105
105
  @chunk_processor.last_key
106
106
  end
107
107
 
108
- # Calucates and returns the read rows limit and row set based on last read key.
108
+ ##
109
+ # Calculates and returns the read rows limit and row set based on last read key.
109
110
  #
110
111
  # @param rows_limit [Integer]
111
112
  # The read will terminate after committing to N rows' worth of results.
@@ -116,7 +117,7 @@ module Google
116
117
  # A hash of the same form as `Google::Bigtable::V2::RowSet`
117
118
  # can also be provided.
118
119
  # @return [Integer, Google::Bigtable::V2::RowSet]
119
-
120
+ #
120
121
  def retry_options rows_limit, row_set
121
122
  return [rows_limit, row_set] unless last_key
122
123
 
@@ -129,14 +130,14 @@ module Google
129
130
  delete_indexes = []
130
131
 
131
132
  row_set.row_ranges.each_with_index do |range, i|
132
- if end_key_read?(range)
133
+ if end_key_read? range
133
134
  delete_indexes << i
134
- elsif start_key_read?(range)
135
+ elsif start_key_read? range
135
136
  range.start_key_open = last_key
136
137
  end
137
138
  end
138
139
 
139
- delete_indexes.each { |i| row_set.row_ranges.delete_at(i) }
140
+ delete_indexes.each { |i| row_set.row_ranges.delete_at i }
140
141
  end
141
142
 
142
143
  if row_set.row_ranges.empty?
@@ -151,15 +152,18 @@ module Google
151
152
  [rows_limit, row_set]
152
153
  end
153
154
 
155
+ ##
154
156
  # Checks if a read operation is retryable.
155
157
  #
156
158
  # @return [Boolean]
159
+ #
157
160
  def retryable?
158
161
  @retry_count < RowsReader::RETRY_LIMIT
159
162
  end
160
163
 
161
164
  private
162
165
 
166
+ ##
163
167
  # Checks if the start key was already read for the range.
164
168
  #
165
169
  # @param range [Google::Bigtable::V2::RowRange]
@@ -175,6 +179,7 @@ module Google
175
179
  start_key.empty? || last_key >= start_key
176
180
  end
177
181
 
182
+ ##
178
183
  # Checks if the end key was already read for the range.
179
184
  #
180
185
  # @param range [Google::Bigtable::V2::RowRange]
@@ -18,6 +18,7 @@
18
18
  module Google
19
19
  module Cloud
20
20
  module Bigtable
21
+ ##
21
22
  # # SampleRowKey
22
23
  #
23
24
  # Sample row key with byte offset.
@@ -30,14 +31,14 @@ module Google
30
31
  # Note: that row keys in this list may not have ever been written to or read
31
32
  # from, and users should therefore not make any assumptions about the row key
32
33
  # structure that are specific to their use case.
33
-
34
+ #
34
35
  # * offset_bytes : Approximate total storage space used by all rows in the table which precede
35
36
  # `row_key`. Buffering the contents of all rows between two subsequent
36
37
  # samples would require space roughly equal to the difference in their
37
38
  # `offset_bytes` fields.
38
39
  #
39
40
  # @example
40
- # require "google/cloud"
41
+ # require "google/cloud/bigtable"
41
42
  #
42
43
  # bigtable = Google::Cloud::Bigtable.new
43
44
  #
@@ -48,9 +49,11 @@ module Google
48
49
  # end
49
50
  #
50
51
  class SampleRowKey
52
+ ##
51
53
  # @return [String] Sample row key.
52
54
  attr_reader :key
53
55
 
56
+ ##
54
57
  # @return [Integer] Row offset in bytes.
55
58
  attr_reader :offset
56
59
 
@@ -74,7 +77,7 @@ module Google
74
77
  # @return [Google::Cloud::Bigtable::SampleRowKey]
75
78
  #
76
79
  def self.from_grpc grpc
77
- new(grpc.row_key, grpc.offset_bytes)
80
+ new grpc.row_key, grpc.offset_bytes
78
81
  end
79
82
  end
80
83
  end