google-cloud-dataflow-v1beta3 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/google/cloud/dataflow/v1beta3/flex_templates_service/client.rb +3 -3
- data/lib/google/cloud/dataflow/v1beta3/flex_templates_service/credentials.rb +1 -3
- data/lib/google/cloud/dataflow/v1beta3/flex_templates_service/rest/client.rb +2 -2
- data/lib/google/cloud/dataflow/v1beta3/flex_templates_service/rest.rb +1 -1
- data/lib/google/cloud/dataflow/v1beta3/flex_templates_service.rb +1 -1
- data/lib/google/cloud/dataflow/v1beta3/jobs/client.rb +28 -7
- data/lib/google/cloud/dataflow/v1beta3/jobs/credentials.rb +1 -3
- data/lib/google/cloud/dataflow/v1beta3/jobs/rest/client.rb +27 -6
- data/lib/google/cloud/dataflow/v1beta3/messages/client.rb +2 -2
- data/lib/google/cloud/dataflow/v1beta3/messages/credentials.rb +1 -3
- data/lib/google/cloud/dataflow/v1beta3/messages/rest/client.rb +1 -1
- data/lib/google/cloud/dataflow/v1beta3/metrics/client.rb +2 -2
- data/lib/google/cloud/dataflow/v1beta3/metrics/credentials.rb +1 -3
- data/lib/google/cloud/dataflow/v1beta3/metrics/rest/client.rb +1 -1
- data/lib/google/cloud/dataflow/v1beta3/snapshots/client.rb +2 -2
- data/lib/google/cloud/dataflow/v1beta3/snapshots/credentials.rb +1 -3
- data/lib/google/cloud/dataflow/v1beta3/snapshots/rest/client.rb +1 -1
- data/lib/google/cloud/dataflow/v1beta3/templates_service/client.rb +27 -8
- data/lib/google/cloud/dataflow/v1beta3/templates_service/credentials.rb +1 -3
- data/lib/google/cloud/dataflow/v1beta3/templates_service/rest/client.rb +26 -7
- data/lib/google/cloud/dataflow/v1beta3/version.rb +1 -1
- data/lib/google/dataflow/v1beta3/environment_pb.rb +4 -1
- data/lib/google/dataflow/v1beta3/jobs_pb.rb +9 -1
- data/lib/google/dataflow/v1beta3/jobs_services_pb.rb +12 -2
- data/lib/google/dataflow/v1beta3/messages_pb.rb +1 -1
- data/lib/google/dataflow/v1beta3/metrics_pb.rb +10 -1
- data/lib/google/dataflow/v1beta3/snapshots_pb.rb +1 -1
- data/lib/google/dataflow/v1beta3/streaming_pb.rb +1 -1
- data/lib/google/dataflow/v1beta3/templates_pb.rb +3 -1
- data/lib/google/dataflow/v1beta3/templates_services_pb.rb +22 -3
- data/proto_docs/google/api/client.rb +6 -0
- data/proto_docs/google/dataflow/v1beta3/environment.rb +96 -16
- data/proto_docs/google/dataflow/v1beta3/jobs.rb +163 -23
- data/proto_docs/google/dataflow/v1beta3/metrics.rb +148 -3
- data/proto_docs/google/dataflow/v1beta3/streaming.rb +3 -0
- data/proto_docs/google/dataflow/v1beta3/templates.rb +183 -35
- data/proto_docs/google/protobuf/field_mask.rb +229 -0
- metadata +9 -14
@@ -53,6 +53,7 @@ module Google
|
|
53
53
|
end
|
54
54
|
|
55
55
|
# Describes the state of a metric.
|
56
|
+
# Next ID: 14
|
56
57
|
# @!attribute [rw] name
|
57
58
|
# @return [::Google::Cloud::Dataflow::V1beta3::MetricStructuredName]
|
58
59
|
# Name of the metric.
|
@@ -90,8 +91,12 @@ module Google
|
|
90
91
|
# @return [::Google::Protobuf::Value]
|
91
92
|
# Worker-computed aggregate value for the "Set" aggregation kind. The only
|
92
93
|
# possible value type is a list of Values whose type can be Long, Double,
|
93
|
-
#
|
94
|
-
# be of the same type.
|
94
|
+
# String, or BoundedTrie according to the metric's type. All Values in the
|
95
|
+
# list must be of the same type.
|
96
|
+
# @!attribute [rw] trie
|
97
|
+
# @return [::Google::Protobuf::Value]
|
98
|
+
# Worker-computed aggregate value for the "Trie" aggregation kind. The only
|
99
|
+
# possible value type is a BoundedTrieNode.
|
95
100
|
# @!attribute [rw] distribution
|
96
101
|
# @return [::Google::Protobuf::Value]
|
97
102
|
# A struct value describing properties of a distribution of numeric values.
|
@@ -137,7 +142,8 @@ module Google
|
|
137
142
|
|
138
143
|
# JobMetrics contains a collection of metrics describing the detailed progress
|
139
144
|
# of a Dataflow job. Metrics correspond to user-defined and system-defined
|
140
|
-
# metrics in the job.
|
145
|
+
# metrics in the job. For more information, see [Dataflow job metrics]
|
146
|
+
# (https://cloud.google.com/dataflow/docs/guides/using-monitoring-intf).
|
141
147
|
#
|
142
148
|
# This resource captures only the most recent values of each metric;
|
143
149
|
# time-series data can be queried for them (under the same metric names)
|
@@ -206,6 +212,139 @@ module Google
|
|
206
212
|
end
|
207
213
|
end
|
208
214
|
|
215
|
+
# Information useful for straggler identification and debugging.
|
216
|
+
# @!attribute [rw] start_time
|
217
|
+
# @return [::Google::Protobuf::Timestamp]
|
218
|
+
# The time when the work item attempt became a straggler.
|
219
|
+
# @!attribute [rw] causes
|
220
|
+
# @return [::Google::Protobuf::Map{::String => ::Google::Cloud::Dataflow::V1beta3::StragglerInfo::StragglerDebuggingInfo}]
|
221
|
+
# The straggler causes, keyed by the string representation of the
|
222
|
+
# StragglerCause enum and contains specialized debugging information for each
|
223
|
+
# straggler cause.
|
224
|
+
class StragglerInfo
|
225
|
+
include ::Google::Protobuf::MessageExts
|
226
|
+
extend ::Google::Protobuf::MessageExts::ClassMethods
|
227
|
+
|
228
|
+
# Information useful for debugging a straggler. Each type will provide
|
229
|
+
# specialized debugging information relevant for a particular cause.
|
230
|
+
# The StragglerDebuggingInfo will be 1:1 mapping to the StragglerCause enum.
|
231
|
+
# @!attribute [rw] hot_key
|
232
|
+
# @return [::Google::Cloud::Dataflow::V1beta3::HotKeyDebuggingInfo]
|
233
|
+
# Hot key debugging details.
|
234
|
+
class StragglerDebuggingInfo
|
235
|
+
include ::Google::Protobuf::MessageExts
|
236
|
+
extend ::Google::Protobuf::MessageExts::ClassMethods
|
237
|
+
end
|
238
|
+
|
239
|
+
# @!attribute [rw] key
|
240
|
+
# @return [::String]
|
241
|
+
# @!attribute [rw] value
|
242
|
+
# @return [::Google::Cloud::Dataflow::V1beta3::StragglerInfo::StragglerDebuggingInfo]
|
243
|
+
class CausesEntry
|
244
|
+
include ::Google::Protobuf::MessageExts
|
245
|
+
extend ::Google::Protobuf::MessageExts::ClassMethods
|
246
|
+
end
|
247
|
+
end
|
248
|
+
|
249
|
+
# Information useful for streaming straggler identification and debugging.
|
250
|
+
# @!attribute [rw] start_time
|
251
|
+
# @return [::Google::Protobuf::Timestamp]
|
252
|
+
# Start time of this straggler.
|
253
|
+
# @!attribute [rw] end_time
|
254
|
+
# @return [::Google::Protobuf::Timestamp]
|
255
|
+
# End time of this straggler.
|
256
|
+
# @!attribute [rw] worker_name
|
257
|
+
# @return [::String]
|
258
|
+
# Name of the worker where the straggler was detected.
|
259
|
+
# @!attribute [rw] data_watermark_lag
|
260
|
+
# @return [::Google::Protobuf::Duration]
|
261
|
+
# The event-time watermark lag at the time of the straggler detection.
|
262
|
+
# @!attribute [rw] system_watermark_lag
|
263
|
+
# @return [::Google::Protobuf::Duration]
|
264
|
+
# The system watermark lag at the time of the straggler detection.
|
265
|
+
class StreamingStragglerInfo
|
266
|
+
include ::Google::Protobuf::MessageExts
|
267
|
+
extend ::Google::Protobuf::MessageExts::ClassMethods
|
268
|
+
end
|
269
|
+
|
270
|
+
# Information for a straggler.
|
271
|
+
# @!attribute [rw] batch_straggler
|
272
|
+
# @return [::Google::Cloud::Dataflow::V1beta3::StragglerInfo]
|
273
|
+
# Batch straggler identification and debugging information.
|
274
|
+
#
|
275
|
+
# Note: The following fields are mutually exclusive: `batch_straggler`, `streaming_straggler`. If a field in that set is populated, all other fields in the set will automatically be cleared.
|
276
|
+
# @!attribute [rw] streaming_straggler
|
277
|
+
# @return [::Google::Cloud::Dataflow::V1beta3::StreamingStragglerInfo]
|
278
|
+
# Streaming straggler identification and debugging information.
|
279
|
+
#
|
280
|
+
# Note: The following fields are mutually exclusive: `streaming_straggler`, `batch_straggler`. If a field in that set is populated, all other fields in the set will automatically be cleared.
|
281
|
+
class Straggler
|
282
|
+
include ::Google::Protobuf::MessageExts
|
283
|
+
extend ::Google::Protobuf::MessageExts::ClassMethods
|
284
|
+
end
|
285
|
+
|
286
|
+
# Information useful for debugging a hot key detection.
|
287
|
+
# @!attribute [rw] detected_hot_keys
|
288
|
+
# @return [::Google::Protobuf::Map{::Integer => ::Google::Cloud::Dataflow::V1beta3::HotKeyDebuggingInfo::HotKeyInfo}]
|
289
|
+
# Debugging information for each detected hot key. Keyed by a hash of the
|
290
|
+
# key.
|
291
|
+
class HotKeyDebuggingInfo
|
292
|
+
include ::Google::Protobuf::MessageExts
|
293
|
+
extend ::Google::Protobuf::MessageExts::ClassMethods
|
294
|
+
|
295
|
+
# Information about a hot key.
|
296
|
+
# @!attribute [rw] hot_key_age
|
297
|
+
# @return [::Google::Protobuf::Duration]
|
298
|
+
# The age of the hot key measured from when it was first detected.
|
299
|
+
# @!attribute [rw] key
|
300
|
+
# @return [::String]
|
301
|
+
# A detected hot key that is causing limited parallelism. This field will
|
302
|
+
# be populated only if the following flag is set to true:
|
303
|
+
# "--enable_hot_key_logging".
|
304
|
+
# @!attribute [rw] key_truncated
|
305
|
+
# @return [::Boolean]
|
306
|
+
# If true, then the above key is truncated and cannot be deserialized. This
|
307
|
+
# occurs if the key above is populated and the key size is >5MB.
|
308
|
+
class HotKeyInfo
|
309
|
+
include ::Google::Protobuf::MessageExts
|
310
|
+
extend ::Google::Protobuf::MessageExts::ClassMethods
|
311
|
+
end
|
312
|
+
|
313
|
+
# @!attribute [rw] key
|
314
|
+
# @return [::Integer]
|
315
|
+
# @!attribute [rw] value
|
316
|
+
# @return [::Google::Cloud::Dataflow::V1beta3::HotKeyDebuggingInfo::HotKeyInfo]
|
317
|
+
class DetectedHotKeysEntry
|
318
|
+
include ::Google::Protobuf::MessageExts
|
319
|
+
extend ::Google::Protobuf::MessageExts::ClassMethods
|
320
|
+
end
|
321
|
+
end
|
322
|
+
|
323
|
+
# Summarized straggler identification details.
|
324
|
+
# @!attribute [rw] total_straggler_count
|
325
|
+
# @return [::Integer]
|
326
|
+
# The total count of stragglers.
|
327
|
+
# @!attribute [rw] straggler_cause_count
|
328
|
+
# @return [::Google::Protobuf::Map{::String => ::Integer}]
|
329
|
+
# Aggregated counts of straggler causes, keyed by the string representation
|
330
|
+
# of the StragglerCause enum.
|
331
|
+
# @!attribute [rw] recent_stragglers
|
332
|
+
# @return [::Array<::Google::Cloud::Dataflow::V1beta3::Straggler>]
|
333
|
+
# The most recent stragglers.
|
334
|
+
class StragglerSummary
|
335
|
+
include ::Google::Protobuf::MessageExts
|
336
|
+
extend ::Google::Protobuf::MessageExts::ClassMethods
|
337
|
+
|
338
|
+
# @!attribute [rw] key
|
339
|
+
# @return [::String]
|
340
|
+
# @!attribute [rw] value
|
341
|
+
# @return [::Integer]
|
342
|
+
class StragglerCauseCountEntry
|
343
|
+
include ::Google::Protobuf::MessageExts
|
344
|
+
extend ::Google::Protobuf::MessageExts::ClassMethods
|
345
|
+
end
|
346
|
+
end
|
347
|
+
|
209
348
|
# Information about a particular execution stage of a job.
|
210
349
|
# @!attribute [rw] stage_id
|
211
350
|
# @return [::String]
|
@@ -229,6 +368,9 @@ module Google
|
|
229
368
|
# @!attribute [rw] metrics
|
230
369
|
# @return [::Array<::Google::Cloud::Dataflow::V1beta3::MetricUpdate>]
|
231
370
|
# Metrics for this stage.
|
371
|
+
# @!attribute [rw] straggler_summary
|
372
|
+
# @return [::Google::Cloud::Dataflow::V1beta3::StragglerSummary]
|
373
|
+
# Straggler summary for this stage.
|
232
374
|
class StageSummary
|
233
375
|
include ::Google::Protobuf::MessageExts
|
234
376
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -310,6 +452,9 @@ module Google
|
|
310
452
|
# @!attribute [rw] metrics
|
311
453
|
# @return [::Array<::Google::Cloud::Dataflow::V1beta3::MetricUpdate>]
|
312
454
|
# Metrics for this work item.
|
455
|
+
# @!attribute [rw] straggler_info
|
456
|
+
# @return [::Google::Cloud::Dataflow::V1beta3::StragglerInfo]
|
457
|
+
# Information about straggler detections for this work item.
|
313
458
|
class WorkItemDetails
|
314
459
|
include ::Google::Protobuf::MessageExts
|
315
460
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -80,6 +80,9 @@ module Google
|
|
80
80
|
# @!attribute [rw] with_attributes
|
81
81
|
# @return [::Boolean]
|
82
82
|
# If true, then the client has requested to get pubsub attributes.
|
83
|
+
# @!attribute [rw] dynamic_destinations
|
84
|
+
# @return [::Boolean]
|
85
|
+
# If true, then this location represents dynamic topics.
|
83
86
|
class PubsubLocation
|
84
87
|
include ::Google::Protobuf::MessageExts
|
85
88
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -44,6 +44,15 @@ module Google
|
|
44
44
|
# @!attribute [rw] default_environment
|
45
45
|
# @return [::Google::Cloud::Dataflow::V1beta3::FlexTemplateRuntimeEnvironment]
|
46
46
|
# Default runtime environment for the job.
|
47
|
+
# @!attribute [rw] image_repository_username_secret_id
|
48
|
+
# @return [::String]
|
49
|
+
# Secret Manager secret id for username to authenticate to private registry.
|
50
|
+
# @!attribute [rw] image_repository_password_secret_id
|
51
|
+
# @return [::String]
|
52
|
+
# Secret Manager secret id for password to authenticate to private registry.
|
53
|
+
# @!attribute [rw] image_repository_cert_path
|
54
|
+
# @return [::String]
|
55
|
+
# Cloud Storage path to self-signed certificate of private registry.
|
47
56
|
class ContainerSpec
|
48
57
|
include ::Google::Protobuf::MessageExts
|
49
58
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -213,19 +222,33 @@ module Google
|
|
213
222
|
# The algorithm to use for autoscaling
|
214
223
|
# @!attribute [rw] dump_heap_on_oom
|
215
224
|
# @return [::Boolean]
|
216
|
-
# If true,
|
217
|
-
#
|
218
|
-
#
|
225
|
+
# If true, when processing time is spent almost entirely
|
226
|
+
# on garbage collection (GC), saves a heap dump before ending the thread
|
227
|
+
# or process. If false, ends the thread or process without saving a heap
|
228
|
+
# dump. Does not save a heap dump when the Java Virtual Machine (JVM) has an
|
229
|
+
# out of memory error during processing. The location of the heap file is
|
230
|
+
# either echoed back to the user, or the user is given the opportunity to
|
219
231
|
# download the heap file.
|
220
232
|
# @!attribute [rw] save_heap_dumps_to_gcs_path
|
221
233
|
# @return [::String]
|
222
|
-
# Cloud Storage bucket (directory) to upload heap dumps to
|
223
|
-
#
|
224
|
-
# (dump_heap_on_oom is set to true).
|
234
|
+
# Cloud Storage bucket (directory) to upload heap dumps to.
|
235
|
+
# Enabling this field implies that `dump_heap_on_oom` is set to true.
|
225
236
|
# @!attribute [rw] launcher_machine_type
|
226
237
|
# @return [::String]
|
227
238
|
# The machine type to use for launching the job. The default is
|
228
239
|
# n1-standard-1.
|
240
|
+
# @!attribute [rw] enable_launcher_vm_serial_port_logging
|
241
|
+
# @return [::Boolean]
|
242
|
+
# If true serial port logging will be enabled for the launcher VM.
|
243
|
+
# @!attribute [rw] streaming_mode
|
244
|
+
# @return [::Google::Cloud::Dataflow::V1beta3::StreamingMode]
|
245
|
+
# Optional. Specifies the Streaming Engine message processing guarantees.
|
246
|
+
# Reduces cost and latency but might result in duplicate messages committed
|
247
|
+
# to storage. Designed to run simple mapping streaming ETL jobs at the lowest
|
248
|
+
# cost. For example, Change Data Capture (CDC) to BigQuery is a canonical use
|
249
|
+
# case. For more information, see
|
250
|
+
# [Set the pipeline streaming
|
251
|
+
# mode](https://cloud.google.com/dataflow/docs/guides/streaming-modes).
|
229
252
|
class FlexTemplateRuntimeEnvironment
|
230
253
|
include ::Google::Protobuf::MessageExts
|
231
254
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -264,51 +287,53 @@ module Google
|
|
264
287
|
# The environment values to set at runtime.
|
265
288
|
# @!attribute [rw] num_workers
|
266
289
|
# @return [::Integer]
|
267
|
-
# The initial number of Google Compute Engine
|
290
|
+
# Optional. The initial number of Google Compute Engine instances for the
|
291
|
+
# job. The default value is 11.
|
268
292
|
# @!attribute [rw] max_workers
|
269
293
|
# @return [::Integer]
|
270
|
-
# The maximum number of Google Compute Engine instances to be made
|
271
|
-
# available to your pipeline during execution, from 1 to 1000.
|
294
|
+
# Optional. The maximum number of Google Compute Engine instances to be made
|
295
|
+
# available to your pipeline during execution, from 1 to 1000. The default
|
296
|
+
# value is 1.
|
272
297
|
# @!attribute [rw] zone
|
273
298
|
# @return [::String]
|
274
|
-
# The Compute Engine [availability
|
299
|
+
# Optional. The Compute Engine [availability
|
275
300
|
# zone](https://cloud.google.com/compute/docs/regions-zones/regions-zones)
|
276
301
|
# for launching worker instances to run your pipeline.
|
277
302
|
# In the future, worker_zone will take precedence.
|
278
303
|
# @!attribute [rw] service_account_email
|
279
304
|
# @return [::String]
|
280
|
-
# The email address of the service account to run the job as.
|
305
|
+
# Optional. The email address of the service account to run the job as.
|
281
306
|
# @!attribute [rw] temp_location
|
282
307
|
# @return [::String]
|
283
|
-
# The Cloud Storage path to use for temporary files.
|
308
|
+
# Required. The Cloud Storage path to use for temporary files.
|
284
309
|
# Must be a valid Cloud Storage URL, beginning with `gs://`.
|
285
310
|
# @!attribute [rw] bypass_temp_dir_validation
|
286
311
|
# @return [::Boolean]
|
287
|
-
# Whether to bypass the safety checks for the job's temporary
|
288
|
-
# Use with caution.
|
312
|
+
# Optional. Whether to bypass the safety checks for the job's temporary
|
313
|
+
# directory. Use with caution.
|
289
314
|
# @!attribute [rw] machine_type
|
290
315
|
# @return [::String]
|
291
|
-
# The machine type to use for the job. Defaults to the value from
|
292
|
-
# template if not specified.
|
316
|
+
# Optional. The machine type to use for the job. Defaults to the value from
|
317
|
+
# the template if not specified.
|
293
318
|
# @!attribute [rw] additional_experiments
|
294
319
|
# @return [::Array<::String>]
|
295
|
-
# Additional experiment flags for the job, specified with the
|
320
|
+
# Optional. Additional experiment flags for the job, specified with the
|
296
321
|
# `--experiments` option.
|
297
322
|
# @!attribute [rw] network
|
298
323
|
# @return [::String]
|
299
|
-
# Network to which VMs will be assigned. If empty or unspecified,
|
324
|
+
# Optional. Network to which VMs will be assigned. If empty or unspecified,
|
300
325
|
# the service will use the network "default".
|
301
326
|
# @!attribute [rw] subnetwork
|
302
327
|
# @return [::String]
|
303
|
-
# Subnetwork to which VMs will be assigned, if desired. You can
|
304
|
-
# subnetwork using either a complete URL or an abbreviated path.
|
305
|
-
#
|
328
|
+
# Optional. Subnetwork to which VMs will be assigned, if desired. You can
|
329
|
+
# specify a subnetwork using either a complete URL or an abbreviated path.
|
330
|
+
# Expected to be of the form
|
306
331
|
# "https://www.googleapis.com/compute/v1/projects/HOST_PROJECT_ID/regions/REGION/subnetworks/SUBNETWORK"
|
307
332
|
# or "regions/REGION/subnetworks/SUBNETWORK". If the subnetwork is located in
|
308
333
|
# a Shared VPC network, you must use the complete URL.
|
309
334
|
# @!attribute [rw] additional_user_labels
|
310
335
|
# @return [::Google::Protobuf::Map{::String => ::String}]
|
311
|
-
# Additional user labels to be specified for the job.
|
336
|
+
# Optional. Additional user labels to be specified for the job.
|
312
337
|
# Keys and values should follow the restrictions specified in the [labeling
|
313
338
|
# restrictions](https://cloud.google.com/compute/docs/labeling-resources#restrictions)
|
314
339
|
# page.
|
@@ -316,22 +341,22 @@ module Google
|
|
316
341
|
# Example: { "name": "wrench", "mass": "1kg", "count": "3" }.
|
317
342
|
# @!attribute [rw] kms_key_name
|
318
343
|
# @return [::String]
|
319
|
-
# Name for the Cloud KMS key for the job.
|
344
|
+
# Optional. Name for the Cloud KMS key for the job.
|
320
345
|
# Key format is:
|
321
346
|
# projects/<project>/locations/<location>/keyRings/<keyring>/cryptoKeys/<key>
|
322
347
|
# @!attribute [rw] ip_configuration
|
323
348
|
# @return [::Google::Cloud::Dataflow::V1beta3::WorkerIPAddressConfiguration]
|
324
|
-
# Configuration for VM IPs.
|
349
|
+
# Optional. Configuration for VM IPs.
|
325
350
|
# @!attribute [rw] worker_region
|
326
351
|
# @return [::String]
|
327
|
-
# The Compute Engine region
|
352
|
+
# Required. The Compute Engine region
|
328
353
|
# (https://cloud.google.com/compute/docs/regions-zones/regions-zones) in
|
329
354
|
# which worker processing should occur, e.g. "us-west1". Mutually exclusive
|
330
355
|
# with worker_zone. If neither worker_region nor worker_zone is specified,
|
331
356
|
# default to the control plane's region.
|
332
357
|
# @!attribute [rw] worker_zone
|
333
358
|
# @return [::String]
|
334
|
-
# The Compute Engine zone
|
359
|
+
# Optional. The Compute Engine zone
|
335
360
|
# (https://cloud.google.com/compute/docs/regions-zones/regions-zones) in
|
336
361
|
# which worker processing should occur, e.g. "us-west1-a". Mutually exclusive
|
337
362
|
# with worker_region. If neither worker_region nor worker_zone is specified,
|
@@ -339,7 +364,20 @@ module Google
|
|
339
364
|
# If both `worker_zone` and `zone` are set, `worker_zone` takes precedence.
|
340
365
|
# @!attribute [rw] enable_streaming_engine
|
341
366
|
# @return [::Boolean]
|
342
|
-
# Whether to enable Streaming Engine for the job.
|
367
|
+
# Optional. Whether to enable Streaming Engine for the job.
|
368
|
+
# @!attribute [rw] disk_size_gb
|
369
|
+
# @return [::Integer]
|
370
|
+
# Optional. The disk size, in gigabytes, to use on each remote Compute Engine
|
371
|
+
# worker instance.
|
372
|
+
# @!attribute [rw] streaming_mode
|
373
|
+
# @return [::Google::Cloud::Dataflow::V1beta3::StreamingMode]
|
374
|
+
# Optional. Specifies the Streaming Engine message processing guarantees.
|
375
|
+
# Reduces cost and latency but might result in duplicate messages committed
|
376
|
+
# to storage. Designed to run simple mapping streaming ETL jobs at the lowest
|
377
|
+
# cost. For example, Change Data Capture (CDC) to BigQuery is a canonical use
|
378
|
+
# case. For more information, see
|
379
|
+
# [Set the pipeline streaming
|
380
|
+
# mode](https://cloud.google.com/dataflow/docs/guides/streaming-modes).
|
343
381
|
class RuntimeEnvironment
|
344
382
|
include ::Google::Protobuf::MessageExts
|
345
383
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -354,6 +392,21 @@ module Google
|
|
354
392
|
end
|
355
393
|
end
|
356
394
|
|
395
|
+
# ParameterMetadataEnumOption specifies the option shown in the enum form.
|
396
|
+
# @!attribute [rw] value
|
397
|
+
# @return [::String]
|
398
|
+
# Required. The value of the enum option.
|
399
|
+
# @!attribute [rw] label
|
400
|
+
# @return [::String]
|
401
|
+
# Optional. The label to display for the enum option.
|
402
|
+
# @!attribute [rw] description
|
403
|
+
# @return [::String]
|
404
|
+
# Optional. The description to display for the enum option.
|
405
|
+
class ParameterMetadataEnumOption
|
406
|
+
include ::Google::Protobuf::MessageExts
|
407
|
+
extend ::Google::Protobuf::MessageExts::ClassMethods
|
408
|
+
end
|
409
|
+
|
357
410
|
# Metadata for a specific parameter.
|
358
411
|
# @!attribute [rw] name
|
359
412
|
# @return [::String]
|
@@ -377,6 +430,35 @@ module Google
|
|
377
430
|
# @!attribute [rw] custom_metadata
|
378
431
|
# @return [::Google::Protobuf::Map{::String => ::String}]
|
379
432
|
# Optional. Additional metadata for describing this parameter.
|
433
|
+
# @!attribute [rw] group_name
|
434
|
+
# @return [::String]
|
435
|
+
# Optional. Specifies a group name for this parameter to be rendered under.
|
436
|
+
# Group header text will be rendered exactly as specified in this field. Only
|
437
|
+
# considered when parent_name is NOT provided.
|
438
|
+
# @!attribute [rw] parent_name
|
439
|
+
# @return [::String]
|
440
|
+
# Optional. Specifies the name of the parent parameter. Used in conjunction
|
441
|
+
# with 'parent_trigger_values' to make this parameter conditional (will only
|
442
|
+
# be rendered conditionally). Should be mappable to a ParameterMetadata.name
|
443
|
+
# field.
|
444
|
+
# @!attribute [rw] parent_trigger_values
|
445
|
+
# @return [::Array<::String>]
|
446
|
+
# Optional. The value(s) of the 'parent_name' parameter which will trigger
|
447
|
+
# this parameter to be shown. If left empty, ANY non-empty value in
|
448
|
+
# parent_name will trigger this parameter to be shown. Only considered when
|
449
|
+
# this parameter is conditional (when 'parent_name' has been provided).
|
450
|
+
# @!attribute [rw] enum_options
|
451
|
+
# @return [::Array<::Google::Cloud::Dataflow::V1beta3::ParameterMetadataEnumOption>]
|
452
|
+
# Optional. The options shown when ENUM ParameterType is specified.
|
453
|
+
# @!attribute [rw] default_value
|
454
|
+
# @return [::String]
|
455
|
+
# Optional. The default values will pre-populate the parameter with the
|
456
|
+
# given value from the proto. If default_value is left empty, the parameter
|
457
|
+
# will be populated with a default of the relevant type, e.g. false for a
|
458
|
+
# boolean.
|
459
|
+
# @!attribute [rw] hidden_ui
|
460
|
+
# @return [::Boolean]
|
461
|
+
# Optional. Whether the parameter should be hidden in the UI.
|
380
462
|
class ParameterMetadata
|
381
463
|
include ::Google::Protobuf::MessageExts
|
382
464
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -401,6 +483,20 @@ module Google
|
|
401
483
|
# @!attribute [rw] parameters
|
402
484
|
# @return [::Array<::Google::Cloud::Dataflow::V1beta3::ParameterMetadata>]
|
403
485
|
# The parameters for the template.
|
486
|
+
# @!attribute [rw] streaming
|
487
|
+
# @return [::Boolean]
|
488
|
+
# Optional. Indicates if the template is streaming or not.
|
489
|
+
# @!attribute [rw] supports_at_least_once
|
490
|
+
# @return [::Boolean]
|
491
|
+
# Optional. Indicates if the streaming template supports at least once mode.
|
492
|
+
# @!attribute [rw] supports_exactly_once
|
493
|
+
# @return [::Boolean]
|
494
|
+
# Optional. Indicates if the streaming template supports exactly once mode.
|
495
|
+
# @!attribute [rw] default_streaming_mode
|
496
|
+
# @return [::String]
|
497
|
+
# Optional. Indicates the default streaming mode for a streaming template.
|
498
|
+
# Only valid if both supports_at_least_once and supports_exactly_once are
|
499
|
+
# true. Possible values: UNSPECIFIED, EXACTLY_ONCE and AT_LEAST_ONCE
|
404
500
|
class TemplateMetadata
|
405
501
|
include ::Google::Protobuf::MessageExts
|
406
502
|
extend ::Google::Protobuf::MessageExts::ClassMethods
|
@@ -427,6 +523,9 @@ module Google
|
|
427
523
|
|
428
524
|
# Python.
|
429
525
|
PYTHON = 2
|
526
|
+
|
527
|
+
# Go.
|
528
|
+
GO = 3
|
430
529
|
end
|
431
530
|
end
|
432
531
|
|
@@ -540,10 +639,16 @@ module Google
|
|
540
639
|
end
|
541
640
|
end
|
542
641
|
|
543
|
-
# Parameters to provide to the template being launched.
|
642
|
+
# Parameters to provide to the template being launched. Note that the
|
643
|
+
# [metadata in the pipeline code]
|
644
|
+
# (https://cloud.google.com/dataflow/docs/guides/templates/creating-templates#metadata)
|
645
|
+
# determines which runtime parameters are valid.
|
544
646
|
# @!attribute [rw] job_name
|
545
647
|
# @return [::String]
|
546
648
|
# Required. The job name to use for the created job.
|
649
|
+
#
|
650
|
+
# The name must match the regular expression
|
651
|
+
# `[a-z]([-a-z0-9]{0,1022}[a-z0-9])?`
|
547
652
|
# @!attribute [rw] parameters
|
548
653
|
# @return [::Google::Protobuf::Map{::String => ::String}]
|
549
654
|
# The runtime parameters to pass to the job.
|
@@ -591,19 +696,19 @@ module Google
|
|
591
696
|
# Defaults to false.
|
592
697
|
# @!attribute [rw] gcs_path
|
593
698
|
# @return [::String]
|
594
|
-
# A Cloud Storage path to the template
|
699
|
+
# A Cloud Storage path to the template to use to create
|
595
700
|
# the job.
|
596
|
-
# Must be valid Cloud Storage URL, beginning with
|
701
|
+
# Must be valid Cloud Storage URL, beginning with `gs://`.
|
597
702
|
#
|
598
703
|
# Note: The following fields are mutually exclusive: `gcs_path`, `dynamic_template`. If a field in that set is populated, all other fields in the set will automatically be cleared.
|
599
704
|
# @!attribute [rw] dynamic_template
|
600
705
|
# @return [::Google::Cloud::Dataflow::V1beta3::DynamicTemplateLaunchParams]
|
601
|
-
#
|
706
|
+
# Parameters for launching a dynamic template.
|
602
707
|
#
|
603
708
|
# Note: The following fields are mutually exclusive: `dynamic_template`, `gcs_path`. If a field in that set is populated, all other fields in the set will automatically be cleared.
|
604
709
|
# @!attribute [rw] launch_parameters
|
605
710
|
# @return [::Google::Cloud::Dataflow::V1beta3::LaunchTemplateParameters]
|
606
|
-
# The parameters of the template to launch.
|
711
|
+
# The parameters of the template to launch. Part of the
|
607
712
|
# body of the POST request.
|
608
713
|
# @!attribute [rw] location
|
609
714
|
# @return [::String]
|
@@ -647,11 +752,11 @@ module Google
|
|
647
752
|
end
|
648
753
|
end
|
649
754
|
|
650
|
-
#
|
755
|
+
# Parameters to pass when launching a dynamic template.
|
651
756
|
# @!attribute [rw] gcs_path
|
652
757
|
# @return [::String]
|
653
|
-
# Path to dynamic template
|
654
|
-
# The file must be a
|
758
|
+
# Path to the dynamic template specification file on Cloud Storage.
|
759
|
+
# The file must be a JSON serialized `DynamicTemplateFileSpec` object.
|
655
760
|
# @!attribute [rw] staging_location
|
656
761
|
# @return [::String]
|
657
762
|
# Cloud Storage path for staging dependencies.
|
@@ -692,6 +797,49 @@ module Google
|
|
692
797
|
|
693
798
|
# The parameter specifies a Pub/Sub Subscription.
|
694
799
|
PUBSUB_SUBSCRIPTION = 9
|
800
|
+
|
801
|
+
# The parameter specifies a BigQuery table.
|
802
|
+
BIGQUERY_TABLE = 10
|
803
|
+
|
804
|
+
# The parameter specifies a JavaScript UDF in Cloud Storage.
|
805
|
+
JAVASCRIPT_UDF_FILE = 11
|
806
|
+
|
807
|
+
# The parameter specifies a Service Account email.
|
808
|
+
SERVICE_ACCOUNT = 12
|
809
|
+
|
810
|
+
# The parameter specifies a Machine Type.
|
811
|
+
MACHINE_TYPE = 13
|
812
|
+
|
813
|
+
# The parameter specifies a KMS Key name.
|
814
|
+
KMS_KEY_NAME = 14
|
815
|
+
|
816
|
+
# The parameter specifies a Worker Region.
|
817
|
+
WORKER_REGION = 15
|
818
|
+
|
819
|
+
# The parameter specifies a Worker Zone.
|
820
|
+
WORKER_ZONE = 16
|
821
|
+
|
822
|
+
# The parameter specifies a boolean input.
|
823
|
+
BOOLEAN = 17
|
824
|
+
|
825
|
+
# The parameter specifies an enum input.
|
826
|
+
ENUM = 18
|
827
|
+
|
828
|
+
# The parameter specifies a number input.
|
829
|
+
NUMBER = 19
|
830
|
+
|
831
|
+
# Deprecated. Please use KAFKA_READ_TOPIC instead.
|
832
|
+
KAFKA_TOPIC = 20
|
833
|
+
|
834
|
+
# The parameter specifies the fully-qualified name of an Apache Kafka topic.
|
835
|
+
# This can be either a Google Managed Kafka topic or a non-managed Kafka
|
836
|
+
# topic.
|
837
|
+
KAFKA_READ_TOPIC = 21
|
838
|
+
|
839
|
+
# The parameter specifies the fully-qualified name of an Apache Kafka topic.
|
840
|
+
# This can be an existing Google Managed Kafka topic, the name for a new
|
841
|
+
# Google Managed Kafka topic, or an existing non-managed Kafka topic.
|
842
|
+
KAFKA_WRITE_TOPIC = 22
|
695
843
|
end
|
696
844
|
end
|
697
845
|
end
|