google-cloud-dataflow-v1beta3 0.12.0 → 0.13.0

data/proto_docs/google/dataflow/v1beta3/environment.rb

@@ -50,13 +50,14 @@ module Google
         #     field for service related experiments is service_options.
         # @!attribute [rw] service_options
         #   @return [::Array<::String>]
-        #     The list of service options to enable. This field should be used for
-        #     service related experiments only. These experiments, when graduating to GA,
-        #     should be replaced by dedicated fields or become default (i.e. always on).
+        #     Optional. The list of service options to enable. This field should be used
+        #     for service related experiments only. These experiments, when graduating to
+        #     GA, should be replaced by dedicated fields or become default (i.e. always
+        #     on).
         # @!attribute [rw] service_kms_key_name
         #   @return [::String]
-        #     If set, contains the Cloud KMS key identifier used to encrypt data
-        #     at rest, AKA a Customer Managed Encryption Key (CMEK).
+        #     Optional. If set, contains the Cloud KMS key identifier used to encrypt
+        #     data at rest, AKA a Customer Managed Encryption Key (CMEK).
         #
         #     Format:
         #       projects/PROJECT_ID/locations/LOCATION/keyRings/KEY_RING/cryptoKeys/KEY
@@ -73,7 +74,7 @@ module Google
         #     are required in order to run the job.
         # @!attribute [rw] dataset
         #   @return [::String]
-        #     The dataset for the current project where various workflow
+        #     Optional. The dataset for the current project where various workflow
         #     related tables are stored.
         #
         #     The supported resource type is:
@@ -91,20 +92,21 @@ module Google
         #     Experimental settings.
         # @!attribute [rw] service_account_email
         #   @return [::String]
-        #     Identity to run virtual machines as. Defaults to the default account.
+        #     Optional. Identity to run virtual machines as. Defaults to the default
+        #     account.
         # @!attribute [rw] flex_resource_scheduling_goal
         #   @return [::Google::Cloud::Dataflow::V1beta3::FlexResourceSchedulingGoal]
-        #     Which Flexible Resource Scheduling mode to run in.
+        #     Optional. Which Flexible Resource Scheduling mode to run in.
         # @!attribute [rw] worker_region
         #   @return [::String]
-        #     The Compute Engine region
+        #     Optional. The Compute Engine region
         #     (https://cloud.google.com/compute/docs/regions-zones/regions-zones) in
         #     which worker processing should occur, e.g. "us-west1". Mutually exclusive
         #     with worker_zone. If neither worker_region nor worker_zone is specified,
         #     default to the control plane's region.
         # @!attribute [rw] worker_zone
         #   @return [::String]
-        #     The Compute Engine zone
+        #     Optional. The Compute Engine zone
         #     (https://cloud.google.com/compute/docs/regions-zones/regions-zones) in
         #     which worker processing should occur, e.g. "us-west1-a". Mutually exclusive
         #     with worker_region. If neither worker_region nor worker_zone is specified,
@@ -114,7 +116,20 @@ module Google
         #     Output only. The shuffle mode used for the job.
         # @!attribute [rw] debug_options
         #   @return [::Google::Cloud::Dataflow::V1beta3::DebugOptions]
-        #     Any debugging options to be supplied to the job.
+        #     Optional. Any debugging options to be supplied to the job.
+        # @!attribute [r] use_streaming_engine_resource_based_billing
+        #   @return [::Boolean]
+        #     Output only. Whether the job uses the Streaming Engine resource-based
+        #     billing model.
+        # @!attribute [rw] streaming_mode
+        #   @return [::Google::Cloud::Dataflow::V1beta3::StreamingMode]
+        #     Optional. Specifies the Streaming Engine message processing guarantees.
+        #     Reduces cost and latency but might result in duplicate messages committed
+        #     to storage. Designed to run simple mapping streaming ETL jobs at the lowest
+        #     cost. For example, Change Data Capture (CDC) to BigQuery is a canonical use
+        #     case. For more information, see
+        #     [Set the pipeline streaming
+        #     mode](https://cloud.google.com/dataflow/docs/guides/streaming-modes).
         class Environment
           include ::Google::Protobuf::MessageExts
           extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -321,7 +336,7 @@ module Google
           extend ::Google::Protobuf::MessageExts::ClassMethods
         end
 
-        # Defines a SDK harness container for executing Dataflow pipelines.
+        # Defines an SDK harness container for executing Dataflow pipelines.
         # @!attribute [rw] container_image
         #   @return [::String]
         #     A docker container image that resides in Google Container Registry.
@@ -339,7 +354,7 @@ module Google
         # @!attribute [rw] capabilities
         #   @return [::Array<::String>]
         #     The set of capabilities enumerated in the above Environment proto. See also
-        #     https://github.com/apache/beam/blob/master/model/pipeline/src/main/proto/beam_runner_api.proto
+        #     [beam_runner_api.proto](https://github.com/apache/beam/blob/master/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/beam_runner_api.proto)
         class SdkHarnessContainerImage
           include ::Google::Protobuf::MessageExts
           extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -467,11 +482,50 @@ module Google
           end
         end
 
+        # Configuration options for sampling elements.
+        # @!attribute [rw] behaviors
+        #   @return [::Array<::Google::Cloud::Dataflow::V1beta3::DataSamplingConfig::DataSamplingBehavior>]
+        #     List of given sampling behaviors to enable. For example, specifying
+        #     behaviors = [ALWAYS_ON] samples in-flight elements but does not sample
+        #     exceptions. Can be used to specify multiple behaviors like,
+        #     behaviors = [ALWAYS_ON, EXCEPTIONS] for specifying periodic sampling and
+        #     exception sampling.
+        #
+        #     If DISABLED is in the list, then sampling will be disabled and ignore the
+        #     other given behaviors.
+        #
+        #     Ordering does not matter.
+        class DataSamplingConfig
+          include ::Google::Protobuf::MessageExts
+          extend ::Google::Protobuf::MessageExts::ClassMethods
+
+          # The following enum defines what to sample for a running job.
+          module DataSamplingBehavior
+            # If given, has no effect on sampling behavior. Used as an unknown or unset
+            # sentinel value.
+            DATA_SAMPLING_BEHAVIOR_UNSPECIFIED = 0
+
+            # When given, disables element sampling. Has same behavior as not setting
+            # the behavior.
+            DISABLED = 1
+
+            # When given, enables sampling in-flight from all PCollections.
+            ALWAYS_ON = 2
+
+            # When given, enables sampling input elements when a user-defined DoFn
+            # causes an exception.
+            EXCEPTIONS = 3
+          end
+        end
+
         # Describes any options that have an effect on the debugging of pipelines.
         # @!attribute [rw] enable_hot_key_logging
         #   @return [::Boolean]
-        #     When true, enables the logging of the literal hot key to the user's Cloud
-        #     Logging.
+        #     Optional. When true, enables the logging of the literal hot key to the
+        #     user's Cloud Logging.
+        # @!attribute [rw] data_sampling
+        #   @return [::Google::Cloud::Dataflow::V1beta3::DataSamplingConfig]
+        #     Configuration options for sampling elements from a running pipeline.
         class DebugOptions
           include ::Google::Protobuf::MessageExts
           extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -555,7 +609,10 @@ module Google
           AUTOSCALING_ALGORITHM_BASIC = 2
         end
 
-        # Specifies how IP addresses should be allocated to the worker machines.
+        # Specifies how to allocate IP addresses to worker machines. You can also use
+        # [pipeline
+        # options](https://cloud.google.com/dataflow/docs/reference/pipeline-options#security_and_networking)
+        # to specify whether Dataflow workers use external IP addresses.
         module WorkerIPAddressConfiguration
           # The configuration is unknown, or unspecified.
           WORKER_IP_UNSPECIFIED = 0
@@ -581,6 +638,29 @@ module Google
           # Shuffle is done on the service side.
           SERVICE_BASED = 2
         end
+
+        # Specifies the Streaming Engine message processing guarantees. Reduces cost
+        # and latency but might result in duplicate messages written to storage.
+        # Designed to run simple mapping streaming ETL jobs at the lowest cost.
+        # For example, Change Data Capture (CDC) to BigQuery is a canonical use
+        # case. For more information, see
+        # [Set the pipeline streaming
+        # mode](https://cloud.google.com/dataflow/docs/guides/streaming-modes).
+        module StreamingMode
+          # Run in the default mode.
+          STREAMING_MODE_UNSPECIFIED = 0
+
+          # In this mode, message deduplication is performed against persistent state
+          # to make sure each message is processed and committed to storage exactly
+          # once.
+          STREAMING_MODE_EXACTLY_ONCE = 1
+
+          # Message deduplication is not performed. Messages might be processed
+          # multiple times, and the results are applied multiple times.
+          # Note: Setting this value also enables Streaming Engine and
+          # Streaming Engine resource-based billing.
+          STREAMING_MODE_AT_LEAST_ONCE = 2
+        end
       end
     end
   end

data/proto_docs/google/dataflow/v1beta3/jobs.rb

@@ -21,33 +21,36 @@ module Google
   module Cloud
     module Dataflow
       module V1beta3
-        # Defines a job to be run by the Cloud Dataflow service.
+        # Defines a job to be run by the Cloud Dataflow service. Do not enter
+        # confidential information when you supply string values using the API.
         # @!attribute [rw] id
         #   @return [::String]
         #     The unique ID of this job.
         #
-        #     This field is set by the Cloud Dataflow service when the Job is
+        #     This field is set by the Dataflow service when the job is
         #     created, and is immutable for the life of the job.
         # @!attribute [rw] project_id
         #   @return [::String]
-        #     The ID of the Cloud Platform project that the job belongs to.
+        #     The ID of the Google Cloud project that the job belongs to.
         # @!attribute [rw] name
         #   @return [::String]
-        #     The user-specified Cloud Dataflow job name.
+        #     Optional. The user-specified Dataflow job name.
         #
-        #     Only one Job with a given name may exist in a project at any
-        #     given time. If a caller attempts to create a Job with the same
-        #     name as an already-existing Job, the attempt returns the
-        #     existing Job.
+        #     Only one active job with a given name can exist in a project within one
+        #     region at
+        #     any given time. Jobs in different regions can have the same name.
+        #     If a caller attempts to create a job with the same
+        #     name as an active job that already exists, the attempt returns the
+        #     existing job.
         #
         #     The name must match the regular expression
         #     `[a-z]([-a-z0-9]{0,1022}[a-z0-9])?`
         # @!attribute [rw] type
         #   @return [::Google::Cloud::Dataflow::V1beta3::JobType]
-        #     The type of Cloud Dataflow job.
+        #     Optional. The type of Dataflow job.
         # @!attribute [rw] environment
         #   @return [::Google::Cloud::Dataflow::V1beta3::Environment]
-        #     The environment for the job.
+        #     Optional. The environment for the job.
         # @!attribute [rw] steps
         #   @return [::Array<::Google::Cloud::Dataflow::V1beta3::Step>]
         #     Exactly one of step or steps_location should be specified.
@@ -68,20 +71,23 @@ module Google
         #     terminal state. After a job has reached a terminal state, no
         #     further state updates may be made.
         #
-        #     This field may be mutated by the Cloud Dataflow service;
+        #     This field might be mutated by the Dataflow service;
         #     callers cannot mutate it.
         # @!attribute [rw] current_state_time
         #   @return [::Google::Protobuf::Timestamp]
         #     The timestamp associated with the current state.
         # @!attribute [rw] requested_state
         #   @return [::Google::Cloud::Dataflow::V1beta3::JobState]
-        #     The job's requested state.
+        #     The job's requested state. Applies to `UpdateJob` requests.
+        #
+        #     Set `requested_state` with `UpdateJob` requests to switch between the
+        #     states `JOB_STATE_STOPPED` and `JOB_STATE_RUNNING`. You can also use
+        #     `UpdateJob` requests to change a job's
+        #     state from `JOB_STATE_RUNNING` to `JOB_STATE_CANCELLED`,
+        #     `JOB_STATE_DONE`,  or `JOB_STATE_DRAINED`. These states irrevocably
+        #     terminate the job if it hasn't already reached a terminal state.
         #
-        #     `UpdateJob` may be used to switch between the `JOB_STATE_STOPPED` and
-        #     `JOB_STATE_RUNNING` states, by setting requested_state.  `UpdateJob` may
-        #     also be used to directly set a job's requested state to
-        #     `JOB_STATE_CANCELLED` or `JOB_STATE_DONE`, irrevocably terminating the
-        #     job if it has not already reached a terminal state.
+        #     This field has no effect on `CreateJob` requests.
         # @!attribute [rw] execution_info
         #   @return [::Google::Cloud::Dataflow::V1beta3::JobExecutionInfo]
         #     Deprecated.
@@ -99,8 +105,8 @@ module Google
         #     transferred to this job.
         # @!attribute [rw] transform_name_mapping
         #   @return [::Google::Protobuf::Map{::String => ::String}]
-        #     The map of transform name prefixes of the job to be replaced to the
-        #     corresponding name prefixes of the new job.
+        #     Optional. The map of transform name prefixes of the job to be replaced to
+        #     the corresponding name prefixes of the new job.
         # @!attribute [rw] client_request_id
         #   @return [::String]
         #     The client's unique identifier of the job, re-used across retried attempts.
@@ -141,7 +147,7 @@ module Google
         #     size.
         # @!attribute [rw] location
         #   @return [::String]
-        #     The [regional endpoint]
+        #     Optional. The [regional endpoint]
         #     (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) that
         #     contains this job.
         # @!attribute [rw] pipeline_description
@@ -175,6 +181,18 @@ module Google
         #   @return [::Boolean]
         #     Reserved for future use. This field is set only in responses from the
         #     server; it is ignored if it is set in any requests.
+        # @!attribute [rw] runtime_updatable_params
+        #   @return [::Google::Cloud::Dataflow::V1beta3::RuntimeUpdatableParams]
+        #     This field may ONLY be modified at runtime using the projects.jobs.update
+        #     method to adjust job behavior. This field has no effect when specified at
+        #     job creation.
+        # @!attribute [r] satisfies_pzi
+        #   @return [::Boolean]
+        #     Output only. Reserved for future use. This field is set only in responses
+        #     from the server; it is ignored if it is set in any requests.
+        # @!attribute [r] service_resources
+        #   @return [::Google::Cloud::Dataflow::V1beta3::ServiceResources]
+        #     Output only. Resources used by the Dataflow Service to run the job.
         class Job
           include ::Google::Protobuf::MessageExts
           extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -198,6 +216,40 @@ module Google
           end
         end
 
+        # Resources used by the Dataflow Service to run the job.
+        # @!attribute [r] zones
+        #   @return [::Array<::String>]
+        #     Output only. List of Cloud Zones being used by the Dataflow Service for
+        #     this job. Example: us-central1-c
+        class ServiceResources
+          include ::Google::Protobuf::MessageExts
+          extend ::Google::Protobuf::MessageExts::ClassMethods
+        end
+
+        # Additional job parameters that can only be updated during runtime using the
+        # projects.jobs.update method. These fields have no effect when specified
+        # during job creation.
+        # @!attribute [rw] max_num_workers
+        #   @return [::Integer]
+        #     The maximum number of workers to cap autoscaling at. This field is
+        #     currently only supported for Streaming Engine jobs.
+        # @!attribute [rw] min_num_workers
+        #   @return [::Integer]
+        #     The minimum number of workers to scale down to. This field is currently
+        #     only supported for Streaming Engine jobs.
+        # @!attribute [rw] worker_utilization_hint
+        #   @return [::Float]
+        #     Target worker utilization, compared against the aggregate utilization of
+        #     the worker pool by autoscaler, to determine upscaling and downscaling when
+        #     absent other constraints such as backlog.
+        #     For more information, see
+        #     [Update an existing
+        #     pipeline](https://cloud.google.com/dataflow/docs/guides/updating-a-pipeline).
+        class RuntimeUpdatableParams
+          include ::Google::Protobuf::MessageExts
+          extend ::Google::Protobuf::MessageExts::ClassMethods
+        end
+
         # Metadata for a Datastore connector used by the job.
         # @!attribute [rw] namespace
         #   @return [::String]
@@ -289,6 +341,9 @@ module Google
         # @!attribute [rw] sdk_support_status
         #   @return [::Google::Cloud::Dataflow::V1beta3::SdkVersion::SdkSupportStatus]
         #     The support status for this SDK version.
+        # @!attribute [r] bugs
+        #   @return [::Array<::Google::Cloud::Dataflow::V1beta3::SdkBug>]
+        #     Output only. Known bugs found in this SDK version.
         class SdkVersion
           include ::Google::Protobuf::MessageExts
           extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -313,6 +368,56 @@ module Google
           end
         end
 
+        # A bug found in the Dataflow SDK.
+        # @!attribute [r] type
+        #   @return [::Google::Cloud::Dataflow::V1beta3::SdkBug::Type]
+        #     Output only. Describes the impact of this SDK bug.
+        # @!attribute [r] severity
+        #   @return [::Google::Cloud::Dataflow::V1beta3::SdkBug::Severity]
+        #     Output only. How severe the SDK bug is.
+        # @!attribute [r] uri
+        #   @return [::String]
+        #     Output only. Link to more information on the bug.
+        class SdkBug
+          include ::Google::Protobuf::MessageExts
+          extend ::Google::Protobuf::MessageExts::ClassMethods
+
+          # Nature of the issue, ordered from least severe to most. Other bug types may
+          # be added to this list in the future.
+          module Type
+            # Unknown issue with this SDK.
+            TYPE_UNSPECIFIED = 0
+
+            # Catch-all for SDK bugs that don't fit in the below categories.
+            GENERAL = 1
+
+            # Using this version of the SDK may result in degraded performance.
+            PERFORMANCE = 2
+
+            # Using this version of the SDK may cause data loss.
+            DATALOSS = 3
+          end
+
+          # Indicates the severity of the bug. Other severities may be added to this
+          # list in the future.
+          module Severity
+            # A bug of unknown severity.
+            SEVERITY_UNSPECIFIED = 0
+
+            # A minor bug that that may reduce reliability or performance for some
+            # jobs. Impact will be minimal or non-existent for most jobs.
+            NOTICE = 1
+
+            # A bug that has some likelihood of causing performance degradation, data
+            # loss, or job failures.
+            WARNING = 2
+
+            # A bug with extremely significant impact. Jobs may fail erroneously,
+            # performance may be severely degraded, and data loss may be very likely.
+            SEVERE = 3
+          end
+        end
+
         # Metadata available primarily for filtering jobs. Will be included in the
         # ListJob response and Job SUMMARY view.
         # @!attribute [rw] sdk_version
@@ -336,9 +441,21 @@ module Google
         # @!attribute [rw] datastore_details
         #   @return [::Array<::Google::Cloud::Dataflow::V1beta3::DatastoreIODetails>]
         #     Identification of a Datastore source used in the Dataflow job.
+        # @!attribute [rw] user_display_properties
+        #   @return [::Google::Protobuf::Map{::String => ::String}]
+        #     List of display properties to help UI filter jobs.
         class JobMetadata
           include ::Google::Protobuf::MessageExts
           extend ::Google::Protobuf::MessageExts::ClassMethods
+
+          # @!attribute [rw] key
+          #   @return [::String]
+          # @!attribute [rw] value
+          #   @return [::String]
+          class UserDisplayPropertiesEntry
+            include ::Google::Protobuf::MessageExts
+            extend ::Google::Protobuf::MessageExts::ClassMethods
+          end
         end
 
         # A message describing the state of a particular execution stage.
@@ -368,6 +485,9 @@ module Google
         # @!attribute [rw] display_data
         #   @return [::Array<::Google::Cloud::Dataflow::V1beta3::DisplayData>]
         #     Pipeline level display data.
+        # @!attribute [rw] step_names_hash
+        #   @return [::String]
+        #     A hash value of the submitted pipeline portable graph step names if exists.
         class PipelineDescription
           include ::Google::Protobuf::MessageExts
           extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -553,6 +673,8 @@ module Google
         # specific operation as part of the overall job.  Data is typically
         # passed from one step to another as part of the job.
         #
+        # **Note:** The properties of this object are not stable and might change.
+        #
         # Here's an example of a sequence of steps which together implement a
         # Map-Reduce job:
         #
@@ -609,7 +731,8 @@ module Google
         end
 
         # Contains information about how a particular
-        # {::Google::Cloud::Dataflow::V1beta3::Step google.dataflow.v1beta3.Step} will be executed.
+        # {::Google::Cloud::Dataflow::V1beta3::Step google.dataflow.v1beta3.Step} will be
+        # executed.
         # @!attribute [rw] step_name
         #   @return [::Array<::String>]
         #     The steps associated with the execution stage.
@@ -679,6 +802,14 @@ module Google
         #     The [regional endpoint]
         #     (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) that
         #     contains this job.
+        # @!attribute [rw] update_mask
+        #   @return [::Google::Protobuf::FieldMask]
+        #     The list of fields to update relative to Job. If empty, only
+        #     RequestedJobState will be considered for update. If the FieldMask is not
+        #     empty and RequestedJobState is none/empty, The fields specified in the
+        #     update mask will be the only ones considered for update. If both
+        #     RequestedJobState and update_mask are specified, an error will be returned
+        #     as we cannot update both state and mask.
         class UpdateJobRequest
           include ::Google::Protobuf::MessageExts
           extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -710,6 +841,9 @@ module Google
         #     The [regional endpoint]
         #     (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) that
         #     contains this job.
+        # @!attribute [rw] name
+        #   @return [::String]
+        #     Optional. The job name.
         class ListJobsRequest
           include ::Google::Protobuf::MessageExts
           extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -842,7 +976,8 @@ module Google
           SHUFFLE_KIND = 8
         end
 
-        # Describes the overall state of a {::Google::Cloud::Dataflow::V1beta3::Job google.dataflow.v1beta3.Job}.
+        # Describes the overall state of a
+        # {::Google::Cloud::Dataflow::V1beta3::Job google.dataflow.v1beta3.Job}.
         module JobState
           # The job's run state isn't specified.
           JOB_STATE_UNKNOWN = 0
@@ -928,7 +1063,12 @@ module Google
           # and Cloud SDK version details.
           JOB_VIEW_SUMMARY = 1
 
-          # Request all information available for this job.
+          # Request all information available for this job. When the job is in
+          # `JOB_STATE_PENDING`, the job has been created but is not yet running, and
+          # not all job information is available. For complete
+          # job information, wait until the job in is `JOB_STATE_RUNNING`. For more
+          # information, see
+          # [JobState](https://cloud.google.com/dataflow/docs/reference/rest/v1b3/projects.jobs#jobstate).
           JOB_VIEW_ALL = 2
 
           # Request summary info and limited job description data for steps, labels and