google-cloud-dataflow-v1beta3 0.12.1 → 0.13.0

data/proto_docs/google/dataflow/v1beta3/jobs.rb

@@ -21,33 +21,36 @@ module Google
   module Cloud
     module Dataflow
       module V1beta3
-        # Defines a job to be run by the Cloud Dataflow service.
+        # Defines a job to be run by the Cloud Dataflow service. Do not enter
+        # confidential information when you supply string values using the API.
         # @!attribute [rw] id
         #   @return [::String]
         #     The unique ID of this job.
         #
-        #     This field is set by the Cloud Dataflow service when the Job is
+        #     This field is set by the Dataflow service when the job is
         #     created, and is immutable for the life of the job.
         # @!attribute [rw] project_id
         #   @return [::String]
-        #     The ID of the Cloud Platform project that the job belongs to.
+        #     The ID of the Google Cloud project that the job belongs to.
         # @!attribute [rw] name
         #   @return [::String]
-        #     The user-specified Cloud Dataflow job name.
+        #     Optional. The user-specified Dataflow job name.
         #
-        #     Only one Job with a given name may exist in a project at any
-        #     given time. If a caller attempts to create a Job with the same
-        #     name as an already-existing Job, the attempt returns the
-        #     existing Job.
+        #     Only one active job with a given name can exist in a project within one
+        #     region at
+        #     any given time. Jobs in different regions can have the same name.
+        #     If a caller attempts to create a job with the same
+        #     name as an active job that already exists, the attempt returns the
+        #     existing job.
         #
         #     The name must match the regular expression
         #     `[a-z]([-a-z0-9]{0,1022}[a-z0-9])?`
         # @!attribute [rw] type
         #   @return [::Google::Cloud::Dataflow::V1beta3::JobType]
-        #     The type of Cloud Dataflow job.
+        #     Optional. The type of Dataflow job.
         # @!attribute [rw] environment
         #   @return [::Google::Cloud::Dataflow::V1beta3::Environment]
-        #     The environment for the job.
+        #     Optional. The environment for the job.
         # @!attribute [rw] steps
         #   @return [::Array<::Google::Cloud::Dataflow::V1beta3::Step>]
         #     Exactly one of step or steps_location should be specified.
@@ -68,20 +71,23 @@ module Google
         #     terminal state. After a job has reached a terminal state, no
         #     further state updates may be made.
         #
-        #     This field may be mutated by the Cloud Dataflow service;
+        #     This field might be mutated by the Dataflow service;
         #     callers cannot mutate it.
         # @!attribute [rw] current_state_time
         #   @return [::Google::Protobuf::Timestamp]
         #     The timestamp associated with the current state.
         # @!attribute [rw] requested_state
         #   @return [::Google::Cloud::Dataflow::V1beta3::JobState]
-        #     The job's requested state.
+        #     The job's requested state. Applies to `UpdateJob` requests.
+        #
+        #     Set `requested_state` with `UpdateJob` requests to switch between the
+        #     states `JOB_STATE_STOPPED` and `JOB_STATE_RUNNING`. You can also use
+        #     `UpdateJob` requests to change a job's
+        #     state from `JOB_STATE_RUNNING` to `JOB_STATE_CANCELLED`,
+        #     `JOB_STATE_DONE`,  or `JOB_STATE_DRAINED`. These states irrevocably
+        #     terminate the job if it hasn't already reached a terminal state.
         #
-        #     `UpdateJob` may be used to switch between the `JOB_STATE_STOPPED` and
-        #     `JOB_STATE_RUNNING` states, by setting requested_state.  `UpdateJob` may
-        #     also be used to directly set a job's requested state to
-        #     `JOB_STATE_CANCELLED` or `JOB_STATE_DONE`, irrevocably terminating the
-        #     job if it has not already reached a terminal state.
+        #     This field has no effect on `CreateJob` requests.
         # @!attribute [rw] execution_info
         #   @return [::Google::Cloud::Dataflow::V1beta3::JobExecutionInfo]
         #     Deprecated.
@@ -99,8 +105,8 @@ module Google
         #     transferred to this job.
         # @!attribute [rw] transform_name_mapping
         #   @return [::Google::Protobuf::Map{::String => ::String}]
-        #     The map of transform name prefixes of the job to be replaced to the
-        #     corresponding name prefixes of the new job.
+        #     Optional. The map of transform name prefixes of the job to be replaced to
+        #     the corresponding name prefixes of the new job.
         # @!attribute [rw] client_request_id
         #   @return [::String]
         #     The client's unique identifier of the job, re-used across retried attempts.
@@ -141,7 +147,7 @@ module Google
         #     size.
         # @!attribute [rw] location
         #   @return [::String]
-        #     The [regional endpoint]
+        #     Optional. The [regional endpoint]
         #     (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) that
         #     contains this job.
         # @!attribute [rw] pipeline_description
@@ -175,6 +181,18 @@ module Google
         #   @return [::Boolean]
         #     Reserved for future use. This field is set only in responses from the
         #     server; it is ignored if it is set in any requests.
+        # @!attribute [rw] runtime_updatable_params
+        #   @return [::Google::Cloud::Dataflow::V1beta3::RuntimeUpdatableParams]
+        #     This field may ONLY be modified at runtime using the projects.jobs.update
+        #     method to adjust job behavior. This field has no effect when specified at
+        #     job creation.
+        # @!attribute [r] satisfies_pzi
+        #   @return [::Boolean]
+        #     Output only. Reserved for future use. This field is set only in responses
+        #     from the server; it is ignored if it is set in any requests.
+        # @!attribute [r] service_resources
+        #   @return [::Google::Cloud::Dataflow::V1beta3::ServiceResources]
+        #     Output only. Resources used by the Dataflow Service to run the job.
         class Job
           include ::Google::Protobuf::MessageExts
           extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -198,6 +216,40 @@ module Google
           end
         end
 
+        # Resources used by the Dataflow Service to run the job.
+        # @!attribute [r] zones
+        #   @return [::Array<::String>]
+        #     Output only. List of Cloud Zones being used by the Dataflow Service for
+        #     this job. Example: us-central1-c
+        class ServiceResources
+          include ::Google::Protobuf::MessageExts
+          extend ::Google::Protobuf::MessageExts::ClassMethods
+        end
+
+        # Additional job parameters that can only be updated during runtime using the
+        # projects.jobs.update method. These fields have no effect when specified
+        # during job creation.
+        # @!attribute [rw] max_num_workers
+        #   @return [::Integer]
+        #     The maximum number of workers to cap autoscaling at. This field is
+        #     currently only supported for Streaming Engine jobs.
+        # @!attribute [rw] min_num_workers
+        #   @return [::Integer]
+        #     The minimum number of workers to scale down to. This field is currently
+        #     only supported for Streaming Engine jobs.
+        # @!attribute [rw] worker_utilization_hint
+        #   @return [::Float]
+        #     Target worker utilization, compared against the aggregate utilization of
+        #     the worker pool by autoscaler, to determine upscaling and downscaling when
+        #     absent other constraints such as backlog.
+        #     For more information, see
+        #     [Update an existing
+        #     pipeline](https://cloud.google.com/dataflow/docs/guides/updating-a-pipeline).
+        class RuntimeUpdatableParams
+          include ::Google::Protobuf::MessageExts
+          extend ::Google::Protobuf::MessageExts::ClassMethods
+        end
+
         # Metadata for a Datastore connector used by the job.
         # @!attribute [rw] namespace
         #   @return [::String]
@@ -289,6 +341,9 @@ module Google
         # @!attribute [rw] sdk_support_status
         #   @return [::Google::Cloud::Dataflow::V1beta3::SdkVersion::SdkSupportStatus]
         #     The support status for this SDK version.
+        # @!attribute [r] bugs
+        #   @return [::Array<::Google::Cloud::Dataflow::V1beta3::SdkBug>]
+        #     Output only. Known bugs found in this SDK version.
         class SdkVersion
           include ::Google::Protobuf::MessageExts
           extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -313,6 +368,56 @@ module Google
           end
         end
 
+        # A bug found in the Dataflow SDK.
+        # @!attribute [r] type
+        #   @return [::Google::Cloud::Dataflow::V1beta3::SdkBug::Type]
+        #     Output only. Describes the impact of this SDK bug.
+        # @!attribute [r] severity
+        #   @return [::Google::Cloud::Dataflow::V1beta3::SdkBug::Severity]
+        #     Output only. How severe the SDK bug is.
+        # @!attribute [r] uri
+        #   @return [::String]
+        #     Output only. Link to more information on the bug.
+        class SdkBug
+          include ::Google::Protobuf::MessageExts
+          extend ::Google::Protobuf::MessageExts::ClassMethods
+
+          # Nature of the issue, ordered from least severe to most. Other bug types may
+          # be added to this list in the future.
+          module Type
+            # Unknown issue with this SDK.
+            TYPE_UNSPECIFIED = 0
+
+            # Catch-all for SDK bugs that don't fit in the below categories.
+            GENERAL = 1
+
+            # Using this version of the SDK may result in degraded performance.
+            PERFORMANCE = 2
+
+            # Using this version of the SDK may cause data loss.
+            DATALOSS = 3
+          end
+
+          # Indicates the severity of the bug. Other severities may be added to this
+          # list in the future.
+          module Severity
+            # A bug of unknown severity.
+            SEVERITY_UNSPECIFIED = 0
+
+            # A minor bug that that may reduce reliability or performance for some
+            # jobs. Impact will be minimal or non-existent for most jobs.
+            NOTICE = 1
+
+            # A bug that has some likelihood of causing performance degradation, data
+            # loss, or job failures.
+            WARNING = 2
+
+            # A bug with extremely significant impact. Jobs may fail erroneously,
+            # performance may be severely degraded, and data loss may be very likely.
+            SEVERE = 3
+          end
+        end
+
         # Metadata available primarily for filtering jobs. Will be included in the
         # ListJob response and Job SUMMARY view.
         # @!attribute [rw] sdk_version
@@ -336,9 +441,21 @@ module Google
         # @!attribute [rw] datastore_details
         #   @return [::Array<::Google::Cloud::Dataflow::V1beta3::DatastoreIODetails>]
         #     Identification of a Datastore source used in the Dataflow job.
+        # @!attribute [rw] user_display_properties
+        #   @return [::Google::Protobuf::Map{::String => ::String}]
+        #     List of display properties to help UI filter jobs.
         class JobMetadata
           include ::Google::Protobuf::MessageExts
           extend ::Google::Protobuf::MessageExts::ClassMethods
+
+          # @!attribute [rw] key
+          #   @return [::String]
+          # @!attribute [rw] value
+          #   @return [::String]
+          class UserDisplayPropertiesEntry
+            include ::Google::Protobuf::MessageExts
+            extend ::Google::Protobuf::MessageExts::ClassMethods
+          end
         end
 
         # A message describing the state of a particular execution stage.
@@ -368,6 +485,9 @@ module Google
         # @!attribute [rw] display_data
         #   @return [::Array<::Google::Cloud::Dataflow::V1beta3::DisplayData>]
         #     Pipeline level display data.
+        # @!attribute [rw] step_names_hash
+        #   @return [::String]
+        #     A hash value of the submitted pipeline portable graph step names if exists.
         class PipelineDescription
           include ::Google::Protobuf::MessageExts
           extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -553,6 +673,8 @@ module Google
         # specific operation as part of the overall job.  Data is typically
         # passed from one step to another as part of the job.
         #
+        # **Note:** The properties of this object are not stable and might change.
+        #
         # Here's an example of a sequence of steps which together implement a
         # Map-Reduce job:
         #
@@ -609,7 +731,8 @@ module Google
         end
 
         # Contains information about how a particular
-        # {::Google::Cloud::Dataflow::V1beta3::Step google.dataflow.v1beta3.Step} will be executed.
+        # {::Google::Cloud::Dataflow::V1beta3::Step google.dataflow.v1beta3.Step} will be
+        # executed.
         # @!attribute [rw] step_name
         #   @return [::Array<::String>]
         #     The steps associated with the execution stage.
@@ -679,6 +802,14 @@ module Google
         #     The [regional endpoint]
         #     (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) that
         #     contains this job.
+        # @!attribute [rw] update_mask
+        #   @return [::Google::Protobuf::FieldMask]
+        #     The list of fields to update relative to Job. If empty, only
+        #     RequestedJobState will be considered for update. If the FieldMask is not
+        #     empty and RequestedJobState is none/empty, The fields specified in the
+        #     update mask will be the only ones considered for update. If both
+        #     RequestedJobState and update_mask are specified, an error will be returned
+        #     as we cannot update both state and mask.
         class UpdateJobRequest
           include ::Google::Protobuf::MessageExts
           extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -710,6 +841,9 @@ module Google
         #     The [regional endpoint]
         #     (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) that
         #     contains this job.
+        # @!attribute [rw] name
+        #   @return [::String]
+        #     Optional. The job name.
         class ListJobsRequest
           include ::Google::Protobuf::MessageExts
           extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -842,7 +976,8 @@ module Google
           SHUFFLE_KIND = 8
         end
 
-        # Describes the overall state of a {::Google::Cloud::Dataflow::V1beta3::Job google.dataflow.v1beta3.Job}.
+        # Describes the overall state of a
+        # {::Google::Cloud::Dataflow::V1beta3::Job google.dataflow.v1beta3.Job}.
         module JobState
           # The job's run state isn't specified.
           JOB_STATE_UNKNOWN = 0
@@ -928,7 +1063,12 @@ module Google
           # and Cloud SDK version details.
           JOB_VIEW_SUMMARY = 1
 
-          # Request all information available for this job.
+          # Request all information available for this job. When the job is in
+          # `JOB_STATE_PENDING`, the job has been created but is not yet running, and
+          # not all job information is available. For complete
+          # job information, wait until the job in is `JOB_STATE_RUNNING`. For more
+          # information, see
+          # [JobState](https://cloud.google.com/dataflow/docs/reference/rest/v1b3/projects.jobs#jobstate).
           JOB_VIEW_ALL = 2
 
           # Request summary info and limited job description data for steps, labels and

data/proto_docs/google/dataflow/v1beta3/metrics.rb

@@ -53,6 +53,7 @@ module Google
         end
 
         # Describes the state of a metric.
+        # Next ID: 14
         # @!attribute [rw] name
         #   @return [::Google::Cloud::Dataflow::V1beta3::MetricStructuredName]
         #     Name of the metric.
@@ -90,8 +91,12 @@ module Google
         #   @return [::Google::Protobuf::Value]
         #     Worker-computed aggregate value for the "Set" aggregation kind.  The only
         #     possible value type is a list of Values whose type can be Long, Double,
-        #     or String, according to the metric's type.  All Values in the list must
-        #     be of the same type.
+        #     String, or BoundedTrie according to the metric's type.  All Values in the
+        #     list must be of the same type.
+        # @!attribute [rw] trie
+        #   @return [::Google::Protobuf::Value]
+        #     Worker-computed aggregate value for the "Trie" aggregation kind.  The only
+        #     possible value type is a BoundedTrieNode.
         # @!attribute [rw] distribution
         #   @return [::Google::Protobuf::Value]
         #     A struct value describing properties of a distribution of numeric values.
@@ -137,7 +142,8 @@ module Google
 
         # JobMetrics contains a collection of metrics describing the detailed progress
         # of a Dataflow job. Metrics correspond to user-defined and system-defined
-        # metrics in the job.
+        # metrics in the job. For more information, see [Dataflow job metrics]
+        # (https://cloud.google.com/dataflow/docs/guides/using-monitoring-intf).
         #
         # This resource captures only the most recent values of each metric;
         # time-series data can be queried for them (under the same metric names)
@@ -206,6 +212,139 @@ module Google
           end
         end
 
+        # Information useful for straggler identification and debugging.
+        # @!attribute [rw] start_time
+        #   @return [::Google::Protobuf::Timestamp]
+        #     The time when the work item attempt became a straggler.
+        # @!attribute [rw] causes
+        #   @return [::Google::Protobuf::Map{::String => ::Google::Cloud::Dataflow::V1beta3::StragglerInfo::StragglerDebuggingInfo}]
+        #     The straggler causes, keyed by the string representation of the
+        #     StragglerCause enum and contains specialized debugging information for each
+        #     straggler cause.
+        class StragglerInfo
+          include ::Google::Protobuf::MessageExts
+          extend ::Google::Protobuf::MessageExts::ClassMethods
+
+          # Information useful for debugging a straggler. Each type will provide
+          # specialized debugging information relevant for a particular cause.
+          # The StragglerDebuggingInfo will be 1:1 mapping to the StragglerCause enum.
+          # @!attribute [rw] hot_key
+          #   @return [::Google::Cloud::Dataflow::V1beta3::HotKeyDebuggingInfo]
+          #     Hot key debugging details.
+          class StragglerDebuggingInfo
+            include ::Google::Protobuf::MessageExts
+            extend ::Google::Protobuf::MessageExts::ClassMethods
+          end
+
+          # @!attribute [rw] key
+          #   @return [::String]
+          # @!attribute [rw] value
+          #   @return [::Google::Cloud::Dataflow::V1beta3::StragglerInfo::StragglerDebuggingInfo]
+          class CausesEntry
+            include ::Google::Protobuf::MessageExts
+            extend ::Google::Protobuf::MessageExts::ClassMethods
+          end
+        end
+
+        # Information useful for streaming straggler identification and debugging.
+        # @!attribute [rw] start_time
+        #   @return [::Google::Protobuf::Timestamp]
+        #     Start time of this straggler.
+        # @!attribute [rw] end_time
+        #   @return [::Google::Protobuf::Timestamp]
+        #     End time of this straggler.
+        # @!attribute [rw] worker_name
+        #   @return [::String]
+        #     Name of the worker where the straggler was detected.
+        # @!attribute [rw] data_watermark_lag
+        #   @return [::Google::Protobuf::Duration]
+        #     The event-time watermark lag at the time of the straggler detection.
+        # @!attribute [rw] system_watermark_lag
+        #   @return [::Google::Protobuf::Duration]
+        #     The system watermark lag at the time of the straggler detection.
+        class StreamingStragglerInfo
+          include ::Google::Protobuf::MessageExts
+          extend ::Google::Protobuf::MessageExts::ClassMethods
+        end
+
+        # Information for a straggler.
+        # @!attribute [rw] batch_straggler
+        #   @return [::Google::Cloud::Dataflow::V1beta3::StragglerInfo]
+        #     Batch straggler identification and debugging information.
+        #
+        #     Note: The following fields are mutually exclusive: `batch_straggler`, `streaming_straggler`. If a field in that set is populated, all other fields in the set will automatically be cleared.
+        # @!attribute [rw] streaming_straggler
+        #   @return [::Google::Cloud::Dataflow::V1beta3::StreamingStragglerInfo]
+        #     Streaming straggler identification and debugging information.
+        #
+        #     Note: The following fields are mutually exclusive: `streaming_straggler`, `batch_straggler`. If a field in that set is populated, all other fields in the set will automatically be cleared.
+        class Straggler
+          include ::Google::Protobuf::MessageExts
+          extend ::Google::Protobuf::MessageExts::ClassMethods
+        end
+
+        # Information useful for debugging a hot key detection.
+        # @!attribute [rw] detected_hot_keys
+        #   @return [::Google::Protobuf::Map{::Integer => ::Google::Cloud::Dataflow::V1beta3::HotKeyDebuggingInfo::HotKeyInfo}]
+        #     Debugging information for each detected hot key. Keyed by a hash of the
+        #     key.
+        class HotKeyDebuggingInfo
+          include ::Google::Protobuf::MessageExts
+          extend ::Google::Protobuf::MessageExts::ClassMethods
+
+          # Information about a hot key.
+          # @!attribute [rw] hot_key_age
+          #   @return [::Google::Protobuf::Duration]
+          #     The age of the hot key measured from when it was first detected.
+          # @!attribute [rw] key
+          #   @return [::String]
+          #     A detected hot key that is causing limited parallelism. This field will
+          #     be populated only if the following flag is set to true:
+          #     "--enable_hot_key_logging".
+          # @!attribute [rw] key_truncated
+          #   @return [::Boolean]
+          #     If true, then the above key is truncated and cannot be deserialized. This
+          #     occurs if the key above is populated and the key size is >5MB.
+          class HotKeyInfo
+            include ::Google::Protobuf::MessageExts
+            extend ::Google::Protobuf::MessageExts::ClassMethods
+          end
+
+          # @!attribute [rw] key
+          #   @return [::Integer]
+          # @!attribute [rw] value
+          #   @return [::Google::Cloud::Dataflow::V1beta3::HotKeyDebuggingInfo::HotKeyInfo]
+          class DetectedHotKeysEntry
+            include ::Google::Protobuf::MessageExts
+            extend ::Google::Protobuf::MessageExts::ClassMethods
+          end
+        end
+
+        # Summarized straggler identification details.
+        # @!attribute [rw] total_straggler_count
+        #   @return [::Integer]
+        #     The total count of stragglers.
+        # @!attribute [rw] straggler_cause_count
+        #   @return [::Google::Protobuf::Map{::String => ::Integer}]
+        #     Aggregated counts of straggler causes, keyed by the string representation
+        #     of the StragglerCause enum.
+        # @!attribute [rw] recent_stragglers
+        #   @return [::Array<::Google::Cloud::Dataflow::V1beta3::Straggler>]
+        #     The most recent stragglers.
+        class StragglerSummary
+          include ::Google::Protobuf::MessageExts
+          extend ::Google::Protobuf::MessageExts::ClassMethods
+
+          # @!attribute [rw] key
+          #   @return [::String]
+          # @!attribute [rw] value
+          #   @return [::Integer]
+          class StragglerCauseCountEntry
+            include ::Google::Protobuf::MessageExts
+            extend ::Google::Protobuf::MessageExts::ClassMethods
+          end
+        end
+
         # Information about a particular execution stage of a job.
         # @!attribute [rw] stage_id
         #   @return [::String]
@@ -229,6 +368,9 @@ module Google
         # @!attribute [rw] metrics
         #   @return [::Array<::Google::Cloud::Dataflow::V1beta3::MetricUpdate>]
         #     Metrics for this stage.
+        # @!attribute [rw] straggler_summary
+        #   @return [::Google::Cloud::Dataflow::V1beta3::StragglerSummary]
+        #     Straggler summary for this stage.
         class StageSummary
           include ::Google::Protobuf::MessageExts
           extend ::Google::Protobuf::MessageExts::ClassMethods
@@ -310,6 +452,9 @@ module Google
         # @!attribute [rw] metrics
         #   @return [::Array<::Google::Cloud::Dataflow::V1beta3::MetricUpdate>]
         #     Metrics for this work item.
+        # @!attribute [rw] straggler_info
+        #   @return [::Google::Cloud::Dataflow::V1beta3::StragglerInfo]
+        #     Information about straggler detections for this work item.
         class WorkItemDetails
           include ::Google::Protobuf::MessageExts
           extend ::Google::Protobuf::MessageExts::ClassMethods

data/proto_docs/google/dataflow/v1beta3/streaming.rb

@@ -80,6 +80,9 @@ module Google
         # @!attribute [rw] with_attributes
         #   @return [::Boolean]
         #     If true, then the client has requested to get pubsub attributes.
+        # @!attribute [rw] dynamic_destinations
+        #   @return [::Boolean]
+        #     If true, then this location represents dynamic topics.
         class PubsubLocation
           include ::Google::Protobuf::MessageExts
           extend ::Google::Protobuf::MessageExts::ClassMethods