google-cloud-dataproc-v1beta2 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. checksums.yaml +7 -0
  2. data/.yardopts +12 -0
  3. data/AUTHENTICATION.md +169 -0
  4. data/LICENSE.md +203 -0
  5. data/README.md +71 -0
  6. data/lib/google-cloud-dataproc-v1beta2.rb +21 -0
  7. data/lib/google/cloud/common_resources_pb.rb +15 -0
  8. data/lib/google/cloud/dataproc/v1beta2.rb +38 -0
  9. data/lib/google/cloud/dataproc/v1beta2/autoscaling_policies_pb.rb +81 -0
  10. data/lib/google/cloud/dataproc/v1beta2/autoscaling_policies_services_pb.rb +58 -0
  11. data/lib/google/cloud/dataproc/v1beta2/autoscaling_policy_service.rb +50 -0
  12. data/lib/google/cloud/dataproc/v1beta2/autoscaling_policy_service/client.rb +734 -0
  13. data/lib/google/cloud/dataproc/v1beta2/autoscaling_policy_service/credentials.rb +51 -0
  14. data/lib/google/cloud/dataproc/v1beta2/autoscaling_policy_service/paths.rb +110 -0
  15. data/lib/google/cloud/dataproc/v1beta2/cluster_controller.rb +50 -0
  16. data/lib/google/cloud/dataproc/v1beta2/cluster_controller/client.rb +979 -0
  17. data/lib/google/cloud/dataproc/v1beta2/cluster_controller/credentials.rb +51 -0
  18. data/lib/google/cloud/dataproc/v1beta2/cluster_controller/operations.rb +564 -0
  19. data/lib/google/cloud/dataproc/v1beta2/clusters_pb.rb +255 -0
  20. data/lib/google/cloud/dataproc/v1beta2/clusters_services_pb.rb +68 -0
  21. data/lib/google/cloud/dataproc/v1beta2/job_controller.rb +49 -0
  22. data/lib/google/cloud/dataproc/v1beta2/job_controller/client.rb +980 -0
  23. data/lib/google/cloud/dataproc/v1beta2/job_controller/credentials.rb +51 -0
  24. data/lib/google/cloud/dataproc/v1beta2/job_controller/operations.rb +564 -0
  25. data/lib/google/cloud/dataproc/v1beta2/jobs_pb.rb +283 -0
  26. data/lib/google/cloud/dataproc/v1beta2/jobs_services_pb.rb +62 -0
  27. data/lib/google/cloud/dataproc/v1beta2/operations_pb.rb +45 -0
  28. data/lib/google/cloud/dataproc/v1beta2/shared_pb.rb +35 -0
  29. data/lib/google/cloud/dataproc/v1beta2/version.rb +28 -0
  30. data/lib/google/cloud/dataproc/v1beta2/workflow_template_service.rb +51 -0
  31. data/lib/google/cloud/dataproc/v1beta2/workflow_template_service/client.rb +1028 -0
  32. data/lib/google/cloud/dataproc/v1beta2/workflow_template_service/credentials.rb +51 -0
  33. data/lib/google/cloud/dataproc/v1beta2/workflow_template_service/operations.rb +564 -0
  34. data/lib/google/cloud/dataproc/v1beta2/workflow_template_service/paths.rb +110 -0
  35. data/lib/google/cloud/dataproc/v1beta2/workflow_templates_pb.rb +189 -0
  36. data/lib/google/cloud/dataproc/v1beta2/workflow_templates_services_pb.rb +104 -0
  37. data/proto_docs/README.md +4 -0
  38. data/proto_docs/google/api/field_behavior.rb +59 -0
  39. data/proto_docs/google/api/resource.rb +247 -0
  40. data/proto_docs/google/cloud/dataproc/v1beta2/autoscaling_policies.rb +272 -0
  41. data/proto_docs/google/cloud/dataproc/v1beta2/clusters.rb +1032 -0
  42. data/proto_docs/google/cloud/dataproc/v1beta2/jobs.rb +984 -0
  43. data/proto_docs/google/cloud/dataproc/v1beta2/operations.rb +98 -0
  44. data/proto_docs/google/cloud/dataproc/v1beta2/shared.rb +68 -0
  45. data/proto_docs/google/cloud/dataproc/v1beta2/workflow_templates.rb +706 -0
  46. data/proto_docs/google/longrunning/operations.rb +150 -0
  47. data/proto_docs/google/protobuf/any.rb +138 -0
  48. data/proto_docs/google/protobuf/duration.rb +98 -0
  49. data/proto_docs/google/protobuf/empty.rb +36 -0
  50. data/proto_docs/google/protobuf/field_mask.rb +229 -0
  51. data/proto_docs/google/protobuf/timestamp.rb +120 -0
  52. data/proto_docs/google/rpc/status.rb +46 -0
  53. metadata +205 -0
@@ -0,0 +1,984 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Copyright 2020 Google LLC
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # https://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+
17
+ # Auto-generated by gapic-generator-ruby. DO NOT EDIT!
18
+
19
+
20
+ module Google
21
+ module Cloud
22
+ module Dataproc
23
+ module V1beta2
24
+ # The runtime logging config of the job.
25
+ # @!attribute [rw] driver_log_levels
26
+ # @return [Google::Protobuf::Map{String => Google::Cloud::Dataproc::V1beta2::LoggingConfig::Level}]
27
+ # The per-package log levels for the driver. This may include
28
+ # "root" package name to configure rootLogger.
29
+ # Examples:
30
+ # 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'
31
+ class LoggingConfig
32
+ include Google::Protobuf::MessageExts
33
+ extend Google::Protobuf::MessageExts::ClassMethods
34
+
35
+ # @!attribute [rw] key
36
+ # @return [String]
37
+ # @!attribute [rw] value
38
+ # @return [Google::Cloud::Dataproc::V1beta2::LoggingConfig::Level]
39
+ class DriverLogLevelsEntry
40
+ include Google::Protobuf::MessageExts
41
+ extend Google::Protobuf::MessageExts::ClassMethods
42
+ end
43
+
44
+ # The Log4j level for job execution. When running an
45
+ # [Apache Hive](http://hive.apache.org/) job, Cloud
46
+ # Dataproc configures the Hive client to an equivalent verbosity level.
47
+ module Level
48
+ # Level is unspecified. Use default level for log4j.
49
+ LEVEL_UNSPECIFIED = 0
50
+
51
+ # Use ALL level for log4j.
52
+ ALL = 1
53
+
54
+ # Use TRACE level for log4j.
55
+ TRACE = 2
56
+
57
+ # Use DEBUG level for log4j.
58
+ DEBUG = 3
59
+
60
+ # Use INFO level for log4j.
61
+ INFO = 4
62
+
63
+ # Use WARN level for log4j.
64
+ WARN = 5
65
+
66
+ # Use ERROR level for log4j.
67
+ ERROR = 6
68
+
69
+ # Use FATAL level for log4j.
70
+ FATAL = 7
71
+
72
+ # Turn off log4j.
73
+ OFF = 8
74
+ end
75
+ end
76
+
77
+ # A Dataproc job for running
78
+ # [Apache Hadoop
79
+ # MapReduce](https://hadoop.apache.org/docs/current/hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapReduceTutorial.html)
80
+ # jobs on [Apache Hadoop
81
+ # YARN](https://hadoop.apache.org/docs/r2.7.1/hadoop-yarn/hadoop-yarn-site/YARN.html).
82
+ # @!attribute [rw] main_jar_file_uri
83
+ # @return [String]
84
+ # The HCFS URI of the jar file containing the main class.
85
+ # Examples:
86
+ # 'gs://foo-bucket/analytics-binaries/extract-useful-metrics-mr.jar'
87
+ # 'hdfs:/tmp/test-samples/custom-wordcount.jar'
88
+ # 'file:///home/usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples.jar'
89
+ # @!attribute [rw] main_class
90
+ # @return [String]
91
+ # The name of the driver's main class. The jar file containing the class
92
+ # must be in the default CLASSPATH or specified in `jar_file_uris`.
93
+ # @!attribute [rw] args
94
+ # @return [Array<String>]
95
+ # Optional. The arguments to pass to the driver. Do not
96
+ # include arguments, such as `-libjars` or `-Dfoo=bar`, that can be set as
97
+ # job properties, since a collision may occur that causes an incorrect job
98
+ # submission.
99
+ # @!attribute [rw] jar_file_uris
100
+ # @return [Array<String>]
101
+ # Optional. Jar file URIs to add to the CLASSPATHs of the
102
+ # Hadoop driver and tasks.
103
+ # @!attribute [rw] file_uris
104
+ # @return [Array<String>]
105
+ # Optional. HCFS (Hadoop Compatible Filesystem) URIs of files to be copied
106
+ # to the working directory of Hadoop drivers and distributed tasks. Useful
107
+ # for naively parallel tasks.
108
+ # @!attribute [rw] archive_uris
109
+ # @return [Array<String>]
110
+ # Optional. HCFS URIs of archives to be extracted in the working directory of
111
+ # Hadoop drivers and tasks. Supported file types:
112
+ # .jar, .tar, .tar.gz, .tgz, or .zip.
113
+ # @!attribute [rw] properties
114
+ # @return [Google::Protobuf::Map{String => String}]
115
+ # Optional. A mapping of property names to values, used to configure Hadoop.
116
+ # Properties that conflict with values set by the Dataproc API may be
117
+ # overwritten. Can include properties set in /etc/hadoop/conf/*-site and
118
+ # classes in user code.
119
+ # @!attribute [rw] logging_config
120
+ # @return [Google::Cloud::Dataproc::V1beta2::LoggingConfig]
121
+ # Optional. The runtime log config for job execution.
122
+ class HadoopJob
123
+ include Google::Protobuf::MessageExts
124
+ extend Google::Protobuf::MessageExts::ClassMethods
125
+
126
+ # @!attribute [rw] key
127
+ # @return [String]
128
+ # @!attribute [rw] value
129
+ # @return [String]
130
+ class PropertiesEntry
131
+ include Google::Protobuf::MessageExts
132
+ extend Google::Protobuf::MessageExts::ClassMethods
133
+ end
134
+ end
135
+
136
+ # A Dataproc job for running [Apache Spark](http://spark.apache.org/)
137
+ # applications on YARN.
138
+ # The specification of the main method to call to drive the job.
139
+ # Specify either the jar file that contains the main class or the main class
140
+ # name. To pass both a main jar and a main class in that jar, add the jar to
141
+ # `CommonJob.jar_file_uris`, and then specify the main class name in
142
+ # `main_class`.
143
+ # @!attribute [rw] main_jar_file_uri
144
+ # @return [String]
145
+ # The HCFS URI of the jar file that contains the main class.
146
+ # @!attribute [rw] main_class
147
+ # @return [String]
148
+ # The name of the driver's main class. The jar file that contains the class
149
+ # must be in the default CLASSPATH or specified in `jar_file_uris`.
150
+ # @!attribute [rw] args
151
+ # @return [Array<String>]
152
+ # Optional. The arguments to pass to the driver. Do not include arguments,
153
+ # such as `--conf`, that can be set as job properties, since a collision may
154
+ # occur that causes an incorrect job submission.
155
+ # @!attribute [rw] jar_file_uris
156
+ # @return [Array<String>]
157
+ # Optional. HCFS URIs of jar files to add to the CLASSPATHs of the
158
+ # Spark driver and tasks.
159
+ # @!attribute [rw] file_uris
160
+ # @return [Array<String>]
161
+ # Optional. HCFS URIs of files to be copied to the working directory of
162
+ # Spark drivers and distributed tasks. Useful for naively parallel tasks.
163
+ # @!attribute [rw] archive_uris
164
+ # @return [Array<String>]
165
+ # Optional. HCFS URIs of archives to be extracted in the working directory
166
+ # of Spark drivers and tasks. Supported file types:
167
+ # .jar, .tar, .tar.gz, .tgz, and .zip.
168
+ # @!attribute [rw] properties
169
+ # @return [Google::Protobuf::Map{String => String}]
170
+ # Optional. A mapping of property names to values, used to configure Spark.
171
+ # Properties that conflict with values set by the Dataproc API may be
172
+ # overwritten. Can include properties set in
173
+ # /etc/spark/conf/spark-defaults.conf and classes in user code.
174
+ # @!attribute [rw] logging_config
175
+ # @return [Google::Cloud::Dataproc::V1beta2::LoggingConfig]
176
+ # Optional. The runtime log config for job execution.
177
+ class SparkJob
178
+ include Google::Protobuf::MessageExts
179
+ extend Google::Protobuf::MessageExts::ClassMethods
180
+
181
+ # @!attribute [rw] key
182
+ # @return [String]
183
+ # @!attribute [rw] value
184
+ # @return [String]
185
+ class PropertiesEntry
186
+ include Google::Protobuf::MessageExts
187
+ extend Google::Protobuf::MessageExts::ClassMethods
188
+ end
189
+ end
190
+
191
+ # A Dataproc job for running
192
+ # [Apache
193
+ # PySpark](https://spark.apache.org/docs/0.9.0/python-programming-guide.html)
194
+ # applications on YARN.
195
+ # @!attribute [rw] main_python_file_uri
196
+ # @return [String]
197
+ # Required. The HCFS URI of the main Python file to use as the driver. Must
198
+ # be a .py file.
199
+ # @!attribute [rw] args
200
+ # @return [Array<String>]
201
+ # Optional. The arguments to pass to the driver. Do not include arguments,
202
+ # such as `--conf`, that can be set as job properties, since a collision may
203
+ # occur that causes an incorrect job submission.
204
+ # @!attribute [rw] python_file_uris
205
+ # @return [Array<String>]
206
+ # Optional. HCFS file URIs of Python files to pass to the PySpark
207
+ # framework. Supported file types: .py, .egg, and .zip.
208
+ # @!attribute [rw] jar_file_uris
209
+ # @return [Array<String>]
210
+ # Optional. HCFS URIs of jar files to add to the CLASSPATHs of the
211
+ # Python driver and tasks.
212
+ # @!attribute [rw] file_uris
213
+ # @return [Array<String>]
214
+ # Optional. HCFS URIs of files to be copied to the working directory of
215
+ # Python drivers and distributed tasks. Useful for naively parallel tasks.
216
+ # @!attribute [rw] archive_uris
217
+ # @return [Array<String>]
218
+ # Optional. HCFS URIs of archives to be extracted in the working directory of
219
+ # .jar, .tar, .tar.gz, .tgz, and .zip.
220
+ # @!attribute [rw] properties
221
+ # @return [Google::Protobuf::Map{String => String}]
222
+ # Optional. A mapping of property names to values, used to configure PySpark.
223
+ # Properties that conflict with values set by the Dataproc API may be
224
+ # overwritten. Can include properties set in
225
+ # /etc/spark/conf/spark-defaults.conf and classes in user code.
226
+ # @!attribute [rw] logging_config
227
+ # @return [Google::Cloud::Dataproc::V1beta2::LoggingConfig]
228
+ # Optional. The runtime log config for job execution.
229
+ class PySparkJob
230
+ include Google::Protobuf::MessageExts
231
+ extend Google::Protobuf::MessageExts::ClassMethods
232
+
233
+ # @!attribute [rw] key
234
+ # @return [String]
235
+ # @!attribute [rw] value
236
+ # @return [String]
237
+ class PropertiesEntry
238
+ include Google::Protobuf::MessageExts
239
+ extend Google::Protobuf::MessageExts::ClassMethods
240
+ end
241
+ end
242
+
243
+ # A list of queries to run on a cluster.
244
+ # @!attribute [rw] queries
245
+ # @return [Array<String>]
246
+ # Required. The queries to execute. You do not need to terminate a query
247
+ # with a semicolon. Multiple queries can be specified in one string
248
+ # by separating each with a semicolon. Here is an example of an Cloud
249
+ # Dataproc API snippet that uses a QueryList to specify a HiveJob:
250
+ #
251
+ # "hiveJob": {
252
+ # "queryList": {
253
+ # "queries": [
254
+ # "query1",
255
+ # "query2",
256
+ # "query3;query4",
257
+ # ]
258
+ # }
259
+ # }
260
+ class QueryList
261
+ include Google::Protobuf::MessageExts
262
+ extend Google::Protobuf::MessageExts::ClassMethods
263
+ end
264
+
265
+ # A Dataproc job for running [Apache Hive](https://hive.apache.org/)
266
+ # queries on YARN.
267
+ # @!attribute [rw] query_file_uri
268
+ # @return [String]
269
+ # The HCFS URI of the script that contains Hive queries.
270
+ # @!attribute [rw] query_list
271
+ # @return [Google::Cloud::Dataproc::V1beta2::QueryList]
272
+ # A list of queries.
273
+ # @!attribute [rw] continue_on_failure
274
+ # @return [Boolean]
275
+ # Optional. Whether to continue executing queries if a query fails.
276
+ # The default value is `false`. Setting to `true` can be useful when
277
+ # executing independent parallel queries.
278
+ # @!attribute [rw] script_variables
279
+ # @return [Google::Protobuf::Map{String => String}]
280
+ # Optional. Mapping of query variable names to values (equivalent to the
281
+ # Hive command: `SET name="value";`).
282
+ # @!attribute [rw] properties
283
+ # @return [Google::Protobuf::Map{String => String}]
284
+ # Optional. A mapping of property names and values, used to configure Hive.
285
+ # Properties that conflict with values set by the Dataproc API may be
286
+ # overwritten. Can include properties set in /etc/hadoop/conf/*-site.xml,
287
+ # /etc/hive/conf/hive-site.xml, and classes in user code.
288
+ # @!attribute [rw] jar_file_uris
289
+ # @return [Array<String>]
290
+ # Optional. HCFS URIs of jar files to add to the CLASSPATH of the
291
+ # Hive server and Hadoop MapReduce (MR) tasks. Can contain Hive SerDes
292
+ # and UDFs.
293
+ class HiveJob
294
+ include Google::Protobuf::MessageExts
295
+ extend Google::Protobuf::MessageExts::ClassMethods
296
+
297
+ # @!attribute [rw] key
298
+ # @return [String]
299
+ # @!attribute [rw] value
300
+ # @return [String]
301
+ class ScriptVariablesEntry
302
+ include Google::Protobuf::MessageExts
303
+ extend Google::Protobuf::MessageExts::ClassMethods
304
+ end
305
+
306
+ # @!attribute [rw] key
307
+ # @return [String]
308
+ # @!attribute [rw] value
309
+ # @return [String]
310
+ class PropertiesEntry
311
+ include Google::Protobuf::MessageExts
312
+ extend Google::Protobuf::MessageExts::ClassMethods
313
+ end
314
+ end
315
+
316
+ # A Dataproc job for running [Apache Spark
317
+ # SQL](http://spark.apache.org/sql/) queries.
318
+ # @!attribute [rw] query_file_uri
319
+ # @return [String]
320
+ # The HCFS URI of the script that contains SQL queries.
321
+ # @!attribute [rw] query_list
322
+ # @return [Google::Cloud::Dataproc::V1beta2::QueryList]
323
+ # A list of queries.
324
+ # @!attribute [rw] script_variables
325
+ # @return [Google::Protobuf::Map{String => String}]
326
+ # Optional. Mapping of query variable names to values (equivalent to the
327
+ # Spark SQL command: SET `name="value";`).
328
+ # @!attribute [rw] properties
329
+ # @return [Google::Protobuf::Map{String => String}]
330
+ # Optional. A mapping of property names to values, used to configure
331
+ # Spark SQL's SparkConf. Properties that conflict with values set by the
332
+ # Dataproc API may be overwritten.
333
+ # @!attribute [rw] jar_file_uris
334
+ # @return [Array<String>]
335
+ # Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.
336
+ # @!attribute [rw] logging_config
337
+ # @return [Google::Cloud::Dataproc::V1beta2::LoggingConfig]
338
+ # Optional. The runtime log config for job execution.
339
+ class SparkSqlJob
340
+ include Google::Protobuf::MessageExts
341
+ extend Google::Protobuf::MessageExts::ClassMethods
342
+
343
+ # @!attribute [rw] key
344
+ # @return [String]
345
+ # @!attribute [rw] value
346
+ # @return [String]
347
+ class ScriptVariablesEntry
348
+ include Google::Protobuf::MessageExts
349
+ extend Google::Protobuf::MessageExts::ClassMethods
350
+ end
351
+
352
+ # @!attribute [rw] key
353
+ # @return [String]
354
+ # @!attribute [rw] value
355
+ # @return [String]
356
+ class PropertiesEntry
357
+ include Google::Protobuf::MessageExts
358
+ extend Google::Protobuf::MessageExts::ClassMethods
359
+ end
360
+ end
361
+
362
+ # A Dataproc job for running [Apache Pig](https://pig.apache.org/)
363
+ # queries on YARN.
364
+ # @!attribute [rw] query_file_uri
365
+ # @return [String]
366
+ # The HCFS URI of the script that contains the Pig queries.
367
+ # @!attribute [rw] query_list
368
+ # @return [Google::Cloud::Dataproc::V1beta2::QueryList]
369
+ # A list of queries.
370
+ # @!attribute [rw] continue_on_failure
371
+ # @return [Boolean]
372
+ # Optional. Whether to continue executing queries if a query fails.
373
+ # The default value is `false`. Setting to `true` can be useful when
374
+ # executing independent parallel queries.
375
+ # @!attribute [rw] script_variables
376
+ # @return [Google::Protobuf::Map{String => String}]
377
+ # Optional. Mapping of query variable names to values (equivalent to the Pig
378
+ # command: `name=[value]`).
379
+ # @!attribute [rw] properties
380
+ # @return [Google::Protobuf::Map{String => String}]
381
+ # Optional. A mapping of property names to values, used to configure Pig.
382
+ # Properties that conflict with values set by the Dataproc API may be
383
+ # overwritten. Can include properties set in /etc/hadoop/conf/*-site.xml,
384
+ # /etc/pig/conf/pig.properties, and classes in user code.
385
+ # @!attribute [rw] jar_file_uris
386
+ # @return [Array<String>]
387
+ # Optional. HCFS URIs of jar files to add to the CLASSPATH of
388
+ # the Pig Client and Hadoop MapReduce (MR) tasks. Can contain Pig UDFs.
389
+ # @!attribute [rw] logging_config
390
+ # @return [Google::Cloud::Dataproc::V1beta2::LoggingConfig]
391
+ # Optional. The runtime log config for job execution.
392
+ class PigJob
393
+ include Google::Protobuf::MessageExts
394
+ extend Google::Protobuf::MessageExts::ClassMethods
395
+
396
+ # @!attribute [rw] key
397
+ # @return [String]
398
+ # @!attribute [rw] value
399
+ # @return [String]
400
+ class ScriptVariablesEntry
401
+ include Google::Protobuf::MessageExts
402
+ extend Google::Protobuf::MessageExts::ClassMethods
403
+ end
404
+
405
+ # @!attribute [rw] key
406
+ # @return [String]
407
+ # @!attribute [rw] value
408
+ # @return [String]
409
+ class PropertiesEntry
410
+ include Google::Protobuf::MessageExts
411
+ extend Google::Protobuf::MessageExts::ClassMethods
412
+ end
413
+ end
414
+
415
+ # A Dataproc job for running
416
+ # [Apache SparkR](https://spark.apache.org/docs/latest/sparkr.html)
417
+ # applications on YARN.
418
+ # @!attribute [rw] main_r_file_uri
419
+ # @return [String]
420
+ # Required. The HCFS URI of the main R file to use as the driver.
421
+ # Must be a .R file.
422
+ # @!attribute [rw] args
423
+ # @return [Array<String>]
424
+ # Optional. The arguments to pass to the driver. Do not include arguments,
425
+ # such as `--conf`, that can be set as job properties, since a collision may
426
+ # occur that causes an incorrect job submission.
427
+ # @!attribute [rw] file_uris
428
+ # @return [Array<String>]
429
+ # Optional. HCFS URIs of files to be copied to the working directory of
430
+ # R drivers and distributed tasks. Useful for naively parallel tasks.
431
+ # @!attribute [rw] archive_uris
432
+ # @return [Array<String>]
433
+ # Optional. HCFS URIs of archives to be extracted in the working directory of
434
+ # Spark drivers and tasks. Supported file types:
435
+ # .jar, .tar, .tar.gz, .tgz, and .zip.
436
+ # @!attribute [rw] properties
437
+ # @return [Google::Protobuf::Map{String => String}]
438
+ # Optional. A mapping of property names to values, used to configure SparkR.
439
+ # Properties that conflict with values set by the Dataproc API may be
440
+ # overwritten. Can include properties set in
441
+ # /etc/spark/conf/spark-defaults.conf and classes in user code.
442
+ # @!attribute [rw] logging_config
443
+ # @return [Google::Cloud::Dataproc::V1beta2::LoggingConfig]
444
+ # Optional. The runtime log config for job execution.
445
+ class SparkRJob
446
+ include Google::Protobuf::MessageExts
447
+ extend Google::Protobuf::MessageExts::ClassMethods
448
+
449
+ # @!attribute [rw] key
450
+ # @return [String]
451
+ # @!attribute [rw] value
452
+ # @return [String]
453
+ class PropertiesEntry
454
+ include Google::Protobuf::MessageExts
455
+ extend Google::Protobuf::MessageExts::ClassMethods
456
+ end
457
+ end
458
+
459
+ # A Dataproc job for running [Presto](https://prestosql.io/) queries.
460
+ # **IMPORTANT**: The [Dataproc Presto Optional
461
+ # Component](/dataproc/docs/concepts/components/presto) must be enabled when
462
+ # the cluster is created to submit a Presto job to the cluster.
463
+ # @!attribute [rw] query_file_uri
464
+ # @return [String]
465
+ # The HCFS URI of the script that contains SQL queries.
466
+ # @!attribute [rw] query_list
467
+ # @return [Google::Cloud::Dataproc::V1beta2::QueryList]
468
+ # A list of queries.
469
+ # @!attribute [rw] continue_on_failure
470
+ # @return [Boolean]
471
+ # Optional. Whether to continue executing queries if a query fails.
472
+ # The default value is `false`. Setting to `true` can be useful when
473
+ # executing independent parallel queries.
474
+ # @!attribute [rw] output_format
475
+ # @return [String]
476
+ # Optional. The format in which query output will be displayed. See the
477
+ # Presto documentation for supported output formats
478
+ # @!attribute [rw] client_tags
479
+ # @return [Array<String>]
480
+ # Optional. Presto client tags to attach to this query
481
+ # @!attribute [rw] properties
482
+ # @return [Google::Protobuf::Map{String => String}]
483
+ # Optional. A mapping of property names to values. Used to set Presto
484
+ # [session properties](https://prestodb.io/docs/current/sql/set-session.html)
485
+ # Equivalent to using the --session flag in the Presto CLI
486
+ # @!attribute [rw] logging_config
487
+ # @return [Google::Cloud::Dataproc::V1beta2::LoggingConfig]
488
+ # Optional. The runtime log config for job execution.
489
+ class PrestoJob
490
+ include Google::Protobuf::MessageExts
491
+ extend Google::Protobuf::MessageExts::ClassMethods
492
+
493
+ # @!attribute [rw] key
494
+ # @return [String]
495
+ # @!attribute [rw] value
496
+ # @return [String]
497
+ class PropertiesEntry
498
+ include Google::Protobuf::MessageExts
499
+ extend Google::Protobuf::MessageExts::ClassMethods
500
+ end
501
+ end
502
+
503
+ # Dataproc job config.
504
+ # @!attribute [rw] cluster_name
505
+ # @return [String]
506
+ # Required. The name of the cluster where the job will be submitted.
507
+ # @!attribute [r] cluster_uuid
508
+ # @return [String]
509
+ # Output only. A cluster UUID generated by the Dataproc service when
510
+ # the job is submitted.
511
+ class JobPlacement
512
+ include Google::Protobuf::MessageExts
513
+ extend Google::Protobuf::MessageExts::ClassMethods
514
+ end
515
+
516
+ # Dataproc job status.
517
+ # @!attribute [r] state
518
+ # @return [Google::Cloud::Dataproc::V1beta2::JobStatus::State]
519
+ # Output only. A state message specifying the overall job state.
520
+ # @!attribute [r] details
521
+ # @return [String]
522
+ # Output only. Optional Job state details, such as an error
523
+ # description if the state is <code>ERROR</code>.
524
+ # @!attribute [r] state_start_time
525
+ # @return [Google::Protobuf::Timestamp]
526
+ # Output only. The time when this state was entered.
527
+ # @!attribute [r] substate
528
+ # @return [Google::Cloud::Dataproc::V1beta2::JobStatus::Substate]
529
+ # Output only. Additional state information, which includes
530
+ # status reported by the agent.
531
+ class JobStatus
532
+ include Google::Protobuf::MessageExts
533
+ extend Google::Protobuf::MessageExts::ClassMethods
534
+
535
+ # The job state.
536
+ module State
537
+ # The job state is unknown.
538
+ STATE_UNSPECIFIED = 0
539
+
540
+ # The job is pending; it has been submitted, but is not yet running.
541
+ PENDING = 1
542
+
543
+ # Job has been received by the service and completed initial setup;
544
+ # it will soon be submitted to the cluster.
545
+ SETUP_DONE = 8
546
+
547
+ # The job is running on the cluster.
548
+ RUNNING = 2
549
+
550
+ # A CancelJob request has been received, but is pending.
551
+ CANCEL_PENDING = 3
552
+
553
+ # Transient in-flight resources have been canceled, and the request to
554
+ # cancel the running job has been issued to the cluster.
555
+ CANCEL_STARTED = 7
556
+
557
+ # The job cancellation was successful.
558
+ CANCELLED = 4
559
+
560
+ # The job has completed successfully.
561
+ DONE = 5
562
+
563
+ # The job has completed, but encountered an error.
564
+ ERROR = 6
565
+
566
+ # Job attempt has failed. The detail field contains failure details for
567
+ # this attempt.
568
+ #
569
+ # Applies to restartable jobs only.
570
+ ATTEMPT_FAILURE = 9
571
+ end
572
+
573
+ # The job substate.
574
+ module Substate
575
+ # The job substate is unknown.
576
+ UNSPECIFIED = 0
577
+
578
+ # The Job is submitted to the agent.
579
+ #
580
+ # Applies to RUNNING state.
581
+ SUBMITTED = 1
582
+
583
+ # The Job has been received and is awaiting execution (it may be waiting
584
+ # for a condition to be met). See the "details" field for the reason for
585
+ # the delay.
586
+ #
587
+ # Applies to RUNNING state.
588
+ QUEUED = 2
589
+
590
+ # The agent-reported status is out of date, which may be caused by a
591
+ # loss of communication between the agent and Dataproc. If the
592
+ # agent does not send a timely update, the job will fail.
593
+ #
594
+ # Applies to RUNNING state.
595
+ STALE_STATUS = 3
596
+ end
597
+ end
598
+
599
+ # Encapsulates the full scoping used to reference a job.
600
+ # @!attribute [rw] project_id
601
+ # @return [String]
602
+ # Required. The ID of the Google Cloud Platform project that the job
603
+ # belongs to.
604
+ # @!attribute [rw] job_id
605
+ # @return [String]
606
+ # Optional. The job ID, which must be unique within the project.
607
+ # The ID must contain only letters (a-z, A-Z), numbers (0-9),
608
+ # underscores (_), or hyphens (-). The maximum length is 100 characters.
609
+ #
610
+ # If not specified by the caller, the job ID will be provided by the server.
611
+ class JobReference
612
+ include Google::Protobuf::MessageExts
613
+ extend Google::Protobuf::MessageExts::ClassMethods
614
+ end
615
+
616
+ # A YARN application created by a job. Application information is a subset of
617
+ # <code>org.apache.hadoop.yarn.proto.YarnProtos.ApplicationReportProto</code>.
618
+ #
619
+ # **Beta Feature**: This report is available for testing purposes only. It may
620
+ # be changed before final release.
621
+ # @!attribute [r] name
622
+ # @return [String]
623
+ # Output only. The application name.
624
+ # @!attribute [r] state
625
+ # @return [Google::Cloud::Dataproc::V1beta2::YarnApplication::State]
626
+ # Output only. The application state.
627
+ # @!attribute [r] progress
628
+ # @return [Float]
629
+ # Output only. The numerical progress of the application, from 1 to 100.
630
+ # @!attribute [r] tracking_url
631
+ # @return [String]
632
+ # Output only. The HTTP URL of the ApplicationMaster, HistoryServer, or
633
+ # TimelineServer that provides application-specific information. The URL uses
634
+ # the internal hostname, and requires a proxy server for resolution and,
635
+ # possibly, access.
636
+ class YarnApplication
637
+ include Google::Protobuf::MessageExts
638
+ extend Google::Protobuf::MessageExts::ClassMethods
639
+
640
+ # The application state, corresponding to
641
+ # <code>YarnProtos.YarnApplicationStateProto</code>.
642
+ module State
643
+ # Status is unspecified.
644
+ STATE_UNSPECIFIED = 0
645
+
646
+ # Status is NEW.
647
+ NEW = 1
648
+
649
+ # Status is NEW_SAVING.
650
+ NEW_SAVING = 2
651
+
652
+ # Status is SUBMITTED.
653
+ SUBMITTED = 3
654
+
655
+ # Status is ACCEPTED.
656
+ ACCEPTED = 4
657
+
658
+ # Status is RUNNING.
659
+ RUNNING = 5
660
+
661
+ # Status is FINISHED.
662
+ FINISHED = 6
663
+
664
+ # Status is FAILED.
665
+ FAILED = 7
666
+
667
+ # Status is KILLED.
668
+ KILLED = 8
669
+ end
670
+ end
671
+
672
+ # A Dataproc job resource.
673
+ # @!attribute [rw] reference
674
+ # @return [Google::Cloud::Dataproc::V1beta2::JobReference]
675
+ # Optional. The fully qualified reference to the job, which can be used to
676
+ # obtain the equivalent REST path of the job resource. If this property
677
+ # is not specified when a job is created, the server generates a
678
+ # <code>job_id</code>.
679
+ # @!attribute [rw] placement
680
+ # @return [Google::Cloud::Dataproc::V1beta2::JobPlacement]
681
+ # Required. Job information, including how, when, and where to
682
+ # run the job.
683
+ # @!attribute [rw] hadoop_job
684
+ # @return [Google::Cloud::Dataproc::V1beta2::HadoopJob]
685
+ # Optional. Job is a Hadoop job.
686
+ # @!attribute [rw] spark_job
687
+ # @return [Google::Cloud::Dataproc::V1beta2::SparkJob]
688
+ # Optional. Job is a Spark job.
689
+ # @!attribute [rw] pyspark_job
690
+ # @return [Google::Cloud::Dataproc::V1beta2::PySparkJob]
691
+ # Optional. Job is a PySpark job.
692
+ # @!attribute [rw] hive_job
693
+ # @return [Google::Cloud::Dataproc::V1beta2::HiveJob]
694
+ # Optional. Job is a Hive job.
695
+ # @!attribute [rw] pig_job
696
+ # @return [Google::Cloud::Dataproc::V1beta2::PigJob]
697
+ # Optional. Job is a Pig job.
698
+ # @!attribute [rw] spark_r_job
699
+ # @return [Google::Cloud::Dataproc::V1beta2::SparkRJob]
700
+ # Optional. Job is a SparkR job.
701
+ # @!attribute [rw] spark_sql_job
702
+ # @return [Google::Cloud::Dataproc::V1beta2::SparkSqlJob]
703
+ # Optional. Job is a SparkSql job.
704
+ # @!attribute [rw] presto_job
705
+ # @return [Google::Cloud::Dataproc::V1beta2::PrestoJob]
706
+ # Optional. Job is a Presto job.
707
+ # @!attribute [r] status
708
+ # @return [Google::Cloud::Dataproc::V1beta2::JobStatus]
709
+ # Output only. The job status. Additional application-specific
710
+ # status information may be contained in the <code>type_job</code>
711
+ # and <code>yarn_applications</code> fields.
712
+ # @!attribute [r] status_history
713
+ # @return [Array<Google::Cloud::Dataproc::V1beta2::JobStatus>]
714
+ # Output only. The previous job status.
715
+ # @!attribute [r] yarn_applications
716
+ # @return [Array<Google::Cloud::Dataproc::V1beta2::YarnApplication>]
717
+ # Output only. The collection of YARN applications spun up by this job.
718
+ #
719
+ # **Beta** Feature: This report is available for testing purposes only. It
720
+ # may be changed before final release.
721
+ # @!attribute [r] submitted_by
722
+ # @return [String]
723
+ # Output only. The email address of the user submitting the job. For jobs
724
+ # submitted on the cluster, the address is <code>username@hostname</code>.
725
+ # @!attribute [r] driver_output_resource_uri
726
+ # @return [String]
727
+ # Output only. A URI pointing to the location of the stdout of the job's
728
+ # driver program.
729
+ # @!attribute [r] driver_control_files_uri
730
+ # @return [String]
731
+ # Output only. If present, the location of miscellaneous control files
732
+ # which may be used as part of job setup and handling. If not present,
733
+ # control files may be placed in the same location as `driver_output_uri`.
734
+ # @!attribute [rw] labels
735
+ # @return [Google::Protobuf::Map{String => String}]
736
+ # Optional. The labels to associate with this job.
737
+ # Label **keys** must contain 1 to 63 characters, and must conform to
738
+ # [RFC 1035](https://www.ietf.org/rfc/rfc1035.txt).
739
+ # Label **values** may be empty, but, if present, must contain 1 to 63
740
+ # characters, and must conform to [RFC
741
+ # 1035](https://www.ietf.org/rfc/rfc1035.txt). No more than 32 labels can be
742
+ # associated with a job.
743
+ # @!attribute [rw] scheduling
744
+ # @return [Google::Cloud::Dataproc::V1beta2::JobScheduling]
745
+ # Optional. Job scheduling configuration.
746
+ # @!attribute [r] job_uuid
747
+ # @return [String]
748
+ # Output only. A UUID that uniquely identifies a job within the project
749
+ # over time. This is in contrast to a user-settable reference.job_id that
750
+ # may be reused over time.
751
+ # @!attribute [r] done
752
+ # @return [Boolean]
753
+ # Output only. Indicates whether the job is completed. If the value is `false`,
754
+ # the job is still in progress. If `true`, the job is completed, and
755
+ # `status.state` field will indicate if it was successful, failed,
756
+ # or cancelled.
757
+ class Job
758
+ include Google::Protobuf::MessageExts
759
+ extend Google::Protobuf::MessageExts::ClassMethods
760
+
761
+ # @!attribute [rw] key
762
+ # @return [String]
763
+ # @!attribute [rw] value
764
+ # @return [String]
765
+ class LabelsEntry
766
+ include Google::Protobuf::MessageExts
767
+ extend Google::Protobuf::MessageExts::ClassMethods
768
+ end
769
+ end
770
+
771
+ # Job scheduling options.
772
+ # @!attribute [rw] max_failures_per_hour
773
+ # @return [Integer]
774
+ # Optional. Maximum number of times per hour a driver may be restarted as
775
+ # a result of driver terminating with non-zero code before job is
776
+ # reported failed.
777
+ #
778
+ # A job may be reported as thrashing if driver exits with non-zero code
779
+ # 4 times within 10 minute window.
780
+ #
781
+ # Maximum value is 10.
782
+ class JobScheduling
783
+ include Google::Protobuf::MessageExts
784
+ extend Google::Protobuf::MessageExts::ClassMethods
785
+ end
786
+
787
+ # Job Operation metadata.
788
+ # @!attribute [r] job_id
789
+ # @return [String]
790
+ # Output only. The job id.
791
+ # @!attribute [r] status
792
+ # @return [Google::Cloud::Dataproc::V1beta2::JobStatus]
793
+ # Output only. Most recent job status.
794
+ # @!attribute [r] operation_type
795
+ # @return [String]
796
+ # Output only. Operation type.
797
+ # @!attribute [r] start_time
798
+ # @return [Google::Protobuf::Timestamp]
799
+ # Output only. Job submission time.
800
+ class JobMetadata
801
+ include Google::Protobuf::MessageExts
802
+ extend Google::Protobuf::MessageExts::ClassMethods
803
+ end
804
+
805
+ # A request to submit a job.
806
+ # @!attribute [rw] project_id
807
+ # @return [String]
808
+ # Required. The ID of the Google Cloud Platform project that the job
809
+ # belongs to.
810
+ # @!attribute [rw] region
811
+ # @return [String]
812
+ # Required. The Dataproc region in which to handle the request.
813
+ # @!attribute [rw] job
814
+ # @return [Google::Cloud::Dataproc::V1beta2::Job]
815
+ # Required. The job resource.
816
+ # @!attribute [rw] request_id
817
+ # @return [String]
818
+ # Optional. A unique id used to identify the request. If the server
819
+ # receives two {Google::Cloud::Dataproc::V1beta2::SubmitJobRequest SubmitJobRequest} requests with the same
820
+ # id, then the second request will be ignored and the
821
+ # first {Google::Cloud::Dataproc::V1beta2::Job Job} created and stored in the backend
822
+ # is returned.
823
+ #
824
+ # It is recommended to always set this value to a
825
+ # [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
826
+ #
827
+ # The id must contain only letters (a-z, A-Z), numbers (0-9),
828
+ # underscores (_), and hyphens (-). The maximum length is 40 characters.
829
+ class SubmitJobRequest
830
+ include Google::Protobuf::MessageExts
831
+ extend Google::Protobuf::MessageExts::ClassMethods
832
+ end
833
+
834
+ # A request to get the resource representation for a job in a project.
835
+ # @!attribute [rw] project_id
836
+ # @return [String]
837
+ # Required. The ID of the Google Cloud Platform project that the job
838
+ # belongs to.
839
+ # @!attribute [rw] region
840
+ # @return [String]
841
+ # Required. The Dataproc region in which to handle the request.
842
+ # @!attribute [rw] job_id
843
+ # @return [String]
844
+ # Required. The job ID.
845
+ class GetJobRequest
846
+ include Google::Protobuf::MessageExts
847
+ extend Google::Protobuf::MessageExts::ClassMethods
848
+ end
849
+
850
+ # A request to list jobs in a project.
851
+ # @!attribute [rw] project_id
852
+ # @return [String]
853
+ # Required. The ID of the Google Cloud Platform project that the job
854
+ # belongs to.
855
+ # @!attribute [rw] region
856
+ # @return [String]
857
+ # Required. The Dataproc region in which to handle the request.
858
+ # @!attribute [rw] page_size
859
+ # @return [Integer]
860
+ # Optional. The number of results to return in each response.
861
+ # @!attribute [rw] page_token
862
+ # @return [String]
863
+ # Optional. The page token, returned by a previous call, to request the
864
+ # next page of results.
865
+ # @!attribute [rw] cluster_name
866
+ # @return [String]
867
+ # Optional. If set, the returned jobs list includes only jobs that were
868
+ # submitted to the named cluster.
869
+ # @!attribute [rw] job_state_matcher
870
+ # @return [Google::Cloud::Dataproc::V1beta2::ListJobsRequest::JobStateMatcher]
871
+ # Optional. Specifies enumerated categories of jobs to list.
872
+ # (default = match ALL jobs).
873
+ #
874
+ # If `filter` is provided, `jobStateMatcher` will be ignored.
875
+ # @!attribute [rw] filter
876
+ # @return [String]
877
+ # Optional. A filter constraining the jobs to list. Filters are
878
+ # case-sensitive and have the following syntax:
879
+ #
880
+ # [field = value] AND [field [= value]] ...
881
+ #
882
+ # where **field** is `status.state` or `labels.[KEY]`, and `[KEY]` is a label
883
+ # key. **value** can be `*` to match all values.
884
+ # `status.state` can be either `ACTIVE` or `NON_ACTIVE`.
885
+ # Only the logical `AND` operator is supported; space-separated items are
886
+ # treated as having an implicit `AND` operator.
887
+ #
888
+ # Example filter:
889
+ #
890
+ # status.state = ACTIVE AND labels.env = staging AND labels.starred = *
891
+ class ListJobsRequest
892
+ include Google::Protobuf::MessageExts
893
+ extend Google::Protobuf::MessageExts::ClassMethods
894
+
895
+ # A matcher that specifies categories of job states.
896
+ module JobStateMatcher
897
+ # Match all jobs, regardless of state.
898
+ ALL = 0
899
+
900
+ # Only match jobs in non-terminal states: PENDING, RUNNING, or
901
+ # CANCEL_PENDING.
902
+ ACTIVE = 1
903
+
904
+ # Only match jobs in terminal states: CANCELLED, DONE, or ERROR.
905
+ NON_ACTIVE = 2
906
+ end
907
+ end
908
+
909
+ # A request to update a job.
910
+ # @!attribute [rw] project_id
911
+ # @return [String]
912
+ # Required. The ID of the Google Cloud Platform project that the job
913
+ # belongs to.
914
+ # @!attribute [rw] region
915
+ # @return [String]
916
+ # Required. The Dataproc region in which to handle the request.
917
+ # @!attribute [rw] job_id
918
+ # @return [String]
919
+ # Required. The job ID.
920
+ # @!attribute [rw] job
921
+ # @return [Google::Cloud::Dataproc::V1beta2::Job]
922
+ # Required. The changes to the job.
923
+ # @!attribute [rw] update_mask
924
+ # @return [Google::Protobuf::FieldMask]
925
+ # Required. Specifies the path, relative to <code>Job</code>, of
926
+ # the field to update. For example, to update the labels of a Job the
927
+ # <code>update_mask</code> parameter would be specified as
928
+ # <code>labels</code>, and the `PATCH` request body would specify the new
929
+ # value. <strong>Note:</strong> Currently, <code>labels</code> is the only
930
+ # field that can be updated.
931
+ class UpdateJobRequest
932
+ include Google::Protobuf::MessageExts
933
+ extend Google::Protobuf::MessageExts::ClassMethods
934
+ end
935
+
936
+ # A list of jobs in a project.
937
+ # @!attribute [r] jobs
938
+ # @return [Array<Google::Cloud::Dataproc::V1beta2::Job>]
939
+ # Output only. Jobs list.
940
+ # @!attribute [rw] next_page_token
941
+ # @return [String]
942
+ # Optional. This token is included in the response if there are more results
943
+ # to fetch. To fetch additional results, provide this value as the
944
+ # `page_token` in a subsequent <code>ListJobsRequest</code>.
945
+ class ListJobsResponse
946
+ include Google::Protobuf::MessageExts
947
+ extend Google::Protobuf::MessageExts::ClassMethods
948
+ end
949
+
950
+ # A request to cancel a job.
951
+ # @!attribute [rw] project_id
952
+ # @return [String]
953
+ # Required. The ID of the Google Cloud Platform project that the job
954
+ # belongs to.
955
+ # @!attribute [rw] region
956
+ # @return [String]
957
+ # Required. The Dataproc region in which to handle the request.
958
+ # @!attribute [rw] job_id
959
+ # @return [String]
960
+ # Required. The job ID.
961
+ class CancelJobRequest
962
+ include Google::Protobuf::MessageExts
963
+ extend Google::Protobuf::MessageExts::ClassMethods
964
+ end
965
+
966
+ # A request to delete a job.
967
+ # @!attribute [rw] project_id
968
+ # @return [String]
969
+ # Required. The ID of the Google Cloud Platform project that the job
970
+ # belongs to.
971
+ # @!attribute [rw] region
972
+ # @return [String]
973
+ # Required. The Dataproc region in which to handle the request.
974
+ # @!attribute [rw] job_id
975
+ # @return [String]
976
+ # Required. The job ID.
977
+ class DeleteJobRequest
978
+ include Google::Protobuf::MessageExts
979
+ extend Google::Protobuf::MessageExts::ClassMethods
980
+ end
981
+ end
982
+ end
983
+ end
984
+ end