@flyteorg/flyteidl 1.5.21 → 1.10.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/gen/pb-js/flyteidl.d.ts +6480 -4250
  2. package/gen/pb-js/flyteidl.js +15117 -9702
  3. package/package.json +1 -1
  4. package/protos/buf.lock +7 -0
  5. package/protos/buf.yaml +1 -0
  6. package/protos/docs/admin/admin.rst +4 -4
  7. package/protos/docs/datacatalog/datacatalog.rst +1 -1
  8. package/protos/docs/service/index.rst +1 -1
  9. package/protos/flyteidl/admin/agent.proto +95 -6
  10. package/protos/flyteidl/admin/cluster_assignment.proto +1 -1
  11. package/protos/flyteidl/admin/common.proto +9 -1
  12. package/protos/flyteidl/admin/description_entity.proto +1 -1
  13. package/protos/flyteidl/admin/event.proto +1 -1
  14. package/protos/flyteidl/admin/execution.proto +10 -1
  15. package/protos/flyteidl/admin/launch_plan.proto +12 -4
  16. package/protos/flyteidl/admin/matchable_resource.proto +9 -3
  17. package/protos/flyteidl/admin/node_execution.proto +13 -1
  18. package/protos/flyteidl/admin/notification.proto +1 -1
  19. package/protos/flyteidl/admin/project.proto +7 -1
  20. package/protos/flyteidl/admin/project_attributes.proto +10 -1
  21. package/protos/flyteidl/admin/project_domain_attributes.proto +11 -2
  22. package/protos/flyteidl/admin/schedule.proto +1 -1
  23. package/protos/flyteidl/admin/signal.proto +2 -2
  24. package/protos/flyteidl/admin/task.proto +1 -1
  25. package/protos/flyteidl/admin/task_execution.proto +1 -1
  26. package/protos/flyteidl/admin/version.proto +1 -1
  27. package/protos/flyteidl/admin/workflow.proto +2 -2
  28. package/protos/flyteidl/admin/workflow_attributes.proto +10 -1
  29. package/protos/flyteidl/core/artifact_id.proto +90 -0
  30. package/protos/flyteidl/core/catalog.proto +3 -1
  31. package/protos/flyteidl/core/compiler.proto +12 -1
  32. package/protos/flyteidl/core/condition.proto +1 -1
  33. package/protos/flyteidl/core/dynamic_job.proto +1 -1
  34. package/protos/flyteidl/core/errors.proto +1 -1
  35. package/protos/flyteidl/core/execution.proto +1 -1
  36. package/protos/flyteidl/core/identifier.proto +7 -1
  37. package/protos/flyteidl/core/interface.proto +14 -1
  38. package/protos/flyteidl/core/literals.proto +4 -1
  39. package/protos/flyteidl/core/metrics.proto +15 -1
  40. package/protos/flyteidl/core/security.proto +1 -1
  41. package/protos/flyteidl/core/tasks.proto +33 -1
  42. package/protos/flyteidl/core/types.proto +7 -1
  43. package/protos/flyteidl/core/workflow.proto +28 -1
  44. package/protos/flyteidl/core/workflow_closure.proto +1 -1
  45. package/protos/flyteidl/datacatalog/datacatalog.proto +10 -2
  46. package/protos/flyteidl/event/cloudevents.proto +73 -0
  47. package/protos/flyteidl/event/event.proto +5 -1
  48. package/protos/flyteidl/plugins/array_job.proto +1 -1
  49. package/protos/flyteidl/plugins/dask.proto +1 -1
  50. package/protos/flyteidl/plugins/kubeflow/common.proto +1 -1
  51. package/protos/flyteidl/plugins/kubeflow/mpi.proto +1 -1
  52. package/protos/flyteidl/plugins/kubeflow/pytorch.proto +1 -1
  53. package/protos/flyteidl/plugins/kubeflow/tensorflow.proto +4 -1
  54. package/protos/flyteidl/plugins/mpi.proto +1 -1
  55. package/protos/flyteidl/plugins/presto.proto +1 -1
  56. package/protos/flyteidl/plugins/pytorch.proto +1 -1
  57. package/protos/flyteidl/plugins/qubole.proto +1 -1
  58. package/protos/flyteidl/plugins/ray.proto +7 -1
  59. package/protos/flyteidl/plugins/spark.proto +1 -1
  60. package/protos/flyteidl/plugins/tensorflow.proto +6 -2
  61. package/protos/flyteidl/plugins/waitable.proto +1 -1
  62. package/protos/flyteidl/service/admin.proto +398 -197
  63. package/protos/flyteidl/service/agent.proto +32 -2
  64. package/protos/flyteidl/service/auth.proto +8 -8
  65. package/protos/flyteidl/service/dataproxy.proto +12 -12
  66. package/protos/flyteidl/service/external_plugin_service.proto +1 -2
  67. package/protos/flyteidl/service/identity.proto +5 -5
  68. package/protos/flyteidl/service/signal.proto +30 -23
  69. package/protos/flyteidl/plugins/sagemaker/hyperparameter_tuning_job.proto +0 -83
  70. package/protos/flyteidl/plugins/sagemaker/parameter_ranges.proto +0 -61
  71. package/protos/flyteidl/plugins/sagemaker/training_job.proto +0 -119
@@ -1,10 +1,12 @@
1
1
  syntax = "proto3";
2
2
  package flyteidl.service;
3
3
 
4
- option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/service";
4
+ option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/service";
5
+
6
+ import "google/api/annotations.proto";
5
7
  import "flyteidl/admin/agent.proto";
6
8
 
7
- // AgentService defines an RPC Service that allows propeller to send the request to the agent server.
9
+ // AsyncAgentService defines an RPC Service that allows propeller to send the request to the agent server.
8
10
  service AsyncAgentService {
9
11
  // Send a task create request to the agent server.
10
12
  rpc CreateTask (flyteidl.admin.CreateTaskRequest) returns (flyteidl.admin.CreateTaskResponse){};
@@ -12,4 +14,32 @@ service AsyncAgentService {
12
14
  rpc GetTask (flyteidl.admin.GetTaskRequest) returns (flyteidl.admin.GetTaskResponse){};
13
15
  // Delete the task resource.
14
16
  rpc DeleteTask (flyteidl.admin.DeleteTaskRequest) returns (flyteidl.admin.DeleteTaskResponse){};
17
+
18
+ // GetTaskMetrics returns one or more task execution metrics, if available.
19
+ //
20
+ // Errors include
21
+ // * OutOfRange if metrics are not available for the specified task time range
22
+ // * various other errors
23
+ rpc GetTaskMetrics(flyteidl.admin.GetTaskMetricsRequest) returns (flyteidl.admin.GetTaskMetricsResponse){};
24
+
25
+ // GetTaskLogs returns task execution logs, if available.
26
+ rpc GetTaskLogs(flyteidl.admin.GetTaskLogsRequest) returns (flyteidl.admin.GetTaskLogsResponse){};
27
+ }
28
+
29
+ // AgentMetadataService defines an RPC service that is also served over HTTP via grpc-gateway.
30
+ // This service allows propeller or users to get the metadata of agents.
31
+ service AgentMetadataService {
32
+ // Fetch a :ref:`ref_flyteidl.admin.Agent` definition.
33
+ rpc GetAgent (flyteidl.admin.GetAgentRequest) returns (flyteidl.admin.GetAgentResponse){
34
+ option (google.api.http) = {
35
+ get: "/api/v1/agent/{name}"
36
+ };
37
+ };
38
+
39
+ // Fetch a list of :ref:`ref_flyteidl.admin.Agent` definitions.
40
+ rpc ListAgents (flyteidl.admin.ListAgentsRequest) returns (flyteidl.admin.ListAgentsResponse){
41
+ option (google.api.http) = {
42
+ get: "/api/v1/agents"
43
+ };
44
+ };
15
45
  }
@@ -1,10 +1,10 @@
1
1
  syntax = "proto3";
2
2
  package flyteidl.service;
3
3
 
4
- option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/service";
4
+ option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/service";
5
5
 
6
6
  import "google/api/annotations.proto";
7
- // import "protoc-gen-swagger/options/annotations.proto";
7
+ import "protoc-gen-openapiv2/options/annotations.proto";
8
8
 
9
9
  message OAuth2MetadataRequest {}
10
10
 
@@ -76,9 +76,9 @@ service AuthMetadataService {
76
76
  option (google.api.http) = {
77
77
  get: "/.well-known/oauth-authorization-server"
78
78
  };
79
- // option (grpc.gateway.protoc_gen_swagger.options.openapiv2_operation) = {
80
- // description: "Retrieves OAuth2 authorization server metadata. This endpoint is anonymously accessible."
81
- // };
79
+ option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
80
+ description: "Retrieves OAuth2 authorization server metadata. This endpoint is anonymously accessible."
81
+ };
82
82
  }
83
83
 
84
84
  // Anonymously accessible. Retrieves the client information clients should use when initiating OAuth2 authorization
@@ -87,8 +87,8 @@ service AuthMetadataService {
87
87
  option (google.api.http) = {
88
88
  get: "/config/v1/flyte_client"
89
89
  };
90
- // option (grpc.gateway.protoc_gen_swagger.options.openapiv2_operation) = {
91
- // description: "Retrieves public flyte client info. This endpoint is anonymously accessible."
92
- // };
90
+ option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
91
+ description: "Retrieves public flyte client info. This endpoint is anonymously accessible."
92
+ };
93
93
  }
94
94
  }
@@ -1,10 +1,10 @@
1
1
  syntax = "proto3";
2
2
  package flyteidl.service;
3
3
 
4
- option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/service";
4
+ option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/service";
5
5
 
6
6
  import "google/api/annotations.proto";
7
- // import "protoc-gen-swagger/options/annotations.proto";
7
+ import "protoc-gen-openapiv2/options/annotations.proto";
8
8
  import "google/protobuf/duration.proto";
9
9
  import "google/protobuf/timestamp.proto";
10
10
  import "flyteidl/core/identifier.proto";
@@ -129,7 +129,7 @@ message PreSignedURLs {
129
129
  // General request artifact to retrieve data from a Flyte artifact url.
130
130
  message GetDataRequest {
131
131
  // A unique identifier in the form of flyte://<something> that uniquely, for a given Flyte
132
- // backend, identifies a Flyte artifact ([i]nput, [o]utput, flyte [d]eck, etc.).
132
+ // backend, identifies a Flyte artifact ([i]nput, [o]output, flyte [d]eck, etc.).
133
133
  // e.g. flyte://v1/proj/development/execid/n2/0/i (for 0th task execution attempt input)
134
134
  // flyte://v1/proj/development/execid/n2/i (for node execution input)
135
135
  // flyte://v1/proj/development/execid/n2/o/o3 (the o3 output of the second node)
@@ -158,9 +158,9 @@ service DataProxyService {
158
158
  post: "/api/v1/dataproxy/artifact_urn"
159
159
  body: "*"
160
160
  };
161
- // option (grpc.gateway.protoc_gen_swagger.options.openapiv2_operation) = {
162
- // description: "Creates a write-only http location that is accessible for tasks at runtime."
163
- // };
161
+ option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
162
+ description: "Creates a write-only http location that is accessible for tasks at runtime."
163
+ };
164
164
  }
165
165
 
166
166
  // CreateDownloadLocation creates a signed url to download artifacts.
@@ -169,9 +169,9 @@ service DataProxyService {
169
169
  option (google.api.http) = {
170
170
  get: "/api/v1/dataproxy/artifact_urn"
171
171
  };
172
- // option (grpc.gateway.protoc_gen_swagger.options.openapiv2_operation) = {
173
- // description: "Deprecated: Please use CreateDownloadLink instead. Creates a read-only http location that is accessible for tasks at runtime."
174
- // };
172
+ option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
173
+ description: "Deprecated: Please use CreateDownloadLink instead. Creates a read-only http location that is accessible for tasks at runtime."
174
+ };
175
175
  }
176
176
 
177
177
  // CreateDownloadLocation creates a signed url to download artifacts.
@@ -180,9 +180,9 @@ service DataProxyService {
180
180
  post: "/api/v1/dataproxy/artifact_link"
181
181
  body: "*"
182
182
  };
183
- // option (grpc.gateway.protoc_gen_swagger.options.openapiv2_operation) = {
184
- // description: "Creates a read-only http location that is accessible for tasks at runtime."
185
- // };
183
+ option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
184
+ description: "Creates a read-only http location that is accessible for tasks at runtime."
185
+ };
186
186
  }
187
187
 
188
188
  rpc GetData (GetDataRequest) returns (GetDataResponse) {
@@ -1,10 +1,9 @@
1
1
  syntax = "proto3";
2
2
  package flyteidl.service;
3
3
 
4
- option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/service";
4
+ option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/service";
5
5
  import "flyteidl/core/literals.proto";
6
6
  import "flyteidl/core/tasks.proto";
7
- import "flyteidl/core/interface.proto";
8
7
 
9
8
  // ExternalPluginService defines an RPC Service that allows propeller to send the request to the backend plugin server.
10
9
  service ExternalPluginService {
@@ -1,11 +1,11 @@
1
1
  syntax = "proto3";
2
2
  package flyteidl.service;
3
3
 
4
- option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/service";
4
+ option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/service";
5
5
 
6
6
  import "google/api/annotations.proto";
7
7
  import "google/protobuf/struct.proto";
8
- // import "protoc-gen-swagger/options/annotations.proto";
8
+ import "protoc-gen-openapiv2/options/annotations.proto";
9
9
 
10
10
  message UserInfoRequest {}
11
11
 
@@ -44,8 +44,8 @@ service IdentityService {
44
44
  option (google.api.http) = {
45
45
  get: "/me"
46
46
  };
47
- // option (grpc.gateway.protoc_gen_swagger.options.openapiv2_operation) = {
48
- // description: "Retrieves authenticated identity info."
49
- // };
47
+ option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
48
+ description: "Retrieves authenticated identity info."
49
+ };
50
50
  }
51
51
  }
@@ -1,11 +1,11 @@
1
1
  syntax = "proto3";
2
2
  package flyteidl.service;
3
3
 
4
- option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/service";
4
+ option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/service";
5
5
 
6
6
  import "google/api/annotations.proto";
7
7
  import "flyteidl/admin/signal.proto";
8
- // import "protoc-gen-swagger/options/annotations.proto";
8
+ import "protoc-gen-openapiv2/options/annotations.proto";
9
9
 
10
10
  // SignalService defines an RPC Service that may create, update, and retrieve signal(s).
11
11
  service SignalService {
@@ -15,19 +15,22 @@ service SignalService {
15
15
  // a signal, meaning the first call will create the signal and all subsequent calls will
16
16
  // fetch the existing signal. This is only useful during Flyte Workflow execution and therefore
17
17
  // is not exposed to mitigate unintended behavior.
18
- // option (grpc.gateway.protoc_gen_swagger.options.openapiv2_operation) = {
19
- // description: "Retrieve a signal, creating it if it does not exist."
20
- // };
18
+ option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
19
+ description: "Retrieve a signal, creating it if it does not exist."
20
+ };
21
21
  }
22
22
 
23
23
  // Fetch a list of :ref:`ref_flyteidl.admin.Signal` definitions.
24
24
  rpc ListSignals (flyteidl.admin.SignalListRequest) returns (flyteidl.admin.SignalList) {
25
25
  option (google.api.http) = {
26
26
  get: "/api/v1/signals/{workflow_execution_id.project}/{workflow_execution_id.domain}/{workflow_execution_id.name}"
27
+ additional_bindings: {
28
+ get: "/api/v1/signals/org/{workflow_execution_id.org}/{workflow_execution_id.project}/{workflow_execution_id.domain}/{workflow_execution_id.name}"
29
+ }
30
+ };
31
+ option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
32
+ description: "Fetch existing signal definitions matching the input signal id filters."
27
33
  };
28
- // option (grpc.gateway.protoc_gen_swagger.options.openapiv2_operation) = {
29
- // description: "Fetch existing signal definitions matching the input signal id filters."
30
- // };
31
34
  }
32
35
 
33
36
  // Sets the value on a :ref:`ref_flyteidl.admin.Signal` definition
@@ -35,21 +38,25 @@ service SignalService {
35
38
  option (google.api.http) = {
36
39
  post: "/api/v1/signals"
37
40
  body: "*"
41
+ additional_bindings: {
42
+ post: "/api/v1/signals/org/{id.execution_id.org}"
43
+ body: "*"
44
+ }
45
+ };
46
+ option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
47
+ description: "Set a signal value."
48
+ responses: {
49
+ key: "400"
50
+ value: {
51
+ description: "Returned for bad request that may have failed validation."
52
+ }
53
+ }
54
+ responses: {
55
+ key: "409"
56
+ value: {
57
+ description: "Returned for a request that references an identical entity that has already been registered."
58
+ }
59
+ }
38
60
  };
39
- // option (grpc.gateway.protoc_gen_swagger.options.openapiv2_operation) = {
40
- // description: "Set a signal value."
41
- // responses: {
42
- // key: "400"
43
- // value: {
44
- // description: "Returned for bad request that may have failed validation."
45
- // }
46
- // }
47
- // responses: {
48
- // key: "409"
49
- // value: {
50
- // description: "Returned for a request that references an identical entity that has already been registered."
51
- // }
52
- // }
53
- // };
54
61
  }
55
62
  }
@@ -1,83 +0,0 @@
1
- syntax = "proto3";
2
-
3
- package flyteidl.plugins.sagemaker;
4
-
5
- option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/plugins";
6
-
7
- import "flyteidl/plugins/sagemaker/parameter_ranges.proto";
8
- import "flyteidl/plugins/sagemaker/training_job.proto";
9
-
10
- // A pass-through for SageMaker's hyperparameter tuning job
11
- message HyperparameterTuningJob {
12
- // The underlying training job that the hyperparameter tuning job will launch during the process
13
- TrainingJob training_job = 1;
14
-
15
- // The maximum number of training jobs that an hpo job can launch. For resource limit purpose.
16
- // https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_ResourceLimits.html
17
- int64 max_number_of_training_jobs = 2;
18
-
19
- // The maximum number of concurrent training job that an hpo job can launch
20
- // https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_ResourceLimits.html
21
- int64 max_parallel_training_jobs = 3;
22
- }
23
-
24
- // HyperparameterTuningObjectiveType determines the direction of the tuning of the Hyperparameter Tuning Job
25
- // with respect to the specified metric.
26
- message HyperparameterTuningObjectiveType {
27
- enum Value {
28
- MINIMIZE = 0;
29
- MAXIMIZE = 1;
30
- }
31
- }
32
-
33
- // The target metric and the objective of the hyperparameter tuning.
34
- // https://docs.aws.amazon.com/sagemaker/latest/dg/automatic-model-tuning-define-metrics.html
35
- message HyperparameterTuningObjective {
36
-
37
- // HyperparameterTuningObjectiveType determines the direction of the tuning of the Hyperparameter Tuning Job
38
- // with respect to the specified metric.
39
- HyperparameterTuningObjectiveType.Value objective_type = 1;
40
-
41
- // The target metric name, which is the user-defined name of the metric specified in the
42
- // training job's algorithm specification
43
- string metric_name = 2;
44
- }
45
-
46
-
47
- // Setting the strategy used when searching in the hyperparameter space
48
- // Refer this doc for more details:
49
- // https://aws.amazon.com/blogs/machine-learning/amazon-sagemaker-automatic-model-tuning-now-supports-random-search-and-hyperparameter-scaling/
50
- message HyperparameterTuningStrategy {
51
- enum Value {
52
- BAYESIAN = 0;
53
- RANDOM = 1;
54
- }
55
- }
56
-
57
- // When the training jobs launched by the hyperparameter tuning job are not improving significantly,
58
- // a hyperparameter tuning job can be stopping early.
59
- // Note that there's only a subset of built-in algorithms that supports early stopping.
60
- // see: https://docs.aws.amazon.com/sagemaker/latest/dg/automatic-model-tuning-early-stopping.html
61
- message TrainingJobEarlyStoppingType {
62
- enum Value {
63
- OFF = 0;
64
- AUTO = 1;
65
- }
66
- }
67
-
68
- // The specification of the hyperparameter tuning process
69
- // https://docs.aws.amazon.com/sagemaker/latest/dg/automatic-model-tuning-ex-tuning-job.html#automatic-model-tuning-ex-low-tuning-config
70
- message HyperparameterTuningJobConfig {
71
- // ParameterRanges is a map that maps hyperparameter name to the corresponding hyperparameter range
72
- ParameterRanges hyperparameter_ranges = 1;
73
-
74
- // Setting the strategy used when searching in the hyperparameter space
75
- HyperparameterTuningStrategy.Value tuning_strategy = 2;
76
-
77
- // The target metric and the objective of the hyperparameter tuning.
78
- HyperparameterTuningObjective tuning_objective = 3;
79
-
80
- // When the training jobs launched by the hyperparameter tuning job are not improving significantly,
81
- // a hyperparameter tuning job can be stopping early.
82
- TrainingJobEarlyStoppingType.Value training_job_early_stopping_type = 4;
83
- }
@@ -1,61 +0,0 @@
1
- syntax = "proto3";
2
-
3
- package flyteidl.plugins.sagemaker;
4
-
5
- option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/plugins";
6
-
7
- // HyperparameterScalingType defines the way to increase or decrease the value of the hyperparameter
8
- // For details, refer to: https://docs.aws.amazon.com/sagemaker/latest/dg/automatic-model-tuning-define-ranges.html
9
- // See examples of these scaling type, refer to: https://aws.amazon.com/blogs/machine-learning/amazon-sagemaker-automatic-model-tuning-now-supports-random-search-and-hyperparameter-scaling/
10
- message HyperparameterScalingType {
11
- enum Value {
12
- AUTO = 0;
13
- LINEAR = 1;
14
- LOGARITHMIC = 2;
15
- REVERSELOGARITHMIC = 3;
16
- }
17
- }
18
-
19
- // ContinuousParameterRange refers to a continuous range of hyperparameter values, allowing
20
- // users to specify the search space of a floating-point hyperparameter
21
- // https://docs.aws.amazon.com/sagemaker/latest/dg/automatic-model-tuning-define-ranges.html
22
- message ContinuousParameterRange {
23
- double max_value = 1;
24
- double min_value = 2;
25
- HyperparameterScalingType.Value scaling_type = 3;
26
- }
27
-
28
- // IntegerParameterRange refers to a discrete range of hyperparameter values, allowing
29
- // users to specify the search space of an integer hyperparameter
30
- // https://docs.aws.amazon.com/sagemaker/latest/dg/automatic-model-tuning-define-ranges.html
31
- message IntegerParameterRange {
32
- int64 max_value = 1;
33
- int64 min_value = 2;
34
- HyperparameterScalingType.Value scaling_type = 3;
35
- }
36
-
37
- // ContinuousParameterRange refers to a continuous range of hyperparameter values, allowing
38
- // users to specify the search space of a floating-point hyperparameter
39
- // https://docs.aws.amazon.com/sagemaker/latest/dg/automatic-model-tuning-define-ranges.html
40
- message CategoricalParameterRange {
41
- repeated string values = 1;
42
- }
43
-
44
-
45
- // ParameterRangeOneOf describes a single ParameterRange, which is a one-of structure that can be one of
46
- // the three possible types: ContinuousParameterRange, IntegerParameterRange, and CategoricalParameterRange.
47
- // This one-of structure in Flyte enables specifying a Parameter in a type-safe manner
48
- // See: https://docs.aws.amazon.com/sagemaker/latest/dg/automatic-model-tuning-define-ranges.html
49
- message ParameterRangeOneOf {
50
- oneof parameter_range_type {
51
- ContinuousParameterRange continuous_parameter_range = 1;
52
- IntegerParameterRange integer_parameter_range = 2;
53
- CategoricalParameterRange categorical_parameter_range = 3;
54
- }
55
- }
56
-
57
- // ParameterRanges is a map that maps hyperparameter name to the corresponding hyperparameter range
58
- // https://docs.aws.amazon.com/sagemaker/latest/dg/automatic-model-tuning-define-ranges.html
59
- message ParameterRanges {
60
- map<string, ParameterRangeOneOf> parameter_range_map = 1;
61
- }
@@ -1,119 +0,0 @@
1
- syntax = "proto3";
2
-
3
- package flyteidl.plugins.sagemaker;
4
-
5
- import "google/protobuf/duration.proto";
6
-
7
- option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/plugins";
8
-
9
- // The input mode that the algorithm supports. When using the File input mode, SageMaker downloads
10
- // the training data from S3 to the provisioned ML storage Volume, and mounts the directory to docker
11
- // volume for training container. When using Pipe input mode, Amazon SageMaker streams data directly
12
- // from S3 to the container.
13
- // See: https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_AlgorithmSpecification.html
14
- // For the input modes that different SageMaker algorithms support, see:
15
- // https://docs.aws.amazon.com/sagemaker/latest/dg/sagemaker-algo-docker-registry-paths.html
16
- message InputMode {
17
- enum Value {
18
- FILE = 0;
19
- PIPE = 1;
20
- }
21
- }
22
-
23
- // The algorithm name is used for deciding which pre-built image to point to.
24
- // This is only required for use cases where SageMaker's built-in algorithm mode is used.
25
- // While we currently only support a subset of the algorithms, more will be added to the list.
26
- // See: https://docs.aws.amazon.com/sagemaker/latest/dg/algos.html
27
- message AlgorithmName {
28
- enum Value {
29
- CUSTOM = 0;
30
- XGBOOST = 1;
31
- }
32
- }
33
-
34
-
35
- // Specifies the type of file for input data. Different SageMaker built-in algorithms require different file types of input data
36
- // See https://docs.aws.amazon.com/sagemaker/latest/dg/cdf-training.html
37
- // https://docs.aws.amazon.com/sagemaker/latest/dg/sagemaker-algo-docker-registry-paths.html
38
- message InputContentType {
39
- enum Value {
40
- TEXT_CSV = 0;
41
- }
42
- }
43
-
44
- // Specifies a metric that the training algorithm writes to stderr or stdout.
45
- // This object is a pass-through.
46
- // See this for details: https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_MetricDefinition.html
47
- message MetricDefinition {
48
- // User-defined name of the metric
49
- string name = 1;
50
- // SageMaker hyperparameter tuning parses your algorithm’s stdout and stderr streams to find algorithm metrics
51
- string regex = 2;
52
- }
53
-
54
-
55
- // Specifies the training algorithm to be used in the training job
56
- // This object is mostly a pass-through, with a couple of exceptions include: (1) in Flyte, users don't need to specify
57
- // TrainingImage; either use the built-in algorithm mode by using Flytekit's Simple Training Job and specifying an algorithm
58
- // name and an algorithm version or (2) when users want to supply custom algorithms they should set algorithm_name field to
59
- // CUSTOM. In this case, the value of the algorithm_version field has no effect
60
- // For pass-through use cases: refer to this AWS official document for more details
61
- // https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_AlgorithmSpecification.html
62
- message AlgorithmSpecification {
63
- // The input mode can be either PIPE or FILE
64
- InputMode.Value input_mode = 1;
65
-
66
- // The algorithm name is used for deciding which pre-built image to point to
67
- AlgorithmName.Value algorithm_name = 2;
68
- // The algorithm version field is used for deciding which pre-built image to point to
69
- // This is only needed for use cases where SageMaker's built-in algorithm mode is chosen
70
- string algorithm_version = 3;
71
-
72
- // A list of metric definitions for SageMaker to evaluate/track on the progress of the training job
73
- // See this: https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_AlgorithmSpecification.html
74
- // and this: https://docs.aws.amazon.com/sagemaker/latest/dg/automatic-model-tuning-define-metrics.html
75
- repeated MetricDefinition metric_definitions = 4;
76
-
77
- // The content type of the input
78
- // See https://docs.aws.amazon.com/sagemaker/latest/dg/cdf-training.html
79
- // https://docs.aws.amazon.com/sagemaker/latest/dg/sagemaker-algo-docker-registry-paths.html
80
- InputContentType.Value input_content_type = 5;
81
- }
82
-
83
- // When enabling distributed training on a training job, the user should use this message to tell Flyte and SageMaker
84
- // what kind of distributed protocol he/she wants to use to distribute the work.
85
- message DistributedProtocol {
86
- enum Value {
87
- // Use this value if the user wishes to use framework-native distributed training interfaces.
88
- // If this value is used, Flyte won't configure SageMaker to initialize unnecessary components such as
89
- // OpenMPI or Parameter Server.
90
- UNSPECIFIED = 0;
91
- // Use this value if the user wishes to use MPI as the underlying protocol for her distributed training job
92
- // MPI is a framework-agnostic distributed protocol. It has multiple implementations. Currently, we have only
93
- // tested the OpenMPI implementation, which is the recommended implementation for Horovod.
94
- MPI = 1;
95
- }
96
- }
97
-
98
- // TrainingJobResourceConfig is a pass-through, specifying the instance type to use for the training job, the
99
- // number of instances to launch, and the size of the ML storage volume the user wants to provision
100
- // Refer to SageMaker official doc for more details: https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_CreateTrainingJob.html
101
- message TrainingJobResourceConfig {
102
- // The number of ML compute instances to use. For distributed training, provide a value greater than 1.
103
- int64 instance_count = 1;
104
- // The ML compute instance type
105
- string instance_type = 2;
106
- // The size of the ML storage volume that you want to provision.
107
- int64 volume_size_in_gb = 3;
108
- // When users specify an instance_count > 1, Flyte will try to configure SageMaker to enable distributed training.
109
- // If the users wish to use framework-agnostic distributed protocol such as MPI or Parameter Server, this
110
- // field should be set to the corresponding enum value
111
- DistributedProtocol.Value distributed_protocol = 4;
112
- }
113
-
114
- // The spec of a training job. This is mostly a pass-through object
115
- // https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_CreateTrainingJob.html
116
- message TrainingJob {
117
- AlgorithmSpecification algorithm_specification = 1;
118
- TrainingJobResourceConfig training_job_resource_config = 2;
119
- }