@flyteorg/flyteidl 1.5.21 → 1.10.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/gen/pb-js/flyteidl.d.ts +6480 -4250
- package/gen/pb-js/flyteidl.js +15117 -9702
- package/package.json +1 -1
- package/protos/buf.lock +7 -0
- package/protos/buf.yaml +1 -0
- package/protos/docs/admin/admin.rst +4 -4
- package/protos/docs/datacatalog/datacatalog.rst +1 -1
- package/protos/docs/service/index.rst +1 -1
- package/protos/flyteidl/admin/agent.proto +95 -6
- package/protos/flyteidl/admin/cluster_assignment.proto +1 -1
- package/protos/flyteidl/admin/common.proto +9 -1
- package/protos/flyteidl/admin/description_entity.proto +1 -1
- package/protos/flyteidl/admin/event.proto +1 -1
- package/protos/flyteidl/admin/execution.proto +10 -1
- package/protos/flyteidl/admin/launch_plan.proto +12 -4
- package/protos/flyteidl/admin/matchable_resource.proto +9 -3
- package/protos/flyteidl/admin/node_execution.proto +13 -1
- package/protos/flyteidl/admin/notification.proto +1 -1
- package/protos/flyteidl/admin/project.proto +7 -1
- package/protos/flyteidl/admin/project_attributes.proto +10 -1
- package/protos/flyteidl/admin/project_domain_attributes.proto +11 -2
- package/protos/flyteidl/admin/schedule.proto +1 -1
- package/protos/flyteidl/admin/signal.proto +2 -2
- package/protos/flyteidl/admin/task.proto +1 -1
- package/protos/flyteidl/admin/task_execution.proto +1 -1
- package/protos/flyteidl/admin/version.proto +1 -1
- package/protos/flyteidl/admin/workflow.proto +2 -2
- package/protos/flyteidl/admin/workflow_attributes.proto +10 -1
- package/protos/flyteidl/core/artifact_id.proto +90 -0
- package/protos/flyteidl/core/catalog.proto +3 -1
- package/protos/flyteidl/core/compiler.proto +12 -1
- package/protos/flyteidl/core/condition.proto +1 -1
- package/protos/flyteidl/core/dynamic_job.proto +1 -1
- package/protos/flyteidl/core/errors.proto +1 -1
- package/protos/flyteidl/core/execution.proto +1 -1
- package/protos/flyteidl/core/identifier.proto +7 -1
- package/protos/flyteidl/core/interface.proto +14 -1
- package/protos/flyteidl/core/literals.proto +4 -1
- package/protos/flyteidl/core/metrics.proto +15 -1
- package/protos/flyteidl/core/security.proto +1 -1
- package/protos/flyteidl/core/tasks.proto +33 -1
- package/protos/flyteidl/core/types.proto +7 -1
- package/protos/flyteidl/core/workflow.proto +28 -1
- package/protos/flyteidl/core/workflow_closure.proto +1 -1
- package/protos/flyteidl/datacatalog/datacatalog.proto +10 -2
- package/protos/flyteidl/event/cloudevents.proto +73 -0
- package/protos/flyteidl/event/event.proto +5 -1
- package/protos/flyteidl/plugins/array_job.proto +1 -1
- package/protos/flyteidl/plugins/dask.proto +1 -1
- package/protos/flyteidl/plugins/kubeflow/common.proto +1 -1
- package/protos/flyteidl/plugins/kubeflow/mpi.proto +1 -1
- package/protos/flyteidl/plugins/kubeflow/pytorch.proto +1 -1
- package/protos/flyteidl/plugins/kubeflow/tensorflow.proto +4 -1
- package/protos/flyteidl/plugins/mpi.proto +1 -1
- package/protos/flyteidl/plugins/presto.proto +1 -1
- package/protos/flyteidl/plugins/pytorch.proto +1 -1
- package/protos/flyteidl/plugins/qubole.proto +1 -1
- package/protos/flyteidl/plugins/ray.proto +7 -1
- package/protos/flyteidl/plugins/spark.proto +1 -1
- package/protos/flyteidl/plugins/tensorflow.proto +6 -2
- package/protos/flyteidl/plugins/waitable.proto +1 -1
- package/protos/flyteidl/service/admin.proto +398 -197
- package/protos/flyteidl/service/agent.proto +32 -2
- package/protos/flyteidl/service/auth.proto +8 -8
- package/protos/flyteidl/service/dataproxy.proto +12 -12
- package/protos/flyteidl/service/external_plugin_service.proto +1 -2
- package/protos/flyteidl/service/identity.proto +5 -5
- package/protos/flyteidl/service/signal.proto +30 -23
- package/protos/flyteidl/plugins/sagemaker/hyperparameter_tuning_job.proto +0 -83
- package/protos/flyteidl/plugins/sagemaker/parameter_ranges.proto +0 -61
- package/protos/flyteidl/plugins/sagemaker/training_job.proto +0 -119
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
syntax = "proto3";
|
|
2
2
|
package flyteidl.service;
|
|
3
3
|
|
|
4
|
-
option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/service";
|
|
4
|
+
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/service";
|
|
5
|
+
|
|
6
|
+
import "google/api/annotations.proto";
|
|
5
7
|
import "flyteidl/admin/agent.proto";
|
|
6
8
|
|
|
7
|
-
//
|
|
9
|
+
// AsyncAgentService defines an RPC Service that allows propeller to send the request to the agent server.
|
|
8
10
|
service AsyncAgentService {
|
|
9
11
|
// Send a task create request to the agent server.
|
|
10
12
|
rpc CreateTask (flyteidl.admin.CreateTaskRequest) returns (flyteidl.admin.CreateTaskResponse){};
|
|
@@ -12,4 +14,32 @@ service AsyncAgentService {
|
|
|
12
14
|
rpc GetTask (flyteidl.admin.GetTaskRequest) returns (flyteidl.admin.GetTaskResponse){};
|
|
13
15
|
// Delete the task resource.
|
|
14
16
|
rpc DeleteTask (flyteidl.admin.DeleteTaskRequest) returns (flyteidl.admin.DeleteTaskResponse){};
|
|
17
|
+
|
|
18
|
+
// GetTaskMetrics returns one or more task execution metrics, if available.
|
|
19
|
+
//
|
|
20
|
+
// Errors include
|
|
21
|
+
// * OutOfRange if metrics are not available for the specified task time range
|
|
22
|
+
// * various other errors
|
|
23
|
+
rpc GetTaskMetrics(flyteidl.admin.GetTaskMetricsRequest) returns (flyteidl.admin.GetTaskMetricsResponse){};
|
|
24
|
+
|
|
25
|
+
// GetTaskLogs returns task execution logs, if available.
|
|
26
|
+
rpc GetTaskLogs(flyteidl.admin.GetTaskLogsRequest) returns (flyteidl.admin.GetTaskLogsResponse){};
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// AgentMetadataService defines an RPC service that is also served over HTTP via grpc-gateway.
|
|
30
|
+
// This service allows propeller or users to get the metadata of agents.
|
|
31
|
+
service AgentMetadataService {
|
|
32
|
+
// Fetch a :ref:`ref_flyteidl.admin.Agent` definition.
|
|
33
|
+
rpc GetAgent (flyteidl.admin.GetAgentRequest) returns (flyteidl.admin.GetAgentResponse){
|
|
34
|
+
option (google.api.http) = {
|
|
35
|
+
get: "/api/v1/agent/{name}"
|
|
36
|
+
};
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
// Fetch a list of :ref:`ref_flyteidl.admin.Agent` definitions.
|
|
40
|
+
rpc ListAgents (flyteidl.admin.ListAgentsRequest) returns (flyteidl.admin.ListAgentsResponse){
|
|
41
|
+
option (google.api.http) = {
|
|
42
|
+
get: "/api/v1/agents"
|
|
43
|
+
};
|
|
44
|
+
};
|
|
15
45
|
}
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
syntax = "proto3";
|
|
2
2
|
package flyteidl.service;
|
|
3
3
|
|
|
4
|
-
option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/service";
|
|
4
|
+
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/service";
|
|
5
5
|
|
|
6
6
|
import "google/api/annotations.proto";
|
|
7
|
-
|
|
7
|
+
import "protoc-gen-openapiv2/options/annotations.proto";
|
|
8
8
|
|
|
9
9
|
message OAuth2MetadataRequest {}
|
|
10
10
|
|
|
@@ -76,9 +76,9 @@ service AuthMetadataService {
|
|
|
76
76
|
option (google.api.http) = {
|
|
77
77
|
get: "/.well-known/oauth-authorization-server"
|
|
78
78
|
};
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
79
|
+
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
|
|
80
|
+
description: "Retrieves OAuth2 authorization server metadata. This endpoint is anonymously accessible."
|
|
81
|
+
};
|
|
82
82
|
}
|
|
83
83
|
|
|
84
84
|
// Anonymously accessible. Retrieves the client information clients should use when initiating OAuth2 authorization
|
|
@@ -87,8 +87,8 @@ service AuthMetadataService {
|
|
|
87
87
|
option (google.api.http) = {
|
|
88
88
|
get: "/config/v1/flyte_client"
|
|
89
89
|
};
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
90
|
+
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
|
|
91
|
+
description: "Retrieves public flyte client info. This endpoint is anonymously accessible."
|
|
92
|
+
};
|
|
93
93
|
}
|
|
94
94
|
}
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
syntax = "proto3";
|
|
2
2
|
package flyteidl.service;
|
|
3
3
|
|
|
4
|
-
option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/service";
|
|
4
|
+
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/service";
|
|
5
5
|
|
|
6
6
|
import "google/api/annotations.proto";
|
|
7
|
-
|
|
7
|
+
import "protoc-gen-openapiv2/options/annotations.proto";
|
|
8
8
|
import "google/protobuf/duration.proto";
|
|
9
9
|
import "google/protobuf/timestamp.proto";
|
|
10
10
|
import "flyteidl/core/identifier.proto";
|
|
@@ -129,7 +129,7 @@ message PreSignedURLs {
|
|
|
129
129
|
// General request artifact to retrieve data from a Flyte artifact url.
|
|
130
130
|
message GetDataRequest {
|
|
131
131
|
// A unique identifier in the form of flyte://<something> that uniquely, for a given Flyte
|
|
132
|
-
// backend, identifies a Flyte artifact ([i]nput, [o]
|
|
132
|
+
// backend, identifies a Flyte artifact ([i]nput, [o]output, flyte [d]eck, etc.).
|
|
133
133
|
// e.g. flyte://v1/proj/development/execid/n2/0/i (for 0th task execution attempt input)
|
|
134
134
|
// flyte://v1/proj/development/execid/n2/i (for node execution input)
|
|
135
135
|
// flyte://v1/proj/development/execid/n2/o/o3 (the o3 output of the second node)
|
|
@@ -158,9 +158,9 @@ service DataProxyService {
|
|
|
158
158
|
post: "/api/v1/dataproxy/artifact_urn"
|
|
159
159
|
body: "*"
|
|
160
160
|
};
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
161
|
+
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
|
|
162
|
+
description: "Creates a write-only http location that is accessible for tasks at runtime."
|
|
163
|
+
};
|
|
164
164
|
}
|
|
165
165
|
|
|
166
166
|
// CreateDownloadLocation creates a signed url to download artifacts.
|
|
@@ -169,9 +169,9 @@ service DataProxyService {
|
|
|
169
169
|
option (google.api.http) = {
|
|
170
170
|
get: "/api/v1/dataproxy/artifact_urn"
|
|
171
171
|
};
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
172
|
+
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
|
|
173
|
+
description: "Deprecated: Please use CreateDownloadLink instead. Creates a read-only http location that is accessible for tasks at runtime."
|
|
174
|
+
};
|
|
175
175
|
}
|
|
176
176
|
|
|
177
177
|
// CreateDownloadLocation creates a signed url to download artifacts.
|
|
@@ -180,9 +180,9 @@ service DataProxyService {
|
|
|
180
180
|
post: "/api/v1/dataproxy/artifact_link"
|
|
181
181
|
body: "*"
|
|
182
182
|
};
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
183
|
+
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
|
|
184
|
+
description: "Creates a read-only http location that is accessible for tasks at runtime."
|
|
185
|
+
};
|
|
186
186
|
}
|
|
187
187
|
|
|
188
188
|
rpc GetData (GetDataRequest) returns (GetDataResponse) {
|
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
syntax = "proto3";
|
|
2
2
|
package flyteidl.service;
|
|
3
3
|
|
|
4
|
-
option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/service";
|
|
4
|
+
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/service";
|
|
5
5
|
import "flyteidl/core/literals.proto";
|
|
6
6
|
import "flyteidl/core/tasks.proto";
|
|
7
|
-
import "flyteidl/core/interface.proto";
|
|
8
7
|
|
|
9
8
|
// ExternalPluginService defines an RPC Service that allows propeller to send the request to the backend plugin server.
|
|
10
9
|
service ExternalPluginService {
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
syntax = "proto3";
|
|
2
2
|
package flyteidl.service;
|
|
3
3
|
|
|
4
|
-
option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/service";
|
|
4
|
+
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/service";
|
|
5
5
|
|
|
6
6
|
import "google/api/annotations.proto";
|
|
7
7
|
import "google/protobuf/struct.proto";
|
|
8
|
-
|
|
8
|
+
import "protoc-gen-openapiv2/options/annotations.proto";
|
|
9
9
|
|
|
10
10
|
message UserInfoRequest {}
|
|
11
11
|
|
|
@@ -44,8 +44,8 @@ service IdentityService {
|
|
|
44
44
|
option (google.api.http) = {
|
|
45
45
|
get: "/me"
|
|
46
46
|
};
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
47
|
+
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
|
|
48
|
+
description: "Retrieves authenticated identity info."
|
|
49
|
+
};
|
|
50
50
|
}
|
|
51
51
|
}
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
syntax = "proto3";
|
|
2
2
|
package flyteidl.service;
|
|
3
3
|
|
|
4
|
-
option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/service";
|
|
4
|
+
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/service";
|
|
5
5
|
|
|
6
6
|
import "google/api/annotations.proto";
|
|
7
7
|
import "flyteidl/admin/signal.proto";
|
|
8
|
-
|
|
8
|
+
import "protoc-gen-openapiv2/options/annotations.proto";
|
|
9
9
|
|
|
10
10
|
// SignalService defines an RPC Service that may create, update, and retrieve signal(s).
|
|
11
11
|
service SignalService {
|
|
@@ -15,19 +15,22 @@ service SignalService {
|
|
|
15
15
|
// a signal, meaning the first call will create the signal and all subsequent calls will
|
|
16
16
|
// fetch the existing signal. This is only useful during Flyte Workflow execution and therefore
|
|
17
17
|
// is not exposed to mitigate unintended behavior.
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
18
|
+
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
|
|
19
|
+
description: "Retrieve a signal, creating it if it does not exist."
|
|
20
|
+
};
|
|
21
21
|
}
|
|
22
22
|
|
|
23
23
|
// Fetch a list of :ref:`ref_flyteidl.admin.Signal` definitions.
|
|
24
24
|
rpc ListSignals (flyteidl.admin.SignalListRequest) returns (flyteidl.admin.SignalList) {
|
|
25
25
|
option (google.api.http) = {
|
|
26
26
|
get: "/api/v1/signals/{workflow_execution_id.project}/{workflow_execution_id.domain}/{workflow_execution_id.name}"
|
|
27
|
+
additional_bindings: {
|
|
28
|
+
get: "/api/v1/signals/org/{workflow_execution_id.org}/{workflow_execution_id.project}/{workflow_execution_id.domain}/{workflow_execution_id.name}"
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
|
|
32
|
+
description: "Fetch existing signal definitions matching the input signal id filters."
|
|
27
33
|
};
|
|
28
|
-
// option (grpc.gateway.protoc_gen_swagger.options.openapiv2_operation) = {
|
|
29
|
-
// description: "Fetch existing signal definitions matching the input signal id filters."
|
|
30
|
-
// };
|
|
31
34
|
}
|
|
32
35
|
|
|
33
36
|
// Sets the value on a :ref:`ref_flyteidl.admin.Signal` definition
|
|
@@ -35,21 +38,25 @@ service SignalService {
|
|
|
35
38
|
option (google.api.http) = {
|
|
36
39
|
post: "/api/v1/signals"
|
|
37
40
|
body: "*"
|
|
41
|
+
additional_bindings: {
|
|
42
|
+
post: "/api/v1/signals/org/{id.execution_id.org}"
|
|
43
|
+
body: "*"
|
|
44
|
+
}
|
|
45
|
+
};
|
|
46
|
+
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
|
|
47
|
+
description: "Set a signal value."
|
|
48
|
+
responses: {
|
|
49
|
+
key: "400"
|
|
50
|
+
value: {
|
|
51
|
+
description: "Returned for bad request that may have failed validation."
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
responses: {
|
|
55
|
+
key: "409"
|
|
56
|
+
value: {
|
|
57
|
+
description: "Returned for a request that references an identical entity that has already been registered."
|
|
58
|
+
}
|
|
59
|
+
}
|
|
38
60
|
};
|
|
39
|
-
// option (grpc.gateway.protoc_gen_swagger.options.openapiv2_operation) = {
|
|
40
|
-
// description: "Set a signal value."
|
|
41
|
-
// responses: {
|
|
42
|
-
// key: "400"
|
|
43
|
-
// value: {
|
|
44
|
-
// description: "Returned for bad request that may have failed validation."
|
|
45
|
-
// }
|
|
46
|
-
// }
|
|
47
|
-
// responses: {
|
|
48
|
-
// key: "409"
|
|
49
|
-
// value: {
|
|
50
|
-
// description: "Returned for a request that references an identical entity that has already been registered."
|
|
51
|
-
// }
|
|
52
|
-
// }
|
|
53
|
-
// };
|
|
54
61
|
}
|
|
55
62
|
}
|
|
@@ -1,83 +0,0 @@
|
|
|
1
|
-
syntax = "proto3";
|
|
2
|
-
|
|
3
|
-
package flyteidl.plugins.sagemaker;
|
|
4
|
-
|
|
5
|
-
option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/plugins";
|
|
6
|
-
|
|
7
|
-
import "flyteidl/plugins/sagemaker/parameter_ranges.proto";
|
|
8
|
-
import "flyteidl/plugins/sagemaker/training_job.proto";
|
|
9
|
-
|
|
10
|
-
// A pass-through for SageMaker's hyperparameter tuning job
|
|
11
|
-
message HyperparameterTuningJob {
|
|
12
|
-
// The underlying training job that the hyperparameter tuning job will launch during the process
|
|
13
|
-
TrainingJob training_job = 1;
|
|
14
|
-
|
|
15
|
-
// The maximum number of training jobs that an hpo job can launch. For resource limit purpose.
|
|
16
|
-
// https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_ResourceLimits.html
|
|
17
|
-
int64 max_number_of_training_jobs = 2;
|
|
18
|
-
|
|
19
|
-
// The maximum number of concurrent training job that an hpo job can launch
|
|
20
|
-
// https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_ResourceLimits.html
|
|
21
|
-
int64 max_parallel_training_jobs = 3;
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
// HyperparameterTuningObjectiveType determines the direction of the tuning of the Hyperparameter Tuning Job
|
|
25
|
-
// with respect to the specified metric.
|
|
26
|
-
message HyperparameterTuningObjectiveType {
|
|
27
|
-
enum Value {
|
|
28
|
-
MINIMIZE = 0;
|
|
29
|
-
MAXIMIZE = 1;
|
|
30
|
-
}
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
// The target metric and the objective of the hyperparameter tuning.
|
|
34
|
-
// https://docs.aws.amazon.com/sagemaker/latest/dg/automatic-model-tuning-define-metrics.html
|
|
35
|
-
message HyperparameterTuningObjective {
|
|
36
|
-
|
|
37
|
-
// HyperparameterTuningObjectiveType determines the direction of the tuning of the Hyperparameter Tuning Job
|
|
38
|
-
// with respect to the specified metric.
|
|
39
|
-
HyperparameterTuningObjectiveType.Value objective_type = 1;
|
|
40
|
-
|
|
41
|
-
// The target metric name, which is the user-defined name of the metric specified in the
|
|
42
|
-
// training job's algorithm specification
|
|
43
|
-
string metric_name = 2;
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
// Setting the strategy used when searching in the hyperparameter space
|
|
48
|
-
// Refer this doc for more details:
|
|
49
|
-
// https://aws.amazon.com/blogs/machine-learning/amazon-sagemaker-automatic-model-tuning-now-supports-random-search-and-hyperparameter-scaling/
|
|
50
|
-
message HyperparameterTuningStrategy {
|
|
51
|
-
enum Value {
|
|
52
|
-
BAYESIAN = 0;
|
|
53
|
-
RANDOM = 1;
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
// When the training jobs launched by the hyperparameter tuning job are not improving significantly,
|
|
58
|
-
// a hyperparameter tuning job can be stopping early.
|
|
59
|
-
// Note that there's only a subset of built-in algorithms that supports early stopping.
|
|
60
|
-
// see: https://docs.aws.amazon.com/sagemaker/latest/dg/automatic-model-tuning-early-stopping.html
|
|
61
|
-
message TrainingJobEarlyStoppingType {
|
|
62
|
-
enum Value {
|
|
63
|
-
OFF = 0;
|
|
64
|
-
AUTO = 1;
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
// The specification of the hyperparameter tuning process
|
|
69
|
-
// https://docs.aws.amazon.com/sagemaker/latest/dg/automatic-model-tuning-ex-tuning-job.html#automatic-model-tuning-ex-low-tuning-config
|
|
70
|
-
message HyperparameterTuningJobConfig {
|
|
71
|
-
// ParameterRanges is a map that maps hyperparameter name to the corresponding hyperparameter range
|
|
72
|
-
ParameterRanges hyperparameter_ranges = 1;
|
|
73
|
-
|
|
74
|
-
// Setting the strategy used when searching in the hyperparameter space
|
|
75
|
-
HyperparameterTuningStrategy.Value tuning_strategy = 2;
|
|
76
|
-
|
|
77
|
-
// The target metric and the objective of the hyperparameter tuning.
|
|
78
|
-
HyperparameterTuningObjective tuning_objective = 3;
|
|
79
|
-
|
|
80
|
-
// When the training jobs launched by the hyperparameter tuning job are not improving significantly,
|
|
81
|
-
// a hyperparameter tuning job can be stopping early.
|
|
82
|
-
TrainingJobEarlyStoppingType.Value training_job_early_stopping_type = 4;
|
|
83
|
-
}
|
|
@@ -1,61 +0,0 @@
|
|
|
1
|
-
syntax = "proto3";
|
|
2
|
-
|
|
3
|
-
package flyteidl.plugins.sagemaker;
|
|
4
|
-
|
|
5
|
-
option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/plugins";
|
|
6
|
-
|
|
7
|
-
// HyperparameterScalingType defines the way to increase or decrease the value of the hyperparameter
|
|
8
|
-
// For details, refer to: https://docs.aws.amazon.com/sagemaker/latest/dg/automatic-model-tuning-define-ranges.html
|
|
9
|
-
// See examples of these scaling type, refer to: https://aws.amazon.com/blogs/machine-learning/amazon-sagemaker-automatic-model-tuning-now-supports-random-search-and-hyperparameter-scaling/
|
|
10
|
-
message HyperparameterScalingType {
|
|
11
|
-
enum Value {
|
|
12
|
-
AUTO = 0;
|
|
13
|
-
LINEAR = 1;
|
|
14
|
-
LOGARITHMIC = 2;
|
|
15
|
-
REVERSELOGARITHMIC = 3;
|
|
16
|
-
}
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
// ContinuousParameterRange refers to a continuous range of hyperparameter values, allowing
|
|
20
|
-
// users to specify the search space of a floating-point hyperparameter
|
|
21
|
-
// https://docs.aws.amazon.com/sagemaker/latest/dg/automatic-model-tuning-define-ranges.html
|
|
22
|
-
message ContinuousParameterRange {
|
|
23
|
-
double max_value = 1;
|
|
24
|
-
double min_value = 2;
|
|
25
|
-
HyperparameterScalingType.Value scaling_type = 3;
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
// IntegerParameterRange refers to a discrete range of hyperparameter values, allowing
|
|
29
|
-
// users to specify the search space of an integer hyperparameter
|
|
30
|
-
// https://docs.aws.amazon.com/sagemaker/latest/dg/automatic-model-tuning-define-ranges.html
|
|
31
|
-
message IntegerParameterRange {
|
|
32
|
-
int64 max_value = 1;
|
|
33
|
-
int64 min_value = 2;
|
|
34
|
-
HyperparameterScalingType.Value scaling_type = 3;
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
// ContinuousParameterRange refers to a continuous range of hyperparameter values, allowing
|
|
38
|
-
// users to specify the search space of a floating-point hyperparameter
|
|
39
|
-
// https://docs.aws.amazon.com/sagemaker/latest/dg/automatic-model-tuning-define-ranges.html
|
|
40
|
-
message CategoricalParameterRange {
|
|
41
|
-
repeated string values = 1;
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
// ParameterRangeOneOf describes a single ParameterRange, which is a one-of structure that can be one of
|
|
46
|
-
// the three possible types: ContinuousParameterRange, IntegerParameterRange, and CategoricalParameterRange.
|
|
47
|
-
// This one-of structure in Flyte enables specifying a Parameter in a type-safe manner
|
|
48
|
-
// See: https://docs.aws.amazon.com/sagemaker/latest/dg/automatic-model-tuning-define-ranges.html
|
|
49
|
-
message ParameterRangeOneOf {
|
|
50
|
-
oneof parameter_range_type {
|
|
51
|
-
ContinuousParameterRange continuous_parameter_range = 1;
|
|
52
|
-
IntegerParameterRange integer_parameter_range = 2;
|
|
53
|
-
CategoricalParameterRange categorical_parameter_range = 3;
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
// ParameterRanges is a map that maps hyperparameter name to the corresponding hyperparameter range
|
|
58
|
-
// https://docs.aws.amazon.com/sagemaker/latest/dg/automatic-model-tuning-define-ranges.html
|
|
59
|
-
message ParameterRanges {
|
|
60
|
-
map<string, ParameterRangeOneOf> parameter_range_map = 1;
|
|
61
|
-
}
|
|
@@ -1,119 +0,0 @@
|
|
|
1
|
-
syntax = "proto3";
|
|
2
|
-
|
|
3
|
-
package flyteidl.plugins.sagemaker;
|
|
4
|
-
|
|
5
|
-
import "google/protobuf/duration.proto";
|
|
6
|
-
|
|
7
|
-
option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/plugins";
|
|
8
|
-
|
|
9
|
-
// The input mode that the algorithm supports. When using the File input mode, SageMaker downloads
|
|
10
|
-
// the training data from S3 to the provisioned ML storage Volume, and mounts the directory to docker
|
|
11
|
-
// volume for training container. When using Pipe input mode, Amazon SageMaker streams data directly
|
|
12
|
-
// from S3 to the container.
|
|
13
|
-
// See: https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_AlgorithmSpecification.html
|
|
14
|
-
// For the input modes that different SageMaker algorithms support, see:
|
|
15
|
-
// https://docs.aws.amazon.com/sagemaker/latest/dg/sagemaker-algo-docker-registry-paths.html
|
|
16
|
-
message InputMode {
|
|
17
|
-
enum Value {
|
|
18
|
-
FILE = 0;
|
|
19
|
-
PIPE = 1;
|
|
20
|
-
}
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
// The algorithm name is used for deciding which pre-built image to point to.
|
|
24
|
-
// This is only required for use cases where SageMaker's built-in algorithm mode is used.
|
|
25
|
-
// While we currently only support a subset of the algorithms, more will be added to the list.
|
|
26
|
-
// See: https://docs.aws.amazon.com/sagemaker/latest/dg/algos.html
|
|
27
|
-
message AlgorithmName {
|
|
28
|
-
enum Value {
|
|
29
|
-
CUSTOM = 0;
|
|
30
|
-
XGBOOST = 1;
|
|
31
|
-
}
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
// Specifies the type of file for input data. Different SageMaker built-in algorithms require different file types of input data
|
|
36
|
-
// See https://docs.aws.amazon.com/sagemaker/latest/dg/cdf-training.html
|
|
37
|
-
// https://docs.aws.amazon.com/sagemaker/latest/dg/sagemaker-algo-docker-registry-paths.html
|
|
38
|
-
message InputContentType {
|
|
39
|
-
enum Value {
|
|
40
|
-
TEXT_CSV = 0;
|
|
41
|
-
}
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
// Specifies a metric that the training algorithm writes to stderr or stdout.
|
|
45
|
-
// This object is a pass-through.
|
|
46
|
-
// See this for details: https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_MetricDefinition.html
|
|
47
|
-
message MetricDefinition {
|
|
48
|
-
// User-defined name of the metric
|
|
49
|
-
string name = 1;
|
|
50
|
-
// SageMaker hyperparameter tuning parses your algorithm’s stdout and stderr streams to find algorithm metrics
|
|
51
|
-
string regex = 2;
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
// Specifies the training algorithm to be used in the training job
|
|
56
|
-
// This object is mostly a pass-through, with a couple of exceptions include: (1) in Flyte, users don't need to specify
|
|
57
|
-
// TrainingImage; either use the built-in algorithm mode by using Flytekit's Simple Training Job and specifying an algorithm
|
|
58
|
-
// name and an algorithm version or (2) when users want to supply custom algorithms they should set algorithm_name field to
|
|
59
|
-
// CUSTOM. In this case, the value of the algorithm_version field has no effect
|
|
60
|
-
// For pass-through use cases: refer to this AWS official document for more details
|
|
61
|
-
// https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_AlgorithmSpecification.html
|
|
62
|
-
message AlgorithmSpecification {
|
|
63
|
-
// The input mode can be either PIPE or FILE
|
|
64
|
-
InputMode.Value input_mode = 1;
|
|
65
|
-
|
|
66
|
-
// The algorithm name is used for deciding which pre-built image to point to
|
|
67
|
-
AlgorithmName.Value algorithm_name = 2;
|
|
68
|
-
// The algorithm version field is used for deciding which pre-built image to point to
|
|
69
|
-
// This is only needed for use cases where SageMaker's built-in algorithm mode is chosen
|
|
70
|
-
string algorithm_version = 3;
|
|
71
|
-
|
|
72
|
-
// A list of metric definitions for SageMaker to evaluate/track on the progress of the training job
|
|
73
|
-
// See this: https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_AlgorithmSpecification.html
|
|
74
|
-
// and this: https://docs.aws.amazon.com/sagemaker/latest/dg/automatic-model-tuning-define-metrics.html
|
|
75
|
-
repeated MetricDefinition metric_definitions = 4;
|
|
76
|
-
|
|
77
|
-
// The content type of the input
|
|
78
|
-
// See https://docs.aws.amazon.com/sagemaker/latest/dg/cdf-training.html
|
|
79
|
-
// https://docs.aws.amazon.com/sagemaker/latest/dg/sagemaker-algo-docker-registry-paths.html
|
|
80
|
-
InputContentType.Value input_content_type = 5;
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
// When enabling distributed training on a training job, the user should use this message to tell Flyte and SageMaker
|
|
84
|
-
// what kind of distributed protocol he/she wants to use to distribute the work.
|
|
85
|
-
message DistributedProtocol {
|
|
86
|
-
enum Value {
|
|
87
|
-
// Use this value if the user wishes to use framework-native distributed training interfaces.
|
|
88
|
-
// If this value is used, Flyte won't configure SageMaker to initialize unnecessary components such as
|
|
89
|
-
// OpenMPI or Parameter Server.
|
|
90
|
-
UNSPECIFIED = 0;
|
|
91
|
-
// Use this value if the user wishes to use MPI as the underlying protocol for her distributed training job
|
|
92
|
-
// MPI is a framework-agnostic distributed protocol. It has multiple implementations. Currently, we have only
|
|
93
|
-
// tested the OpenMPI implementation, which is the recommended implementation for Horovod.
|
|
94
|
-
MPI = 1;
|
|
95
|
-
}
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
// TrainingJobResourceConfig is a pass-through, specifying the instance type to use for the training job, the
|
|
99
|
-
// number of instances to launch, and the size of the ML storage volume the user wants to provision
|
|
100
|
-
// Refer to SageMaker official doc for more details: https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_CreateTrainingJob.html
|
|
101
|
-
message TrainingJobResourceConfig {
|
|
102
|
-
// The number of ML compute instances to use. For distributed training, provide a value greater than 1.
|
|
103
|
-
int64 instance_count = 1;
|
|
104
|
-
// The ML compute instance type
|
|
105
|
-
string instance_type = 2;
|
|
106
|
-
// The size of the ML storage volume that you want to provision.
|
|
107
|
-
int64 volume_size_in_gb = 3;
|
|
108
|
-
// When users specify an instance_count > 1, Flyte will try to configure SageMaker to enable distributed training.
|
|
109
|
-
// If the users wish to use framework-agnostic distributed protocol such as MPI or Parameter Server, this
|
|
110
|
-
// field should be set to the corresponding enum value
|
|
111
|
-
DistributedProtocol.Value distributed_protocol = 4;
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
// The spec of a training job. This is mostly a pass-through object
|
|
115
|
-
// https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_CreateTrainingJob.html
|
|
116
|
-
message TrainingJob {
|
|
117
|
-
AlgorithmSpecification algorithm_specification = 1;
|
|
118
|
-
TrainingJobResourceConfig training_job_resource_config = 2;
|
|
119
|
-
}
|