@flyteorg/flyteidl 1.16.0-b3 → 2.0.0-alpha1

package/protos/flyteidl/plugins/pytorch.proto

@@ -1,25 +0,0 @@
-syntax = "proto3";
-
-package flyteidl.plugins;
-
-option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/plugins";
-
-// Custom proto for torch elastic config for distributed training using 
-// https://github.com/kubeflow/training-operator/blob/master/pkg/apis/kubeflow.org/v1/pytorch_types.go
-message ElasticConfig {
-  string rdzv_backend = 1;
-  int32 min_replicas = 2;
-  int32 max_replicas = 3;
-  int32 nproc_per_node = 4;
-  int32 max_restarts = 5;
-}
-
-// Custom proto for plugin that enables distributed training using https://github.com/kubeflow/pytorch-operator
-message DistributedPyTorchTrainingTask {
-  // number of worker replicas spawned in the cluster for this job
-  int32 workers = 1;
-
-  // config for an elastic pytorch job
-  // 
-  ElasticConfig elastic_config = 2;
-}

package/protos/flyteidl/plugins/qubole.proto

@@ -1,26 +0,0 @@
-syntax = "proto3";
-
-package flyteidl.plugins;
-
-option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/plugins";
-
-// Defines a query to execute on a hive cluster.
-message HiveQuery {
-    string query = 1;
-    uint32 timeout_sec = 2;
-    uint32 retryCount = 3;
-}
-
-// Defines a collection of hive queries.
-message HiveQueryCollection {
-    repeated HiveQuery queries = 2;
-}
-
-// This message works with the 'hive' task type in the SDK and is the object that will be in the 'custom' field
-// of a hive task's TaskTemplate
-message QuboleHiveJob {
-    string cluster_label = 1;
-    HiveQueryCollection query_collection = 2 [deprecated=true];
-    repeated string tags = 3;
-    HiveQuery query = 4;
-}

package/protos/flyteidl/plugins/ray.proto

@@ -1,59 +0,0 @@
-syntax = "proto3";
-
-package flyteidl.plugins;
-
-option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/plugins";
-
-import "flyteidl/core/tasks.proto";
-
-// RayJobSpec defines the desired state of RayJob
-message RayJob {
-  // RayClusterSpec is the cluster template to run the job
-  RayCluster ray_cluster = 1;
-  // runtime_env is base64 encoded.
-  // Ray runtime environments: https://docs.ray.io/en/latest/ray-core/handling-dependencies.html#runtime-environments
-  string runtime_env = 2 [deprecated = true];
-  // shutdown_after_job_finishes specifies whether the RayCluster should be deleted after the RayJob finishes.
-  bool shutdown_after_job_finishes = 3;
-  // ttl_seconds_after_finished specifies the number of seconds after which the RayCluster will be deleted after the RayJob finishes.
-  int32 ttl_seconds_after_finished = 4;
-  // RuntimeEnvYAML represents the runtime environment configuration
-  // provided as a multi-line YAML string.
-  string runtime_env_yaml = 5;
-}
-
-// Define Ray cluster defines the desired state of RayCluster
-message RayCluster {
-  // HeadGroupSpecs are the spec for the head pod
-  HeadGroupSpec head_group_spec = 1;
-  // WorkerGroupSpecs are the specs for the worker pods
-  repeated WorkerGroupSpec worker_group_spec = 2;
-  // Whether to enable autoscaling.
-  bool enable_autoscaling = 3;
-}
-
-// HeadGroupSpec are the spec for the head pod
-message HeadGroupSpec {
-  // Optional. RayStartParams are the params of the start command: address, object-store-memory.
-  // Refer to https://docs.ray.io/en/latest/ray-core/package-ref.html#ray-start
-  map<string, string> ray_start_params = 1;
-  // Pod Spec for the ray head pod
-  core.K8sPod k8s_pod = 2;
-}
-
-// WorkerGroupSpec are the specs for the worker pods
-message WorkerGroupSpec {
-  // Required. RayCluster can have multiple worker groups, and it distinguishes them by name
-  string group_name = 1;
-  // Required. Desired replicas of the worker group. Defaults to 1.
-  int32 replicas = 2;
-  // Optional. Min replicas of the worker group. MinReplicas defaults to 1.
-  int32 min_replicas = 3;
-  // Optional. Max replicas of the worker group. MaxReplicas defaults to maxInt32
-  int32 max_replicas = 4;
-  // Optional. RayStartParams are the params of the start command: address, object-store-memory.
-  // Refer to https://docs.ray.io/en/latest/ray-core/package-ref.html#ray-start
-  map<string, string> ray_start_params = 5;
-  // Pod Spec for ray worker pods
-  core.K8sPod k8s_pod = 6;
-}

package/protos/flyteidl/plugins/spark.proto

@@ -1,42 +0,0 @@
-syntax = "proto3";
-
-package flyteidl.plugins;
-
-option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/plugins";
-
-import "google/protobuf/struct.proto";
-import "flyteidl/core/tasks.proto";
-
-message SparkApplication {
-    enum Type {
-        PYTHON = 0;
-        JAVA = 1;
-        SCALA = 2;
-        R = 3;
-    }
-}
-
-// Custom Proto for Spark Plugin.
-message SparkJob {
-    SparkApplication.Type applicationType = 1;
-    string mainApplicationFile = 2;
-    string mainClass = 3;
-    map<string, string> sparkConf = 4;
-    map<string, string> hadoopConf = 5;
-    string executorPath = 6; // Executor path for Python jobs.
-    // Databricks job configuration.
-    // Config structure can be found here. https://docs.databricks.com/dev-tools/api/2.0/jobs.html#request-structure.
-    google.protobuf.Struct databricksConf = 7;
-    // Databricks access token. https://docs.databricks.com/dev-tools/api/latest/authentication.html
-    // This token can be set in either flytepropeller or flytekit.
-    string databricksToken = 8;
-    // Domain name of your deployment. Use the form <account>.cloud.databricks.com.
-    // This instance name can be set in either flytepropeller or flytekit.
-    string databricksInstance = 9;
-
-    // Pod Spec for the Spark driver pod
-    core.K8sPod driverPod = 10;
-
-    // Pod Spec for the Spark executor pod
-    core.K8sPod executorPod = 11;
-}

package/protos/flyteidl/plugins/tensorflow.proto

@@ -1,18 +0,0 @@
-syntax = "proto3";
-
-package flyteidl.plugins;
-
-option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/plugins";
-
-// Custom proto for plugin that enables distributed training using https://github.com/kubeflow/tf-operator
-message DistributedTensorflowTrainingTask {
-  // number of worker replicas spawned in the cluster for this job
-  int32 workers = 1;
-  // PS -> Parameter server
-  // number of ps replicas spawned in the cluster for this job
-  int32 ps_replicas = 2;
-  // number of chief replicas spawned in the cluster for this job
-  int32 chief_replicas = 3;
-  // number of evaluator replicas spawned in the cluster for this job
-  int32 evaluator_replicas = 4;
-} 

package/protos/flyteidl/plugins/waitable.proto

@@ -1,15 +0,0 @@
-syntax = "proto3";
-
-import "flyteidl/core/execution.proto";
-import "flyteidl/core/identifier.proto";
-
-package flyteidl.plugins;
-
-option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/plugins";
-
-// Represents an Execution that was launched and could be waited on.
-message Waitable {
-    core.WorkflowExecutionIdentifier wf_exec_id = 1;
-    core.WorkflowExecution.Phase phase = 2;
-    string workflow_id = 3;
-}