@flyteorg/flyteidl 1.5.10 → 1.5.12

package/gen/pb-js/flyteidl.d.ts

@@ -1355,6 +1355,79 @@ export namespace flyteidl {
             public static verify(message: { [k: string]: any }): (string|null);
         }
 
+        /** Properties of an ArrayNode. */
+        interface IArrayNode {
+
+            /** ArrayNode node */
+            node?: (flyteidl.core.INode|null);
+
+            /** ArrayNode parallelism */
+            parallelism?: (number|null);
+
+            /** ArrayNode minSuccesses */
+            minSuccesses?: (number|null);
+
+            /** ArrayNode minSuccessRatio */
+            minSuccessRatio?: (number|null);
+        }
+
+        /** Represents an ArrayNode. */
+        class ArrayNode implements IArrayNode {
+
+            /**
+             * Constructs a new ArrayNode.
+             * @param [properties] Properties to set
+             */
+            constructor(properties?: flyteidl.core.IArrayNode);
+
+            /** ArrayNode node. */
+            public node?: (flyteidl.core.INode|null);
+
+            /** ArrayNode parallelism. */
+            public parallelism: number;
+
+            /** ArrayNode minSuccesses. */
+            public minSuccesses: number;
+
+            /** ArrayNode minSuccessRatio. */
+            public minSuccessRatio: number;
+
+            /** ArrayNode successCriteria. */
+            public successCriteria?: ("minSuccesses"|"minSuccessRatio");
+
+            /**
+             * Creates a new ArrayNode instance using the specified properties.
+             * @param [properties] Properties to set
+             * @returns ArrayNode instance
+             */
+            public static create(properties?: flyteidl.core.IArrayNode): flyteidl.core.ArrayNode;
+
+            /**
+             * Encodes the specified ArrayNode message. Does not implicitly {@link flyteidl.core.ArrayNode.verify|verify} messages.
+             * @param message ArrayNode message or plain object to encode
+             * @param [writer] Writer to encode to
+             * @returns Writer
+             */
+            public static encode(message: flyteidl.core.IArrayNode, writer?: $protobuf.Writer): $protobuf.Writer;
+
+            /**
+             * Decodes an ArrayNode message from the specified reader or buffer.
+             * @param reader Reader or buffer to decode from
+             * @param [length] Message length if known beforehand
+             * @returns ArrayNode
+             * @throws {Error} If the payload is not a reader or valid buffer
+             * @throws {$protobuf.util.ProtocolError} If required fields are missing
+             */
+            public static decode(reader: ($protobuf.Reader|Uint8Array), length?: number): flyteidl.core.ArrayNode;
+
+            /**
+             * Verifies an ArrayNode message.
+             * @param message Plain object to verify
+             * @returns `null` if valid, otherwise the reason why it is not
+             */
+            public static verify(message: { [k: string]: any }): (string|null);
+        }
+
         /** Properties of a NodeMetadata. */
         interface INodeMetadata {
 
@@ -1515,6 +1588,9 @@ export namespace flyteidl {
 
             /** Node gateNode */
             gateNode?: (flyteidl.core.IGateNode|null);
+
+            /** Node arrayNode */
+            arrayNode?: (flyteidl.core.IArrayNode|null);
         }
 
         /** Represents a Node. */
@@ -1553,8 +1629,11 @@ export namespace flyteidl {
             /** Node gateNode. */
             public gateNode?: (flyteidl.core.IGateNode|null);
 
+            /** Node arrayNode. */
+            public arrayNode?: (flyteidl.core.IArrayNode|null);
+
             /** Node target. */
-            public target?: ("taskNode"|"workflowNode"|"branchNode"|"gateNode");
+            public target?: ("taskNode"|"workflowNode"|"branchNode"|"gateNode"|"arrayNode");
 
             /**
              * Creates a new Node instance using the specified properties.
@@ -19686,6 +19765,9 @@ export namespace flyteidl {
 
             /** CreateUploadLocationRequest contentMd5 */
             contentMd5?: (Uint8Array|null);
+
+            /** CreateUploadLocationRequest filenameRoot */
+            filenameRoot?: (string|null);
         }
 
         /** Represents a CreateUploadLocationRequest. */
@@ -19712,6 +19794,9 @@ export namespace flyteidl {
             /** CreateUploadLocationRequest contentMd5. */
             public contentMd5: Uint8Array;
 
+            /** CreateUploadLocationRequest filenameRoot. */
+            public filenameRoot: string;
+
             /**
              * Creates a new CreateUploadLocationRequest instance using the specified properties.
              * @param [properties] Properties to set

package/gen/pb-js/flyteidl.js

@@ -3200,6 +3200,190 @@
                 return GateNode;
             })();
     
+            core.ArrayNode = (function() {
+    
+                /**
+                 * Properties of an ArrayNode.
+                 * @memberof flyteidl.core
+                 * @interface IArrayNode
+                 * @property {flyteidl.core.INode|null} [node] ArrayNode node
+                 * @property {number|null} [parallelism] ArrayNode parallelism
+                 * @property {number|null} [minSuccesses] ArrayNode minSuccesses
+                 * @property {number|null} [minSuccessRatio] ArrayNode minSuccessRatio
+                 */
+    
+                /**
+                 * Constructs a new ArrayNode.
+                 * @memberof flyteidl.core
+                 * @classdesc Represents an ArrayNode.
+                 * @implements IArrayNode
+                 * @constructor
+                 * @param {flyteidl.core.IArrayNode=} [properties] Properties to set
+                 */
+                function ArrayNode(properties) {
+                    if (properties)
+                        for (var keys = Object.keys(properties), i = 0; i < keys.length; ++i)
+                            if (properties[keys[i]] != null)
+                                this[keys[i]] = properties[keys[i]];
+                }
+    
+                /**
+                 * ArrayNode node.
+                 * @member {flyteidl.core.INode|null|undefined} node
+                 * @memberof flyteidl.core.ArrayNode
+                 * @instance
+                 */
+                ArrayNode.prototype.node = null;
+    
+                /**
+                 * ArrayNode parallelism.
+                 * @member {number} parallelism
+                 * @memberof flyteidl.core.ArrayNode
+                 * @instance
+                 */
+                ArrayNode.prototype.parallelism = 0;
+    
+                /**
+                 * ArrayNode minSuccesses.
+                 * @member {number} minSuccesses
+                 * @memberof flyteidl.core.ArrayNode
+                 * @instance
+                 */
+                ArrayNode.prototype.minSuccesses = 0;
+    
+                /**
+                 * ArrayNode minSuccessRatio.
+                 * @member {number} minSuccessRatio
+                 * @memberof flyteidl.core.ArrayNode
+                 * @instance
+                 */
+                ArrayNode.prototype.minSuccessRatio = 0;
+    
+                // OneOf field names bound to virtual getters and setters
+                var $oneOfFields;
+    
+                /**
+                 * ArrayNode successCriteria.
+                 * @member {"minSuccesses"|"minSuccessRatio"|undefined} successCriteria
+                 * @memberof flyteidl.core.ArrayNode
+                 * @instance
+                 */
+                Object.defineProperty(ArrayNode.prototype, "successCriteria", {
+                    get: $util.oneOfGetter($oneOfFields = ["minSuccesses", "minSuccessRatio"]),
+                    set: $util.oneOfSetter($oneOfFields)
+                });
+    
+                /**
+                 * Creates a new ArrayNode instance using the specified properties.
+                 * @function create
+                 * @memberof flyteidl.core.ArrayNode
+                 * @static
+                 * @param {flyteidl.core.IArrayNode=} [properties] Properties to set
+                 * @returns {flyteidl.core.ArrayNode} ArrayNode instance
+                 */
+                ArrayNode.create = function create(properties) {
+                    return new ArrayNode(properties);
+                };
+    
+                /**
+                 * Encodes the specified ArrayNode message. Does not implicitly {@link flyteidl.core.ArrayNode.verify|verify} messages.
+                 * @function encode
+                 * @memberof flyteidl.core.ArrayNode
+                 * @static
+                 * @param {flyteidl.core.IArrayNode} message ArrayNode message or plain object to encode
+                 * @param {$protobuf.Writer} [writer] Writer to encode to
+                 * @returns {$protobuf.Writer} Writer
+                 */
+                ArrayNode.encode = function encode(message, writer) {
+                    if (!writer)
+                        writer = $Writer.create();
+                    if (message.node != null && message.hasOwnProperty("node"))
+                        $root.flyteidl.core.Node.encode(message.node, writer.uint32(/* id 1, wireType 2 =*/10).fork()).ldelim();
+                    if (message.parallelism != null && message.hasOwnProperty("parallelism"))
+                        writer.uint32(/* id 2, wireType 0 =*/16).uint32(message.parallelism);
+                    if (message.minSuccesses != null && message.hasOwnProperty("minSuccesses"))
+                        writer.uint32(/* id 3, wireType 0 =*/24).uint32(message.minSuccesses);
+                    if (message.minSuccessRatio != null && message.hasOwnProperty("minSuccessRatio"))
+                        writer.uint32(/* id 4, wireType 5 =*/37).float(message.minSuccessRatio);
+                    return writer;
+                };
+    
+                /**
+                 * Decodes an ArrayNode message from the specified reader or buffer.
+                 * @function decode
+                 * @memberof flyteidl.core.ArrayNode
+                 * @static
+                 * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from
+                 * @param {number} [length] Message length if known beforehand
+                 * @returns {flyteidl.core.ArrayNode} ArrayNode
+                 * @throws {Error} If the payload is not a reader or valid buffer
+                 * @throws {$protobuf.util.ProtocolError} If required fields are missing
+                 */
+                ArrayNode.decode = function decode(reader, length) {
+                    if (!(reader instanceof $Reader))
+                        reader = $Reader.create(reader);
+                    var end = length === undefined ? reader.len : reader.pos + length, message = new $root.flyteidl.core.ArrayNode();
+                    while (reader.pos < end) {
+                        var tag = reader.uint32();
+                        switch (tag >>> 3) {
+                        case 1:
+                            message.node = $root.flyteidl.core.Node.decode(reader, reader.uint32());
+                            break;
+                        case 2:
+                            message.parallelism = reader.uint32();
+                            break;
+                        case 3:
+                            message.minSuccesses = reader.uint32();
+                            break;
+                        case 4:
+                            message.minSuccessRatio = reader.float();
+                            break;
+                        default:
+                            reader.skipType(tag & 7);
+                            break;
+                        }
+                    }
+                    return message;
+                };
+    
+                /**
+                 * Verifies an ArrayNode message.
+                 * @function verify
+                 * @memberof flyteidl.core.ArrayNode
+                 * @static
+                 * @param {Object.<string,*>} message Plain object to verify
+                 * @returns {string|null} `null` if valid, otherwise the reason why it is not
+                 */
+                ArrayNode.verify = function verify(message) {
+                    if (typeof message !== "object" || message === null)
+                        return "object expected";
+                    var properties = {};
+                    if (message.node != null && message.hasOwnProperty("node")) {
+                        var error = $root.flyteidl.core.Node.verify(message.node);
+                        if (error)
+                            return "node." + error;
+                    }
+                    if (message.parallelism != null && message.hasOwnProperty("parallelism"))
+                        if (!$util.isInteger(message.parallelism))
+                            return "parallelism: integer expected";
+                    if (message.minSuccesses != null && message.hasOwnProperty("minSuccesses")) {
+                        properties.successCriteria = 1;
+                        if (!$util.isInteger(message.minSuccesses))
+                            return "minSuccesses: integer expected";
+                    }
+                    if (message.minSuccessRatio != null && message.hasOwnProperty("minSuccessRatio")) {
+                        if (properties.successCriteria === 1)
+                            return "successCriteria: multiple values";
+                        properties.successCriteria = 1;
+                        if (typeof message.minSuccessRatio !== "number")
+                            return "minSuccessRatio: number expected";
+                    }
+                    return null;
+                };
+    
+                return ArrayNode;
+            })();
+    
             core.NodeMetadata = (function() {
     
                 /**
@@ -3524,6 +3708,7 @@
                  * @property {flyteidl.core.IWorkflowNode|null} [workflowNode] Node workflowNode
                  * @property {flyteidl.core.IBranchNode|null} [branchNode] Node branchNode
                  * @property {flyteidl.core.IGateNode|null} [gateNode] Node gateNode
+                 * @property {flyteidl.core.IArrayNode|null} [arrayNode] Node arrayNode
                  */
     
                 /**
@@ -3616,17 +3801,25 @@
                  */
                 Node.prototype.gateNode = null;
     
+                /**
+                 * Node arrayNode.
+                 * @member {flyteidl.core.IArrayNode|null|undefined} arrayNode
+                 * @memberof flyteidl.core.Node
+                 * @instance
+                 */
+                Node.prototype.arrayNode = null;
+    
                 // OneOf field names bound to virtual getters and setters
                 var $oneOfFields;
     
                 /**
                  * Node target.
-                 * @member {"taskNode"|"workflowNode"|"branchNode"|"gateNode"|undefined} target
+                 * @member {"taskNode"|"workflowNode"|"branchNode"|"gateNode"|"arrayNode"|undefined} target
                  * @memberof flyteidl.core.Node
                  * @instance
                  */
                 Object.defineProperty(Node.prototype, "target", {
-                    get: $util.oneOfGetter($oneOfFields = ["taskNode", "workflowNode", "branchNode", "gateNode"]),
+                    get: $util.oneOfGetter($oneOfFields = ["taskNode", "workflowNode", "branchNode", "gateNode", "arrayNode"]),
                     set: $util.oneOfSetter($oneOfFields)
                 });
     
@@ -3675,6 +3868,8 @@
                         $root.flyteidl.core.BranchNode.encode(message.branchNode, writer.uint32(/* id 8, wireType 2 =*/66).fork()).ldelim();
                     if (message.gateNode != null && message.hasOwnProperty("gateNode"))
                         $root.flyteidl.core.GateNode.encode(message.gateNode, writer.uint32(/* id 9, wireType 2 =*/74).fork()).ldelim();
+                    if (message.arrayNode != null && message.hasOwnProperty("arrayNode"))
+                        $root.flyteidl.core.ArrayNode.encode(message.arrayNode, writer.uint32(/* id 10, wireType 2 =*/82).fork()).ldelim();
                     return writer;
                 };
     
@@ -3729,6 +3924,9 @@
                         case 9:
                             message.gateNode = $root.flyteidl.core.GateNode.decode(reader, reader.uint32());
                             break;
+                        case 10:
+                            message.arrayNode = $root.flyteidl.core.ArrayNode.decode(reader, reader.uint32());
+                            break;
                         default:
                             reader.skipType(tag & 7);
                             break;
@@ -3820,6 +4018,16 @@
                                 return "gateNode." + error;
                         }
                     }
+                    if (message.arrayNode != null && message.hasOwnProperty("arrayNode")) {
+                        if (properties.target === 1)
+                            return "target: multiple values";
+                        properties.target = 1;
+                        {
+                            var error = $root.flyteidl.core.ArrayNode.verify(message.arrayNode);
+                            if (error)
+                                return "arrayNode." + error;
+                        }
+                    }
                     return null;
                 };
     
@@ -45833,6 +46041,7 @@
                  * @property {string|null} [filename] CreateUploadLocationRequest filename
                  * @property {google.protobuf.IDuration|null} [expiresIn] CreateUploadLocationRequest expiresIn
                  * @property {Uint8Array|null} [contentMd5] CreateUploadLocationRequest contentMd5
+                 * @property {string|null} [filenameRoot] CreateUploadLocationRequest filenameRoot
                  */
     
                 /**
@@ -45890,6 +46099,14 @@
                  */
                 CreateUploadLocationRequest.prototype.contentMd5 = $util.newBuffer([]);
     
+                /**
+                 * CreateUploadLocationRequest filenameRoot.
+                 * @member {string} filenameRoot
+                 * @memberof flyteidl.service.CreateUploadLocationRequest
+                 * @instance
+                 */
+                CreateUploadLocationRequest.prototype.filenameRoot = "";
+    
                 /**
                  * Creates a new CreateUploadLocationRequest instance using the specified properties.
                  * @function create
@@ -45924,6 +46141,8 @@
                         $root.google.protobuf.Duration.encode(message.expiresIn, writer.uint32(/* id 4, wireType 2 =*/34).fork()).ldelim();
                     if (message.contentMd5 != null && message.hasOwnProperty("contentMd5"))
                         writer.uint32(/* id 5, wireType 2 =*/42).bytes(message.contentMd5);
+                    if (message.filenameRoot != null && message.hasOwnProperty("filenameRoot"))
+                        writer.uint32(/* id 6, wireType 2 =*/50).string(message.filenameRoot);
                     return writer;
                 };
     
@@ -45960,6 +46179,9 @@
                         case 5:
                             message.contentMd5 = reader.bytes();
                             break;
+                        case 6:
+                            message.filenameRoot = reader.string();
+                            break;
                         default:
                             reader.skipType(tag & 7);
                             break;
@@ -45996,6 +46218,9 @@
                     if (message.contentMd5 != null && message.hasOwnProperty("contentMd5"))
                         if (!(message.contentMd5 && typeof message.contentMd5.length === "number" || $util.isString(message.contentMd5)))
                             return "contentMd5: buffer expected";
+                    if (message.filenameRoot != null && message.hasOwnProperty("filenameRoot"))
+                        if (!$util.isString(message.filenameRoot))
+                            return "filenameRoot: string expected";
                     return null;
                 };
     

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flyteorg/flyteidl",
-  "version": "1.5.10",
+  "version": "1.5.12",
   "description": "Compiled protocol buffers and gRPC service clients/servers for Flyte IDLs",
   "repository": {
     "type": "git",

package/protos/flyteidl/core/workflow.proto

@@ -107,6 +107,32 @@ message GateNode {
     }
 }
 
+// ArrayNode is a Flyte node type that simplifies the execution of a sub-node over a list of input
+// values. An ArrayNode can be executed with configurable parallelism (separate from the parent
+// workflow) and can be configured to succeed when a certain number of sub-nodes succeed.
+message ArrayNode {
+    // node is the sub-node that will be executed for each element in the array.
+    Node node = 1;
+
+    // parallelism defines the minimum number of instances to bring up concurrently at any given
+    // point. Note that this is an optimistic restriction and that, due to network partitioning or
+    // other failures, the actual number of currently running instances might be more. This has to
+    // be a positive number if assigned. Default value is size.
+    uint32 parallelism = 2;
+
+    oneof success_criteria {
+        // min_successes is an absolute number of the minimum number of successful completions of
+        // sub-nodes. As soon as this criteria is met, the ArrayNode will be marked as successful
+        // and outputs will be computed. This has to be a non-negative number if assigned. Default
+        // value is size (if specified).
+        uint32 min_successes = 3;
+
+        // If the array job size is not known beforehand, the min_success_ratio can instead be used
+        // to determine when an ArrayNode can be marked successful.
+        float min_success_ratio = 4;
+    }
+}
+
 // Defines extra information about the Node.
 message NodeMetadata {
     // A friendly name for the Node
@@ -171,6 +197,10 @@ message Node {
 
         // Information about the condition to evaluate in this node.
         GateNode gate_node = 9;
+
+        // Information about the sub-node executions for each value in the list of this nodes
+        // inputs values.
+        ArrayNode array_node = 10;
     }
 }
 

package/protos/flyteidl/service/dataproxy.proto

@@ -23,6 +23,10 @@ message CreateUploadLocationResponse {
 }
 
 // CreateUploadLocationRequest specified request for the CreateUploadLocation API.
+// The implementation in data proxy service will create the s3 location with some server side configured prefixes,
+// and then:
+//   - project/domain/(a deterministic str representation of the content_md5)/filename (if present); OR
+//   - project/domain/filename_root (if present)/filename (if present).
 message CreateUploadLocationRequest {
   // Project to create the upload location for
   // +required
@@ -45,6 +49,12 @@ message CreateUploadLocationRequest {
   // generated path.
   // +required
   bytes content_md5 = 5;
+
+  // If present, data proxy will use this string in lieu of the md5 hash in the path. When the filename is also included
+  // this makes the upload location deterministic. The native url will still be prefixed by the upload location prefix
+  // in data proxy config. This option is useful when uploading multiple files.
+  // +optional
+  string filename_root = 6;
 }
 
 // CreateDownloadLocationRequest specified request for the CreateDownloadLocation API.