@flyteorg/flyteidl 0.21.16 → 0.21.20

package/gen/pb-js/flyteidl.d.ts

@@ -2177,6 +2177,116 @@ export namespace flyteidl {
             public static verify(message: { [k: string]: any }): (string|null);
         }
 
+        /** Properties of a StructuredDatasetMetadata. */
+        interface IStructuredDatasetMetadata {
+
+            /** StructuredDatasetMetadata structuredDatasetType */
+            structuredDatasetType?: (flyteidl.core.IStructuredDatasetType|null);
+        }
+
+        /** Represents a StructuredDatasetMetadata. */
+        class StructuredDatasetMetadata implements IStructuredDatasetMetadata {
+
+            /**
+             * Constructs a new StructuredDatasetMetadata.
+             * @param [properties] Properties to set
+             */
+            constructor(properties?: flyteidl.core.IStructuredDatasetMetadata);
+
+            /** StructuredDatasetMetadata structuredDatasetType. */
+            public structuredDatasetType?: (flyteidl.core.IStructuredDatasetType|null);
+
+            /**
+             * Creates a new StructuredDatasetMetadata instance using the specified properties.
+             * @param [properties] Properties to set
+             * @returns StructuredDatasetMetadata instance
+             */
+            public static create(properties?: flyteidl.core.IStructuredDatasetMetadata): flyteidl.core.StructuredDatasetMetadata;
+
+            /**
+             * Encodes the specified StructuredDatasetMetadata message. Does not implicitly {@link flyteidl.core.StructuredDatasetMetadata.verify|verify} messages.
+             * @param message StructuredDatasetMetadata message or plain object to encode
+             * @param [writer] Writer to encode to
+             * @returns Writer
+             */
+            public static encode(message: flyteidl.core.IStructuredDatasetMetadata, writer?: $protobuf.Writer): $protobuf.Writer;
+
+            /**
+             * Decodes a StructuredDatasetMetadata message from the specified reader or buffer.
+             * @param reader Reader or buffer to decode from
+             * @param [length] Message length if known beforehand
+             * @returns StructuredDatasetMetadata
+             * @throws {Error} If the payload is not a reader or valid buffer
+             * @throws {$protobuf.util.ProtocolError} If required fields are missing
+             */
+            public static decode(reader: ($protobuf.Reader|Uint8Array), length?: number): flyteidl.core.StructuredDatasetMetadata;
+
+            /**
+             * Verifies a StructuredDatasetMetadata message.
+             * @param message Plain object to verify
+             * @returns `null` if valid, otherwise the reason why it is not
+             */
+            public static verify(message: { [k: string]: any }): (string|null);
+        }
+
+        /** Properties of a StructuredDataset. */
+        interface IStructuredDataset {
+
+            /** StructuredDataset uri */
+            uri?: (string|null);
+
+            /** StructuredDataset metadata */
+            metadata?: (flyteidl.core.IStructuredDatasetMetadata|null);
+        }
+
+        /** Represents a StructuredDataset. */
+        class StructuredDataset implements IStructuredDataset {
+
+            /**
+             * Constructs a new StructuredDataset.
+             * @param [properties] Properties to set
+             */
+            constructor(properties?: flyteidl.core.IStructuredDataset);
+
+            /** StructuredDataset uri. */
+            public uri: string;
+
+            /** StructuredDataset metadata. */
+            public metadata?: (flyteidl.core.IStructuredDatasetMetadata|null);
+
+            /**
+             * Creates a new StructuredDataset instance using the specified properties.
+             * @param [properties] Properties to set
+             * @returns StructuredDataset instance
+             */
+            public static create(properties?: flyteidl.core.IStructuredDataset): flyteidl.core.StructuredDataset;
+
+            /**
+             * Encodes the specified StructuredDataset message. Does not implicitly {@link flyteidl.core.StructuredDataset.verify|verify} messages.
+             * @param message StructuredDataset message or plain object to encode
+             * @param [writer] Writer to encode to
+             * @returns Writer
+             */
+            public static encode(message: flyteidl.core.IStructuredDataset, writer?: $protobuf.Writer): $protobuf.Writer;
+
+            /**
+             * Decodes a StructuredDataset message from the specified reader or buffer.
+             * @param reader Reader or buffer to decode from
+             * @param [length] Message length if known beforehand
+             * @returns StructuredDataset
+             * @throws {Error} If the payload is not a reader or valid buffer
+             * @throws {$protobuf.util.ProtocolError} If required fields are missing
+             */
+            public static decode(reader: ($protobuf.Reader|Uint8Array), length?: number): flyteidl.core.StructuredDataset;
+
+            /**
+             * Verifies a StructuredDataset message.
+             * @param message Plain object to verify
+             * @returns `null` if valid, otherwise the reason why it is not
+             */
+            public static verify(message: { [k: string]: any }): (string|null);
+        }
+
         /** Properties of a Scalar. */
         interface IScalar {
 
@@ -2200,6 +2310,9 @@ export namespace flyteidl {
 
             /** Scalar generic */
             generic?: (google.protobuf.IStruct|null);
+
+            /** Scalar structuredDataset */
+            structuredDataset?: (flyteidl.core.IStructuredDataset|null);
         }
 
         /** Represents a Scalar. */
@@ -2232,8 +2345,11 @@ export namespace flyteidl {
             /** Scalar generic. */
             public generic?: (google.protobuf.IStruct|null);
 
+            /** Scalar structuredDataset. */
+            public structuredDataset?: (flyteidl.core.IStructuredDataset|null);
+
             /** Scalar value. */
-            public value?: ("primitive"|"blob"|"binary"|"schema"|"noneType"|"error"|"generic");
+            public value?: ("primitive"|"blob"|"binary"|"schema"|"noneType"|"error"|"generic"|"structuredDataset");
 
             /**
              * Creates a new Scalar instance using the specified properties.
@@ -2924,6 +3040,137 @@ export namespace flyteidl {
             }
         }
 
+        /** Properties of a StructuredDatasetType. */
+        interface IStructuredDatasetType {
+
+            /** StructuredDatasetType columns */
+            columns?: (flyteidl.core.StructuredDatasetType.IDatasetColumn[]|null);
+
+            /** StructuredDatasetType format */
+            format?: (string|null);
+
+            /** StructuredDatasetType externalSchemaType */
+            externalSchemaType?: (string|null);
+
+            /** StructuredDatasetType externalSchemaBytes */
+            externalSchemaBytes?: (Uint8Array|null);
+        }
+
+        /** Represents a StructuredDatasetType. */
+        class StructuredDatasetType implements IStructuredDatasetType {
+
+            /**
+             * Constructs a new StructuredDatasetType.
+             * @param [properties] Properties to set
+             */
+            constructor(properties?: flyteidl.core.IStructuredDatasetType);
+
+            /** StructuredDatasetType columns. */
+            public columns: flyteidl.core.StructuredDatasetType.IDatasetColumn[];
+
+            /** StructuredDatasetType format. */
+            public format: string;
+
+            /** StructuredDatasetType externalSchemaType. */
+            public externalSchemaType: string;
+
+            /** StructuredDatasetType externalSchemaBytes. */
+            public externalSchemaBytes: Uint8Array;
+
+            /**
+             * Creates a new StructuredDatasetType instance using the specified properties.
+             * @param [properties] Properties to set
+             * @returns StructuredDatasetType instance
+             */
+            public static create(properties?: flyteidl.core.IStructuredDatasetType): flyteidl.core.StructuredDatasetType;
+
+            /**
+             * Encodes the specified StructuredDatasetType message. Does not implicitly {@link flyteidl.core.StructuredDatasetType.verify|verify} messages.
+             * @param message StructuredDatasetType message or plain object to encode
+             * @param [writer] Writer to encode to
+             * @returns Writer
+             */
+            public static encode(message: flyteidl.core.IStructuredDatasetType, writer?: $protobuf.Writer): $protobuf.Writer;
+
+            /**
+             * Decodes a StructuredDatasetType message from the specified reader or buffer.
+             * @param reader Reader or buffer to decode from
+             * @param [length] Message length if known beforehand
+             * @returns StructuredDatasetType
+             * @throws {Error} If the payload is not a reader or valid buffer
+             * @throws {$protobuf.util.ProtocolError} If required fields are missing
+             */
+            public static decode(reader: ($protobuf.Reader|Uint8Array), length?: number): flyteidl.core.StructuredDatasetType;
+
+            /**
+             * Verifies a StructuredDatasetType message.
+             * @param message Plain object to verify
+             * @returns `null` if valid, otherwise the reason why it is not
+             */
+            public static verify(message: { [k: string]: any }): (string|null);
+        }
+
+        namespace StructuredDatasetType {
+
+            /** Properties of a DatasetColumn. */
+            interface IDatasetColumn {
+
+                /** DatasetColumn name */
+                name?: (string|null);
+
+                /** DatasetColumn literalType */
+                literalType?: (flyteidl.core.ILiteralType|null);
+            }
+
+            /** Represents a DatasetColumn. */
+            class DatasetColumn implements IDatasetColumn {
+
+                /**
+                 * Constructs a new DatasetColumn.
+                 * @param [properties] Properties to set
+                 */
+                constructor(properties?: flyteidl.core.StructuredDatasetType.IDatasetColumn);
+
+                /** DatasetColumn name. */
+                public name: string;
+
+                /** DatasetColumn literalType. */
+                public literalType?: (flyteidl.core.ILiteralType|null);
+
+                /**
+                 * Creates a new DatasetColumn instance using the specified properties.
+                 * @param [properties] Properties to set
+                 * @returns DatasetColumn instance
+                 */
+                public static create(properties?: flyteidl.core.StructuredDatasetType.IDatasetColumn): flyteidl.core.StructuredDatasetType.DatasetColumn;
+
+                /**
+                 * Encodes the specified DatasetColumn message. Does not implicitly {@link flyteidl.core.StructuredDatasetType.DatasetColumn.verify|verify} messages.
+                 * @param message DatasetColumn message or plain object to encode
+                 * @param [writer] Writer to encode to
+                 * @returns Writer
+                 */
+                public static encode(message: flyteidl.core.StructuredDatasetType.IDatasetColumn, writer?: $protobuf.Writer): $protobuf.Writer;
+
+                /**
+                 * Decodes a DatasetColumn message from the specified reader or buffer.
+                 * @param reader Reader or buffer to decode from
+                 * @param [length] Message length if known beforehand
+                 * @returns DatasetColumn
+                 * @throws {Error} If the payload is not a reader or valid buffer
+                 * @throws {$protobuf.util.ProtocolError} If required fields are missing
+                 */
+                public static decode(reader: ($protobuf.Reader|Uint8Array), length?: number): flyteidl.core.StructuredDatasetType.DatasetColumn;
+
+                /**
+                 * Verifies a DatasetColumn message.
+                 * @param message Plain object to verify
+                 * @returns `null` if valid, otherwise the reason why it is not
+                 */
+                public static verify(message: { [k: string]: any }): (string|null);
+            }
+        }
+
         /** Properties of a BlobType. */
         interface IBlobType {
 
@@ -3064,6 +3311,9 @@ export namespace flyteidl {
             /** LiteralType enumType */
             enumType?: (flyteidl.core.IEnumType|null);
 
+            /** LiteralType structuredDatasetType */
+            structuredDatasetType?: (flyteidl.core.IStructuredDatasetType|null);
+
             /** LiteralType metadata */
             metadata?: (google.protobuf.IStruct|null);
         }
@@ -3095,11 +3345,14 @@ export namespace flyteidl {
             /** LiteralType enumType. */
             public enumType?: (flyteidl.core.IEnumType|null);
 
+            /** LiteralType structuredDatasetType. */
+            public structuredDatasetType?: (flyteidl.core.IStructuredDatasetType|null);
+
             /** LiteralType metadata. */
             public metadata?: (google.protobuf.IStruct|null);
 
             /** LiteralType type. */
-            public type?: ("simple"|"schema"|"collectionType"|"mapValueType"|"blob"|"enumType");
+            public type?: ("simple"|"schema"|"collectionType"|"mapValueType"|"blob"|"enumType"|"structuredDatasetType");
 
             /**
              * Creates a new LiteralType instance using the specified properties.

package/gen/pb-js/flyteidl.js

@@ -5207,6 +5207,247 @@ export const flyteidl = $root.flyteidl = (() => {
             return Schema;
         })();
 
+        core.StructuredDatasetMetadata = (function() {
+
+            /**
+             * Properties of a StructuredDatasetMetadata.
+             * @memberof flyteidl.core
+             * @interface IStructuredDatasetMetadata
+             * @property {flyteidl.core.IStructuredDatasetType|null} [structuredDatasetType] StructuredDatasetMetadata structuredDatasetType
+             */
+
+            /**
+             * Constructs a new StructuredDatasetMetadata.
+             * @memberof flyteidl.core
+             * @classdesc Represents a StructuredDatasetMetadata.
+             * @implements IStructuredDatasetMetadata
+             * @constructor
+             * @param {flyteidl.core.IStructuredDatasetMetadata=} [properties] Properties to set
+             */
+            function StructuredDatasetMetadata(properties) {
+                if (properties)
+                    for (let keys = Object.keys(properties), i = 0; i < keys.length; ++i)
+                        if (properties[keys[i]] != null)
+                            this[keys[i]] = properties[keys[i]];
+            }
+
+            /**
+             * StructuredDatasetMetadata structuredDatasetType.
+             * @member {flyteidl.core.IStructuredDatasetType|null|undefined} structuredDatasetType
+             * @memberof flyteidl.core.StructuredDatasetMetadata
+             * @instance
+             */
+            StructuredDatasetMetadata.prototype.structuredDatasetType = null;
+
+            /**
+             * Creates a new StructuredDatasetMetadata instance using the specified properties.
+             * @function create
+             * @memberof flyteidl.core.StructuredDatasetMetadata
+             * @static
+             * @param {flyteidl.core.IStructuredDatasetMetadata=} [properties] Properties to set
+             * @returns {flyteidl.core.StructuredDatasetMetadata} StructuredDatasetMetadata instance
+             */
+            StructuredDatasetMetadata.create = function create(properties) {
+                return new StructuredDatasetMetadata(properties);
+            };
+
+            /**
+             * Encodes the specified StructuredDatasetMetadata message. Does not implicitly {@link flyteidl.core.StructuredDatasetMetadata.verify|verify} messages.
+             * @function encode
+             * @memberof flyteidl.core.StructuredDatasetMetadata
+             * @static
+             * @param {flyteidl.core.IStructuredDatasetMetadata} message StructuredDatasetMetadata message or plain object to encode
+             * @param {$protobuf.Writer} [writer] Writer to encode to
+             * @returns {$protobuf.Writer} Writer
+             */
+            StructuredDatasetMetadata.encode = function encode(message, writer) {
+                if (!writer)
+                    writer = $Writer.create();
+                if (message.structuredDatasetType != null && message.hasOwnProperty("structuredDatasetType"))
+                    $root.flyteidl.core.StructuredDatasetType.encode(message.structuredDatasetType, writer.uint32(/* id 1, wireType 2 =*/10).fork()).ldelim();
+                return writer;
+            };
+
+            /**
+             * Decodes a StructuredDatasetMetadata message from the specified reader or buffer.
+             * @function decode
+             * @memberof flyteidl.core.StructuredDatasetMetadata
+             * @static
+             * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from
+             * @param {number} [length] Message length if known beforehand
+             * @returns {flyteidl.core.StructuredDatasetMetadata} StructuredDatasetMetadata
+             * @throws {Error} If the payload is not a reader or valid buffer
+             * @throws {$protobuf.util.ProtocolError} If required fields are missing
+             */
+            StructuredDatasetMetadata.decode = function decode(reader, length) {
+                if (!(reader instanceof $Reader))
+                    reader = $Reader.create(reader);
+                let end = length === undefined ? reader.len : reader.pos + length, message = new $root.flyteidl.core.StructuredDatasetMetadata();
+                while (reader.pos < end) {
+                    let tag = reader.uint32();
+                    switch (tag >>> 3) {
+                    case 1:
+                        message.structuredDatasetType = $root.flyteidl.core.StructuredDatasetType.decode(reader, reader.uint32());
+                        break;
+                    default:
+                        reader.skipType(tag & 7);
+                        break;
+                    }
+                }
+                return message;
+            };
+
+            /**
+             * Verifies a StructuredDatasetMetadata message.
+             * @function verify
+             * @memberof flyteidl.core.StructuredDatasetMetadata
+             * @static
+             * @param {Object.<string,*>} message Plain object to verify
+             * @returns {string|null} `null` if valid, otherwise the reason why it is not
+             */
+            StructuredDatasetMetadata.verify = function verify(message) {
+                if (typeof message !== "object" || message === null)
+                    return "object expected";
+                if (message.structuredDatasetType != null && message.hasOwnProperty("structuredDatasetType")) {
+                    let error = $root.flyteidl.core.StructuredDatasetType.verify(message.structuredDatasetType);
+                    if (error)
+                        return "structuredDatasetType." + error;
+                }
+                return null;
+            };
+
+            return StructuredDatasetMetadata;
+        })();
+
+        core.StructuredDataset = (function() {
+
+            /**
+             * Properties of a StructuredDataset.
+             * @memberof flyteidl.core
+             * @interface IStructuredDataset
+             * @property {string|null} [uri] StructuredDataset uri
+             * @property {flyteidl.core.IStructuredDatasetMetadata|null} [metadata] StructuredDataset metadata
+             */
+
+            /**
+             * Constructs a new StructuredDataset.
+             * @memberof flyteidl.core
+             * @classdesc Represents a StructuredDataset.
+             * @implements IStructuredDataset
+             * @constructor
+             * @param {flyteidl.core.IStructuredDataset=} [properties] Properties to set
+             */
+            function StructuredDataset(properties) {
+                if (properties)
+                    for (let keys = Object.keys(properties), i = 0; i < keys.length; ++i)
+                        if (properties[keys[i]] != null)
+                            this[keys[i]] = properties[keys[i]];
+            }
+
+            /**
+             * StructuredDataset uri.
+             * @member {string} uri
+             * @memberof flyteidl.core.StructuredDataset
+             * @instance
+             */
+            StructuredDataset.prototype.uri = "";
+
+            /**
+             * StructuredDataset metadata.
+             * @member {flyteidl.core.IStructuredDatasetMetadata|null|undefined} metadata
+             * @memberof flyteidl.core.StructuredDataset
+             * @instance
+             */
+            StructuredDataset.prototype.metadata = null;
+
+            /**
+             * Creates a new StructuredDataset instance using the specified properties.
+             * @function create
+             * @memberof flyteidl.core.StructuredDataset
+             * @static
+             * @param {flyteidl.core.IStructuredDataset=} [properties] Properties to set
+             * @returns {flyteidl.core.StructuredDataset} StructuredDataset instance
+             */
+            StructuredDataset.create = function create(properties) {
+                return new StructuredDataset(properties);
+            };
+
+            /**
+             * Encodes the specified StructuredDataset message. Does not implicitly {@link flyteidl.core.StructuredDataset.verify|verify} messages.
+             * @function encode
+             * @memberof flyteidl.core.StructuredDataset
+             * @static
+             * @param {flyteidl.core.IStructuredDataset} message StructuredDataset message or plain object to encode
+             * @param {$protobuf.Writer} [writer] Writer to encode to
+             * @returns {$protobuf.Writer} Writer
+             */
+            StructuredDataset.encode = function encode(message, writer) {
+                if (!writer)
+                    writer = $Writer.create();
+                if (message.uri != null && message.hasOwnProperty("uri"))
+                    writer.uint32(/* id 1, wireType 2 =*/10).string(message.uri);
+                if (message.metadata != null && message.hasOwnProperty("metadata"))
+                    $root.flyteidl.core.StructuredDatasetMetadata.encode(message.metadata, writer.uint32(/* id 2, wireType 2 =*/18).fork()).ldelim();
+                return writer;
+            };
+
+            /**
+             * Decodes a StructuredDataset message from the specified reader or buffer.
+             * @function decode
+             * @memberof flyteidl.core.StructuredDataset
+             * @static
+             * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from
+             * @param {number} [length] Message length if known beforehand
+             * @returns {flyteidl.core.StructuredDataset} StructuredDataset
+             * @throws {Error} If the payload is not a reader or valid buffer
+             * @throws {$protobuf.util.ProtocolError} If required fields are missing
+             */
+            StructuredDataset.decode = function decode(reader, length) {
+                if (!(reader instanceof $Reader))
+                    reader = $Reader.create(reader);
+                let end = length === undefined ? reader.len : reader.pos + length, message = new $root.flyteidl.core.StructuredDataset();
+                while (reader.pos < end) {
+                    let tag = reader.uint32();
+                    switch (tag >>> 3) {
+                    case 1:
+                        message.uri = reader.string();
+                        break;
+                    case 2:
+                        message.metadata = $root.flyteidl.core.StructuredDatasetMetadata.decode(reader, reader.uint32());
+                        break;
+                    default:
+                        reader.skipType(tag & 7);
+                        break;
+                    }
+                }
+                return message;
+            };
+
+            /**
+             * Verifies a StructuredDataset message.
+             * @function verify
+             * @memberof flyteidl.core.StructuredDataset
+             * @static
+             * @param {Object.<string,*>} message Plain object to verify
+             * @returns {string|null} `null` if valid, otherwise the reason why it is not
+             */
+            StructuredDataset.verify = function verify(message) {
+                if (typeof message !== "object" || message === null)
+                    return "object expected";
+                if (message.uri != null && message.hasOwnProperty("uri"))
+                    if (!$util.isString(message.uri))
+                        return "uri: string expected";
+                if (message.metadata != null && message.hasOwnProperty("metadata")) {
+                    let error = $root.flyteidl.core.StructuredDatasetMetadata.verify(message.metadata);
+                    if (error)
+                        return "metadata." + error;
+                }
+                return null;
+            };
+
+            return StructuredDataset;
+        })();
+
         core.Scalar = (function() {
 
             /**
@@ -5220,6 +5461,7 @@ export const flyteidl = $root.flyteidl = (() => {
              * @property {flyteidl.core.IVoid|null} [noneType] Scalar noneType
              * @property {flyteidl.core.IError|null} [error] Scalar error
              * @property {google.protobuf.IStruct|null} [generic] Scalar generic
+             * @property {flyteidl.core.IStructuredDataset|null} [structuredDataset] Scalar structuredDataset
              */
 
             /**
@@ -5293,17 +5535,25 @@ export const flyteidl = $root.flyteidl = (() => {
              */
             Scalar.prototype.generic = null;
 
+            /**
+             * Scalar structuredDataset.
+             * @member {flyteidl.core.IStructuredDataset|null|undefined} structuredDataset
+             * @memberof flyteidl.core.Scalar
+             * @instance
+             */
+            Scalar.prototype.structuredDataset = null;
+
             // OneOf field names bound to virtual getters and setters
             let $oneOfFields;
 
             /**
              * Scalar value.
-             * @member {"primitive"|"blob"|"binary"|"schema"|"noneType"|"error"|"generic"|undefined} value
+             * @member {"primitive"|"blob"|"binary"|"schema"|"noneType"|"error"|"generic"|"structuredDataset"|undefined} value
              * @memberof flyteidl.core.Scalar
              * @instance
              */
             Object.defineProperty(Scalar.prototype, "value", {
-                get: $util.oneOfGetter($oneOfFields = ["primitive", "blob", "binary", "schema", "noneType", "error", "generic"]),
+                get: $util.oneOfGetter($oneOfFields = ["primitive", "blob", "binary", "schema", "noneType", "error", "generic", "structuredDataset"]),
                 set: $util.oneOfSetter($oneOfFields)
             });
 
@@ -5345,6 +5595,8 @@ export const flyteidl = $root.flyteidl = (() => {
                     $root.flyteidl.core.Error.encode(message.error, writer.uint32(/* id 6, wireType 2 =*/50).fork()).ldelim();
                 if (message.generic != null && message.hasOwnProperty("generic"))
                     $root.google.protobuf.Struct.encode(message.generic, writer.uint32(/* id 7, wireType 2 =*/58).fork()).ldelim();
+                if (message.structuredDataset != null && message.hasOwnProperty("structuredDataset"))
+                    $root.flyteidl.core.StructuredDataset.encode(message.structuredDataset, writer.uint32(/* id 8, wireType 2 =*/66).fork()).ldelim();
                 return writer;
             };
 
@@ -5387,6 +5639,9 @@ export const flyteidl = $root.flyteidl = (() => {
                     case 7:
                         message.generic = $root.google.protobuf.Struct.decode(reader, reader.uint32());
                         break;
+                    case 8:
+                        message.structuredDataset = $root.flyteidl.core.StructuredDataset.decode(reader, reader.uint32());
+                        break;
                     default:
                         reader.skipType(tag & 7);
                         break;
@@ -5475,6 +5730,16 @@ export const flyteidl = $root.flyteidl = (() => {
                             return "generic." + error;
                     }
                 }
+                if (message.structuredDataset != null && message.hasOwnProperty("structuredDataset")) {
+                    if (properties.value === 1)
+                        return "value: multiple values";
+                    properties.value = 1;
+                    {
+                        let error = $root.flyteidl.core.StructuredDataset.verify(message.structuredDataset);
+                        if (error)
+                            return "structuredDataset." + error;
+                    }
+                }
                 return null;
             };
 
@@ -7027,6 +7292,306 @@ export const flyteidl = $root.flyteidl = (() => {
             return SchemaType;
         })();
 
+        core.StructuredDatasetType = (function() {
+
+            /**
+             * Properties of a StructuredDatasetType.
+             * @memberof flyteidl.core
+             * @interface IStructuredDatasetType
+             * @property {Array.<flyteidl.core.StructuredDatasetType.IDatasetColumn>|null} [columns] StructuredDatasetType columns
+             * @property {string|null} [format] StructuredDatasetType format
+             * @property {string|null} [externalSchemaType] StructuredDatasetType externalSchemaType
+             * @property {Uint8Array|null} [externalSchemaBytes] StructuredDatasetType externalSchemaBytes
+             */
+
+            /**
+             * Constructs a new StructuredDatasetType.
+             * @memberof flyteidl.core
+             * @classdesc Represents a StructuredDatasetType.
+             * @implements IStructuredDatasetType
+             * @constructor
+             * @param {flyteidl.core.IStructuredDatasetType=} [properties] Properties to set
+             */
+            function StructuredDatasetType(properties) {
+                this.columns = [];
+                if (properties)
+                    for (let keys = Object.keys(properties), i = 0; i < keys.length; ++i)
+                        if (properties[keys[i]] != null)
+                            this[keys[i]] = properties[keys[i]];
+            }
+
+            /**
+             * StructuredDatasetType columns.
+             * @member {Array.<flyteidl.core.StructuredDatasetType.IDatasetColumn>} columns
+             * @memberof flyteidl.core.StructuredDatasetType
+             * @instance
+             */
+            StructuredDatasetType.prototype.columns = $util.emptyArray;
+
+            /**
+             * StructuredDatasetType format.
+             * @member {string} format
+             * @memberof flyteidl.core.StructuredDatasetType
+             * @instance
+             */
+            StructuredDatasetType.prototype.format = "";
+
+            /**
+             * StructuredDatasetType externalSchemaType.
+             * @member {string} externalSchemaType
+             * @memberof flyteidl.core.StructuredDatasetType
+             * @instance
+             */
+            StructuredDatasetType.prototype.externalSchemaType = "";
+
+            /**
+             * StructuredDatasetType externalSchemaBytes.
+             * @member {Uint8Array} externalSchemaBytes
+             * @memberof flyteidl.core.StructuredDatasetType
+             * @instance
+             */
+            StructuredDatasetType.prototype.externalSchemaBytes = $util.newBuffer([]);
+
+            /**
+             * Creates a new StructuredDatasetType instance using the specified properties.
+             * @function create
+             * @memberof flyteidl.core.StructuredDatasetType
+             * @static
+             * @param {flyteidl.core.IStructuredDatasetType=} [properties] Properties to set
+             * @returns {flyteidl.core.StructuredDatasetType} StructuredDatasetType instance
+             */
+            StructuredDatasetType.create = function create(properties) {
+                return new StructuredDatasetType(properties);
+            };
+
+            /**
+             * Encodes the specified StructuredDatasetType message. Does not implicitly {@link flyteidl.core.StructuredDatasetType.verify|verify} messages.
+             * @function encode
+             * @memberof flyteidl.core.StructuredDatasetType
+             * @static
+             * @param {flyteidl.core.IStructuredDatasetType} message StructuredDatasetType message or plain object to encode
+             * @param {$protobuf.Writer} [writer] Writer to encode to
+             * @returns {$protobuf.Writer} Writer
+             */
+            StructuredDatasetType.encode = function encode(message, writer) {
+                if (!writer)
+                    writer = $Writer.create();
+                if (message.columns != null && message.columns.length)
+                    for (let i = 0; i < message.columns.length; ++i)
+                        $root.flyteidl.core.StructuredDatasetType.DatasetColumn.encode(message.columns[i], writer.uint32(/* id 1, wireType 2 =*/10).fork()).ldelim();
+                if (message.format != null && message.hasOwnProperty("format"))
+                    writer.uint32(/* id 2, wireType 2 =*/18).string(message.format);
+                if (message.externalSchemaType != null && message.hasOwnProperty("externalSchemaType"))
+                    writer.uint32(/* id 3, wireType 2 =*/26).string(message.externalSchemaType);
+                if (message.externalSchemaBytes != null && message.hasOwnProperty("externalSchemaBytes"))
+                    writer.uint32(/* id 4, wireType 2 =*/34).bytes(message.externalSchemaBytes);
+                return writer;
+            };
+
+            /**
+             * Decodes a StructuredDatasetType message from the specified reader or buffer.
+             * @function decode
+             * @memberof flyteidl.core.StructuredDatasetType
+             * @static
+             * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from
+             * @param {number} [length] Message length if known beforehand
+             * @returns {flyteidl.core.StructuredDatasetType} StructuredDatasetType
+             * @throws {Error} If the payload is not a reader or valid buffer
+             * @throws {$protobuf.util.ProtocolError} If required fields are missing
+             */
+            StructuredDatasetType.decode = function decode(reader, length) {
+                if (!(reader instanceof $Reader))
+                    reader = $Reader.create(reader);
+                let end = length === undefined ? reader.len : reader.pos + length, message = new $root.flyteidl.core.StructuredDatasetType();
+                while (reader.pos < end) {
+                    let tag = reader.uint32();
+                    switch (tag >>> 3) {
+                    case 1:
+                        if (!(message.columns && message.columns.length))
+                            message.columns = [];
+                        message.columns.push($root.flyteidl.core.StructuredDatasetType.DatasetColumn.decode(reader, reader.uint32()));
+                        break;
+                    case 2:
+                        message.format = reader.string();
+                        break;
+                    case 3:
+                        message.externalSchemaType = reader.string();
+                        break;
+                    case 4:
+                        message.externalSchemaBytes = reader.bytes();
+                        break;
+                    default:
+                        reader.skipType(tag & 7);
+                        break;
+                    }
+                }
+                return message;
+            };
+
+            /**
+             * Verifies a StructuredDatasetType message.
+             * @function verify
+             * @memberof flyteidl.core.StructuredDatasetType
+             * @static
+             * @param {Object.<string,*>} message Plain object to verify
+             * @returns {string|null} `null` if valid, otherwise the reason why it is not
+             */
+            StructuredDatasetType.verify = function verify(message) {
+                if (typeof message !== "object" || message === null)
+                    return "object expected";
+                if (message.columns != null && message.hasOwnProperty("columns")) {
+                    if (!Array.isArray(message.columns))
+                        return "columns: array expected";
+                    for (let i = 0; i < message.columns.length; ++i) {
+                        let error = $root.flyteidl.core.StructuredDatasetType.DatasetColumn.verify(message.columns[i]);
+                        if (error)
+                            return "columns." + error;
+                    }
+                }
+                if (message.format != null && message.hasOwnProperty("format"))
+                    if (!$util.isString(message.format))
+                        return "format: string expected";
+                if (message.externalSchemaType != null && message.hasOwnProperty("externalSchemaType"))
+                    if (!$util.isString(message.externalSchemaType))
+                        return "externalSchemaType: string expected";
+                if (message.externalSchemaBytes != null && message.hasOwnProperty("externalSchemaBytes"))
+                    if (!(message.externalSchemaBytes && typeof message.externalSchemaBytes.length === "number" || $util.isString(message.externalSchemaBytes)))
+                        return "externalSchemaBytes: buffer expected";
+                return null;
+            };
+
+            StructuredDatasetType.DatasetColumn = (function() {
+
+                /**
+                 * Properties of a DatasetColumn.
+                 * @memberof flyteidl.core.StructuredDatasetType
+                 * @interface IDatasetColumn
+                 * @property {string|null} [name] DatasetColumn name
+                 * @property {flyteidl.core.ILiteralType|null} [literalType] DatasetColumn literalType
+                 */
+
+                /**
+                 * Constructs a new DatasetColumn.
+                 * @memberof flyteidl.core.StructuredDatasetType
+                 * @classdesc Represents a DatasetColumn.
+                 * @implements IDatasetColumn
+                 * @constructor
+                 * @param {flyteidl.core.StructuredDatasetType.IDatasetColumn=} [properties] Properties to set
+                 */
+                function DatasetColumn(properties) {
+                    if (properties)
+                        for (let keys = Object.keys(properties), i = 0; i < keys.length; ++i)
+                            if (properties[keys[i]] != null)
+                                this[keys[i]] = properties[keys[i]];
+                }
+
+                /**
+                 * DatasetColumn name.
+                 * @member {string} name
+                 * @memberof flyteidl.core.StructuredDatasetType.DatasetColumn
+                 * @instance
+                 */
+                DatasetColumn.prototype.name = "";
+
+                /**
+                 * DatasetColumn literalType.
+                 * @member {flyteidl.core.ILiteralType|null|undefined} literalType
+                 * @memberof flyteidl.core.StructuredDatasetType.DatasetColumn
+                 * @instance
+                 */
+                DatasetColumn.prototype.literalType = null;
+
+                /**
+                 * Creates a new DatasetColumn instance using the specified properties.
+                 * @function create
+                 * @memberof flyteidl.core.StructuredDatasetType.DatasetColumn
+                 * @static
+                 * @param {flyteidl.core.StructuredDatasetType.IDatasetColumn=} [properties] Properties to set
+                 * @returns {flyteidl.core.StructuredDatasetType.DatasetColumn} DatasetColumn instance
+                 */
+                DatasetColumn.create = function create(properties) {
+                    return new DatasetColumn(properties);
+                };
+
+                /**
+                 * Encodes the specified DatasetColumn message. Does not implicitly {@link flyteidl.core.StructuredDatasetType.DatasetColumn.verify|verify} messages.
+                 * @function encode
+                 * @memberof flyteidl.core.StructuredDatasetType.DatasetColumn
+                 * @static
+                 * @param {flyteidl.core.StructuredDatasetType.IDatasetColumn} message DatasetColumn message or plain object to encode
+                 * @param {$protobuf.Writer} [writer] Writer to encode to
+                 * @returns {$protobuf.Writer} Writer
+                 */
+                DatasetColumn.encode = function encode(message, writer) {
+                    if (!writer)
+                        writer = $Writer.create();
+                    if (message.name != null && message.hasOwnProperty("name"))
+                        writer.uint32(/* id 1, wireType 2 =*/10).string(message.name);
+                    if (message.literalType != null && message.hasOwnProperty("literalType"))
+                        $root.flyteidl.core.LiteralType.encode(message.literalType, writer.uint32(/* id 2, wireType 2 =*/18).fork()).ldelim();
+                    return writer;
+                };
+
+                /**
+                 * Decodes a DatasetColumn message from the specified reader or buffer.
+                 * @function decode
+                 * @memberof flyteidl.core.StructuredDatasetType.DatasetColumn
+                 * @static
+                 * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from
+                 * @param {number} [length] Message length if known beforehand
+                 * @returns {flyteidl.core.StructuredDatasetType.DatasetColumn} DatasetColumn
+                 * @throws {Error} If the payload is not a reader or valid buffer
+                 * @throws {$protobuf.util.ProtocolError} If required fields are missing
+                 */
+                DatasetColumn.decode = function decode(reader, length) {
+                    if (!(reader instanceof $Reader))
+                        reader = $Reader.create(reader);
+                    let end = length === undefined ? reader.len : reader.pos + length, message = new $root.flyteidl.core.StructuredDatasetType.DatasetColumn();
+                    while (reader.pos < end) {
+                        let tag = reader.uint32();
+                        switch (tag >>> 3) {
+                        case 1:
+                            message.name = reader.string();
+                            break;
+                        case 2:
+                            message.literalType = $root.flyteidl.core.LiteralType.decode(reader, reader.uint32());
+                            break;
+                        default:
+                            reader.skipType(tag & 7);
+                            break;
+                        }
+                    }
+                    return message;
+                };
+
+                /**
+                 * Verifies a DatasetColumn message.
+                 * @function verify
+                 * @memberof flyteidl.core.StructuredDatasetType.DatasetColumn
+                 * @static
+                 * @param {Object.<string,*>} message Plain object to verify
+                 * @returns {string|null} `null` if valid, otherwise the reason why it is not
+                 */
+                DatasetColumn.verify = function verify(message) {
+                    if (typeof message !== "object" || message === null)
+                        return "object expected";
+                    if (message.name != null && message.hasOwnProperty("name"))
+                        if (!$util.isString(message.name))
+                            return "name: string expected";
+                    if (message.literalType != null && message.hasOwnProperty("literalType")) {
+                        let error = $root.flyteidl.core.LiteralType.verify(message.literalType);
+                        if (error)
+                            return "literalType." + error;
+                    }
+                    return null;
+                };
+
+                return DatasetColumn;
+            })();
+
+            return StructuredDatasetType;
+        })();
+
         core.BlobType = (function() {
 
             /**
@@ -7303,6 +7868,7 @@ export const flyteidl = $root.flyteidl = (() => {
              * @property {flyteidl.core.ILiteralType|null} [mapValueType] LiteralType mapValueType
              * @property {flyteidl.core.IBlobType|null} [blob] LiteralType blob
              * @property {flyteidl.core.IEnumType|null} [enumType] LiteralType enumType
+             * @property {flyteidl.core.IStructuredDatasetType|null} [structuredDatasetType] LiteralType structuredDatasetType
              * @property {google.protobuf.IStruct|null} [metadata] LiteralType metadata
              */
 
@@ -7369,6 +7935,14 @@ export const flyteidl = $root.flyteidl = (() => {
              */
             LiteralType.prototype.enumType = null;
 
+            /**
+             * LiteralType structuredDatasetType.
+             * @member {flyteidl.core.IStructuredDatasetType|null|undefined} structuredDatasetType
+             * @memberof flyteidl.core.LiteralType
+             * @instance
+             */
+            LiteralType.prototype.structuredDatasetType = null;
+
             /**
              * LiteralType metadata.
              * @member {google.protobuf.IStruct|null|undefined} metadata
@@ -7382,12 +7956,12 @@ export const flyteidl = $root.flyteidl = (() => {
 
             /**
              * LiteralType type.
-             * @member {"simple"|"schema"|"collectionType"|"mapValueType"|"blob"|"enumType"|undefined} type
+             * @member {"simple"|"schema"|"collectionType"|"mapValueType"|"blob"|"enumType"|"structuredDatasetType"|undefined} type
              * @memberof flyteidl.core.LiteralType
              * @instance
              */
             Object.defineProperty(LiteralType.prototype, "type", {
-                get: $util.oneOfGetter($oneOfFields = ["simple", "schema", "collectionType", "mapValueType", "blob", "enumType"]),
+                get: $util.oneOfGetter($oneOfFields = ["simple", "schema", "collectionType", "mapValueType", "blob", "enumType", "structuredDatasetType"]),
                 set: $util.oneOfSetter($oneOfFields)
             });
 
@@ -7429,6 +8003,8 @@ export const flyteidl = $root.flyteidl = (() => {
                     $root.google.protobuf.Struct.encode(message.metadata, writer.uint32(/* id 6, wireType 2 =*/50).fork()).ldelim();
                 if (message.enumType != null && message.hasOwnProperty("enumType"))
                     $root.flyteidl.core.EnumType.encode(message.enumType, writer.uint32(/* id 7, wireType 2 =*/58).fork()).ldelim();
+                if (message.structuredDatasetType != null && message.hasOwnProperty("structuredDatasetType"))
+                    $root.flyteidl.core.StructuredDatasetType.encode(message.structuredDatasetType, writer.uint32(/* id 8, wireType 2 =*/66).fork()).ldelim();
                 return writer;
             };
 
@@ -7468,6 +8044,9 @@ export const flyteidl = $root.flyteidl = (() => {
                     case 7:
                         message.enumType = $root.flyteidl.core.EnumType.decode(reader, reader.uint32());
                         break;
+                    case 8:
+                        message.structuredDatasetType = $root.flyteidl.core.StructuredDatasetType.decode(reader, reader.uint32());
+                        break;
                     case 6:
                         message.metadata = $root.google.protobuf.Struct.decode(reader, reader.uint32());
                         break;
@@ -7559,6 +8138,16 @@ export const flyteidl = $root.flyteidl = (() => {
                             return "enumType." + error;
                     }
                 }
+                if (message.structuredDatasetType != null && message.hasOwnProperty("structuredDatasetType")) {
+                    if (properties.type === 1)
+                        return "type: multiple values";
+                    properties.type = 1;
+                    {
+                        let error = $root.flyteidl.core.StructuredDatasetType.verify(message.structuredDatasetType);
+                        if (error)
+                            return "structuredDatasetType." + error;
+                    }
+                }
                 if (message.metadata != null && message.hasOwnProperty("metadata")) {
                     let error = $root.google.protobuf.Struct.verify(message.metadata);
                     if (error)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flyteorg/flyteidl",
-  "version": "0.21.16",
+  "version": "0.21.20",
   "description": "Compiled protocol buffers and gRPC service clients/servers for Flyte IDLs",
   "repository": {
     "type": "git",

package/protos/flyteidl/core/literals.proto

@@ -52,6 +52,25 @@ message Schema {
     SchemaType type = 3;
 }
 
+message StructuredDatasetMetadata {
+    // Bundle the type information along with the literal.
+    // This is here because StructuredDatasets can often be more defined at run time than at compile time.
+    // That is, at compile time you might only declare a task to return a pandas dataframe or a StructuredDataset,
+    // without any column information, but at run time, you might have that column information.
+    // flytekit python will copy this type information into the literal, from the type information, if not provided by
+    // the various plugins (encoders).
+    // Since this field is run time generated, it's not used for any type checking.
+    StructuredDatasetType structured_dataset_type = 1;
+}
+
+message StructuredDataset {
+    // String location uniquely identifying where the data is.
+    // Should start with the storage location (e.g. s3://, gs://, bq://, etc.)
+    string uri = 1;
+
+    StructuredDatasetMetadata metadata = 2;
+}
+
 message Scalar {
     oneof value {
         Primitive primitive = 1;
@@ -61,6 +80,7 @@ message Scalar {
         Void none_type = 5;
         Error error = 6;
         google.protobuf.Struct generic = 7;
+        StructuredDataset structured_dataset = 8;
     }
 }
 

package/protos/flyteidl/core/types.proto

@@ -43,6 +43,32 @@ message SchemaType {
     repeated SchemaColumn columns = 3;
 }
 
+message StructuredDatasetType {
+    message DatasetColumn {
+        // A unique name within the schema type for the column.
+        string name = 1;
+
+        // The column type.
+        LiteralType literal_type = 2;
+    }
+
+    // A list of ordered columns this schema comprises of.
+    repeated DatasetColumn columns = 1;
+
+    // This is the storage format, the format of the bits at rest
+    // parquet, feather, csv, etc.
+    // For two types to be compatible, the format will need to be an exact match.
+    string format = 2;
+
+    // This is a string representing the type that the bytes in external_schema_bytes are formatted in.
+    // This is an optional field that will not be used for type checking.
+    string external_schema_type = 3;
+
+    // The serialized bytes of a third-party schema library like Arrow.
+    // This is an optional field that will not be used for type checking.
+    bytes external_schema_bytes = 4;
+}
+
 // Defines type behavior for blob objects
 message BlobType {
     enum BlobDimensionality {
@@ -84,6 +110,9 @@ message LiteralType {
 
         // Defines an enum with pre-defined string values.
         EnumType enum_type = 7;
+
+        // Generalized schema support
+        StructuredDatasetType structured_dataset_type = 8;
     }
 
     // This field contains type metadata that is descriptive of the type, but is NOT considered in type-checking.  This might be used by