@aws-sdk/client-appflow 3.211.0 → 3.213.0

package/dist-types/models/models_0.d.ts

@@ -25,6 +25,13 @@ export interface AggregationConfig {
      *       leave them unaggregated. </p>
      */
     aggregationType?: AggregationType | string;
+    /**
+     * <p>The desired file size, in MB, for each output file that Amazon AppFlow writes to the
+     *       flow destination. For each file, Amazon AppFlow attempts to achieve the size that you
+     *       specify. The actual file sizes might differ from this target based on the number and size of
+     *       the records that each file contains.</p>
+     */
+    targetFileSize?: number;
 }
 export declare enum AmplitudeConnectorOperator {
     BETWEEN = "BETWEEN"
@@ -233,6 +240,9 @@ export interface BasicAuthCredentials {
      */
     password: string | undefined;
 }
+export declare enum CatalogType {
+    GLUE = "GLUE"
+}
 /**
  * <p> There was a conflict when processing the request (for example, a flow with the given name
  *       already exists within the account. Check for conflicting resource names and try again. </p>
@@ -2423,6 +2433,10 @@ export declare enum FileType {
     JSON = "JSON",
     PARQUET = "PARQUET"
 }
+export declare enum PathPrefix {
+    EXECUTION_ID = "EXECUTION_ID",
+    SCHEMA_VERSION = "SCHEMA_VERSION"
+}
 export declare enum PrefixFormat {
     DAY = "DAY",
     HOUR = "HOUR",
@@ -2436,19 +2450,47 @@ export declare enum PrefixType {
     PATH_AND_FILENAME = "PATH_AND_FILENAME"
 }
 /**
- * <p> Determines the prefix that Amazon AppFlow applies to the destination folder name.
- *       You can name your destination folders according to the flow frequency and date. </p>
+ * <p>Specifies elements that Amazon AppFlow includes in the file and folder names in the flow
+ *       destination.</p>
  */
 export interface PrefixConfig {
     /**
-     * <p> Determines the format of the prefix, and whether it applies to the file name, file path,
+     * <p>Determines the format of the prefix, and whether it applies to the file name, file path,
      *       or both. </p>
      */
     prefixType?: PrefixType | string;
     /**
-     * <p> Determines the level of granularity that's included in the prefix. </p>
+     * <p>Determines the level of granularity for the date and time that's included in the prefix. </p>
      */
     prefixFormat?: PrefixFormat | string;
+    /**
+     * <p>Specifies whether the destination file path includes either or both of the following
+     *       elements:</p>
+     *          <dl>
+     *             <dt>EXECUTION_ID</dt>
+     *             <dd>
+     *                <p>The ID that Amazon AppFlow assigns to the flow run.</p>
+     *             </dd>
+     *             <dt>SCHEMA_VERSION</dt>
+     *             <dd>
+     *                <p>The version number of your data schema. Amazon AppFlow assigns this version
+     *             number. The version number increases by one when you change any of the following
+     *             settings in your flow configuration:</p>
+     *                <ul>
+     *                   <li>
+     *                      <p>Source-to-destination field mappings</p>
+     *                   </li>
+     *                   <li>
+     *                      <p>Field data types</p>
+     *                   </li>
+     *                   <li>
+     *                      <p>Partition keys</p>
+     *                   </li>
+     *                </ul>
+     *             </dd>
+     *          </dl>
+     */
+    pathPrefixHierarchy?: (PathPrefix | string)[];
 }
 /**
  * <p> The configuration that determines how Amazon AppFlow should format the flow output
@@ -2662,8 +2704,8 @@ export interface UpsolverS3OutputFormatConfig {
      */
     fileType?: FileType | string;
     /**
-     * <p> Determines the prefix that Amazon AppFlow applies to the destination folder name.
-     *       You can name your destination folders according to the flow frequency and date. </p>
+     * <p>Specifies elements that Amazon AppFlow includes in the file and folder names in the flow
+     *       destination.</p>
      */
     prefixConfig: PrefixConfig | undefined;
     /**
@@ -2798,6 +2840,52 @@ export interface DestinationFlowConfig {
      */
     destinationConnectorProperties: DestinationConnectorProperties | undefined;
 }
+/**
+ * <p>Specifies the configuration that Amazon AppFlow uses when it catalogs your data with
+ *       the Glue Data Catalog. When Amazon AppFlow catalogs your data, it stores
+ *       metadata in Data Catalog tables. This metadata represents the data that's transferred
+ *       by the flow that you configure with these settings.</p>
+ *          <note>
+ *             <p>You can configure a flow with these settings only when the flow destination is Amazon S3.</p>
+ *          </note>
+ */
+export interface GlueDataCatalogConfig {
+    /**
+     * <p>The Amazon Resource Name (ARN) of an IAM role that grants Amazon AppFlow the permissions it needs to create Data Catalog tables, databases, and
+     *       partitions.</p>
+     *          <p>For an example IAM policy that has the required permissions, see <a href="https://docs.aws.amazon.com/appflow/latest/userguide/security_iam_id-based-policy-examples.html">Identity-based
+     *         policy examples for Amazon AppFlow</a>.</p>
+     */
+    roleArn: string | undefined;
+    /**
+     * <p>The name of the Data Catalog database that stores the metadata tables that
+     *         Amazon AppFlow creates in your Amazon Web Services account. These tables contain
+     *       metadata for the data that's transferred by the flow that you configure with this
+     *       parameter.</p>
+     *          <note>
+     *             <p>When you configure a new flow with this parameter, you must specify an existing
+     *         database.</p>
+     *          </note>
+     */
+    databaseName: string | undefined;
+    /**
+     * <p>A naming prefix for each Data Catalog table that Amazon AppFlow creates for
+     *       the flow that you configure with this setting. Amazon AppFlow adds the prefix to the
+     *       beginning of the each table name.</p>
+     */
+    tablePrefix: string | undefined;
+}
+/**
+ * <p>Specifies the configuration that Amazon AppFlow uses when it catalogs your data.
+ *       When Amazon AppFlow catalogs your data, it stores metadata in a data catalog.</p>
+ */
+export interface MetadataCatalogConfig {
+    /**
+     * <p>Specifies the configuration that Amazon AppFlow uses when it catalogs your data with the
+     *         Glue Data Catalog.</p>
+     */
+    glueDataCatalog?: GlueDataCatalogConfig;
+}
 /**
  * <p> Specifies the configuration used when importing incremental records from the source.
  *     </p>
@@ -3149,6 +3237,7 @@ export declare enum OperatorPropertiesKeys {
     MASK_LENGTH = "MASK_LENGTH",
     MASK_VALUE = "MASK_VALUE",
     MATH_OPERATION_FIELDS_ORDER = "MATH_OPERATION_FIELDS_ORDER",
+    ORDERED_PARTITION_KEYS_LIST = "ORDERED_PARTITION_KEYS_LIST",
     SOURCE_DATA_TYPE = "SOURCE_DATA_TYPE",
     SUBFIELD_CATEGORY_MAP = "SUBFIELD_CATEGORY_MAP",
     TRUNCATE_LENGTH = "TRUNCATE_LENGTH",
@@ -3164,6 +3253,7 @@ export declare enum TaskType {
     MAP_ALL = "Map_all",
     MASK = "Mask",
     MERGE = "Merge",
+    PARTITION = "Partition",
     PASSTHROUGH = "Passthrough",
     TRUNCATE = "Truncate",
     VALIDATE = "Validate"
@@ -3318,6 +3408,12 @@ export interface CreateFlowRequest {
      * <p> The tags used to organize, track, or control access for your flow. </p>
      */
     tags?: Record<string, string>;
+    /**
+     * <p>Specifies the configuration that Amazon AppFlow uses when it catalogs the data
+     *       that's transferred by the associated flow. When Amazon AppFlow catalogs the data from a
+     *       flow, it stores metadata in a data catalog.</p>
+     */
+    metadataCatalogConfig?: MetadataCatalogConfig;
 }
 export declare enum FlowStatus {
     ACTIVE = "Active",
@@ -3519,6 +3615,65 @@ export interface ExecutionDetails {
      */
     mostRecentExecutionStatus?: ExecutionStatus | string;
 }
+/**
+ * <p>Describes the status of an attempt from Amazon AppFlow to register a
+ *       resource.</p>
+ *          <p>When you run a flow that you've configured to use a metadata catalog, Amazon AppFlow
+ *       registers a metadata table and data partitions with that catalog. This operation provides the
+ *       status of that registration attempt. The operation also indicates how many related resources
+ *         Amazon AppFlow created or updated.</p>
+ */
+export interface RegistrationOutput {
+    /**
+     * <p>Explains the status of the registration attempt from Amazon AppFlow. If the attempt
+     *       fails, the message explains why.</p>
+     */
+    message?: string;
+    /**
+     * <p>Indicates the number of resources that Amazon AppFlow created or updated. Possible
+     *       resources include metadata tables and data partitions.</p>
+     */
+    result?: string;
+    /**
+     * <p>Indicates the status of the registration attempt from Amazon AppFlow.</p>
+     */
+    status?: ExecutionStatus | string;
+}
+/**
+ * <p>Describes the metadata catalog, metadata table, and data partitions that Amazon AppFlow used for the associated flow run.</p>
+ */
+export interface MetadataCatalogDetail {
+    /**
+     * <p>The type of metadata catalog that Amazon AppFlow used for the associated flow run.
+     *       This parameter returns the following value:</p>
+     *          <dl>
+     *             <dt>GLUE</dt>
+     *             <dd>
+     *                <p>The metadata catalog is provided by the Glue Data Catalog. Glue includes the Glue Data Catalog as a component.</p>
+     *             </dd>
+     *          </dl>
+     */
+    catalogType?: CatalogType | string;
+    /**
+     * <p>The name of the table that stores the metadata for the associated flow run. The table
+     *       stores metadata that represents the data that the flow transferred. Amazon AppFlow
+     *       stores the table in the metadata catalog.</p>
+     */
+    tableName?: string;
+    /**
+     * <p>Describes the status of the attempt from Amazon AppFlow to register the metadata
+     *       table with the metadata catalog. Amazon AppFlow creates or updates this table for the
+     *       associated flow run.</p>
+     */
+    tableRegistrationOutput?: RegistrationOutput;
+    /**
+     * <p>Describes the status of the attempt from Amazon AppFlow to register the data
+     *       partitions with the metadata catalog. The data partitions organize the flow output into a
+     *       hierarchical path, such as a folder path in an  S3 bucket. Amazon AppFlow creates the
+     *       partitions (if they don't already exist) based on your flow configuration.</p>
+     */
+    partitionRegistrationOutput?: RegistrationOutput;
+}
 export interface DescribeFlowResponse {
     /**
      * <p> The flow's Amazon Resource Name (ARN). </p>
@@ -3591,6 +3746,33 @@ export interface DescribeFlowResponse {
      * <p> The tags used to organize, track, or control access for your flow. </p>
      */
     tags?: Record<string, string>;
+    /**
+     * <p>Specifies the configuration that Amazon AppFlow uses when it catalogs the data
+     *       that's transferred by the associated flow. When Amazon AppFlow catalogs the data from a
+     *       flow, it stores metadata in a data catalog.</p>
+     */
+    metadataCatalogConfig?: MetadataCatalogConfig;
+    /**
+     * <p>Describes the metadata catalog, metadata table, and data partitions that Amazon AppFlow used for the associated flow run.</p>
+     */
+    lastRunMetadataCatalogDetails?: MetadataCatalogDetail[];
+    /**
+     * <p>The version number of your data schema. Amazon AppFlow assigns this version number.
+     *       The version number increases by one when you change any of the following settings in your flow
+     *       configuration:</p>
+     *          <ul>
+     *             <li>
+     *                <p>Source-to-destination field mappings</p>
+     *             </li>
+     *             <li>
+     *                <p>Field data types</p>
+     *             </li>
+     *             <li>
+     *                <p>Partition keys</p>
+     *             </li>
+     *          </ul>
+     */
+    schemaVersion?: number;
 }
 export interface DescribeFlowExecutionRecordsRequest {
     /**
@@ -3678,6 +3860,10 @@ export interface ExecutionRecord {
      *       run. </p>
      */
     dataPullEndTime?: Date;
+    /**
+     * <p>Describes the metadata catalog, metadata table, and data partitions that Amazon AppFlow used for the associated flow run.</p>
+     */
+    metadataCatalogDetails?: MetadataCatalogDetail[];
 }
 export interface DescribeFlowExecutionRecordsResponse {
     /**
@@ -4022,6 +4208,12 @@ export interface UpdateFlowRequest {
      *       flow run. </p>
      */
     tasks: Task[] | undefined;
+    /**
+     * <p>Specifies the configuration that Amazon AppFlow uses when it catalogs the data
+     *       that's transferred by the associated flow. When Amazon AppFlow catalogs the data from a
+     *       flow, it stores metadata in a data catalog.</p>
+     */
+    metadataCatalogConfig?: MetadataCatalogConfig;
 }
 export interface UpdateFlowResponse {
     /**
@@ -4481,6 +4673,14 @@ export declare const DestinationConnectorPropertiesFilterSensitiveLog: (obj: Des
  * @internal
  */
 export declare const DestinationFlowConfigFilterSensitiveLog: (obj: DestinationFlowConfig) => any;
+/**
+ * @internal
+ */
+export declare const GlueDataCatalogConfigFilterSensitiveLog: (obj: GlueDataCatalogConfig) => any;
+/**
+ * @internal
+ */
+export declare const MetadataCatalogConfigFilterSensitiveLog: (obj: MetadataCatalogConfig) => any;
 /**
  * @internal
  */
@@ -4637,6 +4837,14 @@ export declare const DescribeFlowRequestFilterSensitiveLog: (obj: DescribeFlowRe
  * @internal
  */
 export declare const ExecutionDetailsFilterSensitiveLog: (obj: ExecutionDetails) => any;
+/**
+ * @internal
+ */
+export declare const RegistrationOutputFilterSensitiveLog: (obj: RegistrationOutput) => any;
+/**
+ * @internal
+ */
+export declare const MetadataCatalogDetailFilterSensitiveLog: (obj: MetadataCatalogDetail) => any;
 /**
  * @internal
  */

package/dist-types/ts3.4/models/models_0.d.ts

@@ -13,6 +13,7 @@ export declare enum AggregationType {
 }
 export interface AggregationConfig {
   aggregationType?: AggregationType | string;
+  targetFileSize?: number;
 }
 export declare enum AmplitudeConnectorOperator {
   BETWEEN = "BETWEEN",
@@ -84,6 +85,9 @@ export interface BasicAuthCredentials {
   username: string | undefined;
   password: string | undefined;
 }
+export declare enum CatalogType {
+  GLUE = "GLUE",
+}
 export declare class ConflictException extends __BaseException {
   readonly name: "ConflictException";
   readonly $fault: "client";
@@ -960,6 +964,10 @@ export declare enum FileType {
   JSON = "JSON",
   PARQUET = "PARQUET",
 }
+export declare enum PathPrefix {
+  EXECUTION_ID = "EXECUTION_ID",
+  SCHEMA_VERSION = "SCHEMA_VERSION",
+}
 export declare enum PrefixFormat {
   DAY = "DAY",
   HOUR = "HOUR",
@@ -975,6 +983,7 @@ export declare enum PrefixType {
 export interface PrefixConfig {
   prefixType?: PrefixType | string;
   prefixFormat?: PrefixFormat | string;
+  pathPrefixHierarchy?: (PathPrefix | string)[];
 }
 export interface S3OutputFormatConfig {
   fileType?: FileType | string;
@@ -1048,6 +1057,14 @@ export interface DestinationFlowConfig {
   connectorProfileName?: string;
   destinationConnectorProperties: DestinationConnectorProperties | undefined;
 }
+export interface GlueDataCatalogConfig {
+  roleArn: string | undefined;
+  databaseName: string | undefined;
+  tablePrefix: string | undefined;
+}
+export interface MetadataCatalogConfig {
+  glueDataCatalog?: GlueDataCatalogConfig;
+}
 export interface IncrementalPullConfig {
   datetimeTypeFieldName?: string;
 }
@@ -1148,6 +1165,7 @@ export declare enum OperatorPropertiesKeys {
   MASK_LENGTH = "MASK_LENGTH",
   MASK_VALUE = "MASK_VALUE",
   MATH_OPERATION_FIELDS_ORDER = "MATH_OPERATION_FIELDS_ORDER",
+  ORDERED_PARTITION_KEYS_LIST = "ORDERED_PARTITION_KEYS_LIST",
   SOURCE_DATA_TYPE = "SOURCE_DATA_TYPE",
   SUBFIELD_CATEGORY_MAP = "SUBFIELD_CATEGORY_MAP",
   TRUNCATE_LENGTH = "TRUNCATE_LENGTH",
@@ -1163,6 +1181,7 @@ export declare enum TaskType {
   MAP_ALL = "Map_all",
   MASK = "Mask",
   MERGE = "Merge",
+  PARTITION = "Partition",
   PASSTHROUGH = "Passthrough",
   TRUNCATE = "Truncate",
   VALIDATE = "Validate",
@@ -1204,6 +1223,7 @@ export interface CreateFlowRequest {
   destinationFlowConfigList: DestinationFlowConfig[] | undefined;
   tasks: Task[] | undefined;
   tags?: Record<string, string>;
+  metadataCatalogConfig?: MetadataCatalogConfig;
 }
 export declare enum FlowStatus {
   ACTIVE = "Active",
@@ -1284,6 +1304,17 @@ export interface ExecutionDetails {
   mostRecentExecutionTime?: Date;
   mostRecentExecutionStatus?: ExecutionStatus | string;
 }
+export interface RegistrationOutput {
+  message?: string;
+  result?: string;
+  status?: ExecutionStatus | string;
+}
+export interface MetadataCatalogDetail {
+  catalogType?: CatalogType | string;
+  tableName?: string;
+  tableRegistrationOutput?: RegistrationOutput;
+  partitionRegistrationOutput?: RegistrationOutput;
+}
 export interface DescribeFlowResponse {
   flowArn?: string;
   description?: string;
@@ -1301,6 +1332,9 @@ export interface DescribeFlowResponse {
   createdBy?: string;
   lastUpdatedBy?: string;
   tags?: Record<string, string>;
+  metadataCatalogConfig?: MetadataCatalogConfig;
+  lastRunMetadataCatalogDetails?: MetadataCatalogDetail[];
+  schemaVersion?: number;
 }
 export interface DescribeFlowExecutionRecordsRequest {
   flowName: string | undefined;
@@ -1325,6 +1359,7 @@ export interface ExecutionRecord {
   lastUpdatedAt?: Date;
   dataPullStartTime?: Date;
   dataPullEndTime?: Date;
+  metadataCatalogDetails?: MetadataCatalogDetail[];
 }
 export interface DescribeFlowExecutionRecordsResponse {
   flowExecutions?: ExecutionRecord[];
@@ -1446,6 +1481,7 @@ export interface UpdateFlowRequest {
   sourceFlowConfig: SourceFlowConfig | undefined;
   destinationFlowConfigList: DestinationFlowConfig[] | undefined;
   tasks: Task[] | undefined;
+  metadataCatalogConfig?: MetadataCatalogConfig;
 }
 export interface UpdateFlowResponse {
   flowStatus?: FlowStatus | string;
@@ -1783,6 +1819,12 @@ export declare const DestinationConnectorPropertiesFilterSensitiveLog: (
 export declare const DestinationFlowConfigFilterSensitiveLog: (
   obj: DestinationFlowConfig
 ) => any;
+export declare const GlueDataCatalogConfigFilterSensitiveLog: (
+  obj: GlueDataCatalogConfig
+) => any;
+export declare const MetadataCatalogConfigFilterSensitiveLog: (
+  obj: MetadataCatalogConfig
+) => any;
 export declare const IncrementalPullConfigFilterSensitiveLog: (
   obj: IncrementalPullConfig
 ) => any;
@@ -1898,6 +1940,12 @@ export declare const DescribeFlowRequestFilterSensitiveLog: (
 export declare const ExecutionDetailsFilterSensitiveLog: (
   obj: ExecutionDetails
 ) => any;
+export declare const RegistrationOutputFilterSensitiveLog: (
+  obj: RegistrationOutput
+) => any;
+export declare const MetadataCatalogDetailFilterSensitiveLog: (
+  obj: MetadataCatalogDetail
+) => any;
 export declare const DescribeFlowResponseFilterSensitiveLog: (
   obj: DescribeFlowResponse
 ) => any;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@aws-sdk/client-appflow",
   "description": "AWS SDK for JavaScript Appflow Client for Node.js, Browser and React Native",
-  "version": "3.211.0",
+  "version": "3.213.0",
   "scripts": {
     "build": "concurrently 'yarn:build:cjs' 'yarn:build:es' 'yarn:build:types'",
     "build:cjs": "tsc -p tsconfig.cjs.json",
@@ -19,36 +19,36 @@
   "dependencies": {
     "@aws-crypto/sha256-browser": "2.0.0",
     "@aws-crypto/sha256-js": "2.0.0",
-    "@aws-sdk/client-sts": "3.211.0",
-    "@aws-sdk/config-resolver": "3.209.0",
-    "@aws-sdk/credential-provider-node": "3.211.0",
-    "@aws-sdk/fetch-http-handler": "3.208.0",
-    "@aws-sdk/hash-node": "3.208.0",
-    "@aws-sdk/invalid-dependency": "3.208.0",
-    "@aws-sdk/middleware-content-length": "3.208.0",
-    "@aws-sdk/middleware-endpoint": "3.208.0",
-    "@aws-sdk/middleware-host-header": "3.208.0",
-    "@aws-sdk/middleware-logger": "3.208.0",
-    "@aws-sdk/middleware-recursion-detection": "3.208.0",
-    "@aws-sdk/middleware-retry": "3.209.0",
-    "@aws-sdk/middleware-serde": "3.208.0",
-    "@aws-sdk/middleware-signing": "3.208.0",
-    "@aws-sdk/middleware-stack": "3.208.0",
-    "@aws-sdk/middleware-user-agent": "3.208.0",
-    "@aws-sdk/node-config-provider": "3.209.0",
-    "@aws-sdk/node-http-handler": "3.208.0",
-    "@aws-sdk/protocol-http": "3.208.0",
-    "@aws-sdk/smithy-client": "3.209.0",
-    "@aws-sdk/types": "3.208.0",
-    "@aws-sdk/url-parser": "3.208.0",
+    "@aws-sdk/client-sts": "3.213.0",
+    "@aws-sdk/config-resolver": "3.212.0",
+    "@aws-sdk/credential-provider-node": "3.212.0",
+    "@aws-sdk/fetch-http-handler": "3.212.0",
+    "@aws-sdk/hash-node": "3.212.0",
+    "@aws-sdk/invalid-dependency": "3.212.0",
+    "@aws-sdk/middleware-content-length": "3.212.0",
+    "@aws-sdk/middleware-endpoint": "3.212.0",
+    "@aws-sdk/middleware-host-header": "3.212.0",
+    "@aws-sdk/middleware-logger": "3.212.0",
+    "@aws-sdk/middleware-recursion-detection": "3.212.0",
+    "@aws-sdk/middleware-retry": "3.212.0",
+    "@aws-sdk/middleware-serde": "3.212.0",
+    "@aws-sdk/middleware-signing": "3.212.0",
+    "@aws-sdk/middleware-stack": "3.212.0",
+    "@aws-sdk/middleware-user-agent": "3.212.0",
+    "@aws-sdk/node-config-provider": "3.212.0",
+    "@aws-sdk/node-http-handler": "3.212.0",
+    "@aws-sdk/protocol-http": "3.212.0",
+    "@aws-sdk/smithy-client": "3.212.0",
+    "@aws-sdk/types": "3.212.0",
+    "@aws-sdk/url-parser": "3.212.0",
     "@aws-sdk/util-base64": "3.208.0",
     "@aws-sdk/util-body-length-browser": "3.188.0",
     "@aws-sdk/util-body-length-node": "3.208.0",
-    "@aws-sdk/util-defaults-mode-browser": "3.209.0",
-    "@aws-sdk/util-defaults-mode-node": "3.209.0",
-    "@aws-sdk/util-endpoints": "3.211.0",
-    "@aws-sdk/util-user-agent-browser": "3.208.0",
-    "@aws-sdk/util-user-agent-node": "3.209.0",
+    "@aws-sdk/util-defaults-mode-browser": "3.212.0",
+    "@aws-sdk/util-defaults-mode-node": "3.212.0",
+    "@aws-sdk/util-endpoints": "3.212.0",
+    "@aws-sdk/util-user-agent-browser": "3.212.0",
+    "@aws-sdk/util-user-agent-node": "3.212.0",
     "@aws-sdk/util-utf8-browser": "3.188.0",
     "@aws-sdk/util-utf8-node": "3.208.0",
     "tslib": "^2.3.1"