@aws-cdk/aws-glue-alpha 2.176.0-alpha.0 → 2.178.0-alpha.0

package/lib/jobs/job.d.ts

@@ -0,0 +1,393 @@
+import * as cloudwatch from 'aws-cdk-lib/aws-cloudwatch';
+import * as events from 'aws-cdk-lib/aws-events';
+import * as iam from 'aws-cdk-lib/aws-iam';
+import * as logs from 'aws-cdk-lib/aws-logs';
+import * as cdk from 'aws-cdk-lib/core';
+import * as constructs from 'constructs';
+import { Code } from '..';
+import { MetricType, JobState, WorkerType, GlueVersion } from '../constants';
+import { IConnection } from '../connection';
+import { ISecurityConfiguration } from '../security-configuration';
+/**
+ * Interface representing a new or an imported Glue Job
+ */
+export interface IJob extends cdk.IResource, iam.IGrantable {
+    /**
+     * The name of the job.
+     * @attribute
+     */
+    readonly jobName: string;
+    /**
+     * The ARN of the job.
+     * @attribute
+     */
+    readonly jobArn: string;
+    /**
+     * Defines a CloudWatch event rule triggered when something happens with this job.
+     *
+     * @see https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/EventTypes.html#glue-event-types
+     */
+    onEvent(id: string, options?: events.OnEventOptions): events.Rule;
+    /**
+     * Defines a CloudWatch event rule triggered when this job moves to the SUCCEEDED state.
+     *
+     * @see https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/EventTypes.html#glue-event-types
+     */
+    onSuccess(id: string, options?: events.OnEventOptions): events.Rule;
+    /**
+     * Defines a CloudWatch event rule triggered when this job moves to the FAILED state.
+     *
+     * @see https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/EventTypes.html#glue-event-types
+     */
+    onFailure(id: string, options?: events.OnEventOptions): events.Rule;
+    /**
+     * Defines a CloudWatch event rule triggered when this job moves to the TIMEOUT state.
+     *
+     * @see https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/EventTypes.html#glue-event-types
+     */
+    onTimeout(id: string, options?: events.OnEventOptions): events.Rule;
+    /**
+     * Create a CloudWatch metric.
+     *
+     * @param metricName name of the metric typically prefixed with `glue.driver.`, `glue.<executorId>.` or `glue.ALL.`.
+     * @param type the metric type.
+     * @param props metric options.
+     *
+     * @see https://docs.aws.amazon.com/glue/latest/dg/monitoring-awsglue-with-cloudwatch-metrics.html
+     */
+    metric(metricName: string, type: MetricType, props?: cloudwatch.MetricOptions): cloudwatch.Metric;
+    /**
+     * Create a CloudWatch Metric indicating job success.
+     */
+    metricSuccess(props?: cloudwatch.MetricOptions): cloudwatch.Metric;
+    /**
+     * Create a CloudWatch Metric indicating job failure.
+     */
+    metricFailure(props?: cloudwatch.MetricOptions): cloudwatch.Metric;
+    /**
+     * Create a CloudWatch Metric indicating job timeout.
+     */
+    metricTimeout(props?: cloudwatch.MetricOptions): cloudwatch.Metric;
+}
+/**
+ * Properties for enabling Continuous Logging for Glue Jobs.
+ *
+ * @see https://docs.aws.amazon.com/glue/latest/dg/monitor-continuous-logging-enable.html
+ * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html
+ */
+export interface ContinuousLoggingProps {
+    /**
+     * Enable continouous logging.
+     */
+    readonly enabled: boolean;
+    /**
+     * Specify a custom CloudWatch log group name.
+     *
+     * @default - a log group is created with name `/aws-glue/jobs/logs-v2/`.
+     */
+    readonly logGroup?: logs.ILogGroup;
+    /**
+     * Specify a custom CloudWatch log stream prefix.
+     *
+     * @default - the job run ID.
+     */
+    readonly logStreamPrefix?: string;
+    /**
+     * Filter out non-useful Apache Spark driver/executor and Apache Hadoop YARN heartbeat log messages.
+     *
+     * @default true
+     */
+    readonly quiet?: boolean;
+    /**
+     * Apply the provided conversion pattern.
+     *
+     * This is a Log4j Conversion Pattern to customize driver and executor logs.
+     *
+     * @default `%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n`
+     */
+    readonly conversionPattern?: string;
+}
+/**
+ * A base class is needed to be able to import existing Jobs into a CDK app to
+ * reference as part of a larger stack or construct. JobBase has the subset
+ * of attribtues required to idenitfy and reference an existing Glue Job,
+ * as well as some CloudWatch metric conveneince functions to configure an
+ * event-driven flow using the job.
+ */
+export declare abstract class JobBase extends cdk.Resource implements IJob {
+    abstract readonly jobArn: string;
+    abstract readonly jobName: string;
+    abstract readonly grantPrincipal: iam.IPrincipal;
+    /**
+     * Create a CloudWatch Event Rule for this Glue Job when it's in a given state
+     *
+     * @param id construct id
+     * @param options event options. Note that some values are overridden if provided, these are
+     *  - eventPattern.source = ['aws.glue']
+     *  - eventPattern.detailType = ['Glue Job State Change', 'Glue Job Run Status']
+     *  - eventPattern.detail.jobName = [this.jobName]
+     *
+     * @see https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/EventTypes.html#glue-event-types
+     */
+    onEvent(id: string, options?: events.OnEventOptions): events.Rule;
+    /**
+     * Create a CloudWatch Event Rule for the transition into the input jobState.
+     *
+     * @param id construct id.
+     * @param jobState the job state.
+     * @param options optional event options.
+     */
+    protected onStateChange(id: string, jobState: JobState, options?: events.OnEventOptions): events.Rule;
+    /**
+     * Create a CloudWatch Event Rule matching JobState.SUCCEEDED.
+     *
+     * @param id construct id.
+     * @param options optional event options. default is {}.
+     */
+    onSuccess(id: string, options?: events.OnEventOptions): events.Rule;
+    /**
+     * Return a CloudWatch Event Rule matching FAILED state.
+     *
+     * @param id construct id.
+     * @param options optional event options. default is {}.
+     */
+    onFailure(id: string, options?: events.OnEventOptions): events.Rule;
+    /**
+     * Return a CloudWatch Event Rule matching TIMEOUT state.
+     *
+     * @param id construct id.
+     * @param options optional event options. default is {}.
+     */
+    onTimeout(id: string, options?: events.OnEventOptions): events.Rule;
+    /**
+     * Create a CloudWatch metric.
+     *
+     * @param metricName name of the metric typically prefixed with `glue.driver.`, `glue.<executorId>.` or `glue.ALL.`.
+     * @param type the metric type.
+     * @param props metric options.
+     *
+     * @see https://docs.aws.amazon.com/glue/latest/dg/monitoring-awsglue-with-cloudwatch-metrics.html
+     */
+    metric(metricName: string, type: MetricType, props?: cloudwatch.MetricOptions): cloudwatch.Metric;
+    /**
+     * Return a CloudWatch Metric indicating job success.
+     *
+     * This metric is based on the Rule returned by no-args onSuccess() call.
+     */
+    metricSuccess(props?: cloudwatch.MetricOptions): cloudwatch.Metric;
+    /**
+     * Return a CloudWatch Metric indicating job failure.
+     *
+     * This metric is based on the Rule returned by no-args onFailure() call.
+     */
+    metricFailure(props?: cloudwatch.MetricOptions): cloudwatch.Metric;
+    /**
+     * Return a CloudWatch Metric indicating job timeout.
+     *
+     * This metric is based on the Rule returned by no-args onTimeout() call.
+     */
+    metricTimeout(props?: cloudwatch.MetricOptions): cloudwatch.Metric;
+    /**
+     * Creates or retrieves a singleton event rule for the input job state for use with the metric JobState methods.
+     *
+     * @param id construct id.
+     * @param jobState the job state.
+     */
+    private metricJobStateRule;
+    /**
+     * Returns the job arn
+     */
+    protected buildJobArn(scope: constructs.Construct, jobName: string): string;
+}
+/**
+ * A subset of Job attributes are required for importing an existing job
+ * into a CDK project. This is ionly used when using fromJobAttributes
+ * to identify and reference the existing job.
+ */
+export interface JobImportAttributes {
+    /**
+     * The name of the job.
+     */
+    readonly jobName: string;
+    /**
+     * The IAM role assumed by Glue to run this job.
+     *
+     * @default - undefined
+     */
+    readonly role?: iam.IRole;
+}
+/**
+ * JobProperties will be used to create new Glue Jobs using this L2 Construct.
+ */
+export interface JobProperties {
+    /**
+     * Script Code Location (required)
+     * Script to run when the Glue job executes. Can be uploaded
+     * from the local directory structure using fromAsset
+     * or referenced via S3 location using fromBucket
+     */
+    readonly script: Code;
+    /**
+     * IAM Role (required)
+     * IAM Role to use for Glue job execution
+     * Must be specified by the developer because the L2 doesn't have visibility
+     * into the actions the script(s) takes during the job execution
+     * The role must trust the Glue service principal (glue.amazonaws.com)
+     * and be granted sufficient permissions.
+     *
+     * @see https://docs.aws.amazon.com/glue/latest/dg/getting-started-access.html
+     */
+    readonly role: iam.IRole;
+    /**
+     * Name of the Glue job (optional)
+     * Developer-specified name of the Glue job
+     *
+     * @default - a name is automatically generated
+     */
+    readonly jobName?: string;
+    /**
+     * Description (optional)
+     * Developer-specified description of the Glue job
+     *
+     * @default - no value
+     */
+    readonly description?: string;
+    /**
+     * Number of Workers (optional)
+     * Number of workers for Glue to use during job execution
+     *
+     * @default 10
+     */
+    readonly numberOfWorkers?: number;
+    /**
+     * Worker Type (optional)
+     * Type of Worker for Glue to use during job execution
+     * Enum options: Standard, G_1X, G_2X, G_025X. G_4X, G_8X, Z_2X
+     *
+     * @default WorkerType.G_1X
+     */
+    readonly workerType?: WorkerType;
+    /**
+     * Max Concurrent Runs (optional)
+     * The maximum number of runs this Glue job can concurrently run
+     *
+     * An error is returned when this threshold is reached. The maximum value
+     * you can specify is controlled by a service limit.
+     *
+     * @default 1
+     */
+    readonly maxConcurrentRuns?: number;
+    /**
+     * Default Arguments (optional)
+     * The default arguments for every run of this Glue job,
+     * specified as name-value pairs.
+     *
+     * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html
+     * for a list of reserved parameters
+     * @default - no arguments
+     */
+    readonly defaultArguments?: {
+        [key: string]: string;
+    };
+    /**
+     * Connections (optional)
+     * List of connections to use for this Glue job
+     * Connections are used to connect to other AWS Service or resources within a VPC.
+     *
+     * @default [] - no connections are added to the job
+     */
+    readonly connections?: IConnection[];
+    /**
+     * Max Retries (optional)
+     * Maximum number of retry attempts Glue performs if the job fails
+     *
+     * @default 0
+     */
+    readonly maxRetries?: number;
+    /**
+     * Timeout (optional)
+     * The maximum time that a job run can consume resources before it is
+     * terminated and enters TIMEOUT status. Specified in minutes.
+     *
+     * @default 2880 (2 days for non-streaming)
+     *
+     */
+    readonly timeout?: cdk.Duration;
+    /**
+     * Security Configuration (optional)
+     * Defines the encryption options for the Glue job
+     *
+     * @default - no security configuration.
+     */
+    readonly securityConfiguration?: ISecurityConfiguration;
+    /**
+     * Tags (optional)
+     * A list of key:value pairs of tags to apply to this Glue job resources
+     *
+     * @default {} - no tags
+     */
+    readonly tags?: {
+        [key: string]: string;
+    };
+    /**
+     * Glue Version
+     * The version of Glue to use to execute this job
+     *
+     * @default 3.0 for ETL
+     */
+    readonly glueVersion?: GlueVersion;
+    /**
+     * Enables the collection of metrics for job profiling.
+     *
+     * @default - no profiling metrics emitted.
+     *
+     * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html
+     */
+    readonly enableProfilingMetrics?: boolean;
+    /**
+     * Enables continuous logging with the specified props.
+     *
+     * @default - continuous logging is enabled.
+     *
+     * @see https://docs.aws.amazon.com/glue/latest/dg/monitor-continuous-logging-enable.html
+     * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html
+     **/
+    readonly continuousLogging?: ContinuousLoggingProps;
+}
+/**
+ * A Glue Job.
+ * @resource AWS::Glue::Job
+ */
+export declare abstract class Job extends JobBase {
+    /**
+     * Identifies an existing Glue Job from a subset of attributes that can
+     * be referenced from within another Stack or Construct.
+     *
+     * @param scope The scope creating construct (usually `this`)
+     * @param id The construct's id.
+     * @param attrs Attributes for the Glue Job we want to import
+     */
+    static fromJobAttributes(scope: constructs.Construct, id: string, attrs: JobImportAttributes): IJob;
+    /**
+     * The IAM role Glue assumes to run this job.
+     */
+    abstract readonly role: iam.IRole;
+    /**
+     * Check no usage of reserved arguments.
+     *
+     * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html
+     */
+    protected checkNoReservedArgs(defaultArguments?: {
+        [key: string]: string;
+    }): {
+        [key: string]: string;
+    } | undefined;
+    /**
+     * Setup Continuous Loggiung Properties
+     * @param role The IAM role to use for continuous logging
+     * @param props The properties for continuous logging configuration
+     * @returns String containing the args for the continuous logging command
+     */
+    setupContinuousLogging(role: iam.IRole, props: ContinuousLoggingProps | undefined): any;
+    protected codeS3ObjectUrl(code: Code): string;
+}