cdk-glue-iceberg-table 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +617 -0
- package/dist/lib/iceberg/iceberg-partition-transform.d.ts +78 -0
- package/dist/lib/iceberg/iceberg-partition-transform.js +219 -0
- package/dist/lib/iceberg/iceberg-table-properties.d.ts +54 -0
- package/dist/lib/iceberg/iceberg-table-properties.js +183 -0
- package/dist/lib/iceberg/iceberg-table.d.ts +187 -0
- package/dist/lib/iceberg/iceberg-table.js +549 -0
- package/dist/lib/iceberg/iceberg-type.d.ts +182 -0
- package/dist/lib/iceberg/iceberg-type.js +365 -0
- package/dist/lib/iceberg/index.d.ts +4 -0
- package/dist/lib/iceberg/index.js +18 -0
- package/package.json +70 -0
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
import { Resource, RemovalPolicy } from 'aws-cdk-lib';
|
|
2
|
+
import { CfnTable } from 'aws-cdk-lib/aws-glue';
|
|
3
|
+
import { Grant, IGrantable } from 'aws-cdk-lib/aws-iam';
|
|
4
|
+
import { Construct } from 'constructs';
|
|
5
|
+
import { IDatabase } from '@aws-cdk/aws-glue-alpha';
|
|
6
|
+
import { IcebergType } from './iceberg-type';
|
|
7
|
+
import { IcebergPartitionTransform } from './iceberg-partition-transform';
|
|
8
|
+
import { IcebergDataFormat, IcebergFormatVersion } from './iceberg-table-properties';
|
|
9
|
+
/** Sort direction for an Iceberg sort field. */
|
|
10
|
+
export declare enum IcebergSortDirection {
|
|
11
|
+
/** Ascending — smaller values first. */
|
|
12
|
+
ASC = "asc",
|
|
13
|
+
/** Descending — larger values first. */
|
|
14
|
+
DESC = "desc"
|
|
15
|
+
}
|
|
16
|
+
/** Null ordering for an Iceberg sort field. */
|
|
17
|
+
export declare enum IcebergNullOrder {
|
|
18
|
+
/** Nulls precede all non-null values. */
|
|
19
|
+
NULLS_FIRST = "nulls-first",
|
|
20
|
+
/** Nulls follow all non-null values. */
|
|
21
|
+
NULLS_LAST = "nulls-last"
|
|
22
|
+
}
|
|
23
|
+
/** One top-level column in an Iceberg table. */
|
|
24
|
+
export interface IcebergColumn {
|
|
25
|
+
/** Column name (unique within the table). */
|
|
26
|
+
readonly name: string;
|
|
27
|
+
/** Column type. */
|
|
28
|
+
readonly type: IcebergType;
|
|
29
|
+
/**
|
|
30
|
+
* Whether the column is non-nullable. Defaults to `false` to
|
|
31
|
+
* match the Iceberg spec default of optional fields.
|
|
32
|
+
*/
|
|
33
|
+
readonly required?: boolean;
|
|
34
|
+
/** Optional documentation string. */
|
|
35
|
+
readonly doc?: string;
|
|
36
|
+
/**
|
|
37
|
+
* Pin this column to a specific Iceberg field id. Highly
|
|
38
|
+
* recommended for production tables: it lets you safely add,
|
|
39
|
+
* remove, and reorder columns across `cdk deploy`s without
|
|
40
|
+
* triggering Iceberg's silent-corruption-on-id-reuse trap. When
|
|
41
|
+
* omitted the construct assigns ids by position (1..N for the
|
|
42
|
+
* top-level columns), which is fine for fresh tables but unsafe
|
|
43
|
+
* once the table has data and you start dropping columns.
|
|
44
|
+
*/
|
|
45
|
+
readonly id?: number;
|
|
46
|
+
}
|
|
47
|
+
/** One partition spec field. */
|
|
48
|
+
export interface IcebergPartitionField {
|
|
49
|
+
/** Name of the top-level table column this partition derives from. */
|
|
50
|
+
readonly sourceColumn: string;
|
|
51
|
+
/** Transform to apply to the source column. */
|
|
52
|
+
readonly transform: IcebergPartitionTransform;
|
|
53
|
+
/**
|
|
54
|
+
* Display name for the partition field. Defaults to
|
|
55
|
+
* `<sourceColumn>` for identity transforms and
|
|
56
|
+
* `<sourceColumn>_<transform>` otherwise.
|
|
57
|
+
*/
|
|
58
|
+
readonly name?: string;
|
|
59
|
+
}
|
|
60
|
+
/** One write-order (sort) field. */
|
|
61
|
+
export interface IcebergSortField {
|
|
62
|
+
/** Name of the top-level table column to sort on. */
|
|
63
|
+
readonly sourceColumn: string;
|
|
64
|
+
/** Transform to apply to the source column. Defaults to `identity`. */
|
|
65
|
+
readonly transform?: IcebergPartitionTransform;
|
|
66
|
+
/** Sort direction. Defaults to `ASC`. */
|
|
67
|
+
readonly direction?: IcebergSortDirection;
|
|
68
|
+
/** Null ordering. Defaults to `NULLS_LAST`. */
|
|
69
|
+
readonly nullOrder?: IcebergNullOrder;
|
|
70
|
+
}
|
|
71
|
+
/** Construction properties for an `IcebergTable`. */
|
|
72
|
+
export interface IcebergTableProps {
|
|
73
|
+
/** Glue database that will hold the table. */
|
|
74
|
+
readonly database: IDatabase;
|
|
75
|
+
/** Table name (lower-case, no spaces — Glue and Athena will fold it). */
|
|
76
|
+
readonly tableName: string;
|
|
77
|
+
/** Top-level columns. Must contain at least one column. */
|
|
78
|
+
readonly columns: IcebergColumn[];
|
|
79
|
+
/** S3 URI where Iceberg metadata + data live. Must start with `s3://`. */
|
|
80
|
+
readonly location: string;
|
|
81
|
+
/** Optional partition spec. Order is preserved in the partition layout. */
|
|
82
|
+
readonly partitionSpec?: IcebergPartitionField[];
|
|
83
|
+
/** Optional default write order. Realized as `IcebergTableInput.writeOrder`. */
|
|
84
|
+
readonly sortOrder?: IcebergSortField[];
|
|
85
|
+
/**
|
|
86
|
+
* Names of columns that together identify a row. Maps to
|
|
87
|
+
* `IcebergSchema.identifierFieldIds`. Per the Iceberg spec these
|
|
88
|
+
* must be primitive, non-nullable, non-floating-point fields.
|
|
89
|
+
*/
|
|
90
|
+
readonly identifierFieldNames?: string[];
|
|
91
|
+
/** Data-file storage format. Defaults to `PARQUET`. */
|
|
92
|
+
readonly dataFormat?: IcebergDataFormat;
|
|
93
|
+
/** Iceberg format version. Defaults to `V2`. */
|
|
94
|
+
readonly formatVersion?: IcebergFormatVersion;
|
|
95
|
+
/**
|
|
96
|
+
* Extra `properties` to publish on the table. Auto-added keys
|
|
97
|
+
* (`format-version`, `write.format.default`) are merged in and
|
|
98
|
+
* must agree with `formatVersion` / `dataFormat` if user-supplied.
|
|
99
|
+
*/
|
|
100
|
+
readonly tableProperties?: {
|
|
101
|
+
[key: string]: string;
|
|
102
|
+
};
|
|
103
|
+
/**
|
|
104
|
+
* Optional table comment. Glue's `OpenTableFormatInput` shape is
|
|
105
|
+
* mutually exclusive with `TableInput.Description`, so the
|
|
106
|
+
* construct cannot surface a description through the regular
|
|
107
|
+
* Glue field. The comment is published in the Iceberg
|
|
108
|
+
* `properties` map under the `comment` key, where Athena's
|
|
109
|
+
* `SHOW TBLPROPERTIES` and Iceberg-aware readers can find it.
|
|
110
|
+
*/
|
|
111
|
+
readonly comment?: string;
|
|
112
|
+
/**
|
|
113
|
+
* Removal policy for the underlying Glue table. Defaults to `RETAIN`
|
|
114
|
+
* to match how CloudFormation treats Glue tables by default and to
|
|
115
|
+
* avoid accidentally dropping production metadata.
|
|
116
|
+
*/
|
|
117
|
+
readonly removalPolicy?: RemovalPolicy;
|
|
118
|
+
}
|
|
119
|
+
/** Public interface implemented by `IcebergTable` and by import shims. */
|
|
120
|
+
export interface IIcebergTable {
|
|
121
|
+
/** Glue ARN of the table. */
|
|
122
|
+
readonly tableArn: string;
|
|
123
|
+
/** Table name. */
|
|
124
|
+
readonly tableName: string;
|
|
125
|
+
/** Glue database holding the table. */
|
|
126
|
+
readonly database: IDatabase;
|
|
127
|
+
/** S3 URI where the table is materialized. */
|
|
128
|
+
readonly location: string;
|
|
129
|
+
/** Grant Glue + S3 read on this table. */
|
|
130
|
+
grantRead(grantee: IGrantable): Grant;
|
|
131
|
+
/** Grant Glue + S3 write on this table. */
|
|
132
|
+
grantWrite(grantee: IGrantable): Grant;
|
|
133
|
+
/** Grant Glue + S3 read + write on this table. */
|
|
134
|
+
grantReadWrite(grantee: IGrantable): Grant;
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* A Glue table created in the Apache Iceberg open table format.
|
|
138
|
+
*
|
|
139
|
+
* The construct emits the
|
|
140
|
+
* `OpenTableFormatInput.IcebergInput.IcebergTableInput` shape that
|
|
141
|
+
* survives CloudFormation `Update` (the alternative — placing columns
|
|
142
|
+
* under `tableInput.storageDescriptor.columns` — silently strips
|
|
143
|
+
* `table_type=ICEBERG` from Glue parameters on the first update and
|
|
144
|
+
* leaves the table un-queryable in Athena).
|
|
145
|
+
*/
|
|
146
|
+
export declare class IcebergTable extends Resource implements IIcebergTable {
|
|
147
|
+
/** Glue ARN of the table. */
|
|
148
|
+
readonly tableArn: string;
|
|
149
|
+
/** Table name as it appears in Glue / Athena. */
|
|
150
|
+
readonly tableName: string;
|
|
151
|
+
/** Glue database holding the table. */
|
|
152
|
+
readonly database: IDatabase;
|
|
153
|
+
/** S3 URI where the Iceberg metadata + data live. */
|
|
154
|
+
readonly location: string;
|
|
155
|
+
/** Resolved data format (after defaulting). */
|
|
156
|
+
readonly dataFormat: IcebergDataFormat;
|
|
157
|
+
/** Resolved format version (after defaulting). */
|
|
158
|
+
readonly formatVersion: IcebergFormatVersion;
|
|
159
|
+
/** The underlying L1 — exposed for escape-hatch use. */
|
|
160
|
+
readonly resource: CfnTable;
|
|
161
|
+
private readonly bucketArn;
|
|
162
|
+
private readonly objectArn;
|
|
163
|
+
private readonly s3PrefixGlob;
|
|
164
|
+
constructor(scope: Construct, id: string, props: IcebergTableProps);
|
|
165
|
+
/**
|
|
166
|
+
* Import an existing Iceberg table by its name + database.
|
|
167
|
+
*
|
|
168
|
+
* Imported tables can be referenced from grants but cannot be
|
|
169
|
+
* mutated; their schema and properties are unknown to CDK.
|
|
170
|
+
*/
|
|
171
|
+
static fromIcebergTableAttributes(scope: Construct, id: string, attrs: IcebergTableAttributes): IIcebergTable;
|
|
172
|
+
/** Grant Glue read + S3 read on this table. */
|
|
173
|
+
grantRead(grantee: IGrantable): Grant;
|
|
174
|
+
/** Grant Glue write + S3 write on this table. */
|
|
175
|
+
grantWrite(grantee: IGrantable): Grant;
|
|
176
|
+
/** Grant Glue + S3 read + write on this table. */
|
|
177
|
+
grantReadWrite(grantee: IGrantable): Grant;
|
|
178
|
+
}
|
|
179
|
+
/** Attributes needed to import an existing Iceberg table. */
|
|
180
|
+
export interface IcebergTableAttributes {
|
|
181
|
+
/** Glue database holding the table. */
|
|
182
|
+
readonly database: IDatabase;
|
|
183
|
+
/** Existing table's name. */
|
|
184
|
+
readonly tableName: string;
|
|
185
|
+
/** S3 URI where the table's data + metadata live. */
|
|
186
|
+
readonly location: string;
|
|
187
|
+
}
|