@aws-mdaa/dataops-dms 1.4.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +103 -99
- package/SCHEMA.md +7882 -1594
- package/lib/config-schema.json +2829 -302
- package/lib/dms-config.d.ts +5 -6
- package/lib/dms-config.js +1 -1
- package/package.json +17 -13
- package/sample_configs/sample-config-cdc.yaml +97 -0
- package/sample_configs/sample-config-comprehensive.yaml +590 -0
- package/sample_configs/sample-config-minimal.yaml +22 -0
- package/sample_configs/sample-config-noproject.yaml +106 -0
- package/mdaa.config.json +0 -3
package/lib/dms-config.d.ts
CHANGED
|
@@ -8,15 +8,14 @@ import { MdaaDataOpsConfigContents, MdaaDataOpsConfigParser } from '@aws-mdaa/da
|
|
|
8
8
|
import { DMSProps } from '@aws-mdaa/dataops-dms-l3-construct';
|
|
9
9
|
export interface DMSConfigContents extends MdaaDataOpsConfigContents {
|
|
10
10
|
/**
|
|
11
|
-
*
|
|
12
|
-
* Required DMS configuration defining database migration and replication capabilities including replication instances, source and target endpoints, migration tasks, and data transformation. Provides complete DMS setup with secure data migration and real-time replication capabilities.
|
|
11
|
+
* DMS migration and replication configuration including instances, endpoints, and tasks.
|
|
13
12
|
*
|
|
14
|
-
* Use cases: Database migration; Real-time replication; data movement
|
|
13
|
+
* Use cases: Database migration; Real-time replication; Cross-engine data movement
|
|
15
14
|
*
|
|
16
|
-
* AWS: AWS Database Migration Service
|
|
15
|
+
* AWS: AWS Database Migration Service
|
|
17
16
|
*
|
|
18
|
-
* Validation:
|
|
19
|
-
|
|
17
|
+
* Validation: Required; valid DMSProps
|
|
18
|
+
*/
|
|
20
19
|
readonly dms: DMSProps;
|
|
21
20
|
}
|
|
22
21
|
export declare class DMSConfigParser extends MdaaDataOpsConfigParser<DMSConfigContents> {
|
package/lib/dms-config.js
CHANGED
|
@@ -14,4 +14,4 @@ class DMSConfigParser extends dataops_shared_1.MdaaDataOpsConfigParser {
|
|
|
14
14
|
}
|
|
15
15
|
}
|
|
16
16
|
exports.DMSConfigParser = DMSConfigParser;
|
|
17
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
17
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZG1zLWNvbmZpZy5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbImRtcy1jb25maWcudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IjtBQUFBOzs7R0FHRzs7O0FBS0gscURBQXFEO0FBQ3JELDZEQUE4RjtBQWdCOUYsTUFBYSxlQUFnQixTQUFRLHdDQUEwQztJQUU3RSxZQUFZLEtBQVksRUFBRSxLQUErQjtRQUN2RCxLQUFLLENBQUMsS0FBSyxFQUFFLEtBQUssRUFBRSxZQUFzQixDQUFDLENBQUM7UUFDNUMsSUFBSSxDQUFDLEdBQUcsR0FBRyxJQUFJLENBQUMsY0FBYyxDQUFDLEdBQUcsQ0FBQztJQUNyQyxDQUFDO0NBQ0Y7QUFORCwwQ0FNQyIsInNvdXJjZXNDb250ZW50IjpbIi8qIVxuICogQ29weXJpZ2h0IEFtYXpvbi5jb20sIEluYy4gb3IgaXRzIGFmZmlsaWF0ZXMuIEFsbCBSaWdodHMgUmVzZXJ2ZWQuXG4gKiBTUERYLUxpY2Vuc2UtSWRlbnRpZmllcjogQXBhY2hlLTIuMFxuICovXG5cbmltcG9ydCB7IE1kYWFBcHBDb25maWdQYXJzZXJQcm9wcyB9IGZyb20gJ0Bhd3MtbWRhYS9hcHAnO1xuaW1wb3J0IHsgU2NoZW1hIH0gZnJvbSAnYWp2JztcbmltcG9ydCB7IFN0YWNrIH0gZnJvbSAnYXdzLWNkay1saWInO1xuaW1wb3J0ICogYXMgY29uZmlnU2NoZW1hIGZyb20gJy4vY29uZmlnLXNjaGVtYS5qc29uJztcbmltcG9ydCB7IE1kYWFEYXRhT3BzQ29uZmlnQ29udGVudHMsIE1kYWFEYXRhT3BzQ29uZmlnUGFyc2VyIH0gZnJvbSAnQGF3cy1tZGFhL2RhdGFvcHMtc2hhcmVkJztcbmltcG9ydCB7IERNU1Byb3BzIH0gZnJvbSAnQGF3cy1tZGFhL2RhdGFvcHMtZG1zLWwzLWNvbnN0cnVjdCc7XG5cbmV4cG9ydCBpbnRlcmZhY2UgRE1TQ29uZmlnQ29udGVudHMgZXh0ZW5kcyBNZGFhRGF0YU9wc0NvbmZpZ0NvbnRlbnRzIHtcbiAgLyoqXG4gICAqIERNUyBtaWdyYXRpb24gYW5kIHJlcGxpY2F0aW9uIGNvbmZpZ3VyYXRpb24gaW5jbHVkaW5nIGluc3RhbmNlcywgZW5kcG9pbnRzLCBhbmQgdGFza3MuXG4gICAqXG4gICAqIFVzZSBjYXNlczogRGF0YWJhc2UgbWlncmF0aW9uOyBSZWFsLXRpbWUgcmVwbGljYXRpb247IENyb3NzLWVuZ2luZSBkYXRhIG1vdmVtZW50XG4gICAqXG4gICAqIEFXUzogQVdTIERhdGFiYXNlIE1pZ3JhdGlvbiBTZXJ2aWNlXG4gICAqXG4gICAqIFZhbGlkYXRpb246IFJlcXVpcmVkOyB2YWxpZCBETVNQcm9wc1xuICAgKi9cbiAgcmVhZG9ubHkgZG1zOiBETVNQcm9wcztcbn1cblxuZXhwb3J0IGNsYXNzIERNU0NvbmZpZ1BhcnNlciBleHRlbmRzIE1kYWFEYXRhT3BzQ29uZmlnUGFyc2VyPERNU0NvbmZpZ0NvbnRlbnRzPiB7XG4gIHB1YmxpYyBkbXM6IERNU1Byb3BzO1xuICBjb25zdHJ1Y3RvcihzdGFjazogU3RhY2ssIHByb3BzOiBNZGFhQXBwQ29uZmlnUGFyc2VyUHJvcHMpIHtcbiAgICBzdXBlcihzdGFjaywgcHJvcHMsIGNvbmZpZ1NjaGVtYSBhcyBTY2hlbWEpO1xuICAgIHRoaXMuZG1zID0gdGhpcy5jb25maWdDb250ZW50cy5kbXM7XG4gIH1cbn1cbiJdfQ==
|
package/package.json
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"name": "Amazon Web Services",
|
|
6
6
|
"url": "https://aws.amazon.com/solutions"
|
|
7
7
|
},
|
|
8
|
-
"version": "1.
|
|
8
|
+
"version": "1.6.0",
|
|
9
9
|
"license": "Apache-2.0",
|
|
10
10
|
"bin": {
|
|
11
11
|
"dms": "bin/dms.js"
|
|
@@ -13,30 +13,31 @@
|
|
|
13
13
|
"main": "lib/index.js",
|
|
14
14
|
"types": "lib/index.d.ts",
|
|
15
15
|
"scripts": {
|
|
16
|
-
"build": "
|
|
16
|
+
"build": "../../../../scripts/build/build_package.sh DMSConfigContents",
|
|
17
17
|
"watch": "tsc -w",
|
|
18
18
|
"test": "jest --passWithNoTests --coverage",
|
|
19
|
-
"lint": "eslint --max-warnings 0 -c
|
|
20
|
-
"cdk": "cdk"
|
|
19
|
+
"lint": "eslint --max-warnings 0 -c ../../../../eslint.config.mjs",
|
|
20
|
+
"cdk": "cdk",
|
|
21
|
+
"test:update-baselines": "UPDATE_BASELINES=true jest --passWithNoTests --testPathPattern='.*\\.diff\\.test\\.ts'",
|
|
22
|
+
"test:package-docs": "../../../../scripts/generate_docs/test_package_docs.sh"
|
|
21
23
|
},
|
|
22
24
|
"devDependencies": {
|
|
23
|
-
"@aws-mdaa/testing": "1.
|
|
25
|
+
"@aws-mdaa/testing": "1.6.0",
|
|
24
26
|
"@types/jest": "29.5.14",
|
|
25
27
|
"@types/node": "22.9.0",
|
|
26
28
|
"@types/prettier": "2.6.0",
|
|
27
29
|
"jest": "29.7.0",
|
|
28
|
-
"ts-jest": "29.4.
|
|
30
|
+
"ts-jest": "29.4.9",
|
|
29
31
|
"ts-node": "10.9.2",
|
|
30
32
|
"typescript": "5.9.3",
|
|
31
|
-
"typescript-json-schema": "0.67.
|
|
33
|
+
"typescript-json-schema": "0.67.4"
|
|
32
34
|
},
|
|
33
35
|
"dependencies": {
|
|
34
|
-
"@aws-mdaa/app": "1.
|
|
35
|
-
"@aws-mdaa/
|
|
36
|
-
"@aws-mdaa/dataops-
|
|
37
|
-
"@aws-mdaa/
|
|
38
|
-
"
|
|
39
|
-
"ajv": "8.17.1",
|
|
36
|
+
"@aws-mdaa/app": "1.6.0",
|
|
37
|
+
"@aws-mdaa/dataops-dms-l3-construct": "1.6.0",
|
|
38
|
+
"@aws-mdaa/dataops-shared": "1.6.0",
|
|
39
|
+
"@aws-mdaa/l3-construct": "1.6.0",
|
|
40
|
+
"ajv": "8.18.0",
|
|
40
41
|
"aws-cdk-lib": "2.220.0",
|
|
41
42
|
"cdk-nag": "2.37.55",
|
|
42
43
|
"constructs": "10.0.96",
|
|
@@ -47,5 +48,8 @@
|
|
|
47
48
|
"repository": {
|
|
48
49
|
"type": "git",
|
|
49
50
|
"url": "https://github.com/aws/modern-data-architecture-accelerator"
|
|
51
|
+
},
|
|
52
|
+
"mdaa": {
|
|
53
|
+
"deployStage": "5"
|
|
50
54
|
}
|
|
51
55
|
}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
# Sample config for the DataOps DMS module - CDC migration variant.
|
|
2
|
+
# Demonstrates CDC and full-load-and-cdc migration types with
|
|
3
|
+
# CDC-specific task properties (cdcStartPosition, cdcStartTime,
|
|
4
|
+
# cdcStopPosition).
|
|
5
|
+
|
|
6
|
+
# DataOps project name for auto-wiring shared resources.
|
|
7
|
+
projectName: test-project
|
|
8
|
+
|
|
9
|
+
# DMS migration and replication configuration.
|
|
10
|
+
dms:
|
|
11
|
+
# Named replication instance configurations.
|
|
12
|
+
replicationInstances:
|
|
13
|
+
test-instance:
|
|
14
|
+
# DMS replication instance class.
|
|
15
|
+
instanceClass: dms.t3.micro
|
|
16
|
+
# VPC ID for replication instance deployment.
|
|
17
|
+
# Often created by your VPC/networking stack.
|
|
18
|
+
# Example SSM: ssm:/path/to/vpc/id
|
|
19
|
+
vpcId: test_vpc_id
|
|
20
|
+
# Subnet IDs spanning at least two AZs.
|
|
21
|
+
# Often created by your VPC/networking stack.
|
|
22
|
+
# Example SSM: ssm:/path/to/subnet/id
|
|
23
|
+
subnetIds:
|
|
24
|
+
- test_subnet_id1
|
|
25
|
+
- test_subnet_id2
|
|
26
|
+
|
|
27
|
+
# Named endpoint configurations.
|
|
28
|
+
endpoints:
|
|
29
|
+
test-source:
|
|
30
|
+
# The type of endpoint. (enum: source, target)
|
|
31
|
+
endpointType: source
|
|
32
|
+
# The endpoint engine name.
|
|
33
|
+
engineName: postgres
|
|
34
|
+
databaseName: test-cdc-db
|
|
35
|
+
# PostgreSQL settings.
|
|
36
|
+
postgreSqlSettings:
|
|
37
|
+
secretsManagerSecretArn: arn:{{partition}}:secretsmanager:{{region}}:{{account}}:secret:test-pg-secret
|
|
38
|
+
secretsManagerSecretKMSArn: arn:{{partition}}:kms:{{region}}:{{account}}:key/test-pg-key
|
|
39
|
+
captureDdls: true
|
|
40
|
+
pluginName: pglogical
|
|
41
|
+
slotName: test_cdc_slot
|
|
42
|
+
test-target:
|
|
43
|
+
endpointType: target
|
|
44
|
+
engineName: s3
|
|
45
|
+
# S3 settings.
|
|
46
|
+
s3Settings:
|
|
47
|
+
bucketName: test-cdc-target-bucket
|
|
48
|
+
serverSideEncryptionKmsKeyId: test-cdc-kms-key-id
|
|
49
|
+
|
|
50
|
+
# Named replication task configurations.
|
|
51
|
+
replicationTasks:
|
|
52
|
+
# CDC-only migration task.
|
|
53
|
+
test-task-cdc:
|
|
54
|
+
replicationInstance: test-instance
|
|
55
|
+
sourceEndpoint: test-source
|
|
56
|
+
targetEndpoint: test-target
|
|
57
|
+
# Migration type: cdc (change data capture only).
|
|
58
|
+
migrationType: cdc
|
|
59
|
+
# CDC start position (date, checkpoint, LSN, or SCN format).
|
|
60
|
+
cdcStartPosition: '2024-01-01T00:00:00'
|
|
61
|
+
# CDC stop position (server_time or commit_time format).
|
|
62
|
+
cdcStopPosition: 'server_time:2024-12-31T23:59:59'
|
|
63
|
+
# Table mappings for the task.
|
|
64
|
+
tableMappings:
|
|
65
|
+
rules:
|
|
66
|
+
- rule-type: selection
|
|
67
|
+
rule-id: '1'
|
|
68
|
+
rule-name: '1'
|
|
69
|
+
object-locator:
|
|
70
|
+
schema-name: public
|
|
71
|
+
table-name: '%'
|
|
72
|
+
rule-action: include
|
|
73
|
+
|
|
74
|
+
# Full-load-and-cdc migration task.
|
|
75
|
+
test-task-full-load-and-cdc:
|
|
76
|
+
replicationInstance: test-instance
|
|
77
|
+
sourceEndpoint: test-source
|
|
78
|
+
targetEndpoint: test-target
|
|
79
|
+
# Migration type: full-load-and-cdc.
|
|
80
|
+
migrationType: full-load-and-cdc
|
|
81
|
+
# CDC start time (epoch timestamp).
|
|
82
|
+
cdcStartTime: 1704067200
|
|
83
|
+
# Table mappings for the task.
|
|
84
|
+
tableMappings:
|
|
85
|
+
rules:
|
|
86
|
+
- rule-type: selection
|
|
87
|
+
rule-id: '1'
|
|
88
|
+
rule-name: '1'
|
|
89
|
+
object-locator:
|
|
90
|
+
schema-name: public
|
|
91
|
+
table-name: '%'
|
|
92
|
+
rule-action: include
|
|
93
|
+
# Overall settings for the task.
|
|
94
|
+
replicationTaskSettings:
|
|
95
|
+
TargetMetadata:
|
|
96
|
+
TargetSchema: ''
|
|
97
|
+
SupportLobs: true
|