@aws-mdaa/dataops-dms 1.4.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +103 -99
- package/SCHEMA.md +7882 -1594
- package/lib/config-schema.json +2829 -302
- package/lib/dms-config.d.ts +5 -6
- package/lib/dms-config.js +1 -1
- package/package.json +17 -13
- package/sample_configs/sample-config-cdc.yaml +97 -0
- package/sample_configs/sample-config-comprehensive.yaml +590 -0
- package/sample_configs/sample-config-minimal.yaml +22 -0
- package/sample_configs/sample-config-noproject.yaml +106 -0
- package/mdaa.config.json +0 -3
|
@@ -0,0 +1,590 @@
|
|
|
1
|
+
# Comprehensive sample config for the DataOps DMS module.
|
|
2
|
+
# Exercises ALL non-excluded schema properties at full depth.
|
|
3
|
+
# Uses projectName for auto-wiring shared resources.
|
|
4
|
+
|
|
5
|
+
# DataOps project name enabling auto-wiring of shared resources
|
|
6
|
+
# (bucket, KMS key, SNS topic, deployment role, security configuration)
|
|
7
|
+
# via SSM parameters.
|
|
8
|
+
projectName: test-project
|
|
9
|
+
|
|
10
|
+
# SNS topic ARN for job notifications and workflow alerts.
|
|
11
|
+
# Auto-resolved from project when projectName is set.
|
|
12
|
+
notificationTopicArn: arn:{{partition}}:sns:{{region}}:{{account}}:test-topic
|
|
13
|
+
|
|
14
|
+
# DMS migration and replication configuration including instances,
|
|
15
|
+
# endpoints, and tasks.
|
|
16
|
+
dms:
|
|
17
|
+
# Whether to create the DMS VPC service role.
|
|
18
|
+
createDmsVpcRole: true
|
|
19
|
+
# Whether to create the DMS CloudWatch Logs service role.
|
|
20
|
+
createDmsLogRole: true
|
|
21
|
+
# Custom IAM role ARN for DMS operations.
|
|
22
|
+
dmsRoleArn: arn:{{partition}}:iam::{{account}}:role/test-dms-role
|
|
23
|
+
|
|
24
|
+
# Named replication instance configurations.
|
|
25
|
+
replicationInstances:
|
|
26
|
+
test-instance:
|
|
27
|
+
# DMS replication instance class.
|
|
28
|
+
instanceClass: dms.t3.micro
|
|
29
|
+
# VPC ID for replication instance deployment.
|
|
30
|
+
# Often created by your VPC/networking stack.
|
|
31
|
+
# Example SSM: ssm:/path/to/vpc/id
|
|
32
|
+
vpcId: test_vpc_id
|
|
33
|
+
# Subnet IDs spanning at least two AZs.
|
|
34
|
+
# Often created by your VPC/networking stack.
|
|
35
|
+
# Example SSM: ssm:/path/to/subnet/id
|
|
36
|
+
subnetIds:
|
|
37
|
+
- test_subnet_id1
|
|
38
|
+
- test_subnet_id2
|
|
39
|
+
# If true, the SG will allow traffic to and from itself.
|
|
40
|
+
addSelfReferenceRule: true
|
|
41
|
+
# Ingress rules to be added to the replication instance SG.
|
|
42
|
+
ingressRules:
|
|
43
|
+
# IPv4 CIDR block rules.
|
|
44
|
+
ipv4:
|
|
45
|
+
- cidr: 10.0.0.0/16
|
|
46
|
+
protocol: tcp
|
|
47
|
+
port: 3306
|
|
48
|
+
# Ending port for port range rules.
|
|
49
|
+
toPort: 3306
|
|
50
|
+
description: Allow MySQL from VPC
|
|
51
|
+
# Prefix list rules.
|
|
52
|
+
prefixList:
|
|
53
|
+
- prefixList: pl-12345678
|
|
54
|
+
protocol: tcp
|
|
55
|
+
port: 5432
|
|
56
|
+
toPort: 5432
|
|
57
|
+
description: Allow PostgreSQL via prefix list
|
|
58
|
+
# Security group peer rules.
|
|
59
|
+
sg:
|
|
60
|
+
- sgId: sg-12345678
|
|
61
|
+
protocol: tcp
|
|
62
|
+
port: 1521
|
|
63
|
+
toPort: 1521
|
|
64
|
+
description: Allow Oracle from app SG
|
|
65
|
+
# Egress rules to be added to the replication instance SG.
|
|
66
|
+
egressRules:
|
|
67
|
+
ipv4:
|
|
68
|
+
- cidr: 0.0.0.0/0
|
|
69
|
+
protocol: tcp
|
|
70
|
+
port: 443
|
|
71
|
+
toPort: 443
|
|
72
|
+
description: Allow HTTPS egress
|
|
73
|
+
|
|
74
|
+
# Named endpoint configurations for source and target databases.
|
|
75
|
+
endpoints:
|
|
76
|
+
# SQL Server source endpoint.
|
|
77
|
+
test-source-sqlserver:
|
|
78
|
+
# The type of endpoint. (enum: source, target)
|
|
79
|
+
endpointType: source
|
|
80
|
+
# The endpoint engine name.
|
|
81
|
+
engineName: sqlserver
|
|
82
|
+
# Optional database name for the endpoint.
|
|
83
|
+
databaseName: test-database
|
|
84
|
+
# Microsoft SQL Server settings.
|
|
85
|
+
microsoftSqlServerSettings:
|
|
86
|
+
# Database name for SQL Server endpoint connectivity.
|
|
87
|
+
databaseName: test-database
|
|
88
|
+
# Secrets Manager secret ARN containing credentials.
|
|
89
|
+
secretsManagerSecretArn: arn:{{partition}}:secretsmanager:{{region}}:{{account}}:secret:test-sqlserver-secret
|
|
90
|
+
# KMS key ARN for the secret.
|
|
91
|
+
secretsManagerSecretKMSArn: arn:{{partition}}:kms:{{region}}:{{account}}:key/test-sqlserver-key
|
|
92
|
+
# IAM role ARN for Secrets Manager access.
|
|
93
|
+
secretsManagerAccessRoleArn: arn:{{partition}}:iam::{{account}}:role/test-sm-access-role
|
|
94
|
+
# BCP packet size in bytes.
|
|
95
|
+
bcpPacketSize: 4096
|
|
96
|
+
controlTablesFileGroup: dms_control
|
|
97
|
+
forceLobLookup: false
|
|
98
|
+
# TCP port number.
|
|
99
|
+
port: 1433
|
|
100
|
+
querySingleAlwaysOnNode: false
|
|
101
|
+
# Read changes only from transaction log backups.
|
|
102
|
+
readBackupOnly: false
|
|
103
|
+
safeguardPolicy: rely-on-sql-server-replication-agent
|
|
104
|
+
serverName: test-sqlserver.example.com
|
|
105
|
+
# Transaction log access mode.
|
|
106
|
+
tlogAccessMode: BackupOnly
|
|
107
|
+
trimSpaceInChar: false
|
|
108
|
+
# Use BCP for full-load operations.
|
|
109
|
+
useBcpFullLoad: true
|
|
110
|
+
useThirdPartyBackupDevice: false
|
|
111
|
+
|
|
112
|
+
# S3 target endpoint.
|
|
113
|
+
test-target-s3:
|
|
114
|
+
endpointType: target
|
|
115
|
+
engineName: s3
|
|
116
|
+
# Amazon S3 settings.
|
|
117
|
+
s3Settings:
|
|
118
|
+
# S3 bucket name for data migration destination.
|
|
119
|
+
bucketName: test-target-bucket
|
|
120
|
+
# KMS key ID for server-side encryption.
|
|
121
|
+
serverSideEncryptionKmsKeyId: test-target-kms-key-id
|
|
122
|
+
# IAM role ARN for DMS service access to S3.
|
|
123
|
+
serviceAccessRoleArn: arn:{{partition}}:iam::{{account}}:role/test-s3-access-role
|
|
124
|
+
# S3 bucket folder for organizing migrated data.
|
|
125
|
+
bucketFolder: dms-output
|
|
126
|
+
# Add column name information to CSV output.
|
|
127
|
+
addColumnName: true
|
|
128
|
+
# Predefined ACL for S3 objects.
|
|
129
|
+
cannedAclForObjects: bucket-owner-full-control
|
|
130
|
+
# Enable CDC INSERT and UPDATE capture.
|
|
131
|
+
cdcInsertsAndUpdates: true
|
|
132
|
+
# CDC INSERT-only capture.
|
|
133
|
+
cdcInsertsOnly: false
|
|
134
|
+
# Maximum batch interval in seconds for CDC.
|
|
135
|
+
cdcMaxBatchInterval: 60
|
|
136
|
+
# Minimum file size in KB for CDC.
|
|
137
|
+
cdcMinFileSize: 32000
|
|
138
|
+
# CDC folder path.
|
|
139
|
+
cdcPath: cdc-data
|
|
140
|
+
# Compression type for S3 target files.
|
|
141
|
+
compressionType: gzip
|
|
142
|
+
# Column delimiter for CSV.
|
|
143
|
+
csvDelimiter: ','
|
|
144
|
+
# String value for columns not in supplemental log.
|
|
145
|
+
csvNoSupValue: ''
|
|
146
|
+
# Null value representation for CSV.
|
|
147
|
+
csvNullValue: 'NULL'
|
|
148
|
+
# Row delimiter for CSV.
|
|
149
|
+
csvRowDelimiter: '\n'
|
|
150
|
+
# Data format for S3 output files.
|
|
151
|
+
dataFormat: parquet
|
|
152
|
+
# Data page size in bytes for Parquet.
|
|
153
|
+
dataPageSize: 1048576
|
|
154
|
+
# Date partition delimiter.
|
|
155
|
+
datePartitionDelimiter: SLASH
|
|
156
|
+
# Enable date-based folder partitioning.
|
|
157
|
+
datePartitionEnabled: true
|
|
158
|
+
# Date format sequence for partitioning.
|
|
159
|
+
datePartitionSequence: YYYYMMDD
|
|
160
|
+
# Time zone for date partition folders.
|
|
161
|
+
datePartitionTimezone: UTC
|
|
162
|
+
# Maximum dictionary page size for Parquet.
|
|
163
|
+
dictPageSizeLimit: 1048576
|
|
164
|
+
# Enable statistics for Parquet pages.
|
|
165
|
+
enableStatistics: true
|
|
166
|
+
# Encoding type for Parquet.
|
|
167
|
+
encodingType: rle-dictionary
|
|
168
|
+
# (Optional) AWS account ID of the S3 bucket owner for cross-account access.
|
|
169
|
+
expectedBucketOwner: '{{context:account-2}}'
|
|
170
|
+
# External table definition for S3 source.
|
|
171
|
+
externalTableDefinition: ''
|
|
172
|
+
# Number of header rows to ignore in CSV.
|
|
173
|
+
ignoreHeaderRows: 1
|
|
174
|
+
# Include INSERT operation indicators in full load CSV.
|
|
175
|
+
includeOpForFullLoad: true
|
|
176
|
+
# Maximum CSV file size in KB.
|
|
177
|
+
maxFileSize: 1048576
|
|
178
|
+
# TIMESTAMP column precision to milliseconds in Parquet.
|
|
179
|
+
parquetTimestampInMillisecond: true
|
|
180
|
+
# Apache Parquet format version.
|
|
181
|
+
parquetVersion: parquet-2-0
|
|
182
|
+
# Preserve transaction order for CDC loads.
|
|
183
|
+
preserveTransactions: true
|
|
184
|
+
# Enable RFC 4180 compliance for CSV.
|
|
185
|
+
rfc4180: true
|
|
186
|
+
# Number of rows in Parquet row group.
|
|
187
|
+
rowGroupLength: 10000
|
|
188
|
+
# Timestamp column name for migration timing.
|
|
189
|
+
timestampColumnName: _dms_timestamp
|
|
190
|
+
# Use CsvNoSupValue for columns not in supplemental log.
|
|
191
|
+
useCsvNoSupValue: false
|
|
192
|
+
# Use task start time for full load timestamp.
|
|
193
|
+
useTaskStartTimeForFullLoadTimestamp: true
|
|
194
|
+
|
|
195
|
+
# MySQL source endpoint.
|
|
196
|
+
test-source-mysql:
|
|
197
|
+
endpointType: source
|
|
198
|
+
engineName: mysql
|
|
199
|
+
databaseName: test-mysql-db
|
|
200
|
+
# MySQL settings.
|
|
201
|
+
mySqlSettings:
|
|
202
|
+
# Secrets Manager secret ARN containing MySQL credentials.
|
|
203
|
+
secretsManagerSecretArn: arn:{{partition}}:secretsmanager:{{region}}:{{account}}:secret:test-mysql-secret
|
|
204
|
+
# KMS key ARN for the MySQL secret.
|
|
205
|
+
secretsManagerSecretKMSArn: arn:{{partition}}:kms:{{region}}:{{account}}:key/test-mysql-key
|
|
206
|
+
# IAM role ARN for Secrets Manager access.
|
|
207
|
+
secretsManagerAccessRoleArn: arn:{{partition}}:iam::{{account}}:role/test-sm-access-role
|
|
208
|
+
# SQL script to execute after connecting.
|
|
209
|
+
afterConnectScript: SET SESSION wait_timeout=28800
|
|
210
|
+
# Clean and recreate table metadata on mismatch.
|
|
211
|
+
cleanSourceMetadataOnMismatch: true
|
|
212
|
+
# Polling interval in seconds for binary log changes.
|
|
213
|
+
eventsPollInterval: 5
|
|
214
|
+
# Maximum CSV file size in KB.
|
|
215
|
+
maxFileSize: 65536
|
|
216
|
+
# Number of parallel threads for loading data.
|
|
217
|
+
parallelLoadThreads: 1
|
|
218
|
+
# Time zone for MySQL source database.
|
|
219
|
+
serverTimezone: UTC
|
|
220
|
+
# Target database type.
|
|
221
|
+
targetDbType: specific-database
|
|
222
|
+
|
|
223
|
+
# PostgreSQL source endpoint.
|
|
224
|
+
test-source-postgres:
|
|
225
|
+
endpointType: source
|
|
226
|
+
engineName: postgres
|
|
227
|
+
databaseName: test-postgres-db
|
|
228
|
+
# PostgreSQL settings.
|
|
229
|
+
postgreSqlSettings:
|
|
230
|
+
# Secrets Manager secret ARN containing PostgreSQL credentials.
|
|
231
|
+
secretsManagerSecretArn: arn:{{partition}}:secretsmanager:{{region}}:{{account}}:secret:test-pg-secret
|
|
232
|
+
# KMS key ARN for the PostgreSQL secret.
|
|
233
|
+
secretsManagerSecretKMSArn: arn:{{partition}}:kms:{{region}}:{{account}}:key/test-pg-key
|
|
234
|
+
# IAM role ARN for Secrets Manager access.
|
|
235
|
+
secretsManagerAccessRoleArn: arn:{{partition}}:iam::{{account}}:role/test-sm-access-role
|
|
236
|
+
# SQL script executed after connecting for CDC.
|
|
237
|
+
afterConnectScript: SET search_path TO public
|
|
238
|
+
# Babelfish for Aurora PostgreSQL database name.
|
|
239
|
+
babelfishDatabaseName: test-babelfish-db
|
|
240
|
+
# Enable DDL event capture.
|
|
241
|
+
captureDdls: true
|
|
242
|
+
# Database mode specification.
|
|
243
|
+
databaseMode: default
|
|
244
|
+
# Schema for operational DDL artifacts.
|
|
245
|
+
ddlArtifactsSchema: cdc_ddl_schema
|
|
246
|
+
# Client statement timeout in seconds.
|
|
247
|
+
executeTimeout: 60
|
|
248
|
+
# Fail task if LOB column exceeds LobMaxSize.
|
|
249
|
+
failTasksOnLobTruncation: false
|
|
250
|
+
# Enable WAL heartbeat.
|
|
251
|
+
heartbeatEnable: true
|
|
252
|
+
# WAL heartbeat frequency in minutes.
|
|
253
|
+
heartbeatFrequency: 5
|
|
254
|
+
# Schema for heartbeat artifacts.
|
|
255
|
+
heartbeatSchema: public
|
|
256
|
+
# Migrate boolean as boolean.
|
|
257
|
+
mapBooleanAsBoolean: true
|
|
258
|
+
# Maximum CSV file size in KB.
|
|
259
|
+
maxFileSize: 32000
|
|
260
|
+
# Plugin for replication slot.
|
|
261
|
+
pluginName: pglogical
|
|
262
|
+
# Logical replication slot name.
|
|
263
|
+
slotName: test_slot
|
|
264
|
+
|
|
265
|
+
# Oracle source endpoint.
|
|
266
|
+
test-source-oracle:
|
|
267
|
+
endpointType: source
|
|
268
|
+
engineName: oracle
|
|
269
|
+
databaseName: test-oracle-db
|
|
270
|
+
# Oracle settings.
|
|
271
|
+
oracleSettings:
|
|
272
|
+
# Secrets Manager secret ARN containing Oracle credentials.
|
|
273
|
+
secretsManagerSecretArn: arn:{{partition}}:secretsmanager:{{region}}:{{account}}:secret:test-oracle-secret
|
|
274
|
+
# KMS key ARN for the Oracle secret.
|
|
275
|
+
secretsManagerSecretKMSArn: arn:{{partition}}:kms:{{region}}:{{account}}:key/test-oracle-key
|
|
276
|
+
# IAM role ARN for Secrets Manager access.
|
|
277
|
+
secretsManagerAccessRoleArn: arn:{{partition}}:iam::{{account}}:role/test-sm-access-role
|
|
278
|
+
# Disable Binary Reader direct file access.
|
|
279
|
+
accessAlternateDirectly: false
|
|
280
|
+
# Enable table-level supplemental logging.
|
|
281
|
+
addSupplementalLogging: true
|
|
282
|
+
# Additional archived log destination ID.
|
|
283
|
+
additionalArchivedLogDestId: 1
|
|
284
|
+
# Enable replication of nested tables.
|
|
285
|
+
allowSelectNestedTables: true
|
|
286
|
+
# Archived redo log destination ID.
|
|
287
|
+
archivedLogDestId: 1
|
|
288
|
+
# Restrict to archived redo logs only.
|
|
289
|
+
archivedLogsOnly: false
|
|
290
|
+
# ASM server address.
|
|
291
|
+
asmServer: test-asm-server
|
|
292
|
+
# Character length semantics.
|
|
293
|
+
charLengthSemantics: byte
|
|
294
|
+
# Enable direct path loading without logging.
|
|
295
|
+
directPathNoLog: false
|
|
296
|
+
# Enable parallel loading with direct path.
|
|
297
|
+
directPathParallelLoad: false
|
|
298
|
+
# Enable homogeneous tablespace replication.
|
|
299
|
+
enableHomogenousTablespace: false
|
|
300
|
+
# Additional archived log destination IDs.
|
|
301
|
+
extraArchivedLogDestIds:
|
|
302
|
+
- 2
|
|
303
|
+
- 3
|
|
304
|
+
# Fail task when LOB exceeds LobMaxSize.
|
|
305
|
+
failTasksOnLobTruncation: false
|
|
306
|
+
# Number data type scale.
|
|
307
|
+
numberDatatypeScale: -1
|
|
308
|
+
# Oracle path prefix for Binary Reader.
|
|
309
|
+
oraclePathPrefix: /rdsdbdata/db/
|
|
310
|
+
# Parallel ASM read threads.
|
|
311
|
+
parallelAsmReadThreads: 2
|
|
312
|
+
# Read-ahead blocks for ASM.
|
|
313
|
+
readAheadBlocks: 1000
|
|
314
|
+
# Enable tablespace name reading.
|
|
315
|
+
readTableSpaceName: false
|
|
316
|
+
# Enable path prefix replacement.
|
|
317
|
+
replacePathPrefix: false
|
|
318
|
+
# Retry interval in seconds.
|
|
319
|
+
retryInterval: 5
|
|
320
|
+
# Secrets Manager ARN for Oracle ASM access role.
|
|
321
|
+
secretsManagerOracleAsmAccessRoleArn: arn:{{partition}}:iam::{{account}}:role/test-asm-role
|
|
322
|
+
# Secrets Manager ARN for Oracle ASM secret.
|
|
323
|
+
secretsManagerOracleAsmSecretArn: arn:{{partition}}:secretsmanager:{{region}}:{{account}}:secret:test-asm-secret
|
|
324
|
+
# Custom function for SDO_GEOMETRY to GEOJSON.
|
|
325
|
+
spatialDataOptionToGeoJsonFunctionName: test_sdo_to_geojson
|
|
326
|
+
# Standby delay time in minutes.
|
|
327
|
+
standbyDelayTime: 0
|
|
328
|
+
# Enable alternate folder for online redo logs.
|
|
329
|
+
useAlternateFolderForOnline: false
|
|
330
|
+
# Enable Binary Reader utility.
|
|
331
|
+
useBFile: false
|
|
332
|
+
# Enable direct path full load.
|
|
333
|
+
useDirectPathFullLoad: true
|
|
334
|
+
# Enable Oracle LogMiner.
|
|
335
|
+
useLogminerReader: true
|
|
336
|
+
# Path prefix for Binary Reader replacement.
|
|
337
|
+
usePathPrefix: /rdsdbdata/log/
|
|
338
|
+
|
|
339
|
+
# MongoDB source endpoint.
|
|
340
|
+
test-source-mongodb:
|
|
341
|
+
endpointType: source
|
|
342
|
+
engineName: mongodb
|
|
343
|
+
databaseName: test-mongo-db
|
|
344
|
+
# MongoDB settings.
|
|
345
|
+
mongoDbSettings:
|
|
346
|
+
# Secrets Manager secret ARN containing MongoDB credentials.
|
|
347
|
+
secretsManagerSecretArn: arn:{{partition}}:secretsmanager:{{region}}:{{account}}:secret:test-mongo-secret
|
|
348
|
+
# KMS key ARN for the MongoDB secret.
|
|
349
|
+
secretsManagerSecretKMSArn: arn:{{partition}}:kms:{{region}}:{{account}}:key/test-mongo-key
|
|
350
|
+
# IAM role ARN for Secrets Manager access.
|
|
351
|
+
secretsManagerAccessRoleArn: arn:{{partition}}:iam::{{account}}:role/test-sm-access-role
|
|
352
|
+
# Authentication mechanism.
|
|
353
|
+
authMechanism: scram-sha-1
|
|
354
|
+
# MongoDB database for authentication.
|
|
355
|
+
authSource: admin
|
|
356
|
+
# Authentication type.
|
|
357
|
+
authType: password
|
|
358
|
+
# Database name on MongoDB source.
|
|
359
|
+
databaseName: test-mongo-db
|
|
360
|
+
# Number of documents to preview.
|
|
361
|
+
docsToInvestigate: '1000'
|
|
362
|
+
# Document ID extraction flag.
|
|
363
|
+
extractDocId: 'true'
|
|
364
|
+
# Nesting level (document or table mode).
|
|
365
|
+
nestingLevel: one
|
|
366
|
+
# Port value for MongoDB source.
|
|
367
|
+
port: 27017
|
|
368
|
+
# Server name.
|
|
369
|
+
serverName: test-mongo-server.example.com
|
|
370
|
+
|
|
371
|
+
# DocumentDB source endpoint.
|
|
372
|
+
test-source-docdb:
|
|
373
|
+
endpointType: source
|
|
374
|
+
engineName: docdb
|
|
375
|
+
databaseName: test-docdb-db
|
|
376
|
+
# DocumentDB settings.
|
|
377
|
+
docDbSettings:
|
|
378
|
+
# Secrets Manager secret ARN containing DocumentDB credentials.
|
|
379
|
+
secretsManagerSecretArn: arn:{{partition}}:secretsmanager:{{region}}:{{account}}:secret:test-docdb-secret
|
|
380
|
+
# KMS key ARN for the DocumentDB secret.
|
|
381
|
+
secretsManagerSecretKMSArn: arn:{{partition}}:kms:{{region}}:{{account}}:key/test-docdb-key
|
|
382
|
+
secretsManagerAccessRoleArn: arn:{{partition}}:iam::{{account}}:role/test-sm-access-role
|
|
383
|
+
# Number of documents to preview.
|
|
384
|
+
docsToInvestigate: 1000
|
|
385
|
+
extractDocId: true
|
|
386
|
+
# Nesting level for migration mode.
|
|
387
|
+
nestingLevel: one
|
|
388
|
+
|
|
389
|
+
# IBM DB2 source endpoint.
|
|
390
|
+
test-source-db2:
|
|
391
|
+
endpointType: source
|
|
392
|
+
engineName: db2
|
|
393
|
+
databaseName: test-db2-db
|
|
394
|
+
# IBM DB2 settings.
|
|
395
|
+
ibmDb2Settings:
|
|
396
|
+
# Secrets Manager secret ARN containing DB2 credentials.
|
|
397
|
+
secretsManagerSecretArn: arn:{{partition}}:secretsmanager:{{region}}:{{account}}:secret:test-db2-secret
|
|
398
|
+
# KMS key ARN for the DB2 secret.
|
|
399
|
+
secretsManagerSecretKMSArn: arn:{{partition}}:kms:{{region}}:{{account}}:key/test-db2-key
|
|
400
|
+
# IAM role ARN for Secrets Manager access.
|
|
401
|
+
secretsManagerAccessRoleArn: arn:{{partition}}:iam::{{account}}:role/test-sm-access-role
|
|
402
|
+
# Log sequence number for CDC starting point.
|
|
403
|
+
currentLsn: '00000000:00000000:0000'
|
|
404
|
+
# Maximum bytes per read operation.
|
|
405
|
+
maxKBytesPerRead: 64
|
|
406
|
+
# Enable ongoing replication (CDC).
|
|
407
|
+
setDataCaptureChanges: true
|
|
408
|
+
|
|
409
|
+
# DynamoDB target endpoint.
|
|
410
|
+
test-target-dynamodb:
|
|
411
|
+
endpointType: target
|
|
412
|
+
engineName: dynamodb
|
|
413
|
+
# DynamoDB settings.
|
|
414
|
+
dynamoDbSettings:
|
|
415
|
+
# IAM service role ARN for DynamoDB endpoint access.
|
|
416
|
+
serviceAccessRoleArn: arn:{{partition}}:iam::{{account}}:role/test-dynamodb-access-role
|
|
417
|
+
|
|
418
|
+
# OpenSearch/Elasticsearch target endpoint.
|
|
419
|
+
test-target-elasticsearch:
|
|
420
|
+
endpointType: target
|
|
421
|
+
engineName: elasticsearch
|
|
422
|
+
# OpenSearch/Elasticsearch settings.
|
|
423
|
+
elasticsearchSettings:
|
|
424
|
+
# OpenSearch cluster endpoint URI.
|
|
425
|
+
endpointUri: https://test-es-domain.{{region}}.es.amazonaws.com
|
|
426
|
+
# Retry duration in seconds.
|
|
427
|
+
errorRetryDuration: 300
|
|
428
|
+
# Maximum percentage of failed records before stopping.
|
|
429
|
+
fullLoadErrorPercentage: 10
|
|
430
|
+
# IAM role ARN for service access.
|
|
431
|
+
serviceAccessRoleArn: arn:{{partition}}:iam::{{account}}:role/test-es-access-role
|
|
432
|
+
|
|
433
|
+
# Kinesis target endpoint.
|
|
434
|
+
test-target-kinesis:
|
|
435
|
+
endpointType: target
|
|
436
|
+
engineName: kinesis
|
|
437
|
+
# Kinesis settings.
|
|
438
|
+
kinesisSettings:
|
|
439
|
+
# Kinesis data stream ARN.
|
|
440
|
+
streamArn: arn:{{partition}}:kinesis:{{region}}:{{account}}:stream/test-stream
|
|
441
|
+
# IAM role ARN for Kinesis access.
|
|
442
|
+
serviceAccessRoleArn: arn:{{partition}}:iam::{{account}}:role/test-kinesis-access-role
|
|
443
|
+
# Show detailed control information.
|
|
444
|
+
includeControlDetails: true
|
|
445
|
+
# Include NULL and empty columns.
|
|
446
|
+
includeNullAndEmpty: true
|
|
447
|
+
# Show partition value in output.
|
|
448
|
+
includePartitionValue: true
|
|
449
|
+
# Include DDL operations.
|
|
450
|
+
includeTableAlterOperations: true
|
|
451
|
+
# Provide detailed transaction information.
|
|
452
|
+
includeTransactionDetails: true
|
|
453
|
+
# Output format for records.
|
|
454
|
+
messageFormat: json
|
|
455
|
+
# Avoid adding '0x' prefix to hex data.
|
|
456
|
+
noHexPrefix: false
|
|
457
|
+
# Prefix schema and table names to partition values.
|
|
458
|
+
partitionIncludeSchemaTable: true
|
|
459
|
+
|
|
460
|
+
# Neptune target endpoint.
|
|
461
|
+
test-target-neptune:
|
|
462
|
+
endpointType: target
|
|
463
|
+
engineName: neptune
|
|
464
|
+
# Neptune settings.
|
|
465
|
+
neptuneSettings:
|
|
466
|
+
# S3 bucket name for temporary graph data storage.
|
|
467
|
+
s3BucketName: test-neptune-staging-bucket
|
|
468
|
+
# S3 bucket folder for staging.
|
|
469
|
+
s3BucketFolder: neptune-staging
|
|
470
|
+
# IAM role ARN for service access.
|
|
471
|
+
serviceAccessRoleArn: arn:{{partition}}:iam::{{account}}:role/test-neptune-access-role
|
|
472
|
+
# Retry duration in milliseconds.
|
|
473
|
+
errorRetryDuration: 300
|
|
474
|
+
maxFileSize: 1048576
|
|
475
|
+
# Maximum retry count.
|
|
476
|
+
maxRetryCount: 3
|
|
477
|
+
|
|
478
|
+
# Redshift target endpoint.
|
|
479
|
+
test-target-redshift:
|
|
480
|
+
endpointType: target
|
|
481
|
+
engineName: redshift
|
|
482
|
+
databaseName: test-redshift-db
|
|
483
|
+
# Redshift settings.
|
|
484
|
+
redshiftSettings:
|
|
485
|
+
# S3 bucket name for intermediate CSV storage.
|
|
486
|
+
bucketName: test-redshift-staging-bucket
|
|
487
|
+
# Secrets Manager secret ARN containing Redshift credentials.
|
|
488
|
+
secretsManagerSecretArn: arn:{{partition}}:secretsmanager:{{region}}:{{account}}:secret:test-redshift-secret
|
|
489
|
+
# KMS key ID for server-side encryption.
|
|
490
|
+
serverSideEncryptionKmsKeyId: test-redshift-kms-key-id
|
|
491
|
+
# KMS key ARN for the Redshift secret.
|
|
492
|
+
secretsManagerSecretKMSArn: arn:{{partition}}:kms:{{region}}:{{account}}:key/test-redshift-key
|
|
493
|
+
# IAM role ARN for Secrets Manager access.
|
|
494
|
+
secretsManagerAccessRoleArn: arn:{{partition}}:iam::{{account}}:role/test-sm-access-role
|
|
495
|
+
# IAM role ARN for DMS service access.
|
|
496
|
+
serviceAccessRoleArn: arn:{{partition}}:iam::{{account}}:role/test-redshift-access-role
|
|
497
|
+
# Allow any date format.
|
|
498
|
+
acceptAnyDate: true
|
|
499
|
+
# SQL script after connecting.
|
|
500
|
+
afterConnectScript: SET search_path TO public
|
|
501
|
+
# S3 folder for CSV staging.
|
|
502
|
+
bucketFolder: redshift-staging
|
|
503
|
+
# Enable case-sensitive schema names.
|
|
504
|
+
caseSensitiveNames: false
|
|
505
|
+
# Enable automatic compression.
|
|
506
|
+
compUpdate: true
|
|
507
|
+
# Connection timeout in milliseconds.
|
|
508
|
+
connectionTimeout: 10000
|
|
509
|
+
# Date format specification.
|
|
510
|
+
dateFormat: auto
|
|
511
|
+
# Migrate empty fields as NULL.
|
|
512
|
+
emptyAsNull: true
|
|
513
|
+
# Override auto-generated IDENTITY values.
|
|
514
|
+
explicitIds: false
|
|
515
|
+
# Parallel threads for file upload.
|
|
516
|
+
fileTransferUploadStreams: 3
|
|
517
|
+
# Timeout in milliseconds for cluster operations.
|
|
518
|
+
loadTimeout: 600000
|
|
519
|
+
# Migrate boolean as native boolean.
|
|
520
|
+
mapBooleanAsBoolean: true
|
|
521
|
+
# Maximum CSV file size in KB.
|
|
522
|
+
maxFileSize: 1048576
|
|
523
|
+
# Remove surrounding quotation marks.
|
|
524
|
+
removeQuotes: true
|
|
525
|
+
# Replacement character for invalid characters.
|
|
526
|
+
replaceChars: '?'
|
|
527
|
+
# Characters to replace during migration.
|
|
528
|
+
replaceInvalidChars: ''
|
|
529
|
+
# Time format specification.
|
|
530
|
+
timeFormat: auto
|
|
531
|
+
# Remove trailing white space from VARCHAR.
|
|
532
|
+
trimBlanks: true
|
|
533
|
+
# Truncate data to fit column size.
|
|
534
|
+
truncateColumns: true
|
|
535
|
+
# In-memory write buffer size in KB.
|
|
536
|
+
writeBufferSize: 512
|
|
537
|
+
|
|
538
|
+
# Sybase (SAP ASE) source endpoint.
|
|
539
|
+
test-source-sybase:
|
|
540
|
+
endpointType: source
|
|
541
|
+
engineName: sybase
|
|
542
|
+
databaseName: test-sybase-db
|
|
543
|
+
# SAP ASE (Sybase) settings.
|
|
544
|
+
sybaseSettings:
|
|
545
|
+
# Secrets Manager secret ARN containing Sybase credentials.
|
|
546
|
+
secretsManagerSecretArn: arn:{{partition}}:secretsmanager:{{region}}:{{account}}:secret:test-sybase-secret
|
|
547
|
+
# KMS key ARN for the Sybase secret.
|
|
548
|
+
secretsManagerSecretKMSArn: arn:{{partition}}:kms:{{region}}:{{account}}:key/test-sybase-key
|
|
549
|
+
# IAM role ARN for Secrets Manager access.
|
|
550
|
+
secretsManagerAccessRoleArn: arn:{{partition}}:iam::{{account}}:role/test-sm-access-role
|
|
551
|
+
|
|
552
|
+
# Named replication task configurations.
|
|
553
|
+
replicationTasks:
|
|
554
|
+
# Full-load migration task.
|
|
555
|
+
test-task-full-load:
|
|
556
|
+
# Replication instance name from replicationInstances section.
|
|
557
|
+
replicationInstance: test-instance
|
|
558
|
+
# Source endpoint name from endpoints section.
|
|
559
|
+
sourceEndpoint: test-source-sqlserver
|
|
560
|
+
# Target endpoint name from endpoints section.
|
|
561
|
+
targetEndpoint: test-target-s3
|
|
562
|
+
# Migration type. (enum: full-load, cdc, full-load-and-cdc)
|
|
563
|
+
migrationType: full-load
|
|
564
|
+
# Overall settings for the task in JSON format.
|
|
565
|
+
replicationTaskSettings:
|
|
566
|
+
TargetMetadata:
|
|
567
|
+
TargetSchema: ''
|
|
568
|
+
SupportLobs: true
|
|
569
|
+
FullLoadSettings:
|
|
570
|
+
TargetTablePrepMode: DROP_AND_CREATE
|
|
571
|
+
# Table mappings for the task.
|
|
572
|
+
tableMappings:
|
|
573
|
+
rules:
|
|
574
|
+
- rule-type: selection
|
|
575
|
+
rule-id: '1'
|
|
576
|
+
rule-name: '1'
|
|
577
|
+
object-locator:
|
|
578
|
+
schema-name: Test
|
|
579
|
+
table-name: '%'
|
|
580
|
+
rule-action: include
|
|
581
|
+
- rule-type: selection
|
|
582
|
+
rule-id: '2'
|
|
583
|
+
rule-name: '2'
|
|
584
|
+
object-locator:
|
|
585
|
+
schema-name: Test
|
|
586
|
+
table-name: DMS%
|
|
587
|
+
rule-action: exclude
|
|
588
|
+
# Supplemental information for certain source/target endpoints.
|
|
589
|
+
taskData:
|
|
590
|
+
supplementalKey: supplementalValue
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# Minimal configuration for DataOps DMS module.
|
|
2
|
+
# Only required properties are included.
|
|
3
|
+
# projectName is included to auto-wire KMS and other shared resources.
|
|
4
|
+
|
|
5
|
+
# DataOps project name for auto-wiring shared resources.
|
|
6
|
+
projectName: test-project
|
|
7
|
+
|
|
8
|
+
# DMS migration and replication configuration.
|
|
9
|
+
dms:
|
|
10
|
+
replicationInstances:
|
|
11
|
+
test-instance:
|
|
12
|
+
instanceClass: dms.t3.micro
|
|
13
|
+
# VPC ID for DMS replication instance deployment
|
|
14
|
+
# Often created by your VPC/networking stack.
|
|
15
|
+
# Example SSM: ssm:/path/to/vpc/id
|
|
16
|
+
vpcId: test_vpc_id
|
|
17
|
+
# Subnet IDs for DMS replication instance placement
|
|
18
|
+
# Often created by your VPC/networking stack.
|
|
19
|
+
# Example SSM: ssm:/path/to/subnet/id
|
|
20
|
+
subnetIds:
|
|
21
|
+
- test_subnet_id1
|
|
22
|
+
- test_subnet_id2
|