@databricks/sdk-pipelines 0.1.0-dev.3 → 0.1.0-dev.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -0
- package/dist/v2/client.d.ts +3 -3
- package/dist/v2/client.d.ts.map +1 -1
- package/dist/v2/client.js +71 -61
- package/dist/v2/client.js.map +1 -1
- package/dist/v2/index.d.ts +1 -1
- package/dist/v2/index.d.ts.map +1 -1
- package/dist/v2/model.d.ts +263 -333
- package/dist/v2/model.d.ts.map +1 -1
- package/dist/v2/model.js +277 -282
- package/dist/v2/model.js.map +1 -1
- package/dist/v2/transport.d.ts +30 -2
- package/dist/v2/transport.d.ts.map +1 -1
- package/dist/v2/transport.js +33 -16
- package/dist/v2/transport.js.map +1 -1
- package/dist/v2/utils.d.ts.map +1 -1
- package/dist/v2/utils.js +2 -1
- package/dist/v2/utils.js.map +1 -1
- package/package.json +9 -5
- package/src/v2/client.ts +0 -630
- package/src/v2/index.ts +0 -154
- package/src/v2/model.ts +0 -5487
- package/src/v2/transport.ts +0 -73
- package/src/v2/utils.ts +0 -180
package/dist/v2/model.js
CHANGED
|
@@ -1,317 +1,308 @@
|
|
|
1
1
|
// Code generated from API definition by Databricks SDK Generator. DO NOT EDIT.
|
|
2
2
|
import { z } from 'zod';
|
|
3
3
|
/** Enum to specify which mode of clone to execute */
|
|
4
|
-
|
|
5
|
-
|
|
4
|
+
// eslint-disable-next-line @typescript-eslint/naming-convention -- Enum-style const object.
|
|
5
|
+
export const CloneMode = {
|
|
6
6
|
/** Data and metadata are copied */
|
|
7
|
-
|
|
8
|
-
}
|
|
7
|
+
MIGRATE_TO_UC: 'MIGRATE_TO_UC',
|
|
8
|
+
};
|
|
9
9
|
/**
|
|
10
10
|
* For certain database sources LakeFlow Connect offers both query based and cdc
|
|
11
11
|
* ingestion, ConnectorType can bse used to convey the type of ingestion.
|
|
12
12
|
* If connection_name is provided for database sources, we default to Query Based ingestion
|
|
13
13
|
*/
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
14
|
+
// eslint-disable-next-line @typescript-eslint/naming-convention -- Enum-style const object.
|
|
15
|
+
export const ConnectorType = {
|
|
16
|
+
CONNECTOR_TYPE_UNSPECIFIED: 'CONNECTOR_TYPE_UNSPECIFIED',
|
|
17
17
|
/**
|
|
18
18
|
* If connector_type = CDC and ingestion_gateway_id is provided then we use Ingestion Gateway pipeline with
|
|
19
19
|
* Cdc Managed Ingestion Pipeline for ingestion, if connector_type = CDC and connection_name is provided
|
|
20
20
|
* then we use Combined Cdc Managed Ingestion Pipeline.
|
|
21
21
|
*/
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
}
|
|
22
|
+
CDC: 'CDC',
|
|
23
|
+
QUERY_BASED: 'QUERY_BASED',
|
|
24
|
+
};
|
|
25
25
|
/**
|
|
26
26
|
* Days of week in which the window is allowed to happen.
|
|
27
27
|
* If not specified all days of the week will be used.
|
|
28
28
|
*/
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
}
|
|
29
|
+
// eslint-disable-next-line @typescript-eslint/naming-convention -- Enum-style const object.
|
|
30
|
+
export const DayOfWeek = {
|
|
31
|
+
DAY_OF_WEEK_UNSPECIFIED: 'DAY_OF_WEEK_UNSPECIFIED',
|
|
32
|
+
MONDAY: 'MONDAY',
|
|
33
|
+
TUESDAY: 'TUESDAY',
|
|
34
|
+
WEDNESDAY: 'WEDNESDAY',
|
|
35
|
+
THURSDAY: 'THURSDAY',
|
|
36
|
+
FRIDAY: 'FRIDAY',
|
|
37
|
+
SATURDAY: 'SATURDAY',
|
|
38
|
+
SUNDAY: 'SUNDAY',
|
|
39
|
+
};
|
|
40
40
|
/**
|
|
41
41
|
* The deployment method that manages the pipeline:
|
|
42
42
|
* - BUNDLE: The pipeline is managed by a Databricks Asset Bundle.
|
|
43
43
|
*/
|
|
44
|
-
|
|
45
|
-
|
|
44
|
+
// eslint-disable-next-line @typescript-eslint/naming-convention -- Enum-style const object.
|
|
45
|
+
export const DeploymentKind = {
|
|
46
46
|
/** Databricks Asset Bundle (DAB) */
|
|
47
|
-
|
|
48
|
-
}
|
|
47
|
+
BUNDLE: 'BUNDLE',
|
|
48
|
+
};
|
|
49
49
|
/** The severity level of the event. */
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
}
|
|
50
|
+
// eslint-disable-next-line @typescript-eslint/naming-convention -- Enum-style const object.
|
|
51
|
+
export const EventLevel = {
|
|
52
|
+
INFO: 'INFO',
|
|
53
|
+
WARN: 'WARN',
|
|
54
|
+
ERROR: 'ERROR',
|
|
55
|
+
METRICS: 'METRICS',
|
|
56
|
+
};
|
|
57
|
+
// eslint-disable-next-line @typescript-eslint/naming-convention -- Enum-style const object.
|
|
58
|
+
export const IngestionSourceType = {
|
|
59
|
+
INGESTION_SOURCE_TYPE_UNSPECIFIED: 'INGESTION_SOURCE_TYPE_UNSPECIFIED',
|
|
60
|
+
MYSQL: 'MYSQL',
|
|
61
|
+
POSTGRESQL: 'POSTGRESQL',
|
|
62
|
+
SQLSERVER: 'SQLSERVER',
|
|
63
|
+
SALESFORCE: 'SALESFORCE',
|
|
64
|
+
BIGQUERY: 'BIGQUERY',
|
|
65
|
+
NETSUITE: 'NETSUITE',
|
|
66
|
+
WORKDAY_RAAS: 'WORKDAY_RAAS',
|
|
67
|
+
GA4_RAW_DATA: 'GA4_RAW_DATA',
|
|
68
|
+
SERVICENOW: 'SERVICENOW',
|
|
69
|
+
MANAGED_POSTGRESQL: 'MANAGED_POSTGRESQL',
|
|
70
|
+
ORACLE: 'ORACLE',
|
|
71
|
+
TERADATA: 'TERADATA',
|
|
72
|
+
SHAREPOINT: 'SHAREPOINT',
|
|
73
|
+
DYNAMICS365: 'DYNAMICS365',
|
|
74
|
+
GOOGLE_DRIVE: 'GOOGLE_DRIVE',
|
|
75
|
+
JIRA: 'JIRA',
|
|
76
|
+
CONFLUENCE: 'CONFLUENCE',
|
|
77
|
+
META_MARKETING: 'META_MARKETING',
|
|
78
|
+
ZENDESK: 'ZENDESK',
|
|
79
|
+
FOREIGN_CATALOG: 'FOREIGN_CATALOG',
|
|
80
|
+
};
|
|
81
81
|
/** Maturity level for EventDetails. */
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
}
|
|
82
|
+
// eslint-disable-next-line @typescript-eslint/naming-convention -- Enum-style const object.
|
|
83
|
+
export const MaturityLevel = {
|
|
84
|
+
STABLE: 'STABLE',
|
|
85
|
+
EVOLVING: 'EVOLVING',
|
|
86
|
+
DEPRECATED: 'DEPRECATED',
|
|
87
|
+
};
|
|
88
88
|
/** Attachment behavior mode for Outlook ingestion */
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
89
|
+
// eslint-disable-next-line @typescript-eslint/naming-convention -- Enum-style const object.
|
|
90
|
+
export const OutlookAttachmentMode = {
|
|
91
|
+
OUTLOOK_ATTACHMENT_MODE_UNSPECIFIED: 'OUTLOOK_ATTACHMENT_MODE_UNSPECIFIED',
|
|
92
92
|
/** Ingest all attachments (both inline and non-inline) */
|
|
93
|
-
|
|
93
|
+
ALL: 'ALL',
|
|
94
94
|
/** Ingest only non-inline attachments (recommended to avoid corporate signature images) */
|
|
95
|
-
|
|
95
|
+
NON_INLINE_ONLY: 'NON_INLINE_ONLY',
|
|
96
96
|
/** Ingest only inline attachments */
|
|
97
|
-
|
|
97
|
+
INLINE_ONLY: 'INLINE_ONLY',
|
|
98
98
|
/** Do not ingest any attachments */
|
|
99
|
-
|
|
100
|
-
}
|
|
99
|
+
NONE: 'NONE',
|
|
100
|
+
};
|
|
101
101
|
/** Body format for Outlook email content */
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
}
|
|
102
|
+
// eslint-disable-next-line @typescript-eslint/naming-convention -- Enum-style const object.
|
|
103
|
+
export const OutlookBodyFormat = {
|
|
104
|
+
OUTLOOK_BODY_FORMAT_UNSPECIFIED: 'OUTLOOK_BODY_FORMAT_UNSPECIFIED',
|
|
105
|
+
TEXT_HTML: 'TEXT_HTML',
|
|
106
|
+
TEXT_PLAIN: 'TEXT_PLAIN',
|
|
107
|
+
};
|
|
108
108
|
/** The health of a pipeline. */
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
}
|
|
109
|
+
// eslint-disable-next-line @typescript-eslint/naming-convention -- Enum-style const object.
|
|
110
|
+
export const PipelineHealthStatus = {
|
|
111
|
+
HEALTHY: 'HEALTHY',
|
|
112
|
+
UNHEALTHY: 'UNHEALTHY',
|
|
113
|
+
};
|
|
114
114
|
/** The set of AWS availability types supported when setting up nodes for a cluster. */
|
|
115
|
-
|
|
116
|
-
|
|
115
|
+
// eslint-disable-next-line @typescript-eslint/naming-convention -- Enum-style const object.
|
|
116
|
+
export const PipelinesAwsAvailability = {
|
|
117
117
|
/** Use spot instances. */
|
|
118
|
-
|
|
118
|
+
SPOT: 'SPOT',
|
|
119
119
|
/** Use on-demand instances. */
|
|
120
|
-
|
|
120
|
+
ON_DEMAND: 'ON_DEMAND',
|
|
121
121
|
/**
|
|
122
122
|
* Preferably use spot instances, but fall back to on-demand instances if spot instances cannot
|
|
123
123
|
* be acquired (e.g., if AWS spot prices are too high).
|
|
124
124
|
*/
|
|
125
|
-
|
|
126
|
-
}
|
|
125
|
+
SPOT_WITH_FALLBACK: 'SPOT_WITH_FALLBACK',
|
|
126
|
+
};
|
|
127
127
|
/** The set of Azure availability types supported when setting up nodes for a cluster. */
|
|
128
|
-
|
|
129
|
-
|
|
128
|
+
// eslint-disable-next-line @typescript-eslint/naming-convention -- Enum-style const object.
|
|
129
|
+
export const PipelinesAzureAvailability = {
|
|
130
130
|
/** Use spot instances. */
|
|
131
|
-
|
|
131
|
+
SPOT_AZURE: 'SPOT_AZURE',
|
|
132
132
|
/** Use on-demand instances. */
|
|
133
|
-
|
|
133
|
+
ON_DEMAND_AZURE: 'ON_DEMAND_AZURE',
|
|
134
134
|
/**
|
|
135
135
|
* Preferably use spot instances, but fall back to on-demand instances if spot instances cannot
|
|
136
136
|
* be acquired (e.g., if Azure is out of Quota).
|
|
137
137
|
*/
|
|
138
|
-
|
|
139
|
-
}
|
|
138
|
+
SPOT_WITH_FALLBACK_AZURE: 'SPOT_WITH_FALLBACK_AZURE',
|
|
139
|
+
};
|
|
140
140
|
/**
|
|
141
141
|
* All EBS volume types that <Databricks> supports.
|
|
142
142
|
* See https://aws.amazon.com/ebs/details/ for details.
|
|
143
143
|
*/
|
|
144
|
-
|
|
145
|
-
|
|
144
|
+
// eslint-disable-next-line @typescript-eslint/naming-convention -- Enum-style const object.
|
|
145
|
+
export const PipelinesEbsVolumeType = {
|
|
146
146
|
/** Provision extra storage using AWS gp2 EBS volumes. */
|
|
147
|
-
|
|
147
|
+
GENERAL_PURPOSE_SSD: 'GENERAL_PURPOSE_SSD',
|
|
148
148
|
/** Provision extra storage using AWS st1 volumes. */
|
|
149
|
-
|
|
150
|
-
}
|
|
149
|
+
THROUGHPUT_OPTIMIZED_HDD: 'THROUGHPUT_OPTIMIZED_HDD',
|
|
150
|
+
};
|
|
151
151
|
/** The set of GCP availability types supported when setting up nodes for a cluster (configurable only for executors). */
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
}
|
|
152
|
+
// eslint-disable-next-line @typescript-eslint/naming-convention -- Enum-style const object.
|
|
153
|
+
export const PipelinesGcpAvailability = {
|
|
154
|
+
PREEMPTIBLE_GCP: 'PREEMPTIBLE_GCP',
|
|
155
|
+
ON_DEMAND_GCP: 'ON_DEMAND_GCP',
|
|
156
|
+
PREEMPTIBLE_WITH_FALLBACK_GCP: 'PREEMPTIBLE_WITH_FALLBACK_GCP',
|
|
157
|
+
};
|
|
158
158
|
/** Enum representing the publishing mode of a pipeline. */
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
}
|
|
159
|
+
// eslint-disable-next-line @typescript-eslint/naming-convention -- Enum-style const object.
|
|
160
|
+
export const PublishingMode = {
|
|
161
|
+
PUBLISHING_MODE_UNSPECIFIED: 'PUBLISHING_MODE_UNSPECIFIED',
|
|
162
|
+
LEGACY_PUBLISHING_MODE: 'LEGACY_PUBLISHING_MODE',
|
|
163
|
+
DEFAULT_PUBLISHING_MODE: 'DEFAULT_PUBLISHING_MODE',
|
|
164
|
+
};
|
|
165
165
|
/** What triggered this update. */
|
|
166
|
-
|
|
167
|
-
|
|
166
|
+
// eslint-disable-next-line @typescript-eslint/naming-convention -- Enum-style const object.
|
|
167
|
+
export const UpdateCause = {
|
|
168
168
|
/** Started through an API call. */
|
|
169
|
-
|
|
169
|
+
API_CALL: 'API_CALL',
|
|
170
170
|
/** Started as a retry for a failed update. */
|
|
171
|
-
|
|
171
|
+
RETRY_ON_FAILURE: 'RETRY_ON_FAILURE',
|
|
172
172
|
/** Started as a result of a service upgrade. */
|
|
173
|
-
|
|
173
|
+
SERVICE_UPGRADE: 'SERVICE_UPGRADE',
|
|
174
174
|
/** Started as a result of a schema change. */
|
|
175
|
-
|
|
175
|
+
SCHEMA_CHANGE: 'SCHEMA_CHANGE',
|
|
176
176
|
/** Started by the Jobs service. */
|
|
177
|
-
|
|
177
|
+
JOB_TASK: 'JOB_TASK',
|
|
178
178
|
/** Started by an action a user performed. */
|
|
179
|
-
|
|
179
|
+
USER_ACTION: 'USER_ACTION',
|
|
180
180
|
/** Started for infrastructure maintenance reason. */
|
|
181
|
-
|
|
182
|
-
}
|
|
181
|
+
INFRASTRUCTURE_MAINTENANCE: 'INFRASTRUCTURE_MAINTENANCE',
|
|
182
|
+
};
|
|
183
183
|
/** The update state. */
|
|
184
|
-
|
|
185
|
-
|
|
184
|
+
// eslint-disable-next-line @typescript-eslint/naming-convention -- Enum-style const object.
|
|
185
|
+
export const UpdateState = {
|
|
186
186
|
/** Update is waiting for previous update to finish. */
|
|
187
|
-
|
|
187
|
+
QUEUED: 'QUEUED',
|
|
188
188
|
/** Initial state of an update. */
|
|
189
|
-
|
|
189
|
+
CREATED: 'CREATED',
|
|
190
190
|
/** Update is waiting for clusters, jobs, or other resources. */
|
|
191
|
-
|
|
191
|
+
WAITING_FOR_RESOURCES: 'WAITING_FOR_RESOURCES',
|
|
192
192
|
/** Update is creating the dataflow graph. */
|
|
193
|
-
|
|
193
|
+
INITIALIZING: 'INITIALIZING',
|
|
194
194
|
/** Update is resetting datasets and checkpoints to the beginning. */
|
|
195
|
-
|
|
195
|
+
RESETTING: 'RESETTING',
|
|
196
196
|
/** If necessary, Update is creating tables or updating their schemas. */
|
|
197
|
-
|
|
197
|
+
SETTING_UP_TABLES: 'SETTING_UP_TABLES',
|
|
198
198
|
/** Update is currently executing queries. */
|
|
199
|
-
|
|
199
|
+
RUNNING: 'RUNNING',
|
|
200
200
|
/** Update is waiting for queries to shut down. */
|
|
201
|
-
|
|
201
|
+
STOPPING: 'STOPPING',
|
|
202
202
|
/** Update is complete and all necessary resources are cleaned up. */
|
|
203
|
-
|
|
203
|
+
COMPLETED: 'COMPLETED',
|
|
204
204
|
/** Update has run into an error that could not be recovered from. */
|
|
205
|
-
|
|
205
|
+
FAILED: 'FAILED',
|
|
206
206
|
/** Update was canceled while it was running or queued. */
|
|
207
|
-
|
|
208
|
-
}
|
|
209
|
-
// eslint-disable-next-line @typescript-eslint/naming-convention --
|
|
210
|
-
export
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
})(FileIngestionOptions_FileFormat || (FileIngestionOptions_FileFormat = {}));
|
|
207
|
+
CANCELED: 'CANCELED',
|
|
208
|
+
};
|
|
209
|
+
// eslint-disable-next-line @typescript-eslint/naming-convention -- Enum-style const object.
|
|
210
|
+
export const FileIngestionOptions_FileFormat = {
|
|
211
|
+
FILE_FORMAT_UNSPECIFIED: 'FILE_FORMAT_UNSPECIFIED',
|
|
212
|
+
BINARYFILE: 'BINARYFILE',
|
|
213
|
+
JSON: 'JSON',
|
|
214
|
+
CSV: 'CSV',
|
|
215
|
+
XML: 'XML',
|
|
216
|
+
EXCEL: 'EXCEL',
|
|
217
|
+
PARQUET: 'PARQUET',
|
|
218
|
+
AVRO: 'AVRO',
|
|
219
|
+
ORC: 'ORC',
|
|
220
|
+
};
|
|
222
221
|
/** Based on https://docs.databricks.com/aws/en/ingestion/cloud-object-storage/auto-loader/schema#how-does-auto-loader-schema-evolution-work */
|
|
223
|
-
// eslint-disable-next-line @typescript-eslint/naming-convention --
|
|
224
|
-
export
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
GoogleDriveOptions_GoogleDriveEntityType["PERMISSION"] = "PERMISSION";
|
|
240
|
-
})(GoogleDriveOptions_GoogleDriveEntityType || (GoogleDriveOptions_GoogleDriveEntityType = {}));
|
|
222
|
+
// eslint-disable-next-line @typescript-eslint/naming-convention -- Enum-style const object.
|
|
223
|
+
export const FileIngestionOptions_SchemaEvolutionMode = {
|
|
224
|
+
SCHEMA_EVOLUTION_MODE_UNSPECIFIED: 'SCHEMA_EVOLUTION_MODE_UNSPECIFIED',
|
|
225
|
+
ADD_NEW_COLUMNS_WITH_TYPE_WIDENING: 'ADD_NEW_COLUMNS_WITH_TYPE_WIDENING',
|
|
226
|
+
ADD_NEW_COLUMNS: 'ADD_NEW_COLUMNS',
|
|
227
|
+
RESCUE: 'RESCUE',
|
|
228
|
+
FAIL_ON_NEW_COLUMNS: 'FAIL_ON_NEW_COLUMNS',
|
|
229
|
+
NONE: 'NONE',
|
|
230
|
+
};
|
|
231
|
+
// eslint-disable-next-line @typescript-eslint/naming-convention -- Enum-style const object.
|
|
232
|
+
export const GoogleDriveOptions_GoogleDriveEntityType = {
|
|
233
|
+
GOOGLE_DRIVE_ENTITY_TYPE_UNSPECIFIED: 'GOOGLE_DRIVE_ENTITY_TYPE_UNSPECIFIED',
|
|
234
|
+
FILE: 'FILE',
|
|
235
|
+
FILE_METADATA: 'FILE_METADATA',
|
|
236
|
+
PERMISSION: 'PERMISSION',
|
|
237
|
+
};
|
|
241
238
|
/** The pipeline state. */
|
|
242
|
-
// eslint-disable-next-line @typescript-eslint/naming-convention --
|
|
243
|
-
export
|
|
244
|
-
(function (PipelineState_PipelineState) {
|
|
239
|
+
// eslint-disable-next-line @typescript-eslint/naming-convention -- Enum-style const object.
|
|
240
|
+
export const PipelineState_PipelineState = {
|
|
245
241
|
/** Pipeline is being deployed and waiting for clusters, jobs, or other resources */
|
|
246
|
-
|
|
242
|
+
DEPLOYING: 'DEPLOYING',
|
|
247
243
|
/** Pipeline is deployed but waiting for streams to start and make progress */
|
|
248
|
-
|
|
244
|
+
STARTING: 'STARTING',
|
|
249
245
|
/** Pipeline is currently executing */
|
|
250
|
-
|
|
246
|
+
RUNNING: 'RUNNING',
|
|
251
247
|
/** Pipeline is waiting for streams to shut down */
|
|
252
|
-
|
|
248
|
+
STOPPING: 'STOPPING',
|
|
253
249
|
/** All clusters, jobs, and other resources associated with the pipeline have been cleaned up */
|
|
254
|
-
|
|
250
|
+
DELETED: 'DELETED',
|
|
255
251
|
/** Pipeline has run into an error, but the daemon is attempting to fix it */
|
|
256
|
-
|
|
252
|
+
RECOVERING: 'RECOVERING',
|
|
257
253
|
/** Pipeline has run into an error that could not be recovered from */
|
|
258
|
-
|
|
254
|
+
FAILED: 'FAILED',
|
|
259
255
|
/** Pipeline is currently being reset */
|
|
260
|
-
|
|
256
|
+
RESETTING: 'RESETTING',
|
|
261
257
|
/** Pipeline is stopped and is not processing data. Can be resumed by calling `run` */
|
|
262
|
-
|
|
263
|
-
}
|
|
258
|
+
IDLE: 'IDLE',
|
|
259
|
+
};
|
|
264
260
|
/** The SCD type to use to ingest the table. */
|
|
265
|
-
// eslint-disable-next-line @typescript-eslint/naming-convention --
|
|
266
|
-
export
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
ScdType_ScdType["SCD_TYPE_2"] = "SCD_TYPE_2";
|
|
261
|
+
// eslint-disable-next-line @typescript-eslint/naming-convention -- Enum-style const object.
|
|
262
|
+
export const ScdType_ScdType = {
|
|
263
|
+
SCD_TYPE_UNSPECIFIED: 'SCD_TYPE_UNSPECIFIED',
|
|
264
|
+
SCD_TYPE_1: 'SCD_TYPE_1',
|
|
265
|
+
SCD_TYPE_2: 'SCD_TYPE_2',
|
|
271
266
|
/**
|
|
272
267
|
* Source data will be appended to destination table rather than merged in
|
|
273
268
|
* the absence of row key.
|
|
274
269
|
*/
|
|
275
|
-
|
|
276
|
-
}
|
|
277
|
-
// eslint-disable-next-line @typescript-eslint/naming-convention --
|
|
278
|
-
export
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
})(SharepointOptions_SharepointEntityType || (SharepointOptions_SharepointEntityType = {}));
|
|
270
|
+
APPEND_ONLY: 'APPEND_ONLY',
|
|
271
|
+
};
|
|
272
|
+
// eslint-disable-next-line @typescript-eslint/naming-convention -- Enum-style const object.
|
|
273
|
+
export const SharepointOptions_SharepointEntityType = {
|
|
274
|
+
SHAREPOINT_ENTITY_TYPE_UNSPECIFIED: 'SHAREPOINT_ENTITY_TYPE_UNSPECIFIED',
|
|
275
|
+
FILE: 'FILE',
|
|
276
|
+
FILE_METADATA: 'FILE_METADATA',
|
|
277
|
+
PERMISSION: 'PERMISSION',
|
|
278
|
+
LIST: 'LIST',
|
|
279
|
+
};
|
|
286
280
|
/** Data level for TikTok Ads report aggregation. */
|
|
287
|
-
// eslint-disable-next-line @typescript-eslint/naming-convention --
|
|
288
|
-
export
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
})(TikTokAdsOptions_TikTokDataLevel || (TikTokAdsOptions_TikTokDataLevel = {}));
|
|
281
|
+
// eslint-disable-next-line @typescript-eslint/naming-convention -- Enum-style const object.
|
|
282
|
+
export const TikTokAdsOptions_TikTokDataLevel = {
|
|
283
|
+
TIK_TOK_DATA_LEVEL_UNSPECIFIED: 'TIK_TOK_DATA_LEVEL_UNSPECIFIED',
|
|
284
|
+
AUCTION_ADVERTISER: 'AUCTION_ADVERTISER',
|
|
285
|
+
AUCTION_CAMPAIGN: 'AUCTION_CAMPAIGN',
|
|
286
|
+
AUCTION_ADGROUP: 'AUCTION_ADGROUP',
|
|
287
|
+
AUCTION_AD: 'AUCTION_AD',
|
|
288
|
+
};
|
|
296
289
|
/** Report type for TikTok Ads API. */
|
|
297
|
-
// eslint-disable-next-line @typescript-eslint/naming-convention --
|
|
298
|
-
export
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
Transformer_Format["JSON"] = "JSON";
|
|
314
|
-
})(Transformer_Format || (Transformer_Format = {}));
|
|
290
|
+
// eslint-disable-next-line @typescript-eslint/naming-convention -- Enum-style const object.
|
|
291
|
+
export const TikTokAdsOptions_TikTokReportType = {
|
|
292
|
+
TIK_TOK_REPORT_TYPE_UNSPECIFIED: 'TIK_TOK_REPORT_TYPE_UNSPECIFIED',
|
|
293
|
+
BASIC: 'BASIC',
|
|
294
|
+
AUDIENCE: 'AUDIENCE',
|
|
295
|
+
PLAYABLE_AD: 'PLAYABLE_AD',
|
|
296
|
+
DSA: 'DSA',
|
|
297
|
+
BUSINESS_CENTER: 'BUSINESS_CENTER',
|
|
298
|
+
GMV_MAX: 'GMV_MAX',
|
|
299
|
+
};
|
|
300
|
+
// eslint-disable-next-line @typescript-eslint/naming-convention -- Enum-style const object.
|
|
301
|
+
export const Transformer_Format = {
|
|
302
|
+
FORMAT_UNSPECIFIED: 'FORMAT_UNSPECIFIED',
|
|
303
|
+
STRING: 'STRING',
|
|
304
|
+
JSON: 'JSON',
|
|
305
|
+
};
|
|
315
306
|
export const unmarshalApplyEnvironmentResponseSchema = z.object({});
|
|
316
307
|
export const unmarshalAutoFullRefreshPolicySchema = z
|
|
317
308
|
.object({
|
|
@@ -514,12 +505,10 @@ export const unmarshalFileFilterSchema = z
|
|
|
514
505
|
}));
|
|
515
506
|
export const unmarshalFileIngestionOptionsSchema = z
|
|
516
507
|
.object({
|
|
517
|
-
format: z.
|
|
508
|
+
format: z.string().optional(),
|
|
518
509
|
file_filters: z.array(z.lazy(() => unmarshalFileFilterSchema)).optional(),
|
|
519
510
|
infer_column_types: z.boolean().optional(),
|
|
520
|
-
schema_evolution_mode: z
|
|
521
|
-
.enum(FileIngestionOptions_SchemaEvolutionMode)
|
|
522
|
-
.optional(),
|
|
511
|
+
schema_evolution_mode: z.string().optional(),
|
|
523
512
|
schema_hints: z.string().optional(),
|
|
524
513
|
ignore_corrupt_files: z.boolean().optional(),
|
|
525
514
|
corrupt_record_column: z.string().optional(),
|
|
@@ -554,11 +543,11 @@ export const unmarshalGetPipelineResponseSchema = z
|
|
|
554
543
|
.object({
|
|
555
544
|
pipeline_id: z.string().optional(),
|
|
556
545
|
spec: z.lazy(() => unmarshalPipelineSpecSchema).optional(),
|
|
557
|
-
state: z.
|
|
546
|
+
state: z.string().optional(),
|
|
558
547
|
cause: z.string().optional(),
|
|
559
548
|
cluster_id: z.string().optional(),
|
|
560
549
|
name: z.string().optional(),
|
|
561
|
-
health: z.
|
|
550
|
+
health: z.string().optional(),
|
|
562
551
|
creator_user_name: z.string().optional(),
|
|
563
552
|
latest_updates: z
|
|
564
553
|
.array(z.lazy(() => unmarshalUpdateStateInfoSchema))
|
|
@@ -569,7 +558,7 @@ export const unmarshalGetPipelineResponseSchema = z
|
|
|
569
558
|
.optional(),
|
|
570
559
|
run_as_user_name: z.string().optional(),
|
|
571
560
|
effective_budget_policy_id: z.string().optional(),
|
|
572
|
-
effective_publishing_mode: z.
|
|
561
|
+
effective_publishing_mode: z.string().optional(),
|
|
573
562
|
run_as: z.lazy(() => unmarshalPipelinesJobRunAsSchema).optional(),
|
|
574
563
|
parameters: z.record(z.string(), z.string()).optional(),
|
|
575
564
|
})
|
|
@@ -618,7 +607,7 @@ export const unmarshalGoogleAdsOptionsSchema = z
|
|
|
618
607
|
export const unmarshalGoogleDriveOptionsSchema = z
|
|
619
608
|
.object({
|
|
620
609
|
url: z.string().optional(),
|
|
621
|
-
entity_type: z.
|
|
610
|
+
entity_type: z.string().optional(),
|
|
622
611
|
file_ingestion_options: z
|
|
623
612
|
.lazy(() => unmarshalFileIngestionOptionsSchema)
|
|
624
613
|
.optional(),
|
|
@@ -655,7 +644,7 @@ export const unmarshalIngestionPipelineDefinitionSchema = z
|
|
|
655
644
|
objects: z
|
|
656
645
|
.array(z.lazy(() => unmarshalIngestionPipelineDefinition_IngestionConfigSchema))
|
|
657
646
|
.optional(),
|
|
658
|
-
source_type: z.
|
|
647
|
+
source_type: z.string().optional(),
|
|
659
648
|
table_configuration: z
|
|
660
649
|
.lazy(() => unmarshalIngestionPipelineDefinition_TableSpecificConfigSchema)
|
|
661
650
|
.optional(),
|
|
@@ -666,7 +655,7 @@ export const unmarshalIngestionPipelineDefinitionSchema = z
|
|
|
666
655
|
full_refresh_window: z
|
|
667
656
|
.lazy(() => unmarshalOperationTimeWindowSchema)
|
|
668
657
|
.optional(),
|
|
669
|
-
connector_type: z.
|
|
658
|
+
connector_type: z.string().optional(),
|
|
670
659
|
data_staging_options: z
|
|
671
660
|
.lazy(() => unmarshalDataStagingOptionsSchema)
|
|
672
661
|
.optional(),
|
|
@@ -788,7 +777,7 @@ export const unmarshalIngestionPipelineDefinition_TableSpecSchema = z
|
|
|
788
777
|
// eslint-disable-next-line @typescript-eslint/naming-convention -- Proto-style nested message name.
|
|
789
778
|
export const unmarshalIngestionPipelineDefinition_TableSpecificConfigSchema = z
|
|
790
779
|
.object({
|
|
791
|
-
scd_type: z.
|
|
780
|
+
scd_type: z.string().optional(),
|
|
792
781
|
primary_keys: z.array(z.string()).optional(),
|
|
793
782
|
sequence_by: z.array(z.string()).optional(),
|
|
794
783
|
include_columns: z.array(z.string()).optional(),
|
|
@@ -804,6 +793,9 @@ export const unmarshalIngestionPipelineDefinition_TableSpecificConfigSchema = z
|
|
|
804
793
|
auto_full_refresh_policy: z
|
|
805
794
|
.lazy(() => unmarshalAutoFullRefreshPolicySchema)
|
|
806
795
|
.optional(),
|
|
796
|
+
table_properties: z.record(z.string(), z.string()).optional(),
|
|
797
|
+
enable_auto_clustering: z.boolean().optional(),
|
|
798
|
+
clustering_columns: z.array(z.string()).optional(),
|
|
807
799
|
})
|
|
808
800
|
.transform(d => ({
|
|
809
801
|
scdType: d.scd_type,
|
|
@@ -816,6 +808,9 @@ export const unmarshalIngestionPipelineDefinition_TableSpecificConfigSchema = z
|
|
|
816
808
|
rowFilter: d.row_filter,
|
|
817
809
|
queryBasedConnectorConfig: d.query_based_connector_config,
|
|
818
810
|
autoFullRefreshPolicy: d.auto_full_refresh_policy,
|
|
811
|
+
tableProperties: d.table_properties,
|
|
812
|
+
enableAutoClustering: d.enable_auto_clustering,
|
|
813
|
+
clusteringColumns: d.clustering_columns,
|
|
819
814
|
}));
|
|
820
815
|
// eslint-disable-next-line @typescript-eslint/naming-convention -- Proto-style nested message name.
|
|
821
816
|
export const unmarshalIngestionPipelineDefinition_TableSpecificConfig_QueryBasedConnectorConfigSchema = z
|
|
@@ -868,9 +863,7 @@ export const unmarshalJsonTransformerOptionsSchema = z
|
|
|
868
863
|
as_variant: z.boolean().optional(),
|
|
869
864
|
schema: z.string().optional(),
|
|
870
865
|
schema_file_path: z.string().optional(),
|
|
871
|
-
schema_evolution_mode: z
|
|
872
|
-
.enum(FileIngestionOptions_SchemaEvolutionMode)
|
|
873
|
-
.optional(),
|
|
866
|
+
schema_evolution_mode: z.string().optional(),
|
|
874
867
|
schema_hints: z.string().optional(),
|
|
875
868
|
})
|
|
876
869
|
.transform(d => ({
|
|
@@ -976,7 +969,7 @@ export const unmarshalNotificationsSchema = z
|
|
|
976
969
|
export const unmarshalOperationTimeWindowSchema = z
|
|
977
970
|
.object({
|
|
978
971
|
start_hour: z.number().optional(),
|
|
979
|
-
days_of_week: z.array(z.
|
|
972
|
+
days_of_week: z.array(z.string()).optional(),
|
|
980
973
|
time_zone_id: z.string().optional(),
|
|
981
974
|
})
|
|
982
975
|
.transform(d => ({
|
|
@@ -1045,8 +1038,8 @@ export const unmarshalOutlookOptionsSchema = z
|
|
|
1045
1038
|
sender_filter: z.array(z.string()).optional(),
|
|
1046
1039
|
subject_filter: z.array(z.string()).optional(),
|
|
1047
1040
|
start_date: z.string().optional(),
|
|
1048
|
-
body_format: z.
|
|
1049
|
-
attachment_mode: z.
|
|
1041
|
+
body_format: z.string().optional(),
|
|
1042
|
+
attachment_mode: z.string().optional(),
|
|
1050
1043
|
include_mailboxes: z.array(z.string()).optional(),
|
|
1051
1044
|
include_folders: z.array(z.string()).optional(),
|
|
1052
1045
|
include_senders: z.array(z.string()).optional(),
|
|
@@ -1129,7 +1122,7 @@ export const unmarshalPipelineClusterSchema = z
|
|
|
1129
1122
|
}));
|
|
1130
1123
|
export const unmarshalPipelineDeploymentSchema = z
|
|
1131
1124
|
.object({
|
|
1132
|
-
kind: z.
|
|
1125
|
+
kind: z.string().optional(),
|
|
1133
1126
|
metadata_file_path: z.string().optional(),
|
|
1134
1127
|
deployment_id: z.string().optional(),
|
|
1135
1128
|
version_id: z.string().optional(),
|
|
@@ -1147,10 +1140,10 @@ export const unmarshalPipelineEventSchema = z
|
|
|
1147
1140
|
origin: z.lazy(() => unmarshalOriginSchema).optional(),
|
|
1148
1141
|
timestamp: z.string().optional(),
|
|
1149
1142
|
message: z.string().optional(),
|
|
1150
|
-
level: z.
|
|
1143
|
+
level: z.string().optional(),
|
|
1151
1144
|
error: z.lazy(() => unmarshalErrorDetailSchema).optional(),
|
|
1152
1145
|
event_type: z.string().optional(),
|
|
1153
|
-
maturity_level: z.
|
|
1146
|
+
maturity_level: z.string().optional(),
|
|
1154
1147
|
truncation: z.lazy(() => unmarshalTruncationSchema).optional(),
|
|
1155
1148
|
})
|
|
1156
1149
|
.transform(d => ({
|
|
@@ -1259,7 +1252,7 @@ export const unmarshalPipelineSpecSchema = z
|
|
|
1259
1252
|
export const unmarshalPipelineStateInfoSchema = z
|
|
1260
1253
|
.object({
|
|
1261
1254
|
pipeline_id: z.string().optional(),
|
|
1262
|
-
state: z.
|
|
1255
|
+
state: z.string().optional(),
|
|
1263
1256
|
cluster_id: z.string().optional(),
|
|
1264
1257
|
name: z.string().optional(),
|
|
1265
1258
|
latest_updates: z
|
|
@@ -1267,7 +1260,7 @@ export const unmarshalPipelineStateInfoSchema = z
|
|
|
1267
1260
|
.optional(),
|
|
1268
1261
|
creator_user_name: z.string().optional(),
|
|
1269
1262
|
run_as_user_name: z.string().optional(),
|
|
1270
|
-
health: z.
|
|
1263
|
+
health: z.string().optional(),
|
|
1271
1264
|
})
|
|
1272
1265
|
.transform(d => ({
|
|
1273
1266
|
pipelineId: d.pipeline_id,
|
|
@@ -1305,11 +1298,11 @@ export const unmarshalPipelinesAutoScaleSchema = z
|
|
|
1305
1298
|
export const unmarshalPipelinesAwsAttributesSchema = z
|
|
1306
1299
|
.object({
|
|
1307
1300
|
first_on_demand: z.number().optional(),
|
|
1308
|
-
availability: z.
|
|
1301
|
+
availability: z.string().optional(),
|
|
1309
1302
|
zone_id: z.string().optional(),
|
|
1310
1303
|
instance_profile_arn: z.string().optional(),
|
|
1311
1304
|
spot_bid_price_percent: z.number().optional(),
|
|
1312
|
-
ebs_volume_type: z.
|
|
1305
|
+
ebs_volume_type: z.string().optional(),
|
|
1313
1306
|
ebs_volume_count: z.number().optional(),
|
|
1314
1307
|
ebs_volume_size: z.number().optional(),
|
|
1315
1308
|
ebs_volume_iops: z.number().optional(),
|
|
@@ -1330,7 +1323,7 @@ export const unmarshalPipelinesAwsAttributesSchema = z
|
|
|
1330
1323
|
export const unmarshalPipelinesAzureAttributesSchema = z
|
|
1331
1324
|
.object({
|
|
1332
1325
|
first_on_demand: z.number().optional(),
|
|
1333
|
-
availability: z.
|
|
1326
|
+
availability: z.string().optional(),
|
|
1334
1327
|
spot_bid_max_price: z.number().optional(),
|
|
1335
1328
|
})
|
|
1336
1329
|
.transform(d => ({
|
|
@@ -1367,7 +1360,7 @@ export const unmarshalPipelinesGcpAttributesSchema = z
|
|
|
1367
1360
|
.object({
|
|
1368
1361
|
google_service_account: z.string().optional(),
|
|
1369
1362
|
boot_disk_size: z.number().optional(),
|
|
1370
|
-
availability: z.
|
|
1363
|
+
availability: z.string().optional(),
|
|
1371
1364
|
zone_id: z.string().optional(),
|
|
1372
1365
|
local_ssd_count: z.number().optional(),
|
|
1373
1366
|
})
|
|
@@ -1454,7 +1447,7 @@ export const unmarshalPostgresSlotConfigSchema = z
|
|
|
1454
1447
|
export const unmarshalRestartWindowSchema = z
|
|
1455
1448
|
.object({
|
|
1456
1449
|
start_hour: z.number().optional(),
|
|
1457
|
-
days_of_week: z.array(z.
|
|
1450
|
+
days_of_week: z.array(z.string()).optional(),
|
|
1458
1451
|
time_zone_id: z.string().optional(),
|
|
1459
1452
|
})
|
|
1460
1453
|
.transform(d => ({
|
|
@@ -1488,7 +1481,7 @@ export const unmarshalSerializedExceptionSchema = z
|
|
|
1488
1481
|
export const unmarshalSharepointOptionsSchema = z
|
|
1489
1482
|
.object({
|
|
1490
1483
|
url: z.string().optional(),
|
|
1491
|
-
entity_type: z.
|
|
1484
|
+
entity_type: z.string().optional(),
|
|
1492
1485
|
file_ingestion_options: z
|
|
1493
1486
|
.lazy(() => unmarshalFileIngestionOptionsSchema)
|
|
1494
1487
|
.optional(),
|
|
@@ -1557,8 +1550,8 @@ export const unmarshalTikTokAdsOptionsSchema = z
|
|
|
1557
1550
|
sync_start_date: z.string().optional(),
|
|
1558
1551
|
dimensions: z.array(z.string()).optional(),
|
|
1559
1552
|
metrics: z.array(z.string()).optional(),
|
|
1560
|
-
report_type: z.
|
|
1561
|
-
data_level: z.
|
|
1553
|
+
report_type: z.string().optional(),
|
|
1554
|
+
data_level: z.string().optional(),
|
|
1562
1555
|
query_lifetime: z.boolean().optional(),
|
|
1563
1556
|
})
|
|
1564
1557
|
.transform(d => ({
|
|
@@ -1572,7 +1565,7 @@ export const unmarshalTikTokAdsOptionsSchema = z
|
|
|
1572
1565
|
}));
|
|
1573
1566
|
export const unmarshalTransformerSchema = z
|
|
1574
1567
|
.object({
|
|
1575
|
-
format: z.
|
|
1568
|
+
format: z.string().optional(),
|
|
1576
1569
|
json_options: z
|
|
1577
1570
|
.lazy(() => unmarshalJsonTransformerOptionsSchema)
|
|
1578
1571
|
.optional(),
|
|
@@ -1605,8 +1598,8 @@ export const unmarshalUpdateInfoSchema = z
|
|
|
1605
1598
|
pipeline_id: z.string().optional(),
|
|
1606
1599
|
update_id: z.string().optional(),
|
|
1607
1600
|
config: z.lazy(() => unmarshalPipelineSpecSchema).optional(),
|
|
1608
|
-
cause: z.
|
|
1609
|
-
state: z.
|
|
1601
|
+
cause: z.string().optional(),
|
|
1602
|
+
state: z.string().optional(),
|
|
1610
1603
|
cluster_id: z.string().optional(),
|
|
1611
1604
|
creation_time: z
|
|
1612
1605
|
.union([z.number(), z.bigint()])
|
|
@@ -1635,7 +1628,7 @@ export const unmarshalUpdateInfoSchema = z
|
|
|
1635
1628
|
export const unmarshalUpdateStateInfoSchema = z
|
|
1636
1629
|
.object({
|
|
1637
1630
|
update_id: z.string().optional(),
|
|
1638
|
-
state: z.
|
|
1631
|
+
state: z.string().optional(),
|
|
1639
1632
|
creation_time: z.string().optional(),
|
|
1640
1633
|
})
|
|
1641
1634
|
.transform(d => ({
|
|
@@ -1703,7 +1696,7 @@ export const marshalClonePipelineRequestSchema = z
|
|
|
1703
1696
|
rootPath: z.string().optional(),
|
|
1704
1697
|
environment: z.lazy(() => marshalPipelinesEnvironmentSchema).optional(),
|
|
1705
1698
|
usagePolicyId: z.string().optional(),
|
|
1706
|
-
cloneMode: z.
|
|
1699
|
+
cloneMode: z.string().optional(),
|
|
1707
1700
|
})
|
|
1708
1701
|
.transform(d => ({
|
|
1709
1702
|
pipeline_id: d.pipelineId,
|
|
@@ -2045,12 +2038,10 @@ export const marshalFileFilterSchema = z
|
|
|
2045
2038
|
}));
|
|
2046
2039
|
export const marshalFileIngestionOptionsSchema = z
|
|
2047
2040
|
.object({
|
|
2048
|
-
format: z.
|
|
2041
|
+
format: z.string().optional(),
|
|
2049
2042
|
fileFilters: z.array(z.lazy(() => marshalFileFilterSchema)).optional(),
|
|
2050
2043
|
inferColumnTypes: z.boolean().optional(),
|
|
2051
|
-
schemaEvolutionMode: z
|
|
2052
|
-
.enum(FileIngestionOptions_SchemaEvolutionMode)
|
|
2053
|
-
.optional(),
|
|
2044
|
+
schemaEvolutionMode: z.string().optional(),
|
|
2054
2045
|
schemaHints: z.string().optional(),
|
|
2055
2046
|
ignoreCorruptFiles: z.boolean().optional(),
|
|
2056
2047
|
corruptRecordColumn: z.string().optional(),
|
|
@@ -2102,7 +2093,7 @@ export const marshalGoogleAdsOptionsSchema = z
|
|
|
2102
2093
|
export const marshalGoogleDriveOptionsSchema = z
|
|
2103
2094
|
.object({
|
|
2104
2095
|
url: z.string().optional(),
|
|
2105
|
-
entityType: z.
|
|
2096
|
+
entityType: z.string().optional(),
|
|
2106
2097
|
fileIngestionOptions: z
|
|
2107
2098
|
.lazy(() => marshalFileIngestionOptionsSchema)
|
|
2108
2099
|
.optional(),
|
|
@@ -2152,7 +2143,7 @@ export const marshalIngestionPipelineDefinitionSchema = z
|
|
|
2152
2143
|
objects: z
|
|
2153
2144
|
.array(z.lazy(() => marshalIngestionPipelineDefinition_IngestionConfigSchema))
|
|
2154
2145
|
.optional(),
|
|
2155
|
-
sourceType: z.
|
|
2146
|
+
sourceType: z.string().optional(),
|
|
2156
2147
|
tableConfiguration: z
|
|
2157
2148
|
.lazy(() => marshalIngestionPipelineDefinition_TableSpecificConfigSchema)
|
|
2158
2149
|
.optional(),
|
|
@@ -2163,7 +2154,7 @@ export const marshalIngestionPipelineDefinitionSchema = z
|
|
|
2163
2154
|
fullRefreshWindow: z
|
|
2164
2155
|
.lazy(() => marshalOperationTimeWindowSchema)
|
|
2165
2156
|
.optional(),
|
|
2166
|
-
connectorType: z.
|
|
2157
|
+
connectorType: z.string().optional(),
|
|
2167
2158
|
dataStagingOptions: z
|
|
2168
2159
|
.lazy(() => marshalDataStagingOptionsSchema)
|
|
2169
2160
|
.optional(),
|
|
@@ -2281,7 +2272,7 @@ export const marshalIngestionPipelineDefinition_TableSpecSchema = z
|
|
|
2281
2272
|
// eslint-disable-next-line @typescript-eslint/naming-convention -- Proto-style nested message name.
|
|
2282
2273
|
export const marshalIngestionPipelineDefinition_TableSpecificConfigSchema = z
|
|
2283
2274
|
.object({
|
|
2284
|
-
scdType: z.
|
|
2275
|
+
scdType: z.string().optional(),
|
|
2285
2276
|
primaryKeys: z.array(z.string()).optional(),
|
|
2286
2277
|
sequenceBy: z.array(z.string()).optional(),
|
|
2287
2278
|
includeColumns: z.array(z.string()).optional(),
|
|
@@ -2297,6 +2288,9 @@ export const marshalIngestionPipelineDefinition_TableSpecificConfigSchema = z
|
|
|
2297
2288
|
autoFullRefreshPolicy: z
|
|
2298
2289
|
.lazy(() => marshalAutoFullRefreshPolicySchema)
|
|
2299
2290
|
.optional(),
|
|
2291
|
+
tableProperties: z.record(z.string(), z.string()).optional(),
|
|
2292
|
+
enableAutoClustering: z.boolean().optional(),
|
|
2293
|
+
clusteringColumns: z.array(z.string()).optional(),
|
|
2300
2294
|
})
|
|
2301
2295
|
.transform(d => ({
|
|
2302
2296
|
scd_type: d.scdType,
|
|
@@ -2309,6 +2303,9 @@ export const marshalIngestionPipelineDefinition_TableSpecificConfigSchema = z
|
|
|
2309
2303
|
row_filter: d.rowFilter,
|
|
2310
2304
|
query_based_connector_config: d.queryBasedConnectorConfig,
|
|
2311
2305
|
auto_full_refresh_policy: d.autoFullRefreshPolicy,
|
|
2306
|
+
table_properties: d.tableProperties,
|
|
2307
|
+
enable_auto_clustering: d.enableAutoClustering,
|
|
2308
|
+
clustering_columns: d.clusteringColumns,
|
|
2312
2309
|
}));
|
|
2313
2310
|
// eslint-disable-next-line @typescript-eslint/naming-convention -- Proto-style nested message name.
|
|
2314
2311
|
export const marshalIngestionPipelineDefinition_TableSpecificConfig_QueryBasedConnectorConfigSchema = z
|
|
@@ -2358,9 +2355,7 @@ export const marshalJsonTransformerOptionsSchema = z
|
|
|
2358
2355
|
asVariant: z.boolean().optional(),
|
|
2359
2356
|
schema: z.string().optional(),
|
|
2360
2357
|
schemaFilePath: z.string().optional(),
|
|
2361
|
-
schemaEvolutionMode: z
|
|
2362
|
-
.enum(FileIngestionOptions_SchemaEvolutionMode)
|
|
2363
|
-
.optional(),
|
|
2358
|
+
schemaEvolutionMode: z.string().optional(),
|
|
2364
2359
|
schemaHints: z.string().optional(),
|
|
2365
2360
|
})
|
|
2366
2361
|
.transform(d => ({
|
|
@@ -2430,7 +2425,7 @@ export const marshalNotificationsSchema = z
|
|
|
2430
2425
|
export const marshalOperationTimeWindowSchema = z
|
|
2431
2426
|
.object({
|
|
2432
2427
|
startHour: z.number().optional(),
|
|
2433
|
-
daysOfWeek: z.array(z.
|
|
2428
|
+
daysOfWeek: z.array(z.string()).optional(),
|
|
2434
2429
|
timeZoneId: z.string().optional(),
|
|
2435
2430
|
})
|
|
2436
2431
|
.transform(d => ({
|
|
@@ -2444,8 +2439,8 @@ export const marshalOutlookOptionsSchema = z
|
|
|
2444
2439
|
senderFilter: z.array(z.string()).optional(),
|
|
2445
2440
|
subjectFilter: z.array(z.string()).optional(),
|
|
2446
2441
|
startDate: z.string().optional(),
|
|
2447
|
-
bodyFormat: z.
|
|
2448
|
-
attachmentMode: z.
|
|
2442
|
+
bodyFormat: z.string().optional(),
|
|
2443
|
+
attachmentMode: z.string().optional(),
|
|
2449
2444
|
includeMailboxes: z.array(z.string()).optional(),
|
|
2450
2445
|
includeFolders: z.array(z.string()).optional(),
|
|
2451
2446
|
includeSenders: z.array(z.string()).optional(),
|
|
@@ -2528,7 +2523,7 @@ export const marshalPipelineClusterSchema = z
|
|
|
2528
2523
|
}));
|
|
2529
2524
|
export const marshalPipelineDeploymentSchema = z
|
|
2530
2525
|
.object({
|
|
2531
|
-
kind: z.
|
|
2526
|
+
kind: z.string().optional(),
|
|
2532
2527
|
metadataFilePath: z.string().optional(),
|
|
2533
2528
|
deploymentId: z.string().optional(),
|
|
2534
2529
|
versionId: z.string().optional(),
|
|
@@ -2605,11 +2600,11 @@ export const marshalPipelinesAutoScaleSchema = z
|
|
|
2605
2600
|
export const marshalPipelinesAwsAttributesSchema = z
|
|
2606
2601
|
.object({
|
|
2607
2602
|
firstOnDemand: z.number().optional(),
|
|
2608
|
-
availability: z.
|
|
2603
|
+
availability: z.string().optional(),
|
|
2609
2604
|
zoneId: z.string().optional(),
|
|
2610
2605
|
instanceProfileArn: z.string().optional(),
|
|
2611
2606
|
spotBidPricePercent: z.number().optional(),
|
|
2612
|
-
ebsVolumeType: z.
|
|
2607
|
+
ebsVolumeType: z.string().optional(),
|
|
2613
2608
|
ebsVolumeCount: z.number().optional(),
|
|
2614
2609
|
ebsVolumeSize: z.number().optional(),
|
|
2615
2610
|
ebsVolumeIops: z.number().optional(),
|
|
@@ -2630,7 +2625,7 @@ export const marshalPipelinesAwsAttributesSchema = z
|
|
|
2630
2625
|
export const marshalPipelinesAzureAttributesSchema = z
|
|
2631
2626
|
.object({
|
|
2632
2627
|
firstOnDemand: z.number().optional(),
|
|
2633
|
-
availability: z.
|
|
2628
|
+
availability: z.string().optional(),
|
|
2634
2629
|
spotBidMaxPrice: z.number().optional(),
|
|
2635
2630
|
})
|
|
2636
2631
|
.transform(d => ({
|
|
@@ -2672,7 +2667,7 @@ export const marshalPipelinesGcpAttributesSchema = z
|
|
|
2672
2667
|
.object({
|
|
2673
2668
|
googleServiceAccount: z.string().optional(),
|
|
2674
2669
|
bootDiskSize: z.number().optional(),
|
|
2675
|
-
availability: z.
|
|
2670
|
+
availability: z.string().optional(),
|
|
2676
2671
|
zoneId: z.string().optional(),
|
|
2677
2672
|
localSsdCount: z.number().optional(),
|
|
2678
2673
|
})
|
|
@@ -2778,7 +2773,7 @@ export const marshalReplaceWhereOverrideSchema = z
|
|
|
2778
2773
|
export const marshalRestartWindowSchema = z
|
|
2779
2774
|
.object({
|
|
2780
2775
|
startHour: z.number().optional(),
|
|
2781
|
-
daysOfWeek: z.array(z.
|
|
2776
|
+
daysOfWeek: z.array(z.string()).optional(),
|
|
2782
2777
|
timeZoneId: z.string().optional(),
|
|
2783
2778
|
})
|
|
2784
2779
|
.transform(d => ({
|
|
@@ -2811,7 +2806,7 @@ export const marshalRewindSpecSchema = z
|
|
|
2811
2806
|
export const marshalSharepointOptionsSchema = z
|
|
2812
2807
|
.object({
|
|
2813
2808
|
url: z.string().optional(),
|
|
2814
|
-
entityType: z.
|
|
2809
|
+
entityType: z.string().optional(),
|
|
2815
2810
|
fileIngestionOptions: z
|
|
2816
2811
|
.lazy(() => marshalFileIngestionOptionsSchema)
|
|
2817
2812
|
.optional(),
|
|
@@ -2866,7 +2861,7 @@ export const marshalStartUpdateRequestSchema = z
|
|
|
2866
2861
|
.object({
|
|
2867
2862
|
pipelineId: z.string().optional(),
|
|
2868
2863
|
fullRefresh: z.boolean().optional(),
|
|
2869
|
-
cause: z.
|
|
2864
|
+
cause: z.string().optional(),
|
|
2870
2865
|
refreshSelection: z.array(z.string()).optional(),
|
|
2871
2866
|
fullRefreshSelection: z.array(z.string()).optional(),
|
|
2872
2867
|
resetCheckpointSelection: z.array(z.string()).optional(),
|
|
@@ -2902,8 +2897,8 @@ export const marshalTikTokAdsOptionsSchema = z
|
|
|
2902
2897
|
syncStartDate: z.string().optional(),
|
|
2903
2898
|
dimensions: z.array(z.string()).optional(),
|
|
2904
2899
|
metrics: z.array(z.string()).optional(),
|
|
2905
|
-
reportType: z.
|
|
2906
|
-
dataLevel: z.
|
|
2900
|
+
reportType: z.string().optional(),
|
|
2901
|
+
dataLevel: z.string().optional(),
|
|
2907
2902
|
queryLifetime: z.boolean().optional(),
|
|
2908
2903
|
})
|
|
2909
2904
|
.transform(d => ({
|
|
@@ -2917,7 +2912,7 @@ export const marshalTikTokAdsOptionsSchema = z
|
|
|
2917
2912
|
}));
|
|
2918
2913
|
export const marshalTransformerSchema = z
|
|
2919
2914
|
.object({
|
|
2920
|
-
format: z.
|
|
2915
|
+
format: z.string().optional(),
|
|
2921
2916
|
config: z
|
|
2922
2917
|
.discriminatedUnion('$case', [
|
|
2923
2918
|
z.object({
|