@whaly/connector-sdk 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +112 -200
- package/dist/index.d.ts +112 -200
- package/dist/index.js +242 -470
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +241 -466
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -38,6 +38,7 @@ __export(index_exports, {
|
|
|
38
38
|
CounterMetric: () => CounterMetric,
|
|
39
39
|
DEFAULT_MAX_CONCURRENT_STREAMS: () => DEFAULT_MAX_CONCURRENT_STREAMS,
|
|
40
40
|
EXECUTION_TIME_METRIC_NAME: () => EXECUTION_TIME_METRIC_NAME,
|
|
41
|
+
GCSStateProvider: () => GCSStateProvider,
|
|
41
42
|
ITarget: () => ITarget,
|
|
42
43
|
MissingFieldInSchemaError: () => MissingFieldInSchemaError,
|
|
43
44
|
MissingSchemaError: () => MissingSchemaError,
|
|
@@ -45,14 +46,13 @@ __export(index_exports, {
|
|
|
45
46
|
ROWS_SYNCED_METRIC_NAME: () => ROWS_SYNCED_METRIC_NAME,
|
|
46
47
|
RecordMessage: () => RecordMessage,
|
|
47
48
|
RenameColumnStore: () => RenameColumnStore,
|
|
49
|
+
ReplicationMethod: () => ReplicationMethod,
|
|
48
50
|
ReplicationMethodMessage: () => ReplicationMethodMessage,
|
|
49
|
-
Resolver: () => Resolver,
|
|
50
51
|
SchemaMessage: () => SchemaMessage,
|
|
51
52
|
SchemaValidationError: () => SchemaValidationError,
|
|
52
53
|
StateMessage: () => StateMessage,
|
|
53
54
|
StateService: () => StateService,
|
|
54
55
|
Stream: () => Stream,
|
|
55
|
-
StreamMetadata: () => StreamMetadata,
|
|
56
56
|
StreamWarehouseSyncService: () => StreamWarehouseSyncService,
|
|
57
57
|
Tap: () => Tap,
|
|
58
58
|
_greaterThan: () => _greaterThan,
|
|
@@ -60,7 +60,6 @@ __export(index_exports, {
|
|
|
60
60
|
createTemporaryFileStream: () => createTemporaryFileStream,
|
|
61
61
|
extractStateForStream: () => extractStateForStream,
|
|
62
62
|
finalizeStateProgressMarkers: () => finalizeStateProgressMarkers,
|
|
63
|
-
findRelatedRelationships: () => findRelatedRelationships,
|
|
64
63
|
flattenSchema: () => flattenSchema,
|
|
65
64
|
getAllMetrics: () => getAllMetrics,
|
|
66
65
|
getAxiosInstance: () => getAxiosInstance,
|
|
@@ -74,7 +73,6 @@ __export(index_exports, {
|
|
|
74
73
|
haltAndCatchFire: () => haltAndCatchFire,
|
|
75
74
|
incrementStreamState: () => incrementStreamState,
|
|
76
75
|
loadJson: () => loadJson,
|
|
77
|
-
loadResolver: () => loadResolver,
|
|
78
76
|
loadSchema: () => loadSchema,
|
|
79
77
|
logger: () => logger,
|
|
80
78
|
postFormDataApiCall: () => postFormDataApiCall,
|
|
@@ -82,189 +80,18 @@ __export(index_exports, {
|
|
|
82
80
|
postUrlEncodedApiCall: () => postUrlEncodedApiCall,
|
|
83
81
|
printMemoryFootprint: () => printMemoryFootprint,
|
|
84
82
|
removeParasiteProperties: () => removeParasiteProperties,
|
|
85
|
-
safePath: () => safePath
|
|
86
|
-
writeStateFile: () => writeStateFile
|
|
83
|
+
safePath: () => safePath
|
|
87
84
|
});
|
|
88
85
|
module.exports = __toCommonJS(index_exports);
|
|
89
86
|
|
|
90
87
|
// src/sdk/utils.ts
|
|
91
|
-
var
|
|
92
|
-
|
|
93
|
-
// src/sdk/service/logger.ts
|
|
94
|
-
var winston2 = __toESM(require("winston"));
|
|
95
|
-
|
|
96
|
-
// src/sdk/models/resolver.ts
|
|
97
|
-
var Resolver = class {
|
|
98
|
-
constructor() {
|
|
99
|
-
}
|
|
100
|
-
};
|
|
101
|
-
|
|
102
|
-
// src/resolvers/local-file/services/files.ts
|
|
103
|
-
var import_fs = __toESM(require("fs"));
|
|
104
|
-
var writeFile = (path, content) => {
|
|
105
|
-
return Promise.resolve(import_fs.default.writeFileSync(path, content));
|
|
106
|
-
};
|
|
107
|
-
var readAndParseJSONFile = (path) => {
|
|
108
|
-
if (!import_fs.default.existsSync(path)) {
|
|
109
|
-
throw new Error(`File \`${path}\` wasn't found at: ${process.cwd()}`);
|
|
110
|
-
}
|
|
111
|
-
const raw = import_fs.default.readFileSync(path);
|
|
112
|
-
const parsed = JSON.parse(raw.toString());
|
|
113
|
-
return parsed;
|
|
114
|
-
};
|
|
115
|
-
|
|
116
|
-
// src/resolvers/local-file/hook/schemaHook.ts
|
|
117
|
-
var import_fs_extra = __toESM(require("fs-extra"));
|
|
118
|
-
var LocalSchemaHook = class {
|
|
119
|
-
async writeSchema(input) {
|
|
120
|
-
await import_fs_extra.default.appendFile("./schema.tmp", JSON.stringify(input) + "\n");
|
|
121
|
-
}
|
|
122
|
-
};
|
|
123
|
-
|
|
124
|
-
// src/resolvers/local-file/resolver.ts
|
|
125
|
-
var import_winston = __toESM(require("winston"));
|
|
126
|
-
|
|
127
|
-
// src/sdk/service/metric.ts
|
|
128
|
-
var metricStore = {};
|
|
129
|
-
var ALL_STREAM_ID_KEYWORD = "all";
|
|
130
|
-
var ROWS_SYNCED_METRIC_NAME = "rows_synced_count";
|
|
131
|
-
var API_CALLS_METRIC_NAME = "api_calls_count";
|
|
132
|
-
var EXECUTION_TIME_METRIC_NAME = "execution_time_ms";
|
|
133
|
-
var getCounterMetrics = (name, streamIds) => {
|
|
134
|
-
return streamIds.map((streamId) => {
|
|
135
|
-
return getCounterMetric(name, streamId);
|
|
136
|
-
});
|
|
137
|
-
};
|
|
138
|
-
var getCounterMetric = (name, streamId = ALL_STREAM_ID_KEYWORD) => {
|
|
139
|
-
if (!metricStore[streamId]) {
|
|
140
|
-
metricStore[streamId] = {};
|
|
141
|
-
}
|
|
142
|
-
const streamMetricStore = metricStore[streamId];
|
|
143
|
-
if (!streamMetricStore[name]) {
|
|
144
|
-
streamMetricStore[name] = new CounterMetric(name, streamId);
|
|
145
|
-
}
|
|
146
|
-
return streamMetricStore[name];
|
|
147
|
-
};
|
|
148
|
-
var getAllMetrics = () => {
|
|
149
|
-
return Object.keys(metricStore).flatMap((streamId) => {
|
|
150
|
-
const streamMetricStore = metricStore[streamId];
|
|
151
|
-
if (!streamMetricStore) {
|
|
152
|
-
return [];
|
|
153
|
-
}
|
|
154
|
-
return Object.keys(streamMetricStore).map((metricName) => streamMetricStore[metricName]).filter((m) => m !== void 0);
|
|
155
|
-
});
|
|
156
|
-
};
|
|
157
|
-
var CounterMetric = class {
|
|
158
|
-
value = 0;
|
|
159
|
-
name;
|
|
160
|
-
streamId;
|
|
161
|
-
constructor(name, streamId) {
|
|
162
|
-
this.name = name;
|
|
163
|
-
this.streamId = streamId;
|
|
164
|
-
}
|
|
165
|
-
increment(inc = 1) {
|
|
166
|
-
this.value = this.value + inc;
|
|
167
|
-
}
|
|
168
|
-
getStreamId() {
|
|
169
|
-
return this.streamId;
|
|
170
|
-
}
|
|
171
|
-
getName() {
|
|
172
|
-
return this.name;
|
|
173
|
-
}
|
|
174
|
-
getValue() {
|
|
175
|
-
return this.value;
|
|
176
|
-
}
|
|
177
|
-
};
|
|
178
|
-
|
|
179
|
-
// src/resolvers/local-file/resolver.ts
|
|
180
|
-
var { combine, prettyPrint, timestamp, errors, splat } = import_winston.default.format;
|
|
181
|
-
var statePath = "./state.json";
|
|
182
|
-
var metricsPath = "./metrics.json";
|
|
183
|
-
var LocalFilesResolver = class extends Resolver {
|
|
184
|
-
checkIfCanSync() {
|
|
185
|
-
return Promise.resolve(true);
|
|
186
|
-
}
|
|
187
|
-
markSyncStarted() {
|
|
188
|
-
return Promise.resolve();
|
|
189
|
-
}
|
|
190
|
-
onError(errorType, errorText, errorDebugText) {
|
|
191
|
-
const path = `./error.json`;
|
|
192
|
-
return writeFile(path, JSON.stringify({
|
|
193
|
-
errorType,
|
|
194
|
-
errorText,
|
|
195
|
-
errorDebugText
|
|
196
|
-
}));
|
|
197
|
-
}
|
|
198
|
-
getSchemaHooks() {
|
|
199
|
-
const wlyHook = new LocalSchemaHook();
|
|
200
|
-
return [wlyHook];
|
|
201
|
-
}
|
|
202
|
-
writeSchema() {
|
|
203
|
-
return Promise.resolve();
|
|
204
|
-
}
|
|
205
|
-
constructor() {
|
|
206
|
-
super();
|
|
207
|
-
}
|
|
208
|
-
getState() {
|
|
209
|
-
logger.info(`\u{1F4C1} Using the LOCAL resolver`);
|
|
210
|
-
const state = readAndParseJSONFile(statePath);
|
|
211
|
-
return Promise.resolve({
|
|
212
|
-
state
|
|
213
|
-
});
|
|
214
|
-
}
|
|
215
|
-
writeCatalog(catalog) {
|
|
216
|
-
return Promise.resolve();
|
|
217
|
-
}
|
|
218
|
-
writeState(state) {
|
|
219
|
-
return writeFile(statePath, state);
|
|
220
|
-
}
|
|
221
|
-
markSyncFailed() {
|
|
222
|
-
logger.info(`\u{1F622} Sync has failed.`);
|
|
223
|
-
return Promise.resolve();
|
|
224
|
-
}
|
|
225
|
-
markSyncComplete() {
|
|
226
|
-
logger.info(`\u{1F389} Sync is complete.`);
|
|
227
|
-
return Promise.resolve();
|
|
228
|
-
}
|
|
229
|
-
markDiscoveryComplete() {
|
|
230
|
-
logger.info(`\u{1F389} Discovery is complete.`);
|
|
231
|
-
return Promise.resolve();
|
|
232
|
-
}
|
|
233
|
-
async flushMetrics() {
|
|
234
|
-
const allMetrics = getAllMetrics();
|
|
235
|
-
const metricsOuput = allMetrics.map((metric) => {
|
|
236
|
-
return {
|
|
237
|
-
streamId: metric.getStreamId(),
|
|
238
|
-
name: metric.getName(),
|
|
239
|
-
value: metric.getValue()
|
|
240
|
-
};
|
|
241
|
-
}).map((obj) => JSON.stringify(obj)).join(`
|
|
242
|
-
`);
|
|
243
|
-
return writeFile(metricsPath, metricsOuput);
|
|
244
|
-
}
|
|
245
|
-
updateSourceValue(optionKey, optionValue) {
|
|
246
|
-
logger.info(`Updating source value ${optionKey} with value ${optionValue}`, { private: true });
|
|
247
|
-
return Promise.resolve();
|
|
248
|
-
}
|
|
249
|
-
getLogFormat() {
|
|
250
|
-
return combine(
|
|
251
|
-
timestamp(),
|
|
252
|
-
errors({ stack: true }),
|
|
253
|
-
splat(),
|
|
254
|
-
prettyPrint()
|
|
255
|
-
);
|
|
256
|
-
}
|
|
257
|
-
};
|
|
258
|
-
|
|
259
|
-
// src/sdk/service/resolver.ts
|
|
260
|
-
var loadResolver = () => {
|
|
261
|
-
return new LocalFilesResolver();
|
|
262
|
-
};
|
|
88
|
+
var import_fs = require("fs");
|
|
263
89
|
|
|
264
90
|
// src/sdk/service/logger.ts
|
|
91
|
+
var winston = __toESM(require("winston"));
|
|
265
92
|
var BATCH_INTERVAL_MS = 100;
|
|
266
93
|
var transports2 = {
|
|
267
|
-
console: new
|
|
94
|
+
console: new winston.transports.Console({
|
|
268
95
|
stderrLevels: [],
|
|
269
96
|
level: "info"
|
|
270
97
|
})
|
|
@@ -274,8 +101,14 @@ var getTransports = () => {
|
|
|
274
101
|
transports2.console
|
|
275
102
|
];
|
|
276
103
|
};
|
|
277
|
-
var
|
|
278
|
-
|
|
104
|
+
var { combine, prettyPrint, timestamp, errors, splat } = winston.format;
|
|
105
|
+
var winstonLogger = winston.createLogger({
|
|
106
|
+
format: combine(
|
|
107
|
+
timestamp(),
|
|
108
|
+
errors({ stack: true }),
|
|
109
|
+
splat(),
|
|
110
|
+
prettyPrint()
|
|
111
|
+
),
|
|
279
112
|
transports: getTransports()
|
|
280
113
|
});
|
|
281
114
|
var isWinstonOpen = true;
|
|
@@ -304,13 +137,9 @@ var logger = {
|
|
|
304
137
|
};
|
|
305
138
|
|
|
306
139
|
// src/sdk/utils.ts
|
|
307
|
-
var writeStateFile = (resolver, state) => {
|
|
308
|
-
logger.info(`\u{1F4DD} Writing the state file.`);
|
|
309
|
-
return resolver.writeState(state);
|
|
310
|
-
};
|
|
311
140
|
function readFile(fileName, filePath) {
|
|
312
141
|
try {
|
|
313
|
-
return (0,
|
|
142
|
+
return (0, import_fs.readFileSync)(filePath);
|
|
314
143
|
} catch (err) {
|
|
315
144
|
logger.error(`Can't open file: \`${fileName}\` at path: \`${filePath}\`. Is it at the proper place?`);
|
|
316
145
|
throw err;
|
|
@@ -337,7 +166,7 @@ var loadSchema = (streamId) => {
|
|
|
337
166
|
// src/sdk/service/network.ts
|
|
338
167
|
var import_axios = __toESM(require("axios"));
|
|
339
168
|
var import_url = require("url");
|
|
340
|
-
var
|
|
169
|
+
var fs = __toESM(require("fs"));
|
|
341
170
|
var qs = __toESM(require("qs"));
|
|
342
171
|
var import_axios_retry = __toESM(require("axios-retry"));
|
|
343
172
|
var shouldRetry = (error) => {
|
|
@@ -450,7 +279,7 @@ var getJSONApiCall = async (endpoint, config, privateLogging) => {
|
|
|
450
279
|
};
|
|
451
280
|
var getDownloadFileApiCall = async (endpoint, config, output, privateLogging) => {
|
|
452
281
|
const instance = getAxiosInstance(config.retryCount);
|
|
453
|
-
const writer =
|
|
282
|
+
const writer = fs.createWriteStream(output);
|
|
454
283
|
const paramsSerializer = (params) => {
|
|
455
284
|
return qs.stringify(params, { arrayFormat: "repeat" });
|
|
456
285
|
};
|
|
@@ -544,6 +373,58 @@ var postUrlEncodedApiCall = async (endpoint, config, payload) => {
|
|
|
544
373
|
}
|
|
545
374
|
};
|
|
546
375
|
|
|
376
|
+
// src/sdk/service/metric.ts
|
|
377
|
+
var metricStore = {};
|
|
378
|
+
var ALL_STREAM_ID_KEYWORD = "all";
|
|
379
|
+
var ROWS_SYNCED_METRIC_NAME = "rows_synced_count";
|
|
380
|
+
var API_CALLS_METRIC_NAME = "api_calls_count";
|
|
381
|
+
var EXECUTION_TIME_METRIC_NAME = "execution_time_ms";
|
|
382
|
+
var getCounterMetrics = (name, streamIds) => {
|
|
383
|
+
return streamIds.map((streamId) => {
|
|
384
|
+
return getCounterMetric(name, streamId);
|
|
385
|
+
});
|
|
386
|
+
};
|
|
387
|
+
var getCounterMetric = (name, streamId = ALL_STREAM_ID_KEYWORD) => {
|
|
388
|
+
if (!metricStore[streamId]) {
|
|
389
|
+
metricStore[streamId] = {};
|
|
390
|
+
}
|
|
391
|
+
const streamMetricStore = metricStore[streamId];
|
|
392
|
+
if (!streamMetricStore[name]) {
|
|
393
|
+
streamMetricStore[name] = new CounterMetric(name, streamId);
|
|
394
|
+
}
|
|
395
|
+
return streamMetricStore[name];
|
|
396
|
+
};
|
|
397
|
+
var getAllMetrics = () => {
|
|
398
|
+
return Object.keys(metricStore).flatMap((streamId) => {
|
|
399
|
+
const streamMetricStore = metricStore[streamId];
|
|
400
|
+
if (!streamMetricStore) {
|
|
401
|
+
return [];
|
|
402
|
+
}
|
|
403
|
+
return Object.keys(streamMetricStore).map((metricName) => streamMetricStore[metricName]).filter((m) => m !== void 0);
|
|
404
|
+
});
|
|
405
|
+
};
|
|
406
|
+
var CounterMetric = class {
|
|
407
|
+
value = 0;
|
|
408
|
+
name;
|
|
409
|
+
streamId;
|
|
410
|
+
constructor(name, streamId) {
|
|
411
|
+
this.name = name;
|
|
412
|
+
this.streamId = streamId;
|
|
413
|
+
}
|
|
414
|
+
increment(inc = 1) {
|
|
415
|
+
this.value = this.value + inc;
|
|
416
|
+
}
|
|
417
|
+
getStreamId() {
|
|
418
|
+
return this.streamId;
|
|
419
|
+
}
|
|
420
|
+
getName() {
|
|
421
|
+
return this.name;
|
|
422
|
+
}
|
|
423
|
+
getValue() {
|
|
424
|
+
return this.value;
|
|
425
|
+
}
|
|
426
|
+
};
|
|
427
|
+
|
|
547
428
|
// src/sdk/service/memory.ts
|
|
548
429
|
var formatMemoryUsage = (data) => `${Math.round(data / 1024 / 1024 * 100) / 100} MB`;
|
|
549
430
|
var printMemoryFootprint = (prefix) => {
|
|
@@ -559,9 +440,6 @@ var printMemoryFootprint = (prefix) => {
|
|
|
559
440
|
|
|
560
441
|
// src/sdk/service/exit.ts
|
|
561
442
|
async function gracefulExit(exitCode) {
|
|
562
|
-
if (exitCode !== 0) {
|
|
563
|
-
await loadResolver().markSyncFailed();
|
|
564
|
-
}
|
|
565
443
|
const loggerFinish = new Promise((resolve, reject) => {
|
|
566
444
|
logger.closeWinston(resolve);
|
|
567
445
|
});
|
|
@@ -581,34 +459,15 @@ var haltAndCatchFire = async (errorType, errorText, errorDebugText) => {
|
|
|
581
459
|
errorText,
|
|
582
460
|
errorDebugText
|
|
583
461
|
);
|
|
584
|
-
const resolver = loadResolver();
|
|
585
|
-
await resolver.onError(errorType, errorText, errorDebugText);
|
|
586
462
|
await gracefulExit(0);
|
|
587
463
|
};
|
|
588
464
|
|
|
589
|
-
// src/sdk/
|
|
590
|
-
var
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
to: rel.to
|
|
596
|
-
};
|
|
597
|
-
if (rel.left === streamId) {
|
|
598
|
-
acc.right.push({
|
|
599
|
-
streamId: rel.right,
|
|
600
|
-
...base
|
|
601
|
-
});
|
|
602
|
-
}
|
|
603
|
-
if (rel.right === streamId) {
|
|
604
|
-
acc.left.push({
|
|
605
|
-
streamId: rel.left,
|
|
606
|
-
...base
|
|
607
|
-
});
|
|
608
|
-
}
|
|
609
|
-
return acc;
|
|
610
|
-
}, { left: [], right: [] });
|
|
611
|
-
};
|
|
465
|
+
// src/sdk/models/replication.ts
|
|
466
|
+
var ReplicationMethod = /* @__PURE__ */ ((ReplicationMethod2) => {
|
|
467
|
+
ReplicationMethod2["INCREMENTAL"] = "INCREMENTAL";
|
|
468
|
+
ReplicationMethod2["FULL_TABLE"] = "FULL_TABLE";
|
|
469
|
+
return ReplicationMethod2;
|
|
470
|
+
})(ReplicationMethod || {});
|
|
612
471
|
|
|
613
472
|
// src/sdk/models/messages.ts
|
|
614
473
|
var RecordMessage = class {
|
|
@@ -668,7 +527,6 @@ var SchemaMessage = class {
|
|
|
668
527
|
displayLabel;
|
|
669
528
|
description;
|
|
670
529
|
propertiesMetadata;
|
|
671
|
-
relationships;
|
|
672
530
|
constructor(opts) {
|
|
673
531
|
const {
|
|
674
532
|
stream,
|
|
@@ -677,7 +535,6 @@ var SchemaMessage = class {
|
|
|
677
535
|
bookmarkProperties,
|
|
678
536
|
displayLabel,
|
|
679
537
|
description,
|
|
680
|
-
relationships,
|
|
681
538
|
propertiesMetadata
|
|
682
539
|
} = opts;
|
|
683
540
|
this.stream = stream;
|
|
@@ -686,7 +543,6 @@ var SchemaMessage = class {
|
|
|
686
543
|
this.bookmarkProperties = bookmarkProperties;
|
|
687
544
|
this.displayLabel = displayLabel;
|
|
688
545
|
this.description = description;
|
|
689
|
-
this.relationships = relationships;
|
|
690
546
|
this.propertiesMetadata = propertiesMetadata;
|
|
691
547
|
}
|
|
692
548
|
toString() {
|
|
@@ -696,8 +552,7 @@ var SchemaMessage = class {
|
|
|
696
552
|
schema: this.schema,
|
|
697
553
|
key_properties: this.keyProperties,
|
|
698
554
|
label: this.displayLabel,
|
|
699
|
-
description: this.description
|
|
700
|
-
relationships: this.relationships
|
|
555
|
+
description: this.description
|
|
701
556
|
};
|
|
702
557
|
if (this.bookmarkProperties) {
|
|
703
558
|
result["bookmark_properties"] = this.bookmarkProperties;
|
|
@@ -706,128 +561,8 @@ var SchemaMessage = class {
|
|
|
706
561
|
}
|
|
707
562
|
};
|
|
708
563
|
|
|
709
|
-
// src/sdk/models/metadata.ts
|
|
710
|
-
var MetadataMap = class extends Map {
|
|
711
|
-
areEquals(array1, array2) {
|
|
712
|
-
return array1.length === array2.length && array1.every(function(value, index) {
|
|
713
|
-
return value === array2[index];
|
|
714
|
-
});
|
|
715
|
-
}
|
|
716
|
-
findExistingEntry(key) {
|
|
717
|
-
return Array.from(super.entries()).find((entry) => {
|
|
718
|
-
if (this.areEquals(entry[0], key)) {
|
|
719
|
-
return true;
|
|
720
|
-
}
|
|
721
|
-
});
|
|
722
|
-
}
|
|
723
|
-
get(key) {
|
|
724
|
-
const existingEntry = this.findExistingEntry(key);
|
|
725
|
-
return existingEntry?.[1];
|
|
726
|
-
}
|
|
727
|
-
set(key, value) {
|
|
728
|
-
const existingKey = this.findExistingEntry(key);
|
|
729
|
-
if (existingKey) {
|
|
730
|
-
return super.set(existingKey?.[0], value);
|
|
731
|
-
} else {
|
|
732
|
-
return super.set(key, value);
|
|
733
|
-
}
|
|
734
|
-
}
|
|
735
|
-
};
|
|
736
|
-
var StreamMetadata = class {
|
|
737
|
-
metadataByBreadcrumb;
|
|
738
|
-
constructor() {
|
|
739
|
-
this.metadataByBreadcrumb = new MetadataMap();
|
|
740
|
-
}
|
|
741
|
-
fromCatalogStreamMetadata(metadataArr) {
|
|
742
|
-
metadataArr.forEach((metadata) => {
|
|
743
|
-
this.metadataByBreadcrumb.set(metadata.breadcrumb, metadata.metadata);
|
|
744
|
-
});
|
|
745
|
-
return this;
|
|
746
|
-
}
|
|
747
|
-
toString() {
|
|
748
|
-
return this.metadataByBreadcrumb;
|
|
749
|
-
}
|
|
750
|
-
get(breadcrumb) {
|
|
751
|
-
return this.metadataByBreadcrumb.get(breadcrumb);
|
|
752
|
-
}
|
|
753
|
-
write(breadcrumb, key, value) {
|
|
754
|
-
if (value === void 0) {
|
|
755
|
-
throw new Error(`The value you're trying to write on the metadata at breadcrumb: \`${breadcrumb}\` and key: \`${key}\` is undefined.
|
|
756
|
-
|
|
757
|
-
Writing an undefined value is not a valid operation. Is there something wrong with your value?`);
|
|
758
|
-
}
|
|
759
|
-
if (!this.metadataByBreadcrumb.get(breadcrumb)) {
|
|
760
|
-
this.metadataByBreadcrumb.set(breadcrumb, {});
|
|
761
|
-
}
|
|
762
|
-
const previousValue = this.metadataByBreadcrumb.get(breadcrumb);
|
|
763
|
-
const newValue = {};
|
|
764
|
-
newValue[key] = value;
|
|
765
|
-
this.metadataByBreadcrumb.set(breadcrumb, { ...previousValue, ...newValue });
|
|
766
|
-
return this;
|
|
767
|
-
}
|
|
768
|
-
getRootBreadcrumbMetadata() {
|
|
769
|
-
const rootBreadcrumbMetadata = this.metadataByBreadcrumb.get([]);
|
|
770
|
-
if (!rootBreadcrumbMetadata) {
|
|
771
|
-
throw new Error(`No metadata was attached to the root breadcrumb \`[\`] while we need one to store metadata at the table level.
|
|
772
|
-
|
|
773
|
-
Is your catalog properly generated?`);
|
|
774
|
-
}
|
|
775
|
-
return rootBreadcrumbMetadata;
|
|
776
|
-
}
|
|
777
|
-
isStreamSelected() {
|
|
778
|
-
const rootBreadcrumbMetadata = this.getRootBreadcrumbMetadata();
|
|
779
|
-
if (rootBreadcrumbMetadata.selected === false) {
|
|
780
|
-
return false;
|
|
781
|
-
}
|
|
782
|
-
if (rootBreadcrumbMetadata.selected === true || rootBreadcrumbMetadata["selected-by-default"] === true) {
|
|
783
|
-
return true;
|
|
784
|
-
}
|
|
785
|
-
return false;
|
|
786
|
-
}
|
|
787
|
-
getKeyProperties() {
|
|
788
|
-
const rootBreadcrumbMetadata = this.getRootBreadcrumbMetadata();
|
|
789
|
-
if (!rootBreadcrumbMetadata["table-key-properties"]) {
|
|
790
|
-
return [];
|
|
791
|
-
}
|
|
792
|
-
return rootBreadcrumbMetadata["table-key-properties"];
|
|
793
|
-
}
|
|
794
|
-
getReplicationKey() {
|
|
795
|
-
const rootBreadcrumbMetadata = this.getRootBreadcrumbMetadata();
|
|
796
|
-
if (!rootBreadcrumbMetadata["replication-key"]) {
|
|
797
|
-
return void 0;
|
|
798
|
-
}
|
|
799
|
-
return rootBreadcrumbMetadata["replication-key"];
|
|
800
|
-
}
|
|
801
|
-
getReplicationMethod() {
|
|
802
|
-
const rootBreadcrumbMetadata = this.getRootBreadcrumbMetadata();
|
|
803
|
-
if (!rootBreadcrumbMetadata["replication-method"]) {
|
|
804
|
-
return void 0;
|
|
805
|
-
}
|
|
806
|
-
return rootBreadcrumbMetadata["replication-method"];
|
|
807
|
-
}
|
|
808
|
-
getForcedReplicationMethod() {
|
|
809
|
-
const rootBreadcrumbMetadata = this.getRootBreadcrumbMetadata();
|
|
810
|
-
if (!rootBreadcrumbMetadata["forced-replication-method"]) {
|
|
811
|
-
return void 0;
|
|
812
|
-
}
|
|
813
|
-
return rootBreadcrumbMetadata["forced-replication-method"];
|
|
814
|
-
}
|
|
815
|
-
toList() {
|
|
816
|
-
return Array.from(this.metadataByBreadcrumb.keys()).map((breadcrumb) => {
|
|
817
|
-
const metadata = this.metadataByBreadcrumb.get(breadcrumb);
|
|
818
|
-
if (!metadata) {
|
|
819
|
-
throw new Error(`There was no metadata entry for breadcrumb: \`${breadcrumb}\` in metadata map: ${this.metadataByBreadcrumb.toString()}`);
|
|
820
|
-
}
|
|
821
|
-
return {
|
|
822
|
-
breadcrumb,
|
|
823
|
-
metadata
|
|
824
|
-
};
|
|
825
|
-
});
|
|
826
|
-
}
|
|
827
|
-
};
|
|
828
|
-
|
|
829
564
|
// src/sdk/models/state.ts
|
|
830
|
-
var
|
|
565
|
+
var import_dayjs = __toESM(require("dayjs"));
|
|
831
566
|
|
|
832
567
|
// src/sdk/constants/date.ts
|
|
833
568
|
var defaultDateTimeFormat = "YYYY-MM-DDTHH:mm:ss.SSSSSSZ";
|
|
@@ -842,12 +577,12 @@ var incrementStreamState = (state, replicationKey, latestRecord, isSorted) => {
|
|
|
842
577
|
if (!newRkValue) {
|
|
843
578
|
throw new Error((0, import_util.format)(`No value for replicationKey: ${replicationKey} in record with keys: %j`, Object.keys(latestRecord)));
|
|
844
579
|
}
|
|
845
|
-
let newRkValueMoment = (0,
|
|
580
|
+
let newRkValueMoment = (0, import_dayjs.default)(newRkValue);
|
|
846
581
|
let prevRkValue;
|
|
847
582
|
let prevRkValueMoment;
|
|
848
583
|
if (isSorted) {
|
|
849
584
|
prevRkValue = state["replicationKeyValue"];
|
|
850
|
-
prevRkValueMoment = (0,
|
|
585
|
+
prevRkValueMoment = (0, import_dayjs.default)(prevRkValue);
|
|
851
586
|
if (prevRkValue && prevRkValueMoment.isAfter(newRkValueMoment)) {
|
|
852
587
|
throw new Error(
|
|
853
588
|
`Unsorted data detected in stream. Latest value '${newRkValue}' is
|
|
@@ -862,24 +597,24 @@ var incrementStreamState = (state, replicationKey, latestRecord, isSorted) => {
|
|
|
862
597
|
state[PROGRESS_MARKERS] = marker;
|
|
863
598
|
}
|
|
864
599
|
prevRkValue = state[PROGRESS_MARKERS]["replicationKeyValue"];
|
|
865
|
-
prevRkValueMoment = (0,
|
|
600
|
+
prevRkValueMoment = (0, import_dayjs.default)(prevRkValue || state["replicationKeyValue"]);
|
|
866
601
|
if (prevRkValue && prevRkValueMoment.isAfter(newRkValueMoment)) {
|
|
867
602
|
newRkValueMoment = prevRkValueMoment;
|
|
868
603
|
newRkValue = prevRkValue;
|
|
869
604
|
}
|
|
870
605
|
}
|
|
871
606
|
const signpostMarker = state[SIGNPOST_MARKER];
|
|
872
|
-
const replicationKeySignpostMoment = (0,
|
|
607
|
+
const replicationKeySignpostMoment = (0, import_dayjs.default)(signpostMarker);
|
|
873
608
|
if (replicationKeySignpostMoment && replicationKeySignpostMoment.isBefore(newRkValueMoment)) {
|
|
874
609
|
newRkValueMoment = replicationKeySignpostMoment;
|
|
875
610
|
newRkValue = replicationKeySignpostMoment.format(defaultDateTimeFormat);
|
|
876
611
|
}
|
|
877
612
|
if (isSorted === false) {
|
|
878
613
|
state[PROGRESS_MARKERS]["replicationKey"] = replicationKey;
|
|
879
|
-
state[PROGRESS_MARKERS]["replicationKeyValue"] = (0,
|
|
614
|
+
state[PROGRESS_MARKERS]["replicationKeyValue"] = (0, import_dayjs.default)(newRkValueMoment).format(defaultDateTimeFormat);
|
|
880
615
|
} else {
|
|
881
616
|
state["replicationKey"] = replicationKey;
|
|
882
|
-
state["replicationKeyValue"] = (0,
|
|
617
|
+
state["replicationKeyValue"] = (0, import_dayjs.default)(newRkValueMoment).format(defaultDateTimeFormat);
|
|
883
618
|
}
|
|
884
619
|
return state;
|
|
885
620
|
};
|
|
@@ -901,7 +636,7 @@ var finalizeStateProgressMarkers = (state) => {
|
|
|
901
636
|
return state;
|
|
902
637
|
};
|
|
903
638
|
var _greaterThan = (aValue, bValue) => {
|
|
904
|
-
return (0,
|
|
639
|
+
return (0, import_dayjs.default)(aValue).isAfter((0, import_dayjs.default)(bValue));
|
|
905
640
|
};
|
|
906
641
|
var extractStateForStream = (state, streamId) => {
|
|
907
642
|
if (!state.bookmarks) {
|
|
@@ -924,26 +659,16 @@ var StateService = class {
|
|
|
924
659
|
}
|
|
925
660
|
return this.instance;
|
|
926
661
|
}
|
|
927
|
-
|
|
928
|
-
setBookmark(streamId, ts) {
|
|
929
|
-
this.bookmarks[streamId] = ts.format(defaultDateTimeFormat);
|
|
930
|
-
}
|
|
931
|
-
setBookmarkV2(streamId, streamState) {
|
|
662
|
+
setBookmark(streamId, streamState) {
|
|
932
663
|
this.bookmarks[streamId] = streamState;
|
|
933
664
|
}
|
|
934
|
-
|
|
665
|
+
setBookmarkSignpost(streamId, signpostValue) {
|
|
935
666
|
if (!this.bookmarks[streamId]) {
|
|
936
667
|
this.bookmarks[streamId] = {};
|
|
937
668
|
}
|
|
938
669
|
this.bookmarks[streamId][SIGNPOST_MARKER] = signpostValue;
|
|
939
670
|
}
|
|
940
671
|
getBookmark(streamId) {
|
|
941
|
-
if (!this.bookmarks[streamId]) {
|
|
942
|
-
return (0, import_moment.default)(0);
|
|
943
|
-
}
|
|
944
|
-
return (0, import_moment.default)(this.bookmarks[streamId]);
|
|
945
|
-
}
|
|
946
|
-
getBookmarkV2(streamId) {
|
|
947
672
|
if (this.bookmarks[streamId]) {
|
|
948
673
|
return this.bookmarks[streamId];
|
|
949
674
|
} else {
|
|
@@ -983,7 +708,7 @@ var import_axios2 = __toESM(require("axios"));
|
|
|
983
708
|
var qs2 = __toESM(require("qs"));
|
|
984
709
|
|
|
985
710
|
// src/sdk/models/tap/stream.ts
|
|
986
|
-
var
|
|
711
|
+
var import_dayjs2 = __toESM(require("dayjs"));
|
|
987
712
|
|
|
988
713
|
// src/sdk/helpers/typing.ts
|
|
989
714
|
var isDatetimeType = (typeDef) => {
|
|
@@ -1006,10 +731,11 @@ var isDatetimeType = (typeDef) => {
|
|
|
1006
731
|
};
|
|
1007
732
|
|
|
1008
733
|
// src/sdk/models/tap/stream.ts
|
|
1009
|
-
var
|
|
734
|
+
var import_cloneDeep2 = __toESM(require("lodash/cloneDeep.js"));
|
|
1010
735
|
var import_bluebird2 = __toESM(require("bluebird"));
|
|
1011
736
|
|
|
1012
737
|
// src/sdk/models/tap/tap.ts
|
|
738
|
+
var import_cloneDeep = __toESM(require("lodash/cloneDeep.js"));
|
|
1013
739
|
var import_bluebird = __toESM(require("bluebird"));
|
|
1014
740
|
var DEFAULT_MAX_CONCURRENT_STREAMS = 5;
|
|
1015
741
|
var Tap = class {
|
|
@@ -1017,12 +743,21 @@ var Tap = class {
|
|
|
1017
743
|
config;
|
|
1018
744
|
streams;
|
|
1019
745
|
concurrency = DEFAULT_MAX_CONCURRENT_STREAMS;
|
|
1020
|
-
|
|
746
|
+
stateProvider;
|
|
747
|
+
tapState;
|
|
748
|
+
constructor(target, config, stateProvider) {
|
|
1021
749
|
this.streams = [];
|
|
1022
750
|
this.target = target;
|
|
1023
751
|
this.config = config;
|
|
752
|
+
this.stateProvider = stateProvider;
|
|
753
|
+
this.tapState = { bookmarks: {} };
|
|
1024
754
|
}
|
|
1025
755
|
sync = async (options) => {
|
|
756
|
+
const initialState = await this.stateProvider.getState();
|
|
757
|
+
if (initialState.state && initialState.state.bookmarks) {
|
|
758
|
+
this.tapState = (0, import_cloneDeep.default)(initialState.state);
|
|
759
|
+
}
|
|
760
|
+
await this.init();
|
|
1026
761
|
logger.info(`\u{1F680} Start syncing`);
|
|
1027
762
|
const state = StateService.getInstance().get();
|
|
1028
763
|
logger.info(`\u{1F4CD} Received state: %j`, state);
|
|
@@ -1059,9 +794,7 @@ var Stream = class {
|
|
|
1059
794
|
config;
|
|
1060
795
|
tapState;
|
|
1061
796
|
target;
|
|
1062
|
-
|
|
1063
|
-
// Replication method that can be forced in the stream implemenation. Will prevail over any user/catalog choice
|
|
1064
|
-
forcedReplicationMethod = void 0;
|
|
797
|
+
replicationMethod = void 0;
|
|
1065
798
|
selectedByDefault = true;
|
|
1066
799
|
STATE_MSG_FREQUENCY = 10;
|
|
1067
800
|
streamId = "default";
|
|
@@ -1069,10 +802,9 @@ var Stream = class {
|
|
|
1069
802
|
displayLabel = void 0;
|
|
1070
803
|
description = void 0;
|
|
1071
804
|
primaryKey = [];
|
|
1072
|
-
|
|
805
|
+
replicationKey = void 0;
|
|
1073
806
|
// When set, use the state from another stream for this stream
|
|
1074
807
|
useStateFromStreamId = void 0;
|
|
1075
|
-
relationships = [];
|
|
1076
808
|
children = [];
|
|
1077
809
|
// If true, it means that the records in this stream are sorted by replicationKey.
|
|
1078
810
|
// If false, then the records are coming in an unsorted manner.
|
|
@@ -1084,9 +816,9 @@ var Stream = class {
|
|
|
1084
816
|
// Metrics configuration
|
|
1085
817
|
rowsSyncedMetricsConf;
|
|
1086
818
|
executionTimeMetricsConf;
|
|
1087
|
-
constructor(config,
|
|
819
|
+
constructor(config, tapState, target, childConcurrency = DEFAULT_MAX_CONCURRENT_STREAMS) {
|
|
1088
820
|
this.config = config;
|
|
1089
|
-
this.tapState = (0,
|
|
821
|
+
this.tapState = (0, import_cloneDeep2.default)(tapState);
|
|
1090
822
|
this.target = target;
|
|
1091
823
|
this.childConcurrency = childConcurrency;
|
|
1092
824
|
this.rowsSyncedMetricsConf = {
|
|
@@ -1159,13 +891,11 @@ var Stream = class {
|
|
|
1159
891
|
if (!this.isSilent) {
|
|
1160
892
|
const schema = await this.getSchema();
|
|
1161
893
|
if (schema !== void 0) {
|
|
1162
|
-
const relatedRels = findRelatedRelationships(this.streamId, this.relationships);
|
|
1163
894
|
const replicationConfig = this.configuredReplicationConfig();
|
|
1164
895
|
const message = new SchemaMessage({
|
|
1165
896
|
keyProperties: this.primaryKey,
|
|
1166
897
|
stream: this.streamId,
|
|
1167
898
|
schema: schema.jsonSchema,
|
|
1168
|
-
relationships: relatedRels,
|
|
1169
899
|
...replicationConfig.replicationKey ? { bookmarkProperties: [replicationConfig.replicationKey] } : {},
|
|
1170
900
|
...this.displayLabel ? { displayLabel: this.displayLabel } : {},
|
|
1171
901
|
...this.description ? { description: this.description } : {},
|
|
@@ -1185,7 +915,7 @@ var Stream = class {
|
|
|
1185
915
|
const signpostMoment = await this.getReplicationKeySignpost();
|
|
1186
916
|
const signpostValue = signpostMoment?.format(defaultDateTimeFormat);
|
|
1187
917
|
if (signpostValue) {
|
|
1188
|
-
StateService.getInstance().
|
|
918
|
+
StateService.getInstance().setBookmarkSignpost(this.streamId, signpostValue);
|
|
1189
919
|
}
|
|
1190
920
|
};
|
|
1191
921
|
/**
|
|
@@ -1201,7 +931,7 @@ var Stream = class {
|
|
|
1201
931
|
if (this.useStateFromStreamId) {
|
|
1202
932
|
return;
|
|
1203
933
|
}
|
|
1204
|
-
if (this.
|
|
934
|
+
if (this.replicationMethod && !this.replicationKey) {
|
|
1205
935
|
return;
|
|
1206
936
|
}
|
|
1207
937
|
if (!replicationConfig.replicationKey) {
|
|
@@ -1211,14 +941,14 @@ var Stream = class {
|
|
|
1211
941
|
);
|
|
1212
942
|
}
|
|
1213
943
|
const stateServiceInst = StateService.getInstance();
|
|
1214
|
-
const state = stateServiceInst.
|
|
944
|
+
const state = stateServiceInst.getBookmark(this.streamId);
|
|
1215
945
|
const newState = incrementStreamState(
|
|
1216
946
|
state,
|
|
1217
947
|
replicationConfig.replicationKey,
|
|
1218
948
|
latestRecord,
|
|
1219
949
|
this.isSorted
|
|
1220
950
|
);
|
|
1221
|
-
StateService.getInstance().
|
|
951
|
+
StateService.getInstance().setBookmark(this.streamId, newState);
|
|
1222
952
|
}
|
|
1223
953
|
}
|
|
1224
954
|
};
|
|
@@ -1258,19 +988,19 @@ var Stream = class {
|
|
|
1258
988
|
this.replicationKey: ${replicationConfig.replicationKey}`);
|
|
1259
989
|
if (state.replicationKeyValue) {
|
|
1260
990
|
logger.debug(`${this.streamId} - getStartingTimestamp -> Returning replication key value: ${state.replicationKeyValue}`);
|
|
1261
|
-
return (0,
|
|
991
|
+
return (0, import_dayjs2.default)(state.replicationKeyValue);
|
|
1262
992
|
} else if (startDate) {
|
|
1263
993
|
logger.debug(`${this.streamId} - getStartingTimestamp -> Returning config start date: ${startDate}`);
|
|
1264
|
-
return (0,
|
|
994
|
+
return (0, import_dayjs2.default)(startDate);
|
|
1265
995
|
} else {
|
|
1266
996
|
logger.debug(`${this.streamId} - getStartingTimestamp -> Returning EPOCH (1970)`);
|
|
1267
|
-
return (0,
|
|
997
|
+
return (0, import_dayjs2.default)(0);
|
|
1268
998
|
}
|
|
1269
999
|
}
|
|
1270
1000
|
/**
|
|
1271
1001
|
* Return the max allowable bookmark value for this stream's replication key.
|
|
1272
1002
|
*
|
|
1273
|
-
* For timestamp-based replication keys, this defaults to `
|
|
1003
|
+
* For timestamp-based replication keys, this defaults to `dayjs()`. For
|
|
1274
1004
|
* non-timestamp replication keys, default to `undefined`.
|
|
1275
1005
|
*
|
|
1276
1006
|
* Override this value to prevent bookmarks from being advanced in cases where we
|
|
@@ -1278,7 +1008,7 @@ var Stream = class {
|
|
|
1278
1008
|
*/
|
|
1279
1009
|
getReplicationKeySignpost = async () => {
|
|
1280
1010
|
if (await this.isTimestampReplicationKey()) {
|
|
1281
|
-
return (0,
|
|
1011
|
+
return (0, import_dayjs2.default)();
|
|
1282
1012
|
}
|
|
1283
1013
|
return void 0;
|
|
1284
1014
|
};
|
|
@@ -1286,22 +1016,22 @@ var Stream = class {
|
|
|
1286
1016
|
* Return the default replication method for the stream that will be used to write the catalog.
|
|
1287
1017
|
*/
|
|
1288
1018
|
defaultReplicationConfig = () => {
|
|
1289
|
-
if (this.
|
|
1019
|
+
if (this.replicationMethod) {
|
|
1290
1020
|
return {
|
|
1291
|
-
replicationMethod: this.
|
|
1292
|
-
replicationKey: this.
|
|
1021
|
+
replicationMethod: this.replicationMethod,
|
|
1022
|
+
replicationKey: this.replicationKey,
|
|
1293
1023
|
isForced: true
|
|
1294
1024
|
};
|
|
1295
1025
|
}
|
|
1296
|
-
if (this.
|
|
1026
|
+
if (this.replicationKey || this.useStateFromStreamId) {
|
|
1297
1027
|
return {
|
|
1298
|
-
replicationMethod: "INCREMENTAL"
|
|
1299
|
-
replicationKey: this.
|
|
1028
|
+
replicationMethod: "INCREMENTAL" /* INCREMENTAL */,
|
|
1029
|
+
replicationKey: this.replicationKey,
|
|
1300
1030
|
isForced: true
|
|
1301
1031
|
};
|
|
1302
1032
|
}
|
|
1303
1033
|
return {
|
|
1304
|
-
replicationMethod: "FULL_TABLE"
|
|
1034
|
+
replicationMethod: "FULL_TABLE" /* FULL_TABLE */,
|
|
1305
1035
|
replicationKey: void 0,
|
|
1306
1036
|
isForced: false
|
|
1307
1037
|
};
|
|
@@ -1362,9 +1092,9 @@ var Stream = class {
|
|
|
1362
1092
|
rowsSent += 1;
|
|
1363
1093
|
}
|
|
1364
1094
|
const stateServiceInst = StateService.getInstance();
|
|
1365
|
-
const state = stateServiceInst.
|
|
1095
|
+
const state = stateServiceInst.getBookmark(this.streamId);
|
|
1366
1096
|
const newState = finalizeStateProgressMarkers(state);
|
|
1367
|
-
stateServiceInst.
|
|
1097
|
+
stateServiceInst.setBookmark(this.streamId, newState);
|
|
1368
1098
|
if (!parent) {
|
|
1369
1099
|
logger.info(`\u2705 Completed sync for stream: ${this.streamId} (${rowsSent} records)`);
|
|
1370
1100
|
} else {
|
|
@@ -1470,7 +1200,7 @@ function getTruncatedParamsForLog(params) {
|
|
|
1470
1200
|
}
|
|
1471
1201
|
|
|
1472
1202
|
// src/sdk/models/tap/restStream.ts
|
|
1473
|
-
var
|
|
1203
|
+
var import_cloneDeep3 = __toESM(require("lodash/cloneDeep.js"));
|
|
1474
1204
|
var import_isEqual = __toESM(require("lodash/isEqual.js"));
|
|
1475
1205
|
var RESTStream = class extends Stream {
|
|
1476
1206
|
axiosInstance;
|
|
@@ -1491,13 +1221,24 @@ var RESTStream = class extends Stream {
|
|
|
1491
1221
|
*/
|
|
1492
1222
|
_prepareRequest = async (nextPageToken, parent) => {
|
|
1493
1223
|
const method = this.httpMethod;
|
|
1494
|
-
const url = this.
|
|
1224
|
+
const url = this.getNextUrl(nextPageToken, parent);
|
|
1495
1225
|
if (!url) {
|
|
1496
1226
|
throw new Error(`StreamId: ${this.streamId} - URL is undefined. Did your properly define the URL for this stream?`);
|
|
1497
1227
|
}
|
|
1498
1228
|
const authenticator = this.authenticator;
|
|
1229
|
+
let finalUrl = url;
|
|
1230
|
+
let urlQueryParams = {};
|
|
1231
|
+
const queryIndex = url.indexOf("?");
|
|
1232
|
+
if (queryIndex !== -1) {
|
|
1233
|
+
finalUrl = url.substring(0, queryIndex);
|
|
1234
|
+
const rawQueryString = url.substring(queryIndex + 1);
|
|
1235
|
+
urlQueryParams = qs2.parse(rawQueryString);
|
|
1236
|
+
}
|
|
1499
1237
|
const params = {
|
|
1500
1238
|
...this.getNextUrlParams(nextPageToken, parent),
|
|
1239
|
+
// Query params embedded in URL should override computed pagination params
|
|
1240
|
+
// but should not override auth params which are appended last
|
|
1241
|
+
...urlQueryParams,
|
|
1501
1242
|
...await authenticator?.getAuthQS(this.config)
|
|
1502
1243
|
};
|
|
1503
1244
|
const requestBody = this.getRequestBodyForNextCall(nextPageToken, parent);
|
|
@@ -1516,7 +1257,7 @@ var RESTStream = class extends Stream {
|
|
|
1516
1257
|
return qs2.stringify(params2, { arrayFormat: "repeat" });
|
|
1517
1258
|
};
|
|
1518
1259
|
const request = {
|
|
1519
|
-
url,
|
|
1260
|
+
url: finalUrl,
|
|
1520
1261
|
method,
|
|
1521
1262
|
data: requestBody,
|
|
1522
1263
|
params,
|
|
@@ -1589,7 +1330,7 @@ var RESTStream = class extends Stream {
|
|
|
1589
1330
|
for await (const row of rows) {
|
|
1590
1331
|
yield row;
|
|
1591
1332
|
}
|
|
1592
|
-
const previousToken = (0,
|
|
1333
|
+
const previousToken = (0, import_cloneDeep3.default)(nextPageToken);
|
|
1593
1334
|
nextPageToken = this.getNextPageToken(resp, previousToken);
|
|
1594
1335
|
if (nextPageToken && (0, import_isEqual.default)(nextPageToken, previousToken)) {
|
|
1595
1336
|
throw new Error(
|
|
@@ -1668,7 +1409,7 @@ var RESTStream = class extends Stream {
|
|
|
1668
1409
|
* TODO: Make URL + Path templatisable and replace those with Extractor (or Tap) config
|
|
1669
1410
|
* @returns
|
|
1670
1411
|
*/
|
|
1671
|
-
|
|
1412
|
+
getNextUrl(previousToken, parent) {
|
|
1672
1413
|
const urlPattern = [this.baseUrl, this.path].join("");
|
|
1673
1414
|
return urlPattern;
|
|
1674
1415
|
}
|
|
@@ -2020,18 +1761,18 @@ var flattenSchema = (streamId, schema) => {
|
|
|
2020
1761
|
|
|
2021
1762
|
// src/sdk/models/target/target.ts
|
|
2022
1763
|
var import_jsonschema = require("jsonschema");
|
|
2023
|
-
var
|
|
1764
|
+
var import_dayjs4 = __toESM(require("dayjs"));
|
|
2024
1765
|
|
|
2025
1766
|
// src/sdk/models/target/streamDbState.ts
|
|
2026
|
-
var
|
|
1767
|
+
var import_dayjs3 = __toESM(require("dayjs"));
|
|
2027
1768
|
|
|
2028
1769
|
// src/sdk/models/target/temporaryFile.ts
|
|
2029
|
-
var
|
|
1770
|
+
var import_fs_extra = __toESM(require("fs-extra"));
|
|
2030
1771
|
var import_uniqueId = __toESM(require("lodash/uniqueId.js"));
|
|
2031
1772
|
var createTemporaryFileStream = (streamId) => {
|
|
2032
|
-
|
|
1773
|
+
import_fs_extra.default.ensureDirSync("./tmp");
|
|
2033
1774
|
const path = `./tmp/${safePath(streamId)}-${(0, import_uniqueId.default)()}.tmp`;
|
|
2034
|
-
const writeStream =
|
|
1775
|
+
const writeStream = import_fs_extra.default.createWriteStream(path, { flags: "w" });
|
|
2035
1776
|
return {
|
|
2036
1777
|
stream: writeStream,
|
|
2037
1778
|
path
|
|
@@ -2079,7 +1820,7 @@ function replaceSymbols(str) {
|
|
|
2079
1820
|
}
|
|
2080
1821
|
|
|
2081
1822
|
// src/sdk/models/target/streamDbState.ts
|
|
2082
|
-
var
|
|
1823
|
+
var fs3 = __toESM(require("fs"));
|
|
2083
1824
|
var StreamState = class {
|
|
2084
1825
|
streamId;
|
|
2085
1826
|
schema;
|
|
@@ -2104,7 +1845,7 @@ var StreamState = class {
|
|
|
2104
1845
|
this.fileToLoad = createTemporaryFileStream(streamId);
|
|
2105
1846
|
this.syncedRowCount = 0;
|
|
2106
1847
|
this.keyProperties = [];
|
|
2107
|
-
this.batchDate = (0,
|
|
1848
|
+
this.batchDate = (0, import_dayjs3.default)();
|
|
2108
1849
|
this.replicationMethod = replicationMethod;
|
|
2109
1850
|
this._hasBeenLoadedYetDuringThisSync = false;
|
|
2110
1851
|
}
|
|
@@ -2129,7 +1870,7 @@ var StreamState = class {
|
|
|
2129
1870
|
resetFileToLoad() {
|
|
2130
1871
|
this.fileToLoad.stream.close();
|
|
2131
1872
|
const oldPath = this.fileToLoad.path;
|
|
2132
|
-
|
|
1873
|
+
fs3.unlinkSync(oldPath);
|
|
2133
1874
|
this.fileToLoad = createTemporaryFileStream(this.streamId);
|
|
2134
1875
|
this.batchedRowCount = 0;
|
|
2135
1876
|
}
|
|
@@ -2160,8 +1901,8 @@ var semaphore = new import_async_mutex.Semaphore(1);
|
|
|
2160
1901
|
var ITarget = class _ITarget {
|
|
2161
1902
|
config;
|
|
2162
1903
|
schemaHooks;
|
|
2163
|
-
resolver;
|
|
2164
1904
|
syncTime;
|
|
1905
|
+
stateProvider;
|
|
2165
1906
|
// Latest state message received from the Tap
|
|
2166
1907
|
// Not yet flushed as we didn't upload the stream data since receiving it
|
|
2167
1908
|
batchedState;
|
|
@@ -2172,15 +1913,15 @@ var ITarget = class _ITarget {
|
|
|
2172
1913
|
streams;
|
|
2173
1914
|
// JSON Schema validator
|
|
2174
1915
|
validator;
|
|
2175
|
-
constructor(config,
|
|
1916
|
+
constructor(config, stateProvider) {
|
|
2176
1917
|
this.config = config;
|
|
2177
|
-
this.
|
|
2178
|
-
this.schemaHooks =
|
|
1918
|
+
this.stateProvider = stateProvider;
|
|
1919
|
+
this.schemaHooks = [];
|
|
2179
1920
|
this.streams = {};
|
|
2180
1921
|
this.batchedState = {};
|
|
2181
1922
|
this.flushedState = {};
|
|
2182
1923
|
this.validator = new import_jsonschema.Validator();
|
|
2183
|
-
this.syncTime = (0,
|
|
1924
|
+
this.syncTime = (0, import_dayjs4.default)();
|
|
2184
1925
|
this.renameColumnStore = new RenameColumnStore();
|
|
2185
1926
|
}
|
|
2186
1927
|
///////////////////////////////////////////
|
|
@@ -2289,7 +2030,7 @@ var ITarget = class _ITarget {
|
|
|
2289
2030
|
this.streams[streamId] = new StreamState(
|
|
2290
2031
|
streamId,
|
|
2291
2032
|
dbSyncInstance,
|
|
2292
|
-
replicationMethod || "FULL_TABLE"
|
|
2033
|
+
replicationMethod || "FULL_TABLE" /* FULL_TABLE */
|
|
2293
2034
|
);
|
|
2294
2035
|
}
|
|
2295
2036
|
replicationMethod = (message) => {
|
|
@@ -2366,39 +2107,7 @@ var ITarget = class _ITarget {
|
|
|
2366
2107
|
return translation;
|
|
2367
2108
|
}
|
|
2368
2109
|
return k;
|
|
2369
|
-
})
|
|
2370
|
-
relationships: {
|
|
2371
|
-
left: message.relationships.left.flatMap((l) => {
|
|
2372
|
-
if (this.renameColumnStore.isReady(l.streamId) && this.renameColumnStore.isReady(message.stream)) {
|
|
2373
|
-
const from = this.renameColumnStore.getColumnTranslation(l.streamId, l.from);
|
|
2374
|
-
const to = this.renameColumnStore.getColumnTranslation(message.stream, l.to);
|
|
2375
|
-
if (from && to) {
|
|
2376
|
-
return [{
|
|
2377
|
-
type: l.type,
|
|
2378
|
-
streamId: l.streamId,
|
|
2379
|
-
from,
|
|
2380
|
-
to
|
|
2381
|
-
}];
|
|
2382
|
-
}
|
|
2383
|
-
}
|
|
2384
|
-
return [];
|
|
2385
|
-
}),
|
|
2386
|
-
right: message.relationships.right.flatMap((r) => {
|
|
2387
|
-
if (this.renameColumnStore.isReady(message.stream) && this.renameColumnStore.isReady(r.streamId)) {
|
|
2388
|
-
const from = this.renameColumnStore.getColumnTranslation(message.stream, r.from);
|
|
2389
|
-
const to = this.renameColumnStore.getColumnTranslation(r.streamId, r.to);
|
|
2390
|
-
if (from && to) {
|
|
2391
|
-
return [{
|
|
2392
|
-
type: r.type,
|
|
2393
|
-
streamId: r.streamId,
|
|
2394
|
-
from,
|
|
2395
|
-
to
|
|
2396
|
-
}];
|
|
2397
|
-
}
|
|
2398
|
-
}
|
|
2399
|
-
return [];
|
|
2400
|
-
})
|
|
2401
|
-
}
|
|
2110
|
+
})
|
|
2402
2111
|
};
|
|
2403
2112
|
await import_bluebird3.default.map(this.schemaHooks, async (hook) => {
|
|
2404
2113
|
const input = {
|
|
@@ -2431,9 +2140,6 @@ var ITarget = class _ITarget {
|
|
|
2431
2140
|
try {
|
|
2432
2141
|
logger.info(`\u{1F44D} Data Extraction is complete. Will load remaining batched stream records.`);
|
|
2433
2142
|
await this.loadAllStreamsInWarehouse({ isFinalLoad: true });
|
|
2434
|
-
const configResolver = loadResolver();
|
|
2435
|
-
await configResolver.markSyncComplete();
|
|
2436
|
-
await configResolver.flushMetrics();
|
|
2437
2143
|
return Promise.resolve();
|
|
2438
2144
|
} catch (err) {
|
|
2439
2145
|
logger.error(`Error while handling complete event.`);
|
|
@@ -2476,8 +2182,7 @@ var ITarget = class _ITarget {
|
|
|
2476
2182
|
}
|
|
2477
2183
|
};
|
|
2478
2184
|
emitState = (state) => {
|
|
2479
|
-
|
|
2480
|
-
return writeStateFile(configResolver, JSON.stringify(state));
|
|
2185
|
+
return this.stateProvider.writeState(JSON.stringify(state));
|
|
2481
2186
|
};
|
|
2482
2187
|
uploadStreamsToWarehouse = async (streamsToUpload) => {
|
|
2483
2188
|
try {
|
|
@@ -2549,15 +2254,15 @@ var safeTableName = (key) => {
|
|
|
2549
2254
|
|
|
2550
2255
|
// src/targets/bigquery/service/record.ts
|
|
2551
2256
|
var import_decimal = __toESM(require("decimal.js"));
|
|
2552
|
-
var
|
|
2257
|
+
var import_dayjs5 = __toESM(require("dayjs"));
|
|
2553
2258
|
var validateDateRange = (record, schema) => {
|
|
2554
2259
|
Object.keys(schema).forEach((key) => {
|
|
2555
|
-
const { format:
|
|
2556
|
-
if (
|
|
2260
|
+
const { format: format6 } = schema[key];
|
|
2261
|
+
if (format6 === "date-time") {
|
|
2557
2262
|
if (record[key]) {
|
|
2558
|
-
const formattedDate = (0,
|
|
2559
|
-
const isNotTooMuchInTheFuture = (0,
|
|
2560
|
-
const isNotTooMuchInThePast = (0,
|
|
2263
|
+
const formattedDate = (0, import_dayjs5.default)(record[key]).format(defaultDateTimeFormat);
|
|
2264
|
+
const isNotTooMuchInTheFuture = (0, import_dayjs5.default)(record[key]).isBefore((0, import_dayjs5.default)("9999-12-31 23:59:59.999999"));
|
|
2265
|
+
const isNotTooMuchInThePast = (0, import_dayjs5.default)(record[key]).isAfter((0, import_dayjs5.default)("0001-01-01 00:00:00"));
|
|
2561
2266
|
if (formattedDate !== "Invalid date" && isNotTooMuchInTheFuture && isNotTooMuchInThePast) {
|
|
2562
2267
|
record[key] = formattedDate;
|
|
2563
2268
|
} else {
|
|
@@ -2612,7 +2317,7 @@ var convertNumberIntoDecimal = (record, schema) => {
|
|
|
2612
2317
|
};
|
|
2613
2318
|
|
|
2614
2319
|
// src/targets/bigquery/service/dbSync.ts
|
|
2615
|
-
var
|
|
2320
|
+
var import_dayjs6 = __toESM(require("dayjs"));
|
|
2616
2321
|
|
|
2617
2322
|
// src/targets/bigquery/service/bigquery.ts
|
|
2618
2323
|
var import_bigquery = require("@google-cloud/bigquery");
|
|
@@ -2693,13 +2398,13 @@ function v4(options, buf, offset) {
|
|
|
2693
2398
|
var v4_default = v4;
|
|
2694
2399
|
|
|
2695
2400
|
// src/targets/bigquery/service/cloudStorage.ts
|
|
2696
|
-
var uploadFileInBucket = async (streamId, gcsBuckerName, filePath) => {
|
|
2401
|
+
var uploadFileInBucket = async (streamId, gcsBuckerName, connectorId, filePath) => {
|
|
2697
2402
|
try {
|
|
2698
2403
|
const retryOptions = { autoRetry: true, maxRetries: 20 };
|
|
2699
2404
|
let storage = new import_storage.Storage({ retryOptions });
|
|
2700
2405
|
const bucketRef = storage.bucket(gcsBuckerName);
|
|
2701
2406
|
await bucketRef.get({ autoCreate: false });
|
|
2702
|
-
const destinationFileName = `${streamId}-${v4_default()}.jsonnl`;
|
|
2407
|
+
const destinationFileName = `${connectorId}/${streamId}-${v4_default()}.jsonnl`;
|
|
2703
2408
|
await bucketRef.upload(filePath, {
|
|
2704
2409
|
destination: destinationFileName
|
|
2705
2410
|
});
|
|
@@ -2767,21 +2472,21 @@ var BigQueryDBSync = class extends StreamWarehouseSyncService {
|
|
|
2767
2472
|
safeColumnName = safeColumnName;
|
|
2768
2473
|
getWarehouseTypeFromJSONSchema = (definition) => {
|
|
2769
2474
|
const type = definition.type;
|
|
2770
|
-
const
|
|
2475
|
+
const format6 = definition.format;
|
|
2771
2476
|
let typeArr;
|
|
2772
2477
|
if (type instanceof Array) {
|
|
2773
2478
|
typeArr = type;
|
|
2774
2479
|
} else {
|
|
2775
2480
|
typeArr = [type];
|
|
2776
2481
|
}
|
|
2777
|
-
if (
|
|
2778
|
-
switch (
|
|
2482
|
+
if (format6) {
|
|
2483
|
+
switch (format6) {
|
|
2779
2484
|
case "date-time":
|
|
2780
2485
|
return "TIMESTAMP";
|
|
2781
2486
|
case "json":
|
|
2782
2487
|
return "STRING";
|
|
2783
2488
|
default:
|
|
2784
|
-
throw new Error(`StreamId: ${this.streamId} - Unsupported format: ${
|
|
2489
|
+
throw new Error(`StreamId: ${this.streamId} - Unsupported format: ${format6}`);
|
|
2785
2490
|
}
|
|
2786
2491
|
} else if (typeArr.includes("number")) {
|
|
2787
2492
|
return "NUMERIC";
|
|
@@ -2981,7 +2686,7 @@ var BigQueryDBSync = class extends StreamWarehouseSyncService {
|
|
|
2981
2686
|
await this.deleteStagingArea();
|
|
2982
2687
|
}
|
|
2983
2688
|
const schema = this.generateBigquerySchema();
|
|
2984
|
-
const expirationTime = (0,
|
|
2689
|
+
const expirationTime = (0, import_dayjs6.default)().add(1, "day").valueOf().toString();
|
|
2985
2690
|
const createTableOptions = {
|
|
2986
2691
|
schema,
|
|
2987
2692
|
expirationTime
|
|
@@ -3004,6 +2709,7 @@ var BigQueryDBSync = class extends StreamWarehouseSyncService {
|
|
|
3004
2709
|
const file = await uploadFileInBucket(
|
|
3005
2710
|
this.streamId,
|
|
3006
2711
|
this.config.loading_deck_gcs_bucket_name,
|
|
2712
|
+
this.config.connector_id,
|
|
3007
2713
|
localFilePath
|
|
3008
2714
|
);
|
|
3009
2715
|
const metadata = {
|
|
@@ -3115,8 +2821,8 @@ var BigQueryDBSync = class extends StreamWarehouseSyncService {
|
|
|
3115
2821
|
|
|
3116
2822
|
// src/targets/bigquery/main.ts
|
|
3117
2823
|
var BigQueryTarget = class extends ITarget {
|
|
3118
|
-
constructor(config,
|
|
3119
|
-
super(config,
|
|
2824
|
+
constructor(config, stateProvider) {
|
|
2825
|
+
super(config, stateProvider);
|
|
3120
2826
|
this.renameColumnStore.setSafeColumnNameConverter(this.safeColumnNameConverter);
|
|
3121
2827
|
}
|
|
3122
2828
|
static requiredConfigKeys = [
|
|
@@ -3139,6 +2845,75 @@ var BigQueryTarget = class extends ITarget {
|
|
|
3139
2845
|
validateDateRange = validateDateRange;
|
|
3140
2846
|
convertNumberIntoDecimal = convertNumberIntoDecimal;
|
|
3141
2847
|
};
|
|
2848
|
+
|
|
2849
|
+
// src/sdk/service/gcs.ts
|
|
2850
|
+
var import_storage2 = require("@google-cloud/storage");
|
|
2851
|
+
var import_util3 = require("util");
|
|
2852
|
+
var getStorage = () => new import_storage2.Storage({ retryOptions: { autoRetry: true, maxRetries: 20 } });
|
|
2853
|
+
var readObjectAsString = async (bucketName, objectPath) => {
|
|
2854
|
+
try {
|
|
2855
|
+
const storage = getStorage();
|
|
2856
|
+
const bucketRef = storage.bucket(bucketName);
|
|
2857
|
+
await bucketRef.get({ autoCreate: false });
|
|
2858
|
+
const fileRef = bucketRef.file(objectPath);
|
|
2859
|
+
const [exists] = await fileRef.exists();
|
|
2860
|
+
if (!exists) {
|
|
2861
|
+
throw new Error(`GCS object not found: gs://${bucketName}/${objectPath}`);
|
|
2862
|
+
}
|
|
2863
|
+
const [contents] = await fileRef.download();
|
|
2864
|
+
return contents.toString("utf8");
|
|
2865
|
+
} catch (err) {
|
|
2866
|
+
throw new Error((0, import_util3.format)(`error reading GCS object gs://${bucketName}/${objectPath}, err: %s`, err?.message));
|
|
2867
|
+
}
|
|
2868
|
+
};
|
|
2869
|
+
var writeStringObject = async (bucketName, objectPath, contents) => {
|
|
2870
|
+
try {
|
|
2871
|
+
const storage = getStorage();
|
|
2872
|
+
const bucketRef = storage.bucket(bucketName);
|
|
2873
|
+
await bucketRef.get({ autoCreate: false });
|
|
2874
|
+
const fileRef = bucketRef.file(objectPath);
|
|
2875
|
+
await fileRef.save(contents, { contentType: "application/json" });
|
|
2876
|
+
logger.info(`\u{1F5F3} Uploaded object to gs://${bucketName}/${objectPath}`);
|
|
2877
|
+
} catch (err) {
|
|
2878
|
+
throw new Error((0, import_util3.format)(`error writing GCS object gs://${bucketName}/${objectPath}, err: %s`, err?.message));
|
|
2879
|
+
}
|
|
2880
|
+
};
|
|
2881
|
+
|
|
2882
|
+
// src/state-providers/gcs/main.ts
|
|
2883
|
+
var import_util4 = require("util");
|
|
2884
|
+
var GCSStateProvider = class {
|
|
2885
|
+
bucketName;
|
|
2886
|
+
connectorId;
|
|
2887
|
+
constructor(connectorId, bucketName) {
|
|
2888
|
+
this.connectorId = connectorId;
|
|
2889
|
+
this.bucketName = bucketName;
|
|
2890
|
+
}
|
|
2891
|
+
getObjectPath() {
|
|
2892
|
+
return `${this.connectorId}/state.json`;
|
|
2893
|
+
}
|
|
2894
|
+
async getState() {
|
|
2895
|
+
const objectPath = this.getObjectPath();
|
|
2896
|
+
try {
|
|
2897
|
+
logger.info(`\u{1F4C1} Loading state from gs://${this.bucketName}/${objectPath}`);
|
|
2898
|
+
const raw = await readObjectAsString(this.bucketName, objectPath);
|
|
2899
|
+
const parsed = JSON.parse(raw);
|
|
2900
|
+
return { state: parsed };
|
|
2901
|
+
} catch (err) {
|
|
2902
|
+
logger.warn((0, import_util4.format)(`error loading state from gs://${this.bucketName}/${objectPath}, will start with empty state, err: %s`, err?.message));
|
|
2903
|
+
return { state: void 0 };
|
|
2904
|
+
}
|
|
2905
|
+
}
|
|
2906
|
+
async writeState(state) {
|
|
2907
|
+
const objectPath = this.getObjectPath();
|
|
2908
|
+
try {
|
|
2909
|
+
logger.info(`\u{1F4DD} Writing state to gs://${this.bucketName}/${objectPath}`);
|
|
2910
|
+
await writeStringObject(this.bucketName, objectPath, state);
|
|
2911
|
+
} catch (err) {
|
|
2912
|
+
logger.error((0, import_util4.format)(`\u{1F4A5} Error while writing state to gs://${this.bucketName}/${objectPath}, err: %s`, err?.message));
|
|
2913
|
+
throw new Error((0, import_util4.format)(`error writing state to gs://${this.bucketName}/${objectPath}, err: %s`, err?.message));
|
|
2914
|
+
}
|
|
2915
|
+
}
|
|
2916
|
+
};
|
|
3142
2917
|
// Annotate the CommonJS export names for ESM import in node:
|
|
3143
2918
|
0 && (module.exports = {
|
|
3144
2919
|
ALL_STREAM_ID_KEYWORD,
|
|
@@ -3149,6 +2924,7 @@ var BigQueryTarget = class extends ITarget {
|
|
|
3149
2924
|
CounterMetric,
|
|
3150
2925
|
DEFAULT_MAX_CONCURRENT_STREAMS,
|
|
3151
2926
|
EXECUTION_TIME_METRIC_NAME,
|
|
2927
|
+
GCSStateProvider,
|
|
3152
2928
|
ITarget,
|
|
3153
2929
|
MissingFieldInSchemaError,
|
|
3154
2930
|
MissingSchemaError,
|
|
@@ -3156,14 +2932,13 @@ var BigQueryTarget = class extends ITarget {
|
|
|
3156
2932
|
ROWS_SYNCED_METRIC_NAME,
|
|
3157
2933
|
RecordMessage,
|
|
3158
2934
|
RenameColumnStore,
|
|
2935
|
+
ReplicationMethod,
|
|
3159
2936
|
ReplicationMethodMessage,
|
|
3160
|
-
Resolver,
|
|
3161
2937
|
SchemaMessage,
|
|
3162
2938
|
SchemaValidationError,
|
|
3163
2939
|
StateMessage,
|
|
3164
2940
|
StateService,
|
|
3165
2941
|
Stream,
|
|
3166
|
-
StreamMetadata,
|
|
3167
2942
|
StreamWarehouseSyncService,
|
|
3168
2943
|
Tap,
|
|
3169
2944
|
_greaterThan,
|
|
@@ -3171,7 +2946,6 @@ var BigQueryTarget = class extends ITarget {
|
|
|
3171
2946
|
createTemporaryFileStream,
|
|
3172
2947
|
extractStateForStream,
|
|
3173
2948
|
finalizeStateProgressMarkers,
|
|
3174
|
-
findRelatedRelationships,
|
|
3175
2949
|
flattenSchema,
|
|
3176
2950
|
getAllMetrics,
|
|
3177
2951
|
getAxiosInstance,
|
|
@@ -3185,7 +2959,6 @@ var BigQueryTarget = class extends ITarget {
|
|
|
3185
2959
|
haltAndCatchFire,
|
|
3186
2960
|
incrementStreamState,
|
|
3187
2961
|
loadJson,
|
|
3188
|
-
loadResolver,
|
|
3189
2962
|
loadSchema,
|
|
3190
2963
|
logger,
|
|
3191
2964
|
postFormDataApiCall,
|
|
@@ -3193,7 +2966,6 @@ var BigQueryTarget = class extends ITarget {
|
|
|
3193
2966
|
postUrlEncodedApiCall,
|
|
3194
2967
|
printMemoryFootprint,
|
|
3195
2968
|
removeParasiteProperties,
|
|
3196
|
-
safePath
|
|
3197
|
-
writeStateFile
|
|
2969
|
+
safePath
|
|
3198
2970
|
});
|
|
3199
2971
|
//# sourceMappingURL=index.js.map
|