azure-kusto-ingest 2.2.3 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.eslintignore +5 -0
- package/.eslintrc.js +247 -0
- package/.mocharc.json +2 -2
- package/.prettierignore +8 -0
- package/.prettierrc.json +5 -0
- package/README.md +64 -62
- package/example.js +20 -30
- package/index.d.ts +24 -2
- package/index.js +46 -6
- package/index.js.map +1 -1
- package/package.json +70 -55
- package/source/abstractKustoClient.d.ts +2 -2
- package/source/abstractKustoClient.js +4 -2
- package/source/abstractKustoClient.js.map +1 -1
- package/source/columnMappings.d.ts +114 -0
- package/source/columnMappings.js +240 -0
- package/source/columnMappings.js.map +1 -0
- package/source/descriptors.d.ts +4 -3
- package/source/descriptors.js +16 -6
- package/source/descriptors.js.map +1 -1
- package/source/errors.d.ts +3 -0
- package/source/errors.js +13 -0
- package/source/errors.js.map +1 -0
- package/source/ingestClient.d.ts +2 -2
- package/source/ingestClient.js +1 -3
- package/source/ingestClient.js.map +1 -1
- package/source/ingestionBlobInfo.js +9 -6
- package/source/ingestionBlobInfo.js.map +1 -1
- package/source/ingestionProperties.d.ts +96 -40
- package/source/ingestionProperties.js +166 -84
- package/source/ingestionProperties.js.map +1 -1
- package/source/managedStreamingIngestClient.d.ts +36 -0
- package/source/managedStreamingIngestClient.js +106 -0
- package/source/managedStreamingIngestClient.js.map +1 -0
- package/source/resourceManager.d.ts +1 -1
- package/source/resourceManager.js +14 -16
- package/source/resourceManager.js.map +1 -1
- package/source/retry.d.ts +10 -0
- package/source/retry.js +44 -0
- package/source/retry.js.map +1 -0
- package/source/status.js +6 -17
- package/source/status.js.map +1 -1
- package/source/statusQ.js +7 -6
- package/source/statusQ.js.map +1 -1
- package/source/streamUtils.d.ts +6 -0
- package/source/streamUtils.js +61 -0
- package/source/streamUtils.js.map +1 -0
- package/source/streamingIngestClient.d.ts +2 -3
- package/source/streamingIngestClient.js +5 -14
- package/source/streamingIngestClient.js.map +1 -1
- package/tsconfig.json +16 -16
- package/tsconfig.tsbuildinfo +1 -1
- package/index.ts +0 -48
- package/tslint.json +0 -18
|
@@ -31,7 +31,7 @@ export declare class ResourceManager {
|
|
|
31
31
|
rows: () => any;
|
|
32
32
|
}, resourceName: string): ResourceURI[];
|
|
33
33
|
refreshAuthorizationContext(): Promise<string>;
|
|
34
|
-
getAuthorizationContextFromService(): Promise<
|
|
34
|
+
getAuthorizationContextFromService(): Promise<string>;
|
|
35
35
|
getIngestionQueues(): Promise<ResourceURI[] | null>;
|
|
36
36
|
getFailedIngestionsQueues(): Promise<ResourceURI[] | null>;
|
|
37
37
|
getSuccessfulIngestionsQueues(): Promise<ResourceURI[] | null>;
|
|
@@ -32,10 +32,10 @@ class ResourceURI {
|
|
|
32
32
|
return new ResourceURI(match[1], match[2], match[3], match[4]);
|
|
33
33
|
}
|
|
34
34
|
getSASConnectionString() {
|
|
35
|
-
if (this.objectType
|
|
35
|
+
if (this.objectType === "queue") {
|
|
36
36
|
return `QueueEndpoint=https://${this.storageAccountName}.queue.core.windows.net/;SharedAccessSignature=${this.sas}`;
|
|
37
37
|
}
|
|
38
|
-
if (this.objectType
|
|
38
|
+
if (this.objectType === "blob") {
|
|
39
39
|
return `BlobEndpoint=https://${this.storageAccountName}.blob.core.windows.net/;SharedAccessSignature=${this.sas}`;
|
|
40
40
|
}
|
|
41
41
|
throw new Error(`Can't make the current object type (${this.objectType}) to connection string`);
|
|
@@ -50,12 +50,7 @@ class IngestClientResources {
|
|
|
50
50
|
this.containers = containers;
|
|
51
51
|
}
|
|
52
52
|
valid() {
|
|
53
|
-
const resources = [
|
|
54
|
-
this.securedReadyForAggregationQueues,
|
|
55
|
-
this.failedIngestionsQueues,
|
|
56
|
-
this.failedIngestionsQueues,
|
|
57
|
-
this.containers
|
|
58
|
-
];
|
|
53
|
+
const resources = [this.securedReadyForAggregationQueues, this.failedIngestionsQueues, this.failedIngestionsQueues, this.containers];
|
|
59
54
|
return resources.reduce((prev, current) => !!(prev && current), true);
|
|
60
55
|
}
|
|
61
56
|
}
|
|
@@ -71,10 +66,10 @@ class ResourceManager {
|
|
|
71
66
|
}
|
|
72
67
|
refreshIngestClientResources() {
|
|
73
68
|
return __awaiter(this, void 0, void 0, function* () {
|
|
74
|
-
const now = moment_1.default();
|
|
69
|
+
const now = (0, moment_1.default)();
|
|
75
70
|
if (!this.ingestClientResources ||
|
|
76
71
|
!this.ingestClientResourcesLastUpdate ||
|
|
77
|
-
|
|
72
|
+
this.ingestClientResourcesLastUpdate.add(this.refreshPeriod) <= now ||
|
|
78
73
|
!this.ingestClientResources.valid()) {
|
|
79
74
|
this.ingestClientResources = yield this.getIngestClientResourcesFromService();
|
|
80
75
|
this.ingestClientResourcesLastUpdate = now;
|
|
@@ -92,8 +87,9 @@ class ResourceManager {
|
|
|
92
87
|
getResourceByName(table, resourceName) {
|
|
93
88
|
const result = [];
|
|
94
89
|
for (const row of table.rows()) {
|
|
95
|
-
|
|
96
|
-
|
|
90
|
+
const typedRow = row;
|
|
91
|
+
if (typedRow.ResourceTypeName === resourceName) {
|
|
92
|
+
result.push(ResourceURI.fromURI(typedRow.StorageRoot));
|
|
97
93
|
}
|
|
98
94
|
}
|
|
99
95
|
return result;
|
|
@@ -102,9 +98,7 @@ class ResourceManager {
|
|
|
102
98
|
var _a;
|
|
103
99
|
return __awaiter(this, void 0, void 0, function* () {
|
|
104
100
|
const now = moment_1.default.utc();
|
|
105
|
-
if (!((_a = this.authorizationContext) === null || _a === void 0 ? void 0 : _a.trim()) ||
|
|
106
|
-
!this.authorizationContextLastUpdate ||
|
|
107
|
-
(this.authorizationContextLastUpdate.add(this.refreshPeriod)) <= now) {
|
|
101
|
+
if (!((_a = this.authorizationContext) === null || _a === void 0 ? void 0 : _a.trim()) || !this.authorizationContextLastUpdate || this.authorizationContextLastUpdate.add(this.refreshPeriod) <= now) {
|
|
108
102
|
this.authorizationContext = yield this.getAuthorizationContextFromService();
|
|
109
103
|
this.authorizationContextLastUpdate = now;
|
|
110
104
|
if (this.authorizationContext == null) {
|
|
@@ -117,7 +111,11 @@ class ResourceManager {
|
|
|
117
111
|
getAuthorizationContextFromService() {
|
|
118
112
|
return __awaiter(this, void 0, void 0, function* () {
|
|
119
113
|
const response = yield this.kustoClient.execute("NetDefaultDB", ".get kusto identity token");
|
|
120
|
-
|
|
114
|
+
const next = response.primaryResults[0].rows().next();
|
|
115
|
+
if (next.done) {
|
|
116
|
+
throw new Error("Failed to get authorization context - got empty results");
|
|
117
|
+
}
|
|
118
|
+
return next.value.toJSON().AuthorizationContext;
|
|
121
119
|
});
|
|
122
120
|
}
|
|
123
121
|
getIngestionQueues() {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"resourceManager.js","sourceRoot":"","sources":["resourceManager.ts"],"names":[],"mappings":";AAAA,uCAAuC;AACvC,kCAAkC;;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"resourceManager.js","sourceRoot":"","sources":["resourceManager.ts"],"names":[],"mappings":";AAAA,uCAAuC;AACvC,kCAAkC;;;;;;;;;;;;;;;AAKlC,oDAA4B;AAE5B,MAAM,UAAU,GAAG,sEAAsE,CAAC;AAE1F,MAAa,WAAW;IACpB,YAAqB,kBAA0B,EAAW,UAAkB,EAAW,UAAkB,EAAW,GAAW;QAA1G,uBAAkB,GAAlB,kBAAkB,CAAQ;QAAW,eAAU,GAAV,UAAU,CAAQ;QAAW,eAAU,GAAV,UAAU,CAAQ;QAAW,QAAG,GAAH,GAAG,CAAQ;IAAG,CAAC;IAEnI,MAAM,CAAC,OAAO,CAAC,GAAW;QACtB,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE;YACnC,MAAM,KAAK,CAAC,4DAA4D,GAAG,GAAG,CAAC,CAAC;SACnF;QACD,OAAO,IAAI,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACnE,CAAC;IAED,sBAAsB;QAClB,IAAI,IAAI,CAAC,UAAU,KAAK,OAAO,EAAE;YAC7B,OAAO,yBAAyB,IAAI,CAAC,kBAAkB,kDAAkD,IAAI,CAAC,GAAG,EAAE,CAAC;SACvH;QACD,IAAI,IAAI,CAAC,UAAU,KAAK,MAAM,EAAE;YAC5B,OAAO,wBAAwB,IAAI,CAAC,kBAAkB,iDAAiD,IAAI,CAAC,GAAG,EAAE,CAAC;SACrH;QAED,MAAM,IAAI,KAAK,CAAC,uCAAuC,IAAI,CAAC,UAAU,wBAAwB,CAAC,CAAC;IACpG,CAAC;CACJ;AArBD,kCAqBC;AAED,MAAa,qBAAqB;IAC9B,YACa,mCAAyD,IAAI,EAC7D,yBAA+C,IAAI,EACnD,6BAAmD,IAAI,EACvD,aAAmC,IAAI;QAHvC,qCAAgC,GAAhC,gCAAgC,CAA6B;QAC7D,2BAAsB,GAAtB,sBAAsB,CAA6B;QACnD,+BAA0B,GAA1B,0BAA0B,CAA6B;QACvD,eAAU,GAAV,UAAU,CAA6B;IACjD,CAAC;IAEJ,KAAK;QACD,MAAM,SAAS,GAAG,CAAC,IAAI,CAAC,gCAAgC,EAAE,IAAI,CAAC,sBAAsB,EAAE,IAAI,CAAC,sBAAsB,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QACrI,OAAO,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,OAAO,CAAC,EAAE,IAAI,CAAC,CAAC;IAC1E,CAAC;CACJ;AAZD,sDAYC;AAED,MAAa,eAAe;IAOxB,YAAqB,WAAmB;QAAnB,gBAAW,GAAX,WAAW,CAAQ;QACpC,IAAI,CAAC,aAAa,GAAG,gBAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAE7C,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC;QAClC,IAAI,CAAC,+BAA+B,GAAG,IAAI,CAAC;QAE5C,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC;QACjC,IAAI,CAAC,8BAA8B,GAAG,IAAI,CAAC;IAC/C,CAAC;IAEK,4BAA4B;;YAC9B,MAAM,GAAG,GAAG,IAAA,gBAAM,GAAE,CAAC;YACrB,IACI,CAAC,IAAI,CAAC,qBAAqB;gBAC3B,CAAC,IAAI,CAAC,+BAA+B;gBACrC,IAAI,CAAC,+BAA+B,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,GAAG;gBACnE,CAAC,IAAI,CAAC,qBAAqB,CAAC,KAAK,EAAE,EACrC;gBACE,IAAI,CAAC,qBAAqB,GAAG,MAAM,IAAI,CAAC,mCAAmC,EAAE,CAAC;gBAC9E,IAAI,CAAC,+BAA+B,GAAG,GAAG,CAAC;aAC9C;YAED,OAAO,IAAI,CAAC,qBAAqB,CAAC;QACtC,CAAC;KAAA;IAEK,mCAAmC;;YACrC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,cAAc,EAAE,0BAA0B,CAAC,CAAC;YAC5F,MAAM,KAAK,GAAG,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;YAEzC,OAAO,IAAI,qBAAqB,CAC5B,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,iCAAiC,CAAC,EAChE,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,uBAAuB,CAAC,EACtD,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,2BAA2B,CAAC,EAC1D,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,aAAa,CAAC,CAC/C,CAAC;QACN,CAAC;KAAA;IAED,iBAAiB,CAAC,KAA0B,EAAE,YAAoB;QAC9D,MAAM,MAAM,GAAG,EAAE,CAAC;QAClB,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE;YAC5B,MAAM,QAAQ,GAAG,GAGhB,CAAC;YACF,IAAI,QAAQ,CAAC,gBAAgB,KAAK,YAAY,EAAE;gBAC5C,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC;aAC1D;SACJ;QACD,OAAO,MAAM,CAAC;IAClB,CAAC;IAEK,2BAA2B;;;YAC7B,MAAM,GAAG,GAAG,gBAAM,CAAC,GAAG,EAAE,CAAC;YACzB,IAAI,CAAC,CAAA,MAAA,IAAI,CAAC,oBAAoB,0CAAE,IAAI,EAAE,CAAA,IAAI,CAAC,IAAI,CAAC,8BAA8B,IAAI,IAAI,CAAC,8BAA8B,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,GAAG,EAAE;gBAClJ,IAAI,CAAC,oBAAoB,GAAG,MAAM,IAAI,CAAC,kCAAkC,EAAE,CAAC;gBAC5E,IAAI,CAAC,8BAA8B,GAAG,GAAG,CAAC;gBAE1C,IAAI,IAAI,CAAC,oBAAoB,IAAI,IAAI,EAAE;oBACnC,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;iBAC1D;aACJ;YAED,OAAO,IAAI,CAAC,oBAAoB,CAAC;;KACpC;IAEK,kCAAkC;;YACpC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,cAAc,EAAE,2BAA2B,CAAC,CAAC;YAC7F,MAAM,IAAI,GAAG,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YACtD,IAAI,IAAI,CAAC,IAAI,EAAE;gBACX,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;aAC9E;YACD,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,EAAoC,CAAC,oBAAoB,CAAC;QACtF,CAAC;KAAA;IAEK,kBAAkB;;YACpB,OAAO,CAAC,MAAM,IAAI,CAAC,4BAA4B,EAAE,CAAC,CAAC,gCAAgC,CAAC;QACxF,CAAC;KAAA;IAEK,yBAAyB;;YAC3B,OAAO,CAAC,MAAM,IAAI,CAAC,4BAA4B,EAAE,CAAC,CAAC,sBAAsB,CAAC;QAC9E,CAAC;KAAA;IAEK,6BAA6B;;YAC/B,OAAO,CAAC,MAAM,IAAI,CAAC,4BAA4B,EAAE,CAAC,CAAC,0BAA0B,CAAC;QAClF,CAAC;KAAA;IAEK,aAAa;;YACf,OAAO,CAAC,MAAM,IAAI,CAAC,4BAA4B,EAAE,CAAC,CAAC,UAAU,CAAC;QAClE,CAAC;KAAA;IAEK,uBAAuB;;YACzB,OAAO,IAAI,CAAC,2BAA2B,EAAE,CAAC;QAC9C,CAAC;KAAA;CACJ;AApGD,0CAoGC;AAED,kBAAe,eAAe,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export declare const sleep: (ms: number) => Promise<void>;
|
|
2
|
+
export declare class ExponentialRetry {
|
|
3
|
+
attemptCount: number;
|
|
4
|
+
sleepBaseSecs: number;
|
|
5
|
+
maxJitterSecs: number;
|
|
6
|
+
currentAttempt: number;
|
|
7
|
+
constructor(attemptCount: number, sleepBaseSecs: number, maxJitterSecs: number);
|
|
8
|
+
backoff(): Promise<void>;
|
|
9
|
+
shouldTry(): boolean;
|
|
10
|
+
}
|
package/source/retry.js
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// Copyright (c) Microsoft Corporation.
|
|
3
|
+
// Licensed under the MIT License.
|
|
4
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
5
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
6
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
7
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
8
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
9
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
10
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
11
|
+
});
|
|
12
|
+
};
|
|
13
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
|
+
exports.ExponentialRetry = exports.sleep = void 0;
|
|
15
|
+
const sleep = (ms) => {
|
|
16
|
+
return new Promise((resolve) => {
|
|
17
|
+
setTimeout(resolve, ms);
|
|
18
|
+
});
|
|
19
|
+
};
|
|
20
|
+
exports.sleep = sleep;
|
|
21
|
+
class ExponentialRetry {
|
|
22
|
+
constructor(attemptCount, sleepBaseSecs, maxJitterSecs) {
|
|
23
|
+
this.attemptCount = attemptCount;
|
|
24
|
+
this.sleepBaseSecs = sleepBaseSecs;
|
|
25
|
+
this.maxJitterSecs = maxJitterSecs;
|
|
26
|
+
this.currentAttempt = 0;
|
|
27
|
+
}
|
|
28
|
+
backoff() {
|
|
29
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
30
|
+
if (!this.shouldTry()) {
|
|
31
|
+
throw new Error("Max retries exceeded");
|
|
32
|
+
}
|
|
33
|
+
const base = this.sleepBaseSecs * Math.pow(2, this.currentAttempt);
|
|
34
|
+
const jitter = Math.floor(this.maxJitterSecs * Math.random());
|
|
35
|
+
yield (0, exports.sleep)(1000 * (base + jitter));
|
|
36
|
+
this.currentAttempt++;
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
shouldTry() {
|
|
40
|
+
return this.currentAttempt < this.attemptCount;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
exports.ExponentialRetry = ExponentialRetry;
|
|
44
|
+
//# sourceMappingURL=retry.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"retry.js","sourceRoot":"","sources":["retry.ts"],"names":[],"mappings":";AAAA,uCAAuC;AACvC,kCAAkC;;;;;;;;;;;;AAE3B,MAAM,KAAK,GAAG,CAAC,EAAU,EAAiB,EAAE;IAC/C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC3B,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAC5B,CAAC,CAAC,CAAC;AACP,CAAC,CAAC;AAJW,QAAA,KAAK,SAIhB;AAEF,MAAa,gBAAgB;IAGzB,YAAmB,YAAoB,EAAS,aAAqB,EAAS,aAAqB;QAAhF,iBAAY,GAAZ,YAAY,CAAQ;QAAS,kBAAa,GAAb,aAAa,CAAQ;QAAS,kBAAa,GAAb,aAAa,CAAQ;QAC/F,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;IAC5B,CAAC;IAEY,OAAO;;YAChB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE;gBACnB,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;aAC3C;YAED,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;YACnE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YAC9D,MAAM,IAAA,aAAK,EAAC,IAAI,GAAG,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC;YACpC,IAAI,CAAC,cAAc,EAAE,CAAC;QAC1B,CAAC;KAAA;IAEM,SAAS;QACZ,OAAO,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,YAAY,CAAC;IACnD,CAAC;CACJ;AArBD,4CAqBC"}
|
package/source/status.js
CHANGED
|
@@ -3,13 +3,11 @@
|
|
|
3
3
|
// Licensed under the MIT License.
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
5
|
exports.KustoIngestStatusQueues = exports.StatusMessage = void 0;
|
|
6
|
+
/* eslint-disable max-classes-per-file -- We want all the Status related classes in this file */
|
|
6
7
|
const statusQ_1 = require("./statusQ");
|
|
7
8
|
class StatusMessage {
|
|
8
9
|
constructor(raw, obj, extraProps) {
|
|
9
|
-
let props = [
|
|
10
|
-
"OperationId", "Database", "Table",
|
|
11
|
-
"IngestionSourceId", "IngestionSourcePath", "RootActivityId"
|
|
12
|
-
];
|
|
10
|
+
let props = ["OperationId", "Database", "Table", "IngestionSourceId", "IngestionSourcePath", "RootActivityId"];
|
|
13
11
|
if (extraProps && extraProps.length > 0) {
|
|
14
12
|
props = props.concat(extraProps);
|
|
15
13
|
}
|
|
@@ -22,27 +20,18 @@ class StatusMessage {
|
|
|
22
20
|
exports.StatusMessage = StatusMessage;
|
|
23
21
|
class SuccessMessage extends StatusMessage {
|
|
24
22
|
constructor(raw, obj) {
|
|
25
|
-
super(raw, obj, [
|
|
26
|
-
"SucceededOn"
|
|
27
|
-
]);
|
|
23
|
+
super(raw, obj, ["SucceededOn"]);
|
|
28
24
|
}
|
|
29
25
|
}
|
|
30
26
|
class FailureMessage extends StatusMessage {
|
|
31
27
|
constructor(raw, obj) {
|
|
32
|
-
super(raw, obj, [
|
|
33
|
-
"FailedOn",
|
|
34
|
-
"Details",
|
|
35
|
-
"ErrorCode",
|
|
36
|
-
"FailureStatus",
|
|
37
|
-
"OriginatesFromUpdatePolicy",
|
|
38
|
-
"ShouldRetry"
|
|
39
|
-
]);
|
|
28
|
+
super(raw, obj, ["FailedOn", "Details", "ErrorCode", "FailureStatus", "OriginatesFromUpdatePolicy", "ShouldRetry"]);
|
|
40
29
|
}
|
|
41
30
|
}
|
|
42
31
|
class KustoIngestStatusQueues {
|
|
43
32
|
constructor(kustoIngestClient) {
|
|
44
|
-
this.success = new statusQ_1.StatusQueue(() => kustoIngestClient.resourceManager.getSuccessfulIngestionsQueues().then(r => r), SuccessMessage);
|
|
45
|
-
this.failure = new statusQ_1.StatusQueue(() => kustoIngestClient.resourceManager.getFailedIngestionsQueues().then(r => r), FailureMessage);
|
|
33
|
+
this.success = new statusQ_1.StatusQueue(() => kustoIngestClient.resourceManager.getSuccessfulIngestionsQueues().then((r) => r), SuccessMessage);
|
|
34
|
+
this.failure = new statusQ_1.StatusQueue(() => kustoIngestClient.resourceManager.getFailedIngestionsQueues().then((r) => r), FailureMessage);
|
|
46
35
|
}
|
|
47
36
|
}
|
|
48
37
|
exports.KustoIngestStatusQueues = KustoIngestStatusQueues;
|
package/source/status.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"status.js","sourceRoot":"","sources":["status.ts"],"names":[],"mappings":";AAAA,uCAAuC;AACvC,kCAAkC;;;AAElC,
|
|
1
|
+
{"version":3,"file":"status.js","sourceRoot":"","sources":["status.ts"],"names":[],"mappings":";AAAA,uCAAuC;AACvC,kCAAkC;;;AAElC,gGAAgG;AAEhG,uCAAwC;AAIxC,MAAa,aAAa;IAStB,YAAY,GAAQ,EAAE,GAAQ,EAAE,UAA2B;QACvD,IAAI,KAAK,GAAa,CAAC,aAAa,EAAE,UAAU,EAAE,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,gBAAgB,CAAC,CAAC;QAEzH,IAAI,UAAU,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;YACrC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;SACpC;QAED,MAAM,IAAI,GAAwB,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;QAEhF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;YACtB,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;SAC3B;IACL,CAAC;CACJ;AAtBD,sCAsBC;AAED,MAAM,cAAe,SAAQ,aAAa;IAGtC,YAAY,GAAQ,EAAE,GAAQ;QAC1B,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC;IACrC,CAAC;CACJ;AAED,MAAM,cAAe,SAAQ,aAAa;IAOtC,YAAY,GAAQ,EAAE,GAAQ;QAC1B,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,eAAe,EAAE,4BAA4B,EAAE,aAAa,CAAC,CAAC,CAAC;IACxH,CAAC;CACJ;AAED,MAAa,uBAAuB;IAGhC,YAAY,iBAAoC;QAC5C,IAAI,CAAC,OAAO,GAAG,IAAI,qBAAW,CAAC,GAAG,EAAE,CAAC,iBAAiB,CAAC,eAAe,CAAC,6BAA6B,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAkB,CAAC,EAAE,cAAc,CAAC,CAAC;QACxJ,IAAI,CAAC,OAAO,GAAG,IAAI,qBAAW,CAAC,GAAG,EAAE,CAAC,iBAAiB,CAAC,eAAe,CAAC,yBAAyB,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAkB,CAAC,EAAE,cAAc,CAAC,CAAC;IACxJ,CAAC;CACJ;AAPD,0DAOC;AAED,kBAAe,uBAAuB,CAAC"}
|
package/source/statusQ.js
CHANGED
|
@@ -12,6 +12,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
12
12
|
};
|
|
13
13
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
14
|
exports.StatusQueue = void 0;
|
|
15
|
+
/* eslint-disable max-classes-per-file -- QueueDetails is a very small class, so we don't need it in a different file */
|
|
15
16
|
const storage_queue_1 = require("@azure/storage-queue");
|
|
16
17
|
class QueueDetails {
|
|
17
18
|
constructor(name, service) {
|
|
@@ -19,7 +20,7 @@ class QueueDetails {
|
|
|
19
20
|
this.service = service;
|
|
20
21
|
}
|
|
21
22
|
}
|
|
22
|
-
|
|
23
|
+
const shuffle = (a) => {
|
|
23
24
|
for (let i = a.length - 1; i > 0; i--) {
|
|
24
25
|
const j = Math.floor(Math.random() * (i + 1));
|
|
25
26
|
const temp = a[j];
|
|
@@ -27,14 +28,14 @@ function shuffle(a) {
|
|
|
27
28
|
a[i] = temp;
|
|
28
29
|
}
|
|
29
30
|
return a;
|
|
30
|
-
}
|
|
31
|
+
};
|
|
31
32
|
class StatusQueue {
|
|
32
33
|
constructor(getQueuesFunc, messageCls) {
|
|
33
34
|
this.getQueuesFunc = getQueuesFunc;
|
|
34
35
|
this.messageCls = messageCls;
|
|
35
36
|
}
|
|
36
37
|
_getQServices(queuesDetails) {
|
|
37
|
-
return queuesDetails.map(
|
|
38
|
+
return queuesDetails.map((q) => {
|
|
38
39
|
const sasConnectionString = q.getSASConnectionString();
|
|
39
40
|
if (!sasConnectionString) {
|
|
40
41
|
throw new Error("Empty or null connection string");
|
|
@@ -67,7 +68,7 @@ class StatusQueue {
|
|
|
67
68
|
for (const m of messages) {
|
|
68
69
|
if (m && Object.keys(m).length > 0) {
|
|
69
70
|
result.push(options && options.raw ? m : this.deserializeMessage(m));
|
|
70
|
-
if (result.length
|
|
71
|
+
if (result.length === n) {
|
|
71
72
|
return { done: true, nonEmptyQs, result };
|
|
72
73
|
}
|
|
73
74
|
}
|
|
@@ -102,9 +103,9 @@ class StatusQueue {
|
|
|
102
103
|
if (m && Object.keys(m).length > 0) {
|
|
103
104
|
result.push(options && options.raw ? m : this.deserializeMessage(m));
|
|
104
105
|
if (!(options && !options.remove)) {
|
|
105
|
-
q.service.deleteMessage(m.messageId, m.popReceipt);
|
|
106
|
+
yield q.service.deleteMessage(m.messageId, m.popReceipt);
|
|
106
107
|
}
|
|
107
|
-
if (result.length
|
|
108
|
+
if (result.length === n) {
|
|
108
109
|
return { done: true, nonEmptyQs, result };
|
|
109
110
|
}
|
|
110
111
|
}
|
package/source/statusQ.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"statusQ.js","sourceRoot":"","sources":["statusQ.ts"],"names":[],"mappings":";AAAA,uCAAuC;AACvC,kCAAkC;;;;;;;;;;;;AAElC,
|
|
1
|
+
{"version":3,"file":"statusQ.js","sourceRoot":"","sources":["statusQ.ts"],"names":[],"mappings":";AAAA,uCAAuC;AACvC,kCAAkC;;;;;;;;;;;;AAElC,wHAAwH;AAExH,wDAAsE;AAItE,MAAM,YAAY;IACd,YAAqB,IAAY,EAAW,OAAoB;QAA3C,SAAI,GAAJ,IAAI,CAAQ;QAAW,YAAO,GAAP,OAAO,CAAa;IAAG,CAAC;CACvE;AAED,MAAM,OAAO,GAAG,CAAI,CAAM,EAAO,EAAE;IAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE;QACnC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC9C,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACZ,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;KACf;IAED,OAAO,CAAC,CAAC;AACb,CAAC,CAAC;AAaF,MAAa,WAAW;IACpB,YAAqB,aAA2C,EAAW,UAAgC;QAAtF,kBAAa,GAAb,aAAa,CAA8B;QAAW,eAAU,GAAV,UAAU,CAAsB;IAAG,CAAC;IAE/G,aAAa,CAAC,aAA4B;QACtC,OAAO,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YAC3B,MAAM,mBAAmB,GAAG,CAAC,CAAC,sBAAsB,EAAE,CAAC;YACvD,IAAI,CAAC,mBAAmB,EAAE;gBACtB,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;aACtD;YACD,OAAO,IAAI,YAAY,CAAC,CAAC,CAAC,UAAU,EAAE,IAAI,2BAAW,CAAC,mBAAmB,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QAC9F,CAAC,CAAC,CAAC;IACP,CAAC;IAEK,OAAO;;YACT,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;YACjD,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC;QAC1C,CAAC;KAAA;IAED,aAAa,CAAC,OAAe;QACzB,OAAO,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC5D,CAAC;IAED,kBAAkB,CAAC,CAAoB;QACnC,OAAO,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;IAC9E,CAAC;IAEK,KAAK,CAAC,EAAkB,EAAE,CAAS,EAAE,OAA0B;;YACjE,MAAM,MAAM,GAAc,EAAE,CAAC;YAC7B,MAAM,UAAU,GAAmB,EAAE,CAAC;YAEtC,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE;gBAChB,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;gBAChD,MAAM,QAAQ,GAAG,QAAQ,CAAC,kBAAkB,CAAC;gBAE7C,IAAI,QAAQ,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE;oBACjC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;iBACtB;gBAED,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE;oBACtB,IAAI,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE;wBAChC,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC;wBAErE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE;4BACrB,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC;yBAC7C;qBACJ;iBACJ;aACJ;YACD,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC;QACjE,CAAC;KAAA;IAEK,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,UAA6B,IAAI;;YAC/C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC1C,MAAM,SAAS,GAAmB,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;YACtE,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC;YAExF,sDAAsD;YACtD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;YAE3D,IAAI,OAAO,CAAC,IAAI,EAAE;gBACd,OAAO,OAAO,CAAC,MAAM,CAAC;aACzB;YACD,MAAM,kBAAkB,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;YAErD,2FAA2F;YAC3F,OAAO,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,kBAAkB,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;QACtF,CAAC;KAAA;IAEK,IAAI,CACN,EAAkB,EAClB,CAAS,EACT,OAAyB;;YAEzB,MAAM,UAAU,GAAU,EAAE,CAAC;YAC7B,MAAM,MAAM,GAAG,EAAE,CAAC;YAElB,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE;gBAChB,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC,CAAC;gBACvE,MAAM,QAAQ,GAAG,QAAQ,CAAC,oBAAoB,CAAC;gBAC/C,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE;oBACtB,IAAI,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE;wBAChC,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC;wBAErE,IAAI,CAAC,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;4BAC/B,MAAM,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC;yBAC5D;wBACD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE;4BACrB,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC;yBAC7C;qBACJ;iBACJ;aACJ;YAED,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC;QACjE,CAAC;KAAA;IAEK,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,UAA4B,IAAI;;;YAC7C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;YACtD,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC;YAExF,sDAAsD;YACtD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;YAC1D,IAAI,OAAO,CAAC,IAAI,EAAE;gBACd,OAAO,OAAO,CAAC,MAAM,CAAC;aACzB;YAED,MAAM,iBAAiB,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;YAEpD,2FAA2F;YAC3F,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,MAAA,OAAO,CAAC,MAAM,CAAC,UAAU,mCAAI,EAAE,EAAE,iBAAiB,EAAE,OAAO,CAAC,CAAC;YAC3F,OAAO,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;;KAC9C;CACJ;AAjHD,kCAiHC"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import { FileDescriptor, StreamDescriptor } from "./descriptors";
|
|
3
|
+
import { Readable } from "stream";
|
|
4
|
+
export declare const fileToStream: (file: FileDescriptor | string) => StreamDescriptor;
|
|
5
|
+
export declare const mergeStreams: (...streams: Readable[]) => Readable;
|
|
6
|
+
export declare const tryStreamToArray: (stream: Readable, maxBufferSize: number) => Promise<Buffer | Readable>;
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// Copyright (c) Microsoft Corporation.
|
|
3
|
+
// Licensed under the MIT License.
|
|
4
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
5
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
6
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
7
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
8
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
9
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
10
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
11
|
+
});
|
|
12
|
+
};
|
|
13
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
14
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
exports.tryStreamToArray = exports.mergeStreams = exports.fileToStream = void 0;
|
|
18
|
+
const descriptors_1 = require("./descriptors");
|
|
19
|
+
const fs_1 = __importDefault(require("fs"));
|
|
20
|
+
const stream_1 = require("stream");
|
|
21
|
+
const stream_array_1 = __importDefault(require("stream-array"));
|
|
22
|
+
const fileToStream = (file) => {
|
|
23
|
+
const fileDescriptor = file instanceof descriptors_1.FileDescriptor ? file : new descriptors_1.FileDescriptor(file);
|
|
24
|
+
const streamFs = fs_1.default.createReadStream(fileDescriptor.filePath);
|
|
25
|
+
const compressionType = fileDescriptor.zipped ? descriptors_1.CompressionType.GZIP : descriptors_1.CompressionType.None;
|
|
26
|
+
return new descriptors_1.StreamDescriptor(streamFs, fileDescriptor.sourceId, compressionType);
|
|
27
|
+
};
|
|
28
|
+
exports.fileToStream = fileToStream;
|
|
29
|
+
const mergeStreams = (...streams) => {
|
|
30
|
+
let pass = new stream_1.PassThrough();
|
|
31
|
+
let waiting = streams.length;
|
|
32
|
+
for (const stream of streams) {
|
|
33
|
+
pass = stream.pipe(pass, { end: false });
|
|
34
|
+
stream.once("end", () => --waiting === 0 && pass.emit("end"));
|
|
35
|
+
}
|
|
36
|
+
return pass;
|
|
37
|
+
};
|
|
38
|
+
exports.mergeStreams = mergeStreams;
|
|
39
|
+
const tryStreamToArray = (stream, maxBufferSize) => __awaiter(void 0, void 0, void 0, function* () {
|
|
40
|
+
return yield new Promise((resolve, reject) => {
|
|
41
|
+
const result = [];
|
|
42
|
+
const endListener = () => resolve(Buffer.concat(result));
|
|
43
|
+
const dataHandler = (chunk) => {
|
|
44
|
+
try {
|
|
45
|
+
result.push(chunk);
|
|
46
|
+
if (result.reduce((sum, b) => sum + b.length, 0) > maxBufferSize) {
|
|
47
|
+
stream.removeListener("data", dataHandler);
|
|
48
|
+
stream.removeListener("end", endListener);
|
|
49
|
+
resolve((0, exports.mergeStreams)((0, stream_array_1.default)(result), stream));
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
catch (e) {
|
|
53
|
+
reject(e);
|
|
54
|
+
}
|
|
55
|
+
};
|
|
56
|
+
stream.on("data", dataHandler);
|
|
57
|
+
stream.on("end", endListener);
|
|
58
|
+
});
|
|
59
|
+
});
|
|
60
|
+
exports.tryStreamToArray = tryStreamToArray;
|
|
61
|
+
//# sourceMappingURL=streamUtils.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"streamUtils.js","sourceRoot":"","sources":["streamUtils.ts"],"names":[],"mappings":";AAAA,uCAAuC;AACvC,kCAAkC;;;;;;;;;;;;;;;AAElC,+CAAkF;AAClF,4CAAoB;AACpB,mCAA+C;AAC/C,gEAAqC;AAE9B,MAAM,YAAY,GAAG,CAAC,IAA6B,EAAoB,EAAE;IAC5E,MAAM,cAAc,GAAG,IAAI,YAAY,4BAAc,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,4BAAc,CAAC,IAAI,CAAC,CAAC;IACxF,MAAM,QAAQ,GAAG,YAAE,CAAC,gBAAgB,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IAC9D,MAAM,eAAe,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,6BAAe,CAAC,IAAI,CAAC,CAAC,CAAC,6BAAe,CAAC,IAAI,CAAC;IAC5F,OAAO,IAAI,8BAAgB,CAAC,QAAQ,EAAE,cAAc,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;AACpF,CAAC,CAAC;AALW,QAAA,YAAY,gBAKvB;AAEK,MAAM,YAAY,GAAG,CAAC,GAAG,OAAmB,EAAY,EAAE;IAC7D,IAAI,IAAI,GAAG,IAAI,oBAAW,EAAE,CAAC;IAC7B,IAAI,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC;IAC7B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;QAC1B,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;QACzC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;KACjE;IACD,OAAO,IAAI,CAAC;AAChB,CAAC,CAAC;AARW,QAAA,YAAY,gBAQvB;AAEK,MAAM,gBAAgB,GAAG,CAAO,MAAgB,EAAE,aAAqB,EAA8B,EAAE;IAC1G,OAAO,MAAM,IAAI,OAAO,CAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC5D,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,WAAW,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QACzD,MAAM,WAAW,GAAG,CAAC,KAAa,EAAE,EAAE;YAClC,IAAI;gBACA,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACnB,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,aAAa,EAAE;oBAC9D,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;oBAC3C,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;oBAC1C,OAAO,CAAC,IAAA,oBAAY,EAAC,IAAA,sBAAS,EAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;iBACpD;aACJ;YAAC,OAAO,CAAC,EAAE;gBACR,MAAM,CAAC,CAAC,CAAC,CAAC;aACb;QACL,CAAC,CAAC;QACF,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAC/B,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;AACP,CAAC,CAAA,CAAC;AAnBW,QAAA,gBAAgB,oBAmB3B"}
|
|
@@ -1,15 +1,14 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
2
|
import IngestionProperties from "./ingestionProperties";
|
|
3
3
|
import { FileDescriptor, StreamDescriptor } from "./descriptors";
|
|
4
|
-
import fs from "fs";
|
|
5
4
|
import { AbstractKustoClient } from "./abstractKustoClient";
|
|
6
5
|
import { KustoConnectionStringBuilder } from "azure-kusto-data";
|
|
7
6
|
import { KustoResponseDataSet } from "azure-kusto-data/source/response";
|
|
7
|
+
import { Readable } from "stream";
|
|
8
8
|
declare class KustoStreamingIngestClient extends AbstractKustoClient {
|
|
9
9
|
private kustoClient;
|
|
10
|
-
private _mapping_required_formats;
|
|
11
10
|
constructor(kcsb: string | KustoConnectionStringBuilder, defaultProps?: IngestionProperties | null);
|
|
12
|
-
ingestFromStream(stream: StreamDescriptor |
|
|
11
|
+
ingestFromStream(stream: StreamDescriptor | Readable, ingestionProperties: IngestionProperties, clientRequestId?: string): Promise<any>;
|
|
13
12
|
ingestFromFile(file: FileDescriptor | string, ingestionProperties: IngestionProperties): Promise<KustoResponseDataSet>;
|
|
14
13
|
}
|
|
15
14
|
export default KustoStreamingIngestClient;
|
|
@@ -14,40 +14,31 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
14
14
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
const ingestionProperties_1 = require("./ingestionProperties");
|
|
18
17
|
const descriptors_1 = require("./descriptors");
|
|
19
18
|
const zlib_1 = __importDefault(require("zlib"));
|
|
20
|
-
const fs_1 = __importDefault(require("fs"));
|
|
21
19
|
const abstractKustoClient_1 = require("./abstractKustoClient");
|
|
22
20
|
const azure_kusto_data_1 = require("azure-kusto-data");
|
|
21
|
+
const streamUtils_1 = require("./streamUtils");
|
|
23
22
|
class KustoStreamingIngestClient extends abstractKustoClient_1.AbstractKustoClient {
|
|
24
23
|
constructor(kcsb, defaultProps = null) {
|
|
25
24
|
super(defaultProps);
|
|
26
25
|
this.kustoClient = new azure_kusto_data_1.Client(kcsb);
|
|
27
|
-
this._mapping_required_formats = Object.freeze([ingestionProperties_1.DataFormat.JSON, ingestionProperties_1.DataFormat.SINGLEJSON, ingestionProperties_1.DataFormat.AVRO, ingestionProperties_1.DataFormat.ORC]);
|
|
28
26
|
}
|
|
29
|
-
ingestFromStream(stream, ingestionProperties) {
|
|
27
|
+
ingestFromStream(stream, ingestionProperties, clientRequestId) {
|
|
30
28
|
var _a;
|
|
31
29
|
return __awaiter(this, void 0, void 0, function* () {
|
|
32
30
|
const props = this._mergeProps(ingestionProperties);
|
|
33
31
|
props.validate();
|
|
34
32
|
const descriptor = stream instanceof descriptors_1.StreamDescriptor ? stream : new descriptors_1.StreamDescriptor(stream);
|
|
35
|
-
const compressedStream = descriptor.compressionType
|
|
36
|
-
|
|
37
|
-
throw new Error(`Mapping reference required for format ${props.foramt}.`);
|
|
38
|
-
}
|
|
39
|
-
return this.kustoClient.executeStreamingIngest(props.database, props.table, compressedStream, props.format, (_a = props.ingestionMappingReference) !== null && _a !== void 0 ? _a : null);
|
|
33
|
+
const compressedStream = descriptor.compressionType === descriptors_1.CompressionType.None ? descriptor.stream.pipe(zlib_1.default.createGzip()) : descriptor.stream;
|
|
34
|
+
return this.kustoClient.executeStreamingIngest(props.database, props.table, compressedStream, props.format, (_a = props.ingestionMappingReference) !== null && _a !== void 0 ? _a : null, clientRequestId);
|
|
40
35
|
});
|
|
41
36
|
}
|
|
42
37
|
ingestFromFile(file, ingestionProperties) {
|
|
43
38
|
return __awaiter(this, void 0, void 0, function* () {
|
|
44
39
|
const props = this._mergeProps(ingestionProperties);
|
|
45
40
|
props.validate();
|
|
46
|
-
|
|
47
|
-
const stream = fs_1.default.createReadStream(fileDescriptor.filePath);
|
|
48
|
-
const compressionType = fileDescriptor.zipped ? descriptors_1.CompressionType.GZIP : descriptors_1.CompressionType.None;
|
|
49
|
-
const streamDescriptor = new descriptors_1.StreamDescriptor(stream, fileDescriptor.sourceId, compressionType);
|
|
50
|
-
return this.ingestFromStream(streamDescriptor, ingestionProperties);
|
|
41
|
+
return this.ingestFromStream((0, streamUtils_1.fileToStream)(file), ingestionProperties);
|
|
51
42
|
});
|
|
52
43
|
}
|
|
53
44
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streamingIngestClient.js","sourceRoot":"","sources":["streamingIngestClient.ts"],"names":[],"mappings":";AAAA,uCAAuC;AACvC,kCAAkC;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"streamingIngestClient.js","sourceRoot":"","sources":["streamingIngestClient.ts"],"names":[],"mappings":";AAAA,uCAAuC;AACvC,kCAAkC;;;;;;;;;;;;;;AAIlC,+CAAkF;AAClF,gDAAwB;AACxB,+DAA4D;AAC5D,uDAAuF;AAEvF,+CAA6C;AAG7C,MAAM,0BAA2B,SAAQ,yCAAmB;IAGxD,YAAY,IAA2C,EAAE,eAA2C,IAAI;QACpG,KAAK,CAAC,YAAY,CAAC,CAAC;QACpB,IAAI,CAAC,WAAW,GAAG,IAAI,yBAAW,CAAC,IAAI,CAAC,CAAC;IAC7C,CAAC;IAEK,gBAAgB,CAAC,MAAmC,EAAE,mBAAwC,EAAE,eAAwB;;;YAC1H,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,mBAAmB,CAAC,CAAC;YACpD,KAAK,CAAC,QAAQ,EAAE,CAAC;YACjB,MAAM,UAAU,GAAqB,MAAM,YAAY,8BAAgB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,8BAAgB,CAAC,MAAM,CAAC,CAAC;YAEhH,MAAM,gBAAgB,GAAG,UAAU,CAAC,eAAe,KAAK,6BAAe,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,cAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;YAC7I,OAAO,IAAI,CAAC,WAAW,CAAC,sBAAsB,CAC1C,KAAK,CAAC,QAAkB,EACxB,KAAK,CAAC,KAAe,EACrB,gBAAgB,EAChB,KAAK,CAAC,MAAM,EACZ,MAAA,KAAK,CAAC,yBAAyB,mCAAI,IAAI,EACvC,eAAe,CAClB,CAAC;;KACL;IAEK,cAAc,CAAC,IAA6B,EAAE,mBAAwC;;YACxF,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,mBAAmB,CAAC,CAAC;YACpD,KAAK,CAAC,QAAQ,EAAE,CAAC;YACjB,OAAO,IAAI,CAAC,gBAAgB,CAAC,IAAA,0BAAY,EAAC,IAAI,CAAC,EAAE,mBAAmB,CAAC,CAAC;QAC1E,CAAC;KAAA;CACJ;AAED,kBAAe,0BAA0B,CAAC"}
|
package/tsconfig.json
CHANGED
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
{
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"compilerOptions": {
|
|
3
|
+
"target": "es6",
|
|
4
|
+
"module": "commonjs",
|
|
5
|
+
"esModuleInterop": true,
|
|
6
|
+
"strict": true,
|
|
7
|
+
"incremental": true,
|
|
8
|
+
"sourceMap": true,
|
|
9
|
+
"resolveJsonModule": true,
|
|
10
|
+
"allowJs": false,
|
|
11
|
+
"declaration": true,
|
|
12
|
+
"noUnusedParameters": true,
|
|
13
|
+
"noUnusedLocals": true
|
|
14
|
+
},
|
|
15
|
+
"include": ["source/**/*.ts", "test/**/*.ts", "index.ts"]
|
|
16
|
+
}
|