azure-kusto-ingest 6.0.2 → 7.0.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist-esm/src/abstractKustoClient.js +8 -15
- package/dist-esm/src/abstractKustoClient.js.map +1 -1
- package/dist-esm/src/columnMappings.js +22 -34
- package/dist-esm/src/columnMappings.js.map +1 -1
- package/dist-esm/src/descriptors.js +15 -27
- package/dist-esm/src/descriptors.js.map +1 -1
- package/dist-esm/src/errors.js +1 -5
- package/dist-esm/src/errors.js.map +1 -1
- package/dist-esm/src/fileDescriptor.browser.js +9 -16
- package/dist-esm/src/fileDescriptor.browser.js.map +1 -1
- package/dist-esm/src/fileDescriptor.js +19 -26
- package/dist-esm/src/fileDescriptor.js.map +1 -1
- package/dist-esm/src/index.js +38 -73
- package/dist-esm/src/index.js.map +1 -1
- package/dist-esm/src/ingestClient.browser.js +11 -15
- package/dist-esm/src/ingestClient.browser.js.map +1 -1
- package/dist-esm/src/ingestClient.js +11 -15
- package/dist-esm/src/ingestClient.js.map +1 -1
- package/dist-esm/src/ingestClientBase.js +25 -32
- package/dist-esm/src/ingestClientBase.js.map +1 -1
- package/dist-esm/src/ingestionBlobInfo.js +4 -8
- package/dist-esm/src/ingestionBlobInfo.js.map +1 -1
- package/dist-esm/src/ingestionProperties.js +29 -36
- package/dist-esm/src/ingestionProperties.js.map +1 -1
- package/dist-esm/src/ingestionResult.js +10 -17
- package/dist-esm/src/ingestionResult.js.map +1 -1
- package/dist-esm/src/managedStreamingIngestClient.js +29 -34
- package/dist-esm/src/managedStreamingIngestClient.js.map +1 -1
- package/dist-esm/src/rankedStorageAccount.js +1 -5
- package/dist-esm/src/rankedStorageAccount.js.map +1 -1
- package/dist-esm/src/rankedStorageAccountSet.js +3 -7
- package/dist-esm/src/rankedStorageAccountSet.js.map +1 -1
- package/dist-esm/src/resourceManager.js +24 -31
- package/dist-esm/src/resourceManager.js.map +1 -1
- package/dist-esm/src/retry.js +3 -8
- package/dist-esm/src/retry.js.map +1 -1
- package/dist-esm/src/status.js +6 -11
- package/dist-esm/src/status.js.map +1 -1
- package/dist-esm/src/statusQ.js +3 -7
- package/dist-esm/src/statusQ.js.map +1 -1
- package/dist-esm/src/streamUtils.browser.js +8 -15
- package/dist-esm/src/streamUtils.browser.js.map +1 -1
- package/dist-esm/src/streamUtils.js +18 -28
- package/dist-esm/src/streamUtils.js.map +1 -1
- package/dist-esm/src/streamingIngestClient.browser.js +9 -11
- package/dist-esm/src/streamingIngestClient.browser.js.map +1 -1
- package/dist-esm/src/streamingIngestClient.js +12 -17
- package/dist-esm/src/streamingIngestClient.js.map +1 -1
- package/dist-esm/src/streamingIngestClientBase.js +7 -11
- package/dist-esm/src/streamingIngestClientBase.js.map +1 -1
- package/package.json +10 -6
- package/types/src/abstractKustoClient.d.ts +2 -2
- package/types/src/columnMappings.d.ts +1 -1
- package/types/src/descriptors.d.ts +2 -2
- package/types/src/fileDescriptor.browser.d.ts +2 -2
- package/types/src/fileDescriptor.d.ts +2 -2
- package/types/src/index.d.ts +13 -13
- package/types/src/ingestClient.browser.d.ts +5 -5
- package/types/src/ingestClient.d.ts +6 -6
- package/types/src/ingestClientBase.d.ts +5 -5
- package/types/src/ingestionBlobInfo.d.ts +3 -3
- package/types/src/ingestionProperties.d.ts +1 -1
- package/types/src/managedStreamingIngestClient.d.ts +6 -6
- package/types/src/rankedStorageAccountSet.d.ts +1 -1
- package/types/src/status.d.ts +2 -2
- package/types/src/statusQ.d.ts +2 -2
- package/types/src/streamUtils.browser.d.ts +3 -3
- package/types/src/streamUtils.d.ts +3 -3
- package/types/src/streamingIngestClient.browser.d.ts +4 -4
- package/types/src/streamingIngestClient.d.ts +5 -5
- package/types/src/streamingIngestClientBase.d.ts +3 -3
|
@@ -1,27 +1,20 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
// Copyright (c) Microsoft Corporation.
|
|
3
2
|
// Licensed under the MIT License.
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
const descriptors_1 = require("./descriptors");
|
|
7
|
-
const fileToStream = async (file) => {
|
|
3
|
+
import { CompressionType, StreamDescriptor } from "./descriptors.js";
|
|
4
|
+
export const fileToStream = async (file) => {
|
|
8
5
|
const streamFs = await file.file.arrayBuffer();
|
|
9
|
-
const compressionType = file.zipped ?
|
|
10
|
-
return new
|
|
6
|
+
const compressionType = file.zipped ? CompressionType.GZIP : CompressionType.None;
|
|
7
|
+
return new StreamDescriptor(streamFs, file.sourceId, compressionType);
|
|
11
8
|
};
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
return await (0, exports.fileToStream)(file);
|
|
9
|
+
export const tryFileToBuffer = async (file) => {
|
|
10
|
+
return await fileToStream(file);
|
|
15
11
|
};
|
|
16
|
-
exports.tryFileToBuffer = tryFileToBuffer;
|
|
17
12
|
// NOT USED
|
|
18
|
-
const tryStreamToArray = async (stream) => {
|
|
13
|
+
export const tryStreamToArray = async (stream) => {
|
|
19
14
|
return Promise.resolve(stream);
|
|
20
15
|
};
|
|
21
|
-
exports.tryStreamToArray = tryStreamToArray;
|
|
22
16
|
// NOT USED
|
|
23
|
-
const readableToStream = (stream) => {
|
|
17
|
+
export const readableToStream = (stream) => {
|
|
24
18
|
return stream;
|
|
25
19
|
};
|
|
26
|
-
exports.readableToStream = readableToStream;
|
|
27
20
|
//# sourceMappingURL=streamUtils.browser.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streamUtils.browser.js","sourceRoot":"","sources":["../../src/streamUtils.browser.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"streamUtils.browser.js","sourceRoot":"","sources":["../../src/streamUtils.browser.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAElC,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAGrE,MAAM,CAAC,MAAM,YAAY,GAAG,KAAK,EAAE,IAAoB,EAA6B,EAAE;IAClF,MAAM,QAAQ,GAAG,MAAO,IAAI,CAAC,IAAa,CAAC,WAAW,EAAE,CAAC;IACzD,MAAM,eAAe,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC;IAClF,OAAO,IAAI,gBAAgB,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;AAC1E,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,EAAE,IAAoB,EAA6B,EAAE;IACrF,OAAO,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC;AACpC,CAAC,CAAC;AAEF,WAAW;AACX,MAAM,CAAC,MAAM,gBAAgB,GAAG,KAAK,EAAE,MAAW,EAAmB,EAAE;IACnE,OAAO,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;AACnC,CAAC,CAAC;AAEF,WAAW;AACX,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,MAAmB,EAAe,EAAE;IACjE,OAAO,MAAM,CAAC;AAClB,CAAC,CAAC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport { CompressionType, StreamDescriptor } from \"./descriptors.js\";\nimport { FileDescriptor } from \"./fileDescriptor.browser.js\";\n\nexport const fileToStream = async (file: FileDescriptor): Promise<StreamDescriptor> => {\n const streamFs = await (file.file as Blob).arrayBuffer();\n const compressionType = file.zipped ? CompressionType.GZIP : CompressionType.None;\n return new StreamDescriptor(streamFs, file.sourceId, compressionType);\n};\n\nexport const tryFileToBuffer = async (file: FileDescriptor): Promise<StreamDescriptor> => {\n return await fileToStream(file);\n};\n\n// NOT USED\nexport const tryStreamToArray = async (stream: any): Promise<Buffer> => {\n return Promise.resolve(stream);\n};\n\n// NOT USED\nexport const readableToStream = (stream: ArrayBuffer): ArrayBuffer => {\n return stream;\n};\n"]}
|
|
@@ -1,36 +1,28 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
// Copyright (c) Microsoft Corporation.
|
|
3
2
|
// Licensed under the MIT License.
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
exports.readableToStream = exports.tryStreamToArray = exports.tryFileToBuffer = exports.fileToStream = void 0;
|
|
9
|
-
const fs_1 = __importDefault(require("fs"));
|
|
10
|
-
const stream_1 = require("stream");
|
|
11
|
-
const stream_array_1 = __importDefault(require("stream-array"));
|
|
12
|
-
const descriptors_1 = require("./descriptors");
|
|
3
|
+
import fs from "fs";
|
|
4
|
+
import { PassThrough } from "stream";
|
|
5
|
+
import streamify from "stream-array";
|
|
6
|
+
import { CompressionType, StreamDescriptor } from "./descriptors.js";
|
|
13
7
|
// Returns fs.ReadStream for node and NodeJS.ReadableStream in browser
|
|
14
|
-
const fileToStream = (fileDescriptor) => {
|
|
15
|
-
const streamFs =
|
|
16
|
-
const compressionType = fileDescriptor.zipped ?
|
|
17
|
-
return Promise.resolve(new
|
|
8
|
+
export const fileToStream = (fileDescriptor) => {
|
|
9
|
+
const streamFs = fs.createReadStream(fileDescriptor.file);
|
|
10
|
+
const compressionType = fileDescriptor.zipped ? CompressionType.GZIP : CompressionType.None;
|
|
11
|
+
return Promise.resolve(new StreamDescriptor(streamFs, fileDescriptor.sourceId, compressionType));
|
|
18
12
|
};
|
|
19
|
-
exports.fileToStream = fileToStream;
|
|
20
13
|
// Used in managed streaming where we buffer the file to memory for retries
|
|
21
|
-
const tryFileToBuffer = async (fileDescriptor) => {
|
|
14
|
+
export const tryFileToBuffer = async (fileDescriptor) => {
|
|
22
15
|
try {
|
|
23
|
-
const buffer =
|
|
24
|
-
const compressionType = fileDescriptor.zipped ?
|
|
25
|
-
return new
|
|
16
|
+
const buffer = fs.readFileSync(fileDescriptor.file);
|
|
17
|
+
const compressionType = fileDescriptor.zipped ? CompressionType.GZIP : CompressionType.None;
|
|
18
|
+
return new StreamDescriptor(buffer, fileDescriptor.sourceId, compressionType);
|
|
26
19
|
}
|
|
27
20
|
catch (error) {
|
|
28
|
-
return await
|
|
21
|
+
return await fileToStream(fileDescriptor);
|
|
29
22
|
}
|
|
30
23
|
};
|
|
31
|
-
exports.tryFileToBuffer = tryFileToBuffer;
|
|
32
24
|
const mergeStreams = (...streams) => {
|
|
33
|
-
let pass = new
|
|
25
|
+
let pass = new PassThrough();
|
|
34
26
|
let waiting = streams.length;
|
|
35
27
|
for (const stream of streams) {
|
|
36
28
|
pass = stream.pipe(pass, { end: false });
|
|
@@ -38,7 +30,7 @@ const mergeStreams = (...streams) => {
|
|
|
38
30
|
}
|
|
39
31
|
return pass;
|
|
40
32
|
};
|
|
41
|
-
const tryStreamToArray = async (stream, maxBufferSize) => {
|
|
33
|
+
export const tryStreamToArray = async (stream, maxBufferSize) => {
|
|
42
34
|
if (stream instanceof Buffer) {
|
|
43
35
|
return stream;
|
|
44
36
|
}
|
|
@@ -51,7 +43,7 @@ const tryStreamToArray = async (stream, maxBufferSize) => {
|
|
|
51
43
|
if (result.reduce((sum, b) => sum + b.length, 0) > maxBufferSize) {
|
|
52
44
|
stream.removeListener("data", dataHandler);
|
|
53
45
|
stream.removeListener("end", endListener);
|
|
54
|
-
resolve(mergeStreams((
|
|
46
|
+
resolve(mergeStreams(streamify(result), stream));
|
|
55
47
|
}
|
|
56
48
|
}
|
|
57
49
|
catch (e) {
|
|
@@ -62,9 +54,7 @@ const tryStreamToArray = async (stream, maxBufferSize) => {
|
|
|
62
54
|
stream.on("end", endListener);
|
|
63
55
|
});
|
|
64
56
|
};
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
return (0, stream_array_1.default)([stream]);
|
|
57
|
+
export const readableToStream = (stream) => {
|
|
58
|
+
return streamify([stream]);
|
|
68
59
|
};
|
|
69
|
-
exports.readableToStream = readableToStream;
|
|
70
60
|
//# sourceMappingURL=streamUtils.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streamUtils.js","sourceRoot":"","sources":["../../src/streamUtils.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"streamUtils.js","sourceRoot":"","sources":["../../src/streamUtils.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAElC,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,WAAW,EAAY,MAAM,QAAQ,CAAC;AAC/C,OAAO,SAAS,MAAM,cAAc,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAGrE,sEAAsE;AACtE,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,cAA8B,EAA6B,EAAE;IACtF,MAAM,QAAQ,GAAG,EAAE,CAAC,gBAAgB,CAAC,cAAc,CAAC,IAAc,CAAC,CAAC;IACpE,MAAM,eAAe,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC;IAC5F,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,gBAAgB,CAAC,QAAQ,EAAE,cAAc,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC,CAAC;AACrG,CAAC,CAAC;AAEF,2EAA2E;AAC3E,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,EAAE,cAA8B,EAA6B,EAAE;IAC/F,IAAI;QACA,MAAM,MAAM,GAAG,EAAE,CAAC,YAAY,CAAC,cAAc,CAAC,IAAc,CAAC,CAAC;QAC9D,MAAM,eAAe,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC;QAC5F,OAAO,IAAI,gBAAgB,CAAC,MAAM,EAAE,cAAc,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;KACjF;IAAC,OAAO,KAAK,EAAE;QACZ,OAAO,MAAM,YAAY,CAAC,cAAc,CAAC,CAAC;KAC7C;AACL,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,GAAG,OAAmB,EAAY,EAAE;IACtD,IAAI,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC;IAC7B,IAAI,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC;IAC7B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;QAC1B,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;QACzC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;KACjE;IACD,OAAO,IAAI,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,gBAAgB,GAAG,KAAK,EAAE,MAAgB,EAAE,aAAqB,EAA8B,EAAE;IAC1G,IAAI,MAAM,YAAY,MAAM,EAAE;QAC1B,OAAO,MAAM,CAAC;KACjB;IACD,OAAO,MAAM,IAAI,OAAO,CAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC5D,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,WAAW,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QACzD,MAAM,WAAW,GAAG,CAAC,KAAa,EAAE,EAAE;YAClC,IAAI;gBACA,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACnB,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,aAAa,EAAE;oBAC9D,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;oBAC3C,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;oBAC1C,OAAO,CAAC,YAAY,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;iBACpD;aACJ;YAAC,OAAO,CAAC,EAAE;gBACR,MAAM,CAAC,CAAC,CAAC,CAAC;aACb;QACL,CAAC,CAAC;QACF,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAC/B,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;AACP,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,MAA8B,EAAY,EAAE;IACzE,OAAO,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;AAC/B,CAAC,CAAC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport fs from \"fs\";\nimport { PassThrough, Readable } from \"stream\";\nimport streamify from \"stream-array\";\nimport { CompressionType, StreamDescriptor } from \"./descriptors.js\";\nimport { FileDescriptor } from \"./fileDescriptor.js\";\n\n// Returns fs.ReadStream for node and NodeJS.ReadableStream in browser\nexport const fileToStream = (fileDescriptor: FileDescriptor): Promise<StreamDescriptor> => {\n const streamFs = fs.createReadStream(fileDescriptor.file as string);\n const compressionType = fileDescriptor.zipped ? CompressionType.GZIP : CompressionType.None;\n return Promise.resolve(new StreamDescriptor(streamFs, fileDescriptor.sourceId, compressionType));\n};\n\n// Used in managed streaming where we buffer the file to memory for retries\nexport const tryFileToBuffer = async (fileDescriptor: FileDescriptor): Promise<StreamDescriptor> => {\n try {\n const buffer = fs.readFileSync(fileDescriptor.file as string);\n const compressionType = fileDescriptor.zipped ? CompressionType.GZIP : CompressionType.None;\n return new StreamDescriptor(buffer, fileDescriptor.sourceId, compressionType);\n } catch (error) {\n return await fileToStream(fileDescriptor);\n }\n};\n\nconst mergeStreams = (...streams: Readable[]): Readable => {\n let pass = new PassThrough();\n let waiting = streams.length;\n for (const stream of streams) {\n pass = stream.pipe(pass, { end: false });\n stream.once(\"end\", () => --waiting === 0 && pass.emit(\"end\"));\n }\n return pass;\n};\n\nexport const tryStreamToArray = async (stream: Readable, maxBufferSize: number): Promise<Buffer | Readable> => {\n if (stream instanceof Buffer) {\n return stream;\n }\n return await new Promise<Buffer | Readable>((resolve, reject) => {\n const result: Buffer[] = [];\n const endListener = () => resolve(Buffer.concat(result));\n const dataHandler = (chunk: Buffer) => {\n try {\n result.push(chunk);\n if (result.reduce((sum, b) => sum + b.length, 0) > maxBufferSize) {\n stream.removeListener(\"data\", dataHandler);\n stream.removeListener(\"end\", endListener);\n resolve(mergeStreams(streamify(result), stream));\n }\n } catch (e) {\n reject(e);\n }\n };\n stream.on(\"data\", dataHandler);\n stream.on(\"end\", endListener);\n });\n};\n\nexport const readableToStream = (stream: Readable | ArrayBuffer): Readable => {\n return streamify([stream]);\n};\n"]}
|
|
@@ -1,12 +1,10 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
// Copyright (c) Microsoft Corporation.
|
|
3
2
|
// Licensed under the MIT License.
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
class KustoStreamingIngestClient extends streamingIngestClientBase_1.KustoStreamingIngestClientBase {
|
|
3
|
+
import { StreamDescriptor } from "./descriptors.js";
|
|
4
|
+
import { FileDescriptor } from "./fileDescriptor.browser.js";
|
|
5
|
+
import { tryFileToBuffer } from "./streamUtils.browser.js";
|
|
6
|
+
import { KustoStreamingIngestClientBase } from "./streamingIngestClientBase.js";
|
|
7
|
+
class KustoStreamingIngestClient extends KustoStreamingIngestClientBase {
|
|
10
8
|
constructor(kcsb, defaultProps, autoCorrectEndpoint) {
|
|
11
9
|
super(kcsb, defaultProps, autoCorrectEndpoint);
|
|
12
10
|
}
|
|
@@ -17,7 +15,7 @@ class KustoStreamingIngestClient extends streamingIngestClientBase_1.KustoStream
|
|
|
17
15
|
var _a;
|
|
18
16
|
this.ensureOpen();
|
|
19
17
|
const props = this._getMergedProps(ingestionProperties);
|
|
20
|
-
const descriptor = stream instanceof
|
|
18
|
+
const descriptor = stream instanceof StreamDescriptor ? stream : new StreamDescriptor(stream);
|
|
21
19
|
return await this.kustoClient.executeStreamingIngest(props.database, props.table, descriptor.stream, props.format, (_a = props.ingestionMappingReference) !== null && _a !== void 0 ? _a : null, clientRequestId);
|
|
22
20
|
}
|
|
23
21
|
/**
|
|
@@ -25,9 +23,9 @@ class KustoStreamingIngestClient extends streamingIngestClientBase_1.KustoStream
|
|
|
25
23
|
*/
|
|
26
24
|
async ingestFromFile(file, ingestionProperties) {
|
|
27
25
|
this.ensureOpen();
|
|
28
|
-
const descriptor = file instanceof
|
|
29
|
-
return this.ingestFromStream(await
|
|
26
|
+
const descriptor = file instanceof FileDescriptor ? file : new FileDescriptor(file);
|
|
27
|
+
return this.ingestFromStream(await tryFileToBuffer(descriptor), ingestionProperties);
|
|
30
28
|
}
|
|
31
29
|
}
|
|
32
|
-
|
|
30
|
+
export default KustoStreamingIngestClient;
|
|
33
31
|
//# sourceMappingURL=streamingIngestClient.browser.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streamingIngestClient.browser.js","sourceRoot":"","sources":["../../src/streamingIngestClient.browser.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"streamingIngestClient.browser.js","sourceRoot":"","sources":["../../src/streamingIngestClient.browser.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAKlC,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,8BAA8B,EAAE,MAAM,gCAAgC,CAAC;AAEhF,MAAM,0BAA2B,SAAQ,8BAA8B;IACnE,YAAY,IAA2C,EAAE,YAAuC,EAAE,mBAA6B;QAC3H,KAAK,CAAC,IAAI,EAAE,YAAY,EAAE,mBAAmB,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB,CAAC,MAAsC,EAAE,mBAA8C,EAAE,eAAwB;;QACnI,IAAI,CAAC,UAAU,EAAE,CAAC;QAElB,MAAM,KAAK,GAAG,IAAI,CAAC,eAAe,CAAC,mBAAmB,CAAC,CAAC;QACxD,MAAM,UAAU,GAAqB,MAAM,YAAY,gBAAgB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAChH,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,sBAAsB,CAChD,KAAK,CAAC,QAAkB,EACxB,KAAK,CAAC,KAAe,EACrB,UAAU,CAAC,MAAM,EACjB,KAAK,CAAC,MAAM,EACZ,MAAA,KAAK,CAAC,yBAAyB,mCAAI,IAAI,EACvC,eAAe,CAClB,CAAC;IACN,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,IAA2B,EAAE,mBAA8C;QAC5F,IAAI,CAAC,UAAU,EAAE,CAAC;QAElB,MAAM,UAAU,GAAmB,IAAI,YAAY,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC;QACpG,OAAO,IAAI,CAAC,gBAAgB,CAAC,MAAM,eAAe,CAAC,UAAU,CAAC,EAAE,mBAAmB,CAAC,CAAC;IACzF,CAAC;CACJ;AAED,eAAe,0BAA0B,CAAC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport { IngestionPropertiesInput } from \"./ingestionProperties.js\";\n\nimport { KustoConnectionStringBuilder, KustoResponseDataSet } from \"azure-kusto-data\";\nimport { StreamDescriptor } from \"./descriptors.js\";\nimport { FileDescriptor } from \"./fileDescriptor.browser.js\";\nimport { tryFileToBuffer } from \"./streamUtils.browser.js\";\nimport { KustoStreamingIngestClientBase } from \"./streamingIngestClientBase.js\";\n\nclass KustoStreamingIngestClient extends KustoStreamingIngestClientBase {\n constructor(kcsb: string | KustoConnectionStringBuilder, defaultProps?: IngestionPropertiesInput, autoCorrectEndpoint?: boolean) {\n super(kcsb, defaultProps, autoCorrectEndpoint);\n }\n\n /**\n * Use Readable for Node.js and ArrayBuffer in browser\n */\n async ingestFromStream(stream: StreamDescriptor | ArrayBuffer, ingestionProperties?: IngestionPropertiesInput, clientRequestId?: string): Promise<any> {\n this.ensureOpen();\n\n const props = this._getMergedProps(ingestionProperties);\n const descriptor: StreamDescriptor = stream instanceof StreamDescriptor ? stream : new StreamDescriptor(stream);\n return await this.kustoClient.executeStreamingIngest(\n props.database as string,\n props.table as string,\n descriptor.stream,\n props.format,\n props.ingestionMappingReference ?? null,\n clientRequestId\n );\n }\n\n /**\n * Use string for Node.js and Blob in browser\n */\n async ingestFromFile(file: FileDescriptor | Blob, ingestionProperties?: IngestionPropertiesInput): Promise<KustoResponseDataSet> {\n this.ensureOpen();\n\n const descriptor: FileDescriptor = file instanceof FileDescriptor ? file : new FileDescriptor(file);\n return this.ingestFromStream(await tryFileToBuffer(descriptor), ingestionProperties);\n }\n}\n\nexport default KustoStreamingIngestClient;\n"]}
|
|
@@ -1,16 +1,11 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
// Copyright (c) Microsoft Corporation.
|
|
3
2
|
// Licensed under the MIT License.
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
};
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
const fileDescriptor_1 = require("./fileDescriptor");
|
|
11
|
-
const streamUtils_1 = require("./streamUtils");
|
|
12
|
-
const streamingIngestClientBase_1 = require("./streamingIngestClientBase");
|
|
13
|
-
class KustoStreamingIngestClient extends streamingIngestClientBase_1.KustoStreamingIngestClientBase {
|
|
3
|
+
import zlib from "zlib";
|
|
4
|
+
import { CompressionType, StreamDescriptor } from "./descriptors.js";
|
|
5
|
+
import { FileDescriptor } from "./fileDescriptor.js";
|
|
6
|
+
import { fileToStream } from "./streamUtils.js";
|
|
7
|
+
import { KustoStreamingIngestClientBase } from "./streamingIngestClientBase.js";
|
|
8
|
+
class KustoStreamingIngestClient extends KustoStreamingIngestClientBase {
|
|
14
9
|
constructor(kcsb, defaultProps, autoCorrectEndpoint) {
|
|
15
10
|
super(kcsb, defaultProps, autoCorrectEndpoint);
|
|
16
11
|
}
|
|
@@ -21,10 +16,10 @@ class KustoStreamingIngestClient extends streamingIngestClientBase_1.KustoStream
|
|
|
21
16
|
var _a;
|
|
22
17
|
this.ensureOpen();
|
|
23
18
|
const props = this._getMergedProps(ingestionProperties);
|
|
24
|
-
const descriptor = stream instanceof
|
|
25
|
-
const compressedStream = descriptor.compressionType ===
|
|
19
|
+
const descriptor = stream instanceof StreamDescriptor ? stream : new StreamDescriptor(stream);
|
|
20
|
+
const compressedStream = descriptor.compressionType === CompressionType.None
|
|
26
21
|
? !(descriptor.stream instanceof ArrayBuffer)
|
|
27
|
-
? descriptor.stream.pipe(
|
|
22
|
+
? descriptor.stream.pipe(zlib.createGzip())
|
|
28
23
|
: descriptor.stream
|
|
29
24
|
: descriptor.stream;
|
|
30
25
|
return await this.kustoClient.executeStreamingIngest(props.database, props.table, compressedStream, props.format, (_a = props.ingestionMappingReference) !== null && _a !== void 0 ? _a : null, clientRequestId);
|
|
@@ -34,9 +29,9 @@ class KustoStreamingIngestClient extends streamingIngestClientBase_1.KustoStream
|
|
|
34
29
|
*/
|
|
35
30
|
async ingestFromFile(file, ingestionProperties) {
|
|
36
31
|
this.ensureOpen();
|
|
37
|
-
const descriptor = file instanceof
|
|
38
|
-
return this.ingestFromStream(await
|
|
32
|
+
const descriptor = file instanceof FileDescriptor ? file : new FileDescriptor(file);
|
|
33
|
+
return this.ingestFromStream(await fileToStream(descriptor), ingestionProperties);
|
|
39
34
|
}
|
|
40
35
|
}
|
|
41
|
-
|
|
36
|
+
export default KustoStreamingIngestClient;
|
|
42
37
|
//# sourceMappingURL=streamingIngestClient.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streamingIngestClient.js","sourceRoot":"","sources":["../../src/streamingIngestClient.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"streamingIngestClient.js","sourceRoot":"","sources":["../../src/streamingIngestClient.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAMlC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACrE,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,8BAA8B,EAAE,MAAM,gCAAgC,CAAC;AAEhF,MAAM,0BAA2B,SAAQ,8BAA8B;IACnE,YAAY,IAA2C,EAAE,YAAuC,EAAE,mBAA6B;QAC3H,KAAK,CAAC,IAAI,EAAE,YAAY,EAAE,mBAAmB,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB,CAClB,MAAiD,EACjD,mBAA8C,EAC9C,eAAwB;;QAExB,IAAI,CAAC,UAAU,EAAE,CAAC;QAElB,MAAM,KAAK,GAAG,IAAI,CAAC,eAAe,CAAC,mBAAmB,CAAC,CAAC;QACxD,MAAM,UAAU,GAAqB,MAAM,YAAY,gBAAgB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAEhH,MAAM,gBAAgB,GAClB,UAAU,CAAC,eAAe,KAAK,eAAe,CAAC,IAAI;YAC/C,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,YAAY,WAAW,CAAC;gBACzC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC3C,CAAC,CAAC,UAAU,CAAC,MAAM;YACvB,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QAC5B,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,sBAAsB,CAChD,KAAK,CAAC,QAAkB,EACxB,KAAK,CAAC,KAAe,EACrB,gBAAgB,EAChB,KAAK,CAAC,MAAM,EACZ,MAAA,KAAK,CAAC,yBAAyB,mCAAI,IAAI,EACvC,eAAe,CAClB,CAAC;IACN,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,IAAoC,EAAE,mBAA8C;QACrG,IAAI,CAAC,UAAU,EAAE,CAAC;QAElB,MAAM,UAAU,GAAmB,IAAI,YAAY,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC;QACpG,OAAO,IAAI,CAAC,gBAAgB,CAAC,MAAM,YAAY,CAAC,UAAU,CAAC,EAAE,mBAAmB,CAAC,CAAC;IACtF,CAAC;CACJ;AAED,eAAe,0BAA0B,CAAC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport { IngestionPropertiesInput } from \"./ingestionProperties.js\";\n\nimport { KustoConnectionStringBuilder, KustoResponseDataSet } from \"azure-kusto-data\";\nimport { Readable } from \"stream\";\nimport zlib from \"zlib\";\nimport { CompressionType, StreamDescriptor } from \"./descriptors.js\";\nimport { FileDescriptor } from \"./fileDescriptor.js\";\nimport { fileToStream } from \"./streamUtils.js\";\nimport { KustoStreamingIngestClientBase } from \"./streamingIngestClientBase.js\";\n\nclass KustoStreamingIngestClient extends KustoStreamingIngestClientBase {\n constructor(kcsb: string | KustoConnectionStringBuilder, defaultProps?: IngestionPropertiesInput, autoCorrectEndpoint?: boolean) {\n super(kcsb, defaultProps, autoCorrectEndpoint);\n }\n\n /**\n * Use Readable for Node.js and ArrayBuffer in browser\n */\n async ingestFromStream(\n stream: StreamDescriptor | Readable | ArrayBuffer,\n ingestionProperties?: IngestionPropertiesInput,\n clientRequestId?: string\n ): Promise<any> {\n this.ensureOpen();\n\n const props = this._getMergedProps(ingestionProperties);\n const descriptor: StreamDescriptor = stream instanceof StreamDescriptor ? stream : new StreamDescriptor(stream);\n\n const compressedStream =\n descriptor.compressionType === CompressionType.None\n ? !(descriptor.stream instanceof ArrayBuffer)\n ? descriptor.stream.pipe(zlib.createGzip())\n : descriptor.stream\n : descriptor.stream;\n return await this.kustoClient.executeStreamingIngest(\n props.database as string,\n props.table as string,\n compressedStream,\n props.format,\n props.ingestionMappingReference ?? null,\n clientRequestId\n );\n }\n\n /**\n * Use string for Node.js and Blob in browser\n */\n async ingestFromFile(file: FileDescriptor | string | Blob, ingestionProperties?: IngestionPropertiesInput): Promise<KustoResponseDataSet> {\n this.ensureOpen();\n\n const descriptor: FileDescriptor = file instanceof FileDescriptor ? file : new FileDescriptor(file);\n return this.ingestFromStream(await fileToStream(descriptor), ingestionProperties);\n }\n}\n\nexport default KustoStreamingIngestClient;\n"]}
|
|
@@ -1,27 +1,24 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
// Copyright (c) Microsoft Corporation.
|
|
3
2
|
// Licensed under the MIT License.
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
const azure_kusto_data_1 = require("azure-kusto-data");
|
|
9
|
-
class KustoStreamingIngestClientBase extends abstractKustoClient_1.AbstractKustoClient {
|
|
3
|
+
import { BlobDescriptor } from "./descriptors.js";
|
|
4
|
+
import { AbstractKustoClient } from "./abstractKustoClient.js";
|
|
5
|
+
import { Client as KustoClient, KustoConnectionStringBuilder } from "azure-kusto-data";
|
|
6
|
+
export class KustoStreamingIngestClientBase extends AbstractKustoClient {
|
|
10
7
|
constructor(kcsb, defaultProps, autoCorrectEndpoint = true) {
|
|
11
8
|
super(defaultProps);
|
|
12
9
|
if (typeof kcsb === "string") {
|
|
13
|
-
kcsb = new
|
|
10
|
+
kcsb = new KustoConnectionStringBuilder(kcsb);
|
|
14
11
|
}
|
|
15
12
|
if (autoCorrectEndpoint) {
|
|
16
13
|
kcsb.dataSource = this.getQueryEndpoint(kcsb.dataSource);
|
|
17
14
|
}
|
|
18
|
-
this.kustoClient = new
|
|
15
|
+
this.kustoClient = new KustoClient(kcsb);
|
|
19
16
|
this.defaultDatabase = this.kustoClient.defaultDatabase;
|
|
20
17
|
}
|
|
21
18
|
async ingestFromBlob(blob, ingestionProperties, clientRequestId) {
|
|
22
19
|
var _a;
|
|
23
20
|
const props = this._getMergedProps(ingestionProperties);
|
|
24
|
-
const descriptor = blob instanceof
|
|
21
|
+
const descriptor = blob instanceof BlobDescriptor ? blob : new BlobDescriptor(blob);
|
|
25
22
|
// No need to check blob size if it was given to us that it's not empty
|
|
26
23
|
await descriptor.fillSize();
|
|
27
24
|
return await this.kustoClient.executeStreamingIngest(props.database, props.table, undefined, props.format, (_a = props.ingestionMappingReference) !== null && _a !== void 0 ? _a : null, descriptor.path, clientRequestId);
|
|
@@ -33,5 +30,4 @@ class KustoStreamingIngestClientBase extends abstractKustoClient_1.AbstractKusto
|
|
|
33
30
|
super.close();
|
|
34
31
|
}
|
|
35
32
|
}
|
|
36
|
-
exports.KustoStreamingIngestClientBase = KustoStreamingIngestClientBase;
|
|
37
33
|
//# sourceMappingURL=streamingIngestClientBase.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streamingIngestClientBase.js","sourceRoot":"","sources":["../../src/streamingIngestClientBase.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"streamingIngestClientBase.js","sourceRoot":"","sources":["../../src/streamingIngestClientBase.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAIlC,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAC/D,OAAO,EAAE,MAAM,IAAI,WAAW,EAAE,4BAA4B,EAAE,MAAM,kBAAkB,CAAC;AAEvF,MAAM,OAAgB,8BAA+B,SAAQ,mBAAmB;IAE5E,YAAY,IAA2C,EAAE,YAAuC,EAAE,sBAA+B,IAAI;QACjI,KAAK,CAAC,YAAY,CAAC,CAAC;QACpB,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;YAC1B,IAAI,GAAG,IAAI,4BAA4B,CAAC,IAAI,CAAC,CAAC;SACjD;QACD,IAAI,mBAAmB,EAAE;YACrB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;SAC5D;QACD,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC;IAC5D,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,IAA6B,EAAE,mBAA8C,EAAE,eAAwB;;QACxH,MAAM,KAAK,GAAG,IAAI,CAAC,eAAe,CAAC,mBAAmB,CAAC,CAAC;QACxD,MAAM,UAAU,GAAG,IAAI,YAAY,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC;QACpF,uEAAuE;QACvE,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC;QAE5B,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,sBAAsB,CAChD,KAAK,CAAC,QAAkB,EACxB,KAAK,CAAC,KAAe,EACrB,SAAS,EACT,KAAK,CAAC,MAAM,EACZ,MAAA,KAAK,CAAC,yBAAyB,mCAAI,IAAI,EACvC,UAAU,CAAC,IAAI,EACf,eAAe,CAClB,CAAC;IACN,CAAC;IAED,KAAK;QACD,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE;YACjB,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;SAC5B;QACD,KAAK,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;CACJ","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport { IngestionPropertiesInput } from \"./ingestionProperties.js\";\n\nimport { BlobDescriptor } from \"./descriptors.js\";\nimport { AbstractKustoClient } from \"./abstractKustoClient.js\";\nimport { Client as KustoClient, KustoConnectionStringBuilder } from \"azure-kusto-data\";\n\nexport abstract class KustoStreamingIngestClientBase extends AbstractKustoClient {\n protected kustoClient: KustoClient;\n constructor(kcsb: string | KustoConnectionStringBuilder, defaultProps?: IngestionPropertiesInput, autoCorrectEndpoint: boolean = true) {\n super(defaultProps);\n if (typeof kcsb === \"string\") {\n kcsb = new KustoConnectionStringBuilder(kcsb);\n }\n if (autoCorrectEndpoint) {\n kcsb.dataSource = this.getQueryEndpoint(kcsb.dataSource);\n }\n this.kustoClient = new KustoClient(kcsb);\n this.defaultDatabase = this.kustoClient.defaultDatabase;\n }\n\n async ingestFromBlob(blob: string | BlobDescriptor, ingestionProperties?: IngestionPropertiesInput, clientRequestId?: string): Promise<any> {\n const props = this._getMergedProps(ingestionProperties);\n const descriptor = blob instanceof BlobDescriptor ? blob : new BlobDescriptor(blob);\n // No need to check blob size if it was given to us that it's not empty\n await descriptor.fillSize();\n\n return await this.kustoClient.executeStreamingIngest(\n props.database as string,\n props.table as string,\n undefined,\n props.format,\n props.ingestionMappingReference ?? null,\n descriptor.path,\n clientRequestId\n );\n }\n\n close() {\n if (!this._isClosed) {\n this.kustoClient.close();\n }\n super.close();\n }\n}\n"]}
|
package/package.json
CHANGED
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "azure-kusto-ingest",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "7.0.0-alpha.0",
|
|
4
4
|
"description": "Azure Data Explorer Ingestion SDK",
|
|
5
|
-
"
|
|
6
|
-
"
|
|
7
|
-
|
|
5
|
+
"type": "module",
|
|
6
|
+
"exports": {
|
|
7
|
+
".": {
|
|
8
|
+
"import": "./dist-esm/src/index.js",
|
|
9
|
+
"types": "./types/src/index.d.ts"
|
|
10
|
+
}
|
|
11
|
+
},
|
|
8
12
|
"scripts": {
|
|
9
13
|
"build": "tsc -b",
|
|
10
14
|
"clean": "rimraf dist/* dist-esm/* types/*"
|
|
@@ -64,7 +68,7 @@
|
|
|
64
68
|
"@types/tmp": "^0.2.3",
|
|
65
69
|
"@types/uuid": "^8.3.4",
|
|
66
70
|
"@types/uuid-validate": "0.0.1",
|
|
67
|
-
"azure-kusto-data": "^
|
|
71
|
+
"azure-kusto-data": "^7.0.0-alpha.0",
|
|
68
72
|
"browserify-zlib": "0.2.0",
|
|
69
73
|
"buffer": "^6.0.3",
|
|
70
74
|
"is-ip": "^3.1.0",
|
|
@@ -79,5 +83,5 @@
|
|
|
79
83
|
"@types/sinon": "^10.0.13",
|
|
80
84
|
"assert": "^2.0.0"
|
|
81
85
|
},
|
|
82
|
-
"gitHead": "
|
|
86
|
+
"gitHead": "e09b5fe882833ca9890bbd332e80206701c28f85"
|
|
83
87
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { IngestionProperties, IngestionPropertiesInput } from "./ingestionProperties";
|
|
2
|
-
import { StreamDescriptor, FileDescriptorBase, BlobDescriptor } from "./descriptors";
|
|
1
|
+
import { IngestionProperties, IngestionPropertiesInput } from "./ingestionProperties.js";
|
|
2
|
+
import { StreamDescriptor, FileDescriptorBase, BlobDescriptor } from "./descriptors.js";
|
|
3
3
|
export declare abstract class AbstractKustoClient {
|
|
4
4
|
defaultProps: IngestionProperties;
|
|
5
5
|
defaultDatabase?: string;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { IngestionMappingKind } from "./ingestionProperties";
|
|
1
|
+
import { IngestionMappingKind } from "./ingestionProperties.js";
|
|
2
2
|
export declare enum FieldTransformation {
|
|
3
3
|
PropertyBagArrayToDictionary = "PropertyBagArrayToDictionary",
|
|
4
4
|
DateTimeFromUnixSeconds = "DateTimeFromUnixSeconds",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
1
|
+
/// <reference types="node" resolution-mode="require"/>
|
|
2
2
|
import { Readable } from "stream";
|
|
3
|
-
import IngestionProperties from "./ingestionProperties";
|
|
3
|
+
import IngestionProperties from "./ingestionProperties.js";
|
|
4
4
|
export declare enum CompressionType {
|
|
5
5
|
ZIP = ".zip",
|
|
6
6
|
GZIP = ".gz",
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { AbstractDescriptor, CompressionType, FileDescriptorBase } from "./descriptors";
|
|
2
|
-
import { IngestionPropertiesInput } from "./ingestionProperties";
|
|
1
|
+
import { AbstractDescriptor, CompressionType, FileDescriptorBase } from "./descriptors.js";
|
|
2
|
+
import { IngestionPropertiesInput } from "./ingestionProperties.js";
|
|
3
3
|
export declare class FileDescriptor extends AbstractDescriptor implements FileDescriptorBase {
|
|
4
4
|
readonly file: Blob;
|
|
5
5
|
readonly extension?: string | undefined;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { AbstractDescriptor, CompressionType, FileDescriptorBase } from "./descriptors";
|
|
2
|
-
import { IngestionPropertiesInput } from "./ingestionProperties";
|
|
1
|
+
import { AbstractDescriptor, CompressionType, FileDescriptorBase } from "./descriptors.js";
|
|
2
|
+
import { IngestionPropertiesInput } from "./ingestionProperties.js";
|
|
3
3
|
/**
|
|
4
4
|
* Describes a file to be ingested. Use string to describe a local path in Node.JS and Blob object in browsers
|
|
5
5
|
*/
|
package/types/src/index.d.ts
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
import KustoIngestClient from "./ingestClient";
|
|
2
|
-
import streamingIngestClient from "./streamingIngestClient";
|
|
3
|
-
import managedStreamingIngestClient from "./managedStreamingIngestClient";
|
|
4
|
-
import KustoIngestStatusQueues from "./status";
|
|
5
|
-
import { IngestionResult, OperationStatus, IngestionStatus, IngestionStatusInTableDescription } from "./ingestionResult";
|
|
6
|
-
import { DataFormat, IngestionMappingKind, ReportLevel, ReportMethod, ValidationImplications, ValidationOptions, ValidationPolicy, dataFormatMappingKind } from "./ingestionProperties";
|
|
7
|
-
import { ApacheAvroColumnMapping, AvroColumnMapping, ColumnMapping, ConstantTransformation, CsvColumnMapping, FieldTransformation, JsonColumnMapping, OrcColumnMapping, ParquetColumnMapping, SStreamColumnMapping, W3CLogFileMapping } from "./columnMappings";
|
|
8
|
-
import { BlobDescriptor, CompressionType, StreamDescriptor } from "./descriptors";
|
|
9
|
-
import { FileDescriptor } from "./fileDescriptor";
|
|
10
|
-
export { Transformation as ColumnMappingTransformation } from "./columnMappings";
|
|
11
|
-
export { IngestionProperties } from "./ingestionProperties";
|
|
1
|
+
import KustoIngestClient from "./ingestClient.js";
|
|
2
|
+
import streamingIngestClient from "./streamingIngestClient.js";
|
|
3
|
+
import managedStreamingIngestClient from "./managedStreamingIngestClient.js";
|
|
4
|
+
import KustoIngestStatusQueues from "./status.js";
|
|
5
|
+
import { type IngestionResult, OperationStatus, type IngestionStatus, IngestionStatusInTableDescription } from "./ingestionResult.js";
|
|
6
|
+
import { DataFormat, IngestionMappingKind, ReportLevel, ReportMethod, ValidationImplications, ValidationOptions, ValidationPolicy, dataFormatMappingKind } from "./ingestionProperties.js";
|
|
7
|
+
import { ApacheAvroColumnMapping, AvroColumnMapping, ColumnMapping, ConstantTransformation, CsvColumnMapping, FieldTransformation, JsonColumnMapping, OrcColumnMapping, ParquetColumnMapping, SStreamColumnMapping, W3CLogFileMapping } from "./columnMappings.js";
|
|
8
|
+
import { BlobDescriptor, CompressionType, StreamDescriptor } from "./descriptors.js";
|
|
9
|
+
import { FileDescriptor } from "./fileDescriptor.js";
|
|
10
|
+
export type { Transformation as ColumnMappingTransformation } from "./columnMappings.js";
|
|
11
|
+
export { IngestionProperties } from "./ingestionProperties.js";
|
|
12
12
|
export { KustoIngestClient as IngestClient, KustoIngestStatusQueues as IngestStatusQueues, managedStreamingIngestClient as ManagedStreamingIngestClient, streamingIngestClient as StreamingIngestClient, };
|
|
13
13
|
/**
|
|
14
14
|
* @deprecated - import directly instead. Export const is not exporting type.
|
|
@@ -18,7 +18,7 @@ export declare const IngestionDescriptors: {
|
|
|
18
18
|
FileDescriptor: typeof FileDescriptor;
|
|
19
19
|
StreamDescriptor: typeof StreamDescriptor;
|
|
20
20
|
};
|
|
21
|
-
export { ApacheAvroColumnMapping, AvroColumnMapping, BlobDescriptor, ColumnMapping, CompressionType, ConstantTransformation, CsvColumnMapping, DataFormat, FieldTransformation, FileDescriptor, IngestionMappingKind, JsonColumnMapping, OrcColumnMapping, ParquetColumnMapping, ReportLevel, ReportMethod, SStreamColumnMapping, StreamDescriptor, ValidationImplications, ValidationOptions, ValidationPolicy, W3CLogFileMapping, dataFormatMappingKind, IngestionResult, OperationStatus, IngestionStatus, IngestionStatusInTableDescription, };
|
|
21
|
+
export { ApacheAvroColumnMapping, AvroColumnMapping, BlobDescriptor, ColumnMapping, CompressionType, ConstantTransformation, CsvColumnMapping, DataFormat, FieldTransformation, FileDescriptor, IngestionMappingKind, JsonColumnMapping, OrcColumnMapping, ParquetColumnMapping, ReportLevel, ReportMethod, SStreamColumnMapping, StreamDescriptor, ValidationImplications, ValidationOptions, ValidationPolicy, W3CLogFileMapping, dataFormatMappingKind, type IngestionResult, OperationStatus, type IngestionStatus, IngestionStatusInTableDescription, };
|
|
22
22
|
/**
|
|
23
23
|
* @deprecated - import directly instead
|
|
24
24
|
*/
|
|
@@ -47,5 +47,5 @@ export declare const IngestionPropertiesEnums: {
|
|
|
47
47
|
FieldTransformation: typeof FieldTransformation;
|
|
48
48
|
ColumnMapping: typeof ColumnMapping;
|
|
49
49
|
};
|
|
50
|
-
export { IngestionPropertiesValidationError } from "./errors";
|
|
50
|
+
export { IngestionPropertiesValidationError } from "./errors.js";
|
|
51
51
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { KustoConnectionStringBuilder } from "azure-kusto-data";
|
|
2
|
-
import { StreamDescriptor } from "./descriptors";
|
|
3
|
-
import { FileDescriptor } from "./fileDescriptor.browser";
|
|
4
|
-
import { IngestionPropertiesInput } from "./ingestionProperties";
|
|
5
|
-
import { KustoIngestClientBase } from "./ingestClientBase";
|
|
6
|
-
import { IngestionResult } from "./ingestionResult";
|
|
2
|
+
import { StreamDescriptor } from "./descriptors.js";
|
|
3
|
+
import { FileDescriptor } from "./fileDescriptor.browser.js";
|
|
4
|
+
import { IngestionPropertiesInput } from "./ingestionProperties.js";
|
|
5
|
+
import { KustoIngestClientBase } from "./ingestClientBase.js";
|
|
6
|
+
import { IngestionResult } from "./ingestionResult.js";
|
|
7
7
|
export declare class KustoIngestClient extends KustoIngestClientBase {
|
|
8
8
|
constructor(kcsb: string | KustoConnectionStringBuilder, defaultProps?: IngestionPropertiesInput, autoCorrectEndpoint?: boolean);
|
|
9
9
|
/**
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
1
|
+
/// <reference types="node" resolution-mode="require"/>
|
|
2
2
|
import { KustoConnectionStringBuilder } from "azure-kusto-data";
|
|
3
|
-
import { StreamDescriptor } from "./descriptors";
|
|
4
|
-
import { FileDescriptor } from "./fileDescriptor";
|
|
5
|
-
import { IngestionPropertiesInput } from "./ingestionProperties";
|
|
6
|
-
import { KustoIngestClientBase } from "./ingestClientBase";
|
|
3
|
+
import { StreamDescriptor } from "./descriptors.js";
|
|
4
|
+
import { FileDescriptor } from "./fileDescriptor.js";
|
|
5
|
+
import { IngestionPropertiesInput } from "./ingestionProperties.js";
|
|
6
|
+
import { KustoIngestClientBase } from "./ingestClientBase.js";
|
|
7
7
|
import { Readable } from "stream";
|
|
8
|
-
import { IngestionResult } from "./ingestionResult";
|
|
8
|
+
import { IngestionResult } from "./ingestionResult.js";
|
|
9
9
|
export declare class KustoIngestClient extends KustoIngestClientBase {
|
|
10
10
|
constructor(kcsb: string | KustoConnectionStringBuilder, defaultProps?: IngestionPropertiesInput, autoCorrectEndpoint?: boolean);
|
|
11
11
|
/**
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { KustoConnectionStringBuilder } from "azure-kusto-data";
|
|
2
|
-
import ResourceManager from "./resourceManager";
|
|
3
|
-
import { IngestionPropertiesInput } from "./ingestionProperties";
|
|
4
|
-
import { AbstractKustoClient } from "./abstractKustoClient";
|
|
5
|
-
import { IngestionResult } from "./ingestionResult";
|
|
6
|
-
import { BlobDescriptor, StreamDescriptor } from "./descriptors";
|
|
2
|
+
import ResourceManager from "./resourceManager.js";
|
|
3
|
+
import { IngestionPropertiesInput } from "./ingestionProperties.js";
|
|
4
|
+
import { AbstractKustoClient } from "./abstractKustoClient.js";
|
|
5
|
+
import { IngestionResult } from "./ingestionResult.js";
|
|
6
|
+
import { BlobDescriptor, StreamDescriptor } from "./descriptors.js";
|
|
7
7
|
export declare abstract class KustoIngestClientBase extends AbstractKustoClient {
|
|
8
8
|
resourceManager: ResourceManager;
|
|
9
9
|
applicationForTracing: string | null;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { BlobDescriptor } from "./descriptors";
|
|
2
|
-
import IngestionProperties, { ReportLevel, ReportMethod } from "./ingestionProperties";
|
|
3
|
-
import { IngestionStatusInTableDescription } from "./ingestionResult";
|
|
1
|
+
import { BlobDescriptor } from "./descriptors.js";
|
|
2
|
+
import IngestionProperties, { ReportLevel, ReportMethod } from "./ingestionProperties.js";
|
|
3
|
+
import { IngestionStatusInTableDescription } from "./ingestionResult.js";
|
|
4
4
|
export declare class IngestionBlobInfo {
|
|
5
5
|
BlobPath: string;
|
|
6
6
|
RawDataSize: number | null;
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
2
|
-
import { IngestionPropertiesInput } from "./ingestionProperties";
|
|
1
|
+
/// <reference types="node" resolution-mode="require"/>
|
|
2
|
+
import { IngestionPropertiesInput } from "./ingestionProperties.js";
|
|
3
3
|
import { KustoConnectionStringBuilder, KustoResponseDataSet } from "azure-kusto-data";
|
|
4
4
|
import { Readable } from "stream";
|
|
5
|
-
import { AbstractKustoClient } from "./abstractKustoClient";
|
|
6
|
-
import { AbstractDescriptor, BlobDescriptor, StreamDescriptor } from "./descriptors";
|
|
7
|
-
import { FileDescriptor } from "./fileDescriptor";
|
|
8
|
-
import { IngestionResult } from "./ingestionResult";
|
|
5
|
+
import { AbstractKustoClient } from "./abstractKustoClient.js";
|
|
6
|
+
import { AbstractDescriptor, BlobDescriptor, StreamDescriptor } from "./descriptors.js";
|
|
7
|
+
import { FileDescriptor } from "./fileDescriptor.js";
|
|
8
|
+
import { IngestionResult } from "./ingestionResult.js";
|
|
9
9
|
declare class KustoManagedStreamingIngestClient extends AbstractKustoClient {
|
|
10
10
|
private streamingIngestClient;
|
|
11
11
|
private queuedIngestClient;
|
package/types/src/status.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { StatusQueue } from "./statusQ";
|
|
2
|
-
import KustoIngestClient from "./ingestClient";
|
|
1
|
+
import { StatusQueue } from "./statusQ.js";
|
|
2
|
+
import KustoIngestClient from "./ingestClient.js";
|
|
3
3
|
export declare class StatusMessage {
|
|
4
4
|
OperationId?: string;
|
|
5
5
|
Database?: string;
|
package/types/src/statusQ.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { PeekedMessageItem, QueueClient } from "@azure/storage-queue";
|
|
2
|
-
import { ResourceURI } from "./resourceManager";
|
|
3
|
-
import { StatusMessage } from "./status";
|
|
2
|
+
import { ResourceURI } from "./resourceManager.js";
|
|
3
|
+
import { StatusMessage } from "./status.js";
|
|
4
4
|
declare class QueueDetails {
|
|
5
5
|
readonly name: string;
|
|
6
6
|
readonly service: QueueClient;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
2
|
-
import { StreamDescriptor } from "./descriptors";
|
|
3
|
-
import { FileDescriptor } from "./fileDescriptor.browser";
|
|
1
|
+
/// <reference types="node" resolution-mode="require"/>
|
|
2
|
+
import { StreamDescriptor } from "./descriptors.js";
|
|
3
|
+
import { FileDescriptor } from "./fileDescriptor.browser.js";
|
|
4
4
|
export declare const fileToStream: (file: FileDescriptor) => Promise<StreamDescriptor>;
|
|
5
5
|
export declare const tryFileToBuffer: (file: FileDescriptor) => Promise<StreamDescriptor>;
|
|
6
6
|
export declare const tryStreamToArray: (stream: any) => Promise<Buffer>;
|