azure-kusto-ingest 6.0.2 → 7.0.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist-esm/src/abstractKustoClient.js +8 -15
- package/dist-esm/src/abstractKustoClient.js.map +1 -1
- package/dist-esm/src/columnMappings.js +22 -34
- package/dist-esm/src/columnMappings.js.map +1 -1
- package/dist-esm/src/descriptors.js +15 -27
- package/dist-esm/src/descriptors.js.map +1 -1
- package/dist-esm/src/errors.js +1 -5
- package/dist-esm/src/errors.js.map +1 -1
- package/dist-esm/src/fileDescriptor.browser.js +9 -16
- package/dist-esm/src/fileDescriptor.browser.js.map +1 -1
- package/dist-esm/src/fileDescriptor.js +20 -28
- package/dist-esm/src/fileDescriptor.js.map +1 -1
- package/dist-esm/src/index.js +38 -73
- package/dist-esm/src/index.js.map +1 -1
- package/dist-esm/src/ingestClient.browser.js +11 -15
- package/dist-esm/src/ingestClient.browser.js.map +1 -1
- package/dist-esm/src/ingestClient.js +11 -15
- package/dist-esm/src/ingestClient.js.map +1 -1
- package/dist-esm/src/ingestClientBase.js +25 -32
- package/dist-esm/src/ingestClientBase.js.map +1 -1
- package/dist-esm/src/ingestionBlobInfo.js +4 -8
- package/dist-esm/src/ingestionBlobInfo.js.map +1 -1
- package/dist-esm/src/ingestionProperties.js +29 -36
- package/dist-esm/src/ingestionProperties.js.map +1 -1
- package/dist-esm/src/ingestionResult.js +10 -17
- package/dist-esm/src/ingestionResult.js.map +1 -1
- package/dist-esm/src/managedStreamingIngestClient.js +29 -34
- package/dist-esm/src/managedStreamingIngestClient.js.map +1 -1
- package/dist-esm/src/rankedStorageAccount.js +1 -5
- package/dist-esm/src/rankedStorageAccount.js.map +1 -1
- package/dist-esm/src/rankedStorageAccountSet.js +3 -7
- package/dist-esm/src/rankedStorageAccountSet.js.map +1 -1
- package/dist-esm/src/resourceManager.js +24 -31
- package/dist-esm/src/resourceManager.js.map +1 -1
- package/dist-esm/src/retry.js +3 -8
- package/dist-esm/src/retry.js.map +1 -1
- package/dist-esm/src/status.js +6 -11
- package/dist-esm/src/status.js.map +1 -1
- package/dist-esm/src/statusQ.js +3 -7
- package/dist-esm/src/statusQ.js.map +1 -1
- package/dist-esm/src/streamUtils.browser.js +8 -15
- package/dist-esm/src/streamUtils.browser.js.map +1 -1
- package/dist-esm/src/streamUtils.js +18 -28
- package/dist-esm/src/streamUtils.js.map +1 -1
- package/dist-esm/src/streamingIngestClient.browser.js +9 -11
- package/dist-esm/src/streamingIngestClient.browser.js.map +1 -1
- package/dist-esm/src/streamingIngestClient.js +12 -17
- package/dist-esm/src/streamingIngestClient.js.map +1 -1
- package/dist-esm/src/streamingIngestClientBase.js +7 -11
- package/dist-esm/src/streamingIngestClientBase.js.map +1 -1
- package/package.json +11 -6
- package/types/src/abstractKustoClient.d.ts +2 -2
- package/types/src/columnMappings.d.ts +1 -1
- package/types/src/descriptors.d.ts +1 -2
- package/types/src/fileDescriptor.browser.d.ts +2 -2
- package/types/src/fileDescriptor.d.ts +2 -2
- package/types/src/index.d.ts +13 -13
- package/types/src/ingestClient.browser.d.ts +5 -5
- package/types/src/ingestClient.d.ts +5 -6
- package/types/src/ingestClientBase.d.ts +5 -5
- package/types/src/ingestionBlobInfo.d.ts +3 -3
- package/types/src/ingestionProperties.d.ts +1 -1
- package/types/src/managedStreamingIngestClient.d.ts +5 -6
- package/types/src/rankedStorageAccountSet.d.ts +1 -1
- package/types/src/status.d.ts +2 -2
- package/types/src/statusQ.d.ts +2 -2
- package/types/src/streamUtils.browser.d.ts +2 -3
- package/types/src/streamUtils.d.ts +2 -3
- package/types/src/streamingIngestClient.browser.d.ts +4 -4
- package/types/src/streamingIngestClient.d.ts +4 -5
- package/types/src/streamingIngestClientBase.d.ts +3 -3
|
@@ -1,45 +1,38 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
// Copyright (c) Microsoft Corporation.
|
|
3
2
|
// Licensed under the MIT License.
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
const fs_1 = __importDefault(require("fs"));
|
|
12
|
-
const tmp_promise_1 = require("tmp-promise");
|
|
13
|
-
const util_1 = require("util");
|
|
14
|
-
const descriptors_1 = require("./descriptors");
|
|
15
|
-
const ingestionProperties_1 = require("./ingestionProperties");
|
|
3
|
+
import zlib from "zlib";
|
|
4
|
+
import pathlib from "path";
|
|
5
|
+
import fs from "fs";
|
|
6
|
+
import { file as tmpFile } from "tmp-promise";
|
|
7
|
+
import { promisify } from "util";
|
|
8
|
+
import { AbstractDescriptor, CompressionType, shouldCompressFileByExtension } from "./descriptors.js";
|
|
9
|
+
import { shouldCompressFileByFormat } from "./ingestionProperties.js";
|
|
16
10
|
/**
|
|
17
11
|
* Describes a file to be ingested. Use string to describe a local path in Node.JS and Blob object in browsers
|
|
18
12
|
*/
|
|
19
|
-
class FileDescriptor extends
|
|
13
|
+
export class FileDescriptor extends AbstractDescriptor {
|
|
20
14
|
constructor(
|
|
21
15
|
/**
|
|
22
16
|
* Use string in Node.JS and Blob in browser
|
|
23
17
|
*/
|
|
24
|
-
file, sourceId = null, size = null, compressionType =
|
|
25
|
-
name
|
|
26
|
-
) {
|
|
18
|
+
file, sourceId = null, size = null, compressionType = CompressionType.None, extension, // Extracted from file name by default
|
|
19
|
+
name) {
|
|
27
20
|
super(sourceId, size);
|
|
28
21
|
this.file = file;
|
|
29
22
|
this.extension = extension;
|
|
30
23
|
this.name = name;
|
|
31
24
|
this.compressionType = compressionType;
|
|
32
|
-
this.name = name ? name :
|
|
33
|
-
this.extension = extension ? extension :
|
|
34
|
-
this.zipped = compressionType !==
|
|
35
|
-
this.shouldNotCompress = !
|
|
25
|
+
this.name = name ? name : pathlib.basename(this.file);
|
|
26
|
+
this.extension = extension ? extension : pathlib.extname(this.file).toLowerCase();
|
|
27
|
+
this.zipped = compressionType !== CompressionType.None || this.extension === ".gz" || this.extension === ".zip";
|
|
28
|
+
this.shouldNotCompress = !shouldCompressFileByExtension(this.extension);
|
|
36
29
|
}
|
|
37
30
|
async _gzip() {
|
|
38
|
-
const { path, cleanup } = await (
|
|
31
|
+
const { path, cleanup } = await tmpFile({ postfix: ".gz", keep: false });
|
|
39
32
|
this.cleanupTmp = cleanup;
|
|
40
|
-
const zipper =
|
|
41
|
-
const input =
|
|
42
|
-
const output =
|
|
33
|
+
const zipper = zlib.createGzip();
|
|
34
|
+
const input = fs.createReadStream(this.file, { autoClose: true });
|
|
35
|
+
const output = fs.createWriteStream(path);
|
|
43
36
|
await new Promise((resolve, reject) => {
|
|
44
37
|
input
|
|
45
38
|
.pipe(zipper)
|
|
@@ -54,7 +47,7 @@ class FileDescriptor extends descriptors_1.AbstractDescriptor {
|
|
|
54
47
|
return path;
|
|
55
48
|
}
|
|
56
49
|
async prepare(ingestionProperties) {
|
|
57
|
-
const shouldNotCompressByFormat = !
|
|
50
|
+
const shouldNotCompressByFormat = !shouldCompressFileByFormat(ingestionProperties);
|
|
58
51
|
if (this.zipped || this.shouldNotCompress || shouldNotCompressByFormat) {
|
|
59
52
|
const estimatedCompressionModifier = 11;
|
|
60
53
|
await this._calculateSize(estimatedCompressionModifier);
|
|
@@ -66,7 +59,7 @@ class FileDescriptor extends descriptors_1.AbstractDescriptor {
|
|
|
66
59
|
}
|
|
67
60
|
async _calculateSize(modifier = 1) {
|
|
68
61
|
if (this.size == null || this.size <= 0) {
|
|
69
|
-
const asyncStat =
|
|
62
|
+
const asyncStat = promisify(fs.stat);
|
|
70
63
|
this.size = (await asyncStat(this.file)).size * modifier;
|
|
71
64
|
}
|
|
72
65
|
}
|
|
@@ -79,5 +72,4 @@ class FileDescriptor extends descriptors_1.AbstractDescriptor {
|
|
|
79
72
|
return this.compressionType ? `.${this.compressionType}` : ".gz";
|
|
80
73
|
}
|
|
81
74
|
}
|
|
82
|
-
exports.FileDescriptor = FileDescriptor;
|
|
83
75
|
//# sourceMappingURL=fileDescriptor.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fileDescriptor.js","sourceRoot":"","sources":["../../src/fileDescriptor.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"fileDescriptor.js","sourceRoot":"","sources":["../../src/fileDescriptor.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAElC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,OAAO,MAAM,MAAM,CAAC;AAC3B,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,IAAI,IAAI,OAAO,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AACjC,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAsB,6BAA6B,EAAE,MAAM,kBAAkB,CAAC;AAC1H,OAAO,EAA4B,0BAA0B,EAAE,MAAM,0BAA0B,CAAC;AAEhG;;GAEG;AACH,MAAM,OAAO,cAAe,SAAQ,kBAAkB;IAMlD;IACI;;OAEG;IACM,IAAmB,EAC5B,WAA0B,IAAI,EAC9B,OAAsB,IAAI,EAC1B,kBAAmC,eAAe,CAAC,IAAI,EAC9C,SAAkB,EAAE,sCAAsC;IAC1D,IAAa;QAEtB,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAPb,SAAI,GAAJ,IAAI,CAAe;QAInB,cAAS,GAAT,SAAS,CAAS;QAClB,SAAI,GAAJ,IAAI,CAAS;QAGtB,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAc,CAAC,CAAC;QAChE,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,IAAc,CAAC,CAAC,WAAW,EAAE,CAAC;QAE5F,IAAI,CAAC,MAAM,GAAG,eAAe,KAAK,eAAe,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,KAAK,KAAK,IAAI,IAAI,CAAC,SAAS,KAAK,MAAM,CAAC;QAChH,IAAI,CAAC,iBAAiB,GAAG,CAAC,6BAA6B,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC5E,CAAC;IAED,KAAK,CAAC,KAAK;QACP,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,MAAM,OAAO,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACzE,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC;QAE1B,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAc,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5E,MAAM,MAAM,GAAG,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;QAE1C,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAClC,KAAK;iBACA,IAAI,CAAC,MAAM,CAAC;iBACZ,IAAI,CAAC,MAAM,CAAC;iBACZ,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;gBACjB,MAAM,CAAC,GAAG,CAAC,CAAC;YAChB,CAAC,CAAC,CAAC;YACP,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE;gBACtB,OAAO,CAAC,IAAI,CAAC,CAAC;YAClB,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,mBAA8C;QACxD,MAAM,yBAAyB,GAAG,CAAC,0BAA0B,CAAC,mBAAmB,CAAC,CAAC;QACnF,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,iBAAiB,IAAI,yBAAyB,EAAE,CAAC;YACrE,MAAM,4BAA4B,GAAG,EAAE,CAAC;YACxC,MAAM,IAAI,CAAC,cAAc,CAAC,4BAA4B,CAAC,CAAC;YACxD,OAAO,IAAI,CAAC,IAAc,CAAC;QAC/B,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QAChC,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,WAAmB,CAAC;QAC7C,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC;YACtC,MAAM,SAAS,GAAG,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;YACrC,IAAI,CAAC,IAAI,GAAG,CAAC,MAAM,SAAS,CAAC,IAAI,CAAC,IAAc,CAAC,CAAC,CAAC,IAAI,GAAG,QAAQ,CAAC;QACvE,CAAC;IACL,CAAC;IAED,KAAK,CAAC,OAAO;QACT,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAClB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAC5B,CAAC;IACL,CAAC;IAED,oBAAoB;QAChB,OAAO,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;IACrE,CAAC;CACJ","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport zlib from \"zlib\";\nimport pathlib from \"path\";\nimport fs from \"fs\";\nimport { file as tmpFile } from \"tmp-promise\";\nimport { promisify } from \"util\";\nimport { AbstractDescriptor, CompressionType, FileDescriptorBase, shouldCompressFileByExtension } from \"./descriptors.js\";\nimport { IngestionPropertiesInput, shouldCompressFileByFormat } from \"./ingestionProperties.js\";\n\n/**\n * Describes a file to be ingested. Use string to describe a local path in Node.JS and Blob object in browsers\n */\nexport class FileDescriptor extends AbstractDescriptor implements FileDescriptorBase {\n zipped: boolean;\n compressionType: CompressionType;\n shouldNotCompress: boolean;\n cleanupTmp?: () => Promise<void>;\n\n constructor(\n /**\n * Use string in Node.JS and Blob in browser\n */\n readonly file: string | Blob,\n sourceId: string | null = null,\n size: number | null = null,\n compressionType: CompressionType = CompressionType.None,\n readonly extension?: string, // Extracted from file name by default\n readonly name?: string, // Extracted from file name by default\n ) {\n super(sourceId, size);\n this.compressionType = compressionType;\n this.name = name ? name : pathlib.basename(this.file as string);\n this.extension = extension ? extension : pathlib.extname(this.file as string).toLowerCase();\n\n this.zipped = compressionType !== CompressionType.None || this.extension === \".gz\" || this.extension === \".zip\";\n this.shouldNotCompress = !shouldCompressFileByExtension(this.extension);\n }\n\n async _gzip(): Promise<string> {\n const { path, cleanup } = await tmpFile({ postfix: \".gz\", keep: false });\n this.cleanupTmp = cleanup;\n\n const zipper = zlib.createGzip();\n const input = fs.createReadStream(this.file as string, { autoClose: true });\n const output = fs.createWriteStream(path);\n\n await new Promise((resolve, reject) => {\n input\n .pipe(zipper)\n .pipe(output)\n .on(\"error\", (err) => {\n reject(err);\n });\n output.once(\"close\", () => {\n resolve(null);\n });\n });\n\n return path;\n }\n\n async prepare(ingestionProperties?: IngestionPropertiesInput): Promise<string> {\n const shouldNotCompressByFormat = !shouldCompressFileByFormat(ingestionProperties);\n if (this.zipped || this.shouldNotCompress || shouldNotCompressByFormat) {\n const estimatedCompressionModifier = 11;\n await this._calculateSize(estimatedCompressionModifier);\n return this.file as string;\n }\n\n const path = await this._gzip();\n await this._calculateSize();\n return path;\n }\n\n private async _calculateSize(modifier: number = 1): Promise<void> {\n if (this.size == null || this.size <= 0) {\n const asyncStat = promisify(fs.stat);\n this.size = (await asyncStat(this.file as string)).size * modifier;\n }\n }\n\n async cleanup(): Promise<void> {\n if (this.cleanupTmp) {\n await this.cleanupTmp();\n }\n }\n\n getCompressionSuffix() {\n return this.compressionType ? `.${this.compressionType}` : \".gz\";\n }\n}\n"]}
|
package/dist-esm/src/index.js
CHANGED
|
@@ -1,88 +1,53 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
// Copyright (c) Microsoft Corporation.
|
|
3
2
|
// Licensed under the MIT License.
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
const status_1 = __importDefault(require("./status"));
|
|
16
|
-
exports.IngestStatusQueues = status_1.default;
|
|
17
|
-
const ingestionResult_1 = require("./ingestionResult");
|
|
18
|
-
Object.defineProperty(exports, "OperationStatus", { enumerable: true, get: function () { return ingestionResult_1.OperationStatus; } });
|
|
19
|
-
Object.defineProperty(exports, "IngestionStatusInTableDescription", { enumerable: true, get: function () { return ingestionResult_1.IngestionStatusInTableDescription; } });
|
|
20
|
-
const ingestionProperties_1 = require("./ingestionProperties");
|
|
21
|
-
Object.defineProperty(exports, "DataFormat", { enumerable: true, get: function () { return ingestionProperties_1.DataFormat; } });
|
|
22
|
-
Object.defineProperty(exports, "IngestionMappingKind", { enumerable: true, get: function () { return ingestionProperties_1.IngestionMappingKind; } });
|
|
23
|
-
Object.defineProperty(exports, "ReportLevel", { enumerable: true, get: function () { return ingestionProperties_1.ReportLevel; } });
|
|
24
|
-
Object.defineProperty(exports, "ReportMethod", { enumerable: true, get: function () { return ingestionProperties_1.ReportMethod; } });
|
|
25
|
-
Object.defineProperty(exports, "ValidationImplications", { enumerable: true, get: function () { return ingestionProperties_1.ValidationImplications; } });
|
|
26
|
-
Object.defineProperty(exports, "ValidationOptions", { enumerable: true, get: function () { return ingestionProperties_1.ValidationOptions; } });
|
|
27
|
-
Object.defineProperty(exports, "ValidationPolicy", { enumerable: true, get: function () { return ingestionProperties_1.ValidationPolicy; } });
|
|
28
|
-
Object.defineProperty(exports, "dataFormatMappingKind", { enumerable: true, get: function () { return ingestionProperties_1.dataFormatMappingKind; } });
|
|
29
|
-
const columnMappings_1 = require("./columnMappings");
|
|
30
|
-
Object.defineProperty(exports, "ApacheAvroColumnMapping", { enumerable: true, get: function () { return columnMappings_1.ApacheAvroColumnMapping; } });
|
|
31
|
-
Object.defineProperty(exports, "AvroColumnMapping", { enumerable: true, get: function () { return columnMappings_1.AvroColumnMapping; } });
|
|
32
|
-
Object.defineProperty(exports, "ColumnMapping", { enumerable: true, get: function () { return columnMappings_1.ColumnMapping; } });
|
|
33
|
-
Object.defineProperty(exports, "ConstantTransformation", { enumerable: true, get: function () { return columnMappings_1.ConstantTransformation; } });
|
|
34
|
-
Object.defineProperty(exports, "CsvColumnMapping", { enumerable: true, get: function () { return columnMappings_1.CsvColumnMapping; } });
|
|
35
|
-
Object.defineProperty(exports, "FieldTransformation", { enumerable: true, get: function () { return columnMappings_1.FieldTransformation; } });
|
|
36
|
-
Object.defineProperty(exports, "JsonColumnMapping", { enumerable: true, get: function () { return columnMappings_1.JsonColumnMapping; } });
|
|
37
|
-
Object.defineProperty(exports, "OrcColumnMapping", { enumerable: true, get: function () { return columnMappings_1.OrcColumnMapping; } });
|
|
38
|
-
Object.defineProperty(exports, "ParquetColumnMapping", { enumerable: true, get: function () { return columnMappings_1.ParquetColumnMapping; } });
|
|
39
|
-
Object.defineProperty(exports, "SStreamColumnMapping", { enumerable: true, get: function () { return columnMappings_1.SStreamColumnMapping; } });
|
|
40
|
-
Object.defineProperty(exports, "W3CLogFileMapping", { enumerable: true, get: function () { return columnMappings_1.W3CLogFileMapping; } });
|
|
41
|
-
const descriptors_1 = require("./descriptors");
|
|
42
|
-
Object.defineProperty(exports, "BlobDescriptor", { enumerable: true, get: function () { return descriptors_1.BlobDescriptor; } });
|
|
43
|
-
Object.defineProperty(exports, "CompressionType", { enumerable: true, get: function () { return descriptors_1.CompressionType; } });
|
|
44
|
-
Object.defineProperty(exports, "StreamDescriptor", { enumerable: true, get: function () { return descriptors_1.StreamDescriptor; } });
|
|
45
|
-
const fileDescriptor_1 = require("./fileDescriptor");
|
|
46
|
-
Object.defineProperty(exports, "FileDescriptor", { enumerable: true, get: function () { return fileDescriptor_1.FileDescriptor; } });
|
|
47
|
-
var ingestionProperties_2 = require("./ingestionProperties");
|
|
48
|
-
Object.defineProperty(exports, "IngestionProperties", { enumerable: true, get: function () { return ingestionProperties_2.IngestionProperties; } });
|
|
3
|
+
import KustoIngestClient from "./ingestClient.js";
|
|
4
|
+
import streamingIngestClient from "./streamingIngestClient.js";
|
|
5
|
+
import managedStreamingIngestClient from "./managedStreamingIngestClient.js";
|
|
6
|
+
import KustoIngestStatusQueues from "./status.js";
|
|
7
|
+
import { OperationStatus, IngestionStatusInTableDescription } from "./ingestionResult.js";
|
|
8
|
+
import { DataFormat, IngestionMappingKind, ReportLevel, ReportMethod, ValidationImplications, ValidationOptions, ValidationPolicy, dataFormatMappingKind, } from "./ingestionProperties.js";
|
|
9
|
+
import { ApacheAvroColumnMapping, AvroColumnMapping, ColumnMapping, ConstantTransformation, CsvColumnMapping, FieldTransformation, JsonColumnMapping, OrcColumnMapping, ParquetColumnMapping, SStreamColumnMapping, W3CLogFileMapping, } from "./columnMappings.js";
|
|
10
|
+
import { BlobDescriptor, CompressionType, StreamDescriptor } from "./descriptors.js";
|
|
11
|
+
import { FileDescriptor } from "./fileDescriptor.js";
|
|
12
|
+
export { IngestionProperties } from "./ingestionProperties.js";
|
|
13
|
+
export { KustoIngestClient as IngestClient, KustoIngestStatusQueues as IngestStatusQueues, managedStreamingIngestClient as ManagedStreamingIngestClient, streamingIngestClient as StreamingIngestClient, };
|
|
49
14
|
/**
|
|
50
15
|
* @deprecated - import directly instead. Export const is not exporting type.
|
|
51
16
|
*/
|
|
52
|
-
|
|
53
|
-
BlobDescriptor
|
|
54
|
-
FileDescriptor
|
|
55
|
-
StreamDescriptor
|
|
17
|
+
export const IngestionDescriptors = {
|
|
18
|
+
BlobDescriptor,
|
|
19
|
+
FileDescriptor,
|
|
20
|
+
StreamDescriptor,
|
|
56
21
|
};
|
|
22
|
+
export { ApacheAvroColumnMapping, AvroColumnMapping, BlobDescriptor, ColumnMapping, CompressionType, ConstantTransformation, CsvColumnMapping, DataFormat, FieldTransformation, FileDescriptor, IngestionMappingKind, JsonColumnMapping, OrcColumnMapping, ParquetColumnMapping, ReportLevel, ReportMethod, SStreamColumnMapping, StreamDescriptor, ValidationImplications, ValidationOptions, ValidationPolicy, W3CLogFileMapping, dataFormatMappingKind, OperationStatus, IngestionStatusInTableDescription, };
|
|
57
23
|
/**
|
|
58
24
|
* @deprecated - import directly instead
|
|
59
25
|
*/
|
|
60
|
-
|
|
61
|
-
JsonColumnMapping
|
|
62
|
-
CsvColumnMapping
|
|
63
|
-
AvroColumnMapping
|
|
64
|
-
ParquetColumnMapping
|
|
65
|
-
OrcColumnMapping
|
|
66
|
-
W3CLogFileMapping
|
|
67
|
-
ValidationPolicy
|
|
68
|
-
ReportLevel
|
|
69
|
-
ReportMethod
|
|
70
|
-
ValidationImplications
|
|
71
|
-
ValidationOptions
|
|
72
|
-
DataFormat
|
|
26
|
+
export const IngestionPropertiesEnums = {
|
|
27
|
+
JsonColumnMapping,
|
|
28
|
+
CsvColumnMapping,
|
|
29
|
+
AvroColumnMapping,
|
|
30
|
+
ParquetColumnMapping,
|
|
31
|
+
OrcColumnMapping,
|
|
32
|
+
W3CLogFileMapping,
|
|
33
|
+
ValidationPolicy,
|
|
34
|
+
ReportLevel,
|
|
35
|
+
ReportMethod,
|
|
36
|
+
ValidationImplications,
|
|
37
|
+
ValidationOptions,
|
|
38
|
+
DataFormat,
|
|
73
39
|
/**
|
|
74
40
|
* @deprecated - use IngestionMappingKind instead
|
|
75
41
|
*/
|
|
76
|
-
IngestionMappingType:
|
|
77
|
-
IngestionMappingKind
|
|
78
|
-
CompressionType
|
|
79
|
-
ApacheAvroColumnMapping
|
|
80
|
-
SStreamColumnMapping
|
|
81
|
-
ConstantTransformation
|
|
82
|
-
FieldTransformation
|
|
83
|
-
ColumnMapping
|
|
42
|
+
IngestionMappingType: IngestionMappingKind,
|
|
43
|
+
IngestionMappingKind,
|
|
44
|
+
CompressionType,
|
|
45
|
+
ApacheAvroColumnMapping,
|
|
46
|
+
SStreamColumnMapping,
|
|
47
|
+
ConstantTransformation,
|
|
48
|
+
FieldTransformation,
|
|
49
|
+
ColumnMapping,
|
|
84
50
|
};
|
|
85
|
-
|
|
86
|
-
Object.defineProperty(exports, "IngestionPropertiesValidationError", { enumerable: true, get: function () { return errors_1.IngestionPropertiesValidationError; } });
|
|
51
|
+
export { IngestionPropertiesValidationError } from "./errors.js";
|
|
87
52
|
// eslint-disable-next-line no-console
|
|
88
53
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAElC,OAAO,iBAAiB,MAAM,mBAAmB,CAAC;AAElD,OAAO,qBAAqB,MAAM,4BAA4B,CAAC;AAE/D,OAAO,4BAA4B,MAAM,mCAAmC,CAAC;AAE7E,OAAO,uBAAuB,MAAM,aAAa,CAAC;AAElD,OAAO,EAAwB,eAAe,EAAwB,iCAAiC,EAAE,MAAM,sBAAsB,CAAC;AACtI,OAAO,EACH,UAAU,EACV,oBAAoB,EACpB,WAAW,EACX,YAAY,EACZ,sBAAsB,EACtB,iBAAiB,EACjB,gBAAgB,EAChB,qBAAqB,GACxB,MAAM,0BAA0B,CAAC;AAElC,OAAO,EACH,uBAAuB,EACvB,iBAAiB,EACjB,aAAa,EACb,sBAAsB,EACtB,gBAAgB,EAChB,mBAAmB,EACnB,iBAAiB,EACjB,gBAAgB,EAChB,oBAAoB,EACpB,oBAAoB,EACpB,iBAAiB,GACpB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACrF,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAGrD,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAC/D,OAAO,EACH,iBAAiB,IAAI,YAAY,EACjC,uBAAuB,IAAI,kBAAkB,EAC7C,4BAA4B,IAAI,4BAA4B,EAC5D,qBAAqB,IAAI,qBAAqB,GACjD,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG;IAChC,cAAc;IACd,cAAc;IACd,gBAAgB;CACnB,CAAC;AAEF,OAAO,EACH,uBAAuB,EACvB,iBAAiB,EACjB,cAAc,EACd,aAAa,EACb,eAAe,EACf,sBAAsB,EACtB,gBAAgB,EAChB,UAAU,EACV,mBAAmB,EACnB,cAAc,EACd,oBAAoB,EACpB,iBAAiB,EACjB,gBAAgB,EAChB,oBAAoB,EACpB,WAAW,EACX,YAAY,EACZ,oBAAoB,EACpB,gBAAgB,EAChB,sBAAsB,EACtB,iBAAiB,EACjB,gBAAgB,EAChB,iBAAiB,EACjB,qBAAqB,EAErB,eAAe,EAEf,iCAAiC,GACpC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG;IACpC,iBAAiB;IACjB,gBAAgB;IAChB,iBAAiB;IACjB,oBAAoB;IACpB,gBAAgB;IAChB,iBAAiB;IACjB,gBAAgB;IAChB,WAAW;IACX,YAAY;IACZ,sBAAsB;IACtB,iBAAiB;IACjB,UAAU;IACV;;OAEG;IACH,oBAAoB,EAAE,oBAAoB;IAC1C,oBAAoB;IACpB,eAAe;IACf,uBAAuB;IACvB,oBAAoB;IACpB,sBAAsB;IACtB,mBAAmB;IACnB,aAAa;CAChB,CAAC;AAEF,OAAO,EAAE,kCAAkC,EAAE,MAAM,aAAa,CAAC;AACjE,sCAAsC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport KustoIngestClient from \"./ingestClient.js\";\n\nimport streamingIngestClient from \"./streamingIngestClient.js\";\n\nimport managedStreamingIngestClient from \"./managedStreamingIngestClient.js\";\n\nimport KustoIngestStatusQueues from \"./status.js\";\n\nimport { type IngestionResult, OperationStatus, type IngestionStatus, IngestionStatusInTableDescription } from \"./ingestionResult.js\";\nimport {\n DataFormat,\n IngestionMappingKind,\n ReportLevel,\n ReportMethod,\n ValidationImplications,\n ValidationOptions,\n ValidationPolicy,\n dataFormatMappingKind,\n} from \"./ingestionProperties.js\";\n\nimport {\n ApacheAvroColumnMapping,\n AvroColumnMapping,\n ColumnMapping,\n ConstantTransformation,\n CsvColumnMapping,\n FieldTransformation,\n JsonColumnMapping,\n OrcColumnMapping,\n ParquetColumnMapping,\n SStreamColumnMapping,\n W3CLogFileMapping,\n} from \"./columnMappings.js\";\nimport { BlobDescriptor, CompressionType, StreamDescriptor } from \"./descriptors.js\";\nimport { FileDescriptor } from \"./fileDescriptor.js\";\n\nexport type { Transformation as ColumnMappingTransformation } from \"./columnMappings.js\";\nexport { IngestionProperties } from \"./ingestionProperties.js\";\nexport {\n KustoIngestClient as IngestClient,\n KustoIngestStatusQueues as IngestStatusQueues,\n managedStreamingIngestClient as ManagedStreamingIngestClient,\n streamingIngestClient as StreamingIngestClient,\n};\n\n/**\n * @deprecated - import directly instead. Export const is not exporting type.\n */\nexport const IngestionDescriptors = {\n BlobDescriptor,\n FileDescriptor,\n StreamDescriptor,\n};\n\nexport {\n ApacheAvroColumnMapping,\n AvroColumnMapping,\n BlobDescriptor,\n ColumnMapping,\n CompressionType,\n ConstantTransformation,\n CsvColumnMapping,\n DataFormat,\n FieldTransformation,\n FileDescriptor,\n IngestionMappingKind,\n JsonColumnMapping,\n OrcColumnMapping,\n ParquetColumnMapping,\n ReportLevel,\n ReportMethod,\n SStreamColumnMapping,\n StreamDescriptor,\n ValidationImplications,\n ValidationOptions,\n ValidationPolicy,\n W3CLogFileMapping,\n dataFormatMappingKind,\n type IngestionResult,\n OperationStatus,\n type IngestionStatus,\n IngestionStatusInTableDescription,\n};\n\n/**\n * @deprecated - import directly instead\n */\nexport const IngestionPropertiesEnums = {\n JsonColumnMapping,\n CsvColumnMapping,\n AvroColumnMapping,\n ParquetColumnMapping,\n OrcColumnMapping,\n W3CLogFileMapping,\n ValidationPolicy,\n ReportLevel,\n ReportMethod,\n ValidationImplications,\n ValidationOptions,\n DataFormat,\n /**\n * @deprecated - use IngestionMappingKind instead\n */\n IngestionMappingType: IngestionMappingKind,\n IngestionMappingKind,\n CompressionType,\n ApacheAvroColumnMapping,\n SStreamColumnMapping,\n ConstantTransformation,\n FieldTransformation,\n ColumnMapping,\n};\n\nexport { IngestionPropertiesValidationError } from \"./errors.js\";\n// eslint-disable-next-line no-console\n"]}
|
|
@@ -1,12 +1,9 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
// Copyright (c) Microsoft Corporation.
|
|
3
2
|
// Licensed under the MIT License.
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
const ingestClientBase_1 = require("./ingestClientBase");
|
|
9
|
-
class KustoIngestClient extends ingestClientBase_1.KustoIngestClientBase {
|
|
3
|
+
import { BlobDescriptor, generateBlobName, StreamDescriptor } from "./descriptors.js";
|
|
4
|
+
import { FileDescriptor } from "./fileDescriptor.browser.js";
|
|
5
|
+
import { KustoIngestClientBase } from "./ingestClientBase.js";
|
|
6
|
+
export class KustoIngestClient extends KustoIngestClientBase {
|
|
10
7
|
constructor(kcsb, defaultProps, autoCorrectEndpoint) {
|
|
11
8
|
super(kcsb, defaultProps, autoCorrectEndpoint, true);
|
|
12
9
|
}
|
|
@@ -15,13 +12,13 @@ class KustoIngestClient extends ingestClientBase_1.KustoIngestClientBase {
|
|
|
15
12
|
*/
|
|
16
13
|
async ingestFromFile(file, ingestionProperties) {
|
|
17
14
|
this.ensureOpen();
|
|
18
|
-
const descriptor = file instanceof
|
|
15
|
+
const descriptor = file instanceof FileDescriptor ? file : new FileDescriptor(file);
|
|
19
16
|
const blob = descriptor.file;
|
|
20
17
|
const props = this._getMergedProps(ingestionProperties);
|
|
21
18
|
const fileToUpload = await descriptor.prepare(ingestionProperties);
|
|
22
|
-
const blobName =
|
|
19
|
+
const blobName = generateBlobName(descriptor, props);
|
|
23
20
|
const blobUri = await this.uploadToBlobWithRetry(fileToUpload, blobName);
|
|
24
|
-
return this.ingestFromBlob(new
|
|
21
|
+
return this.ingestFromBlob(new BlobDescriptor(blobUri, blob.size, descriptor.sourceId), props);
|
|
25
22
|
}
|
|
26
23
|
/**
|
|
27
24
|
* Use Readable for Node.js and ArrayBuffer in browser
|
|
@@ -29,12 +26,11 @@ class KustoIngestClient extends ingestClientBase_1.KustoIngestClientBase {
|
|
|
29
26
|
async ingestFromStream(stream, ingestionProperties) {
|
|
30
27
|
this.ensureOpen();
|
|
31
28
|
const props = this._getMergedProps(ingestionProperties);
|
|
32
|
-
const descriptor = stream instanceof
|
|
33
|
-
const blobName =
|
|
29
|
+
const descriptor = stream instanceof StreamDescriptor ? stream : new StreamDescriptor(stream);
|
|
30
|
+
const blobName = generateBlobName(descriptor, props);
|
|
34
31
|
const blobUri = await this.uploadToBlobWithRetry(descriptor.stream, blobName);
|
|
35
|
-
return this.ingestFromBlob(new
|
|
32
|
+
return this.ingestFromBlob(new BlobDescriptor(blobUri, descriptor.size, descriptor.sourceId), props);
|
|
36
33
|
}
|
|
37
34
|
}
|
|
38
|
-
|
|
39
|
-
exports.default = KustoIngestClient;
|
|
35
|
+
export default KustoIngestClient;
|
|
40
36
|
//# sourceMappingURL=ingestClient.browser.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ingestClient.browser.js","sourceRoot":"","sources":["../../src/ingestClient.browser.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"ingestClient.browser.js","sourceRoot":"","sources":["../../src/ingestClient.browser.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAIlC,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACtF,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAG7D,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAG9D,MAAM,OAAO,iBAAkB,SAAQ,qBAAqB;IACxD,YAAY,IAA2C,EAAE,YAAuC,EAAE,mBAA6B;QAC3H,KAAK,CAAC,IAAI,EAAE,YAAY,EAAE,mBAAmB,EAAE,IAAI,CAAC,CAAC;IACzD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,IAA2B,EAAE,mBAA8C;QAC5F,IAAI,CAAC,UAAU,EAAE,CAAC;QAClB,MAAM,UAAU,GAAG,IAAI,YAAY,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC;QAEpF,MAAM,IAAI,GAAG,UAAU,CAAC,IAAY,CAAC;QACrC,MAAM,KAAK,GAAG,IAAI,CAAC,eAAe,CAAC,mBAAmB,CAAC,CAAC;QAExD,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;QACnE,MAAM,QAAQ,GAAG,gBAAgB,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QAErD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;QAEzE,OAAO,IAAI,CAAC,cAAc,CAAC,IAAI,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,EAAE,KAAK,CAAC,CAAC;IACnG,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB,CAAC,MAAsC,EAAE,mBAA8C;QACzG,IAAI,CAAC,UAAU,EAAE,CAAC;QAClB,MAAM,KAAK,GAAG,IAAI,CAAC,eAAe,CAAC,mBAAmB,CAAC,CAAC;QACxD,MAAM,UAAU,GAAqB,MAAM,YAAY,gBAAgB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAChH,MAAM,QAAQ,GAAG,gBAAgB,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QACrD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,MAAqB,EAAE,QAAQ,CAAC,CAAC;QAC7F,OAAO,IAAI,CAAC,cAAc,CAAC,IAAI,cAAc,CAAC,OAAO,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,EAAE,KAAK,CAAC,CAAC;IACzG,CAAC;CACJ;AAED,eAAe,iBAAiB,CAAC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport { KustoConnectionStringBuilder } from \"azure-kusto-data\";\n\nimport { BlobDescriptor, generateBlobName, StreamDescriptor } from \"./descriptors.js\";\nimport { FileDescriptor } from \"./fileDescriptor.browser.js\";\n\nimport { IngestionPropertiesInput } from \"./ingestionProperties.js\";\nimport { KustoIngestClientBase } from \"./ingestClientBase.js\";\nimport { IngestionResult } from \"./ingestionResult.js\";\n\nexport class KustoIngestClient extends KustoIngestClientBase {\n constructor(kcsb: string | KustoConnectionStringBuilder, defaultProps?: IngestionPropertiesInput, autoCorrectEndpoint?: boolean) {\n super(kcsb, defaultProps, autoCorrectEndpoint, true);\n }\n\n /**\n * Use string for Node.js and Blob in browser\n */\n async ingestFromFile(file: Blob | FileDescriptor, ingestionProperties?: IngestionPropertiesInput): Promise<IngestionResult> {\n this.ensureOpen();\n const descriptor = file instanceof FileDescriptor ? file : new FileDescriptor(file);\n\n const blob = descriptor.file as Blob;\n const props = this._getMergedProps(ingestionProperties);\n\n const fileToUpload = await descriptor.prepare(ingestionProperties);\n const blobName = generateBlobName(descriptor, props);\n\n const blobUri = await this.uploadToBlobWithRetry(fileToUpload, blobName);\n\n return this.ingestFromBlob(new BlobDescriptor(blobUri, blob.size, descriptor.sourceId), props);\n }\n\n /**\n * Use Readable for Node.js and ArrayBuffer in browser\n */\n async ingestFromStream(stream: ArrayBuffer | StreamDescriptor, ingestionProperties?: IngestionPropertiesInput): Promise<IngestionResult> {\n this.ensureOpen();\n const props = this._getMergedProps(ingestionProperties);\n const descriptor: StreamDescriptor = stream instanceof StreamDescriptor ? stream : new StreamDescriptor(stream);\n const blobName = generateBlobName(descriptor, props);\n const blobUri = await this.uploadToBlobWithRetry(descriptor.stream as ArrayBuffer, blobName);\n return this.ingestFromBlob(new BlobDescriptor(blobUri, descriptor.size, descriptor.sourceId), props);\n }\n}\n\nexport default KustoIngestClient;\n"]}
|
|
@@ -1,12 +1,9 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
// Copyright (c) Microsoft Corporation.
|
|
3
2
|
// Licensed under the MIT License.
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
const ingestClientBase_1 = require("./ingestClientBase");
|
|
9
|
-
class KustoIngestClient extends ingestClientBase_1.KustoIngestClientBase {
|
|
3
|
+
import { BlobDescriptor, generateBlobName, StreamDescriptor } from "./descriptors.js";
|
|
4
|
+
import { FileDescriptor } from "./fileDescriptor.js";
|
|
5
|
+
import { KustoIngestClientBase } from "./ingestClientBase.js";
|
|
6
|
+
export class KustoIngestClient extends KustoIngestClientBase {
|
|
10
7
|
constructor(kcsb, defaultProps, autoCorrectEndpoint) {
|
|
11
8
|
super(kcsb, defaultProps, autoCorrectEndpoint);
|
|
12
9
|
}
|
|
@@ -16,12 +13,12 @@ class KustoIngestClient extends ingestClientBase_1.KustoIngestClientBase {
|
|
|
16
13
|
async ingestFromFile(file, ingestionProperties) {
|
|
17
14
|
this.ensureOpen();
|
|
18
15
|
const props = this._getMergedProps(ingestionProperties);
|
|
19
|
-
const descriptor = file instanceof
|
|
16
|
+
const descriptor = file instanceof FileDescriptor ? file : new FileDescriptor(file);
|
|
20
17
|
try {
|
|
21
|
-
const blobName =
|
|
18
|
+
const blobName = generateBlobName(descriptor, props);
|
|
22
19
|
const fileToUpload = await descriptor.prepare(ingestionProperties);
|
|
23
20
|
const blobUri = await this.uploadToBlobWithRetry(fileToUpload, blobName);
|
|
24
|
-
return this.ingestFromBlob(new
|
|
21
|
+
return this.ingestFromBlob(new BlobDescriptor(blobUri, descriptor.size, descriptor.sourceId), props);
|
|
25
22
|
}
|
|
26
23
|
finally {
|
|
27
24
|
await descriptor.cleanup();
|
|
@@ -33,13 +30,12 @@ class KustoIngestClient extends ingestClientBase_1.KustoIngestClientBase {
|
|
|
33
30
|
async ingestFromStream(stream, ingestionProperties) {
|
|
34
31
|
this.ensureOpen();
|
|
35
32
|
const props = this._getMergedProps(ingestionProperties);
|
|
36
|
-
const descriptor = stream instanceof
|
|
33
|
+
const descriptor = stream instanceof StreamDescriptor ? stream : new StreamDescriptor(stream);
|
|
37
34
|
// TODO we don't gzip on stream here, just ont streamingIngestClient
|
|
38
|
-
const blobName =
|
|
35
|
+
const blobName = generateBlobName(descriptor, props);
|
|
39
36
|
const blobUri = await this.uploadToBlobWithRetry(descriptor, blobName);
|
|
40
|
-
return this.ingestFromBlob(new
|
|
37
|
+
return this.ingestFromBlob(new BlobDescriptor(blobUri), props); // descriptor.size?
|
|
41
38
|
}
|
|
42
39
|
}
|
|
43
|
-
|
|
44
|
-
exports.default = KustoIngestClient;
|
|
40
|
+
export default KustoIngestClient;
|
|
45
41
|
//# sourceMappingURL=ingestClient.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ingestClient.js","sourceRoot":"","sources":["../../src/ingestClient.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"ingestClient.js","sourceRoot":"","sources":["../../src/ingestClient.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAIlC,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACtF,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAGrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAI9D,MAAM,OAAO,iBAAkB,SAAQ,qBAAqB;IACxD,YAAY,IAA2C,EAAE,YAAuC,EAAE,mBAA6B;QAC3H,KAAK,CAAC,IAAI,EAAE,YAAY,EAAE,mBAAmB,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,IAAoC,EAAE,mBAA8C;QACrG,IAAI,CAAC,UAAU,EAAE,CAAC;QAClB,MAAM,KAAK,GAAG,IAAI,CAAC,eAAe,CAAC,mBAAmB,CAAC,CAAC;QAExD,MAAM,UAAU,GAAG,IAAI,YAAY,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC;QAEpF,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,gBAAgB,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YACrD,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;YACnE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;YACzE,OAAO,IAAI,CAAC,cAAc,CAAC,IAAI,cAAc,CAAC,OAAO,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,EAAE,KAAK,CAAC,CAAC;QACzG,CAAC;gBAAS,CAAC;YACP,MAAM,UAAU,CAAC,OAAO,EAAE,CAAC;QAC/B,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB,CAAC,MAAiD,EAAE,mBAA8C;QACpH,IAAI,CAAC,UAAU,EAAE,CAAC;QAClB,MAAM,KAAK,GAAG,IAAI,CAAC,eAAe,CAAC,mBAAmB,CAAC,CAAC;QACxD,MAAM,UAAU,GAAqB,MAAM,YAAY,gBAAgB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAChH,oEAAoE;QACpE,MAAM,QAAQ,GAAG,gBAAgB,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QAErD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QAEvE,OAAO,IAAI,CAAC,cAAc,CAAC,IAAI,cAAc,CAAC,OAAO,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,mBAAmB;IACvF,CAAC;CACJ;AAED,eAAe,iBAAiB,CAAC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport { KustoConnectionStringBuilder } from \"azure-kusto-data\";\n\nimport { BlobDescriptor, generateBlobName, StreamDescriptor } from \"./descriptors.js\";\nimport { FileDescriptor } from \"./fileDescriptor.js\";\n\nimport { IngestionPropertiesInput } from \"./ingestionProperties.js\";\nimport { KustoIngestClientBase } from \"./ingestClientBase.js\";\nimport { Readable } from \"stream\";\nimport { IngestionResult } from \"./ingestionResult.js\";\n\nexport class KustoIngestClient extends KustoIngestClientBase {\n constructor(kcsb: string | KustoConnectionStringBuilder, defaultProps?: IngestionPropertiesInput, autoCorrectEndpoint?: boolean) {\n super(kcsb, defaultProps, autoCorrectEndpoint);\n }\n\n /**\n * Use string in Node.JS and Blob in browser\n */\n async ingestFromFile(file: FileDescriptor | string | Blob, ingestionProperties?: IngestionPropertiesInput): Promise<IngestionResult> {\n this.ensureOpen();\n const props = this._getMergedProps(ingestionProperties);\n\n const descriptor = file instanceof FileDescriptor ? file : new FileDescriptor(file);\n\n try {\n const blobName = generateBlobName(descriptor, props);\n const fileToUpload = await descriptor.prepare(ingestionProperties);\n const blobUri = await this.uploadToBlobWithRetry(fileToUpload, blobName);\n return this.ingestFromBlob(new BlobDescriptor(blobUri, descriptor.size, descriptor.sourceId), props);\n } finally {\n await descriptor.cleanup();\n }\n }\n\n /**\n * Use Readable in Node.JS and ArrayBuffer in browser\n */\n async ingestFromStream(stream: StreamDescriptor | Readable | ArrayBuffer, ingestionProperties?: IngestionPropertiesInput): Promise<IngestionResult> {\n this.ensureOpen();\n const props = this._getMergedProps(ingestionProperties);\n const descriptor: StreamDescriptor = stream instanceof StreamDescriptor ? stream : new StreamDescriptor(stream);\n // TODO we don't gzip on stream here, just ont streamingIngestClient\n const blobName = generateBlobName(descriptor, props);\n\n const blobUri = await this.uploadToBlobWithRetry(descriptor, blobName);\n\n return this.ingestFromBlob(new BlobDescriptor(blobUri), props); // descriptor.size?\n }\n}\n\nexport default KustoIngestClient;\n"]}
|
|
@@ -1,31 +1,25 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
// Copyright (c) Microsoft Corporation.
|
|
3
2
|
// Licensed under the MIT License.
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
const ingestionProperties_1 = require("./ingestionProperties");
|
|
15
|
-
const abstractKustoClient_1 = require("./abstractKustoClient");
|
|
16
|
-
const ingestionResult_1 = require("./ingestionResult");
|
|
17
|
-
const descriptors_1 = require("./descriptors");
|
|
18
|
-
class KustoIngestClientBase extends abstractKustoClient_1.AbstractKustoClient {
|
|
3
|
+
import { Client as KustoClient, KustoConnectionStringBuilder } from "azure-kusto-data";
|
|
4
|
+
import ResourceManager from "./resourceManager.js";
|
|
5
|
+
import IngestionBlobInfo from "./ingestionBlobInfo.js";
|
|
6
|
+
import { ContainerClient } from "@azure/storage-blob";
|
|
7
|
+
import { QueueClient } from "@azure/storage-queue";
|
|
8
|
+
import { ReportLevel, ReportMethod } from "./ingestionProperties.js";
|
|
9
|
+
import { AbstractKustoClient } from "./abstractKustoClient.js";
|
|
10
|
+
import { TableReportIngestionResult, IngestionStatusInTableDescription, IngestionStatusResult, OperationStatus, putRecordInTable, } from "./ingestionResult.js";
|
|
11
|
+
import { BlobDescriptor, StreamDescriptor } from "./descriptors.js";
|
|
12
|
+
export class KustoIngestClientBase extends AbstractKustoClient {
|
|
19
13
|
constructor(kcsb, defaultProps, autoCorrectEndpoint = true, isBrowser) {
|
|
20
14
|
super(defaultProps);
|
|
21
15
|
if (typeof kcsb === "string") {
|
|
22
|
-
kcsb = new
|
|
16
|
+
kcsb = new KustoConnectionStringBuilder(kcsb);
|
|
23
17
|
}
|
|
24
18
|
if (autoCorrectEndpoint) {
|
|
25
19
|
kcsb.dataSource = this.getIngestionEndpoint(kcsb.dataSource);
|
|
26
20
|
}
|
|
27
|
-
const kustoClient = new
|
|
28
|
-
this.resourceManager = new
|
|
21
|
+
const kustoClient = new KustoClient(kcsb);
|
|
22
|
+
this.resourceManager = new ResourceManager(kustoClient, isBrowser);
|
|
29
23
|
this.defaultDatabase = kustoClient.defaultDatabase;
|
|
30
24
|
const clientDetails = kcsb.clientDetails();
|
|
31
25
|
this.applicationForTracing = clientDetails.applicationNameForTracing;
|
|
@@ -34,21 +28,21 @@ class KustoIngestClientBase extends abstractKustoClient_1.AbstractKustoClient {
|
|
|
34
28
|
async ingestFromBlob(blob, ingestionProperties, maxRetries = KustoIngestClientBase.MaxNumberOfRetryAttempts) {
|
|
35
29
|
this.ensureOpen();
|
|
36
30
|
const props = this._getMergedProps(ingestionProperties);
|
|
37
|
-
const descriptor = blob instanceof
|
|
31
|
+
const descriptor = blob instanceof BlobDescriptor ? blob : new BlobDescriptor(blob);
|
|
38
32
|
const authorizationContext = await this.resourceManager.getAuthorizationContext();
|
|
39
|
-
const ingestionBlobInfo = new
|
|
40
|
-
const reportToTable = props.reportLevel !==
|
|
33
|
+
const ingestionBlobInfo = new IngestionBlobInfo(descriptor, props, authorizationContext, this.applicationForTracing, this.clientVersionForTracing);
|
|
34
|
+
const reportToTable = props.reportLevel !== ReportLevel.DoNotReport && props.reportMethod !== ReportMethod.Queue;
|
|
41
35
|
if (reportToTable) {
|
|
42
36
|
const statusTableClient = await this.resourceManager.createStatusTable();
|
|
43
|
-
const status = this.createStatusObject(props,
|
|
44
|
-
await
|
|
45
|
-
const desc = new
|
|
37
|
+
const status = this.createStatusObject(props, OperationStatus.Pending, ingestionBlobInfo);
|
|
38
|
+
await putRecordInTable(statusTableClient, Object.assign(Object.assign({}, status), { partitionKey: ingestionBlobInfo.Id, rowKey: ingestionBlobInfo.Id }));
|
|
39
|
+
const desc = new IngestionStatusInTableDescription(statusTableClient.url, ingestionBlobInfo.Id, ingestionBlobInfo.Id);
|
|
46
40
|
ingestionBlobInfo.IngestionStatusInTable = desc;
|
|
47
41
|
await this.sendQueueMessage(maxRetries, ingestionBlobInfo);
|
|
48
|
-
return new
|
|
42
|
+
return new TableReportIngestionResult(desc, statusTableClient);
|
|
49
43
|
}
|
|
50
44
|
await this.sendQueueMessage(maxRetries, ingestionBlobInfo);
|
|
51
|
-
return new
|
|
45
|
+
return new IngestionStatusResult(this.createStatusObject(props, OperationStatus.Queued, ingestionBlobInfo));
|
|
52
46
|
}
|
|
53
47
|
async sendQueueMessage(maxRetries, blobInfo) {
|
|
54
48
|
const queues = await this.resourceManager.getIngestionQueues();
|
|
@@ -59,7 +53,7 @@ class KustoIngestClientBase extends abstractKustoClient_1.AbstractKustoClient {
|
|
|
59
53
|
const encoded = Buffer.from(ingestionBlobInfoJson).toString("base64");
|
|
60
54
|
const retryCount = Math.min(maxRetries, queues.length);
|
|
61
55
|
for (let i = 0; i < retryCount; i++) {
|
|
62
|
-
const queueClient = new
|
|
56
|
+
const queueClient = new QueueClient(queues[i].uri);
|
|
63
57
|
try {
|
|
64
58
|
const queueResponse = await queueClient.sendMessage(encoded);
|
|
65
59
|
this.resourceManager.reportResourceUsageResult(queueClient.accountName, true);
|
|
@@ -92,13 +86,13 @@ class KustoIngestClientBase extends abstractKustoClient_1.AbstractKustoClient {
|
|
|
92
86
|
const retryCount = Math.min(maxRetries, containers.length);
|
|
93
87
|
// Go over all containers and try to upload the file to the first one that succeeds
|
|
94
88
|
for (let i = 0; i < retryCount; i++) {
|
|
95
|
-
const containerClient = new
|
|
89
|
+
const containerClient = new ContainerClient(containers[i].uri);
|
|
96
90
|
const blockBlobClient = containerClient.getBlockBlobClient(blobName);
|
|
97
91
|
try {
|
|
98
92
|
if (typeof descriptor == "string") {
|
|
99
93
|
await blockBlobClient.uploadFile(descriptor);
|
|
100
94
|
}
|
|
101
|
-
else if (descriptor instanceof
|
|
95
|
+
else if (descriptor instanceof StreamDescriptor) {
|
|
102
96
|
if (descriptor.stream instanceof Buffer) {
|
|
103
97
|
await blockBlobClient.uploadData(descriptor.stream);
|
|
104
98
|
}
|
|
@@ -125,7 +119,6 @@ class KustoIngestClientBase extends abstractKustoClient_1.AbstractKustoClient {
|
|
|
125
119
|
super.close();
|
|
126
120
|
}
|
|
127
121
|
}
|
|
128
|
-
exports.KustoIngestClientBase = KustoIngestClientBase;
|
|
129
122
|
KustoIngestClientBase.MaxNumberOfRetryAttempts = 3;
|
|
130
|
-
|
|
123
|
+
export default KustoIngestClientBase;
|
|
131
124
|
//# sourceMappingURL=ingestClientBase.js.map
|