azure-kusto-ingest 5.0.3 → 5.1.0

package/dist-esm/src/descriptors.js

@@ -5,9 +5,10 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.generateBlobName = exports.BlobDescriptor = exports.StreamDescriptor = exports.getSourceId = exports.CompressionType = void 0;
+exports.generateBlobName = exports.BlobDescriptor = exports.StreamDescriptor = exports.AbstractDescriptor = exports.getSourceId = exports.CompressionType = void 0;
 const uuid_1 = require("uuid");
 const uuid_validate_1 = __importDefault(require("uuid-validate"));
+const storage_blob_1 = require("@azure/storage-blob");
 var CompressionType;
 (function (CompressionType) {
     CompressionType["ZIP"] = ".zip";
@@ -24,15 +25,22 @@ const getSourceId = (sourceId) => {
     return (0, uuid_1.v4)();
 };
 exports.getSourceId = getSourceId;
-class StreamDescriptor {
+class AbstractDescriptor {
+    constructor(sourceId = null, size = null) {
+        this.sourceId = sourceId;
+        this.size = size;
+        this.sourceId = (0, exports.getSourceId)(sourceId);
+    }
+}
+exports.AbstractDescriptor = AbstractDescriptor;
+class StreamDescriptor extends AbstractDescriptor {
     /**
      * Use Readable for Node.js and ArrayBuffer in browser
      */
-    constructor(stream, sourceId = null, compressionType = CompressionType.None) {
+    constructor(stream, sourceId = null, compressionType = CompressionType.None, size = null) {
+        super(sourceId, size);
         this.stream = stream;
-        this.size = null;
         this.compressionType = compressionType;
-        this.sourceId = (0, exports.getSourceId)(sourceId);
     }
     merge(other) {
         this.size = other.size;
@@ -46,11 +54,23 @@ class StreamDescriptor {
     }
 }
 exports.StreamDescriptor = StreamDescriptor;
-class BlobDescriptor {
+class BlobDescriptor extends AbstractDescriptor {
     constructor(path, size = null, sourceId = null) {
+        super(sourceId, size);
         this.path = path;
-        this.size = size;
-        this.sourceId = (0, exports.getSourceId)(sourceId);
+    }
+    async fillSize() {
+        if (!this.size) {
+            const blobClient = new storage_blob_1.BlobClient(this.path);
+            const blobProps = await blobClient.getProperties();
+            const length = blobProps.contentLength;
+            if (length !== undefined) {
+                if (length === 0) {
+                    throw new Error("Empty blob.");
+                }
+                this.size = length;
+            }
+        }
     }
 }
 exports.BlobDescriptor = BlobDescriptor;

package/dist-esm/src/fileDescriptor.browser.js

@@ -8,13 +8,12 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.FileDescriptor = void 0;
 const pako_1 = __importDefault(require("pako"));
 const descriptors_1 = require("./descriptors");
-class FileDescriptor {
+class FileDescriptor extends descriptors_1.AbstractDescriptor {
     constructor(file, sourceId = null, size = null, compressionType = descriptors_1.CompressionType.None, extension, name) {
+        super(sourceId);
         this.file = file;
-        this.sourceId = sourceId;
         this.extension = extension;
         this.name = name;
-        this.sourceId = (0, descriptors_1.getSourceId)(sourceId);
         this.compressionType = compressionType;
         this.size = size || file.size;
         this.zipped = compressionType !== descriptors_1.CompressionType.None || this.extension === ".gz" || this.extension === ".zip";

package/dist-esm/src/fileDescriptor.js

@@ -15,7 +15,7 @@ const descriptors_1 = require("./descriptors");
 /**
  * Describes a file to be ingested. Use string to describe a local path in Node.JS and Blob object in browsers
  */
-class FileDescriptor {
+class FileDescriptor extends descriptors_1.AbstractDescriptor {
     constructor(
     /**
      * Use string in Node.JS and Blob in browser
@@ -23,14 +23,13 @@ class FileDescriptor {
     file, sourceId = null, size = null, compressionType = descriptors_1.CompressionType.None, extension, // Extracted from file name by default
     name // Extracted from file name by default
     ) {
+        super(sourceId, size);
         this.file = file;
         this.extension = extension;
         this.name = name;
-        this.sourceId = (0, descriptors_1.getSourceId)(sourceId);
         this.compressionType = compressionType;
         this.name = name ? name : path_1.default.basename(this.file);
         this.extension = extension ? extension : path_1.default.extname(this.file).toLowerCase();
-        this.size = size;
         this.zipped = compressionType !== descriptors_1.CompressionType.None || this.extension === ".gz" || this.extension === ".zip";
     }
     async _gzip() {

package/dist-esm/src/index.js

@@ -5,11 +5,15 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.IngestionPropertiesValidationError = exports.IngestionPropertiesEnums = exports.ColumnMapping = exports.FieldTransformation = exports.ConstantTransformation = exports.SStreamColumnMapping = exports.ApacheAvroColumnMapping = exports.CompressionType = exports.IngestionMappingKind = exports.DataFormat = exports.ValidationOptions = exports.ValidationImplications = exports.ReportMethod = exports.ReportLevel = exports.ValidationPolicy = exports.W3CLogFileMapping = exports.OrcColumnMapping = exports.ParquetColumnMapping = exports.AvroColumnMapping = exports.CsvColumnMapping = exports.JsonColumnMapping = exports.IngestionDescriptors = exports.IngestionProperties = exports.IngestStatusQueues = exports.ManagedStreamingIngestClient = exports.StreamingIngestClient = exports.IngestClient = void 0;
+exports.IngestionPropertiesValidationError = exports.IngestionPropertiesEnums = exports.StreamDescriptor = exports.BlobDescriptor = exports.ColumnMapping = exports.FieldTransformation = exports.ConstantTransformation = exports.SStreamColumnMapping = exports.ApacheAvroColumnMapping = exports.CompressionType = exports.IngestionMappingKind = exports.DataFormat = exports.ValidationOptions = exports.ValidationImplications = exports.ReportMethod = exports.ReportLevel = exports.ValidationPolicy = exports.W3CLogFileMapping = exports.OrcColumnMapping = exports.ParquetColumnMapping = exports.AvroColumnMapping = exports.CsvColumnMapping = exports.JsonColumnMapping = exports.IngestionDescriptors = exports.IngestionProperties = exports.IngestStatusQueues = exports.ManagedStreamingIngestClient = exports.StreamingIngestClient = exports.IngestClient = void 0;
 const ingestClient_1 = __importDefault(require("./ingestClient"));
+exports.IngestClient = ingestClient_1.default;
 const streamingIngestClient_1 = __importDefault(require("./streamingIngestClient"));
+exports.StreamingIngestClient = streamingIngestClient_1.default;
 const managedStreamingIngestClient_1 = __importDefault(require("./managedStreamingIngestClient"));
+exports.ManagedStreamingIngestClient = managedStreamingIngestClient_1.default;
 const status_1 = __importDefault(require("./status"));
+exports.IngestStatusQueues = status_1.default;
 const ingestionProperties_1 = require("./ingestionProperties");
 Object.defineProperty(exports, "DataFormat", { enumerable: true, get: function () { return ingestionProperties_1.DataFormat; } });
 Object.defineProperty(exports, "IngestionMappingKind", { enumerable: true, get: function () { return ingestionProperties_1.IngestionMappingKind; } });
@@ -19,7 +23,9 @@ Object.defineProperty(exports, "ValidationImplications", { enumerable: true, get
 Object.defineProperty(exports, "ValidationOptions", { enumerable: true, get: function () { return ingestionProperties_1.ValidationOptions; } });
 Object.defineProperty(exports, "ValidationPolicy", { enumerable: true, get: function () { return ingestionProperties_1.ValidationPolicy; } });
 const descriptors_1 = require("./descriptors");
+Object.defineProperty(exports, "BlobDescriptor", { enumerable: true, get: function () { return descriptors_1.BlobDescriptor; } });
 Object.defineProperty(exports, "CompressionType", { enumerable: true, get: function () { return descriptors_1.CompressionType; } });
+Object.defineProperty(exports, "StreamDescriptor", { enumerable: true, get: function () { return descriptors_1.StreamDescriptor; } });
 const fileDescriptor_1 = require("./fileDescriptor");
 const columnMappings_1 = require("./columnMappings");
 Object.defineProperty(exports, "ApacheAvroColumnMapping", { enumerable: true, get: function () { return columnMappings_1.ApacheAvroColumnMapping; } });
@@ -33,12 +39,11 @@ Object.defineProperty(exports, "OrcColumnMapping", { enumerable: true, get: func
 Object.defineProperty(exports, "ParquetColumnMapping", { enumerable: true, get: function () { return columnMappings_1.ParquetColumnMapping; } });
 Object.defineProperty(exports, "SStreamColumnMapping", { enumerable: true, get: function () { return columnMappings_1.SStreamColumnMapping; } });
 Object.defineProperty(exports, "W3CLogFileMapping", { enumerable: true, get: function () { return columnMappings_1.W3CLogFileMapping; } });
-exports.IngestClient = ingestClient_1.default;
-exports.StreamingIngestClient = streamingIngestClient_1.default;
-exports.ManagedStreamingIngestClient = managedStreamingIngestClient_1.default;
-exports.IngestStatusQueues = status_1.default;
 var ingestionProperties_2 = require("./ingestionProperties");
 Object.defineProperty(exports, "IngestionProperties", { enumerable: true, get: function () { return ingestionProperties_2.IngestionProperties; } });
+/**
+ * @deprecated - import directly instead. Export const is not exporting type.
+ */
 exports.IngestionDescriptors = {
     BlobDescriptor: descriptors_1.BlobDescriptor,
     FileDescriptor: fileDescriptor_1.FileDescriptor,

package/dist-esm/src/ingestClient.js

@@ -36,7 +36,12 @@ class KustoIngestClient extends ingestClientBase_1.KustoIngestClientBase {
         const descriptor = stream instanceof descriptors_1.StreamDescriptor ? stream : new descriptors_1.StreamDescriptor(stream);
         const blobName = (0, descriptors_1.generateBlobName)(descriptor, props);
         const blockBlobClient = await this.resourceManager.getBlockBlobClient(blobName);
-        await blockBlobClient.uploadStream(descriptor.stream);
+        if (descriptor.stream instanceof Buffer) {
+            await blockBlobClient.uploadData(descriptor.stream);
+        }
+        else {
+            await blockBlobClient.uploadStream(descriptor.stream);
+        }
         return this.ingestFromBlob(new descriptors_1.BlobDescriptor(blockBlobClient.url), props); // descriptor.size?
     }
 }

package/dist-esm/src/ingestionBlobInfo.js

@@ -52,6 +52,9 @@ class IngestionBlobInfo {
         if (ingestionProperties.format) {
             additionalProperties.format = ingestionProperties.format;
         }
+        if (ingestionProperties.ignoreFirstRecord) {
+            additionalProperties.ignoreFirstRecord = ingestionProperties.ignoreFirstRecord;
+        }
         this.AdditionalProperties = additionalProperties;
     }
 }

package/dist-esm/src/managedStreamingIngestClient.js

@@ -67,20 +67,53 @@ class KustoManagedStreamingIngestClient extends abstractKustoClient_1.AbstractKu
     /**
      * Use Readable for Node.js and ArrayBuffer in browser
      */
-    async ingestFromStream(stream, ingestionProperties) {
+    async ingestFromStream(stream, ingestionProperties, clientRequestId) {
+        var _a;
         this.ensureOpen();
         const props = this._getMergedProps(ingestionProperties);
-        const descriptor = stream instanceof descriptors_1.StreamDescriptor ? stream : new descriptors_1.StreamDescriptor(stream);
+        let descriptor = stream instanceof descriptors_1.StreamDescriptor ? stream : new descriptors_1.StreamDescriptor(stream);
         let result = core_util_1.isNode ? await (0, streamUtils_1.tryStreamToArray)(descriptor.stream, maxStreamSize) : descriptor.stream;
-        if ((core_util_1.isNode && result instanceof Buffer) || (!core_util_1.isNode && descriptor.stream.byteLength < maxStreamSize)) {
+        descriptor = new descriptors_1.StreamDescriptor(result).merge(descriptor);
+        let streamingResult = null;
+        // tryStreamToArray returns a Buffer in NodeJS impl if stream size is small enouph
+        if ((core_util_1.isNode && result instanceof Buffer) || !core_util_1.isNode) {
+            streamingResult = await this.streamWithRetries(core_util_1.isNode ? (_a = descriptor.size) !== null && _a !== void 0 ? _a : 0 : descriptor.stream.byteLength, descriptor, props, clientRequestId, result);
+            result = core_util_1.isNode ? (0, stream_array_1.default)([result]) : descriptor.stream;
+        }
+        return streamingResult !== null && streamingResult !== void 0 ? streamingResult : this.queuedIngestClient.ingestFromStream(new descriptors_1.StreamDescriptor(result).merge(descriptor), props);
+    }
+    /**
+     * Use string for Node.js and Blob in browser
+     */
+    async ingestFromFile(file, ingestionProperties) {
+        this.ensureOpen();
+        const stream = file instanceof fileDescriptor_1.FileDescriptor ? await (0, streamUtils_1.tryFileToBuffer)(file) : await (0, streamUtils_1.tryFileToBuffer)(new fileDescriptor_1.FileDescriptor(file));
+        return await this.ingestFromStream(stream, ingestionProperties);
+    }
+    async ingestFromBlob(blob, ingestionProperties, clientRequestId) {
+        var _a;
+        const props = this._getMergedProps(ingestionProperties);
+        const descriptor = blob instanceof descriptors_1.BlobDescriptor ? blob : new descriptors_1.BlobDescriptor(blob);
+        // No need to check blob size if it was given to us that it's not empty
+        await descriptor.fillSize();
+        const streamingResult = await this.streamWithRetries((_a = descriptor.size) !== null && _a !== void 0 ? _a : 0, descriptor, props, clientRequestId);
+        return streamingResult !== null && streamingResult !== void 0 ? streamingResult : this.queuedIngestClient.ingestFromBlob(descriptor, props);
+    }
+    async streamWithRetries(length, descriptor, props, clientRequestId, stream) {
+        const isBlob = descriptor instanceof descriptors_1.BlobDescriptor;
+        if (length <= maxStreamSize) {
             // If we get buffer that means it was less than the max size, so we can do streamingIngestion
             const retry = new retry_1.ExponentialRetry(attemptCount, this.baseSleepTimeSecs, this.baseJitterSecs);
             while (retry.shouldTry()) {
                 try {
-                    const sourceId = `KNC.executeManagedStreamingIngest;${descriptor.sourceId};${retry.currentAttempt}`;
-                    return core_util_1.isNode
-                        ? await this.streamingIngestClient.ingestFromStream(new descriptors_1.StreamDescriptor((0, stream_array_1.default)([result])).merge(descriptor), props, sourceId)
-                        : await this.streamingIngestClient.ingestFromStream(descriptor, props, sourceId);
+                    const sourceId = clientRequestId !== null && clientRequestId !== void 0 ? clientRequestId : `KNC.executeManagedStreamingIngest${isBlob ? "FromBlob" : "FromStream"};${descriptor.sourceId};${retry.currentAttempt}`;
+                    if (isBlob) {
+                        return this.streamingIngestClient.ingestFromBlob(descriptor, props, sourceId);
+                    }
+                    if (core_util_1.isNode) {
+                        return await this.streamingIngestClient.ingestFromStream(new descriptors_1.StreamDescriptor((0, stream_array_1.default)([stream])).merge(descriptor), props, sourceId);
+                    }
+                    return await this.streamingIngestClient.ingestFromStream(descriptor, props, sourceId);
                 }
                 catch (err) {
                     const oneApiError = err;
@@ -90,17 +123,9 @@ class KustoManagedStreamingIngestClient extends abstractKustoClient_1.AbstractKu
                     await retry.backoff();
                 }
             }
-            result = core_util_1.isNode ? (0, stream_array_1.default)([result]) : descriptor.stream;
+            stream = isBlob ? undefined : core_util_1.isNode ? (0, stream_array_1.default)([stream]) : descriptor.stream;
         }
-        return await this.queuedIngestClient.ingestFromStream(new descriptors_1.StreamDescriptor(result).merge(descriptor), ingestionProperties);
-    }
-    /**
-     * Use string for Node.js and Blob in browser
-     */
-    async ingestFromFile(file, ingestionProperties) {
-        this.ensureOpen();
-        const stream = file instanceof fileDescriptor_1.FileDescriptor ? await (0, streamUtils_1.tryFileToBuffer)(file) : await (0, streamUtils_1.tryFileToBuffer)(new fileDescriptor_1.FileDescriptor(file));
-        return await this.ingestFromStream(stream, ingestionProperties);
+        return null;
     }
     close() {
         if (!this._isClosed) {

package/dist-esm/src/resourceManager.js

@@ -33,19 +33,16 @@ class ResourceManager {
         this.baseSleepTimeSecs = 1;
         this.baseJitterSecs = 1;
         this.refreshPeriod = azure_kusto_data_1.TimeUtils.toMilliseconds(1, 0, 0);
+        this.refreshPeriodOnError = azure_kusto_data_1.TimeUtils.toMilliseconds(0, 10, 0);
         this.ingestClientResources = null;
         this.ingestClientResourcesLastUpdate = null;
         this.authorizationContext = null;
         this.authorizationContextLastUpdate = null;
     }
     async refreshIngestClientResources() {
-        const now = Date.now();
-        if (!this.ingestClientResources ||
-            !this.ingestClientResourcesLastUpdate ||
-            this.ingestClientResourcesLastUpdate + this.refreshPeriod <= now ||
-            !this.ingestClientResources.valid()) {
-            this.ingestClientResources = await this.getIngestClientResourcesFromService();
-            this.ingestClientResourcesLastUpdate = now;
+        const error = await this.tryRefresh(false);
+        if (!this.ingestClientResources) {
+            throw new Error(`Failed to fetch ingestion resources from service.  ${error === null || error === void 0 ? void 0 : error.message}.\n ${error === null || error === void 0 ? void 0 : error.stack}`);
         }
         return this.ingestClientResources;
     }
@@ -56,7 +53,11 @@ class ResourceManager {
                 const cmd = `.get ingestion resources ${this.isBrowser ? `with (EnableBlobCors='true', EnableQueueCors='true', EnableTableCors='true')` : ""}`;
                 const response = await this.kustoClient.execute("NetDefaultDB", cmd);
                 const table = response.primaryResults[0];
-                return new IngestClientResources(this.getResourceByName(table, "SecuredReadyForAggregationQueue"), this.getResourceByName(table, "FailedIngestionsQueue"), this.getResourceByName(table, "SuccessfulIngestionsQueue"), this.getResourceByName(table, "TempStorage"));
+                const resoures = new IngestClientResources(this.getResourceByName(table, "SecuredReadyForAggregationQueue"), this.getResourceByName(table, "FailedIngestionsQueue"), this.getResourceByName(table, "SuccessfulIngestionsQueue"), this.getResourceByName(table, "TempStorage"));
+                if (!resoures.valid()) {
+                    throw new Error("Unexpected error occured - fetched data returned missing resource");
+                }
+                return resoures;
             }
             catch (error) {
                 if (!(error instanceof azure_kusto_data_1.KustoDataErrors.ThrottlingError)) {
@@ -78,16 +79,34 @@ class ResourceManager {
         return result;
     }
     async refreshAuthorizationContext() {
+        const error = await this.tryRefresh(true);
+        if (this.authorizationContext == null) {
+            throw new Error(`Failed to fetch Authorization context from service.  ${error === null || error === void 0 ? void 0 : error.message}.\n ${error === null || error === void 0 ? void 0 : error.stack}`);
+        }
+        return this.authorizationContext;
+    }
+    async tryRefresh(isAuthContextFetch) {
         var _a;
+        const resource = isAuthContextFetch ? (_a = this.authorizationContext) === null || _a === void 0 ? void 0 : _a.trim() : this.ingestClientResources;
+        const lastRefresh = isAuthContextFetch ? this.authorizationContextLastUpdate : this.ingestClientResourcesLastUpdate;
         const now = Date.now();
-        if (!((_a = this.authorizationContext) === null || _a === void 0 ? void 0 : _a.trim()) || !this.authorizationContextLastUpdate || this.authorizationContextLastUpdate + this.refreshPeriod <= now) {
-            this.authorizationContext = await this.getAuthorizationContextFromService();
-            this.authorizationContextLastUpdate = now;
-            if (this.authorizationContext == null) {
-                throw new Error("Authorization context can't be null");
+        let error = null;
+        if (!resource || !lastRefresh || lastRefresh + this.refreshPeriod <= now) {
+            try {
+                if (isAuthContextFetch) {
+                    this.authorizationContext = await this.getAuthorizationContextFromService();
+                    this.authorizationContextLastUpdate = now;
+                }
+                else {
+                    this.ingestClientResources = await this.getIngestClientResourcesFromService();
+                    this.ingestClientResourcesLastUpdate = now;
+                }
+            }
+            catch (e) {
+                error = e;
             }
         }
-        return this.authorizationContext;
+        return error;
     }
     async getAuthorizationContextFromService() {
         const retry = new retry_1.ExponentialRetry(ATTEMPT_COUNT, this.baseSleepTimeSecs, this.baseJitterSecs);

package/dist-esm/src/streamingIngestClient.browser.js

@@ -4,14 +4,11 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 const descriptors_1 = require("./descriptors");
 const fileDescriptor_browser_1 = require("./fileDescriptor.browser");
-const abstractKustoClient_1 = require("./abstractKustoClient");
-const azure_kusto_data_1 = require("azure-kusto-data");
 const streamUtils_browser_1 = require("./streamUtils.browser");
-class KustoStreamingIngestClient extends abstractKustoClient_1.AbstractKustoClient {
+const streamingIngestClientBase_1 = require("./streamingIngestClientBase");
+class KustoStreamingIngestClient extends streamingIngestClientBase_1.KustoStreamingIngestClientBase {
     constructor(kcsb, defaultProps) {
-        super(defaultProps);
-        this.kustoClient = new azure_kusto_data_1.Client(kcsb);
-        this.defaultDatabase = this.kustoClient.defaultDatabase;
+        super(kcsb, defaultProps);
     }
     /**
      * Use Readable for Node.js and ArrayBuffer in browser
@@ -31,12 +28,6 @@ class KustoStreamingIngestClient extends abstractKustoClient_1.AbstractKustoClie
         const descriptor = file instanceof fileDescriptor_browser_1.FileDescriptor ? file : new fileDescriptor_browser_1.FileDescriptor(file);
         return this.ingestFromStream(await (0, streamUtils_browser_1.tryFileToBuffer)(descriptor), ingestionProperties);
     }
-    close() {
-        if (!this._isClosed) {
-            this.kustoClient.close();
-        }
-        super.close();
-    }
 }
 exports.default = KustoStreamingIngestClient;
 //# sourceMappingURL=streamingIngestClient.browser.js.map

package/dist-esm/src/streamingIngestClient.js

@@ -8,14 +8,11 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const descriptors_1 = require("./descriptors");
 const fileDescriptor_1 = require("./fileDescriptor");
 const zlib_1 = __importDefault(require("zlib"));
-const abstractKustoClient_1 = require("./abstractKustoClient");
-const azure_kusto_data_1 = require("azure-kusto-data");
 const streamUtils_1 = require("./streamUtils");
-class KustoStreamingIngestClient extends abstractKustoClient_1.AbstractKustoClient {
+const streamingIngestClientBase_1 = require("./streamingIngestClientBase");
+class KustoStreamingIngestClient extends streamingIngestClientBase_1.KustoStreamingIngestClientBase {
     constructor(kcsb, defaultProps) {
-        super(defaultProps);
-        this.kustoClient = new azure_kusto_data_1.Client(kcsb);
-        this.defaultDatabase = this.kustoClient.defaultDatabase;
+        super(kcsb, defaultProps);
     }
     /**
      * Use Readable for Node.js and ArrayBuffer in browser
@@ -40,12 +37,6 @@ class KustoStreamingIngestClient extends abstractKustoClient_1.AbstractKustoClie
         const descriptor = file instanceof fileDescriptor_1.FileDescriptor ? file : new fileDescriptor_1.FileDescriptor(file);
         return this.ingestFromStream(await (0, streamUtils_1.fileToStream)(descriptor), ingestionProperties);
     }
-    close() {
-        if (!this._isClosed) {
-            this.kustoClient.close();
-        }
-        super.close();
-    }
 }
 exports.default = KustoStreamingIngestClient;
 //# sourceMappingURL=streamingIngestClient.js.map

package/dist-esm/src/streamingIngestClientBase.js

@@ -0,0 +1,31 @@
+"use strict";
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.KustoStreamingIngestClientBase = void 0;
+const descriptors_1 = require("./descriptors");
+const abstractKustoClient_1 = require("./abstractKustoClient");
+const azure_kusto_data_1 = require("azure-kusto-data");
+class KustoStreamingIngestClientBase extends abstractKustoClient_1.AbstractKustoClient {
+    constructor(kcsb, defaultProps) {
+        super(defaultProps);
+        this.kustoClient = new azure_kusto_data_1.Client(kcsb);
+        this.defaultDatabase = this.kustoClient.defaultDatabase;
+    }
+    async ingestFromBlob(blob, ingestionProperties, clientRequestId) {
+        var _a;
+        const props = this._getMergedProps(ingestionProperties);
+        const descriptor = blob instanceof descriptors_1.BlobDescriptor ? blob : new descriptors_1.BlobDescriptor(blob);
+        // No need to check blob size if it was given to us that it's not empty
+        await descriptor.fillSize();
+        return await this.kustoClient.executeStreamingIngest(props.database, props.table, undefined, props.format, (_a = props.ingestionMappingReference) !== null && _a !== void 0 ? _a : null, descriptor.path, clientRequestId);
+    }
+    close() {
+        if (!this._isClosed) {
+            this.kustoClient.close();
+        }
+        super.close();
+    }
+}
+exports.KustoStreamingIngestClientBase = KustoStreamingIngestClientBase;
+//# sourceMappingURL=streamingIngestClientBase.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "azure-kusto-ingest",
-    "version": "5.0.3",
+    "version": "5.1.0",
     "description": "Azure Data Explorer Ingestion SDK",
     "module": "dist-esm/src/index.js",
     "types": "./types/src/index.d.ts",
@@ -61,7 +61,7 @@
         "@types/tmp": "^0.2.3",
         "@types/uuid": "^8.3.4",
         "@types/uuid-validate": "0.0.1",
-        "azure-kusto-data": "^5.0.3",
+        "azure-kusto-data": "^5.1.0",
         "browserify-zlib": "0.2.0",
         "stream-array": "^1.1.2",
         "stream-browserify": "3.0.0",
@@ -74,5 +74,5 @@
         "@types/sinon": "^10.0.13",
         "assert": "^2.0.0"
     },
-    "gitHead": "f8bf719744dfee126942678185a2315511ca0dfa"
+    "gitHead": "120a8d91fde4b0e520225e09895a28b085b85a7e"
 }

package/types/src/abstractKustoClient.d.ts

@@ -1,5 +1,5 @@
 import { IngestionProperties, IngestionPropertiesInput } from "./ingestionProperties";
-import { StreamDescriptor, FileDescriptorBase } from "./descriptors";
+import { StreamDescriptor, FileDescriptorBase, BlobDescriptor } from "./descriptors";
 export declare abstract class AbstractKustoClient {
     defaultProps: IngestionProperties;
     defaultDatabase?: string;
@@ -8,6 +8,7 @@ export declare abstract class AbstractKustoClient {
     _getMergedProps(newProperties?: IngestionPropertiesInput): IngestionProperties;
     abstract ingestFromStream(stream: StreamDescriptor, ingestionProperties: IngestionPropertiesInput): Promise<any>;
     abstract ingestFromFile(file: FileDescriptorBase | string | Blob, ingestionProperties: IngestionPropertiesInput): Promise<any>;
+    abstract ingestFromBlob(blob: string | BlobDescriptor, ingestionProperties?: IngestionPropertiesInput): Promise<any>;
     close(): void;
     protected ensureOpen(): void;
 }

package/types/src/descriptors.d.ts

@@ -7,23 +7,25 @@ export declare enum CompressionType {
     None = ""
 }
 export declare const getSourceId: (sourceId: string | null) => string;
-export declare class StreamDescriptor {
-    readonly stream: Readable | ArrayBuffer;
+export declare abstract class AbstractDescriptor {
+    sourceId: string | null;
     size: number | null;
+    constructor(sourceId?: string | null, size?: number | null);
+}
+export declare class StreamDescriptor extends AbstractDescriptor {
+    readonly stream: Readable | ArrayBuffer;
     compressionType: CompressionType;
-    sourceId: string;
     /**
      * Use Readable for Node.js and ArrayBuffer in browser
      */
-    constructor(stream: Readable | ArrayBuffer, sourceId?: string | null, compressionType?: CompressionType);
+    constructor(stream: Readable | ArrayBuffer, sourceId?: string | null, compressionType?: CompressionType, size?: number | null);
     merge(other: StreamDescriptor): this;
     getCompressionSuffix(): string;
 }
-export declare class BlobDescriptor {
+export declare class BlobDescriptor extends AbstractDescriptor {
     readonly path: string;
-    size: number | null;
-    sourceId: string;
     constructor(path: string, size?: number | null, sourceId?: string | null);
+    fillSize(): Promise<void>;
 }
 export interface FileDescriptorBase {
     size: number | null;

package/types/src/fileDescriptor.browser.d.ts

@@ -1,7 +1,6 @@
-import { CompressionType, FileDescriptorBase } from "./descriptors";
-export declare class FileDescriptor implements FileDescriptorBase {
+import { AbstractDescriptor, CompressionType, FileDescriptorBase } from "./descriptors";
+export declare class FileDescriptor extends AbstractDescriptor implements FileDescriptorBase {
     readonly file: Blob;
-    readonly sourceId: string | null;
     readonly extension?: string | undefined;
     readonly name?: string | undefined;
     size: number | null;

package/types/src/fileDescriptor.d.ts

@@ -1,19 +1,17 @@
-import { CompressionType, FileDescriptorBase } from "./descriptors";
+import { AbstractDescriptor, CompressionType, FileDescriptorBase } from "./descriptors";
 /**
  * Describes a file to be ingested. Use string to describe a local path in Node.JS and Blob object in browsers
  */
-export declare class FileDescriptor implements FileDescriptorBase {
+export declare class FileDescriptor extends AbstractDescriptor implements FileDescriptorBase {
     /**
      * Use string in Node.JS and Blob in browser
      */
     readonly file: string | Blob;
     readonly extension?: string | undefined;
     readonly name?: string | undefined;
-    size: number | null;
     zipped: boolean;
     compressionType: CompressionType;
     cleanupTmp?: () => Promise<void>;
-    sourceId: string;
     constructor(
     /**
      * Use string in Node.JS and Blob in browser

package/types/src/index.d.ts

@@ -7,17 +7,20 @@ import { BlobDescriptor, CompressionType, StreamDescriptor } from "./descriptors
 import { FileDescriptor } from "./fileDescriptor";
 import { ApacheAvroColumnMapping, AvroColumnMapping, ColumnMapping, ConstantTransformation, CsvColumnMapping, FieldTransformation, JsonColumnMapping, OrcColumnMapping, ParquetColumnMapping, SStreamColumnMapping, W3CLogFileMapping } from "./columnMappings";
 export { Transformation as ColumnMappingTransformation } from "./columnMappings";
-export declare const IngestClient: typeof KustoIngestClient;
-export declare const StreamingIngestClient: typeof streamingIngestClient;
-export declare const ManagedStreamingIngestClient: typeof managedStreamingIngestClient;
-export declare const IngestStatusQueues: typeof KustoIngestStatusQueues;
+export { KustoIngestClient as IngestClient };
+export { streamingIngestClient as StreamingIngestClient };
+export { managedStreamingIngestClient as ManagedStreamingIngestClient };
+export { KustoIngestStatusQueues as IngestStatusQueues };
 export { IngestionProperties } from "./ingestionProperties";
+/**
+ * @deprecated - import directly instead. Export const is not exporting type.
+ */
 export declare const IngestionDescriptors: {
     BlobDescriptor: typeof BlobDescriptor;
     FileDescriptor: typeof FileDescriptor;
     StreamDescriptor: typeof StreamDescriptor;
 };
-export { JsonColumnMapping, CsvColumnMapping, AvroColumnMapping, ParquetColumnMapping, OrcColumnMapping, W3CLogFileMapping, ValidationPolicy, ReportLevel, ReportMethod, ValidationImplications, ValidationOptions, DataFormat, IngestionMappingKind, CompressionType, ApacheAvroColumnMapping, SStreamColumnMapping, ConstantTransformation, FieldTransformation, ColumnMapping, };
+export { JsonColumnMapping, CsvColumnMapping, AvroColumnMapping, ParquetColumnMapping, OrcColumnMapping, W3CLogFileMapping, ValidationPolicy, ReportLevel, ReportMethod, ValidationImplications, ValidationOptions, DataFormat, IngestionMappingKind, CompressionType, ApacheAvroColumnMapping, SStreamColumnMapping, ConstantTransformation, FieldTransformation, ColumnMapping, BlobDescriptor, StreamDescriptor, };
 /**
  * @deprecated - import directly instead
  */

package/types/src/managedStreamingIngestClient.d.ts

@@ -1,6 +1,6 @@
 /// <reference types="node" />
 import { IngestionPropertiesInput } from "./ingestionProperties";
-import { StreamDescriptor } from "./descriptors";
+import { AbstractDescriptor, BlobDescriptor, StreamDescriptor } from "./descriptors";
 import { FileDescriptor } from "./fileDescriptor";
 import { AbstractKustoClient } from "./abstractKustoClient";
 import { KustoConnectionStringBuilder } from "azure-kusto-data";
@@ -34,11 +34,13 @@ declare class KustoManagedStreamingIngestClient extends AbstractKustoClient {
     /**
      * Use Readable for Node.js and ArrayBuffer in browser
      */
-    ingestFromStream(stream: StreamDescriptor | Readable | ArrayBuffer, ingestionProperties?: IngestionPropertiesInput): Promise<any>;
+    ingestFromStream(stream: StreamDescriptor | Readable | ArrayBuffer, ingestionProperties?: IngestionPropertiesInput, clientRequestId?: string): Promise<any>;
     /**
      * Use string for Node.js and Blob in browser
      */
     ingestFromFile(file: FileDescriptor | string | Blob, ingestionProperties?: IngestionPropertiesInput): Promise<KustoResponseDataSet | QueueSendMessageResponse>;
+    ingestFromBlob(blob: string | BlobDescriptor, ingestionProperties?: IngestionPropertiesInput, clientRequestId?: string): Promise<any>;
+    streamWithRetries(length: number, descriptor: AbstractDescriptor, props?: IngestionPropertiesInput, clientRequestId?: string, stream?: Readable | ArrayBuffer): Promise<any>;
     close(): void;
 }
 export default KustoManagedStreamingIngestClient;

package/types/src/resourceManager.d.ts

@@ -15,6 +15,7 @@ export declare class ResourceManager {
     readonly kustoClient: Client;
     readonly isBrowser: boolean;
     readonly refreshPeriod: number;
+    refreshPeriodOnError: number;
     ingestClientResources: IngestClientResources | null;
     ingestClientResourcesLastUpdate: number | null;
     authorizationContext: string | null;
@@ -28,6 +29,7 @@ export declare class ResourceManager {
         rows: () => any;
     }, resourceName: string): ResourceURI[];
     refreshAuthorizationContext(): Promise<string>;
+    tryRefresh(isAuthContextFetch: boolean): Promise<Error | null>;
     getAuthorizationContextFromService(): Promise<string>;
     getIngestionQueues(): Promise<ResourceURI[] | null>;
     getFailedIngestionsQueues(): Promise<ResourceURI[] | null>;

package/types/src/streamingIngestClient.browser.d.ts

@@ -1,11 +1,10 @@
 import { IngestionPropertiesInput } from "./ingestionProperties";
 import { StreamDescriptor } from "./descriptors";
 import { FileDescriptor } from "./fileDescriptor.browser";
-import { AbstractKustoClient } from "./abstractKustoClient";
 import { KustoConnectionStringBuilder } from "azure-kusto-data";
 import { KustoResponseDataSet } from "azure-kusto-data/src/response";
-declare class KustoStreamingIngestClient extends AbstractKustoClient {
-    private kustoClient;
+import { KustoStreamingIngestClientBase } from "./streamingIngestClientBase";
+declare class KustoStreamingIngestClient extends KustoStreamingIngestClientBase {
     constructor(kcsb: string | KustoConnectionStringBuilder, defaultProps?: IngestionPropertiesInput);
     /**
      * Use Readable for Node.js and ArrayBuffer in browser
@@ -15,7 +14,6 @@ declare class KustoStreamingIngestClient extends AbstractKustoClient {
      * Use string for Node.js and Blob in browser
      */
     ingestFromFile(file: FileDescriptor | Blob, ingestionProperties?: IngestionPropertiesInput): Promise<KustoResponseDataSet>;
-    close(): void;
 }
 export default KustoStreamingIngestClient;
 //# sourceMappingURL=streamingIngestClient.browser.d.ts.map

package/types/src/streamingIngestClient.d.ts

@@ -2,12 +2,11 @@
 import { IngestionPropertiesInput } from "./ingestionProperties";
 import { StreamDescriptor } from "./descriptors";
 import { FileDescriptor } from "./fileDescriptor";
-import { AbstractKustoClient } from "./abstractKustoClient";
 import { KustoConnectionStringBuilder } from "azure-kusto-data";
 import { KustoResponseDataSet } from "azure-kusto-data/src/response";
 import { Readable } from "stream";
-declare class KustoStreamingIngestClient extends AbstractKustoClient {
-    private kustoClient;
+import { KustoStreamingIngestClientBase } from "./streamingIngestClientBase";
+declare class KustoStreamingIngestClient extends KustoStreamingIngestClientBase {
     constructor(kcsb: string | KustoConnectionStringBuilder, defaultProps?: IngestionPropertiesInput);
     /**
      * Use Readable for Node.js and ArrayBuffer in browser
@@ -17,7 +16,6 @@ declare class KustoStreamingIngestClient extends AbstractKustoClient {
      * Use string for Node.js and Blob in browser
      */
     ingestFromFile(file: FileDescriptor | string | Blob, ingestionProperties?: IngestionPropertiesInput): Promise<KustoResponseDataSet>;
-    close(): void;
 }
 export default KustoStreamingIngestClient;
 //# sourceMappingURL=streamingIngestClient.d.ts.map

package/types/src/streamingIngestClientBase.d.ts

@@ -0,0 +1,11 @@
+import { IngestionPropertiesInput } from "./ingestionProperties";
+import { BlobDescriptor } from "./descriptors";
+import { AbstractKustoClient } from "./abstractKustoClient";
+import { Client as KustoClient, KustoConnectionStringBuilder } from "azure-kusto-data";
+export declare abstract class KustoStreamingIngestClientBase extends AbstractKustoClient {
+    protected kustoClient: KustoClient;
+    constructor(kcsb: string | KustoConnectionStringBuilder, defaultProps?: IngestionPropertiesInput);
+    ingestFromBlob(blob: string | BlobDescriptor, ingestionProperties?: IngestionPropertiesInput, clientRequestId?: string): Promise<any>;
+    close(): void;
+}
+//# sourceMappingURL=streamingIngestClientBase.d.ts.map