azure-kusto-ingest 4.0.5 → 5.0.0

package/dist-esm/src/resourceManager.js

@@ -0,0 +1,142 @@
+"use strict";
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ResourceManager = exports.IngestClientResources = exports.ResourceURI = void 0;
+const azure_kusto_data_1 = require("azure-kusto-data");
+const retry_1 = require("./retry");
+const storage_blob_1 = require("@azure/storage-blob");
+const ATTEMPT_COUNT = 4;
+class ResourceURI {
+    constructor(uri) {
+        this.uri = uri;
+    }
+}
+exports.ResourceURI = ResourceURI;
+class IngestClientResources {
+    constructor(securedReadyForAggregationQueues = null, failedIngestionsQueues = null, successfulIngestionsQueues = null, containers = null) {
+        this.securedReadyForAggregationQueues = securedReadyForAggregationQueues;
+        this.failedIngestionsQueues = failedIngestionsQueues;
+        this.successfulIngestionsQueues = successfulIngestionsQueues;
+        this.containers = containers;
+    }
+    valid() {
+        const resources = [this.securedReadyForAggregationQueues, this.failedIngestionsQueues, this.failedIngestionsQueues, this.containers];
+        return resources.reduce((prev, current) => !!(prev && current), true);
+    }
+}
+exports.IngestClientResources = IngestClientResources;
+class ResourceManager {
+    constructor(kustoClient, isBrowser = false) {
+        this.kustoClient = kustoClient;
+        this.isBrowser = isBrowser;
+        this.baseSleepTimeSecs = 1;
+        this.baseJitterSecs = 1;
+        this.refreshPeriod = azure_kusto_data_1.TimeUtils.toMilliseconds(1, 0, 0);
+        this.ingestClientResources = null;
+        this.ingestClientResourcesLastUpdate = null;
+        this.authorizationContext = null;
+        this.authorizationContextLastUpdate = null;
+    }
+    async refreshIngestClientResources() {
+        const now = Date.now();
+        if (!this.ingestClientResources ||
+            !this.ingestClientResourcesLastUpdate ||
+            this.ingestClientResourcesLastUpdate + this.refreshPeriod <= now ||
+            !this.ingestClientResources.valid()) {
+            this.ingestClientResources = await this.getIngestClientResourcesFromService();
+            this.ingestClientResourcesLastUpdate = now;
+        }
+        return this.ingestClientResources;
+    }
+    async getIngestClientResourcesFromService() {
+        const retry = new retry_1.ExponentialRetry(ATTEMPT_COUNT, this.baseSleepTimeSecs, this.baseJitterSecs);
+        while (retry.shouldTry()) {
+            try {
+                const cmd = `.get ingestion resources ${this.isBrowser ? `with (EnableBlobCors='true', EnableQueueCors='true', EnableTableCors='true')` : ""}`;
+                const response = await this.kustoClient.execute("NetDefaultDB", cmd);
+                const table = response.primaryResults[0];
+                return new IngestClientResources(this.getResourceByName(table, "SecuredReadyForAggregationQueue"), this.getResourceByName(table, "FailedIngestionsQueue"), this.getResourceByName(table, "SuccessfulIngestionsQueue"), this.getResourceByName(table, "TempStorage"));
+            }
+            catch (error) {
+                if (!(error instanceof azure_kusto_data_1.KustoDataErrors.ThrottlingError)) {
+                    throw error;
+                }
+                await retry.backoff();
+            }
+        }
+        throw new Error(`Failed to get ingestion resources from server - the request was throttled ${ATTEMPT_COUNT} times.`);
+    }
+    getResourceByName(table, resourceName) {
+        const result = [];
+        for (const row of table.rows()) {
+            const typedRow = row;
+            if (typedRow.ResourceTypeName === resourceName) {
+                result.push(new ResourceURI(typedRow.StorageRoot));
+            }
+        }
+        return result;
+    }
+    async refreshAuthorizationContext() {
+        var _a;
+        const now = Date.now();
+        if (!((_a = this.authorizationContext) === null || _a === void 0 ? void 0 : _a.trim()) || !this.authorizationContextLastUpdate || this.authorizationContextLastUpdate + this.refreshPeriod <= now) {
+            this.authorizationContext = await this.getAuthorizationContextFromService();
+            this.authorizationContextLastUpdate = now;
+            if (this.authorizationContext == null) {
+                throw new Error("Authorization context can't be null");
+            }
+        }
+        return this.authorizationContext;
+    }
+    async getAuthorizationContextFromService() {
+        const retry = new retry_1.ExponentialRetry(ATTEMPT_COUNT, this.baseSleepTimeSecs, this.baseJitterSecs);
+        while (retry.shouldTry()) {
+            try {
+                const response = await this.kustoClient.execute("NetDefaultDB", ".get kusto identity token");
+                const next = response.primaryResults[0].rows().next();
+                if (next.done) {
+                    throw new Error("Failed to get authorization context - got empty results");
+                }
+                return next.value.toJSON().AuthorizationContext;
+            }
+            catch (error) {
+                if (!(error instanceof azure_kusto_data_1.KustoDataErrors.ThrottlingError)) {
+                    throw error;
+                }
+                await retry.backoff();
+            }
+        }
+        throw new Error(`Failed to get identity token from server - the request was throttled ${ATTEMPT_COUNT} times.`);
+    }
+    async getIngestionQueues() {
+        return (await this.refreshIngestClientResources()).securedReadyForAggregationQueues;
+    }
+    async getFailedIngestionsQueues() {
+        return (await this.refreshIngestClientResources()).failedIngestionsQueues;
+    }
+    async getSuccessfulIngestionsQueues() {
+        return (await this.refreshIngestClientResources()).successfulIngestionsQueues;
+    }
+    async getContainers() {
+        return (await this.refreshIngestClientResources()).containers;
+    }
+    async getAuthorizationContext() {
+        return this.refreshAuthorizationContext();
+    }
+    async getBlockBlobClient(blobName) {
+        const containers = await this.getContainers();
+        if (containers == null) {
+            throw new Error("Failed to get containers");
+        }
+        const container = containers[Math.floor(Math.random() * containers.length)];
+        const containerClient = new storage_blob_1.ContainerClient(container.uri);
+        return containerClient.getBlockBlobClient(blobName);
+    }
+    close() {
+        this.kustoClient.close();
+    }
+}
+exports.ResourceManager = ResourceManager;
+exports.default = ResourceManager;
+//# sourceMappingURL=resourceManager.js.map

package/dist-esm/src/retry.js

@@ -0,0 +1,37 @@
+"use strict";
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ExponentialRetry = exports.sleep = void 0;
+const sleep = (ms) => {
+    return new Promise((resolve) => {
+        setTimeout(resolve, ms);
+    });
+};
+exports.sleep = sleep;
+class ExponentialRetry {
+    constructor(attemptCount, sleepBaseSecs, maxJitterSecs) {
+        this.attemptCount = attemptCount;
+        this.sleepBaseSecs = sleepBaseSecs;
+        this.maxJitterSecs = maxJitterSecs;
+        this.currentAttempt = 0;
+    }
+    async backoff() {
+        if (!this.shouldTry()) {
+            throw new Error("Max retries exceeded");
+        }
+        this.currentAttempt++;
+        if (!this.shouldTry()) {
+            // This was the last retry - no need to sleep
+            return;
+        }
+        const base = this.sleepBaseSecs * Math.pow(2, this.currentAttempt - 1);
+        const jitter = Math.floor(this.maxJitterSecs * Math.random());
+        await (0, exports.sleep)(1000 * (base + jitter));
+    }
+    shouldTry() {
+        return this.currentAttempt < this.attemptCount;
+    }
+}
+exports.ExponentialRetry = ExponentialRetry;
+//# sourceMappingURL=retry.js.map

package/dist-esm/src/statusQ.js

@@ -0,0 +1,119 @@
+"use strict";
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StatusQueue = void 0;
+const storage_queue_1 = require("@azure/storage-queue");
+class QueueDetails {
+    constructor(name, service) {
+        this.name = name;
+        this.service = service;
+    }
+}
+const shuffle = (a) => {
+    for (let i = a.length - 1; i > 0; i--) {
+        const j = Math.floor(Math.random() * (i + 1));
+        const temp = a[j];
+        a[j] = a[i];
+        a[i] = temp;
+    }
+    return a;
+};
+class StatusQueue {
+    constructor(getQueuesFunc, messageCls) {
+        this.getQueuesFunc = getQueuesFunc;
+        this.messageCls = messageCls;
+    }
+    _getQServices(queuesDetails) {
+        return queuesDetails.map((q) => {
+            const fullUri = q.uri;
+            if (!fullUri) {
+                throw new Error("Empty or null connection string");
+            }
+            // chop off sas
+            const indexOfSas = q.uri.indexOf("?");
+            const name = indexOfSas > 0 ? q.uri.substring(0, indexOfSas) : q.uri;
+            return new QueueDetails(name, new storage_queue_1.QueueClient(fullUri));
+        });
+    }
+    async isEmpty() {
+        const result = await this.peek(1, { raw: true });
+        return !result || result.length === 0;
+    }
+    decodeContent(content) {
+        return Buffer.from(content, "base64").toString("ascii");
+    }
+    deserializeMessage(m) {
+        return new this.messageCls(this.decodeContent(m.messageText), null, null);
+    }
+    async _peek(qs, n, options) {
+        const result = [];
+        const nonEmptyQs = [];
+        for (const q of qs) {
+            const response = await q.service.peekMessages();
+            const messages = response.peekedMessageItems;
+            if (messages && messages.length > 0) {
+                nonEmptyQs.push(q);
+            }
+            for (const m of messages) {
+                if (m && Object.keys(m).length > 0) {
+                    result.push(options && options.raw ? m : this.deserializeMessage(m));
+                    if (result.length === n) {
+                        return { done: true, nonEmptyQs, result };
+                    }
+                }
+            }
+        }
+        return { done: nonEmptyQs.length === 0, nonEmptyQs, result };
+    }
+    async peek(n = 1, options = null) {
+        const queues = await this.getQueuesFunc();
+        const qServices = shuffle(this._getQServices(queues));
+        const perQ = qServices.length > 1 ? Math.floor(n / qServices.length) : qServices.length;
+        // First, iterate evenly and randomly on status queues
+        const partial = await this._peek(qServices, perQ, options);
+        if (partial.done) {
+            return partial.result;
+        }
+        const messagesLeftToPeek = n - partial.result.length;
+        // In case queues are uneven, iterate again. This time, request for all n messages and trim
+        return (await this._peek(partial.nonEmptyQs, messagesLeftToPeek, options)).result;
+    }
+    async _pop(qs, n, options) {
+        const nonEmptyQs = [];
+        const result = [];
+        for (const q of qs) {
+            const response = await q.service.receiveMessages({ numOfMessages: n });
+            const messages = response.receivedMessageItems;
+            for (const m of messages) {
+                if (m && Object.keys(m).length > 0) {
+                    result.push(options && options.raw ? m : this.deserializeMessage(m));
+                    if (!(options && !options.remove)) {
+                        await q.service.deleteMessage(m.messageId, m.popReceipt);
+                    }
+                    if (result.length === n) {
+                        return { done: true, nonEmptyQs, result };
+                    }
+                }
+            }
+        }
+        return { done: nonEmptyQs.length === 0, nonEmptyQs, result };
+    }
+    async pop(n = 1, options = null) {
+        var _a;
+        const queues = await this.getQueuesFunc();
+        const qServices = shuffle(this._getQServices(queues));
+        const perQ = qServices.length > 1 ? Math.floor(n / qServices.length) : qServices.length;
+        // First, iterate evenly and randomly on status queues
+        const partial = await this._pop(qServices, perQ, options);
+        if (partial.done) {
+            return partial.result;
+        }
+        const messagesLeftToPop = n - partial.result.length;
+        // In case queues are uneven, iterate again. This time, request for all n messages and trim
+        const final = await this._pop((_a = partial.result.nonEmptyQs) !== null && _a !== void 0 ? _a : [], messagesLeftToPop, options);
+        return partial.result.concat(final.result);
+    }
+}
+exports.StatusQueue = StatusQueue;
+//# sourceMappingURL=statusQ.js.map

package/dist-esm/src/streamUtils.browser.js

@@ -0,0 +1,22 @@
+"use strict";
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.tryStreamToArray = exports.tryFileToBuffer = exports.fileToStream = void 0;
+const descriptors_1 = require("./descriptors");
+const fileToStream = async (file) => {
+    const streamFs = await file.file.arrayBuffer();
+    const compressionType = file.zipped ? descriptors_1.CompressionType.GZIP : descriptors_1.CompressionType.None;
+    return new descriptors_1.StreamDescriptor(streamFs, file.sourceId, compressionType);
+};
+exports.fileToStream = fileToStream;
+const tryFileToBuffer = async (file) => {
+    return await (0, exports.fileToStream)(file);
+};
+exports.tryFileToBuffer = tryFileToBuffer;
+// NOT USED
+const tryStreamToArray = async (stream) => {
+    return Promise.resolve(stream);
+};
+exports.tryStreamToArray = tryStreamToArray;
+//# sourceMappingURL=streamUtils.browser.js.map

package/{source → dist-esm/src}/streamUtils.js

@@ -1,31 +1,34 @@
 "use strict";
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.tryStreamToArray = exports.mergeStreams = exports.fileToStream = void 0;
+exports.tryStreamToArray = exports.tryFileToBuffer = exports.fileToStream = void 0;
 const descriptors_1 = require("./descriptors");
 const fs_1 = __importDefault(require("fs"));
 const stream_1 = require("stream");
 const stream_array_1 = __importDefault(require("stream-array"));
-const fileToStream = (file) => {
-    const fileDescriptor = file instanceof descriptors_1.FileDescriptor ? file : new descriptors_1.FileDescriptor(file);
-    const streamFs = fs_1.default.createReadStream(fileDescriptor.filePath);
+// Returns fs.ReadStream for node and NodeJS.ReadableStream in browser
+const fileToStream = (fileDescriptor) => {
+    const streamFs = fs_1.default.createReadStream(fileDescriptor.file);
     const compressionType = fileDescriptor.zipped ? descriptors_1.CompressionType.GZIP : descriptors_1.CompressionType.None;
-    return new descriptors_1.StreamDescriptor(streamFs, fileDescriptor.sourceId, compressionType);
+    return Promise.resolve(new descriptors_1.StreamDescriptor(streamFs, fileDescriptor.sourceId, compressionType));
 };
 exports.fileToStream = fileToStream;
+// Used in managed streaming where we buffer the file to memory for retries
+const tryFileToBuffer = async (fileDescriptor) => {
+    try {
+        const buffer = fs_1.default.readFileSync(fileDescriptor.file);
+        const compressionType = fileDescriptor.zipped ? descriptors_1.CompressionType.GZIP : descriptors_1.CompressionType.None;
+        return new descriptors_1.StreamDescriptor(buffer, fileDescriptor.sourceId, compressionType);
+    }
+    catch (error) {
+        return await (0, exports.fileToStream)(fileDescriptor);
+    }
+};
+exports.tryFileToBuffer = tryFileToBuffer;
 const mergeStreams = (...streams) => {
     let pass = new stream_1.PassThrough();
     let waiting = streams.length;
@@ -35,9 +38,11 @@ const mergeStreams = (...streams) => {
     }
     return pass;
 };
-exports.mergeStreams = mergeStreams;
-const tryStreamToArray = (stream, maxBufferSize) => __awaiter(void 0, void 0, void 0, function* () {
-    return yield new Promise((resolve, reject) => {
+const tryStreamToArray = async (stream, maxBufferSize) => {
+    if (stream instanceof Buffer) {
+        return stream;
+    }
+    return await new Promise((resolve, reject) => {
         const result = [];
         const endListener = () => resolve(Buffer.concat(result));
         const dataHandler = (chunk) => {
@@ -46,7 +51,7 @@ const tryStreamToArray = (stream, maxBufferSize) => __awaiter(void 0, void 0, vo
                 if (result.reduce((sum, b) => sum + b.length, 0) > maxBufferSize) {
                     stream.removeListener("data", dataHandler);
                     stream.removeListener("end", endListener);
-                    resolve((0, exports.mergeStreams)((0, stream_array_1.default)(result), stream));
+                    resolve(mergeStreams((0, stream_array_1.default)(result), stream));
                 }
             }
             catch (e) {
@@ -56,6 +61,6 @@ const tryStreamToArray = (stream, maxBufferSize) => __awaiter(void 0, void 0, vo
         stream.on("data", dataHandler);
         stream.on("end", endListener);
     });
-});
+};
 exports.tryStreamToArray = tryStreamToArray;
 //# sourceMappingURL=streamUtils.js.map

package/dist-esm/src/streamingIngestClient.browser.js

@@ -0,0 +1,42 @@
+"use strict";
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+Object.defineProperty(exports, "__esModule", { value: true });
+const descriptors_1 = require("./descriptors");
+const fileDescriptor_browser_1 = require("./fileDescriptor.browser");
+const abstractKustoClient_1 = require("./abstractKustoClient");
+const azure_kusto_data_1 = require("azure-kusto-data");
+const streamUtils_browser_1 = require("./streamUtils.browser");
+class KustoStreamingIngestClient extends abstractKustoClient_1.AbstractKustoClient {
+    constructor(kcsb, defaultProps) {
+        super(defaultProps);
+        this.kustoClient = new azure_kusto_data_1.Client(kcsb);
+        this.defaultDatabase = this.kustoClient.defaultDatabase;
+    }
+    /**
+     * Use Readable for Node.js and ArrayBuffer in browser
+     */
+    async ingestFromStream(stream, ingestionProperties, clientRequestId) {
+        var _a;
+        this.ensureOpen();
+        const props = this._getMergedProps(ingestionProperties);
+        const descriptor = stream instanceof descriptors_1.StreamDescriptor ? stream : new descriptors_1.StreamDescriptor(stream);
+        return await this.kustoClient.executeStreamingIngest(props.database, props.table, descriptor.stream, props.format, (_a = props.ingestionMappingReference) !== null && _a !== void 0 ? _a : null, clientRequestId);
+    }
+    /**
+     * Use string for Node.js and Blob in browser
+     */
+    async ingestFromFile(file, ingestionProperties) {
+        this.ensureOpen();
+        const descriptor = file instanceof fileDescriptor_browser_1.FileDescriptor ? file : new fileDescriptor_browser_1.FileDescriptor(file);
+        return this.ingestFromStream(await (0, streamUtils_browser_1.tryFileToBuffer)(descriptor), ingestionProperties);
+    }
+    close() {
+        if (!this._isClosed) {
+            this.kustoClient.close();
+        }
+        super.close();
+    }
+}
+exports.default = KustoStreamingIngestClient;
+//# sourceMappingURL=streamingIngestClient.browser.js.map

package/dist-esm/src/streamingIngestClient.js

@@ -0,0 +1,51 @@
+"use strict";
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const descriptors_1 = require("./descriptors");
+const fileDescriptor_1 = require("./fileDescriptor");
+const zlib_1 = __importDefault(require("zlib"));
+const abstractKustoClient_1 = require("./abstractKustoClient");
+const azure_kusto_data_1 = require("azure-kusto-data");
+const streamUtils_1 = require("./streamUtils");
+class KustoStreamingIngestClient extends abstractKustoClient_1.AbstractKustoClient {
+    constructor(kcsb, defaultProps) {
+        super(defaultProps);
+        this.kustoClient = new azure_kusto_data_1.Client(kcsb);
+        this.defaultDatabase = this.kustoClient.defaultDatabase;
+    }
+    /**
+     * Use Readable for Node.js and ArrayBuffer in browser
+     */
+    async ingestFromStream(stream, ingestionProperties, clientRequestId) {
+        var _a;
+        this.ensureOpen();
+        const props = this._getMergedProps(ingestionProperties);
+        const descriptor = stream instanceof descriptors_1.StreamDescriptor ? stream : new descriptors_1.StreamDescriptor(stream);
+        const compressedStream = descriptor.compressionType === descriptors_1.CompressionType.None
+            ? !(descriptor.stream instanceof ArrayBuffer)
+                ? descriptor.stream.pipe(zlib_1.default.createGzip())
+                : descriptor.stream
+            : descriptor.stream;
+        return await this.kustoClient.executeStreamingIngest(props.database, props.table, compressedStream, props.format, (_a = props.ingestionMappingReference) !== null && _a !== void 0 ? _a : null, clientRequestId);
+    }
+    /**
+     * Use string for Node.js and Blob in browser
+     */
+    async ingestFromFile(file, ingestionProperties) {
+        this.ensureOpen();
+        const descriptor = file instanceof fileDescriptor_1.FileDescriptor ? file : new fileDescriptor_1.FileDescriptor(file);
+        return this.ingestFromStream(await (0, streamUtils_1.fileToStream)(descriptor), ingestionProperties);
+    }
+    close() {
+        if (!this._isClosed) {
+            this.kustoClient.close();
+        }
+        super.close();
+    }
+}
+exports.default = KustoStreamingIngestClient;
+//# sourceMappingURL=streamingIngestClient.js.map