azure-kusto-ingest 3.3.0 → 3.4.0

package/package.json

@@ -1,12 +1,18 @@
 {
     "name": "azure-kusto-ingest",
-    "version": "3.3.0",
+    "version": "3.4.0",
     "description": "Azure Data Explorer Ingestion SDK",
     "main": "index.js",
     "types": "index.d.ts",
+    "scripts": {
+        "build": "tsc"
+    },
     "engines": {
         "node": ">= 14.0.0"
     },
+    "publishConfig": {
+        "access": "public"
+    },
     "tags": [
         "azure",
         "kusto",
@@ -17,40 +23,41 @@
         "azure",
         "kusto"
     ],
-    "scripts": {
-        "build": "npm link ../azure-kusto-data && tsc -b",
-        "prepublish": "npm run build",
-        "example": "npm run build && node example.js",
-        "lint": "npx eslint -c .eslintrc.js --ext .ts .",
-        "test": "npm run build && mocha --parallel --recursive test/*.js",
-        "e2e": "npm run build && mocha --parallel --timeout 240000 test/e2eTests/e2eTest.js",
-        "allTests": "npm run build && mocha --parallel --timeout 240000 --recursive",
-        "testPipeline": "npm run build && nyc --reporter lcovonly mocha --timeout 240000 --recursive --reporter mocha-junit-reporter ",
-        "format": "npx prettier --write .",
-        "checkFormat": "npx prettier --check  --end-of-line lf ."
-    },
     "repository": {
         "type": "git",
         "url": "https://github.com/Azure/azure-kusto-node.git",
         "directory": "azure-kusto-ingest"
     },
+    "files": [
+        "source/**/*.js",
+        "source/**/*.d.ts",
+        "index.js",
+        "index.d.ts",
+        "tsconfig.tsbuildinfo",
+        "example.js"
+    ],
     "author": "",
     "license": "ISC",
     "dependencies": {
-        "@azure/storage-blob": "12.1.2",
-        "@azure/storage-queue": "12.0.5",
-        "@types/node": "^14.14.13",
-        "@types/sinon": "^9.0.9",
+        "@azure/storage-blob": "^12.11.0",
+        "@azure/storage-queue": "^12.10.0",
+        "@types/node": "^18.6.4",
+        "@types/sinon": "^10.0.13",
         "@types/stream-array": "^1.1.0",
         "@types/stream-to-array": "^2.3.0",
-        "@types/uuid": "^8.3.0",
+        "@types/uuid": "^8.3.4",
         "@types/uuid-validate": "0.0.1",
-        "azure-kusto-data": "file:../azure-kusto-data",
-        "moment": "^2.22.2",
-        "stream-array": "^1.1.0",
+        "azure-kusto-data": "^3.4.0",
+        "moment": "^2.29.4",
+        "stream-array": "^1.1.2",
         "stream-to-array": "^2.3.0",
+        "tmp-promise": "^3.0.3",
+        "ts-node": "^10.9.1",
         "uuid": "^8.3.2",
         "uuid-validate": "0.0.3"
     },
-    "gitHead": "c0b1e333ceddca341e05b3e0add4ef939d1bcea3"
+    "gitHead": "7133736d0867b15b126c62e14dd72bbaa69bf905",
+    "devDependencies": {
+        "@types/tmp": "^0.2.3"
+    }
 }

package/source/columnMappings.js

@@ -3,7 +3,6 @@
 // Licensed under the MIT License.
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.W3CLogFileMapping = exports.OrcColumnMapping = exports.ParquetColumnMapping = exports.SStreamColumnMapping = exports.ApacheAvroColumnMapping = exports.AvroColumnMapping = exports.JsonColumnMapping = exports.CsvColumnMapping = exports.ColumnMapping = exports.ConstantTransformation = exports.FieldTransformation = void 0;
-/* eslint-disable max-classes-per-file -- We want all the Column Mappings to be in this file */
 /* eslint-disable @typescript-eslint/ban-types -- We legitimately want to use {} as a "any non-nullable type" */
 const ingestionProperties_1 = require("./ingestionProperties");
 var FieldTransformation;

package/source/descriptors.d.ts

@@ -12,9 +12,13 @@ export declare class FileDescriptor {
     size: number | null;
     sourceId: string;
     zipped: boolean;
-    constructor(filePath: string, sourceId?: string | null, size?: number | null);
+    compressionType: CompressionType;
+    cleanupTmp?: () => Promise<void>;
+    constructor(filePath: string, sourceId?: string | null, size?: number | null, compressionType?: CompressionType);
     _gzip(): Promise<string>;
     prepare(): Promise<string>;
+    private calculateSize;
+    cleanup(): Promise<void>;
 }
 export declare class StreamDescriptor {
     readonly stream: Readable;

package/source/descriptors.js

@@ -15,12 +15,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.BlobDescriptor = exports.StreamDescriptor = exports.FileDescriptor = exports.CompressionType = void 0;
-/* eslint-disable max-classes-per-file -- We want all the Descriptors to be in this file */
 const uuid_1 = require("uuid");
 const uuid_validate_1 = __importDefault(require("uuid-validate"));
 const zlib_1 = __importDefault(require("zlib"));
 const path_1 = __importDefault(require("path"));
 const fs_1 = __importDefault(require("fs"));
+const tmp_promise_1 = require("tmp-promise");
+const util_1 = require("util");
 var CompressionType;
 (function (CompressionType) {
     CompressionType["ZIP"] = ".zip";
@@ -37,19 +38,22 @@ const getSourceId = (sourceId) => {
     return (0, uuid_1.v4)();
 };
 class FileDescriptor {
-    constructor(filePath, sourceId = null, size = null) {
+    constructor(filePath, sourceId = null, size = null, compressionType = CompressionType.None) {
         this.filePath = filePath;
+        this.compressionType = compressionType;
         this.name = path_1.default.basename(this.filePath);
         this.extension = path_1.default.extname(this.filePath).toLowerCase();
         this.size = size;
-        this.zipped = this.extension === ".gz" || this.extension === ".zip";
+        this.zipped = compressionType !== CompressionType.None || this.extension === ".gz" || this.extension === ".zip";
         this.sourceId = getSourceId(sourceId);
     }
     _gzip() {
         return __awaiter(this, void 0, void 0, function* () {
+            const { path, cleanup } = yield (0, tmp_promise_1.file)({ postfix: ".gz", keep: false });
+            this.cleanupTmp = cleanup;
             const zipper = zlib_1.default.createGzip();
             const input = fs_1.default.createReadStream(this.filePath, { autoClose: true });
-            const output = fs_1.default.createWriteStream(this.filePath + ".gz");
+            const output = fs_1.default.createWriteStream(path);
             yield new Promise((resolve, reject) => {
                 input
                     .pipe(zipper)
@@ -61,22 +65,34 @@ class FileDescriptor {
                     resolve(null);
                 });
             });
-            return this.filePath + ".gz";
+            return path;
         });
     }
     prepare() {
         return __awaiter(this, void 0, void 0, function* () {
             if (this.zipped) {
-                if (this.size == null || this.size <= 0) {
-                    this.size = fs_1.default.statSync(this.filePath).size * 11;
-                }
+                const estimatedCompressionModifier = 11;
+                yield this.calculateSize(estimatedCompressionModifier);
                 return this.filePath;
             }
-            yield this._gzip();
+            const path = yield this._gzip();
+            yield this.calculateSize();
+            return path;
+        });
+    }
+    calculateSize(modifier = 1) {
+        return __awaiter(this, void 0, void 0, function* () {
             if (this.size == null || this.size <= 0) {
-                this.size = fs_1.default.statSync(this.filePath).size;
+                const asyncStat = (0, util_1.promisify)(fs_1.default.stat);
+                this.size = (yield asyncStat(this.filePath)).size * modifier;
+            }
+        });
+    }
+    cleanup() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.cleanupTmp) {
+                yield this.cleanupTmp();
             }
-            return this.filePath + ".gz";
         });
     }
 }

package/source/ingestClient.js

@@ -22,6 +22,7 @@ const ingestionBlobInfo_1 = __importDefault(require("./ingestionBlobInfo"));
 const storage_queue_1 = require("@azure/storage-queue");
 const storage_blob_1 = require("@azure/storage-blob");
 const abstractKustoClient_1 = require("./abstractKustoClient");
+const path_1 = __importDefault(require("path"));
 class KustoIngestClient extends abstractKustoClient_1.AbstractKustoClient {
     constructor(kcsb, defaultProps) {
         super(defaultProps);
@@ -40,7 +41,7 @@ class KustoIngestClient extends abstractKustoClient_1.AbstractKustoClient {
                 throw new Error("Failed to get containers");
             }
             const container = containers[Math.floor(Math.random() * containers.length)];
-            const containerClient = new storage_blob_1.ContainerClient(container.getSASConnectionString(), container.objectName);
+            const containerClient = new storage_blob_1.ContainerClient(container.uri);
             return containerClient.getBlockBlobClient(blobName);
         });
     }
@@ -59,11 +60,16 @@ class KustoIngestClient extends abstractKustoClient_1.AbstractKustoClient {
         return __awaiter(this, void 0, void 0, function* () {
             const props = this._getMergedProps(ingestionProperties);
             const descriptor = file instanceof descriptors_1.FileDescriptor ? file : new descriptors_1.FileDescriptor(file);
-            const fileToUpload = yield descriptor.prepare();
-            const blobName = `${props.database}__${props.table}__${descriptor.sourceId}__${fileToUpload}`;
-            const blockBlobClient = yield this._getBlockBlobClient(blobName);
-            yield blockBlobClient.uploadFile(fileToUpload);
-            return this.ingestFromBlob(new descriptors_1.BlobDescriptor(blockBlobClient.url, descriptor.size, descriptor.sourceId), props);
+            try {
+                const fileToUpload = yield descriptor.prepare();
+                const blobName = `${props.database}__${props.table}__${descriptor.sourceId}__${descriptor.name}__.${path_1.default.extname(fileToUpload)}`;
+                const blockBlobClient = yield this._getBlockBlobClient(blobName);
+                yield blockBlobClient.uploadFile(fileToUpload);
+                return this.ingestFromBlob(new descriptors_1.BlobDescriptor(blockBlobClient.url, descriptor.size, descriptor.sourceId), props);
+            }
+            finally {
+                yield descriptor.cleanup();
+            }
         });
     }
     ingestFromBlob(blob, ingestionProperties) {
@@ -76,7 +82,7 @@ class KustoIngestClient extends abstractKustoClient_1.AbstractKustoClient {
             }
             const authorizationContext = yield this.resourceManager.getAuthorizationContext();
             const queueDetails = queues[Math.floor(Math.random() * queues.length)];
-            const queueClient = new storage_queue_1.QueueClient(queueDetails.getSASConnectionString(), queueDetails.objectName);
+            const queueClient = new storage_queue_1.QueueClient(queueDetails.uri);
             const ingestionBlobInfo = new ingestionBlobInfo_1.default(descriptor, props, authorizationContext);
             const ingestionBlobInfoJson = JSON.stringify(ingestionBlobInfo);
             const encoded = Buffer.from(ingestionBlobInfoJson).toString("base64");

package/source/ingestionProperties.js

@@ -3,7 +3,6 @@
 // Licensed under the MIT License.
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.IngestionProperties = exports.ReportMethod = exports.ReportLevel = exports.ValidationPolicy = exports.ValidationImplications = exports.ValidationOptions = exports.dataFormatMappingKind = exports.IngestionMappingKind = exports.DataFormat = void 0;
-/* eslint-disable max-classes-per-file -- the main class is IngestionProperties, ValidationPolicy is a tiny class */
 const errors_1 = require("./errors");
 /**
  * Data formats supported for Kusto ingestion.

package/source/resourceManager.d.ts

@@ -1,13 +1,8 @@
 import { Client } from "azure-kusto-data";
 import moment from "moment";
 export declare class ResourceURI {
-    readonly storageAccountName: string;
-    readonly objectType: string;
-    readonly objectName: string;
-    readonly sas: string;
-    constructor(storageAccountName: string, objectType: string, objectName: string, sas: string);
-    static fromURI(uri: string): ResourceURI;
-    getSASConnectionString(): string;
+    readonly uri: string;
+    constructor(uri: string);
 }
 export declare class IngestClientResources {
     readonly securedReadyForAggregationQueues: ResourceURI[] | null;
@@ -24,6 +19,8 @@ export declare class ResourceManager {
     ingestClientResourcesLastUpdate: moment.Moment | null;
     authorizationContext: string | null;
     authorizationContextLastUpdate: moment.Moment | null;
+    private baseSleepTimeSecs;
+    private baseJitterSecs;
     constructor(kustoClient: Client);
     refreshIngestClientResources(): Promise<IngestClientResources>;
     getIngestClientResourcesFromService(): Promise<IngestClientResources>;

package/source/resourceManager.js

@@ -15,30 +15,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ResourceManager = exports.IngestClientResources = exports.ResourceURI = void 0;
+const azure_kusto_data_1 = require("azure-kusto-data");
+const retry_1 = require("./retry");
 const moment_1 = __importDefault(require("moment"));
-const URI_FORMAT = /https:\/\/(\w+).(queue|blob|table).core.windows.net\/([\w,-]+)\?(.*)/;
+const ATTEMPT_COUNT = 4;
 class ResourceURI {
-    constructor(storageAccountName, objectType, objectName, sas) {
-        this.storageAccountName = storageAccountName;
-        this.objectType = objectType;
-        this.objectName = objectName;
-        this.sas = sas;
-    }
-    static fromURI(uri) {
-        const match = URI_FORMAT.exec(uri);
-        if (match == null || match.length < 5) {
-            throw Error(`Failed to create ResourceManager from URI - invalid uri (${uri})`);
-        }
-        return new ResourceURI(match[1], match[2], match[3], match[4]);
-    }
-    getSASConnectionString() {
-        if (this.objectType === "queue") {
-            return `QueueEndpoint=https://${this.storageAccountName}.queue.core.windows.net/;SharedAccessSignature=${this.sas}`;
-        }
-        if (this.objectType === "blob") {
-            return `BlobEndpoint=https://${this.storageAccountName}.blob.core.windows.net/;SharedAccessSignature=${this.sas}`;
-        }
-        throw new Error(`Can't make the current object type (${this.objectType}) to connection string`);
+    constructor(uri) {
+        this.uri = uri;
     }
 }
 exports.ResourceURI = ResourceURI;
@@ -58,6 +41,8 @@ exports.IngestClientResources = IngestClientResources;
 class ResourceManager {
     constructor(kustoClient) {
         this.kustoClient = kustoClient;
+        this.baseSleepTimeSecs = 1;
+        this.baseJitterSecs = 1;
         this.refreshPeriod = moment_1.default.duration(1, "h");
         this.ingestClientResources = null;
         this.ingestClientResourcesLastUpdate = null;
@@ -79,9 +64,21 @@ class ResourceManager {
     }
     getIngestClientResourcesFromService() {
         return __awaiter(this, void 0, void 0, function* () {
-            const response = yield this.kustoClient.execute("NetDefaultDB", ".get ingestion resources");
-            const table = response.primaryResults[0];
-            return new IngestClientResources(this.getResourceByName(table, "SecuredReadyForAggregationQueue"), this.getResourceByName(table, "FailedIngestionsQueue"), this.getResourceByName(table, "SuccessfulIngestionsQueue"), this.getResourceByName(table, "TempStorage"));
+            const retry = new retry_1.ExponentialRetry(ATTEMPT_COUNT, this.baseSleepTimeSecs, this.baseJitterSecs);
+            while (retry.shouldTry()) {
+                try {
+                    const response = yield this.kustoClient.execute("NetDefaultDB", ".get ingestion resources");
+                    const table = response.primaryResults[0];
+                    return new IngestClientResources(this.getResourceByName(table, "SecuredReadyForAggregationQueue"), this.getResourceByName(table, "FailedIngestionsQueue"), this.getResourceByName(table, "SuccessfulIngestionsQueue"), this.getResourceByName(table, "TempStorage"));
+                }
+                catch (error) {
+                    if (!(error instanceof azure_kusto_data_1.KustoDataErrors.ThrottlingError)) {
+                        throw error;
+                    }
+                    yield retry.backoff();
+                }
+            }
+            throw new Error(`Failed to get ingestion resources from server - the request was throttled ${ATTEMPT_COUNT} times.`);
         });
     }
     getResourceByName(table, resourceName) {
@@ -89,7 +86,7 @@ class ResourceManager {
         for (const row of table.rows()) {
             const typedRow = row;
             if (typedRow.ResourceTypeName === resourceName) {
-                result.push(ResourceURI.fromURI(typedRow.StorageRoot));
+                result.push(new ResourceURI(typedRow.StorageRoot));
             }
         }
         return result;
@@ -110,12 +107,24 @@ class ResourceManager {
     }
     getAuthorizationContextFromService() {
         return __awaiter(this, void 0, void 0, function* () {
-            const response = yield this.kustoClient.execute("NetDefaultDB", ".get kusto identity token");
-            const next = response.primaryResults[0].rows().next();
-            if (next.done) {
-                throw new Error("Failed to get authorization context - got empty results");
+            const retry = new retry_1.ExponentialRetry(ATTEMPT_COUNT, this.baseSleepTimeSecs, this.baseJitterSecs);
+            while (retry.shouldTry()) {
+                try {
+                    const response = yield this.kustoClient.execute("NetDefaultDB", ".get kusto identity token");
+                    const next = response.primaryResults[0].rows().next();
+                    if (next.done) {
+                        throw new Error("Failed to get authorization context - got empty results");
+                    }
+                    return next.value.toJSON().AuthorizationContext;
+                }
+                catch (error) {
+                    if (!(error instanceof azure_kusto_data_1.KustoDataErrors.ThrottlingError)) {
+                        throw error;
+                    }
+                    yield retry.backoff();
+                }
             }
-            return next.value.toJSON().AuthorizationContext;
+            throw new Error(`Failed to get identity token from server - the request was throttled ${ATTEMPT_COUNT} times.`);
         });
     }
     getIngestionQueues() {

package/source/retry.js

@@ -30,10 +30,14 @@ class ExponentialRetry {
             if (!this.shouldTry()) {
                 throw new Error("Max retries exceeded");
             }
-            const base = this.sleepBaseSecs * Math.pow(2, this.currentAttempt);
+            this.currentAttempt++;
+            if (!this.shouldTry()) {
+                // This was the last retry - no need to sleep
+                return;
+            }
+            const base = this.sleepBaseSecs * Math.pow(2, this.currentAttempt - 1);
             const jitter = Math.floor(this.maxJitterSecs * Math.random());
             yield (0, exports.sleep)(1000 * (base + jitter));
-            this.currentAttempt++;
         });
     }
     shouldTry() {

package/source/status.js

@@ -3,7 +3,6 @@
 // Licensed under the MIT License.
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.KustoIngestStatusQueues = exports.StatusMessage = void 0;
-/* eslint-disable max-classes-per-file -- We want all the Status related classes in this file */
 const statusQ_1 = require("./statusQ");
 class StatusMessage {
     constructor(raw, obj, extraProps) {

package/source/statusQ.js

@@ -12,7 +12,6 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.StatusQueue = void 0;
-/* eslint-disable max-classes-per-file -- QueueDetails is a very small class, so we don't need it in a different file */
 const storage_queue_1 = require("@azure/storage-queue");
 class QueueDetails {
     constructor(name, service) {
@@ -36,11 +35,14 @@ class StatusQueue {
     }
     _getQServices(queuesDetails) {
         return queuesDetails.map((q) => {
-            const sasConnectionString = q.getSASConnectionString();
-            if (!sasConnectionString) {
+            const fullUri = q.uri;
+            if (!fullUri) {
                 throw new Error("Empty or null connection string");
             }
-            return new QueueDetails(q.objectName, new storage_queue_1.QueueClient(sasConnectionString, q.objectName));
+            // chop off sas
+            const indexOfSas = q.uri.indexOf("?");
+            const name = indexOfSas > 0 ? q.uri.substring(0, indexOfSas) : q.uri;
+            return new QueueDetails(name, new storage_queue_1.QueueClient(fullUri));
         });
     }
     isEmpty() {