azure-kusto-ingest 2.2.2 → 3.1.0

package/.eslintignore

@@ -0,0 +1,5 @@
+node_modules
+**/*.d.ts
+**/*.js
+coverage
+.nyc_output

package/.eslintrc.js

@@ -0,0 +1,247 @@
+/*
+👋 Hi! This file was autogenerated by tslint-to-eslint-config.
+https://github.com/typescript-eslint/tslint-to-eslint-config
+
+It represents the closest reasonable ESLint configuration to this
+project's original TSLint configuration.
+
+We recommend eventually switching this configuration to extend from
+the recommended rulesets in typescript-eslint.
+https://github.com/typescript-eslint/tslint-to-eslint-config/blob/master/docs/FAQs.md
+
+Happy linting! 💖
+*/
+module.exports = {
+    "env": {
+        "browser": true,
+        "es6": true,
+        "node": true
+    },
+    "extends": [
+        "eslint:recommended",
+        "plugin:@typescript-eslint/recommended",
+        "plugin:@typescript-eslint/recommended-requiring-type-checking",
+        "prettier"
+    ],
+    "ignorePatterns": [
+        "node_modules",
+        "**/*.d.ts",
+        "**/*.js"
+    ],
+    "parser": "@typescript-eslint/parser",
+    "parserOptions": {
+        "project": "tsconfig.json",
+        "sourceType": "module"
+    },
+    "plugins": [
+        "eslint-plugin-jsdoc",
+        "eslint-plugin-prefer-arrow",
+        "@typescript-eslint",
+        "header"
+    ],
+    "rules": {
+        "@typescript-eslint/adjacent-overload-signatures": "error",
+        "@typescript-eslint/array-type": [
+            "error",
+            {
+                "default": "array"
+            }
+        ],
+        "@typescript-eslint/ban-ts-comment": "error",
+        "@typescript-eslint/ban-types": [
+            "error",
+            {
+                "types": {
+                    "Object": {
+                        "message": "Avoid using the `Object` type. Did you mean `object`?"
+                    },
+                    "Function": {
+                        "message": "Avoid using the `Function` type. Prefer a specific function type, like `() => void`."
+                    },
+                    "Boolean": {
+                        "message": "Avoid using the `Boolean` type. Did you mean `boolean`?"
+                    },
+                    "Number": {
+                        "message": "Avoid using the `Number` type. Did you mean `number`?"
+                    },
+                    "String": {
+                        "message": "Avoid using the `String` type. Did you mean `string`?"
+                    },
+                    "Symbol": {
+                        "message": "Avoid using the `Symbol` type. Did you mean `symbol`?"
+                    }
+                }
+            }
+        ],
+        "@typescript-eslint/consistent-type-assertions": "error",
+        "@typescript-eslint/dot-notation": "error",
+        "@typescript-eslint/no-array-constructor": "error",
+        "@typescript-eslint/no-empty-interface": "error",
+        "@typescript-eslint/no-explicit-any": "off",
+        "@typescript-eslint/no-extra-non-null-assertion": "error",
+        "@typescript-eslint/no-extra-semi": "error",
+        "@typescript-eslint/no-loss-of-precision": "error",
+        "@typescript-eslint/no-misused-new": "error",
+        "@typescript-eslint/no-namespace": "error",
+        "@typescript-eslint/no-non-null-asserted-optional-chain": "error",
+        "@typescript-eslint/no-parameter-properties": "off",
+        "@typescript-eslint/no-shadow": [
+            "error",
+            {
+                "hoist": "all"
+            }
+        ],
+        "@typescript-eslint/no-this-alias": "error",
+        "@typescript-eslint/no-unnecessary-type-constraint": "error",
+        "@typescript-eslint/no-unused-expressions": "error",
+        "@typescript-eslint/no-unused-vars": "warn",
+        "@typescript-eslint/no-use-before-define": "off",
+        "@typescript-eslint/no-var-requires": "error",
+        "@typescript-eslint/prefer-as-const": "error",
+        "@typescript-eslint/prefer-for-of": "error",
+        "@typescript-eslint/prefer-function-type": "error",
+        "@typescript-eslint/prefer-namespace-keyword": "error",
+        "@typescript-eslint/triple-slash-reference": [
+            "error",
+            {
+                "path": "always",
+                "types": "prefer-import",
+                "lib": "always"
+            }
+        ],
+        "@typescript-eslint/unified-signatures": "error",
+        "complexity": "off",
+        "constructor-super": "error",
+        "dot-notation": "error",
+        "eqeqeq": [
+            "error",
+            "smart"
+        ],
+        "for-direction": "error",
+        "getter-return": "error",
+        "guard-for-in": "error",
+        "id-denylist": [
+            "error",
+            "any",
+            "Number",
+            "number",
+            "String",
+            "string",
+            "Boolean",
+            "boolean",
+            "Undefined",
+            "undefined"
+        ],
+        "id-match": "error",
+        "jsdoc/check-alignment": "error",
+        "jsdoc/check-indentation": "error",
+        "jsdoc/newline-after-description": "error",
+        "max-classes-per-file": [
+            "error",
+            1
+        ],
+        "new-parens": "error",
+        "no-array-constructor": "off",
+        "no-async-promise-executor": "error",
+        "no-bitwise": "error",
+        "no-caller": "error",
+        "no-case-declarations": "error",
+        "no-class-assign": "error",
+        "no-compare-neg-zero": "error",
+        "no-cond-assign": "error",
+        "no-console": "error",
+        "no-const-assign": "error",
+        "no-control-regex": "error",
+        "no-debugger": "error",
+        "no-delete-var": "error",
+        "no-dupe-args": "error",
+        "no-dupe-class-members": "error",
+        "no-dupe-else-if": "error",
+        "no-dupe-keys": "error",
+        "no-duplicate-case": "error",
+        "no-empty": "error",
+        "no-empty-character-class": "error",
+        "no-empty-pattern": "error",
+        "no-eval": "error",
+        "no-ex-assign": "error",
+        "no-extra-boolean-cast": "error",
+        "no-extra-semi": "off",
+        "no-fallthrough": "off",
+        "no-func-assign": "error",
+        "no-global-assign": "error",
+        "no-import-assign": "error",
+        "no-inner-declarations": "error",
+        "no-invalid-regexp": "error",
+        "no-invalid-this": "off",
+        "no-irregular-whitespace": "error",
+        "no-loss-of-precision": "off",
+        "no-misleading-character-class": "error",
+        "no-mixed-spaces-and-tabs": "error",
+        "no-new-symbol": "error",
+        "no-new-wrappers": "error",
+        "no-nonoctal-decimal-escape": "error",
+        "no-obj-calls": "error",
+        "no-octal": "error",
+        "no-prototype-builtins": "error",
+        "no-redeclare": "error",
+        "no-regex-spaces": "error",
+        "no-self-assign": "error",
+        "no-setter-return": "error",
+        "no-shadow-restricted-names": "error",
+        "no-sparse-arrays": "error",
+        "no-this-before-super": "error",
+        "no-throw-literal": "error",
+        "no-trailing-spaces": "error",
+        "no-undef-init": "error",
+        "no-unexpected-multiline": "error",
+        "no-unreachable": "error",
+        "no-unsafe-finally": "error",
+        "no-unsafe-negation": "error",
+        "no-unsafe-optional-chaining": "error",
+        "no-unused-expressions": "error",
+        "no-unused-labels": "error",
+        "no-unused-vars": "off",
+        "no-use-before-define": "off",
+        "no-useless-backreference": "error",
+        "no-useless-catch": "error",
+        "no-useless-escape": "error",
+        "no-var": "error",
+        "no-with": "error",
+        "object-shorthand": "error",
+        "one-var": [
+            "error",
+            "never"
+        ],
+        "prefer-arrow/prefer-arrow-functions": "error",
+        "prefer-const": "error",
+        "radix": "error",
+        "require-yield": "error",
+        "spaced-comment": [
+            "error",
+            "always",
+            {
+                "markers": [
+                    "/"
+                ]
+            }
+        ],
+        "use-isnan": "error",
+        "valid-typeof": "off",
+
+        // Rules that were added manually
+
+        "header/header": ["error", "line", " Copyright (c) Microsoft Corporation.\n Licensed under the MIT License."],
+
+        // Disabled rules to avoid too many changes, consider re-enabling in the future
+        "@typescript-eslint/no-unsafe-assignment": "off",
+        "@typescript-eslint/no-unsafe-argument": "off",
+        "@typescript-eslint/no-unsafe-return": "off",
+        "@typescript-eslint/restrict-template-expressions": "off",
+        "@typescript-eslint/no-empty-function": "off",
+        "@typescript-eslint/no-inferrable-types": "off",
+        "no-undef":"off",
+        "@typescript-eslint/no-non-null-assertion":"off",
+        "@typescript-eslint/no-unnecessary-type-assertion":"off",
+        "no-constant-condition": "off",
+    }
+};

package/.mocharc.json

@@ -1,3 +1,3 @@
 {
-  "enable-source-maps": true
-}
+    "enable-source-maps": true
+}

package/.prettierignore

@@ -0,0 +1,8 @@
+node_modules
+**/*.d.ts
+**/*.js
+!example.js
+coverage
+.nyc_output
+test/data/*
+test/e2eTests/e2eData/*

package/.prettierrc.json

@@ -0,0 +1,5 @@
+{
+    "endOfLine": "auto",
+    "printWidth": 160,
+    "tabWidth": 4
+}

package/README.md

@@ -6,41 +6,35 @@
 
 ## Quick Start
 
-```javascript 
+```javascript
 const IngestClient = require("azure-kusto-ingest").IngestClient;
 const IngestionProps = require("azure-kusto-ingest").IngestionProperties;
 const KustoConnectionStringBuilder = require("azure-kusto-data").KustoConnectionStringBuilder;
-const { DataFormat, JsonColumnMapping } = require("azure-kusto-ingest").IngestionPropertiesEnums;
+const { DataFormat, JsonColumnMapping } = require("azure-kusto-ingest");
 
 const kcsb = KustoConnectionStringBuilder.withAadApplicationKeyAuthentication(`https://ingest-${cluster}.kusto.windows.net`, appId, appKey, authorityId);
 
-const ingestionProps = new IngestionProps(
-        { 
-            database: "Database",
-            table: "Table",
-            format: DataFormat.JSON,
-            ingestionMapping: [
-                new JsonColumnMapping("TargetColumn1", "$.sourceProp1"),
-                new JsonColumnMapping("TargetColumn2", "$.sourceProp2"),
-                new JsonColumnMapping("TargetColumn3", "$.sourceProp3")
-            ]
-        }
-);
+const ingestionProps = new IngestionProps({
+    database: "Database",
+    table: "Table",
+    format: DataFormat.JSON,
+    ingestionMapping: [
+        new JsonColumnMapping("TargetColumn1", "$.sourceProp1"),
+        new JsonColumnMapping("TargetColumn2", "$.sourceProp2"),
+        new JsonColumnMapping("TargetColumn3", "$.sourceProp3"),
+    ],
+});
+
+const ingestClient = new IngestClient(kcsb, ingestionProps);
 
-const ingestClient = new IngestClient(
-    kcsb,
-    ingestionProps
-);
-    
 console.log("Ingest from file");
 
 Ingest();
 
 async function Ingest() {
-    try{
+    try {
         await ingestClient.ingestFromFile("file.json", null);
-    }
-    catch(err){
+    } catch (err) {
         console.log(err);
     }
     console.log("Wait for ingestion status...");
@@ -49,36 +43,52 @@ async function Ingest() {
 ```
 
 ## Authentication
+
 There are several authentication methods
 
 ### AAD App
+
 The are two ways to authenticate is to use app id and key
 
 1. Using app key
+
 ```javascript
-const kcsb = KustoConnectionStringBuilder.withAadApplicationKeyAuthentication(`https://ingest-${clusterName}.kusto.windows.net`,'appid','appkey','authorityId');
+const kcsb = KustoConnectionStringBuilder.withAadApplicationKeyAuthentication(
+    `https://ingest-${clusterName}.kusto.windows.net`,
+    "appid",
+    "appkey",
+    "authorityId"
+);
 ```
 
 1. Using a certificate:
 
 ```javascript
-const kcsb = KustoConnectionStringBuilder.withAadApplicationCertificateAuthentication(`https://ingest-${clusterName}.kusto.windows.net`, 'appid', 'certificate', 'thumbprint', 'authorityId');
+const kcsb = KustoConnectionStringBuilder.withAadApplicationCertificateAuthentication(
+    `https://ingest-${clusterName}.kusto.windows.net`,
+    "appid",
+    "certificate",
+    "thumbprint",
+    "authorityId"
+);
 ```
 
-
 ### Username/Password
+
 ```javascript
-KustoConnectionStringBuilder.withAadUserPasswordAuthentication(`https://${clusterName}.kusto.windows.net`,'username','password');
+KustoConnectionStringBuilder.withAadUserPasswordAuthentication(`https://${clusterName}.kusto.windows.net`, "username", "password");
 ```
 
-Authority is optional *when it can be inferred from the domain* ('user@microsoft.com' would make the authority 'microsoft.com'). 
+Authority is optional _when it can be inferred from the domain_ ('user@microsoft.com' would make the authority 'microsoft.com').
 In any case it is possible to pass the authority id
+
 ```javascript
-KustoConnectionStringBuilder.withAadUserPasswordAuthentication(`https://ingest-${clusterName}.kusto.windows.net`,'username','password','authorityId');
+KustoConnectionStringBuilder.withAadUserPasswordAuthentication(`https://ingest-${clusterName}.kusto.windows.net`, "username", "password", "authorityId");
 ```
 
 ### Device
-Using this method will write a token to the console, which can be used to authenticate at https://login.microsoftonline.com/common/oauth2/deviceauth and will allow temporary access. 
+
+Using this method will write a token to the console, which can be used to authenticate at https://login.microsoftonline.com/common/oauth2/deviceauth and will allow temporary access.
 
 **<!>It is not ment for production purposes<!>**
 
@@ -101,6 +111,7 @@ A Quick Overview is available at https://docs.microsoft.com/en-us/azure/data-exp
 Notice ingestion is done against the ingestion endpoint, which usually include `ingest-` prefix on the cluster name.
 
 ### Ingestion Properties
+
 Ingestion Props are instructions for Kusto on how to process the data.
 
 The easiest way to provide ingestion properties is to set them on the ingestion client like in the sample above.
@@ -109,26 +120,22 @@ It is also possible to pass them on each ingestion (will merge them with default
 Example props:
 
 ```javascript
-const ingestionProps = new IngestionProps(
-        "Database",
-        "Table",
-        DataFormat.JSON,
-        [
-            new JsonColumnMapping("TargetColumn1", "$.sourceProp1"),
-            new JsonColumnMapping("TargetColumn2", "$.sourceProp2"),
-            new JsonColumnMapping("TargetColumn3", "$.sourceProp3")
-        ]
-);
+const ingestionProps = new IngestionProps("Database", "Table", DataFormat.JSON, [
+    new JsonColumnMapping("TargetColumn1", "$.sourceProp1"),
+    new JsonColumnMapping("TargetColumn2", "$.sourceProp2"),
+    new JsonColumnMapping("TargetColumn3", "$.sourceProp3"),
+]);
 ```
 
 ### Ingestion Sources
+
 There are several methods of ingesting data into Kusto (Azure Data Explorer) using this library
 
 #### From Stream
 
 This is useful for cases you already have streams available (http respinse, file stream, etc..)
 
-```javascript
+````javascript
 try{
     await ingestClient.ingestFromStream(readable, null);
 }
@@ -150,19 +157,17 @@ try{
 catch(err){
     console.log(err);
 }
-```
+````
 
 #### From Azure Storage Blob
 
 Probably the easiest way would be to provide a uri (with [SAS](https://docs.microsoft.com/en-us/azure/storage/common/storage-dotnet-shared-access-signature-part-1)).
 
 ```javascript
-
 let blob = new BlobDescriptor(blobUri, size);
-try{
+try {
     await ingestClient.ingestFromBlob(blob, null);
-}
-catch(err){
+} catch (err) {
     console.log(err);
 }
 ```
@@ -179,38 +184,36 @@ For Example:
 const IngestClient = require("azure-kusto-ingest").IngestClient;
 const IngestStatusQueues = require("azure-kusto-ingest").IngestStatusQueues;
 const IngestionProps = require("azure-kusto-ingest").IngestionProperties;
-const { ReportLevel, ReportMethod } = require("azure-kusto-ingest").IngestionPropertiesEnums;
+const { ReportLevel, ReportMethod } = require("azure-kusto-ingest");
 const KustoConnectionStringBuilder = require("azure-kusto-data").KustoConnectionStringBuilder;
-const { DataFormat, JsonColumnMapping } = require("azure-kusto-ingest").IngestionPropertiesEnums;
+const { DataFormat, JsonColumnMapping } = require("azure-kusto-ingest");
 const fs = require("fs");
 
-
 const ingestClient = new IngestClient(
     KustoConnectionStringBuilder.withAadApplicationKeyAuthentication(`https://ingest-${clusterName}.kusto.windows.net`, appId, appKey, authorityId),
     new IngestionProps(
         "db",
         "table",
         DataFormat.JSON,
-        [
-            new JsonColumnMapping("Id", "$.id"),
-            new JsonColumnMapping("Type", "$.type"),
-            new JsonColumnMapping("Value", "$.type"),
-        ],
+        [new JsonColumnMapping("Id", "$.id"), new JsonColumnMapping("Type", "$.type"), new JsonColumnMapping("Value", "$.type")],
+        null,
         null,
         null,
         null,
         null,
         null,
-        null,      
         ReportLevel.FailuresAndSuccesses,
-        ReportMethod.Queue)
+        ReportMethod.Queue
+    )
 );
 
 const statusQueues = new IngestStatusQueues(ingestClient);
 
 async function waitForStatus() {
-    while (await statusQueues.failure.isEmpty() && await statusQueues.success.isEmpty()) {
-        await new Promise((resolve) => { setTimeout(resolve, 1000); });
+    while ((await statusQueues.failure.isEmpty()) && (await statusQueues.success.isEmpty())) {
+        await new Promise((resolve) => {
+            setTimeout(resolve, 1000);
+        });
     }
 
     const successes = statusQueues.success.pop();
@@ -218,20 +221,19 @@ async function waitForStatus() {
         console.log(JSON.stringify(success));
     }
 
-    const failures = statusQueues.failure.pop()
+    const failures = statusQueues.failure.pop();
     for (let failure of failures) {
         console.log(JSON.stringify(failure));
     }
 }
 
 async function ingestFromFile() {
-    try{
+    try {
         await ingestClient.ingestFromFile("file.json", null);
-    }
-    catch(err){
+    } catch (err) {
         console.log(err);
     }
     console.log("Wait for ingestion status...");
     await waitForStatus();
 }
-
+```

package/example.js

@@ -4,12 +4,11 @@
 const IngestClient = require("azure-kusto-ingest").IngestClient;
 const IngestStatusQueues = require("azure-kusto-ingest").IngestStatusQueues;
 const IngestionProps = require("azure-kusto-ingest").IngestionProperties;
-const { ReportLevel, ReportMethod } = require("azure-kusto-ingest").IngestionPropertiesEnums;
 const KustoConnectionStringBuilder = require("azure-kusto-data").KustoConnectionStringBuilder;
-const { DataFormat, JsonColumnMapping, IngestionMappingType, CompressionType } = require("azure-kusto-ingest").IngestionPropertiesEnums;
+const { DataFormat, JsonColumnMapping, IngestionMappingKind, CompressionType, ReportLevel, ReportMethod } = require("azure-kusto-ingest");
 const { BlobDescriptor, StreamDescriptor } = require("azure-kusto-ingest").IngestionDescriptors;
 const StreamingIngestClient = require("azure-kusto-ingest").StreamingIngestClient;
-const fs = require('fs');
+const fs = require("fs");
 
 const clusterName = null;
 const appId = null;
@@ -23,18 +22,15 @@ const props = new IngestionProps({
     ingestionMapping: [
         new JsonColumnMapping("TargetColumn1", "$.sourceProp1"),
         new JsonColumnMapping("TargetColumn2", "$.sourceProp2"),
-        new JsonColumnMapping("TargetColumn3", "$.sourceProp3")
+        new JsonColumnMapping("TargetColumn3", "$.sourceProp3"),
     ],
-    ingestionMappingType: IngestionMappingType.JSON,
+    ingestionMappingType: IngestionMappingKind.JSON,
     reportLevel: ReportLevel.FailuresAndSuccesses,
-    reportMethod: ReportMethod.Queue
-
+    reportMethod: ReportMethod.Queue,
 });
 
 const ingestClient = new IngestClient(
-    KustoConnectionStringBuilder.withAadApplicationKeyAuthentication(
-        `https://ingest-${clusterName}.kusto.windows.net`, appId, appKey, authorityId
-    ),
+    KustoConnectionStringBuilder.withAadApplicationKeyAuthentication(`https://ingest-${clusterName}.kusto.windows.net`, appId, appKey, authorityId),
     props
 );
 
@@ -42,19 +38,17 @@ const statusQueues = new IngestStatusQueues(ingestClient);
 
 startIngestion();
 
-// Streaming ingest client 
+// Streaming ingest client
 const props2 = new IngestionProps({
     database: "Database",
     table: "Table",
     format: DataFormat.JSON,
-    ingestionMappingReference: "Pre-defiend mapping name" // For json format mapping is required
+    ingestionMappingReference: "Pre-defined mapping name",
 });
 
 // Init with engine endpoint
 const streamingIngestClient = new StreamingIngestClient(
-    KustoConnectionStringBuilder.withAadApplicationKeyAuthentication(
-        `https://${clusterName}.kusto.windows.net`, appId, appKey, authorityId
-    ),
+    KustoConnectionStringBuilder.withAadApplicationKeyAuthentication(`https://${clusterName}.kusto.windows.net`, appId, appKey, authorityId),
     props2
 );
 
@@ -67,8 +61,7 @@ async function startIngestion() {
         console.log("Ingestion done?");
 
         await waitForStatus();
-    }
-    catch (err) {
+    } catch (err) {
         console.log(err);
     }
 
@@ -77,14 +70,13 @@ async function startIngestion() {
         console.log("Ingestion done?");
 
         await waitForStatus();
-    }
-    catch (err) {
+    } catch (err) {
         console.log(err);
     }
 }
 
 async function waitForStatus(numberOFIngestions = 1) {
-    while (await statusQueues.failure.isEmpty() && await statusQueues.success.isEmpty()) {
+    while ((await statusQueues.failure.isEmpty()) && (await statusQueues.success.isEmpty())) {
         console.log("Waiting for status...");
         await sleep(1000);
     }
@@ -100,27 +92,26 @@ async function waitForStatus(numberOFIngestions = 1) {
 }
 
 function sleep(ms) {
-    return new Promise((resolve) => { setTimeout(resolve, ms); });
+    return new Promise((resolve) => {
+        setTimeout(resolve, ms);
+    });
 }
 
 async function startStreamingIngestion() {
-
     // Ingest from file with either file path or FileDescriptor
     try {
         await streamingIngestClient.ingestFromFile("file.json", props2);
         console.log("Ingestion done");
-    }
-    catch (err) {
+    } catch (err) {
         console.log(err);
     }
 
     // Ingest from stream with either ReadStream or StreamDescriptor
     let stream = fs.createReadStream("file.json");
     try {
-        await streamingIngestClient.ingestFromStream("file.json", props2);
+        await streamingIngestClient.ingestFromStream(stream, props2);
         console.log("Ingestion done");
-    }
-    catch (err) {
+    } catch (err) {
         console.log(err);
     }
 
@@ -130,8 +121,7 @@ async function startStreamingIngestion() {
     try {
         await streamingIngestClient.ingestFromStream(streamDescriptor, props2);
         console.log("Ingestion done");
-    }
-    catch (err) {
+    } catch (err) {
         console.log(err);
     }
-}
+}