azure-kusto-ingest 2.2.2 → 3.1.0

package/source/ingestionProperties.d.ts

@@ -1,24 +1,88 @@
+import { ColumnMapping } from "./columnMappings";
+/**
+ * Data formats supported for Kusto ingestion.
+ */
 export declare enum DataFormat {
+    /**
+     * Comma-separated value.
+     */
     CSV = "csv",
+    /**
+     * Tab-separated value.
+     */
     TSV = "tsv",
+    /**
+     * Semicolon-separated value (the unique Azure Storage log format).
+     */
     SCSV = "scsv",
+    /**
+     * Start-Of-Header (CTRL-A)-separated value.
+     */
     SOHSV = "sohsv",
+    /**
+     * Pipeline-separated value (used by Cosmos).
+     */
     PSV = "psv",
+    /**
+     * Each record is a line and has just one field.
+     */
     TXT = "txt",
+    /**
+     * Whole stream is a single record with a single field.
+     */
+    RAW = "raw",
+    /**
+     * Tab-separated value with '\' escaping character.
+     */
+    TSVE = "tsve",
+    /**
+     * Data is in a JSON format, each line is record with a single JSON value.
+     */
     JSON = "json",
+    /**
+     * Data stream holds a single JSON value -- newlines are regular whitespace.
+     */
     SINGLEJSON = "singlejson",
+    /**
+     * The data stream is a concatenation of JSON documents (property bags all).
+     */
+    MULTIJSON = "multijson",
+    /**
+     * Avro format.
+     */
     AVRO = "avro",
+    /**
+     * Parquet format.
+     */
     PARQUET = "parquet",
-    TSVE = "tsve",
-    ORC = "orc"
+    /**
+     * Microsoft Cosmos structured streams format
+     */
+    SSTREAM = "sstream",
+    /**
+     * The Optimized Row Columnar (ORC)
+     */
+    ORC = "orc",
+    /**
+     * Avro format for ingesting through avro2json.
+     */
+    APACHEAVRO = "apacheavro",
+    /**
+     * W3C Extended Log File format.
+     */
+    W3CLogFile = "w3clogfile"
 }
-export declare enum IngestionMappingType {
+export declare enum IngestionMappingKind {
     CSV = "Csv",
-    PARQUET = "Parquet",
-    AVRO = "Avro",
     JSON = "Json",
-    ORC = "orc"
+    AVRO = "Avro",
+    PARQUET = "Parquet",
+    SSTREAM = "SStream",
+    ORC = "orc",
+    APACHEAVRO = "ApacheAvro",
+    W3CLOGFILE = "W3CLogFile"
 }
+export declare const dataFormatMappingKind: (dataFormat: DataFormat) => IngestionMappingKind;
 export declare enum ValidationOptions {
     DoNotValidate = 0,
     ValidateCsvInputConstantColumns = 1,
@@ -32,6 +96,7 @@ export declare class ValidationPolicy {
     readonly validationOptions: ValidationOptions;
     readonly validationImplications: ValidationImplications;
     constructor(validationOptions?: ValidationOptions, validationImplications?: ValidationImplications);
+    toJSON(): Record<string, number>;
 }
 export declare enum ReportLevel {
     FailuresOnly = 0,
@@ -39,45 +104,44 @@ export declare enum ReportLevel {
     FailuresAndSuccesses = 2
 }
 export declare enum ReportMethod {
-    Queue = 0
-}
-declare class ColumnMapping {
-}
-export declare class CsvColumnMapping extends ColumnMapping {
-    readonly columnName: string;
-    readonly cslDataType: string;
-    readonly ordinal: string;
-    constructor(columnName: string, cslDataType: string, ordinal: string);
+    Queue = 0,
+    Table = 1,
+    QueueAndTable = 2
 }
-export declare class JsonColumnMapping extends ColumnMapping {
-    readonly columnName: string;
-    readonly jsonPath: string;
-    readonly cslDataType: string | null;
-    constructor(columnName: string, jsonPath: string, cslDataType?: string | null);
-}
-declare class IngestionPropertiesFields {
-    database?: string | null;
-    table?: string | null;
-    format?: string | null;
-    ingestionMapping?: ColumnMapping[] | null;
-    ingestionMappingReference?: string | null;
-    ingestionMappingType?: string | null;
-    additionalTags?: string | null;
-    ingestIfNotExists?: string | null;
-    ingestByTags?: string[] | null;
-    dropByTags?: string[] | null;
-    flushImmediately?: boolean | null;
-    reportLevel?: ReportLevel | null;
-    reportMethod?: ReportMethod | null;
-    validationPolicy?: string | null;
+export interface IngestionPropertiesFields {
+    database?: string;
+    table?: string;
+    format?: DataFormat;
+    /**
+     * @deprecated. Use ingestionMappingColumns instead.
+     */
+    ingestionMapping?: ColumnMapping[];
+    ingestionMappingColumns?: ColumnMapping[];
+    ingestionMappingReference?: string;
+    /**
+     * @deprecated. Use ingestionMappingKind instead.
+     */
+    ingestionMappingType?: IngestionMappingKind;
+    ingestionMappingKind?: IngestionMappingKind;
+    additionalTags?: string;
+    ingestIfNotExists?: string;
+    ingestByTags?: string[];
+    dropByTags?: string[];
+    flushImmediately?: boolean;
+    reportLevel?: ReportLevel;
+    reportMethod?: ReportMethod;
+    validationPolicy?: ValidationPolicy;
     additionalProperties?: {
         [additional: string]: any;
     } | null;
 }
-export declare class IngestionProperties extends IngestionPropertiesFields {
-    constructor({ database, table, format, ingestionMapping, ingestionMappingReference, ingestionMappingType, additionalTags, ingestIfNotExists, ingestByTags, dropByTags, flushImmediately, reportLevel, reportMethod, validationPolicy, additionalProperties }: IngestionPropertiesFields);
+export interface IngestionProperties extends IngestionPropertiesFields {
+}
+export declare class IngestionProperties {
+    constructor(data: Partial<IngestionPropertiesFields>);
     validate(): void;
-    [extraProps: string]: any;
-    merge(extraProps: any): IngestionProperties;
+    merge(extraProps: IngestionPropertiesInput): IngestionProperties;
+    setDefaults(): void;
 }
+export declare type IngestionPropertiesInput = IngestionProperties | IngestionPropertiesFields | null | undefined;
 export default IngestionProperties;

package/source/ingestionProperties.js

@@ -2,30 +2,135 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.IngestionProperties = exports.JsonColumnMapping = exports.CsvColumnMapping = exports.ReportMethod = exports.ReportLevel = exports.ValidationPolicy = exports.ValidationImplications = exports.ValidationOptions = exports.IngestionMappingType = exports.DataFormat = void 0;
+exports.IngestionProperties = exports.ReportMethod = exports.ReportLevel = exports.ValidationPolicy = exports.ValidationImplications = exports.ValidationOptions = exports.dataFormatMappingKind = exports.IngestionMappingKind = exports.DataFormat = void 0;
+/* eslint-disable max-classes-per-file -- the main class is IngestionProperties, ValidationPolicy is a tiny class */
+const errors_1 = require("./errors");
+/**
+ * Data formats supported for Kusto ingestion.
+ */
 var DataFormat;
 (function (DataFormat) {
+    /**
+     * Comma-separated value.
+     */
     DataFormat["CSV"] = "csv";
+    /**
+     * Tab-separated value.
+     */
     DataFormat["TSV"] = "tsv";
+    /**
+     * Semicolon-separated value (the unique Azure Storage log format).
+     */
     DataFormat["SCSV"] = "scsv";
+    /**
+     * Start-Of-Header (CTRL-A)-separated value.
+     */
     DataFormat["SOHSV"] = "sohsv";
+    /**
+     * Pipeline-separated value (used by Cosmos).
+     */
     DataFormat["PSV"] = "psv";
+    /**
+     * Each record is a line and has just one field.
+     */
     DataFormat["TXT"] = "txt";
+    /**
+     * Whole stream is a single record with a single field.
+     */
+    DataFormat["RAW"] = "raw";
+    /**
+     * Tab-separated value with '\' escaping character.
+     */
+    DataFormat["TSVE"] = "tsve";
+    /**
+     * Data is in a JSON format, each line is record with a single JSON value.
+     */
     DataFormat["JSON"] = "json";
+    /**
+     * Data stream holds a single JSON value -- newlines are regular whitespace.
+     */
     DataFormat["SINGLEJSON"] = "singlejson";
+    /**
+     * The data stream is a concatenation of JSON documents (property bags all).
+     */
+    DataFormat["MULTIJSON"] = "multijson";
+    /**
+     * Avro format.
+     */
     DataFormat["AVRO"] = "avro";
+    /**
+     * Parquet format.
+     */
     DataFormat["PARQUET"] = "parquet";
-    DataFormat["TSVE"] = "tsve";
+    /**
+     * Microsoft Cosmos structured streams format
+     */
+    DataFormat["SSTREAM"] = "sstream";
+    /**
+     * The Optimized Row Columnar (ORC)
+     */
     DataFormat["ORC"] = "orc";
+    /**
+     * Avro format for ingesting through avro2json.
+     */
+    DataFormat["APACHEAVRO"] = "apacheavro";
+    /**
+     * W3C Extended Log File format.
+     */
+    DataFormat["W3CLogFile"] = "w3clogfile";
 })(DataFormat = exports.DataFormat || (exports.DataFormat = {}));
-var IngestionMappingType;
-(function (IngestionMappingType) {
-    IngestionMappingType["CSV"] = "Csv";
-    IngestionMappingType["PARQUET"] = "Parquet";
-    IngestionMappingType["AVRO"] = "Avro";
-    IngestionMappingType["JSON"] = "Json";
-    IngestionMappingType["ORC"] = "orc";
-})(IngestionMappingType = exports.IngestionMappingType || (exports.IngestionMappingType = {}));
+var IngestionMappingKind;
+(function (IngestionMappingKind) {
+    IngestionMappingKind["CSV"] = "Csv";
+    IngestionMappingKind["JSON"] = "Json";
+    IngestionMappingKind["AVRO"] = "Avro";
+    IngestionMappingKind["PARQUET"] = "Parquet";
+    IngestionMappingKind["SSTREAM"] = "SStream";
+    IngestionMappingKind["ORC"] = "orc";
+    IngestionMappingKind["APACHEAVRO"] = "ApacheAvro";
+    IngestionMappingKind["W3CLOGFILE"] = "W3CLogFile";
+})(IngestionMappingKind = exports.IngestionMappingKind || (exports.IngestionMappingKind = {}));
+const dataFormatMappingKind = (dataFormat) => {
+    switch (dataFormat) {
+        case DataFormat.CSV:
+            return IngestionMappingKind.CSV;
+        case DataFormat.TSV:
+            return IngestionMappingKind.CSV;
+        case DataFormat.SCSV:
+            return IngestionMappingKind.CSV;
+        case DataFormat.SOHSV:
+            return IngestionMappingKind.CSV;
+        case DataFormat.PSV:
+            return IngestionMappingKind.CSV;
+        case DataFormat.TXT:
+            return IngestionMappingKind.CSV;
+        case DataFormat.RAW:
+            return IngestionMappingKind.CSV;
+        case DataFormat.TSVE:
+            return IngestionMappingKind.CSV;
+        case DataFormat.JSON:
+            return IngestionMappingKind.JSON;
+        case DataFormat.SINGLEJSON:
+            return IngestionMappingKind.JSON;
+        case DataFormat.MULTIJSON:
+            return IngestionMappingKind.JSON;
+        case DataFormat.AVRO:
+            return IngestionMappingKind.AVRO;
+        case DataFormat.PARQUET:
+            return IngestionMappingKind.PARQUET;
+        case DataFormat.SSTREAM:
+            return IngestionMappingKind.SSTREAM;
+        case DataFormat.ORC:
+            return IngestionMappingKind.ORC;
+        case DataFormat.APACHEAVRO:
+            return IngestionMappingKind.APACHEAVRO;
+        case DataFormat.W3CLogFile:
+            return IngestionMappingKind.W3CLOGFILE;
+        default:
+            throw new errors_1.IngestionPropertiesValidationError(`Unsupported data format: ${dataFormat}`);
+    }
+};
+exports.dataFormatMappingKind = dataFormatMappingKind;
 var ValidationOptions;
 (function (ValidationOptions) {
     ValidationOptions[ValidationOptions["DoNotValidate"] = 0] = "DoNotValidate";
@@ -42,6 +147,12 @@ class ValidationPolicy {
         this.validationOptions = validationOptions;
         this.validationImplications = validationImplications;
     }
+    toJSON() {
+        return {
+            ValidationOptions: this.validationOptions,
+            ValidationImplications: this.validationImplications,
+        };
+    }
 }
 exports.ValidationPolicy = ValidationPolicy;
 var ReportLevel;
@@ -53,94 +164,79 @@ var ReportLevel;
 var ReportMethod;
 (function (ReportMethod) {
     ReportMethod[ReportMethod["Queue"] = 0] = "Queue";
+    ReportMethod[ReportMethod["Table"] = 1] = "Table";
+    ReportMethod[ReportMethod["QueueAndTable"] = 2] = "QueueAndTable";
 })(ReportMethod = exports.ReportMethod || (exports.ReportMethod = {}));
-class ColumnMapping {
-}
-class CsvColumnMapping extends ColumnMapping {
-    constructor(columnName, cslDataType, ordinal) {
-        super();
-        this.columnName = columnName;
-        this.cslDataType = cslDataType;
-        this.ordinal = ordinal;
-    }
-}
-exports.CsvColumnMapping = CsvColumnMapping;
-class JsonColumnMapping extends ColumnMapping {
-    constructor(columnName, jsonPath, cslDataType = null) {
-        super();
-        this.columnName = columnName;
-        this.jsonPath = jsonPath;
-        this.cslDataType = cslDataType;
-    }
-}
-exports.JsonColumnMapping = JsonColumnMapping;
-class IngestionPropertiesFields {
-    constructor() {
-        this.database = null;
-        this.table = null;
-        this.format = null;
-        this.ingestionMapping = null;
-        this.ingestionMappingReference = null;
-        this.ingestionMappingType = null;
-        this.additionalTags = null;
-        this.ingestIfNotExists = null;
-        this.ingestByTags = null;
-        this.dropByTags = null;
-        this.flushImmediately = null;
-        this.reportLevel = null;
-        this.reportMethod = null;
-        this.validationPolicy = null;
-        this.additionalProperties = null;
-    }
-}
-class IngestionProperties extends IngestionPropertiesFields {
-    constructor({ database = null, table = null, format = null, ingestionMapping = null, ingestionMappingReference = null, ingestionMappingType = null, additionalTags = null, ingestIfNotExists = null, ingestByTags = null, dropByTags = null, flushImmediately = null, reportLevel = null, reportMethod = null, validationPolicy = null, additionalProperties = null }) {
-        super();
-        if (ingestionMapping && ingestionMappingReference)
-            throw new Error("Both mapping and a mapping reference detected");
-        this.database = database;
-        this.table = table;
-        this.format = format;
-        this.ingestionMapping = ingestionMapping;
-        this.ingestionMappingType = ingestionMappingType;
-        this.ingestionMappingReference = ingestionMappingReference;
-        this.additionalTags = additionalTags;
-        this.ingestIfNotExists = ingestIfNotExists;
-        this.ingestByTags = ingestByTags;
-        this.dropByTags = dropByTags;
-        this.flushImmediately = flushImmediately;
-        this.reportLevel = reportLevel;
-        this.reportMethod = reportMethod;
-        this.validationPolicy = validationPolicy;
-        this.additionalProperties = additionalProperties;
+// eslint-disable-next-line no-redeclare
+class IngestionProperties {
+    constructor(data) {
+        Object.assign(this, data);
     }
     validate() {
-        if (!this.flushImmediately)
-            this.flushImmediately = false;
-        if (!this.reportLevel)
-            this.reportLevel = ReportLevel.DoNotReport;
-        if (!this.reportMethod)
-            this.reportMethod = ReportMethod.Queue;
         if (!this.database)
-            throw new Error("Must define a target database");
+            throw new errors_1.IngestionPropertiesValidationError("Must define a target database");
         if (!this.table)
-            throw new Error("Must define a target table");
+            throw new errors_1.IngestionPropertiesValidationError("Must define a target table");
         if (!this.format)
-            throw new Error("Must define a data format");
-        if (this.ingestionMapping && this.ingestionMappingReference)
-            throw new Error("Both mapping and a mapping reference detected");
-        if (!this.ingestionMapping && !this.ingestionMappingReference && this.format === DataFormat.JSON)
-            throw new Error("Json must have a mapping defined");
+            throw new errors_1.IngestionPropertiesValidationError("Must define a data format");
+        if (this.ingestionMappingType && !this.ingestionMappingKind) {
+            this.ingestionMappingKind = this.ingestionMappingType;
+        }
+        if (this.ingestionMapping && !this.ingestionMappingColumns) {
+            this.ingestionMappingColumns = this.ingestionMapping;
+        }
+        if (!this.ingestionMappingColumns && !this.ingestionMappingReference) {
+            if (this.ingestionMappingKind) {
+                throw new errors_1.IngestionPropertiesValidationError("Cannot define ingestionMappingKind without either ingestionMappingColumns or" + " ingestionMappingReference");
+            }
+        }
+        else {
+            const mappingKind = (0, exports.dataFormatMappingKind)(this.format);
+            if (this.ingestionMappingKind && this.ingestionMappingKind !== mappingKind) {
+                throw new errors_1.IngestionPropertiesValidationError(`Mapping kind '${this.ingestionMappingKind}' does not match format '${this.format}' (should be '${mappingKind}')`);
+            }
+            if (this.ingestionMappingColumns) {
+                if (this.ingestionMappingReference) {
+                    throw new errors_1.IngestionPropertiesValidationError("Cannot define both ingestionMappingColumns and ingestionMappingReference");
+                }
+                if (this.ingestionMappingColumns.length === 0) {
+                    throw new errors_1.IngestionPropertiesValidationError("Must define at least one column mapping");
+                }
+                const wrongMappings = this.ingestionMappingColumns
+                    .filter((m) => m.mappingKind !== mappingKind)
+                    .map((m) => `Mapping kind mismatch for column '${m.columnName}' - expected data format kind -  '${mappingKind}', but was '${m.mappingKind}'`);
+                if (wrongMappings.length > 0) {
+                    throw new errors_1.IngestionPropertiesValidationError(`Invalid columns:\n${wrongMappings.join("\n")}`);
+                }
+            }
+        }
     }
     merge(extraProps) {
         const merged = new IngestionProperties(this);
+        if (!extraProps) {
+            return merged;
+        }
+        const assign = (obj, prop, value) => {
+            obj[prop] = value;
+        };
         for (const key of Object.keys(extraProps)) {
-            if (extraProps[key] != null) {
-                merged[key] = extraProps[key];
+            if (extraProps[key]) {
+                assign(merged, key, extraProps[key]);
             }
         }
         return merged;
     }
+    setDefaults() {
+        if (!this.format) {
+            this.format = DataFormat.CSV;
+        }
+        if (!this.reportLevel) {
+            this.reportLevel = ReportLevel.FailuresOnly;
+        }
+        if (!this.reportMethod) {
+            this.reportMethod = ReportMethod.Queue;
+        }
+    }
 }
 exports.IngestionProperties = IngestionProperties;
 exports.default = IngestionProperties;

package/source/ingestionProperties.js.map

@@ -1 +1 @@
-{"version":3,"file":"ingestionProperties.js","sourceRoot":"","sources":["ingestionProperties.ts"],"names":[],"mappings":";AAAA,uCAAuC;AACvC,kCAAkC;;;AAElC,IAAY,UAaX;AAbD,WAAY,UAAU;IAClB,yBAAW,CAAA;IACX,yBAAW,CAAA;IACX,2BAAa,CAAA;IACb,6BAAe,CAAA;IACf,yBAAW,CAAA;IACX,yBAAW,CAAA;IACX,2BAAa,CAAA;IACb,uCAAyB,CAAA;IACzB,2BAAa,CAAA;IACb,iCAAmB,CAAA;IACnB,2BAAa,CAAA;IACb,yBAAW,CAAA;AACf,CAAC,EAbW,UAAU,GAAV,kBAAU,KAAV,kBAAU,QAarB;AAED,IAAY,oBAMX;AAND,WAAY,oBAAoB;IAC5B,mCAAW,CAAA;IACX,2CAAmB,CAAA;IACnB,qCAAa,CAAA;IACb,qCAAa,CAAA;IACb,mCAAW,CAAA;AACf,CAAC,EANW,oBAAoB,GAApB,4BAAoB,KAApB,4BAAoB,QAM/B;AAED,IAAY,iBAIX;AAJD,WAAY,iBAAiB;IACzB,2EAAiB,CAAA;IACjB,+GAAmC,CAAA;IACnC,+GAAmC,CAAA;AACvC,CAAC,EAJW,iBAAiB,GAAjB,yBAAiB,KAAjB,yBAAiB,QAI5B;AAGD,IAAY,sBAGX;AAHD,WAAY,sBAAsB;IAC9B,mEAAQ,CAAA;IACR,+EAAc,CAAA;AAClB,CAAC,EAHW,sBAAsB,GAAtB,8BAAsB,KAAtB,8BAAsB,QAGjC;AAGD,MAAa,gBAAgB;IACzB,YAAqB,oBAAuC,iBAAiB,CAAC,aAAa,EAAW,yBAAiD,sBAAsB,CAAC,UAAU;QAAnK,sBAAiB,GAAjB,iBAAiB,CAAqD;QAAW,2BAAsB,GAAtB,sBAAsB,CAA4D;IACxL,CAAC;CACJ;AAHD,4CAGC;AAED,IAAY,WAIX;AAJD,WAAY,WAAW;IACnB,6DAAgB,CAAA;IAChB,2DAAe,CAAA;IACf,6EAAwB,CAAA;AAC5B,CAAC,EAJW,WAAW,GAAX,mBAAW,KAAX,mBAAW,QAItB;AAED,IAAY,YAEX;AAFD,WAAY,YAAY;IACpB,iDAAS,CAAA;AACb,CAAC,EAFW,YAAY,GAAZ,oBAAY,KAAZ,oBAAY,QAEvB;AAED,MAAM,aAAa;CAClB;AAED,MAAa,gBAAiB,SAAQ,aAAa;IAC/C,YAAqB,UAAkB,EAAW,WAAmB,EAAW,OAAe;QAC3F,KAAK,EAAE,CAAC;QADS,eAAU,GAAV,UAAU,CAAQ;QAAW,gBAAW,GAAX,WAAW,CAAQ;QAAW,YAAO,GAAP,OAAO,CAAQ;IAE/F,CAAC;CACJ;AAJD,4CAIC;AAED,MAAa,iBAAkB,SAAQ,aAAa;IAChD,YAAqB,UAAkB,EAAW,QAAgB,EAAW,cAA6B,IAAI;QAC1G,KAAK,EAAE,CAAC;QADS,eAAU,GAAV,UAAU,CAAQ;QAAW,aAAQ,GAAR,QAAQ,CAAQ;QAAW,gBAAW,GAAX,WAAW,CAAsB;IAE9G,CAAC;CACJ;AAJD,8CAIC;AAED,MAAM,yBAAyB;IAA/B;QACI,aAAQ,GAAmB,IAAI,CAAC;QAChC,UAAK,GAAmB,IAAI,CAAC;QAC7B,WAAM,GAAmB,IAAI,CAAC;QAC9B,qBAAgB,GAA4B,IAAI,CAAC;QACjD,8BAAyB,GAAmB,IAAI,CAAC;QACjD,yBAAoB,GAAmB,IAAI,CAAC;QAC5C,mBAAc,GAAmB,IAAI,CAAC;QACtC,sBAAiB,GAAmB,IAAI,CAAC;QACzC,iBAAY,GAAqB,IAAI,CAAC;QACtC,eAAU,GAAqB,IAAI,CAAC;QACpC,qBAAgB,GAAoB,IAAI,CAAC;QACzC,gBAAW,GAAwB,IAAI,CAAC;QACxC,iBAAY,GAAyB,IAAI,CAAC;QAC1C,qBAAgB,GAAmB,IAAI,CAAC;QACxC,yBAAoB,GAAwC,IAAI,CAAC;IACrE,CAAC;CAAA;AAED,MAAa,mBAAoB,SAAQ,yBAAyB;IAC9D,YAAY,EACI,QAAQ,GAAG,IAAI,EACf,KAAK,GAAG,IAAI,EACZ,MAAM,GAAG,IAAI,EACb,gBAAgB,GAAG,IAAI,EACvB,yBAAyB,GAAG,IAAI,EAChC,oBAAoB,GAAG,IAAI,EAC3B,cAAc,GAAG,IAAI,EACrB,iBAAiB,GAAG,IAAI,EACxB,YAAY,GAAG,IAAI,EACnB,UAAU,GAAG,IAAI,EACjB,gBAAgB,GAAG,IAAI,EACvB,WAAW,GAAG,IAAI,EAClB,YAAY,GAAG,IAAI,EACnB,gBAAgB,GAAG,IAAI,EACvB,oBAAoB,GAAG,IAAI,EACH;QACpC,KAAK,EAAE,CAAC;QACR,IAAI,gBAAgB,IAAI,yBAAyB;YAAE,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;QAEpH,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;QACjD,IAAI,CAAC,yBAAyB,GAAG,yBAAyB,CAAC;QAC3D,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAC3C,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;IACrD,CAAC;IAED,QAAQ;QAEJ,IAAI,CAAC,IAAI,CAAC,gBAAgB;YAAE,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;QAC1D,IAAI,CAAC,IAAI,CAAC,WAAW;YAAE,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC,WAAW,CAAC;QAClE,IAAI,CAAC,IAAI,CAAC,YAAY;YAAE,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC,KAAK,CAAC;QAE/D,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACrE,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAC/D,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/D,IAAI,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,yBAAyB;YACvD,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;QACrE,IAAI,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,IAAI,CAAC,yBAAyB,IAAI,IAAI,CAAC,MAAM,KAAK,UAAU,CAAC,IAAI;YAC5F,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IAC5D,CAAC;IAID,KAAK,CAAC,UAAe;QACjB,MAAM,MAAM,GAAG,IAAI,mBAAmB,CAAC,IAAI,CAAC,CAAC;QAE7C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE;YACvC,IAAI,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE;gBACzB,MAAM,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;aACjC;SACJ;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;CACJ;AAlED,kDAkEC;AAED,kBAAe,mBAAmB,CAAC"}
+{"version":3,"file":"ingestionProperties.js","sourceRoot":"","sources":["ingestionProperties.ts"],"names":[],"mappings":";AAAA,uCAAuC;AACvC,kCAAkC;;;AAElC,oHAAoH;AAEpH,qCAA8D;AAG9D;;GAEG;AACH,IAAY,UAqEX;AArED,WAAY,UAAU;IAClB;;OAEG;IACH,yBAAW,CAAA;IACX;;OAEG;IACH,yBAAW,CAAA;IACX;;OAEG;IACH,2BAAa,CAAA;IACb;;OAEG;IACH,6BAAe,CAAA;IACf;;OAEG;IACH,yBAAW,CAAA;IACX;;OAEG;IACH,yBAAW,CAAA;IACX;;OAEG;IACH,yBAAW,CAAA;IACX;;OAEG;IACH,2BAAa,CAAA;IACb;;OAEG;IACH,2BAAa,CAAA;IACb;;OAEG;IACH,uCAAyB,CAAA;IACzB;;OAEG;IACH,qCAAuB,CAAA;IACvB;;OAEG;IACH,2BAAa,CAAA;IACb;;OAEG;IACH,iCAAmB,CAAA;IACnB;;OAEG;IACH,iCAAmB,CAAA;IACnB;;OAEG;IACH,yBAAW,CAAA;IACX;;OAEG;IACH,uCAAyB,CAAA;IACzB;;OAEG;IACH,uCAAyB,CAAA;AAC7B,CAAC,EArEW,UAAU,GAAV,kBAAU,KAAV,kBAAU,QAqErB;AAED,IAAY,oBASX;AATD,WAAY,oBAAoB;IAC5B,mCAAW,CAAA;IACX,qCAAa,CAAA;IACb,qCAAa,CAAA;IACb,2CAAmB,CAAA;IACnB,2CAAmB,CAAA;IACnB,mCAAW,CAAA;IACX,iDAAyB,CAAA;IACzB,iDAAyB,CAAA;AAC7B,CAAC,EATW,oBAAoB,GAApB,4BAAoB,KAApB,4BAAoB,QAS/B;AAEM,MAAM,qBAAqB,GAAG,CAAC,UAAsB,EAAwB,EAAE;IAClF,QAAQ,UAAU,EAAE;QAChB,KAAK,UAAU,CAAC,GAAG;YACf,OAAO,oBAAoB,CAAC,GAAG,CAAC;QACpC,KAAK,UAAU,CAAC,GAAG;YACf,OAAO,oBAAoB,CAAC,GAAG,CAAC;QACpC,KAAK,UAAU,CAAC,IAAI;YAChB,OAAO,oBAAoB,CAAC,GAAG,CAAC;QACpC,KAAK,UAAU,CAAC,KAAK;YACjB,OAAO,oBAAoB,CAAC,GAAG,CAAC;QACpC,KAAK,UAAU,CAAC,GAAG;YACf,OAAO,oBAAoB,CAAC,GAAG,CAAC;QACpC,KAAK,UAAU,CAAC,GAAG;YACf,OAAO,oBAAoB,CAAC,GAAG,CAAC;QACpC,KAAK,UAAU,CAAC,GAAG;YACf,OAAO,oBAAoB,CAAC,GAAG,CAAC;QACpC,KAAK,UAAU,CAAC,IAAI;YAChB,OAAO,oBAAoB,CAAC,GAAG,CAAC;QACpC,KAAK,UAAU,CAAC,IAAI;YAChB,OAAO,oBAAoB,CAAC,IAAI,CAAC;QACrC,KAAK,UAAU,CAAC,UAAU;YACtB,OAAO,oBAAoB,CAAC,IAAI,CAAC;QACrC,KAAK,UAAU,CAAC,SAAS;YACrB,OAAO,oBAAoB,CAAC,IAAI,CAAC;QACrC,KAAK,UAAU,CAAC,IAAI;YAChB,OAAO,oBAAoB,CAAC,IAAI,CAAC;QACrC,KAAK,UAAU,CAAC,OAAO;YACnB,OAAO,oBAAoB,CAAC,OAAO,CAAC;QACxC,KAAK,UAAU,CAAC,OAAO;YACnB,OAAO,oBAAoB,CAAC,OAAO,CAAC;QACxC,KAAK,UAAU,CAAC,GAAG;YACf,OAAO,oBAAoB,CAAC,GAAG,CAAC;QACpC,KAAK,UAAU,CAAC,UAAU;YACtB,OAAO,oBAAoB,CAAC,UAAU,CAAC;QAC3C,KAAK,UAAU,CAAC,UAAU;YACtB,OAAO,oBAAoB,CAAC,UAAU,CAAC;QAC3C;YACI,MAAM,IAAI,2CAAkC,CAAC,4BAA4B,UAAU,EAAE,CAAC,CAAC;KAC9F;AACL,CAAC,CAAC;AAvCW,QAAA,qBAAqB,yBAuChC;AAEF,IAAY,iBAIX;AAJD,WAAY,iBAAiB;IACzB,2EAAiB,CAAA;IACjB,+GAAmC,CAAA;IACnC,+GAAmC,CAAA;AACvC,CAAC,EAJW,iBAAiB,GAAjB,yBAAiB,KAAjB,yBAAiB,QAI5B;AAED,IAAY,sBAGX;AAHD,WAAY,sBAAsB;IAC9B,mEAAQ,CAAA;IACR,+EAAc,CAAA;AAClB,CAAC,EAHW,sBAAsB,GAAtB,8BAAsB,KAAtB,8BAAsB,QAGjC;AAED,MAAa,gBAAgB;IACzB,YACa,oBAAuC,iBAAiB,CAAC,aAAa,EACtE,yBAAiD,sBAAsB,CAAC,UAAU;QADlF,sBAAiB,GAAjB,iBAAiB,CAAqD;QACtE,2BAAsB,GAAtB,sBAAsB,CAA4D;IAC5F,CAAC;IAEJ,MAAM;QACF,OAAO;YACH,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,sBAAsB,EAAE,IAAI,CAAC,sBAAsB;SACtD,CAAC;IACN,CAAC;CACJ;AAZD,4CAYC;AAED,IAAY,WAIX;AAJD,WAAY,WAAW;IACnB,6DAAgB,CAAA;IAChB,2DAAe,CAAA;IACf,6EAAwB,CAAA;AAC5B,CAAC,EAJW,WAAW,GAAX,mBAAW,KAAX,mBAAW,QAItB;AAED,IAAY,YAIX;AAJD,WAAY,YAAY;IACpB,iDAAS,CAAA;IACT,iDAAK,CAAA;IACL,iEAAa,CAAA;AACjB,CAAC,EAJW,YAAY,GAAZ,oBAAY,KAAZ,oBAAY,QAIvB;AAgCD,wCAAwC;AACxC,MAAa,mBAAmB;IAC5B,YAAY,IAAwC;QAChD,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC9B,CAAC;IAED,QAAQ;QACJ,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,MAAM,IAAI,2CAAkC,CAAC,+BAA+B,CAAC,CAAC;QAClG,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,MAAM,IAAI,2CAAkC,CAAC,4BAA4B,CAAC,CAAC;QAC5F,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,2CAAkC,CAAC,2BAA2B,CAAC,CAAC;QAE5F,IAAI,IAAI,CAAC,oBAAoB,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE;YACzD,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC;SACzD;QAED,IAAI,IAAI,CAAC,gBAAgB,IAAI,CAAC,IAAI,CAAC,uBAAuB,EAAE;YACxD,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC,gBAAgB,CAAC;SACxD;QAED,IAAI,CAAC,IAAI,CAAC,uBAAuB,IAAI,CAAC,IAAI,CAAC,yBAAyB,EAAE;YAClE,IAAI,IAAI,CAAC,oBAAoB,EAAE;gBAC3B,MAAM,IAAI,2CAAkC,CACxC,8EAA8E,GAAG,4BAA4B,CAChH,CAAC;aACL;SACJ;aAAM;YACH,MAAM,WAAW,GAAG,IAAA,6BAAqB,EAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvD,IAAI,IAAI,CAAC,oBAAoB,IAAI,IAAI,CAAC,oBAAoB,KAAK,WAAW,EAAE;gBACxE,MAAM,IAAI,2CAAkC,CACxC,iBAAiB,IAAI,CAAC,oBAAoB,4BAA4B,IAAI,CAAC,MAAM,iBAAiB,WAAW,IAAI,CACpH,CAAC;aACL;YACD,IAAI,IAAI,CAAC,uBAAuB,EAAE;gBAC9B,IAAI,IAAI,CAAC,yBAAyB,EAAE;oBAChC,MAAM,IAAI,2CAAkC,CAAC,0EAA0E,CAAC,CAAC;iBAC5H;gBAED,IAAI,IAAI,CAAC,uBAAuB,CAAC,MAAM,KAAK,CAAC,EAAE;oBAC3C,MAAM,IAAI,2CAAkC,CAAC,yCAAyC,CAAC,CAAC;iBAC3F;gBAED,MAAM,aAAa,GAAG,IAAI,CAAC,uBAAuB;qBAC7C,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,WAAW,CAAC;qBAC5C,GAAG,CACA,CAAC,CAAC,EAAE,EAAE,CAAC,qCAAqC,CAAC,CAAC,UAAU,qCAAqC,WAAW,eAAe,CAAC,CAAC,WAAW,GAAG,CAC1I,CAAC;gBACN,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE;oBAC1B,MAAM,IAAI,2CAAkC,CAAC,qBAAqB,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;iBACjG;aACJ;SACJ;IACL,CAAC;IAED,KAAK,CAAC,UAAoC;QACtC,MAAM,MAAM,GAAG,IAAI,mBAAmB,CAAC,IAAI,CAAC,CAAC;QAE7C,IAAI,CAAC,UAAU,EAAE;YACb,OAAO,MAAM,CAAC;SACjB;QAED,MAAM,MAAM,GAAG,CACX,GAA8B,EAC9B,IAAO,EACP,KAAQ,EACV,EAAE;YACA,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;QACtB,CAAC,CAAC;QAEF,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAwC,EAAE;YAC9E,IAAI,UAAU,CAAC,GAAG,CAAC,EAAE;gBACjB,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;aACxC;SACJ;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,WAAW;QACP,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YACd,IAAI,CAAC,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC;SAChC;QACD,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;YACnB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC,YAAY,CAAC;SAC/C;QACD,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;YACpB,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC,KAAK,CAAC;SAC1C;IACL,CAAC;CACJ;AAvFD,kDAuFC;AAID,kBAAe,mBAAmB,CAAC"}

package/source/managedStreamingIngestClient.d.ts

@@ -0,0 +1,36 @@
+/// <reference types="node" />
+import { IngestionPropertiesInput } from "./ingestionProperties";
+import { FileDescriptor, StreamDescriptor } from "./descriptors";
+import { AbstractKustoClient } from "./abstractKustoClient";
+import { KustoConnectionStringBuilder } from "azure-kusto-data";
+import { KustoResponseDataSet } from "azure-kusto-data/source/response";
+import { QueueSendMessageResponse } from "@azure/storage-queue";
+import { Readable } from "stream";
+declare class KustoManagedStreamingIngestClient extends AbstractKustoClient {
+    private streamingIngestClient;
+    private queuedIngestClient;
+    private baseSleepTimeSecs;
+    private baseJitterSecs;
+    /**
+     * Creates a KustoManagedStreamingIngestClient from a DM connection string.
+     * This method infers the engine connection string.
+     * For advanced usage, use the constructor that takes a DM connection string and an engine connection string.
+     *
+     * @param dmConnectionString The DM connection string.
+     * @param defaultProps The default ingestion properties.
+     */
+    static fromDmConnectionString(dmConnectionString: KustoConnectionStringBuilder, defaultProps?: IngestionPropertiesInput): KustoManagedStreamingIngestClient;
+    /**
+     * Creates a KustoManagedStreamingIngestClient from a engine connection string.
+     * This method infers the engine connection string.
+     * For advanced usage, use the constructor that takes an engine connection string and an engine connection string.
+     *
+     * @param engineConnectionString The engine connection string.
+     * @param defaultProps The default ingestion properties.
+     */
+    static fromEngineConnectionString(engineConnectionString: KustoConnectionStringBuilder, defaultProps?: IngestionPropertiesInput): KustoManagedStreamingIngestClient;
+    constructor(engineKcsb: string | KustoConnectionStringBuilder, dmKcsb: string | KustoConnectionStringBuilder, defaultProps?: IngestionPropertiesInput);
+    ingestFromStream(stream: StreamDescriptor | Readable, ingestionProperties?: IngestionPropertiesInput): Promise<any>;
+    ingestFromFile(file: FileDescriptor | string, ingestionProperties?: IngestionPropertiesInput): Promise<KustoResponseDataSet | QueueSendMessageResponse>;
+}
+export default KustoManagedStreamingIngestClient;