azure-kusto-ingest 6.0.2 → 7.0.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist-esm/src/abstractKustoClient.js +8 -15
- package/dist-esm/src/abstractKustoClient.js.map +1 -1
- package/dist-esm/src/columnMappings.js +22 -34
- package/dist-esm/src/columnMappings.js.map +1 -1
- package/dist-esm/src/descriptors.js +15 -27
- package/dist-esm/src/descriptors.js.map +1 -1
- package/dist-esm/src/errors.js +1 -5
- package/dist-esm/src/errors.js.map +1 -1
- package/dist-esm/src/fileDescriptor.browser.js +9 -16
- package/dist-esm/src/fileDescriptor.browser.js.map +1 -1
- package/dist-esm/src/fileDescriptor.js +20 -28
- package/dist-esm/src/fileDescriptor.js.map +1 -1
- package/dist-esm/src/index.js +38 -73
- package/dist-esm/src/index.js.map +1 -1
- package/dist-esm/src/ingestClient.browser.js +11 -15
- package/dist-esm/src/ingestClient.browser.js.map +1 -1
- package/dist-esm/src/ingestClient.js +11 -15
- package/dist-esm/src/ingestClient.js.map +1 -1
- package/dist-esm/src/ingestClientBase.js +25 -32
- package/dist-esm/src/ingestClientBase.js.map +1 -1
- package/dist-esm/src/ingestionBlobInfo.js +4 -8
- package/dist-esm/src/ingestionBlobInfo.js.map +1 -1
- package/dist-esm/src/ingestionProperties.js +29 -36
- package/dist-esm/src/ingestionProperties.js.map +1 -1
- package/dist-esm/src/ingestionResult.js +10 -17
- package/dist-esm/src/ingestionResult.js.map +1 -1
- package/dist-esm/src/managedStreamingIngestClient.js +29 -34
- package/dist-esm/src/managedStreamingIngestClient.js.map +1 -1
- package/dist-esm/src/rankedStorageAccount.js +1 -5
- package/dist-esm/src/rankedStorageAccount.js.map +1 -1
- package/dist-esm/src/rankedStorageAccountSet.js +3 -7
- package/dist-esm/src/rankedStorageAccountSet.js.map +1 -1
- package/dist-esm/src/resourceManager.js +24 -31
- package/dist-esm/src/resourceManager.js.map +1 -1
- package/dist-esm/src/retry.js +3 -8
- package/dist-esm/src/retry.js.map +1 -1
- package/dist-esm/src/status.js +6 -11
- package/dist-esm/src/status.js.map +1 -1
- package/dist-esm/src/statusQ.js +3 -7
- package/dist-esm/src/statusQ.js.map +1 -1
- package/dist-esm/src/streamUtils.browser.js +8 -15
- package/dist-esm/src/streamUtils.browser.js.map +1 -1
- package/dist-esm/src/streamUtils.js +18 -28
- package/dist-esm/src/streamUtils.js.map +1 -1
- package/dist-esm/src/streamingIngestClient.browser.js +9 -11
- package/dist-esm/src/streamingIngestClient.browser.js.map +1 -1
- package/dist-esm/src/streamingIngestClient.js +12 -17
- package/dist-esm/src/streamingIngestClient.js.map +1 -1
- package/dist-esm/src/streamingIngestClientBase.js +7 -11
- package/dist-esm/src/streamingIngestClientBase.js.map +1 -1
- package/package.json +11 -6
- package/types/src/abstractKustoClient.d.ts +2 -2
- package/types/src/columnMappings.d.ts +1 -1
- package/types/src/descriptors.d.ts +1 -2
- package/types/src/fileDescriptor.browser.d.ts +2 -2
- package/types/src/fileDescriptor.d.ts +2 -2
- package/types/src/index.d.ts +13 -13
- package/types/src/ingestClient.browser.d.ts +5 -5
- package/types/src/ingestClient.d.ts +5 -6
- package/types/src/ingestClientBase.d.ts +5 -5
- package/types/src/ingestionBlobInfo.d.ts +3 -3
- package/types/src/ingestionProperties.d.ts +1 -1
- package/types/src/managedStreamingIngestClient.d.ts +5 -6
- package/types/src/rankedStorageAccountSet.d.ts +1 -1
- package/types/src/status.d.ts +2 -2
- package/types/src/statusQ.d.ts +2 -2
- package/types/src/streamUtils.browser.d.ts +2 -3
- package/types/src/streamUtils.d.ts +2 -3
- package/types/src/streamingIngestClient.browser.d.ts +4 -4
- package/types/src/streamingIngestClient.d.ts +4 -5
- package/types/src/streamingIngestClientBase.d.ts +3 -3
|
@@ -1,26 +1,20 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
// Copyright (c) Microsoft Corporation.
|
|
3
2
|
// Licensed under the MIT License.
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
};
|
|
7
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8
|
-
exports.AbstractKustoClient = void 0;
|
|
9
|
-
const ingestionProperties_1 = require("./ingestionProperties");
|
|
10
|
-
const is_ip_1 = __importDefault(require("is-ip"));
|
|
3
|
+
import { IngestionProperties } from "./ingestionProperties.js";
|
|
4
|
+
import isIP from "is-ip";
|
|
11
5
|
const INGEST_PREFIX = "ingest-";
|
|
12
6
|
const PROTOCOL_SUFFIX = "://";
|
|
13
|
-
class AbstractKustoClient {
|
|
7
|
+
export class AbstractKustoClient {
|
|
14
8
|
constructor(defaultProps) {
|
|
15
9
|
this._isClosed = false;
|
|
16
10
|
if (!defaultProps) {
|
|
17
|
-
this.defaultProps = new
|
|
11
|
+
this.defaultProps = new IngestionProperties({});
|
|
18
12
|
}
|
|
19
|
-
else if (!(defaultProps instanceof
|
|
20
|
-
this.defaultProps = new
|
|
13
|
+
else if (!(defaultProps instanceof IngestionProperties)) {
|
|
14
|
+
this.defaultProps = new IngestionProperties(defaultProps);
|
|
21
15
|
}
|
|
22
16
|
else {
|
|
23
|
-
this.defaultProps = new
|
|
17
|
+
this.defaultProps = new IngestionProperties(defaultProps);
|
|
24
18
|
}
|
|
25
19
|
}
|
|
26
20
|
_getMergedProps(newProperties) {
|
|
@@ -59,7 +53,7 @@ class AbstractKustoClient {
|
|
|
59
53
|
if (!authority) {
|
|
60
54
|
return true;
|
|
61
55
|
}
|
|
62
|
-
const is_ip = (
|
|
56
|
+
const is_ip = isIP(authority);
|
|
63
57
|
const is_localhost = authority.includes("localhost");
|
|
64
58
|
return is_localhost || is_ip || authority.toLowerCase() === "onebox.dev.kusto.windows.net";
|
|
65
59
|
}
|
|
@@ -68,5 +62,4 @@ class AbstractKustoClient {
|
|
|
68
62
|
}
|
|
69
63
|
}
|
|
70
64
|
}
|
|
71
|
-
exports.AbstractKustoClient = AbstractKustoClient;
|
|
72
65
|
//# sourceMappingURL=abstractKustoClient.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"abstractKustoClient.js","sourceRoot":"","sources":["../../src/abstractKustoClient.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"abstractKustoClient.js","sourceRoot":"","sources":["../../src/abstractKustoClient.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAElC,OAAO,EAAE,mBAAmB,EAA4B,MAAM,0BAA0B,CAAC;AAEzF,OAAO,IAAI,MAAM,OAAO,CAAC;AAEzB,MAAM,aAAa,GAAG,SAAS,CAAC;AAChC,MAAM,eAAe,GAAG,KAAK,CAAC;AAE9B,MAAM,OAAgB,mBAAmB;IAKrC,YAAsB,YAAsC;QAFlD,cAAS,GAAY,KAAK,CAAC;QAGjC,IAAI,CAAC,YAAY,EAAE,CAAC;YAChB,IAAI,CAAC,YAAY,GAAG,IAAI,mBAAmB,CAAC,EAAE,CAAC,CAAC;QACpD,CAAC;aAAM,IAAI,CAAC,CAAC,YAAY,YAAY,mBAAmB,CAAC,EAAE,CAAC;YACxD,IAAI,CAAC,YAAY,GAAG,IAAI,mBAAmB,CAAC,YAAY,CAAC,CAAC;QAC9D,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,YAAY,GAAG,IAAI,mBAAmB,CAAC,YAAY,CAAC,CAAC;QAC9D,CAAC;IACL,CAAC;IAED,eAAe,CAAC,aAAwC;QACpD,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QACrD,KAAK,CAAC,WAAW,EAAE,CAAC;QACpB,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;YAClB,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC;QAC1C,CAAC;QAED,KAAK,CAAC,QAAQ,EAAE,CAAC;QACjB,OAAO,KAAK,CAAC;IACjB,CAAC;IAQM,KAAK;QACR,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;IAC1B,CAAC;IAES,UAAU;QAChB,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACxC,CAAC;IACL,CAAC;IAED,oBAAoB,CAAC,UAAmB;QACpC,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,EAAE,CAAC;YAC3F,OAAO,UAAU,CAAC;QACtB,CAAC;QACD,OAAO,UAAU,CAAC,OAAO,CAAC,eAAe,EAAE,eAAe,GAAG,aAAa,CAAC,CAAC;IAChF,CAAC;IAED,gBAAgB,CAAC,UAAmB;QAChC,IAAI,UAAU,IAAI,UAAU,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;YACnD,OAAO,UAAU,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;QACjD,CAAC;QACD,OAAO,UAAU,CAAC;IACtB,CAAC;IAED,kBAAkB,CAAC,UAAkB;QACjC,IAAI,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC;YACtC,MAAM,SAAS,GAAG,SAAS,CAAC,QAAQ,CAAC;YACrC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACb,OAAO,IAAI,CAAC;YAChB,CAAC;YACD,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;YAC9B,MAAM,YAAY,GAAG,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YACrD,OAAO,YAAY,IAAI,KAAK,IAAI,SAAS,CAAC,WAAW,EAAE,KAAK,8BAA8B,CAAC;QAC/F,CAAC;QAAC,WAAM,CAAC;YACL,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC;CACJ","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport { IngestionProperties, IngestionPropertiesInput } from \"./ingestionProperties.js\";\nimport { StreamDescriptor, FileDescriptorBase, BlobDescriptor } from \"./descriptors.js\";\nimport isIP from \"is-ip\";\n\nconst INGEST_PREFIX = \"ingest-\";\nconst PROTOCOL_SUFFIX = \"://\";\n\nexport abstract class AbstractKustoClient {\n public defaultProps: IngestionProperties;\n public defaultDatabase?: string;\n protected _isClosed: boolean = false;\n\n protected constructor(defaultProps: IngestionPropertiesInput) {\n if (!defaultProps) {\n this.defaultProps = new IngestionProperties({});\n } else if (!(defaultProps instanceof IngestionProperties)) {\n this.defaultProps = new IngestionProperties(defaultProps);\n } else {\n this.defaultProps = new IngestionProperties(defaultProps);\n }\n }\n\n _getMergedProps(newProperties?: IngestionPropertiesInput): IngestionProperties {\n const props = this.defaultProps.merge(newProperties);\n props.setDefaults();\n if (!props.database) {\n props.database = this.defaultDatabase;\n }\n\n props.validate();\n return props;\n }\n\n abstract ingestFromStream(stream: StreamDescriptor, ingestionProperties: IngestionPropertiesInput): Promise<any>;\n\n abstract ingestFromFile(file: FileDescriptorBase | string | Blob, ingestionProperties: IngestionPropertiesInput): Promise<any>;\n\n abstract ingestFromBlob(blob: string | BlobDescriptor, ingestionProperties?: IngestionPropertiesInput): Promise<any>;\n\n public close(): void {\n this._isClosed = true;\n }\n\n protected ensureOpen() {\n if (this._isClosed) {\n throw new Error(\"Client is closed\");\n }\n }\n\n getIngestionEndpoint(clusterUrl?: string): string | undefined {\n if (!clusterUrl || clusterUrl.includes(INGEST_PREFIX) || this.isReservedHostname(clusterUrl)) {\n return clusterUrl;\n }\n return clusterUrl.replace(PROTOCOL_SUFFIX, PROTOCOL_SUFFIX + INGEST_PREFIX);\n }\n\n getQueryEndpoint(clusterUrl?: string): string | undefined {\n if (clusterUrl && clusterUrl.includes(INGEST_PREFIX)) {\n return clusterUrl.replace(INGEST_PREFIX, \"\");\n }\n return clusterUrl;\n }\n\n isReservedHostname(clusterUrl: string): boolean {\n try {\n const parsedUrl = new URL(clusterUrl);\n const authority = parsedUrl.hostname;\n if (!authority) {\n return true;\n }\n const is_ip = isIP(authority);\n const is_localhost = authority.includes(\"localhost\");\n return is_localhost || is_ip || authority.toLowerCase() === \"onebox.dev.kusto.windows.net\";\n } catch {\n return false;\n }\n }\n}\n"]}
|
|
@@ -1,24 +1,21 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
// Copyright (c) Microsoft Corporation.
|
|
3
2
|
// Licensed under the MIT License.
|
|
4
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.W3CLogFileMapping = exports.OrcColumnMapping = exports.ParquetColumnMapping = exports.SStreamColumnMapping = exports.ApacheAvroColumnMapping = exports.AvroColumnMapping = exports.JsonColumnMapping = exports.CsvColumnMapping = exports.ColumnMapping = exports.ConstantTransformation = exports.FieldTransformation = void 0;
|
|
6
3
|
/* eslint-disable @typescript-eslint/ban-types -- We legitimately want to use {} as a "any non-nullable type" */
|
|
7
|
-
|
|
8
|
-
var FieldTransformation;
|
|
4
|
+
import { IngestionMappingKind } from "./ingestionProperties.js";
|
|
5
|
+
export var FieldTransformation;
|
|
9
6
|
(function (FieldTransformation) {
|
|
10
7
|
FieldTransformation["PropertyBagArrayToDictionary"] = "PropertyBagArrayToDictionary";
|
|
11
8
|
FieldTransformation["DateTimeFromUnixSeconds"] = "DateTimeFromUnixSeconds";
|
|
12
9
|
FieldTransformation["DateTimeFromUnixMilliseconds"] = "DateTimeFromUnixMilliseconds";
|
|
13
10
|
FieldTransformation["DateTimeFromUnixMicroseconds"] = "DateTimeFromUnixMicroseconds";
|
|
14
11
|
FieldTransformation["DateTimeFromUnixNanoseconds"] = "DateTimeFromUnixNanoseconds";
|
|
15
|
-
})(FieldTransformation || (
|
|
16
|
-
var ConstantTransformation;
|
|
12
|
+
})(FieldTransformation || (FieldTransformation = {}));
|
|
13
|
+
export var ConstantTransformation;
|
|
17
14
|
(function (ConstantTransformation) {
|
|
18
15
|
ConstantTransformation["SourceLocation"] = "SourceLocation";
|
|
19
16
|
ConstantTransformation["SourceLineNumber"] = "SourceLineNumber";
|
|
20
|
-
})(ConstantTransformation || (
|
|
21
|
-
class ColumnMapping {
|
|
17
|
+
})(ConstantTransformation || (ConstantTransformation = {}));
|
|
18
|
+
export class ColumnMapping {
|
|
22
19
|
constructor(columnName, cslDataType, Properties) {
|
|
23
20
|
this.columnName = columnName;
|
|
24
21
|
this.cslDataType = cslDataType;
|
|
@@ -47,8 +44,7 @@ class ColumnMapping {
|
|
|
47
44
|
return result;
|
|
48
45
|
}
|
|
49
46
|
}
|
|
50
|
-
|
|
51
|
-
class CsvColumnMapping extends ColumnMapping {
|
|
47
|
+
export class CsvColumnMapping extends ColumnMapping {
|
|
52
48
|
/**
|
|
53
49
|
* @deprecated Use the factory methods instead.
|
|
54
50
|
*/
|
|
@@ -60,7 +56,7 @@ class CsvColumnMapping extends ColumnMapping {
|
|
|
60
56
|
this.columnName = columnName;
|
|
61
57
|
this.cslDataType = cslDataType;
|
|
62
58
|
this.ordinal = ordinal;
|
|
63
|
-
this.mappingKind =
|
|
59
|
+
this.mappingKind = IngestionMappingKind.CSV;
|
|
64
60
|
}
|
|
65
61
|
static withOrdinal(columnName, ordinal, cslDataType) {
|
|
66
62
|
return new CsvColumnMapping(columnName, cslDataType, ordinal.toString());
|
|
@@ -69,8 +65,7 @@ class CsvColumnMapping extends ColumnMapping {
|
|
|
69
65
|
return new CsvColumnMapping(columnName, cslDataType, undefined, constantValue);
|
|
70
66
|
}
|
|
71
67
|
}
|
|
72
|
-
|
|
73
|
-
class JsonColumnMapping extends ColumnMapping {
|
|
68
|
+
export class JsonColumnMapping extends ColumnMapping {
|
|
74
69
|
/**
|
|
75
70
|
* @deprecated Use the factory methods instead.
|
|
76
71
|
*/
|
|
@@ -82,7 +77,7 @@ class JsonColumnMapping extends ColumnMapping {
|
|
|
82
77
|
});
|
|
83
78
|
this.columnName = columnName;
|
|
84
79
|
this.jsonPath = jsonPath;
|
|
85
|
-
this.mappingKind =
|
|
80
|
+
this.mappingKind = IngestionMappingKind.JSON;
|
|
86
81
|
}
|
|
87
82
|
static withPath(columnName, path, cslDataType, transform) {
|
|
88
83
|
return new JsonColumnMapping(columnName, path, cslDataType, undefined, transform);
|
|
@@ -94,8 +89,7 @@ class JsonColumnMapping extends ColumnMapping {
|
|
|
94
89
|
return new JsonColumnMapping(columnName, undefined, cslDataType, undefined, transform);
|
|
95
90
|
}
|
|
96
91
|
}
|
|
97
|
-
|
|
98
|
-
class AvroColumnMapping extends ColumnMapping {
|
|
92
|
+
export class AvroColumnMapping extends ColumnMapping {
|
|
99
93
|
constructor(columnName, cslDataType, path, field, constantValue, transform) {
|
|
100
94
|
super(columnName, cslDataType !== null && cslDataType !== void 0 ? cslDataType : undefined, {
|
|
101
95
|
Path: path,
|
|
@@ -104,7 +98,7 @@ class AvroColumnMapping extends ColumnMapping {
|
|
|
104
98
|
Transform: transform,
|
|
105
99
|
});
|
|
106
100
|
this.columnName = columnName;
|
|
107
|
-
this.mappingKind =
|
|
101
|
+
this.mappingKind = IngestionMappingKind.AVRO;
|
|
108
102
|
}
|
|
109
103
|
static withPath(columnName, path, cslDataType, transform) {
|
|
110
104
|
return new AvroColumnMapping(columnName, cslDataType, path, undefined, undefined, transform);
|
|
@@ -119,8 +113,7 @@ class AvroColumnMapping extends ColumnMapping {
|
|
|
119
113
|
return new AvroColumnMapping(columnName, cslDataType, undefined, undefined, undefined, transform);
|
|
120
114
|
}
|
|
121
115
|
}
|
|
122
|
-
|
|
123
|
-
class ApacheAvroColumnMapping extends ColumnMapping {
|
|
116
|
+
export class ApacheAvroColumnMapping extends ColumnMapping {
|
|
124
117
|
constructor(columnName, cslDataType, path, field, constantValue, transform) {
|
|
125
118
|
super(columnName, cslDataType !== null && cslDataType !== void 0 ? cslDataType : undefined, {
|
|
126
119
|
Path: path,
|
|
@@ -129,7 +122,7 @@ class ApacheAvroColumnMapping extends ColumnMapping {
|
|
|
129
122
|
Transform: transform,
|
|
130
123
|
});
|
|
131
124
|
this.columnName = columnName;
|
|
132
|
-
this.mappingKind =
|
|
125
|
+
this.mappingKind = IngestionMappingKind.APACHEAVRO;
|
|
133
126
|
}
|
|
134
127
|
static withPath(columnName, path, cslDataType, transform) {
|
|
135
128
|
return new ApacheAvroColumnMapping(columnName, cslDataType, path, undefined, undefined, transform);
|
|
@@ -144,8 +137,7 @@ class ApacheAvroColumnMapping extends ColumnMapping {
|
|
|
144
137
|
return new ApacheAvroColumnMapping(columnName, cslDataType, undefined, undefined, undefined, transform);
|
|
145
138
|
}
|
|
146
139
|
}
|
|
147
|
-
|
|
148
|
-
class SStreamColumnMapping extends ColumnMapping {
|
|
140
|
+
export class SStreamColumnMapping extends ColumnMapping {
|
|
149
141
|
constructor(columnName, cslDataType, path, field, constantValue, transform) {
|
|
150
142
|
super(columnName, cslDataType !== null && cslDataType !== void 0 ? cslDataType : undefined, {
|
|
151
143
|
Path: path,
|
|
@@ -154,7 +146,7 @@ class SStreamColumnMapping extends ColumnMapping {
|
|
|
154
146
|
Transform: transform,
|
|
155
147
|
});
|
|
156
148
|
this.columnName = columnName;
|
|
157
|
-
this.mappingKind =
|
|
149
|
+
this.mappingKind = IngestionMappingKind.SSTREAM;
|
|
158
150
|
}
|
|
159
151
|
static withPath(columnName, path, cslDataType, transform) {
|
|
160
152
|
return new SStreamColumnMapping(columnName, cslDataType, path, undefined, undefined, transform);
|
|
@@ -169,8 +161,7 @@ class SStreamColumnMapping extends ColumnMapping {
|
|
|
169
161
|
return new SStreamColumnMapping(columnName, cslDataType, undefined, undefined, undefined, transform);
|
|
170
162
|
}
|
|
171
163
|
}
|
|
172
|
-
|
|
173
|
-
class ParquetColumnMapping extends ColumnMapping {
|
|
164
|
+
export class ParquetColumnMapping extends ColumnMapping {
|
|
174
165
|
constructor(columnName, cslDataType, path, field, constantValue, transform) {
|
|
175
166
|
super(columnName, cslDataType !== null && cslDataType !== void 0 ? cslDataType : undefined, {
|
|
176
167
|
Path: path,
|
|
@@ -179,7 +170,7 @@ class ParquetColumnMapping extends ColumnMapping {
|
|
|
179
170
|
Transform: transform,
|
|
180
171
|
});
|
|
181
172
|
this.columnName = columnName;
|
|
182
|
-
this.mappingKind =
|
|
173
|
+
this.mappingKind = IngestionMappingKind.PARQUET;
|
|
183
174
|
}
|
|
184
175
|
static withPath(columnName, path, cslDataType, transform) {
|
|
185
176
|
return new ParquetColumnMapping(columnName, cslDataType, path, undefined, undefined, transform);
|
|
@@ -194,8 +185,7 @@ class ParquetColumnMapping extends ColumnMapping {
|
|
|
194
185
|
return new ParquetColumnMapping(columnName, cslDataType, undefined, undefined, undefined, transform);
|
|
195
186
|
}
|
|
196
187
|
}
|
|
197
|
-
|
|
198
|
-
class OrcColumnMapping extends ColumnMapping {
|
|
188
|
+
export class OrcColumnMapping extends ColumnMapping {
|
|
199
189
|
constructor(columnName, cslDataType, path, field, constantValue, transform) {
|
|
200
190
|
super(columnName, cslDataType !== null && cslDataType !== void 0 ? cslDataType : undefined, {
|
|
201
191
|
Path: path,
|
|
@@ -204,7 +194,7 @@ class OrcColumnMapping extends ColumnMapping {
|
|
|
204
194
|
Transform: transform,
|
|
205
195
|
});
|
|
206
196
|
this.columnName = columnName;
|
|
207
|
-
this.mappingKind =
|
|
197
|
+
this.mappingKind = IngestionMappingKind.ORC;
|
|
208
198
|
}
|
|
209
199
|
static withPath(columnName, path, cslDataType, transform) {
|
|
210
200
|
return new OrcColumnMapping(columnName, cslDataType, path, undefined, undefined, transform);
|
|
@@ -219,12 +209,11 @@ class OrcColumnMapping extends ColumnMapping {
|
|
|
219
209
|
return new OrcColumnMapping(columnName, cslDataType, undefined, undefined, undefined, transform);
|
|
220
210
|
}
|
|
221
211
|
}
|
|
222
|
-
|
|
223
|
-
class W3CLogFileMapping extends ColumnMapping {
|
|
212
|
+
export class W3CLogFileMapping extends ColumnMapping {
|
|
224
213
|
constructor(columnName, cslDataType, field, constantValue, transform) {
|
|
225
214
|
super(columnName, cslDataType !== null && cslDataType !== void 0 ? cslDataType : undefined, { Field: field, ConstValue: constantValue, Transform: transform });
|
|
226
215
|
this.columnName = columnName;
|
|
227
|
-
this.mappingKind =
|
|
216
|
+
this.mappingKind = IngestionMappingKind.W3CLOGFILE;
|
|
228
217
|
}
|
|
229
218
|
static withField(columnName, field, cslDataType, transform) {
|
|
230
219
|
return new W3CLogFileMapping(columnName, cslDataType, field, undefined, transform);
|
|
@@ -236,5 +225,4 @@ class W3CLogFileMapping extends ColumnMapping {
|
|
|
236
225
|
return new W3CLogFileMapping(columnName, cslDataType, undefined, undefined, transform);
|
|
237
226
|
}
|
|
238
227
|
}
|
|
239
|
-
exports.W3CLogFileMapping = W3CLogFileMapping;
|
|
240
228
|
//# sourceMappingURL=columnMappings.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"columnMappings.js","sourceRoot":"","sources":["../../src/columnMappings.ts"],"names":[],"mappings":";AAAA,uCAAuC;AACvC,kCAAkC;;;AAElC,gHAAgH;AAEhH,+DAA6D;AAE7D,IAAY,mBAMX;AAND,WAAY,mBAAmB;IAC3B,oFAA6D,CAAA;IAC7D,0EAAmD,CAAA;IACnD,oFAA6D,CAAA;IAC7D,oFAA6D,CAAA;IAC7D,kFAA2D,CAAA;AAC/D,CAAC,EANW,mBAAmB,mCAAnB,mBAAmB,QAM9B;AAED,IAAY,sBAGX;AAHD,WAAY,sBAAsB;IAC9B,2DAAiC,CAAA;IACjC,+DAAqC,CAAA;AACzC,CAAC,EAHW,sBAAsB,sCAAtB,sBAAsB,QAGjC;AAsBD,MAAsB,aAAa;IAC/B,YAA+B,UAAkB,EAAW,WAAoB,EAAW,UAA8B;QAA1F,eAAU,GAAV,UAAU,CAAQ;QAAW,gBAAW,GAAX,WAAW,CAAS;QAAW,eAAU,GAAV,UAAU,CAAoB;IAAG,CAAC;IAItH,YAAY;QACf,MAAM,MAAM,GAAqB;YAC7B,MAAM,EAAE,IAAI,CAAC,UAAU;SAC1B,CAAC;QACF,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC;SACtC;QAED,IAAI,IAAI,CAAC,UAAU,EAAE;YACjB,MAAM,CAAC,UAAU,GAAG,EAAE,CAAC;YACvB,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,UAAU,EAAE;gBAC/B,IAAI,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,EAAE;oBAC5D,MAAM,QAAQ,GAAG,GAA8B,CAAC;oBAChD,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;oBAE3C,iEAAiE;oBACjE,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,IAAI,EAAE;wBAC7C,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAC;qBACrD;iBACJ;aACJ;SACJ;QACD,OAAO,MAAM,CAAC;IAClB,CAAC;CACJ;AA7BD,sCA6BC;AAED,MAAa,gBAAiB,SAAQ,aAAa;IAC/C;;OAEG;IACH,YAA+B,UAAkB,EAAW,WAAoB,EAAW,OAAgB,EAAE,aAAkB;QAC3H,KAAK,CAAC,UAAU,EAAE,WAAW,EAAE;YAC3B,OAAO,EAAE,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;YAClE,UAAU,EAAE,aAAa;SAC5B,CAAC,CAAC;QAJwB,eAAU,GAAV,UAAU,CAAQ;QAAW,gBAAW,GAAX,WAAW,CAAS;QAAW,YAAO,GAAP,OAAO,CAAS;QAe3G,gBAAW,GAAG,0CAAoB,CAAC,GAAG,CAAC;IAVvC,CAAC;IAEM,MAAM,CAAC,WAAW,CAAC,UAAkB,EAAE,OAAe,EAAE,WAAoB;QAC/E,OAAO,IAAI,gBAAgB,CAAC,UAAU,EAAE,WAAW,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC7E,CAAC;IAEM,MAAM,CAAC,iBAAiB,CAAC,UAAkB,EAAE,aAAiB,EAAE,WAAoB;QACvF,OAAO,IAAI,gBAAgB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;IACnF,CAAC;CAGJ;AApBD,4CAoBC;AAED,MAAa,iBAAkB,SAAQ,aAAa;IAChD;;OAEG;IACH,YAAqB,UAAkB,EAAW,QAAiB,EAAE,cAA6B,IAAI,EAAE,aAAkB,EAAE,SAA0B;QAClJ,KAAK,CAAC,UAAU,EAAE,WAAW,aAAX,WAAW,cAAX,WAAW,GAAI,SAAS,EAAE;YACxC,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE,aAAa;YACzB,SAAS,EAAE,SAAS;SACvB,CAAC,CAAC;QALc,eAAU,GAAV,UAAU,CAAQ;QAAW,aAAQ,GAAR,QAAQ,CAAS;QAoBnE,gBAAW,GAAG,0CAAoB,CAAC,IAAI,CAAC;IAdxC,CAAC;IAEM,MAAM,CAAC,QAAQ,CAAC,UAAkB,EAAE,IAAY,EAAE,WAAoB,EAAE,SAA+B;QAC1G,OAAO,IAAI,iBAAiB,CAAC,UAAU,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACtF,CAAC;IAEM,MAAM,CAAC,iBAAiB,CAAC,UAAkB,EAAE,aAAiB,EAAE,WAAoB;QACvF,OAAO,IAAI,iBAAiB,CAAC,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC;IACpF,CAAC;IAEM,MAAM,CAAC,aAAa,CAAC,UAAkB,EAAE,SAAiC,EAAE,WAAoB;QACnG,OAAO,IAAI,iBAAiB,CAAC,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAC3F,CAAC;CAGJ;AAzBD,8CAyBC;AAED,MAAa,iBAAkB,SAAQ,aAAa;IAChD,YAA6B,UAAkB,EAAE,WAAoB,EAAE,IAAa,EAAE,KAAc,EAAE,aAAkB,EAAE,SAA0B;QAChJ,KAAK,CAAC,UAAU,EAAE,WAAW,aAAX,WAAW,cAAX,WAAW,GAAI,SAAS,EAAE;YACxC,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,KAAK;YACZ,UAAU,EAAE,aAAa;YACzB,SAAS,EAAE,SAAS;SACvB,CAAC,CAAC;QANsB,eAAU,GAAV,UAAU,CAAQ;QAyB/C,gBAAW,GAAG,0CAAoB,CAAC,IAAI,CAAC;IAlBxC,CAAC;IAEM,MAAM,CAAC,QAAQ,CAAC,UAAkB,EAAE,IAAY,EAAE,WAAoB,EAAE,SAA+B;QAC1G,OAAO,IAAI,iBAAiB,CAAC,UAAU,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACjG,CAAC;IAEM,MAAM,CAAC,SAAS,CAAC,UAAkB,EAAE,KAAa,EAAE,WAAoB,EAAE,SAA+B;QAC5G,OAAO,IAAI,iBAAiB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAClG,CAAC;IAEM,MAAM,CAAC,iBAAiB,CAAC,UAAkB,EAAE,aAAiB,EAAE,WAAoB;QACvF,OAAO,IAAI,iBAAiB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;IAC/F,CAAC;IAEM,MAAM,CAAC,aAAa,CAAC,UAAkB,EAAE,SAAiC,EAAE,WAAoB;QACnG,OAAO,IAAI,iBAAiB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACtG,CAAC;CAGJ;AA3BD,8CA2BC;AAED,MAAa,uBAAwB,SAAQ,aAAa;IACtD,YAA6B,UAAkB,EAAE,WAAoB,EAAE,IAAa,EAAE,KAAc,EAAE,aAAkB,EAAE,SAA0B;QAChJ,KAAK,CAAC,UAAU,EAAE,WAAW,aAAX,WAAW,cAAX,WAAW,GAAI,SAAS,EAAE;YACxC,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,KAAK;YACZ,UAAU,EAAE,aAAa;YACzB,SAAS,EAAE,SAAS;SACvB,CAAC,CAAC;QANsB,eAAU,GAAV,UAAU,CAAQ;QAyB/C,gBAAW,GAAG,0CAAoB,CAAC,UAAU,CAAC;IAlB9C,CAAC;IAEM,MAAM,CAAC,QAAQ,CAAC,UAAkB,EAAE,IAAY,EAAE,WAAoB,EAAE,SAA+B;QAC1G,OAAO,IAAI,uBAAuB,CAAC,UAAU,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACvG,CAAC;IAEM,MAAM,CAAC,SAAS,CAAC,UAAkB,EAAE,KAAa,EAAE,WAAoB,EAAE,SAA+B;QAC5G,OAAO,IAAI,uBAAuB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACxG,CAAC;IAEM,MAAM,CAAC,iBAAiB,CAAC,UAAkB,EAAE,aAAiB,EAAE,WAAoB;QACvF,OAAO,IAAI,uBAAuB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;IACrG,CAAC;IAEM,MAAM,CAAC,aAAa,CAAC,UAAkB,EAAE,SAAiC,EAAE,WAAoB;QACnG,OAAO,IAAI,uBAAuB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAC5G,CAAC;CAGJ;AA3BD,0DA2BC;AAED,MAAa,oBAAqB,SAAQ,aAAa;IACnD,YAA6B,UAAkB,EAAE,WAAoB,EAAE,IAAa,EAAE,KAAc,EAAE,aAAkB,EAAE,SAA0B;QAChJ,KAAK,CAAC,UAAU,EAAE,WAAW,aAAX,WAAW,cAAX,WAAW,GAAI,SAAS,EAAE;YACxC,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,KAAK;YACZ,UAAU,EAAE,aAAa;YACzB,SAAS,EAAE,SAAS;SACvB,CAAC,CAAC;QANsB,eAAU,GAAV,UAAU,CAAQ;QAyB/C,gBAAW,GAAG,0CAAoB,CAAC,OAAO,CAAC;IAlB3C,CAAC;IAEM,MAAM,CAAC,QAAQ,CAAC,UAAkB,EAAE,IAAY,EAAE,WAAoB,EAAE,SAA+B;QAC1G,OAAO,IAAI,oBAAoB,CAAC,UAAU,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACpG,CAAC;IAEM,MAAM,CAAC,SAAS,CAAC,UAAkB,EAAE,KAAa,EAAE,WAAoB,EAAE,SAA+B;QAC5G,OAAO,IAAI,oBAAoB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACrG,CAAC;IAEM,MAAM,CAAC,iBAAiB,CAAC,UAAkB,EAAE,aAAiB,EAAE,WAAoB;QACvF,OAAO,IAAI,oBAAoB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;IAClG,CAAC;IAEM,MAAM,CAAC,aAAa,CAAC,UAAkB,EAAE,SAAiC,EAAE,WAAoB;QACnG,OAAO,IAAI,oBAAoB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACzG,CAAC;CAGJ;AA3BD,oDA2BC;AAED,MAAa,oBAAqB,SAAQ,aAAa;IACnD,YAA6B,UAAkB,EAAE,WAAoB,EAAE,IAAa,EAAE,KAAc,EAAE,aAAkB,EAAE,SAA0B;QAChJ,KAAK,CAAC,UAAU,EAAE,WAAW,aAAX,WAAW,cAAX,WAAW,GAAI,SAAS,EAAE;YACxC,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,KAAK;YACZ,UAAU,EAAE,aAAa;YACzB,SAAS,EAAE,SAAS;SACvB,CAAC,CAAC;QANsB,eAAU,GAAV,UAAU,CAAQ;QAyB/C,gBAAW,GAAG,0CAAoB,CAAC,OAAO,CAAC;IAlB3C,CAAC;IAEM,MAAM,CAAC,QAAQ,CAAC,UAAkB,EAAE,IAAY,EAAE,WAAoB,EAAE,SAA+B;QAC1G,OAAO,IAAI,oBAAoB,CAAC,UAAU,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACpG,CAAC;IAEM,MAAM,CAAC,SAAS,CAAC,UAAkB,EAAE,KAAa,EAAE,WAAoB,EAAE,SAA+B;QAC5G,OAAO,IAAI,oBAAoB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACrG,CAAC;IAEM,MAAM,CAAC,iBAAiB,CAAC,UAAkB,EAAE,aAAiB,EAAE,WAAoB;QACvF,OAAO,IAAI,oBAAoB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;IAClG,CAAC;IAEM,MAAM,CAAC,aAAa,CAAC,UAAkB,EAAE,SAAiC,EAAE,WAAoB;QACnG,OAAO,IAAI,oBAAoB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACzG,CAAC;CAGJ;AA3BD,oDA2BC;AAED,MAAa,gBAAiB,SAAQ,aAAa;IAC/C,YAA6B,UAAkB,EAAE,WAAoB,EAAE,IAAa,EAAE,KAAc,EAAE,aAAkB,EAAE,SAA0B;QAChJ,KAAK,CAAC,UAAU,EAAE,WAAW,aAAX,WAAW,cAAX,WAAW,GAAI,SAAS,EAAE;YACxC,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,KAAK;YACZ,UAAU,EAAE,aAAa;YACzB,SAAS,EAAE,SAAS;SACvB,CAAC,CAAC;QANsB,eAAU,GAAV,UAAU,CAAQ;QAyB/C,gBAAW,GAAG,0CAAoB,CAAC,GAAG,CAAC;IAlBvC,CAAC;IAEM,MAAM,CAAC,QAAQ,CAAC,UAAkB,EAAE,IAAY,EAAE,WAAoB,EAAE,SAA+B;QAC1G,OAAO,IAAI,gBAAgB,CAAC,UAAU,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAChG,CAAC;IAEM,MAAM,CAAC,SAAS,CAAC,UAAkB,EAAE,KAAa,EAAE,WAAoB,EAAE,SAA+B;QAC5G,OAAO,IAAI,gBAAgB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACjG,CAAC;IAEM,MAAM,CAAC,iBAAiB,CAAC,UAAkB,EAAE,aAAiB,EAAE,WAAoB;QACvF,OAAO,IAAI,gBAAgB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;IAC9F,CAAC;IAEM,MAAM,CAAC,aAAa,CAAC,UAAkB,EAAE,SAAiC,EAAE,WAAoB;QACnG,OAAO,IAAI,gBAAgB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACrG,CAAC;CAGJ;AA3BD,4CA2BC;AAED,MAAa,iBAAkB,SAAQ,aAAa;IAChD,YAA6B,UAAkB,EAAE,WAAoB,EAAE,KAAc,EAAE,aAAkB,EAAE,SAA0B;QACjI,KAAK,CAAC,UAAU,EAAE,WAAW,aAAX,WAAW,cAAX,WAAW,GAAI,SAAS,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;QADtF,eAAU,GAAV,UAAU,CAAQ;QAgB/C,gBAAW,GAAG,0CAAoB,CAAC,UAAU,CAAC;IAd9C,CAAC;IAEM,MAAM,CAAC,SAAS,CAAC,UAAkB,EAAE,KAAa,EAAE,WAAoB,EAAE,SAA+B;QAC5G,OAAO,IAAI,iBAAiB,CAAC,UAAU,EAAE,WAAW,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACvF,CAAC;IAEM,MAAM,CAAC,iBAAiB,CAAC,UAAkB,EAAE,aAAiB,EAAE,WAAoB;QACvF,OAAO,IAAI,iBAAiB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;IACpF,CAAC;IAEM,MAAM,CAAC,aAAa,CAAC,UAAkB,EAAE,SAAiC,EAAE,WAAoB;QACnG,OAAO,IAAI,iBAAiB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAC3F,CAAC;CAGJ;AAlBD,8CAkBC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\n/* eslint-disable @typescript-eslint/ban-types -- We legitimately want to use {} as a \"any non-nullable type\" */\n\nimport { IngestionMappingKind } from \"./ingestionProperties\";\n\nexport enum FieldTransformation {\n PropertyBagArrayToDictionary = \"PropertyBagArrayToDictionary\",\n DateTimeFromUnixSeconds = \"DateTimeFromUnixSeconds\",\n DateTimeFromUnixMilliseconds = \"DateTimeFromUnixMilliseconds\",\n DateTimeFromUnixMicroseconds = \"DateTimeFromUnixMicroseconds\",\n DateTimeFromUnixNanoseconds = \"DateTimeFromUnixNanoseconds\",\n}\n\nexport enum ConstantTransformation {\n SourceLocation = \"SourceLocation\",\n SourceLineNumber = \"SourceLineNumber\",\n}\n\nexport type Transformation = FieldTransformation | ConstantTransformation;\n\ninterface MappingProperties {\n Field?: string;\n Path?: string;\n Ordinal?: number;\n ConstValue?: {};\n Transform?: Transformation;\n}\n\ntype MappingPropertiesStrings = {\n [key in keyof MappingProperties]: string;\n};\n\ninterface ApiColumnMapping {\n Column: string;\n DataType?: string;\n Properties?: MappingPropertiesStrings;\n}\n\nexport abstract class ColumnMapping {\n protected constructor(readonly columnName: string, readonly cslDataType?: string, readonly Properties?: MappingProperties) {}\n\n public abstract mappingKind: IngestionMappingKind;\n\n public toApiMapping(): ApiColumnMapping {\n const result: ApiColumnMapping = {\n Column: this.columnName,\n };\n if (this.cslDataType) {\n result.DataType = this.cslDataType;\n }\n\n if (this.Properties) {\n result.Properties = {};\n for (const key in this.Properties) {\n if (Object.prototype.hasOwnProperty.call(this.Properties, key)) {\n const typedKey = key as keyof MappingProperties;\n const property = this.Properties[typedKey];\n\n // We don't do if (property) because we '0' is a legitimate value\n if (property !== undefined && property !== null) {\n result.Properties[typedKey] = property.toString();\n }\n }\n }\n }\n return result;\n }\n}\n\nexport class CsvColumnMapping extends ColumnMapping {\n /**\n * @deprecated Use the factory methods instead.\n */\n protected constructor(readonly columnName: string, readonly cslDataType?: string, readonly ordinal?: string, constantValue?: {}) {\n super(columnName, cslDataType, {\n Ordinal: ordinal === undefined ? undefined : parseInt(ordinal, 10),\n ConstValue: constantValue,\n });\n }\n\n public static withOrdinal(columnName: string, ordinal: number, cslDataType?: string): CsvColumnMapping {\n return new CsvColumnMapping(columnName, cslDataType, ordinal.toString());\n }\n\n public static withConstantValue(columnName: string, constantValue: {}, cslDataType?: string): CsvColumnMapping {\n return new CsvColumnMapping(columnName, cslDataType, undefined, constantValue);\n }\n\n mappingKind = IngestionMappingKind.CSV;\n}\n\nexport class JsonColumnMapping extends ColumnMapping {\n /**\n * @deprecated Use the factory methods instead.\n */\n constructor(readonly columnName: string, readonly jsonPath?: string, cslDataType: string | null = null, constantValue?: {}, transform?: Transformation) {\n super(columnName, cslDataType ?? undefined, {\n Path: jsonPath,\n ConstValue: constantValue,\n Transform: transform,\n });\n }\n\n public static withPath(columnName: string, path: string, cslDataType?: string, transform?: FieldTransformation): JsonColumnMapping {\n return new JsonColumnMapping(columnName, path, cslDataType, undefined, transform);\n }\n\n public static withConstantValue(columnName: string, constantValue: {}, cslDataType?: string): JsonColumnMapping {\n return new JsonColumnMapping(columnName, undefined, cslDataType, constantValue);\n }\n\n public static withTransform(columnName: string, transform: ConstantTransformation, cslDataType?: string): JsonColumnMapping {\n return new JsonColumnMapping(columnName, undefined, cslDataType, undefined, transform);\n }\n\n mappingKind = IngestionMappingKind.JSON;\n}\n\nexport class AvroColumnMapping extends ColumnMapping {\n private constructor(readonly columnName: string, cslDataType?: string, path?: string, field?: string, constantValue?: {}, transform?: Transformation) {\n super(columnName, cslDataType ?? undefined, {\n Path: path,\n Field: field,\n ConstValue: constantValue,\n Transform: transform,\n });\n }\n\n public static withPath(columnName: string, path: string, cslDataType?: string, transform?: FieldTransformation): AvroColumnMapping {\n return new AvroColumnMapping(columnName, cslDataType, path, undefined, undefined, transform);\n }\n\n public static withField(columnName: string, field: string, cslDataType?: string, transform?: FieldTransformation): AvroColumnMapping {\n return new AvroColumnMapping(columnName, cslDataType, undefined, field, undefined, transform);\n }\n\n public static withConstantValue(columnName: string, constantValue: {}, cslDataType?: string): AvroColumnMapping {\n return new AvroColumnMapping(columnName, cslDataType, undefined, undefined, constantValue);\n }\n\n public static withTransform(columnName: string, transform: ConstantTransformation, cslDataType?: string): AvroColumnMapping {\n return new AvroColumnMapping(columnName, cslDataType, undefined, undefined, undefined, transform);\n }\n\n mappingKind = IngestionMappingKind.AVRO;\n}\n\nexport class ApacheAvroColumnMapping extends ColumnMapping {\n private constructor(readonly columnName: string, cslDataType?: string, path?: string, field?: string, constantValue?: {}, transform?: Transformation) {\n super(columnName, cslDataType ?? undefined, {\n Path: path,\n Field: field,\n ConstValue: constantValue,\n Transform: transform,\n });\n }\n\n public static withPath(columnName: string, path: string, cslDataType?: string, transform?: FieldTransformation): ApacheAvroColumnMapping {\n return new ApacheAvroColumnMapping(columnName, cslDataType, path, undefined, undefined, transform);\n }\n\n public static withField(columnName: string, field: string, cslDataType?: string, transform?: FieldTransformation): ApacheAvroColumnMapping {\n return new ApacheAvroColumnMapping(columnName, cslDataType, undefined, field, undefined, transform);\n }\n\n public static withConstantValue(columnName: string, constantValue: {}, cslDataType?: string): ApacheAvroColumnMapping {\n return new ApacheAvroColumnMapping(columnName, cslDataType, undefined, undefined, constantValue);\n }\n\n public static withTransform(columnName: string, transform: ConstantTransformation, cslDataType?: string): ApacheAvroColumnMapping {\n return new ApacheAvroColumnMapping(columnName, cslDataType, undefined, undefined, undefined, transform);\n }\n\n mappingKind = IngestionMappingKind.APACHEAVRO;\n}\n\nexport class SStreamColumnMapping extends ColumnMapping {\n private constructor(readonly columnName: string, cslDataType?: string, path?: string, field?: string, constantValue?: {}, transform?: Transformation) {\n super(columnName, cslDataType ?? undefined, {\n Path: path,\n Field: field,\n ConstValue: constantValue,\n Transform: transform,\n });\n }\n\n public static withPath(columnName: string, path: string, cslDataType?: string, transform?: FieldTransformation): SStreamColumnMapping {\n return new SStreamColumnMapping(columnName, cslDataType, path, undefined, undefined, transform);\n }\n\n public static withField(columnName: string, field: string, cslDataType?: string, transform?: FieldTransformation): SStreamColumnMapping {\n return new SStreamColumnMapping(columnName, cslDataType, undefined, field, undefined, transform);\n }\n\n public static withConstantValue(columnName: string, constantValue: {}, cslDataType?: string): SStreamColumnMapping {\n return new SStreamColumnMapping(columnName, cslDataType, undefined, undefined, constantValue);\n }\n\n public static withTransform(columnName: string, transform: ConstantTransformation, cslDataType?: string): SStreamColumnMapping {\n return new SStreamColumnMapping(columnName, cslDataType, undefined, undefined, undefined, transform);\n }\n\n mappingKind = IngestionMappingKind.SSTREAM;\n}\n\nexport class ParquetColumnMapping extends ColumnMapping {\n private constructor(readonly columnName: string, cslDataType?: string, path?: string, field?: string, constantValue?: {}, transform?: Transformation) {\n super(columnName, cslDataType ?? undefined, {\n Path: path,\n Field: field,\n ConstValue: constantValue,\n Transform: transform,\n });\n }\n\n public static withPath(columnName: string, path: string, cslDataType?: string, transform?: FieldTransformation): ParquetColumnMapping {\n return new ParquetColumnMapping(columnName, cslDataType, path, undefined, undefined, transform);\n }\n\n public static withField(columnName: string, field: string, cslDataType?: string, transform?: FieldTransformation): ParquetColumnMapping {\n return new ParquetColumnMapping(columnName, cslDataType, undefined, field, undefined, transform);\n }\n\n public static withConstantValue(columnName: string, constantValue: {}, cslDataType?: string): ParquetColumnMapping {\n return new ParquetColumnMapping(columnName, cslDataType, undefined, undefined, constantValue);\n }\n\n public static withTransform(columnName: string, transform: ConstantTransformation, cslDataType?: string): ParquetColumnMapping {\n return new ParquetColumnMapping(columnName, cslDataType, undefined, undefined, undefined, transform);\n }\n\n mappingKind = IngestionMappingKind.PARQUET;\n}\n\nexport class OrcColumnMapping extends ColumnMapping {\n private constructor(readonly columnName: string, cslDataType?: string, path?: string, field?: string, constantValue?: {}, transform?: Transformation) {\n super(columnName, cslDataType ?? undefined, {\n Path: path,\n Field: field,\n ConstValue: constantValue,\n Transform: transform,\n });\n }\n\n public static withPath(columnName: string, path: string, cslDataType?: string, transform?: FieldTransformation): OrcColumnMapping {\n return new OrcColumnMapping(columnName, cslDataType, path, undefined, undefined, transform);\n }\n\n public static withField(columnName: string, field: string, cslDataType?: string, transform?: FieldTransformation): OrcColumnMapping {\n return new OrcColumnMapping(columnName, cslDataType, undefined, field, undefined, transform);\n }\n\n public static withConstantValue(columnName: string, constantValue: {}, cslDataType?: string): OrcColumnMapping {\n return new OrcColumnMapping(columnName, cslDataType, undefined, undefined, constantValue);\n }\n\n public static withTransform(columnName: string, transform: ConstantTransformation, cslDataType?: string): OrcColumnMapping {\n return new OrcColumnMapping(columnName, cslDataType, undefined, undefined, undefined, transform);\n }\n\n mappingKind = IngestionMappingKind.ORC;\n}\n\nexport class W3CLogFileMapping extends ColumnMapping {\n private constructor(readonly columnName: string, cslDataType?: string, field?: string, constantValue?: {}, transform?: Transformation) {\n super(columnName, cslDataType ?? undefined, { Field: field, ConstValue: constantValue, Transform: transform });\n }\n\n public static withField(columnName: string, field: string, cslDataType?: string, transform?: FieldTransformation): W3CLogFileMapping {\n return new W3CLogFileMapping(columnName, cslDataType, field, undefined, transform);\n }\n\n public static withConstantValue(columnName: string, constantValue: {}, cslDataType?: string): W3CLogFileMapping {\n return new W3CLogFileMapping(columnName, cslDataType, undefined, constantValue);\n }\n\n public static withTransform(columnName: string, transform: ConstantTransformation, cslDataType?: string): W3CLogFileMapping {\n return new W3CLogFileMapping(columnName, cslDataType, undefined, undefined, transform);\n }\n\n mappingKind = IngestionMappingKind.W3CLOGFILE;\n}\n"]}
|
|
1
|
+
{"version":3,"file":"columnMappings.js","sourceRoot":"","sources":["../../src/columnMappings.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAElC,gHAAgH;AAEhH,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAEhE,MAAM,CAAN,IAAY,mBAMX;AAND,WAAY,mBAAmB;IAC3B,oFAA6D,CAAA;IAC7D,0EAAmD,CAAA;IACnD,oFAA6D,CAAA;IAC7D,oFAA6D,CAAA;IAC7D,kFAA2D,CAAA;AAC/D,CAAC,EANW,mBAAmB,KAAnB,mBAAmB,QAM9B;AAED,MAAM,CAAN,IAAY,sBAGX;AAHD,WAAY,sBAAsB;IAC9B,2DAAiC,CAAA;IACjC,+DAAqC,CAAA;AACzC,CAAC,EAHW,sBAAsB,KAAtB,sBAAsB,QAGjC;AAsBD,MAAM,OAAgB,aAAa;IAC/B,YACa,UAAkB,EAClB,WAAoB,EACpB,UAA8B;QAF9B,eAAU,GAAV,UAAU,CAAQ;QAClB,gBAAW,GAAX,WAAW,CAAS;QACpB,eAAU,GAAV,UAAU,CAAoB;IACxC,CAAC;IAIG,YAAY;QACf,MAAM,MAAM,GAAqB;YAC7B,MAAM,EAAE,IAAI,CAAC,UAAU;SAC1B,CAAC;QACF,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC;QACvC,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAClB,MAAM,CAAC,UAAU,GAAG,EAAE,CAAC;YACvB,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChC,IAAI,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,EAAE,CAAC;oBAC7D,MAAM,QAAQ,GAAG,GAA8B,CAAC;oBAChD,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;oBAE3C,iEAAiE;oBACjE,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;wBAC9C,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAC;oBACtD,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;QACD,OAAO,MAAM,CAAC;IAClB,CAAC;CACJ;AAED,MAAM,OAAO,gBAAiB,SAAQ,aAAa;IAC/C;;OAEG;IACH,YACa,UAAkB,EAClB,WAAoB,EACpB,OAAgB,EACzB,aAAkB;QAElB,KAAK,CAAC,UAAU,EAAE,WAAW,EAAE;YAC3B,OAAO,EAAE,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;YAClE,UAAU,EAAE,aAAa;SAC5B,CAAC,CAAC;QARM,eAAU,GAAV,UAAU,CAAQ;QAClB,gBAAW,GAAX,WAAW,CAAS;QACpB,YAAO,GAAP,OAAO,CAAS;QAiB7B,gBAAW,GAAG,oBAAoB,CAAC,GAAG,CAAC;IAVvC,CAAC;IAEM,MAAM,CAAC,WAAW,CAAC,UAAkB,EAAE,OAAe,EAAE,WAAoB;QAC/E,OAAO,IAAI,gBAAgB,CAAC,UAAU,EAAE,WAAW,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC7E,CAAC;IAEM,MAAM,CAAC,iBAAiB,CAAC,UAAkB,EAAE,aAAiB,EAAE,WAAoB;QACvF,OAAO,IAAI,gBAAgB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;IACnF,CAAC;CAGJ;AAED,MAAM,OAAO,iBAAkB,SAAQ,aAAa;IAChD;;OAEG;IACH,YACa,UAAkB,EAClB,QAAiB,EAC1B,cAA6B,IAAI,EACjC,aAAkB,EAClB,SAA0B;QAE1B,KAAK,CAAC,UAAU,EAAE,WAAW,aAAX,WAAW,cAAX,WAAW,GAAI,SAAS,EAAE;YACxC,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE,aAAa;YACzB,SAAS,EAAE,SAAS;SACvB,CAAC,CAAC;QAVM,eAAU,GAAV,UAAU,CAAQ;QAClB,aAAQ,GAAR,QAAQ,CAAS;QAwB9B,gBAAW,GAAG,oBAAoB,CAAC,IAAI,CAAC;IAdxC,CAAC;IAEM,MAAM,CAAC,QAAQ,CAAC,UAAkB,EAAE,IAAY,EAAE,WAAoB,EAAE,SAA+B;QAC1G,OAAO,IAAI,iBAAiB,CAAC,UAAU,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACtF,CAAC;IAEM,MAAM,CAAC,iBAAiB,CAAC,UAAkB,EAAE,aAAiB,EAAE,WAAoB;QACvF,OAAO,IAAI,iBAAiB,CAAC,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC;IACpF,CAAC;IAEM,MAAM,CAAC,aAAa,CAAC,UAAkB,EAAE,SAAiC,EAAE,WAAoB;QACnG,OAAO,IAAI,iBAAiB,CAAC,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAC3F,CAAC;CAGJ;AAED,MAAM,OAAO,iBAAkB,SAAQ,aAAa;IAChD,YACa,UAAkB,EAC3B,WAAoB,EACpB,IAAa,EACb,KAAc,EACd,aAAkB,EAClB,SAA0B;QAE1B,KAAK,CAAC,UAAU,EAAE,WAAW,aAAX,WAAW,cAAX,WAAW,GAAI,SAAS,EAAE;YACxC,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,KAAK;YACZ,UAAU,EAAE,aAAa;YACzB,SAAS,EAAE,SAAS;SACvB,CAAC,CAAC;QAZM,eAAU,GAAV,UAAU,CAAQ;QA+B/B,gBAAW,GAAG,oBAAoB,CAAC,IAAI,CAAC;IAlBxC,CAAC;IAEM,MAAM,CAAC,QAAQ,CAAC,UAAkB,EAAE,IAAY,EAAE,WAAoB,EAAE,SAA+B;QAC1G,OAAO,IAAI,iBAAiB,CAAC,UAAU,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACjG,CAAC;IAEM,MAAM,CAAC,SAAS,CAAC,UAAkB,EAAE,KAAa,EAAE,WAAoB,EAAE,SAA+B;QAC5G,OAAO,IAAI,iBAAiB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAClG,CAAC;IAEM,MAAM,CAAC,iBAAiB,CAAC,UAAkB,EAAE,aAAiB,EAAE,WAAoB;QACvF,OAAO,IAAI,iBAAiB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;IAC/F,CAAC;IAEM,MAAM,CAAC,aAAa,CAAC,UAAkB,EAAE,SAAiC,EAAE,WAAoB;QACnG,OAAO,IAAI,iBAAiB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACtG,CAAC;CAGJ;AAED,MAAM,OAAO,uBAAwB,SAAQ,aAAa;IACtD,YACa,UAAkB,EAC3B,WAAoB,EACpB,IAAa,EACb,KAAc,EACd,aAAkB,EAClB,SAA0B;QAE1B,KAAK,CAAC,UAAU,EAAE,WAAW,aAAX,WAAW,cAAX,WAAW,GAAI,SAAS,EAAE;YACxC,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,KAAK;YACZ,UAAU,EAAE,aAAa;YACzB,SAAS,EAAE,SAAS;SACvB,CAAC,CAAC;QAZM,eAAU,GAAV,UAAU,CAAQ;QA+B/B,gBAAW,GAAG,oBAAoB,CAAC,UAAU,CAAC;IAlB9C,CAAC;IAEM,MAAM,CAAC,QAAQ,CAAC,UAAkB,EAAE,IAAY,EAAE,WAAoB,EAAE,SAA+B;QAC1G,OAAO,IAAI,uBAAuB,CAAC,UAAU,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACvG,CAAC;IAEM,MAAM,CAAC,SAAS,CAAC,UAAkB,EAAE,KAAa,EAAE,WAAoB,EAAE,SAA+B;QAC5G,OAAO,IAAI,uBAAuB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACxG,CAAC;IAEM,MAAM,CAAC,iBAAiB,CAAC,UAAkB,EAAE,aAAiB,EAAE,WAAoB;QACvF,OAAO,IAAI,uBAAuB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;IACrG,CAAC;IAEM,MAAM,CAAC,aAAa,CAAC,UAAkB,EAAE,SAAiC,EAAE,WAAoB;QACnG,OAAO,IAAI,uBAAuB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAC5G,CAAC;CAGJ;AAED,MAAM,OAAO,oBAAqB,SAAQ,aAAa;IACnD,YACa,UAAkB,EAC3B,WAAoB,EACpB,IAAa,EACb,KAAc,EACd,aAAkB,EAClB,SAA0B;QAE1B,KAAK,CAAC,UAAU,EAAE,WAAW,aAAX,WAAW,cAAX,WAAW,GAAI,SAAS,EAAE;YACxC,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,KAAK;YACZ,UAAU,EAAE,aAAa;YACzB,SAAS,EAAE,SAAS;SACvB,CAAC,CAAC;QAZM,eAAU,GAAV,UAAU,CAAQ;QA+B/B,gBAAW,GAAG,oBAAoB,CAAC,OAAO,CAAC;IAlB3C,CAAC;IAEM,MAAM,CAAC,QAAQ,CAAC,UAAkB,EAAE,IAAY,EAAE,WAAoB,EAAE,SAA+B;QAC1G,OAAO,IAAI,oBAAoB,CAAC,UAAU,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACpG,CAAC;IAEM,MAAM,CAAC,SAAS,CAAC,UAAkB,EAAE,KAAa,EAAE,WAAoB,EAAE,SAA+B;QAC5G,OAAO,IAAI,oBAAoB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACrG,CAAC;IAEM,MAAM,CAAC,iBAAiB,CAAC,UAAkB,EAAE,aAAiB,EAAE,WAAoB;QACvF,OAAO,IAAI,oBAAoB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;IAClG,CAAC;IAEM,MAAM,CAAC,aAAa,CAAC,UAAkB,EAAE,SAAiC,EAAE,WAAoB;QACnG,OAAO,IAAI,oBAAoB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACzG,CAAC;CAGJ;AAED,MAAM,OAAO,oBAAqB,SAAQ,aAAa;IACnD,YACa,UAAkB,EAC3B,WAAoB,EACpB,IAAa,EACb,KAAc,EACd,aAAkB,EAClB,SAA0B;QAE1B,KAAK,CAAC,UAAU,EAAE,WAAW,aAAX,WAAW,cAAX,WAAW,GAAI,SAAS,EAAE;YACxC,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,KAAK;YACZ,UAAU,EAAE,aAAa;YACzB,SAAS,EAAE,SAAS;SACvB,CAAC,CAAC;QAZM,eAAU,GAAV,UAAU,CAAQ;QA+B/B,gBAAW,GAAG,oBAAoB,CAAC,OAAO,CAAC;IAlB3C,CAAC;IAEM,MAAM,CAAC,QAAQ,CAAC,UAAkB,EAAE,IAAY,EAAE,WAAoB,EAAE,SAA+B;QAC1G,OAAO,IAAI,oBAAoB,CAAC,UAAU,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACpG,CAAC;IAEM,MAAM,CAAC,SAAS,CAAC,UAAkB,EAAE,KAAa,EAAE,WAAoB,EAAE,SAA+B;QAC5G,OAAO,IAAI,oBAAoB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACrG,CAAC;IAEM,MAAM,CAAC,iBAAiB,CAAC,UAAkB,EAAE,aAAiB,EAAE,WAAoB;QACvF,OAAO,IAAI,oBAAoB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;IAClG,CAAC;IAEM,MAAM,CAAC,aAAa,CAAC,UAAkB,EAAE,SAAiC,EAAE,WAAoB;QACnG,OAAO,IAAI,oBAAoB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACzG,CAAC;CAGJ;AAED,MAAM,OAAO,gBAAiB,SAAQ,aAAa;IAC/C,YACa,UAAkB,EAC3B,WAAoB,EACpB,IAAa,EACb,KAAc,EACd,aAAkB,EAClB,SAA0B;QAE1B,KAAK,CAAC,UAAU,EAAE,WAAW,aAAX,WAAW,cAAX,WAAW,GAAI,SAAS,EAAE;YACxC,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,KAAK;YACZ,UAAU,EAAE,aAAa;YACzB,SAAS,EAAE,SAAS;SACvB,CAAC,CAAC;QAZM,eAAU,GAAV,UAAU,CAAQ;QA+B/B,gBAAW,GAAG,oBAAoB,CAAC,GAAG,CAAC;IAlBvC,CAAC;IAEM,MAAM,CAAC,QAAQ,CAAC,UAAkB,EAAE,IAAY,EAAE,WAAoB,EAAE,SAA+B;QAC1G,OAAO,IAAI,gBAAgB,CAAC,UAAU,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAChG,CAAC;IAEM,MAAM,CAAC,SAAS,CAAC,UAAkB,EAAE,KAAa,EAAE,WAAoB,EAAE,SAA+B;QAC5G,OAAO,IAAI,gBAAgB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACjG,CAAC;IAEM,MAAM,CAAC,iBAAiB,CAAC,UAAkB,EAAE,aAAiB,EAAE,WAAoB;QACvF,OAAO,IAAI,gBAAgB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;IAC9F,CAAC;IAEM,MAAM,CAAC,aAAa,CAAC,UAAkB,EAAE,SAAiC,EAAE,WAAoB;QACnG,OAAO,IAAI,gBAAgB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACrG,CAAC;CAGJ;AAED,MAAM,OAAO,iBAAkB,SAAQ,aAAa;IAChD,YACa,UAAkB,EAC3B,WAAoB,EACpB,KAAc,EACd,aAAkB,EAClB,SAA0B;QAE1B,KAAK,CAAC,UAAU,EAAE,WAAW,aAAX,WAAW,cAAX,WAAW,GAAI,SAAS,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;QANtG,eAAU,GAAV,UAAU,CAAQ;QAqB/B,gBAAW,GAAG,oBAAoB,CAAC,UAAU,CAAC;IAd9C,CAAC;IAEM,MAAM,CAAC,SAAS,CAAC,UAAkB,EAAE,KAAa,EAAE,WAAoB,EAAE,SAA+B;QAC5G,OAAO,IAAI,iBAAiB,CAAC,UAAU,EAAE,WAAW,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACvF,CAAC;IAEM,MAAM,CAAC,iBAAiB,CAAC,UAAkB,EAAE,aAAiB,EAAE,WAAoB;QACvF,OAAO,IAAI,iBAAiB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;IACpF,CAAC;IAEM,MAAM,CAAC,aAAa,CAAC,UAAkB,EAAE,SAAiC,EAAE,WAAoB;QACnG,OAAO,IAAI,iBAAiB,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAC3F,CAAC;CAGJ","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\n/* eslint-disable @typescript-eslint/ban-types -- We legitimately want to use {} as a \"any non-nullable type\" */\n\nimport { IngestionMappingKind } from \"./ingestionProperties.js\";\n\nexport enum FieldTransformation {\n PropertyBagArrayToDictionary = \"PropertyBagArrayToDictionary\",\n DateTimeFromUnixSeconds = \"DateTimeFromUnixSeconds\",\n DateTimeFromUnixMilliseconds = \"DateTimeFromUnixMilliseconds\",\n DateTimeFromUnixMicroseconds = \"DateTimeFromUnixMicroseconds\",\n DateTimeFromUnixNanoseconds = \"DateTimeFromUnixNanoseconds\",\n}\n\nexport enum ConstantTransformation {\n SourceLocation = \"SourceLocation\",\n SourceLineNumber = \"SourceLineNumber\",\n}\n\nexport type Transformation = FieldTransformation | ConstantTransformation;\n\ninterface MappingProperties {\n Field?: string;\n Path?: string;\n Ordinal?: number;\n ConstValue?: {};\n Transform?: Transformation;\n}\n\ntype MappingPropertiesStrings = {\n [key in keyof MappingProperties]: string;\n};\n\ninterface ApiColumnMapping {\n Column: string;\n DataType?: string;\n Properties?: MappingPropertiesStrings;\n}\n\nexport abstract class ColumnMapping {\n protected constructor(\n readonly columnName: string,\n readonly cslDataType?: string,\n readonly Properties?: MappingProperties,\n ) {}\n\n public abstract mappingKind: IngestionMappingKind;\n\n public toApiMapping(): ApiColumnMapping {\n const result: ApiColumnMapping = {\n Column: this.columnName,\n };\n if (this.cslDataType) {\n result.DataType = this.cslDataType;\n }\n\n if (this.Properties) {\n result.Properties = {};\n for (const key in this.Properties) {\n if (Object.prototype.hasOwnProperty.call(this.Properties, key)) {\n const typedKey = key as keyof MappingProperties;\n const property = this.Properties[typedKey];\n\n // We don't do if (property) because we '0' is a legitimate value\n if (property !== undefined && property !== null) {\n result.Properties[typedKey] = property.toString();\n }\n }\n }\n }\n return result;\n }\n}\n\nexport class CsvColumnMapping extends ColumnMapping {\n /**\n * @deprecated Use the factory methods instead.\n */\n protected constructor(\n readonly columnName: string,\n readonly cslDataType?: string,\n readonly ordinal?: string,\n constantValue?: {},\n ) {\n super(columnName, cslDataType, {\n Ordinal: ordinal === undefined ? undefined : parseInt(ordinal, 10),\n ConstValue: constantValue,\n });\n }\n\n public static withOrdinal(columnName: string, ordinal: number, cslDataType?: string): CsvColumnMapping {\n return new CsvColumnMapping(columnName, cslDataType, ordinal.toString());\n }\n\n public static withConstantValue(columnName: string, constantValue: {}, cslDataType?: string): CsvColumnMapping {\n return new CsvColumnMapping(columnName, cslDataType, undefined, constantValue);\n }\n\n mappingKind = IngestionMappingKind.CSV;\n}\n\nexport class JsonColumnMapping extends ColumnMapping {\n /**\n * @deprecated Use the factory methods instead.\n */\n constructor(\n readonly columnName: string,\n readonly jsonPath?: string,\n cslDataType: string | null = null,\n constantValue?: {},\n transform?: Transformation,\n ) {\n super(columnName, cslDataType ?? undefined, {\n Path: jsonPath,\n ConstValue: constantValue,\n Transform: transform,\n });\n }\n\n public static withPath(columnName: string, path: string, cslDataType?: string, transform?: FieldTransformation): JsonColumnMapping {\n return new JsonColumnMapping(columnName, path, cslDataType, undefined, transform);\n }\n\n public static withConstantValue(columnName: string, constantValue: {}, cslDataType?: string): JsonColumnMapping {\n return new JsonColumnMapping(columnName, undefined, cslDataType, constantValue);\n }\n\n public static withTransform(columnName: string, transform: ConstantTransformation, cslDataType?: string): JsonColumnMapping {\n return new JsonColumnMapping(columnName, undefined, cslDataType, undefined, transform);\n }\n\n mappingKind = IngestionMappingKind.JSON;\n}\n\nexport class AvroColumnMapping extends ColumnMapping {\n private constructor(\n readonly columnName: string,\n cslDataType?: string,\n path?: string,\n field?: string,\n constantValue?: {},\n transform?: Transformation,\n ) {\n super(columnName, cslDataType ?? undefined, {\n Path: path,\n Field: field,\n ConstValue: constantValue,\n Transform: transform,\n });\n }\n\n public static withPath(columnName: string, path: string, cslDataType?: string, transform?: FieldTransformation): AvroColumnMapping {\n return new AvroColumnMapping(columnName, cslDataType, path, undefined, undefined, transform);\n }\n\n public static withField(columnName: string, field: string, cslDataType?: string, transform?: FieldTransformation): AvroColumnMapping {\n return new AvroColumnMapping(columnName, cslDataType, undefined, field, undefined, transform);\n }\n\n public static withConstantValue(columnName: string, constantValue: {}, cslDataType?: string): AvroColumnMapping {\n return new AvroColumnMapping(columnName, cslDataType, undefined, undefined, constantValue);\n }\n\n public static withTransform(columnName: string, transform: ConstantTransformation, cslDataType?: string): AvroColumnMapping {\n return new AvroColumnMapping(columnName, cslDataType, undefined, undefined, undefined, transform);\n }\n\n mappingKind = IngestionMappingKind.AVRO;\n}\n\nexport class ApacheAvroColumnMapping extends ColumnMapping {\n private constructor(\n readonly columnName: string,\n cslDataType?: string,\n path?: string,\n field?: string,\n constantValue?: {},\n transform?: Transformation,\n ) {\n super(columnName, cslDataType ?? undefined, {\n Path: path,\n Field: field,\n ConstValue: constantValue,\n Transform: transform,\n });\n }\n\n public static withPath(columnName: string, path: string, cslDataType?: string, transform?: FieldTransformation): ApacheAvroColumnMapping {\n return new ApacheAvroColumnMapping(columnName, cslDataType, path, undefined, undefined, transform);\n }\n\n public static withField(columnName: string, field: string, cslDataType?: string, transform?: FieldTransformation): ApacheAvroColumnMapping {\n return new ApacheAvroColumnMapping(columnName, cslDataType, undefined, field, undefined, transform);\n }\n\n public static withConstantValue(columnName: string, constantValue: {}, cslDataType?: string): ApacheAvroColumnMapping {\n return new ApacheAvroColumnMapping(columnName, cslDataType, undefined, undefined, constantValue);\n }\n\n public static withTransform(columnName: string, transform: ConstantTransformation, cslDataType?: string): ApacheAvroColumnMapping {\n return new ApacheAvroColumnMapping(columnName, cslDataType, undefined, undefined, undefined, transform);\n }\n\n mappingKind = IngestionMappingKind.APACHEAVRO;\n}\n\nexport class SStreamColumnMapping extends ColumnMapping {\n private constructor(\n readonly columnName: string,\n cslDataType?: string,\n path?: string,\n field?: string,\n constantValue?: {},\n transform?: Transformation,\n ) {\n super(columnName, cslDataType ?? undefined, {\n Path: path,\n Field: field,\n ConstValue: constantValue,\n Transform: transform,\n });\n }\n\n public static withPath(columnName: string, path: string, cslDataType?: string, transform?: FieldTransformation): SStreamColumnMapping {\n return new SStreamColumnMapping(columnName, cslDataType, path, undefined, undefined, transform);\n }\n\n public static withField(columnName: string, field: string, cslDataType?: string, transform?: FieldTransformation): SStreamColumnMapping {\n return new SStreamColumnMapping(columnName, cslDataType, undefined, field, undefined, transform);\n }\n\n public static withConstantValue(columnName: string, constantValue: {}, cslDataType?: string): SStreamColumnMapping {\n return new SStreamColumnMapping(columnName, cslDataType, undefined, undefined, constantValue);\n }\n\n public static withTransform(columnName: string, transform: ConstantTransformation, cslDataType?: string): SStreamColumnMapping {\n return new SStreamColumnMapping(columnName, cslDataType, undefined, undefined, undefined, transform);\n }\n\n mappingKind = IngestionMappingKind.SSTREAM;\n}\n\nexport class ParquetColumnMapping extends ColumnMapping {\n private constructor(\n readonly columnName: string,\n cslDataType?: string,\n path?: string,\n field?: string,\n constantValue?: {},\n transform?: Transformation,\n ) {\n super(columnName, cslDataType ?? undefined, {\n Path: path,\n Field: field,\n ConstValue: constantValue,\n Transform: transform,\n });\n }\n\n public static withPath(columnName: string, path: string, cslDataType?: string, transform?: FieldTransformation): ParquetColumnMapping {\n return new ParquetColumnMapping(columnName, cslDataType, path, undefined, undefined, transform);\n }\n\n public static withField(columnName: string, field: string, cslDataType?: string, transform?: FieldTransformation): ParquetColumnMapping {\n return new ParquetColumnMapping(columnName, cslDataType, undefined, field, undefined, transform);\n }\n\n public static withConstantValue(columnName: string, constantValue: {}, cslDataType?: string): ParquetColumnMapping {\n return new ParquetColumnMapping(columnName, cslDataType, undefined, undefined, constantValue);\n }\n\n public static withTransform(columnName: string, transform: ConstantTransformation, cslDataType?: string): ParquetColumnMapping {\n return new ParquetColumnMapping(columnName, cslDataType, undefined, undefined, undefined, transform);\n }\n\n mappingKind = IngestionMappingKind.PARQUET;\n}\n\nexport class OrcColumnMapping extends ColumnMapping {\n private constructor(\n readonly columnName: string,\n cslDataType?: string,\n path?: string,\n field?: string,\n constantValue?: {},\n transform?: Transformation,\n ) {\n super(columnName, cslDataType ?? undefined, {\n Path: path,\n Field: field,\n ConstValue: constantValue,\n Transform: transform,\n });\n }\n\n public static withPath(columnName: string, path: string, cslDataType?: string, transform?: FieldTransformation): OrcColumnMapping {\n return new OrcColumnMapping(columnName, cslDataType, path, undefined, undefined, transform);\n }\n\n public static withField(columnName: string, field: string, cslDataType?: string, transform?: FieldTransformation): OrcColumnMapping {\n return new OrcColumnMapping(columnName, cslDataType, undefined, field, undefined, transform);\n }\n\n public static withConstantValue(columnName: string, constantValue: {}, cslDataType?: string): OrcColumnMapping {\n return new OrcColumnMapping(columnName, cslDataType, undefined, undefined, constantValue);\n }\n\n public static withTransform(columnName: string, transform: ConstantTransformation, cslDataType?: string): OrcColumnMapping {\n return new OrcColumnMapping(columnName, cslDataType, undefined, undefined, undefined, transform);\n }\n\n mappingKind = IngestionMappingKind.ORC;\n}\n\nexport class W3CLogFileMapping extends ColumnMapping {\n private constructor(\n readonly columnName: string,\n cslDataType?: string,\n field?: string,\n constantValue?: {},\n transform?: Transformation,\n ) {\n super(columnName, cslDataType ?? undefined, { Field: field, ConstValue: constantValue, Transform: transform });\n }\n\n public static withField(columnName: string, field: string, cslDataType?: string, transform?: FieldTransformation): W3CLogFileMapping {\n return new W3CLogFileMapping(columnName, cslDataType, field, undefined, transform);\n }\n\n public static withConstantValue(columnName: string, constantValue: {}, cslDataType?: string): W3CLogFileMapping {\n return new W3CLogFileMapping(columnName, cslDataType, undefined, constantValue);\n }\n\n public static withTransform(columnName: string, transform: ConstantTransformation, cslDataType?: string): W3CLogFileMapping {\n return new W3CLogFileMapping(columnName, cslDataType, undefined, undefined, transform);\n }\n\n mappingKind = IngestionMappingKind.W3CLOGFILE;\n}\n"]}
|
|
@@ -1,43 +1,34 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
// Copyright (c) Microsoft Corporation.
|
|
3
2
|
// Licensed under the MIT License.
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
};
|
|
7
|
-
|
|
8
|
-
exports.generateBlobName = exports.BlobDescriptor = exports.StreamDescriptor = exports.AbstractDescriptor = exports.shouldCompressFileByExtension = exports.getSourceId = exports.CompressionType = void 0;
|
|
9
|
-
const uuid_1 = require("uuid");
|
|
10
|
-
const uuid_validate_1 = __importDefault(require("uuid-validate"));
|
|
11
|
-
const storage_blob_1 = require("@azure/storage-blob");
|
|
12
|
-
var CompressionType;
|
|
3
|
+
import { v4 as uuidv4 } from "uuid";
|
|
4
|
+
import uuidValidate from "uuid-validate";
|
|
5
|
+
import { BlobClient } from "@azure/storage-blob";
|
|
6
|
+
export var CompressionType;
|
|
13
7
|
(function (CompressionType) {
|
|
14
8
|
CompressionType["ZIP"] = ".zip";
|
|
15
9
|
CompressionType["GZIP"] = ".gz";
|
|
16
10
|
CompressionType["None"] = "";
|
|
17
|
-
})(CompressionType || (
|
|
18
|
-
const getSourceId = (sourceId) => {
|
|
11
|
+
})(CompressionType || (CompressionType = {}));
|
|
12
|
+
export const getSourceId = (sourceId) => {
|
|
19
13
|
if (sourceId) {
|
|
20
|
-
if (!(
|
|
14
|
+
if (!uuidValidate(sourceId, 4)) {
|
|
21
15
|
throw Error("sourceId is not a valid uuid/v4");
|
|
22
16
|
}
|
|
23
17
|
return sourceId;
|
|
24
18
|
}
|
|
25
|
-
return (
|
|
19
|
+
return uuidv4();
|
|
26
20
|
};
|
|
27
|
-
|
|
28
|
-
function shouldCompressFileByExtension(extension) {
|
|
21
|
+
export function shouldCompressFileByExtension(extension) {
|
|
29
22
|
return !(extension === ".avro" || extension === ".apacheavro" || extension === ".parquet" || extension === ".sstream" || extension === ".orc");
|
|
30
23
|
}
|
|
31
|
-
|
|
32
|
-
class AbstractDescriptor {
|
|
24
|
+
export class AbstractDescriptor {
|
|
33
25
|
constructor(sourceId = null, size = null) {
|
|
34
26
|
this.sourceId = sourceId;
|
|
35
27
|
this.size = size;
|
|
36
|
-
this.sourceId =
|
|
28
|
+
this.sourceId = getSourceId(sourceId);
|
|
37
29
|
}
|
|
38
30
|
}
|
|
39
|
-
|
|
40
|
-
class StreamDescriptor extends AbstractDescriptor {
|
|
31
|
+
export class StreamDescriptor extends AbstractDescriptor {
|
|
41
32
|
/**
|
|
42
33
|
* Use Readable for Node.js and ArrayBuffer in browser
|
|
43
34
|
*/
|
|
@@ -57,15 +48,14 @@ class StreamDescriptor extends AbstractDescriptor {
|
|
|
57
48
|
return this.compressionType ? `.${this.compressionType}` : "";
|
|
58
49
|
}
|
|
59
50
|
}
|
|
60
|
-
|
|
61
|
-
class BlobDescriptor extends AbstractDescriptor {
|
|
51
|
+
export class BlobDescriptor extends AbstractDescriptor {
|
|
62
52
|
constructor(path, size = null, sourceId = null) {
|
|
63
53
|
super(sourceId, size);
|
|
64
54
|
this.path = path;
|
|
65
55
|
}
|
|
66
56
|
async fillSize() {
|
|
67
57
|
if (!this.size) {
|
|
68
|
-
const blobClient = new
|
|
58
|
+
const blobClient = new BlobClient(this.path);
|
|
69
59
|
const blobProps = await blobClient.getProperties();
|
|
70
60
|
const length = blobProps.contentLength;
|
|
71
61
|
if (length !== undefined) {
|
|
@@ -77,12 +67,10 @@ class BlobDescriptor extends AbstractDescriptor {
|
|
|
77
67
|
}
|
|
78
68
|
}
|
|
79
69
|
}
|
|
80
|
-
|
|
81
|
-
const generateBlobName = (desc, props) => {
|
|
70
|
+
export const generateBlobName = (desc, props) => {
|
|
82
71
|
const extension = desc instanceof StreamDescriptor ? null : `${desc.name ? "__" + desc.name : `${desc.extension ? "." + desc.extension : ""}`}`;
|
|
83
72
|
const formatSuffix = props.format ? `.${props.format}` : ".csv";
|
|
84
73
|
const compressionString = desc.getCompressionSuffix();
|
|
85
74
|
return `${props.database}__${props.table}__${desc.sourceId}${extension || formatSuffix}${compressionString}`;
|
|
86
75
|
};
|
|
87
|
-
exports.generateBlobName = generateBlobName;
|
|
88
76
|
//# sourceMappingURL=descriptors.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"descriptors.js","sourceRoot":"","sources":["../../src/descriptors.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"descriptors.js","sourceRoot":"","sources":["../../src/descriptors.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAElC,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AACpC,OAAO,YAAY,MAAM,eAAe,CAAC;AAGzC,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,MAAM,CAAN,IAAY,eAIX;AAJD,WAAY,eAAe;IACvB,+BAAY,CAAA;IACZ,+BAAY,CAAA;IACZ,4BAAS,CAAA;AACb,CAAC,EAJW,eAAe,KAAf,eAAe,QAI1B;AAED,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,QAAuB,EAAU,EAAE;IAC3D,IAAI,QAAQ,EAAE,CAAC;QACX,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,CAAC;YAC7B,MAAM,KAAK,CAAC,iCAAiC,CAAC,CAAC;QACnD,CAAC;QACD,OAAO,QAAQ,CAAC;IACpB,CAAC;IACD,OAAO,MAAM,EAAE,CAAC;AACpB,CAAC,CAAC;AAEF,MAAM,UAAU,6BAA6B,CAAC,SAAkB;IAC5D,OAAO,CAAC,CAAC,SAAS,KAAK,OAAO,IAAI,SAAS,KAAK,aAAa,IAAI,SAAS,KAAK,UAAU,IAAI,SAAS,KAAK,UAAU,IAAI,SAAS,KAAK,MAAM,CAAC,CAAC;AACnJ,CAAC;AAED,MAAM,OAAgB,kBAAkB;IACpC,YACW,WAA0B,IAAI,EAC9B,OAAsB,IAAI;QAD1B,aAAQ,GAAR,QAAQ,CAAsB;QAC9B,SAAI,GAAJ,IAAI,CAAsB;QAEjC,IAAI,CAAC,QAAQ,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC1C,CAAC;CACJ;AAED,MAAM,OAAO,gBAAiB,SAAQ,kBAAkB;IACpD;;OAEG;IACH,YACa,MAA8B,EACvC,WAA0B,IAAI,EACvB,kBAAmC,eAAe,CAAC,IAAI,EAC9D,OAAsB,IAAI;QAE1B,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QALb,WAAM,GAAN,MAAM,CAAwB;QAEhC,oBAAe,GAAf,eAAe,CAAwC;IAIlE,CAAC;IAED,KAAK,CAAC,KAAuB;QACzB,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;QACvB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC,eAAe,CAAC;QAC7C,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;QAC/B,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,6CAA6C;IAC7C,oBAAoB;QAChB,OAAO,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAClE,CAAC;CACJ;AAED,MAAM,OAAO,cAAe,SAAQ,kBAAkB;IAClD,YACa,IAAY,EACrB,OAAsB,IAAI,EAC1B,WAA0B,IAAI;QAE9B,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAJb,SAAI,GAAJ,IAAI,CAAQ;IAKzB,CAAC;IAED,KAAK,CAAC,QAAQ;QACV,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACb,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7C,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,aAAa,EAAE,CAAC;YACnD,MAAM,MAAM,GAAG,SAAS,CAAC,aAAa,CAAC;YACvC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gBACvB,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;oBACf,MAAM,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC;gBACnC,CAAC;gBACD,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC;YACvB,CAAC;QACL,CAAC;IACL,CAAC;CACJ;AAaD,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,IAA2C,EAAE,KAA0B,EAAU,EAAE;IAChH,MAAM,SAAS,GAAG,IAAI,YAAY,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC;IAEhJ,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,iBAAiB,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAC;IACtD,OAAO,GAAG,KAAK,CAAC,QAAQ,KAAK,KAAK,CAAC,KAAK,KAAK,IAAI,CAAC,QAAQ,GAAG,SAAS,IAAI,YAAY,GAAG,iBAAiB,EAAE,CAAC;AACjH,CAAC,CAAC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport { v4 as uuidv4 } from \"uuid\";\nimport uuidValidate from \"uuid-validate\";\nimport { Readable } from \"stream\";\nimport IngestionProperties from \"./ingestionProperties.js\";\nimport { BlobClient } from \"@azure/storage-blob\";\n\nexport enum CompressionType {\n ZIP = \".zip\",\n GZIP = \".gz\",\n None = \"\",\n}\n\nexport const getSourceId = (sourceId: string | null): string => {\n if (sourceId) {\n if (!uuidValidate(sourceId, 4)) {\n throw Error(\"sourceId is not a valid uuid/v4\");\n }\n return sourceId;\n }\n return uuidv4();\n};\n\nexport function shouldCompressFileByExtension(extension?: string): boolean {\n return !(extension === \".avro\" || extension === \".apacheavro\" || extension === \".parquet\" || extension === \".sstream\" || extension === \".orc\");\n}\n\nexport abstract class AbstractDescriptor {\n constructor(\n public sourceId: string | null = null,\n public size: number | null = null,\n ) {\n this.sourceId = getSourceId(sourceId);\n }\n}\n\nexport class StreamDescriptor extends AbstractDescriptor {\n /**\n * Use Readable for Node.js and ArrayBuffer in browser\n */\n constructor(\n readonly stream: Readable | ArrayBuffer,\n sourceId: string | null = null,\n public compressionType: CompressionType = CompressionType.None,\n size: number | null = null,\n ) {\n super(sourceId, size);\n }\n\n merge(other: StreamDescriptor) {\n this.size = other.size;\n this.compressionType = other.compressionType;\n this.sourceId = other.sourceId;\n return this;\n }\n\n // Currently streams are not compressed by us\n getCompressionSuffix() {\n return this.compressionType ? `.${this.compressionType}` : \"\";\n }\n}\n\nexport class BlobDescriptor extends AbstractDescriptor {\n constructor(\n readonly path: string,\n size: number | null = null,\n sourceId: string | null = null,\n ) {\n super(sourceId, size);\n }\n\n async fillSize(): Promise<void> {\n if (!this.size) {\n const blobClient = new BlobClient(this.path);\n const blobProps = await blobClient.getProperties();\n const length = blobProps.contentLength;\n if (length !== undefined) {\n if (length === 0) {\n throw new Error(\"Empty blob.\");\n }\n this.size = length;\n }\n }\n }\n}\n\nexport interface FileDescriptorBase {\n size: number | null;\n zipped: boolean;\n compressionType: CompressionType;\n cleanupTmp?: () => Promise<void>;\n extension?: string;\n name?: string;\n sourceId: string | null;\n getCompressionSuffix: () => string;\n}\n\nexport const generateBlobName = (desc: StreamDescriptor | FileDescriptorBase, props: IngestionProperties): string => {\n const extension = desc instanceof StreamDescriptor ? null : `${desc.name ? \"__\" + desc.name : `${desc.extension ? \".\" + desc.extension : \"\"}`}`;\n\n const formatSuffix = props.format ? `.${props.format}` : \".csv\";\n const compressionString = desc.getCompressionSuffix();\n return `${props.database}__${props.table}__${desc.sourceId}${extension || formatSuffix}${compressionString}`;\n};\n"]}
|
package/dist-esm/src/errors.js
CHANGED
|
@@ -1,13 +1,9 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
// Copyright (c) Microsoft Corporation.
|
|
3
2
|
// Licensed under the MIT License.
|
|
4
|
-
|
|
5
|
-
exports.IngestionPropertiesValidationError = void 0;
|
|
6
|
-
class IngestionPropertiesValidationError extends Error {
|
|
3
|
+
export class IngestionPropertiesValidationError extends Error {
|
|
7
4
|
constructor(message) {
|
|
8
5
|
super(message);
|
|
9
6
|
this.name = "IngestionPropertiesValidationError";
|
|
10
7
|
}
|
|
11
8
|
}
|
|
12
|
-
exports.IngestionPropertiesValidationError = IngestionPropertiesValidationError;
|
|
13
9
|
//# sourceMappingURL=errors.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../src/errors.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../src/errors.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAElC,MAAM,OAAO,kCAAmC,SAAQ,KAAK;IACzD,YAAY,OAAe;QACvB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,oCAAoC,CAAC;IACrD,CAAC;CACJ","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nexport class IngestionPropertiesValidationError extends Error {\n constructor(message: string) {\n super(message);\n this.name = \"IngestionPropertiesValidationError\";\n }\n}\n"]}
|
|
@@ -1,30 +1,24 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
// Copyright (c) Microsoft Corporation.
|
|
3
2
|
// Licensed under the MIT License.
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
};
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
const pako_1 = __importDefault(require("pako"));
|
|
10
|
-
const descriptors_1 = require("./descriptors");
|
|
11
|
-
const ingestionProperties_1 = require("./ingestionProperties");
|
|
12
|
-
class FileDescriptor extends descriptors_1.AbstractDescriptor {
|
|
13
|
-
constructor(file, sourceId = null, size = null, compressionType = descriptors_1.CompressionType.None, extension, name) {
|
|
3
|
+
import pako from "pako";
|
|
4
|
+
import { AbstractDescriptor, CompressionType, shouldCompressFileByExtension } from "./descriptors.js";
|
|
5
|
+
import { shouldCompressFileByFormat } from "./ingestionProperties.js";
|
|
6
|
+
export class FileDescriptor extends AbstractDescriptor {
|
|
7
|
+
constructor(file, sourceId = null, size = null, compressionType = CompressionType.None, extension, name) {
|
|
14
8
|
super(sourceId);
|
|
15
9
|
this.file = file;
|
|
16
10
|
this.extension = extension;
|
|
17
11
|
this.name = name;
|
|
18
12
|
this.compressionType = compressionType;
|
|
19
13
|
this.size = size || file.size;
|
|
20
|
-
this.zipped = compressionType !==
|
|
21
|
-
this.shouldNotCompress = !
|
|
14
|
+
this.zipped = compressionType !== CompressionType.None || this.extension === ".gz" || this.extension === ".zip";
|
|
15
|
+
this.shouldNotCompress = !shouldCompressFileByExtension(this.extension);
|
|
22
16
|
}
|
|
23
17
|
async prepare(ingestionProperties) {
|
|
24
|
-
const shouldNotCompressByFormat = !
|
|
18
|
+
const shouldNotCompressByFormat = !shouldCompressFileByFormat(ingestionProperties);
|
|
25
19
|
if (!this.zipped && !this.shouldNotCompress && !shouldNotCompressByFormat) {
|
|
26
20
|
try {
|
|
27
|
-
const gzipped =
|
|
21
|
+
const gzipped = pako.gzip(await this.file.arrayBuffer());
|
|
28
22
|
return new Blob([gzipped]);
|
|
29
23
|
}
|
|
30
24
|
catch (e) {
|
|
@@ -42,5 +36,4 @@ class FileDescriptor extends descriptors_1.AbstractDescriptor {
|
|
|
42
36
|
return this.compressionType ? `.${this.compressionType}` : ".gz";
|
|
43
37
|
}
|
|
44
38
|
}
|
|
45
|
-
exports.FileDescriptor = FileDescriptor;
|
|
46
39
|
//# sourceMappingURL=fileDescriptor.browser.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fileDescriptor.browser.js","sourceRoot":"","sources":["../../src/fileDescriptor.browser.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"fileDescriptor.browser.js","sourceRoot":"","sources":["../../src/fileDescriptor.browser.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAElC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAsB,6BAA6B,EAAE,MAAM,kBAAkB,CAAC;AAC1H,OAAO,EAA4B,0BAA0B,EAAE,MAAM,0BAA0B,CAAC;AAEhG,MAAM,OAAO,cAAe,SAAQ,kBAAkB;IAOlD,YACa,IAAU,EACnB,WAA0B,IAAI,EAC9B,OAAsB,IAAI,EAC1B,kBAAmC,eAAe,CAAC,IAAI,EAC9C,SAAkB,EAClB,IAAa;QAEtB,KAAK,CAAC,QAAQ,CAAC,CAAC;QAPP,SAAI,GAAJ,IAAI,CAAM;QAIV,cAAS,GAAT,SAAS,CAAS;QAClB,SAAI,GAAJ,IAAI,CAAS;QAGtB,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,IAAI,GAAG,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC;QAC9B,IAAI,CAAC,MAAM,GAAG,eAAe,KAAK,eAAe,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,KAAK,KAAK,IAAI,IAAI,CAAC,SAAS,KAAK,MAAM,CAAC;QAChH,IAAI,CAAC,iBAAiB,GAAG,CAAC,6BAA6B,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC5E,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,mBAA8C;QACxD,MAAM,yBAAyB,GAAG,CAAC,0BAA0B,CAAC,mBAAmB,CAAC,CAAC;QACnF,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,iBAAiB,IAAI,CAAC,yBAAyB,EAAE,CAAC;YACxE,IAAI,CAAC;gBACD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;gBACzD,OAAO,IAAI,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,kCAAkC;YACtC,CAAC;QACL,CAAC;QAED,OAAO,IAAI,CAAC,IAAI,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,OAAO;QACT,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAClB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAC5B,CAAC;IACL,CAAC;IAED,oBAAoB;QAChB,OAAO,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;IACrE,CAAC;CACJ","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport pako from \"pako\";\nimport { AbstractDescriptor, CompressionType, FileDescriptorBase, shouldCompressFileByExtension } from \"./descriptors.js\";\nimport { IngestionPropertiesInput, shouldCompressFileByFormat } from \"./ingestionProperties.js\";\n\nexport class FileDescriptor extends AbstractDescriptor implements FileDescriptorBase {\n size: number | null;\n zipped: boolean;\n compressionType: CompressionType;\n shouldNotCompress: boolean;\n cleanupTmp?: () => Promise<void>;\n\n constructor(\n readonly file: Blob,\n sourceId: string | null = null,\n size: number | null = null,\n compressionType: CompressionType = CompressionType.None,\n readonly extension?: string,\n readonly name?: string,\n ) {\n super(sourceId);\n this.compressionType = compressionType;\n this.size = size || file.size;\n this.zipped = compressionType !== CompressionType.None || this.extension === \".gz\" || this.extension === \".zip\";\n this.shouldNotCompress = !shouldCompressFileByExtension(this.extension);\n }\n\n async prepare(ingestionProperties?: IngestionPropertiesInput): Promise<Blob> {\n const shouldNotCompressByFormat = !shouldCompressFileByFormat(ingestionProperties);\n if (!this.zipped && !this.shouldNotCompress && !shouldNotCompressByFormat) {\n try {\n const gzipped = pako.gzip(await this.file.arrayBuffer());\n return new Blob([gzipped]);\n } catch (e) {\n // Ignore - return the file itself\n }\n }\n\n return this.file;\n }\n\n async cleanup(): Promise<void> {\n if (this.cleanupTmp) {\n await this.cleanupTmp();\n }\n }\n\n getCompressionSuffix() {\n return this.compressionType ? `.${this.compressionType}` : \".gz\";\n }\n}\n"]}
|