azure-kusto-ingest 6.0.2 → 7.0.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist-esm/src/abstractKustoClient.js +8 -15
- package/dist-esm/src/abstractKustoClient.js.map +1 -1
- package/dist-esm/src/columnMappings.js +22 -34
- package/dist-esm/src/columnMappings.js.map +1 -1
- package/dist-esm/src/descriptors.js +15 -27
- package/dist-esm/src/descriptors.js.map +1 -1
- package/dist-esm/src/errors.js +1 -5
- package/dist-esm/src/errors.js.map +1 -1
- package/dist-esm/src/fileDescriptor.browser.js +9 -16
- package/dist-esm/src/fileDescriptor.browser.js.map +1 -1
- package/dist-esm/src/fileDescriptor.js +20 -28
- package/dist-esm/src/fileDescriptor.js.map +1 -1
- package/dist-esm/src/index.js +38 -73
- package/dist-esm/src/index.js.map +1 -1
- package/dist-esm/src/ingestClient.browser.js +11 -15
- package/dist-esm/src/ingestClient.browser.js.map +1 -1
- package/dist-esm/src/ingestClient.js +11 -15
- package/dist-esm/src/ingestClient.js.map +1 -1
- package/dist-esm/src/ingestClientBase.js +25 -32
- package/dist-esm/src/ingestClientBase.js.map +1 -1
- package/dist-esm/src/ingestionBlobInfo.js +4 -8
- package/dist-esm/src/ingestionBlobInfo.js.map +1 -1
- package/dist-esm/src/ingestionProperties.js +29 -36
- package/dist-esm/src/ingestionProperties.js.map +1 -1
- package/dist-esm/src/ingestionResult.js +10 -17
- package/dist-esm/src/ingestionResult.js.map +1 -1
- package/dist-esm/src/managedStreamingIngestClient.js +29 -34
- package/dist-esm/src/managedStreamingIngestClient.js.map +1 -1
- package/dist-esm/src/rankedStorageAccount.js +1 -5
- package/dist-esm/src/rankedStorageAccount.js.map +1 -1
- package/dist-esm/src/rankedStorageAccountSet.js +3 -7
- package/dist-esm/src/rankedStorageAccountSet.js.map +1 -1
- package/dist-esm/src/resourceManager.js +24 -31
- package/dist-esm/src/resourceManager.js.map +1 -1
- package/dist-esm/src/retry.js +3 -8
- package/dist-esm/src/retry.js.map +1 -1
- package/dist-esm/src/status.js +6 -11
- package/dist-esm/src/status.js.map +1 -1
- package/dist-esm/src/statusQ.js +3 -7
- package/dist-esm/src/statusQ.js.map +1 -1
- package/dist-esm/src/streamUtils.browser.js +8 -15
- package/dist-esm/src/streamUtils.browser.js.map +1 -1
- package/dist-esm/src/streamUtils.js +18 -28
- package/dist-esm/src/streamUtils.js.map +1 -1
- package/dist-esm/src/streamingIngestClient.browser.js +9 -11
- package/dist-esm/src/streamingIngestClient.browser.js.map +1 -1
- package/dist-esm/src/streamingIngestClient.js +12 -17
- package/dist-esm/src/streamingIngestClient.js.map +1 -1
- package/dist-esm/src/streamingIngestClientBase.js +7 -11
- package/dist-esm/src/streamingIngestClientBase.js.map +1 -1
- package/package.json +11 -6
- package/types/src/abstractKustoClient.d.ts +2 -2
- package/types/src/columnMappings.d.ts +1 -1
- package/types/src/descriptors.d.ts +1 -2
- package/types/src/fileDescriptor.browser.d.ts +2 -2
- package/types/src/fileDescriptor.d.ts +2 -2
- package/types/src/index.d.ts +13 -13
- package/types/src/ingestClient.browser.d.ts +5 -5
- package/types/src/ingestClient.d.ts +5 -6
- package/types/src/ingestClientBase.d.ts +5 -5
- package/types/src/ingestionBlobInfo.d.ts +3 -3
- package/types/src/ingestionProperties.d.ts +1 -1
- package/types/src/managedStreamingIngestClient.d.ts +5 -6
- package/types/src/rankedStorageAccountSet.d.ts +1 -1
- package/types/src/status.d.ts +2 -2
- package/types/src/statusQ.d.ts +2 -2
- package/types/src/streamUtils.browser.d.ts +2 -3
- package/types/src/streamUtils.d.ts +2 -3
- package/types/src/streamingIngestClient.browser.d.ts +4 -4
- package/types/src/streamingIngestClient.d.ts +4 -5
- package/types/src/streamingIngestClientBase.d.ts +3 -3
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"statusQ.js","sourceRoot":"","sources":["../../src/statusQ.ts"],"names":[],"mappings":";AAAA,uCAAuC;AACvC,kCAAkC;;;AAElC,wDAAsE;AAItE,MAAM,YAAY;IACd,YAAqB,IAAY,EAAW,OAAoB;QAA3C,SAAI,GAAJ,IAAI,CAAQ;QAAW,YAAO,GAAP,OAAO,CAAa;IAAG,CAAC;CACvE;AAED,MAAM,OAAO,GAAG,CAAI,CAAM,EAAO,EAAE;IAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE;QACnC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC9C,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACZ,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;KACf;IAED,OAAO,CAAC,CAAC;AACb,CAAC,CAAC;AAaF,MAAa,WAAW;IACpB,YAAqB,aAA2C,EAAW,UAAgC;QAAtF,kBAAa,GAAb,aAAa,CAA8B;QAAW,eAAU,GAAV,UAAU,CAAsB;IAAG,CAAC;IAE/G,aAAa,CAAC,aAA4B;QACtC,OAAO,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YAC3B,MAAM,OAAO,GAAG,CAAC,CAAC,GAAG,CAAC;YACtB,IAAI,CAAC,OAAO,EAAE;gBACV,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;aACtD;YACD,eAAe;YACf,MAAM,UAAU,GAAG,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,IAAI,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;YACrE,OAAO,IAAI,YAAY,CAAC,IAAI,EAAE,IAAI,2BAAW,CAAC,OAAO,CAAC,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;IACP,CAAC;IAED,KAAK,CAAC,OAAO;QACT,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;QACjD,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC;IAC1C,CAAC;IAED,aAAa,CAAC,OAAe;QACzB,OAAO,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC5D,CAAC;IAED,kBAAkB,CAAC,CAAoB;QACnC,OAAO,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;IAC9E,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,EAAkB,EAAE,CAAS,EAAE,OAA0B;QACjE,MAAM,MAAM,GAAc,EAAE,CAAC;QAC7B,MAAM,UAAU,GAAmB,EAAE,CAAC;QAEtC,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE;YAChB,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;YAChD,MAAM,QAAQ,GAAG,QAAQ,CAAC,kBAAkB,CAAC;YAE7C,IAAI,QAAQ,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE;gBACjC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACtB;YAED,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE;gBACtB,IAAI,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE;oBAChC,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC;oBAErE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE;wBACrB,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC;qBAC7C;iBACJ;aACJ;SACJ;QACD,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC;IACjE,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,UAA6B,IAAI;QAC/C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC1C,MAAM,SAAS,GAAmB,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;QACtE,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC;QAExF,sDAAsD;QACtD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;QAE3D,IAAI,OAAO,CAAC,IAAI,EAAE;YACd,OAAO,OAAO,CAAC,MAAM,CAAC;SACzB;QACD,MAAM,kBAAkB,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;QAErD,2FAA2F;QAC3F,OAAO,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,kBAAkB,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;IACtF,CAAC;IAED,KAAK,CAAC,IAAI,CACN,EAAkB,EAClB,CAAS,EACT,OAAyB;QAEzB,MAAM,UAAU,GAAU,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,EAAE,CAAC;QAElB,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE;YAChB,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC,CAAC;YACvE,MAAM,QAAQ,GAAG,QAAQ,CAAC,oBAAoB,CAAC;YAC/C,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE;gBACtB,IAAI,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE;oBAChC,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC;oBAErE,IAAI,CAAC,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;wBAC/B,MAAM,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC;qBAC5D;oBACD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE;wBACrB,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC;qBAC7C;iBACJ;aACJ;SACJ;QAED,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC;IACjE,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,UAA4B,IAAI;;QAC7C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;QACtD,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC;QAExF,sDAAsD;QACtD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;QAC1D,IAAI,OAAO,CAAC,IAAI,EAAE;YACd,OAAO,OAAO,CAAC,MAAM,CAAC;SACzB;QAED,MAAM,iBAAiB,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;QAEpD,2FAA2F;QAC3F,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,MAAA,OAAO,CAAC,MAAM,CAAC,UAAU,mCAAI,EAAE,EAAE,iBAAiB,EAAE,OAAO,CAAC,CAAC;QAC3F,OAAO,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC/C,CAAC;CACJ;AApHD,kCAoHC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport { PeekedMessageItem, QueueClient } from \"@azure/storage-queue\";\nimport { ResourceURI } from \"./resourceManager\";\nimport { StatusMessage } from \"./status\";\n\nclass QueueDetails {\n constructor(readonly name: string, readonly service: QueueClient) {}\n}\n\nconst shuffle = <T>(a: T[]): T[] => {\n for (let i = a.length - 1; i > 0; i--) {\n const j = Math.floor(Math.random() * (i + 1));\n const temp = a[j];\n a[j] = a[i];\n a[i] = temp;\n }\n\n return a;\n};\n\ninterface PeekParams {\n raw: boolean;\n}\n\ninterface PopParams {\n raw: boolean;\n remove: boolean;\n}\n\ntype Message = PeekedMessageItem | StatusMessage;\n\nexport class StatusQueue {\n constructor(readonly getQueuesFunc: () => Promise<ResourceURI[]>, readonly messageCls: typeof StatusMessage) {}\n\n _getQServices(queuesDetails: ResourceURI[]) {\n return queuesDetails.map((q) => {\n const fullUri = q.uri;\n if (!fullUri) {\n throw new Error(\"Empty or null connection string\");\n }\n // chop off sas\n const indexOfSas = q.uri.indexOf(\"?\");\n const name = indexOfSas > 0 ? q.uri.substring(0, indexOfSas) : q.uri;\n return new QueueDetails(name, new QueueClient(fullUri));\n });\n }\n\n async isEmpty() {\n const result = await this.peek(1, { raw: true });\n return !result || result.length === 0;\n }\n\n decodeContent(content: string) {\n return Buffer.from(content, \"base64\").toString(\"ascii\");\n }\n\n deserializeMessage(m: PeekedMessageItem): StatusMessage {\n return new this.messageCls(this.decodeContent(m.messageText), null, null);\n }\n\n async _peek(qs: QueueDetails[], n: number, options: PeekParams | null): Promise<{ result: Message[]; nonEmptyQs: QueueDetails[]; done: boolean }> {\n const result: Message[] = [];\n const nonEmptyQs: QueueDetails[] = [];\n\n for (const q of qs) {\n const response = await q.service.peekMessages();\n const messages = response.peekedMessageItems;\n\n if (messages && messages.length > 0) {\n nonEmptyQs.push(q);\n }\n\n for (const m of messages) {\n if (m && Object.keys(m).length > 0) {\n result.push(options && options.raw ? m : this.deserializeMessage(m));\n\n if (result.length === n) {\n return { done: true, nonEmptyQs, result };\n }\n }\n }\n }\n return { done: nonEmptyQs.length === 0, nonEmptyQs, result };\n }\n\n async peek(n = 1, options: PeekParams | null = null): Promise<Message[]> {\n const queues = await this.getQueuesFunc();\n const qServices: QueueDetails[] = shuffle(this._getQServices(queues));\n const perQ = qServices.length > 1 ? Math.floor(n / qServices.length) : qServices.length;\n\n // First, iterate evenly and randomly on status queues\n const partial = await this._peek(qServices, perQ, options);\n\n if (partial.done) {\n return partial.result;\n }\n const messagesLeftToPeek = n - partial.result.length;\n\n // In case queues are uneven, iterate again. This time, request for all n messages and trim\n return (await this._peek(partial.nonEmptyQs, messagesLeftToPeek, options)).result;\n }\n\n async _pop(\n qs: QueueDetails[],\n n: number,\n options: PopParams | null\n ): Promise<{ result: Message[] & { nonEmptyQs?: QueueDetails[] }; nonEmptyQs: any[]; done: boolean }> {\n const nonEmptyQs: any[] = [];\n const result = [];\n\n for (const q of qs) {\n const response = await q.service.receiveMessages({ numOfMessages: n });\n const messages = response.receivedMessageItems;\n for (const m of messages) {\n if (m && Object.keys(m).length > 0) {\n result.push(options && options.raw ? m : this.deserializeMessage(m));\n\n if (!(options && !options.remove)) {\n await q.service.deleteMessage(m.messageId, m.popReceipt);\n }\n if (result.length === n) {\n return { done: true, nonEmptyQs, result };\n }\n }\n }\n }\n\n return { done: nonEmptyQs.length === 0, nonEmptyQs, result };\n }\n\n async pop(n = 1, options: PopParams | null = null): Promise<Message[]> {\n const queues = await this.getQueuesFunc();\n const qServices = shuffle(this._getQServices(queues));\n const perQ = qServices.length > 1 ? Math.floor(n / qServices.length) : qServices.length;\n\n // First, iterate evenly and randomly on status queues\n const partial = await this._pop(qServices, perQ, options);\n if (partial.done) {\n return partial.result;\n }\n\n const messagesLeftToPop = n - partial.result.length;\n\n // In case queues are uneven, iterate again. This time, request for all n messages and trim\n const final = await this._pop(partial.result.nonEmptyQs ?? [], messagesLeftToPop, options);\n return partial.result.concat(final.result);\n }\n}\n"]}
|
|
1
|
+
{"version":3,"file":"statusQ.js","sourceRoot":"","sources":["../../src/statusQ.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAElC,OAAO,EAAqB,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAItE,MAAM,YAAY;IACd,YACa,IAAY,EACZ,OAAoB;QADpB,SAAI,GAAJ,IAAI,CAAQ;QACZ,YAAO,GAAP,OAAO,CAAa;IAC9B,CAAC;CACP;AAED,MAAM,OAAO,GAAG,CAAI,CAAM,EAAO,EAAE;IAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC9C,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACZ,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAChB,CAAC;IAED,OAAO,CAAC,CAAC;AACb,CAAC,CAAC;AAaF,MAAM,OAAO,WAAW;IACpB,YACa,aAA2C,EAC3C,UAAgC;QADhC,kBAAa,GAAb,aAAa,CAA8B;QAC3C,eAAU,GAAV,UAAU,CAAsB;IAC1C,CAAC;IAEJ,aAAa,CAAC,aAA4B;QACtC,OAAO,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YAC3B,MAAM,OAAO,GAAG,CAAC,CAAC,GAAG,CAAC;YACtB,IAAI,CAAC,OAAO,EAAE,CAAC;gBACX,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;YACvD,CAAC;YACD,eAAe;YACf,MAAM,UAAU,GAAG,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,IAAI,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;YACrE,OAAO,IAAI,YAAY,CAAC,IAAI,EAAE,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;IACP,CAAC;IAED,KAAK,CAAC,OAAO;QACT,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;QACjD,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC;IAC1C,CAAC;IAED,aAAa,CAAC,OAAe;QACzB,OAAO,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC5D,CAAC;IAED,kBAAkB,CAAC,CAAoB;QACnC,OAAO,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;IAC9E,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,EAAkB,EAAE,CAAS,EAAE,OAA0B;QACjE,MAAM,MAAM,GAAc,EAAE,CAAC;QAC7B,MAAM,UAAU,GAAmB,EAAE,CAAC;QAEtC,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;YACjB,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;YAChD,MAAM,QAAQ,GAAG,QAAQ,CAAC,kBAAkB,CAAC;YAE7C,IAAI,QAAQ,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACvB,CAAC;YAED,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;gBACvB,IAAI,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACjC,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC;oBAErE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;wBACtB,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC;oBAC9C,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC;IACjE,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,UAA6B,IAAI;QAC/C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC1C,MAAM,SAAS,GAAmB,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;QACtE,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC;QAExF,sDAAsD;QACtD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;QAE3D,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACf,OAAO,OAAO,CAAC,MAAM,CAAC;QAC1B,CAAC;QACD,MAAM,kBAAkB,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;QAErD,2FAA2F;QAC3F,OAAO,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,kBAAkB,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;IACtF,CAAC;IAED,KAAK,CAAC,IAAI,CACN,EAAkB,EAClB,CAAS,EACT,OAAyB;QAEzB,MAAM,UAAU,GAAU,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,EAAE,CAAC;QAElB,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;YACjB,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC,CAAC;YACvE,MAAM,QAAQ,GAAG,QAAQ,CAAC,oBAAoB,CAAC;YAC/C,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;gBACvB,IAAI,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACjC,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC;oBAErE,IAAI,CAAC,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;wBAChC,MAAM,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC;oBAC7D,CAAC;oBACD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;wBACtB,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC;oBAC9C,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC;IACjE,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,UAA4B,IAAI;;QAC7C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;QACtD,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC;QAExF,sDAAsD;QACtD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;QAC1D,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACf,OAAO,OAAO,CAAC,MAAM,CAAC;QAC1B,CAAC;QAED,MAAM,iBAAiB,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;QAEpD,2FAA2F;QAC3F,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,MAAA,OAAO,CAAC,MAAM,CAAC,UAAU,mCAAI,EAAE,EAAE,iBAAiB,EAAE,OAAO,CAAC,CAAC;QAC3F,OAAO,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC/C,CAAC;CACJ","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport { PeekedMessageItem, QueueClient } from \"@azure/storage-queue\";\nimport { ResourceURI } from \"./resourceManager.js\";\nimport { StatusMessage } from \"./status.js\";\n\nclass QueueDetails {\n constructor(\n readonly name: string,\n readonly service: QueueClient,\n ) {}\n}\n\nconst shuffle = <T>(a: T[]): T[] => {\n for (let i = a.length - 1; i > 0; i--) {\n const j = Math.floor(Math.random() * (i + 1));\n const temp = a[j];\n a[j] = a[i];\n a[i] = temp;\n }\n\n return a;\n};\n\ninterface PeekParams {\n raw: boolean;\n}\n\ninterface PopParams {\n raw: boolean;\n remove: boolean;\n}\n\ntype Message = PeekedMessageItem | StatusMessage;\n\nexport class StatusQueue {\n constructor(\n readonly getQueuesFunc: () => Promise<ResourceURI[]>,\n readonly messageCls: typeof StatusMessage,\n ) {}\n\n _getQServices(queuesDetails: ResourceURI[]) {\n return queuesDetails.map((q) => {\n const fullUri = q.uri;\n if (!fullUri) {\n throw new Error(\"Empty or null connection string\");\n }\n // chop off sas\n const indexOfSas = q.uri.indexOf(\"?\");\n const name = indexOfSas > 0 ? q.uri.substring(0, indexOfSas) : q.uri;\n return new QueueDetails(name, new QueueClient(fullUri));\n });\n }\n\n async isEmpty() {\n const result = await this.peek(1, { raw: true });\n return !result || result.length === 0;\n }\n\n decodeContent(content: string) {\n return Buffer.from(content, \"base64\").toString(\"ascii\");\n }\n\n deserializeMessage(m: PeekedMessageItem): StatusMessage {\n return new this.messageCls(this.decodeContent(m.messageText), null, null);\n }\n\n async _peek(qs: QueueDetails[], n: number, options: PeekParams | null): Promise<{ result: Message[]; nonEmptyQs: QueueDetails[]; done: boolean }> {\n const result: Message[] = [];\n const nonEmptyQs: QueueDetails[] = [];\n\n for (const q of qs) {\n const response = await q.service.peekMessages();\n const messages = response.peekedMessageItems;\n\n if (messages && messages.length > 0) {\n nonEmptyQs.push(q);\n }\n\n for (const m of messages) {\n if (m && Object.keys(m).length > 0) {\n result.push(options && options.raw ? m : this.deserializeMessage(m));\n\n if (result.length === n) {\n return { done: true, nonEmptyQs, result };\n }\n }\n }\n }\n return { done: nonEmptyQs.length === 0, nonEmptyQs, result };\n }\n\n async peek(n = 1, options: PeekParams | null = null): Promise<Message[]> {\n const queues = await this.getQueuesFunc();\n const qServices: QueueDetails[] = shuffle(this._getQServices(queues));\n const perQ = qServices.length > 1 ? Math.floor(n / qServices.length) : qServices.length;\n\n // First, iterate evenly and randomly on status queues\n const partial = await this._peek(qServices, perQ, options);\n\n if (partial.done) {\n return partial.result;\n }\n const messagesLeftToPeek = n - partial.result.length;\n\n // In case queues are uneven, iterate again. This time, request for all n messages and trim\n return (await this._peek(partial.nonEmptyQs, messagesLeftToPeek, options)).result;\n }\n\n async _pop(\n qs: QueueDetails[],\n n: number,\n options: PopParams | null,\n ): Promise<{ result: Message[] & { nonEmptyQs?: QueueDetails[] }; nonEmptyQs: any[]; done: boolean }> {\n const nonEmptyQs: any[] = [];\n const result = [];\n\n for (const q of qs) {\n const response = await q.service.receiveMessages({ numOfMessages: n });\n const messages = response.receivedMessageItems;\n for (const m of messages) {\n if (m && Object.keys(m).length > 0) {\n result.push(options && options.raw ? m : this.deserializeMessage(m));\n\n if (!(options && !options.remove)) {\n await q.service.deleteMessage(m.messageId, m.popReceipt);\n }\n if (result.length === n) {\n return { done: true, nonEmptyQs, result };\n }\n }\n }\n }\n\n return { done: nonEmptyQs.length === 0, nonEmptyQs, result };\n }\n\n async pop(n = 1, options: PopParams | null = null): Promise<Message[]> {\n const queues = await this.getQueuesFunc();\n const qServices = shuffle(this._getQServices(queues));\n const perQ = qServices.length > 1 ? Math.floor(n / qServices.length) : qServices.length;\n\n // First, iterate evenly and randomly on status queues\n const partial = await this._pop(qServices, perQ, options);\n if (partial.done) {\n return partial.result;\n }\n\n const messagesLeftToPop = n - partial.result.length;\n\n // In case queues are uneven, iterate again. This time, request for all n messages and trim\n const final = await this._pop(partial.result.nonEmptyQs ?? [], messagesLeftToPop, options);\n return partial.result.concat(final.result);\n }\n}\n"]}
|
|
@@ -1,27 +1,20 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
// Copyright (c) Microsoft Corporation.
|
|
3
2
|
// Licensed under the MIT License.
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
const descriptors_1 = require("./descriptors");
|
|
7
|
-
const fileToStream = async (file) => {
|
|
3
|
+
import { CompressionType, StreamDescriptor } from "./descriptors.js";
|
|
4
|
+
export const fileToStream = async (file) => {
|
|
8
5
|
const streamFs = await file.file.arrayBuffer();
|
|
9
|
-
const compressionType = file.zipped ?
|
|
10
|
-
return new
|
|
6
|
+
const compressionType = file.zipped ? CompressionType.GZIP : CompressionType.None;
|
|
7
|
+
return new StreamDescriptor(streamFs, file.sourceId, compressionType);
|
|
11
8
|
};
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
return await (0, exports.fileToStream)(file);
|
|
9
|
+
export const tryFileToBuffer = async (file) => {
|
|
10
|
+
return await fileToStream(file);
|
|
15
11
|
};
|
|
16
|
-
exports.tryFileToBuffer = tryFileToBuffer;
|
|
17
12
|
// NOT USED
|
|
18
|
-
const tryStreamToArray = async (stream) => {
|
|
13
|
+
export const tryStreamToArray = async (stream) => {
|
|
19
14
|
return Promise.resolve(stream);
|
|
20
15
|
};
|
|
21
|
-
exports.tryStreamToArray = tryStreamToArray;
|
|
22
16
|
// NOT USED
|
|
23
|
-
const readableToStream = (stream) => {
|
|
17
|
+
export const readableToStream = (stream) => {
|
|
24
18
|
return stream;
|
|
25
19
|
};
|
|
26
|
-
exports.readableToStream = readableToStream;
|
|
27
20
|
//# sourceMappingURL=streamUtils.browser.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streamUtils.browser.js","sourceRoot":"","sources":["../../src/streamUtils.browser.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"streamUtils.browser.js","sourceRoot":"","sources":["../../src/streamUtils.browser.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAElC,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAGrE,MAAM,CAAC,MAAM,YAAY,GAAG,KAAK,EAAE,IAAoB,EAA6B,EAAE;IAClF,MAAM,QAAQ,GAAG,MAAO,IAAI,CAAC,IAAa,CAAC,WAAW,EAAE,CAAC;IACzD,MAAM,eAAe,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC;IAClF,OAAO,IAAI,gBAAgB,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;AAC1E,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,EAAE,IAAoB,EAA6B,EAAE;IACrF,OAAO,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC;AACpC,CAAC,CAAC;AAEF,WAAW;AACX,MAAM,CAAC,MAAM,gBAAgB,GAAG,KAAK,EAAE,MAAW,EAAmB,EAAE;IACnE,OAAO,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;AACnC,CAAC,CAAC;AAEF,WAAW;AACX,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,MAAmB,EAAe,EAAE;IACjE,OAAO,MAAM,CAAC;AAClB,CAAC,CAAC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport { CompressionType, StreamDescriptor } from \"./descriptors.js\";\nimport { FileDescriptor } from \"./fileDescriptor.browser.js\";\n\nexport const fileToStream = async (file: FileDescriptor): Promise<StreamDescriptor> => {\n const streamFs = await (file.file as Blob).arrayBuffer();\n const compressionType = file.zipped ? CompressionType.GZIP : CompressionType.None;\n return new StreamDescriptor(streamFs, file.sourceId, compressionType);\n};\n\nexport const tryFileToBuffer = async (file: FileDescriptor): Promise<StreamDescriptor> => {\n return await fileToStream(file);\n};\n\n// NOT USED\nexport const tryStreamToArray = async (stream: any): Promise<Buffer> => {\n return Promise.resolve(stream);\n};\n\n// NOT USED\nexport const readableToStream = (stream: ArrayBuffer): ArrayBuffer => {\n return stream;\n};\n"]}
|
|
@@ -1,36 +1,28 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
// Copyright (c) Microsoft Corporation.
|
|
3
2
|
// Licensed under the MIT License.
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
exports.readableToStream = exports.tryStreamToArray = exports.tryFileToBuffer = exports.fileToStream = void 0;
|
|
9
|
-
const fs_1 = __importDefault(require("fs"));
|
|
10
|
-
const stream_1 = require("stream");
|
|
11
|
-
const stream_array_1 = __importDefault(require("stream-array"));
|
|
12
|
-
const descriptors_1 = require("./descriptors");
|
|
3
|
+
import fs from "fs";
|
|
4
|
+
import { PassThrough } from "stream";
|
|
5
|
+
import streamify from "stream-array";
|
|
6
|
+
import { CompressionType, StreamDescriptor } from "./descriptors.js";
|
|
13
7
|
// Returns fs.ReadStream for node and NodeJS.ReadableStream in browser
|
|
14
|
-
const fileToStream = (fileDescriptor) => {
|
|
15
|
-
const streamFs =
|
|
16
|
-
const compressionType = fileDescriptor.zipped ?
|
|
17
|
-
return Promise.resolve(new
|
|
8
|
+
export const fileToStream = (fileDescriptor) => {
|
|
9
|
+
const streamFs = fs.createReadStream(fileDescriptor.file);
|
|
10
|
+
const compressionType = fileDescriptor.zipped ? CompressionType.GZIP : CompressionType.None;
|
|
11
|
+
return Promise.resolve(new StreamDescriptor(streamFs, fileDescriptor.sourceId, compressionType));
|
|
18
12
|
};
|
|
19
|
-
exports.fileToStream = fileToStream;
|
|
20
13
|
// Used in managed streaming where we buffer the file to memory for retries
|
|
21
|
-
const tryFileToBuffer = async (fileDescriptor) => {
|
|
14
|
+
export const tryFileToBuffer = async (fileDescriptor) => {
|
|
22
15
|
try {
|
|
23
|
-
const buffer =
|
|
24
|
-
const compressionType = fileDescriptor.zipped ?
|
|
25
|
-
return new
|
|
16
|
+
const buffer = fs.readFileSync(fileDescriptor.file);
|
|
17
|
+
const compressionType = fileDescriptor.zipped ? CompressionType.GZIP : CompressionType.None;
|
|
18
|
+
return new StreamDescriptor(buffer, fileDescriptor.sourceId, compressionType);
|
|
26
19
|
}
|
|
27
20
|
catch (error) {
|
|
28
|
-
return await
|
|
21
|
+
return await fileToStream(fileDescriptor);
|
|
29
22
|
}
|
|
30
23
|
};
|
|
31
|
-
exports.tryFileToBuffer = tryFileToBuffer;
|
|
32
24
|
const mergeStreams = (...streams) => {
|
|
33
|
-
let pass = new
|
|
25
|
+
let pass = new PassThrough();
|
|
34
26
|
let waiting = streams.length;
|
|
35
27
|
for (const stream of streams) {
|
|
36
28
|
pass = stream.pipe(pass, { end: false });
|
|
@@ -38,7 +30,7 @@ const mergeStreams = (...streams) => {
|
|
|
38
30
|
}
|
|
39
31
|
return pass;
|
|
40
32
|
};
|
|
41
|
-
const tryStreamToArray = async (stream, maxBufferSize) => {
|
|
33
|
+
export const tryStreamToArray = async (stream, maxBufferSize) => {
|
|
42
34
|
if (stream instanceof Buffer) {
|
|
43
35
|
return stream;
|
|
44
36
|
}
|
|
@@ -51,7 +43,7 @@ const tryStreamToArray = async (stream, maxBufferSize) => {
|
|
|
51
43
|
if (result.reduce((sum, b) => sum + b.length, 0) > maxBufferSize) {
|
|
52
44
|
stream.removeListener("data", dataHandler);
|
|
53
45
|
stream.removeListener("end", endListener);
|
|
54
|
-
resolve(mergeStreams((
|
|
46
|
+
resolve(mergeStreams(streamify(result), stream));
|
|
55
47
|
}
|
|
56
48
|
}
|
|
57
49
|
catch (e) {
|
|
@@ -62,9 +54,7 @@ const tryStreamToArray = async (stream, maxBufferSize) => {
|
|
|
62
54
|
stream.on("end", endListener);
|
|
63
55
|
});
|
|
64
56
|
};
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
return (0, stream_array_1.default)([stream]);
|
|
57
|
+
export const readableToStream = (stream) => {
|
|
58
|
+
return streamify([stream]);
|
|
68
59
|
};
|
|
69
|
-
exports.readableToStream = readableToStream;
|
|
70
60
|
//# sourceMappingURL=streamUtils.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streamUtils.js","sourceRoot":"","sources":["../../src/streamUtils.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"streamUtils.js","sourceRoot":"","sources":["../../src/streamUtils.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAElC,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,WAAW,EAAY,MAAM,QAAQ,CAAC;AAC/C,OAAO,SAAS,MAAM,cAAc,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAGrE,sEAAsE;AACtE,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,cAA8B,EAA6B,EAAE;IACtF,MAAM,QAAQ,GAAG,EAAE,CAAC,gBAAgB,CAAC,cAAc,CAAC,IAAc,CAAC,CAAC;IACpE,MAAM,eAAe,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC;IAC5F,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,gBAAgB,CAAC,QAAQ,EAAE,cAAc,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC,CAAC;AACrG,CAAC,CAAC;AAEF,2EAA2E;AAC3E,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,EAAE,cAA8B,EAA6B,EAAE;IAC/F,IAAI,CAAC;QACD,MAAM,MAAM,GAAG,EAAE,CAAC,YAAY,CAAC,cAAc,CAAC,IAAc,CAAC,CAAC;QAC9D,MAAM,eAAe,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC;QAC5F,OAAO,IAAI,gBAAgB,CAAC,MAAM,EAAE,cAAc,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;IAClF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,OAAO,MAAM,YAAY,CAAC,cAAc,CAAC,CAAC;IAC9C,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,GAAG,OAAmB,EAAY,EAAE;IACtD,IAAI,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC;IAC7B,IAAI,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC;IAC7B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC3B,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;QACzC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAClE,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,gBAAgB,GAAG,KAAK,EAAE,MAAgB,EAAE,aAAqB,EAA8B,EAAE;IAC1G,IAAI,MAAM,YAAY,MAAM,EAAE,CAAC;QAC3B,OAAO,MAAM,CAAC;IAClB,CAAC;IACD,OAAO,MAAM,IAAI,OAAO,CAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC5D,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,WAAW,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QACzD,MAAM,WAAW,GAAG,CAAC,KAAa,EAAE,EAAE;YAClC,IAAI,CAAC;gBACD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACnB,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,aAAa,EAAE,CAAC;oBAC/D,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;oBAC3C,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;oBAC1C,OAAO,CAAC,YAAY,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;gBACrD,CAAC;YACL,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,MAAM,CAAC,CAAC,CAAC,CAAC;YACd,CAAC;QACL,CAAC,CAAC;QACF,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAC/B,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;AACP,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,MAA8B,EAAY,EAAE;IACzE,OAAO,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;AAC/B,CAAC,CAAC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport fs from \"fs\";\nimport { PassThrough, Readable } from \"stream\";\nimport streamify from \"stream-array\";\nimport { CompressionType, StreamDescriptor } from \"./descriptors.js\";\nimport { FileDescriptor } from \"./fileDescriptor.js\";\n\n// Returns fs.ReadStream for node and NodeJS.ReadableStream in browser\nexport const fileToStream = (fileDescriptor: FileDescriptor): Promise<StreamDescriptor> => {\n const streamFs = fs.createReadStream(fileDescriptor.file as string);\n const compressionType = fileDescriptor.zipped ? CompressionType.GZIP : CompressionType.None;\n return Promise.resolve(new StreamDescriptor(streamFs, fileDescriptor.sourceId, compressionType));\n};\n\n// Used in managed streaming where we buffer the file to memory for retries\nexport const tryFileToBuffer = async (fileDescriptor: FileDescriptor): Promise<StreamDescriptor> => {\n try {\n const buffer = fs.readFileSync(fileDescriptor.file as string);\n const compressionType = fileDescriptor.zipped ? CompressionType.GZIP : CompressionType.None;\n return new StreamDescriptor(buffer, fileDescriptor.sourceId, compressionType);\n } catch (error) {\n return await fileToStream(fileDescriptor);\n }\n};\n\nconst mergeStreams = (...streams: Readable[]): Readable => {\n let pass = new PassThrough();\n let waiting = streams.length;\n for (const stream of streams) {\n pass = stream.pipe(pass, { end: false });\n stream.once(\"end\", () => --waiting === 0 && pass.emit(\"end\"));\n }\n return pass;\n};\n\nexport const tryStreamToArray = async (stream: Readable, maxBufferSize: number): Promise<Buffer | Readable> => {\n if (stream instanceof Buffer) {\n return stream;\n }\n return await new Promise<Buffer | Readable>((resolve, reject) => {\n const result: Buffer[] = [];\n const endListener = () => resolve(Buffer.concat(result));\n const dataHandler = (chunk: Buffer) => {\n try {\n result.push(chunk);\n if (result.reduce((sum, b) => sum + b.length, 0) > maxBufferSize) {\n stream.removeListener(\"data\", dataHandler);\n stream.removeListener(\"end\", endListener);\n resolve(mergeStreams(streamify(result), stream));\n }\n } catch (e) {\n reject(e);\n }\n };\n stream.on(\"data\", dataHandler);\n stream.on(\"end\", endListener);\n });\n};\n\nexport const readableToStream = (stream: Readable | ArrayBuffer): Readable => {\n return streamify([stream]);\n};\n"]}
|
|
@@ -1,12 +1,10 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
// Copyright (c) Microsoft Corporation.
|
|
3
2
|
// Licensed under the MIT License.
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
class KustoStreamingIngestClient extends streamingIngestClientBase_1.KustoStreamingIngestClientBase {
|
|
3
|
+
import { StreamDescriptor } from "./descriptors.js";
|
|
4
|
+
import { FileDescriptor } from "./fileDescriptor.browser.js";
|
|
5
|
+
import { tryFileToBuffer } from "./streamUtils.browser.js";
|
|
6
|
+
import { KustoStreamingIngestClientBase } from "./streamingIngestClientBase.js";
|
|
7
|
+
class KustoStreamingIngestClient extends KustoStreamingIngestClientBase {
|
|
10
8
|
constructor(kcsb, defaultProps, autoCorrectEndpoint) {
|
|
11
9
|
super(kcsb, defaultProps, autoCorrectEndpoint);
|
|
12
10
|
}
|
|
@@ -17,7 +15,7 @@ class KustoStreamingIngestClient extends streamingIngestClientBase_1.KustoStream
|
|
|
17
15
|
var _a;
|
|
18
16
|
this.ensureOpen();
|
|
19
17
|
const props = this._getMergedProps(ingestionProperties);
|
|
20
|
-
const descriptor = stream instanceof
|
|
18
|
+
const descriptor = stream instanceof StreamDescriptor ? stream : new StreamDescriptor(stream);
|
|
21
19
|
return await this.kustoClient.executeStreamingIngest(props.database, props.table, descriptor.stream, props.format, (_a = props.ingestionMappingReference) !== null && _a !== void 0 ? _a : null, clientRequestId);
|
|
22
20
|
}
|
|
23
21
|
/**
|
|
@@ -25,9 +23,9 @@ class KustoStreamingIngestClient extends streamingIngestClientBase_1.KustoStream
|
|
|
25
23
|
*/
|
|
26
24
|
async ingestFromFile(file, ingestionProperties) {
|
|
27
25
|
this.ensureOpen();
|
|
28
|
-
const descriptor = file instanceof
|
|
29
|
-
return this.ingestFromStream(await
|
|
26
|
+
const descriptor = file instanceof FileDescriptor ? file : new FileDescriptor(file);
|
|
27
|
+
return this.ingestFromStream(await tryFileToBuffer(descriptor), ingestionProperties);
|
|
30
28
|
}
|
|
31
29
|
}
|
|
32
|
-
|
|
30
|
+
export default KustoStreamingIngestClient;
|
|
33
31
|
//# sourceMappingURL=streamingIngestClient.browser.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streamingIngestClient.browser.js","sourceRoot":"","sources":["../../src/streamingIngestClient.browser.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"streamingIngestClient.browser.js","sourceRoot":"","sources":["../../src/streamingIngestClient.browser.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAKlC,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,8BAA8B,EAAE,MAAM,gCAAgC,CAAC;AAEhF,MAAM,0BAA2B,SAAQ,8BAA8B;IACnE,YAAY,IAA2C,EAAE,YAAuC,EAAE,mBAA6B;QAC3H,KAAK,CAAC,IAAI,EAAE,YAAY,EAAE,mBAAmB,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB,CAAC,MAAsC,EAAE,mBAA8C,EAAE,eAAwB;;QACnI,IAAI,CAAC,UAAU,EAAE,CAAC;QAElB,MAAM,KAAK,GAAG,IAAI,CAAC,eAAe,CAAC,mBAAmB,CAAC,CAAC;QACxD,MAAM,UAAU,GAAqB,MAAM,YAAY,gBAAgB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAChH,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,sBAAsB,CAChD,KAAK,CAAC,QAAkB,EACxB,KAAK,CAAC,KAAe,EACrB,UAAU,CAAC,MAAM,EACjB,KAAK,CAAC,MAAM,EACZ,MAAA,KAAK,CAAC,yBAAyB,mCAAI,IAAI,EACvC,eAAe,CAClB,CAAC;IACN,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,IAA2B,EAAE,mBAA8C;QAC5F,IAAI,CAAC,UAAU,EAAE,CAAC;QAElB,MAAM,UAAU,GAAmB,IAAI,YAAY,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC;QACpG,OAAO,IAAI,CAAC,gBAAgB,CAAC,MAAM,eAAe,CAAC,UAAU,CAAC,EAAE,mBAAmB,CAAC,CAAC;IACzF,CAAC;CACJ;AAED,eAAe,0BAA0B,CAAC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport { IngestionPropertiesInput } from \"./ingestionProperties.js\";\n\nimport { KustoConnectionStringBuilder, KustoResponseDataSet } from \"azure-kusto-data\";\nimport { StreamDescriptor } from \"./descriptors.js\";\nimport { FileDescriptor } from \"./fileDescriptor.browser.js\";\nimport { tryFileToBuffer } from \"./streamUtils.browser.js\";\nimport { KustoStreamingIngestClientBase } from \"./streamingIngestClientBase.js\";\n\nclass KustoStreamingIngestClient extends KustoStreamingIngestClientBase {\n constructor(kcsb: string | KustoConnectionStringBuilder, defaultProps?: IngestionPropertiesInput, autoCorrectEndpoint?: boolean) {\n super(kcsb, defaultProps, autoCorrectEndpoint);\n }\n\n /**\n * Use Readable for Node.js and ArrayBuffer in browser\n */\n async ingestFromStream(stream: StreamDescriptor | ArrayBuffer, ingestionProperties?: IngestionPropertiesInput, clientRequestId?: string): Promise<any> {\n this.ensureOpen();\n\n const props = this._getMergedProps(ingestionProperties);\n const descriptor: StreamDescriptor = stream instanceof StreamDescriptor ? stream : new StreamDescriptor(stream);\n return await this.kustoClient.executeStreamingIngest(\n props.database as string,\n props.table as string,\n descriptor.stream,\n props.format,\n props.ingestionMappingReference ?? null,\n clientRequestId,\n );\n }\n\n /**\n * Use string for Node.js and Blob in browser\n */\n async ingestFromFile(file: FileDescriptor | Blob, ingestionProperties?: IngestionPropertiesInput): Promise<KustoResponseDataSet> {\n this.ensureOpen();\n\n const descriptor: FileDescriptor = file instanceof FileDescriptor ? file : new FileDescriptor(file);\n return this.ingestFromStream(await tryFileToBuffer(descriptor), ingestionProperties);\n }\n}\n\nexport default KustoStreamingIngestClient;\n"]}
|
|
@@ -1,16 +1,11 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
// Copyright (c) Microsoft Corporation.
|
|
3
2
|
// Licensed under the MIT License.
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
};
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
const fileDescriptor_1 = require("./fileDescriptor");
|
|
11
|
-
const streamUtils_1 = require("./streamUtils");
|
|
12
|
-
const streamingIngestClientBase_1 = require("./streamingIngestClientBase");
|
|
13
|
-
class KustoStreamingIngestClient extends streamingIngestClientBase_1.KustoStreamingIngestClientBase {
|
|
3
|
+
import zlib from "zlib";
|
|
4
|
+
import { CompressionType, StreamDescriptor } from "./descriptors.js";
|
|
5
|
+
import { FileDescriptor } from "./fileDescriptor.js";
|
|
6
|
+
import { fileToStream } from "./streamUtils.js";
|
|
7
|
+
import { KustoStreamingIngestClientBase } from "./streamingIngestClientBase.js";
|
|
8
|
+
class KustoStreamingIngestClient extends KustoStreamingIngestClientBase {
|
|
14
9
|
constructor(kcsb, defaultProps, autoCorrectEndpoint) {
|
|
15
10
|
super(kcsb, defaultProps, autoCorrectEndpoint);
|
|
16
11
|
}
|
|
@@ -21,10 +16,10 @@ class KustoStreamingIngestClient extends streamingIngestClientBase_1.KustoStream
|
|
|
21
16
|
var _a;
|
|
22
17
|
this.ensureOpen();
|
|
23
18
|
const props = this._getMergedProps(ingestionProperties);
|
|
24
|
-
const descriptor = stream instanceof
|
|
25
|
-
const compressedStream = descriptor.compressionType ===
|
|
19
|
+
const descriptor = stream instanceof StreamDescriptor ? stream : new StreamDescriptor(stream);
|
|
20
|
+
const compressedStream = descriptor.compressionType === CompressionType.None
|
|
26
21
|
? !(descriptor.stream instanceof ArrayBuffer)
|
|
27
|
-
? descriptor.stream.pipe(
|
|
22
|
+
? descriptor.stream.pipe(zlib.createGzip())
|
|
28
23
|
: descriptor.stream
|
|
29
24
|
: descriptor.stream;
|
|
30
25
|
return await this.kustoClient.executeStreamingIngest(props.database, props.table, compressedStream, props.format, (_a = props.ingestionMappingReference) !== null && _a !== void 0 ? _a : null, clientRequestId);
|
|
@@ -34,9 +29,9 @@ class KustoStreamingIngestClient extends streamingIngestClientBase_1.KustoStream
|
|
|
34
29
|
*/
|
|
35
30
|
async ingestFromFile(file, ingestionProperties) {
|
|
36
31
|
this.ensureOpen();
|
|
37
|
-
const descriptor = file instanceof
|
|
38
|
-
return this.ingestFromStream(await
|
|
32
|
+
const descriptor = file instanceof FileDescriptor ? file : new FileDescriptor(file);
|
|
33
|
+
return this.ingestFromStream(await fileToStream(descriptor), ingestionProperties);
|
|
39
34
|
}
|
|
40
35
|
}
|
|
41
|
-
|
|
36
|
+
export default KustoStreamingIngestClient;
|
|
42
37
|
//# sourceMappingURL=streamingIngestClient.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streamingIngestClient.js","sourceRoot":"","sources":["../../src/streamingIngestClient.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"streamingIngestClient.js","sourceRoot":"","sources":["../../src/streamingIngestClient.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAMlC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACrE,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,8BAA8B,EAAE,MAAM,gCAAgC,CAAC;AAEhF,MAAM,0BAA2B,SAAQ,8BAA8B;IACnE,YAAY,IAA2C,EAAE,YAAuC,EAAE,mBAA6B;QAC3H,KAAK,CAAC,IAAI,EAAE,YAAY,EAAE,mBAAmB,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB,CAClB,MAAiD,EACjD,mBAA8C,EAC9C,eAAwB;;QAExB,IAAI,CAAC,UAAU,EAAE,CAAC;QAElB,MAAM,KAAK,GAAG,IAAI,CAAC,eAAe,CAAC,mBAAmB,CAAC,CAAC;QACxD,MAAM,UAAU,GAAqB,MAAM,YAAY,gBAAgB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAEhH,MAAM,gBAAgB,GAClB,UAAU,CAAC,eAAe,KAAK,eAAe,CAAC,IAAI;YAC/C,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,YAAY,WAAW,CAAC;gBACzC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC3C,CAAC,CAAC,UAAU,CAAC,MAAM;YACvB,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QAC5B,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,sBAAsB,CAChD,KAAK,CAAC,QAAkB,EACxB,KAAK,CAAC,KAAe,EACrB,gBAAgB,EAChB,KAAK,CAAC,MAAM,EACZ,MAAA,KAAK,CAAC,yBAAyB,mCAAI,IAAI,EACvC,eAAe,CAClB,CAAC;IACN,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,IAAoC,EAAE,mBAA8C;QACrG,IAAI,CAAC,UAAU,EAAE,CAAC;QAElB,MAAM,UAAU,GAAmB,IAAI,YAAY,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC;QACpG,OAAO,IAAI,CAAC,gBAAgB,CAAC,MAAM,YAAY,CAAC,UAAU,CAAC,EAAE,mBAAmB,CAAC,CAAC;IACtF,CAAC;CACJ;AAED,eAAe,0BAA0B,CAAC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport { IngestionPropertiesInput } from \"./ingestionProperties.js\";\n\nimport { KustoConnectionStringBuilder, KustoResponseDataSet } from \"azure-kusto-data\";\nimport { Readable } from \"stream\";\nimport zlib from \"zlib\";\nimport { CompressionType, StreamDescriptor } from \"./descriptors.js\";\nimport { FileDescriptor } from \"./fileDescriptor.js\";\nimport { fileToStream } from \"./streamUtils.js\";\nimport { KustoStreamingIngestClientBase } from \"./streamingIngestClientBase.js\";\n\nclass KustoStreamingIngestClient extends KustoStreamingIngestClientBase {\n constructor(kcsb: string | KustoConnectionStringBuilder, defaultProps?: IngestionPropertiesInput, autoCorrectEndpoint?: boolean) {\n super(kcsb, defaultProps, autoCorrectEndpoint);\n }\n\n /**\n * Use Readable for Node.js and ArrayBuffer in browser\n */\n async ingestFromStream(\n stream: StreamDescriptor | Readable | ArrayBuffer,\n ingestionProperties?: IngestionPropertiesInput,\n clientRequestId?: string,\n ): Promise<any> {\n this.ensureOpen();\n\n const props = this._getMergedProps(ingestionProperties);\n const descriptor: StreamDescriptor = stream instanceof StreamDescriptor ? stream : new StreamDescriptor(stream);\n\n const compressedStream =\n descriptor.compressionType === CompressionType.None\n ? !(descriptor.stream instanceof ArrayBuffer)\n ? descriptor.stream.pipe(zlib.createGzip())\n : descriptor.stream\n : descriptor.stream;\n return await this.kustoClient.executeStreamingIngest(\n props.database as string,\n props.table as string,\n compressedStream,\n props.format,\n props.ingestionMappingReference ?? null,\n clientRequestId,\n );\n }\n\n /**\n * Use string for Node.js and Blob in browser\n */\n async ingestFromFile(file: FileDescriptor | string | Blob, ingestionProperties?: IngestionPropertiesInput): Promise<KustoResponseDataSet> {\n this.ensureOpen();\n\n const descriptor: FileDescriptor = file instanceof FileDescriptor ? file : new FileDescriptor(file);\n return this.ingestFromStream(await fileToStream(descriptor), ingestionProperties);\n }\n}\n\nexport default KustoStreamingIngestClient;\n"]}
|
|
@@ -1,27 +1,24 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
// Copyright (c) Microsoft Corporation.
|
|
3
2
|
// Licensed under the MIT License.
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
const azure_kusto_data_1 = require("azure-kusto-data");
|
|
9
|
-
class KustoStreamingIngestClientBase extends abstractKustoClient_1.AbstractKustoClient {
|
|
3
|
+
import { BlobDescriptor } from "./descriptors.js";
|
|
4
|
+
import { AbstractKustoClient } from "./abstractKustoClient.js";
|
|
5
|
+
import { Client as KustoClient, KustoConnectionStringBuilder } from "azure-kusto-data";
|
|
6
|
+
export class KustoStreamingIngestClientBase extends AbstractKustoClient {
|
|
10
7
|
constructor(kcsb, defaultProps, autoCorrectEndpoint = true) {
|
|
11
8
|
super(defaultProps);
|
|
12
9
|
if (typeof kcsb === "string") {
|
|
13
|
-
kcsb = new
|
|
10
|
+
kcsb = new KustoConnectionStringBuilder(kcsb);
|
|
14
11
|
}
|
|
15
12
|
if (autoCorrectEndpoint) {
|
|
16
13
|
kcsb.dataSource = this.getQueryEndpoint(kcsb.dataSource);
|
|
17
14
|
}
|
|
18
|
-
this.kustoClient = new
|
|
15
|
+
this.kustoClient = new KustoClient(kcsb);
|
|
19
16
|
this.defaultDatabase = this.kustoClient.defaultDatabase;
|
|
20
17
|
}
|
|
21
18
|
async ingestFromBlob(blob, ingestionProperties, clientRequestId) {
|
|
22
19
|
var _a;
|
|
23
20
|
const props = this._getMergedProps(ingestionProperties);
|
|
24
|
-
const descriptor = blob instanceof
|
|
21
|
+
const descriptor = blob instanceof BlobDescriptor ? blob : new BlobDescriptor(blob);
|
|
25
22
|
// No need to check blob size if it was given to us that it's not empty
|
|
26
23
|
await descriptor.fillSize();
|
|
27
24
|
return await this.kustoClient.executeStreamingIngest(props.database, props.table, undefined, props.format, (_a = props.ingestionMappingReference) !== null && _a !== void 0 ? _a : null, descriptor.path, clientRequestId);
|
|
@@ -33,5 +30,4 @@ class KustoStreamingIngestClientBase extends abstractKustoClient_1.AbstractKusto
|
|
|
33
30
|
super.close();
|
|
34
31
|
}
|
|
35
32
|
}
|
|
36
|
-
exports.KustoStreamingIngestClientBase = KustoStreamingIngestClientBase;
|
|
37
33
|
//# sourceMappingURL=streamingIngestClientBase.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streamingIngestClientBase.js","sourceRoot":"","sources":["../../src/streamingIngestClientBase.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"streamingIngestClientBase.js","sourceRoot":"","sources":["../../src/streamingIngestClientBase.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAIlC,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAC/D,OAAO,EAAE,MAAM,IAAI,WAAW,EAAE,4BAA4B,EAAE,MAAM,kBAAkB,CAAC;AAEvF,MAAM,OAAgB,8BAA+B,SAAQ,mBAAmB;IAE5E,YAAY,IAA2C,EAAE,YAAuC,EAAE,sBAA+B,IAAI;QACjI,KAAK,CAAC,YAAY,CAAC,CAAC;QACpB,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,IAAI,GAAG,IAAI,4BAA4B,CAAC,IAAI,CAAC,CAAC;QAClD,CAAC;QACD,IAAI,mBAAmB,EAAE,CAAC;YACtB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC7D,CAAC;QACD,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC;IAC5D,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,IAA6B,EAAE,mBAA8C,EAAE,eAAwB;;QACxH,MAAM,KAAK,GAAG,IAAI,CAAC,eAAe,CAAC,mBAAmB,CAAC,CAAC;QACxD,MAAM,UAAU,GAAG,IAAI,YAAY,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC;QACpF,uEAAuE;QACvE,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC;QAE5B,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,sBAAsB,CAChD,KAAK,CAAC,QAAkB,EACxB,KAAK,CAAC,KAAe,EACrB,SAAS,EACT,KAAK,CAAC,MAAM,EACZ,MAAA,KAAK,CAAC,yBAAyB,mCAAI,IAAI,EACvC,UAAU,CAAC,IAAI,EACf,eAAe,CAClB,CAAC;IACN,CAAC;IAED,KAAK;QACD,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YAClB,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QAC7B,CAAC;QACD,KAAK,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;CACJ","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport { IngestionPropertiesInput } from \"./ingestionProperties.js\";\n\nimport { BlobDescriptor } from \"./descriptors.js\";\nimport { AbstractKustoClient } from \"./abstractKustoClient.js\";\nimport { Client as KustoClient, KustoConnectionStringBuilder } from \"azure-kusto-data\";\n\nexport abstract class KustoStreamingIngestClientBase extends AbstractKustoClient {\n protected kustoClient: KustoClient;\n constructor(kcsb: string | KustoConnectionStringBuilder, defaultProps?: IngestionPropertiesInput, autoCorrectEndpoint: boolean = true) {\n super(defaultProps);\n if (typeof kcsb === \"string\") {\n kcsb = new KustoConnectionStringBuilder(kcsb);\n }\n if (autoCorrectEndpoint) {\n kcsb.dataSource = this.getQueryEndpoint(kcsb.dataSource);\n }\n this.kustoClient = new KustoClient(kcsb);\n this.defaultDatabase = this.kustoClient.defaultDatabase;\n }\n\n async ingestFromBlob(blob: string | BlobDescriptor, ingestionProperties?: IngestionPropertiesInput, clientRequestId?: string): Promise<any> {\n const props = this._getMergedProps(ingestionProperties);\n const descriptor = blob instanceof BlobDescriptor ? blob : new BlobDescriptor(blob);\n // No need to check blob size if it was given to us that it's not empty\n await descriptor.fillSize();\n\n return await this.kustoClient.executeStreamingIngest(\n props.database as string,\n props.table as string,\n undefined,\n props.format,\n props.ingestionMappingReference ?? null,\n descriptor.path,\n clientRequestId,\n );\n }\n\n close() {\n if (!this._isClosed) {\n this.kustoClient.close();\n }\n super.close();\n }\n}\n"]}
|
package/package.json
CHANGED
|
@@ -1,10 +1,15 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "azure-kusto-ingest",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "7.0.0-alpha.1",
|
|
4
4
|
"description": "Azure Data Explorer Ingestion SDK",
|
|
5
|
-
"
|
|
6
|
-
"
|
|
7
|
-
"
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "dist-esm/src/index.js",
|
|
7
|
+
"exports": {
|
|
8
|
+
".": {
|
|
9
|
+
"import": "./dist-esm/src/index.js",
|
|
10
|
+
"types": "./types/src/index.d.ts"
|
|
11
|
+
}
|
|
12
|
+
},
|
|
8
13
|
"scripts": {
|
|
9
14
|
"build": "tsc -b",
|
|
10
15
|
"clean": "rimraf dist/* dist-esm/* types/*"
|
|
@@ -64,7 +69,7 @@
|
|
|
64
69
|
"@types/tmp": "^0.2.3",
|
|
65
70
|
"@types/uuid": "^8.3.4",
|
|
66
71
|
"@types/uuid-validate": "0.0.1",
|
|
67
|
-
"azure-kusto-data": "^
|
|
72
|
+
"azure-kusto-data": "^7.0.0-alpha.1",
|
|
68
73
|
"browserify-zlib": "0.2.0",
|
|
69
74
|
"buffer": "^6.0.3",
|
|
70
75
|
"is-ip": "^3.1.0",
|
|
@@ -79,5 +84,5 @@
|
|
|
79
84
|
"@types/sinon": "^10.0.13",
|
|
80
85
|
"assert": "^2.0.0"
|
|
81
86
|
},
|
|
82
|
-
"gitHead": "
|
|
87
|
+
"gitHead": "7d97fb4354a6546333e194c29d6bceb1f4d78efa"
|
|
83
88
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { IngestionProperties, IngestionPropertiesInput } from "./ingestionProperties";
|
|
2
|
-
import { StreamDescriptor, FileDescriptorBase, BlobDescriptor } from "./descriptors";
|
|
1
|
+
import { IngestionProperties, IngestionPropertiesInput } from "./ingestionProperties.js";
|
|
2
|
+
import { StreamDescriptor, FileDescriptorBase, BlobDescriptor } from "./descriptors.js";
|
|
3
3
|
export declare abstract class AbstractKustoClient {
|
|
4
4
|
defaultProps: IngestionProperties;
|
|
5
5
|
defaultDatabase?: string;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { IngestionMappingKind } from "./ingestionProperties";
|
|
1
|
+
import { IngestionMappingKind } from "./ingestionProperties.js";
|
|
2
2
|
export declare enum FieldTransformation {
|
|
3
3
|
PropertyBagArrayToDictionary = "PropertyBagArrayToDictionary",
|
|
4
4
|
DateTimeFromUnixSeconds = "DateTimeFromUnixSeconds",
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { AbstractDescriptor, CompressionType, FileDescriptorBase } from "./descriptors";
|
|
2
|
-
import { IngestionPropertiesInput } from "./ingestionProperties";
|
|
1
|
+
import { AbstractDescriptor, CompressionType, FileDescriptorBase } from "./descriptors.js";
|
|
2
|
+
import { IngestionPropertiesInput } from "./ingestionProperties.js";
|
|
3
3
|
export declare class FileDescriptor extends AbstractDescriptor implements FileDescriptorBase {
|
|
4
4
|
readonly file: Blob;
|
|
5
5
|
readonly extension?: string | undefined;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { AbstractDescriptor, CompressionType, FileDescriptorBase } from "./descriptors";
|
|
2
|
-
import { IngestionPropertiesInput } from "./ingestionProperties";
|
|
1
|
+
import { AbstractDescriptor, CompressionType, FileDescriptorBase } from "./descriptors.js";
|
|
2
|
+
import { IngestionPropertiesInput } from "./ingestionProperties.js";
|
|
3
3
|
/**
|
|
4
4
|
* Describes a file to be ingested. Use string to describe a local path in Node.JS and Blob object in browsers
|
|
5
5
|
*/
|
package/types/src/index.d.ts
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
import KustoIngestClient from "./ingestClient";
|
|
2
|
-
import streamingIngestClient from "./streamingIngestClient";
|
|
3
|
-
import managedStreamingIngestClient from "./managedStreamingIngestClient";
|
|
4
|
-
import KustoIngestStatusQueues from "./status";
|
|
5
|
-
import { IngestionResult, OperationStatus, IngestionStatus, IngestionStatusInTableDescription } from "./ingestionResult";
|
|
6
|
-
import { DataFormat, IngestionMappingKind, ReportLevel, ReportMethod, ValidationImplications, ValidationOptions, ValidationPolicy, dataFormatMappingKind } from "./ingestionProperties";
|
|
7
|
-
import { ApacheAvroColumnMapping, AvroColumnMapping, ColumnMapping, ConstantTransformation, CsvColumnMapping, FieldTransformation, JsonColumnMapping, OrcColumnMapping, ParquetColumnMapping, SStreamColumnMapping, W3CLogFileMapping } from "./columnMappings";
|
|
8
|
-
import { BlobDescriptor, CompressionType, StreamDescriptor } from "./descriptors";
|
|
9
|
-
import { FileDescriptor } from "./fileDescriptor";
|
|
10
|
-
export { Transformation as ColumnMappingTransformation } from "./columnMappings";
|
|
11
|
-
export { IngestionProperties } from "./ingestionProperties";
|
|
1
|
+
import KustoIngestClient from "./ingestClient.js";
|
|
2
|
+
import streamingIngestClient from "./streamingIngestClient.js";
|
|
3
|
+
import managedStreamingIngestClient from "./managedStreamingIngestClient.js";
|
|
4
|
+
import KustoIngestStatusQueues from "./status.js";
|
|
5
|
+
import { type IngestionResult, OperationStatus, type IngestionStatus, IngestionStatusInTableDescription } from "./ingestionResult.js";
|
|
6
|
+
import { DataFormat, IngestionMappingKind, ReportLevel, ReportMethod, ValidationImplications, ValidationOptions, ValidationPolicy, dataFormatMappingKind } from "./ingestionProperties.js";
|
|
7
|
+
import { ApacheAvroColumnMapping, AvroColumnMapping, ColumnMapping, ConstantTransformation, CsvColumnMapping, FieldTransformation, JsonColumnMapping, OrcColumnMapping, ParquetColumnMapping, SStreamColumnMapping, W3CLogFileMapping } from "./columnMappings.js";
|
|
8
|
+
import { BlobDescriptor, CompressionType, StreamDescriptor } from "./descriptors.js";
|
|
9
|
+
import { FileDescriptor } from "./fileDescriptor.js";
|
|
10
|
+
export type { Transformation as ColumnMappingTransformation } from "./columnMappings.js";
|
|
11
|
+
export { IngestionProperties } from "./ingestionProperties.js";
|
|
12
12
|
export { KustoIngestClient as IngestClient, KustoIngestStatusQueues as IngestStatusQueues, managedStreamingIngestClient as ManagedStreamingIngestClient, streamingIngestClient as StreamingIngestClient, };
|
|
13
13
|
/**
|
|
14
14
|
* @deprecated - import directly instead. Export const is not exporting type.
|
|
@@ -18,7 +18,7 @@ export declare const IngestionDescriptors: {
|
|
|
18
18
|
FileDescriptor: typeof FileDescriptor;
|
|
19
19
|
StreamDescriptor: typeof StreamDescriptor;
|
|
20
20
|
};
|
|
21
|
-
export { ApacheAvroColumnMapping, AvroColumnMapping, BlobDescriptor, ColumnMapping, CompressionType, ConstantTransformation, CsvColumnMapping, DataFormat, FieldTransformation, FileDescriptor, IngestionMappingKind, JsonColumnMapping, OrcColumnMapping, ParquetColumnMapping, ReportLevel, ReportMethod, SStreamColumnMapping, StreamDescriptor, ValidationImplications, ValidationOptions, ValidationPolicy, W3CLogFileMapping, dataFormatMappingKind, IngestionResult, OperationStatus, IngestionStatus, IngestionStatusInTableDescription, };
|
|
21
|
+
export { ApacheAvroColumnMapping, AvroColumnMapping, BlobDescriptor, ColumnMapping, CompressionType, ConstantTransformation, CsvColumnMapping, DataFormat, FieldTransformation, FileDescriptor, IngestionMappingKind, JsonColumnMapping, OrcColumnMapping, ParquetColumnMapping, ReportLevel, ReportMethod, SStreamColumnMapping, StreamDescriptor, ValidationImplications, ValidationOptions, ValidationPolicy, W3CLogFileMapping, dataFormatMappingKind, type IngestionResult, OperationStatus, type IngestionStatus, IngestionStatusInTableDescription, };
|
|
22
22
|
/**
|
|
23
23
|
* @deprecated - import directly instead
|
|
24
24
|
*/
|
|
@@ -47,5 +47,5 @@ export declare const IngestionPropertiesEnums: {
|
|
|
47
47
|
FieldTransformation: typeof FieldTransformation;
|
|
48
48
|
ColumnMapping: typeof ColumnMapping;
|
|
49
49
|
};
|
|
50
|
-
export { IngestionPropertiesValidationError } from "./errors";
|
|
50
|
+
export { IngestionPropertiesValidationError } from "./errors.js";
|
|
51
51
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { KustoConnectionStringBuilder } from "azure-kusto-data";
|
|
2
|
-
import { StreamDescriptor } from "./descriptors";
|
|
3
|
-
import { FileDescriptor } from "./fileDescriptor.browser";
|
|
4
|
-
import { IngestionPropertiesInput } from "./ingestionProperties";
|
|
5
|
-
import { KustoIngestClientBase } from "./ingestClientBase";
|
|
6
|
-
import { IngestionResult } from "./ingestionResult";
|
|
2
|
+
import { StreamDescriptor } from "./descriptors.js";
|
|
3
|
+
import { FileDescriptor } from "./fileDescriptor.browser.js";
|
|
4
|
+
import { IngestionPropertiesInput } from "./ingestionProperties.js";
|
|
5
|
+
import { KustoIngestClientBase } from "./ingestClientBase.js";
|
|
6
|
+
import { IngestionResult } from "./ingestionResult.js";
|
|
7
7
|
export declare class KustoIngestClient extends KustoIngestClientBase {
|
|
8
8
|
constructor(kcsb: string | KustoConnectionStringBuilder, defaultProps?: IngestionPropertiesInput, autoCorrectEndpoint?: boolean);
|
|
9
9
|
/**
|
|
@@ -1,11 +1,10 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
2
1
|
import { KustoConnectionStringBuilder } from "azure-kusto-data";
|
|
3
|
-
import { StreamDescriptor } from "./descriptors";
|
|
4
|
-
import { FileDescriptor } from "./fileDescriptor";
|
|
5
|
-
import { IngestionPropertiesInput } from "./ingestionProperties";
|
|
6
|
-
import { KustoIngestClientBase } from "./ingestClientBase";
|
|
2
|
+
import { StreamDescriptor } from "./descriptors.js";
|
|
3
|
+
import { FileDescriptor } from "./fileDescriptor.js";
|
|
4
|
+
import { IngestionPropertiesInput } from "./ingestionProperties.js";
|
|
5
|
+
import { KustoIngestClientBase } from "./ingestClientBase.js";
|
|
7
6
|
import { Readable } from "stream";
|
|
8
|
-
import { IngestionResult } from "./ingestionResult";
|
|
7
|
+
import { IngestionResult } from "./ingestionResult.js";
|
|
9
8
|
export declare class KustoIngestClient extends KustoIngestClientBase {
|
|
10
9
|
constructor(kcsb: string | KustoConnectionStringBuilder, defaultProps?: IngestionPropertiesInput, autoCorrectEndpoint?: boolean);
|
|
11
10
|
/**
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { KustoConnectionStringBuilder } from "azure-kusto-data";
|
|
2
|
-
import ResourceManager from "./resourceManager";
|
|
3
|
-
import { IngestionPropertiesInput } from "./ingestionProperties";
|
|
4
|
-
import { AbstractKustoClient } from "./abstractKustoClient";
|
|
5
|
-
import { IngestionResult } from "./ingestionResult";
|
|
6
|
-
import { BlobDescriptor, StreamDescriptor } from "./descriptors";
|
|
2
|
+
import ResourceManager from "./resourceManager.js";
|
|
3
|
+
import { IngestionPropertiesInput } from "./ingestionProperties.js";
|
|
4
|
+
import { AbstractKustoClient } from "./abstractKustoClient.js";
|
|
5
|
+
import { IngestionResult } from "./ingestionResult.js";
|
|
6
|
+
import { BlobDescriptor, StreamDescriptor } from "./descriptors.js";
|
|
7
7
|
export declare abstract class KustoIngestClientBase extends AbstractKustoClient {
|
|
8
8
|
resourceManager: ResourceManager;
|
|
9
9
|
applicationForTracing: string | null;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { BlobDescriptor } from "./descriptors";
|
|
2
|
-
import IngestionProperties, { ReportLevel, ReportMethod } from "./ingestionProperties";
|
|
3
|
-
import { IngestionStatusInTableDescription } from "./ingestionResult";
|
|
1
|
+
import { BlobDescriptor } from "./descriptors.js";
|
|
2
|
+
import IngestionProperties, { ReportLevel, ReportMethod } from "./ingestionProperties.js";
|
|
3
|
+
import { IngestionStatusInTableDescription } from "./ingestionResult.js";
|
|
4
4
|
export declare class IngestionBlobInfo {
|
|
5
5
|
BlobPath: string;
|
|
6
6
|
RawDataSize: number | null;
|