@azure/storage-blob-changefeed 12.0.0-alpha.20211214.3 → 12.0.0-alpha.20220111.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +47 -47
- package/dist/index.js.map +1 -1
- package/dist-esm/storage-blob-changefeed/src/BlobChangeFeedClient.js +4 -4
- package/dist-esm/storage-blob-changefeed/src/BlobChangeFeedClient.js.map +1 -1
- package/dist-esm/storage-blob-changefeed/src/ChangeFeed.js +8 -8
- package/dist-esm/storage-blob-changefeed/src/ChangeFeed.js.map +1 -1
- package/dist-esm/storage-blob-changefeed/src/ChangeFeedFactory.js +7 -7
- package/dist-esm/storage-blob-changefeed/src/ChangeFeedFactory.js.map +1 -1
- package/dist-esm/storage-blob-changefeed/src/ChunkFactory.js +1 -1
- package/dist-esm/storage-blob-changefeed/src/ChunkFactory.js.map +1 -1
- package/dist-esm/storage-blob-changefeed/src/LazyLoadingBlobStream.js +5 -5
- package/dist-esm/storage-blob-changefeed/src/LazyLoadingBlobStream.js.map +1 -1
- package/dist-esm/storage-blob-changefeed/src/Segment.js +3 -3
- package/dist-esm/storage-blob-changefeed/src/Segment.js.map +1 -1
- package/dist-esm/storage-blob-changefeed/src/SegmentFactory.js +3 -3
- package/dist-esm/storage-blob-changefeed/src/SegmentFactory.js.map +1 -1
- package/dist-esm/storage-blob-changefeed/src/Shard.js +3 -3
- package/dist-esm/storage-blob-changefeed/src/Shard.js.map +1 -1
- package/dist-esm/storage-blob-changefeed/src/ShardFactory.js +3 -3
- package/dist-esm/storage-blob-changefeed/src/ShardFactory.js.map +1 -1
- package/dist-esm/storage-blob-changefeed/src/utils/tracing.js +1 -1
- package/dist-esm/storage-blob-changefeed/src/utils/tracing.js.map +1 -1
- package/dist-esm/storage-blob-changefeed/src/utils/utils.common.js +4 -4
- package/dist-esm/storage-blob-changefeed/src/utils/utils.common.js.map +1 -1
- package/dist-esm/storage-internal-avro/src/AvroParser.js.map +1 -1
- package/dist-esm/storage-internal-avro/src/AvroReader.js +8 -8
- package/dist-esm/storage-internal-avro/src/AvroReader.js.map +1 -1
- package/package.json +8 -8
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Copyright (c) Microsoft Corporation.
|
|
2
2
|
// Licensed under the MIT license.
|
|
3
3
|
import { __asyncGenerator, __await } from "tslib";
|
|
4
|
-
import { BlobServiceClient, Pipeline } from "@azure/storage-blob";
|
|
4
|
+
import { BlobServiceClient, Pipeline, } from "@azure/storage-blob";
|
|
5
5
|
import { ChangeFeedFactory } from "./ChangeFeedFactory";
|
|
6
6
|
import { CHANGE_FEED_MAX_PAGE_SIZE, SDK_VERSION } from "./utils/constants";
|
|
7
7
|
/**
|
|
@@ -81,7 +81,7 @@ export class BlobChangeFeedClient {
|
|
|
81
81
|
while (changeFeed.hasNext()) {
|
|
82
82
|
const event = yield __await(changeFeed.getChange({
|
|
83
83
|
abortSignal: options.abortSignal,
|
|
84
|
-
tracingOptions: options.tracingOptions
|
|
84
|
+
tracingOptions: options.tracingOptions,
|
|
85
85
|
}));
|
|
86
86
|
if (event) {
|
|
87
87
|
yield yield __await(event);
|
|
@@ -104,7 +104,7 @@ export class BlobChangeFeedClient {
|
|
|
104
104
|
while (changeFeed.hasNext() && eventPage.events.length < maxPageSize) {
|
|
105
105
|
const event = yield __await(changeFeed.getChange({
|
|
106
106
|
abortSignal: options.abortSignal,
|
|
107
|
-
tracingOptions: options.tracingOptions
|
|
107
|
+
tracingOptions: options.tracingOptions,
|
|
108
108
|
}));
|
|
109
109
|
if (event) {
|
|
110
110
|
eventPage.events.push(event);
|
|
@@ -214,7 +214,7 @@ export class BlobChangeFeedClient {
|
|
|
214
214
|
*/
|
|
215
215
|
byPage: (settings = {}) => {
|
|
216
216
|
return this.getPage(settings.continuationToken, settings.maxPageSize, options);
|
|
217
|
-
}
|
|
217
|
+
},
|
|
218
218
|
};
|
|
219
219
|
}
|
|
220
220
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"BlobChangeFeedClient.js","sourceRoot":"","sources":["../../../src/BlobChangeFeedClient.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;;AAElC,OAAO,EACL,iBAAiB,EAIjB,QAAQ,EACT,MAAM,qBAAqB,CAAC;AAG7B,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAExD,OAAO,EAAE,yBAAyB,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAI3E;;GAEG;AACH,MAAM,OAAO,uBAAuB;IAWlC;QACE,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;QACjB,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC;IAC9B,CAAC;CACF;AAED;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CACzB,UAA+E,EAC/E,kBAA0C,EAAE;IAE5C,OAAO,WAAW,CAAC,UAAU,EAAE,qBAAqB,CAAC,eAAe,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,SAAS,qBAAqB,CAAC,OAAgC;IAC7D,IAAI,CAAC,OAAO,EAAE;QACZ,OAAO,GAAG,EAAE,CAAC;KACd;IACD,IAAI,OAAO,CAAC,gBAAgB,KAAK,SAAS,EAAE;QAC1C,OAAO,CAAC,gBAAgB,GAAG,EAAE,CAAC;KAC/B;IAED,IAAI,OAAO,CAAC,gBAAgB,CAAC,eAAe,KAAK,SAAS,EAAE;QAC1D,OAAO,CAAC,gBAAgB,CAAC,eAAe,GAAG,EAAE,CAAC;KAC/C;SAAM,IAAI,OAAO,CAAC,gBAAgB,CAAC,eAAe,KAAK,EAAE,EAAE;QAC1D,OAAO,CAAC,gBAAgB,CAAC,eAAe,IAAI,GAAG,CAAC;KACjD;IACD,OAAO,CAAC,gBAAgB,CAAC,eAAe,IAAI,iBAAiB,WAAW,EAAE,CAAC;IAC3E,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;GAGG;AACH,MAAM,OAAO,oBAAoB;IAqF/B,YACE,WAAmB,EACnB,oBAIY;IACZ,2FAA2F;IAC3F,iEAAiE;IACjE,OAAgC;QAEhC,IAAI,CAAC,iBAAiB,GAAG,IAAI,iBAAiB,EAAE,CAAC;QAEjD,IAAI,oBAAoB,YAAY,QAAQ,EAAE;YAC5C,IAAI,CAAC,iBAAiB,GAAG,IAAI,iBAAiB,CAAC,WAAW,EAAE,oBAAoB,CAAC,CAAC;SACnF;aAAM;YACL,IAAI,CAAC,iBAAiB,GAAG,IAAI,iBAAiB,CAC5C,WAAW,EACX,oBAAoB,EACpB,qBAAqB,CAAC,OAAO,CAAC,CAC/B,CAAC;SACH;IACH,CAAC;IApGD;;;;;;;;;;;OAWG;IACI,MAAM,CAAC,oBAAoB,CAChC,gBAAwB;IACxB,2FAA2F;IAC3F,iEAAiE;IACjE,OAAgC;QAEhC,MAAM,iBAAiB,GAAG,iBAAiB,CAAC,oBAAoB,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC;QAC5F,OAAO,IAAI,oBAAoB,CAC7B,iBAAiB,CAAC,GAAG,EACrB,iBAAiB,CAAC,UAAU,EAC5B,qBAAqB,CAAC,OAAO,CAAC,CAC/B,CAAC;IACJ,CAAC;IA8Ec,SAAS,CACtB,UAA4C,EAAE;;YAE9C,MAAM,UAAU,GAAe,cAAM,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAChE,IAAI,CAAC,iBAAiB,EACtB,SAAS,EACT,OAAO,CACR,CAAA,CAAC;YAEF,OAAO,UAAU,CAAC,OAAO,EAAE,EAAE;gBAC3B,MAAM,KAAK,GAAG,cAAM,UAAU,CAAC,SAAS,CAAC;oBACvC,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,cAAc,EAAE,OAAO,CAAC,cAAc;iBACvC,CAAC,CAAA,CAAC;gBACH,IAAI,KAAK,EAAE;oBACT,oBAAM,KAAK,CAAA,CAAC;iBACb;qBAAM;oBACL,6BAAO;iBACR;aACF;QACH,CAAC;KAAA;IAED,6FAA6F;IAC9E,OAAO,CACpB,iBAA0B,EAC1B,WAAoB,EACpB,UAA4C,EAAE;;YAE9C,MAAM,UAAU,GAAe,cAAM,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAChE,IAAI,CAAC,iBAAiB,EACtB,iBAAiB,EACjB,OAAO,CACR,CAAA,CAAC;YAEF,IAAI,CAAC,WAAW,IAAI,WAAW,GAAG,yBAAyB,EAAE;gBAC3D,WAAW,GAAG,yBAAyB,CAAC;aACzC;YACD,OAAO,UAAU,CAAC,OAAO,EAAE,EAAE;gBAC3B,MAAM,SAAS,GAAG,IAAI,uBAAuB,EAAE,CAAC;gBAChD,OAAO,UAAU,CAAC,OAAO,EAAE,IAAI,SAAS,CAAC,MAAM,CAAC,MAAM,GAAG,WAAW,EAAE;oBACpE,MAAM,KAAK,GAAG,cAAM,UAAU,CAAC,SAAS,CAAC;wBACvC,WAAW,EAAE,OAAO,CAAC,WAAW;wBAChC,cAAc,EAAE,OAAO,CAAC,cAAc;qBACvC,CAAC,CAAA,CAAC;oBACH,IAAI,KAAK,EAAE;wBACT,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;qBAC9B;iBACF;gBACD,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE;oBACxB,SAAS,CAAC,iBAAiB,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC,CAAC;iBACtE;gBACD,IAAI,SAAS,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;oBAC/B,oBAAM,SAAS,CAAA,CAAC;iBACjB;qBAAM;oBACL,6BAAO;iBACR;aACF;QACH,CAAC;KAAA;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAuEG;IACI,WAAW,CAChB,UAA4C,EAAE;QAE9C,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACrC,OAAO;YACL;;eAEG;YACH,KAAK,CAAC,IAAI;gBACR,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;YACrB,CAAC;YACD;;eAEG;YACH,CAAC,MAAM,CAAC,aAAa,CAAC;gBACpB,OAAO,IAAI,CAAC;YACd,CAAC;YACD;;eAEG;YACH,MAAM,EAAE,CAAC,WAAyB,EAAE,EAAE,EAAE;gBACtC,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,EAAE,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;YACjF,CAAC;SACF,CAAC;IACJ,CAAC;CACF","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\nimport {\n BlobServiceClient,\n StoragePipelineOptions,\n StorageSharedKeyCredential,\n AnonymousCredential,\n Pipeline\n} from \"@azure/storage-blob\";\nimport { PagedAsyncIterableIterator, PageSettings } from \"@azure/core-paging\";\nimport { BlobChangeFeedEvent } from \"./models/BlobChangeFeedEvent\";\nimport { ChangeFeedFactory } from \"./ChangeFeedFactory\";\nimport { ChangeFeed } from \"./ChangeFeed\";\nimport { CHANGE_FEED_MAX_PAGE_SIZE, SDK_VERSION } from \"./utils/constants\";\nimport { BlobChangeFeedListChangesOptions } from \"./models/models\";\nimport { TokenCredential } from \"@azure/core-http\";\n\n/**\n * Contains paged response data for the {@link BlobChangeFeedClient.listChanges} operation.\n */\nexport class BlobChangeFeedEventPage {\n /**\n * Array of {@link BlobChangeFeedEvent}.\n */\n public events: BlobChangeFeedEvent[];\n\n /**\n * The token that keeps track of where to continue the iterator.\n */\n public continuationToken: string;\n\n constructor() {\n this.events = [];\n this.continuationToken = \"\";\n }\n}\n\n/**\n * Creates a new Pipeline object with Credential provided.\n *\n * @param credential - Such as AnonymousCredential, StorageSharedKeyCredential or any credential from the `@azure/identity` package to authenticate requests to the service. You can also provide an object that implements the TokenCredential interface. If not specified, AnonymousCredential is used.\n * @param pipelineOptions - Optional. Options.\n * @returns A new Pipeline object.\n */\nexport function newPipeline(\n credential?: StorageSharedKeyCredential | AnonymousCredential | TokenCredential,\n pipelineOptions: StoragePipelineOptions = {}\n): Pipeline {\n return newPipeline(credential, appendUserAgentPrefix(pipelineOptions));\n}\n\nfunction appendUserAgentPrefix(options?: StoragePipelineOptions): StoragePipelineOptions {\n if (!options) {\n options = {};\n }\n if (options.userAgentOptions === undefined) {\n options.userAgentOptions = {};\n }\n\n if (options.userAgentOptions.userAgentPrefix === undefined) {\n options.userAgentOptions.userAgentPrefix = \"\";\n } else if (options.userAgentOptions.userAgentPrefix !== \"\") {\n options.userAgentOptions.userAgentPrefix += \" \";\n }\n options.userAgentOptions.userAgentPrefix += `changefeed-js/${SDK_VERSION}`;\n return options;\n}\n\n/**\n * BlobChangeFeedClient.\n * @see https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blob-change-feed?tabs=azure-portal\n */\nexport class BlobChangeFeedClient {\n /**\n * blobServiceClient provided by `@azure/storage-blob` package.\n */\n private blobServiceClient: BlobServiceClient;\n private changeFeedFactory: ChangeFeedFactory;\n\n /**\n *\n * Creates an instance of BlobChangeFeedClient from connection string.\n *\n * @param connectionString - Account connection string or a SAS connection string of an Azure storage account.\n * [ Note - Account connection string can only be used in NODE.JS runtime. ]\n * Account connection string example -\n * `DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=accountKey;EndpointSuffix=core.windows.net`\n * SAS connection string example -\n * `BlobEndpoint=https://myaccount.blob.core.windows.net/;QueueEndpoint=https://myaccount.queue.core.windows.net/;FileEndpoint=https://myaccount.file.core.windows.net/;TableEndpoint=https://myaccount.table.core.windows.net/;SharedAccessSignature=sasString`\n * @param options - Optional. Options to configure the HTTP pipeline.\n */\n public static fromConnectionString(\n connectionString: string,\n // Legacy, no way to fix the eslint error without breaking. Disable the rule for this line.\n /* eslint-disable-next-line @azure/azure-sdk/ts-naming-options */\n options?: StoragePipelineOptions\n ): BlobChangeFeedClient {\n const blobServiceClient = BlobServiceClient.fromConnectionString(connectionString, options);\n return new BlobChangeFeedClient(\n blobServiceClient.url,\n blobServiceClient.credential,\n appendUserAgentPrefix(options)\n );\n }\n\n /**\n * Creates an instance of BlobChangeFeedClient.\n *\n * @param url - A Client string pointing to Azure Storage blob service, such as\n * \"https://myaccount.blob.core.windows.net\". You can append a SAS\n * if using AnonymousCredential, such as \"https://myaccount.blob.core.windows.net?sasString\".\n * @param credential - Such as AnonymousCredential, StorageSharedKeyCredential or any credential from the `@azure/identity` package to authenticate requests to the service. You can also provide an object that implements the TokenCredential interface. If not specified, AnonymousCredential is used.\n * @param options - Optional. Options to configure the HTTP pipeline.\n *\n * Example using DefaultAzureCredential from `@azure/identity`:\n *\n * ```js\n * const account = \"<storage account name>\";\n *\n * const defaultAzureCredential = new DefaultAzureCredential();\n *\n * const blobChangeFeedClient = new BlobChangeFeedClient(\n * `https://${account}.blob.core.windows.net`,\n * defaultAzureCredential\n * );\n * ```\n *\n * Example using an account name/key:\n *\n * ```js\n * const account = \"<storage account name>\"\n * const sharedKeyCredential = new StorageSharedKeyCredential(account, \"<account key>\");\n *\n * const blobChangeFeedClient = new BlobChangeFeedClient(\n * `https://${account}.blob.core.windows.net`,\n * sharedKeyCredential\n * );\n * ```\n */\n constructor(\n url: string,\n credential?: StorageSharedKeyCredential | AnonymousCredential | TokenCredential,\n // Legacy, no way to fix the eslint error without breaking. Disable the rule for this line.\n /* eslint-disable-next-line @azure/azure-sdk/ts-naming-options */\n options?: StoragePipelineOptions\n );\n\n /**\n * Creates an instance of BlobChangeFeedClient.\n *\n * @param url - A Client string pointing to Azure Storage blob service, such as\n * \"https://myaccount.blob.core.windows.net\". You can append a SAS\n * if using AnonymousCredential, such as \"https://myaccount.blob.core.windows.net?sasString\".\n * @param pipeline - Call newPipeline() to create a default\n * pipeline, or provide a customized pipeline.\n */\n constructor(url: string, pipeline: Pipeline);\n constructor(\n urlOrClient: string,\n credentialOrPipeline?:\n | StorageSharedKeyCredential\n | AnonymousCredential\n | TokenCredential\n | Pipeline,\n // Legacy, no way to fix the eslint error without breaking. Disable the rule for this line.\n /* eslint-disable-next-line @azure/azure-sdk/ts-naming-options */\n options?: StoragePipelineOptions\n ) {\n this.changeFeedFactory = new ChangeFeedFactory();\n\n if (credentialOrPipeline instanceof Pipeline) {\n this.blobServiceClient = new BlobServiceClient(urlOrClient, credentialOrPipeline);\n } else {\n this.blobServiceClient = new BlobServiceClient(\n urlOrClient,\n credentialOrPipeline,\n appendUserAgentPrefix(options)\n );\n }\n }\n\n private async *getChange(\n options: BlobChangeFeedListChangesOptions = {}\n ): AsyncIterableIterator<BlobChangeFeedEvent> {\n const changeFeed: ChangeFeed = await this.changeFeedFactory.create(\n this.blobServiceClient,\n undefined,\n options\n );\n\n while (changeFeed.hasNext()) {\n const event = await changeFeed.getChange({\n abortSignal: options.abortSignal,\n tracingOptions: options.tracingOptions\n });\n if (event) {\n yield event;\n } else {\n return;\n }\n }\n }\n\n // start in ChangeFeedListChangesOptions will be ignored when continuationToken is specified.\n private async *getPage(\n continuationToken?: string,\n maxPageSize?: number,\n options: BlobChangeFeedListChangesOptions = {}\n ): AsyncIterableIterator<BlobChangeFeedEventPage> {\n const changeFeed: ChangeFeed = await this.changeFeedFactory.create(\n this.blobServiceClient,\n continuationToken,\n options\n );\n\n if (!maxPageSize || maxPageSize > CHANGE_FEED_MAX_PAGE_SIZE) {\n maxPageSize = CHANGE_FEED_MAX_PAGE_SIZE;\n }\n while (changeFeed.hasNext()) {\n const eventPage = new BlobChangeFeedEventPage();\n while (changeFeed.hasNext() && eventPage.events.length < maxPageSize) {\n const event = await changeFeed.getChange({\n abortSignal: options.abortSignal,\n tracingOptions: options.tracingOptions\n });\n if (event) {\n eventPage.events.push(event);\n }\n }\n if (changeFeed.hasNext()) {\n eventPage.continuationToken = JSON.stringify(changeFeed.getCursor());\n }\n if (eventPage.events.length > 0) {\n yield eventPage;\n } else {\n return;\n }\n }\n }\n\n /**\n * Returns an async iterable iterator to list all the change feed events\n * in the specified account.\n *\n * .byPage() returns an async iterable iterator to list the change feed events in pages.\n *\n * Example using `for await` syntax:\n *\n * ```js\n * let i = 1;\n * for await (const event of blobChangeFeedClient.listChanges()) {\n * console.log(`Event ${i++}, type: ${event.eventType}`);\n * }\n * ```\n *\n * Example using `iter.next()`:\n *\n * ```js\n * let i = 1;\n * const iter = blobChangeFeedClient.listChanges();\n * let eventItem = await iter.next();\n * while (!eventItem.done) {\n * console.log(`Event ${i++}, type: ${eventItem.eventType}`);\n * eventItem = await iter.next();\n * }\n * ```\n *\n * Example using `byPage()`:\n *\n * ```js\n * // passing optional maxPageSize in the page settings\n * let i = 1;\n * for await (const eventPage of blobChangeFeedClient.listChanges().byPage({ maxPageSize: 20 })) {\n * if (eventPage.events) {\n * for (const event of eventPage.events) {\n * console.log(`Event ${i++}, type: ${event.eventType}`);\n * }\n * }\n * }\n * ```\n *\n * Example using paging with a marker:\n *\n * ```js\n * let i = 1;\n * let iterator = blobChangeFeedClient.listChanges().byPage({ maxPageSize: 2 });\n * let eventPage = (await iterator.next()).value;\n *\n * if (eventPage.events) {\n * for (const container of eventPage.events) {\n * console.log(`Event ${i++}, type: ${event.eventType}`);\n * }\n * }\n *\n * // Gets next marker\n * let marker = eventPage.continuationToken;\n * // Passing next marker as continuationToken\n * iterator = blobChangeFeedClient\n * .listChanges()\n * .byPage({ continuationToken: marker, maxPageSize: 10 });\n * eventPage = (await iterator.next()).value;\n *\n * if (eventPage.events) {\n * for (const container of eventPage.events) {\n * console.log(`Event ${i++}, type: ${event.eventType}`);\n * }\n * }\n * ```\n *\n * @param options - Options to list change feed events.\n * @returns An asyncIterableIterator that supports paging.\n */\n public listChanges(\n options: BlobChangeFeedListChangesOptions = {}\n ): PagedAsyncIterableIterator<BlobChangeFeedEvent, BlobChangeFeedEventPage> {\n const iter = this.getChange(options);\n return {\n /**\n * The next method, part of the iteration protocol\n */\n async next() {\n return iter.next();\n },\n /**\n * The connection to the async iterator, part of the iteration protocol\n */\n [Symbol.asyncIterator]() {\n return this;\n },\n /**\n * Return an AsyncIterableIterator that works a page at a time\n */\n byPage: (settings: PageSettings = {}) => {\n return this.getPage(settings.continuationToken, settings.maxPageSize, options);\n }\n };\n }\n}\n"]}
|
|
1
|
+
{"version":3,"file":"BlobChangeFeedClient.js","sourceRoot":"","sources":["../../../src/BlobChangeFeedClient.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;;AAElC,OAAO,EACL,iBAAiB,EAIjB,QAAQ,GACT,MAAM,qBAAqB,CAAC;AAG7B,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAExD,OAAO,EAAE,yBAAyB,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAI3E;;GAEG;AACH,MAAM,OAAO,uBAAuB;IAWlC;QACE,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;QACjB,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC;IAC9B,CAAC;CACF;AAED;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CACzB,UAA+E,EAC/E,kBAA0C,EAAE;IAE5C,OAAO,WAAW,CAAC,UAAU,EAAE,qBAAqB,CAAC,eAAe,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,SAAS,qBAAqB,CAAC,OAAgC;IAC7D,IAAI,CAAC,OAAO,EAAE;QACZ,OAAO,GAAG,EAAE,CAAC;KACd;IACD,IAAI,OAAO,CAAC,gBAAgB,KAAK,SAAS,EAAE;QAC1C,OAAO,CAAC,gBAAgB,GAAG,EAAE,CAAC;KAC/B;IAED,IAAI,OAAO,CAAC,gBAAgB,CAAC,eAAe,KAAK,SAAS,EAAE;QAC1D,OAAO,CAAC,gBAAgB,CAAC,eAAe,GAAG,EAAE,CAAC;KAC/C;SAAM,IAAI,OAAO,CAAC,gBAAgB,CAAC,eAAe,KAAK,EAAE,EAAE;QAC1D,OAAO,CAAC,gBAAgB,CAAC,eAAe,IAAI,GAAG,CAAC;KACjD;IACD,OAAO,CAAC,gBAAgB,CAAC,eAAe,IAAI,iBAAiB,WAAW,EAAE,CAAC;IAC3E,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;GAGG;AACH,MAAM,OAAO,oBAAoB;IAqF/B,YACE,WAAmB,EACnB,oBAIY;IACZ,2FAA2F;IAC3F,iEAAiE;IACjE,OAAgC;QAEhC,IAAI,CAAC,iBAAiB,GAAG,IAAI,iBAAiB,EAAE,CAAC;QAEjD,IAAI,oBAAoB,YAAY,QAAQ,EAAE;YAC5C,IAAI,CAAC,iBAAiB,GAAG,IAAI,iBAAiB,CAAC,WAAW,EAAE,oBAAoB,CAAC,CAAC;SACnF;aAAM;YACL,IAAI,CAAC,iBAAiB,GAAG,IAAI,iBAAiB,CAC5C,WAAW,EACX,oBAAoB,EACpB,qBAAqB,CAAC,OAAO,CAAC,CAC/B,CAAC;SACH;IACH,CAAC;IApGD;;;;;;;;;;;OAWG;IACI,MAAM,CAAC,oBAAoB,CAChC,gBAAwB;IACxB,2FAA2F;IAC3F,iEAAiE;IACjE,OAAgC;QAEhC,MAAM,iBAAiB,GAAG,iBAAiB,CAAC,oBAAoB,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC;QAC5F,OAAO,IAAI,oBAAoB,CAC7B,iBAAiB,CAAC,GAAG,EACrB,iBAAiB,CAAC,UAAU,EAC5B,qBAAqB,CAAC,OAAO,CAAC,CAC/B,CAAC;IACJ,CAAC;IA8Ec,SAAS,CACtB,UAA4C,EAAE;;YAE9C,MAAM,UAAU,GAAe,cAAM,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAChE,IAAI,CAAC,iBAAiB,EACtB,SAAS,EACT,OAAO,CACR,CAAA,CAAC;YAEF,OAAO,UAAU,CAAC,OAAO,EAAE,EAAE;gBAC3B,MAAM,KAAK,GAAG,cAAM,UAAU,CAAC,SAAS,CAAC;oBACvC,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,cAAc,EAAE,OAAO,CAAC,cAAc;iBACvC,CAAC,CAAA,CAAC;gBACH,IAAI,KAAK,EAAE;oBACT,oBAAM,KAAK,CAAA,CAAC;iBACb;qBAAM;oBACL,6BAAO;iBACR;aACF;QACH,CAAC;KAAA;IAED,6FAA6F;IAC9E,OAAO,CACpB,iBAA0B,EAC1B,WAAoB,EACpB,UAA4C,EAAE;;YAE9C,MAAM,UAAU,GAAe,cAAM,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAChE,IAAI,CAAC,iBAAiB,EACtB,iBAAiB,EACjB,OAAO,CACR,CAAA,CAAC;YAEF,IAAI,CAAC,WAAW,IAAI,WAAW,GAAG,yBAAyB,EAAE;gBAC3D,WAAW,GAAG,yBAAyB,CAAC;aACzC;YACD,OAAO,UAAU,CAAC,OAAO,EAAE,EAAE;gBAC3B,MAAM,SAAS,GAAG,IAAI,uBAAuB,EAAE,CAAC;gBAChD,OAAO,UAAU,CAAC,OAAO,EAAE,IAAI,SAAS,CAAC,MAAM,CAAC,MAAM,GAAG,WAAW,EAAE;oBACpE,MAAM,KAAK,GAAG,cAAM,UAAU,CAAC,SAAS,CAAC;wBACvC,WAAW,EAAE,OAAO,CAAC,WAAW;wBAChC,cAAc,EAAE,OAAO,CAAC,cAAc;qBACvC,CAAC,CAAA,CAAC;oBACH,IAAI,KAAK,EAAE;wBACT,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;qBAC9B;iBACF;gBACD,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE;oBACxB,SAAS,CAAC,iBAAiB,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC,CAAC;iBACtE;gBACD,IAAI,SAAS,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;oBAC/B,oBAAM,SAAS,CAAA,CAAC;iBACjB;qBAAM;oBACL,6BAAO;iBACR;aACF;QACH,CAAC;KAAA;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAuEG;IACI,WAAW,CAChB,UAA4C,EAAE;QAE9C,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACrC,OAAO;YACL;;eAEG;YACH,KAAK,CAAC,IAAI;gBACR,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;YACrB,CAAC;YACD;;eAEG;YACH,CAAC,MAAM,CAAC,aAAa,CAAC;gBACpB,OAAO,IAAI,CAAC;YACd,CAAC;YACD;;eAEG;YACH,MAAM,EAAE,CAAC,WAAyB,EAAE,EAAE,EAAE;gBACtC,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,EAAE,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;YACjF,CAAC;SACF,CAAC;IACJ,CAAC;CACF","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\nimport {\n BlobServiceClient,\n StoragePipelineOptions,\n StorageSharedKeyCredential,\n AnonymousCredential,\n Pipeline,\n} from \"@azure/storage-blob\";\nimport { PagedAsyncIterableIterator, PageSettings } from \"@azure/core-paging\";\nimport { BlobChangeFeedEvent } from \"./models/BlobChangeFeedEvent\";\nimport { ChangeFeedFactory } from \"./ChangeFeedFactory\";\nimport { ChangeFeed } from \"./ChangeFeed\";\nimport { CHANGE_FEED_MAX_PAGE_SIZE, SDK_VERSION } from \"./utils/constants\";\nimport { BlobChangeFeedListChangesOptions } from \"./models/models\";\nimport { TokenCredential } from \"@azure/core-http\";\n\n/**\n * Contains paged response data for the {@link BlobChangeFeedClient.listChanges} operation.\n */\nexport class BlobChangeFeedEventPage {\n /**\n * Array of {@link BlobChangeFeedEvent}.\n */\n public events: BlobChangeFeedEvent[];\n\n /**\n * The token that keeps track of where to continue the iterator.\n */\n public continuationToken: string;\n\n constructor() {\n this.events = [];\n this.continuationToken = \"\";\n }\n}\n\n/**\n * Creates a new Pipeline object with Credential provided.\n *\n * @param credential - Such as AnonymousCredential, StorageSharedKeyCredential or any credential from the `@azure/identity` package to authenticate requests to the service. You can also provide an object that implements the TokenCredential interface. If not specified, AnonymousCredential is used.\n * @param pipelineOptions - Optional. Options.\n * @returns A new Pipeline object.\n */\nexport function newPipeline(\n credential?: StorageSharedKeyCredential | AnonymousCredential | TokenCredential,\n pipelineOptions: StoragePipelineOptions = {}\n): Pipeline {\n return newPipeline(credential, appendUserAgentPrefix(pipelineOptions));\n}\n\nfunction appendUserAgentPrefix(options?: StoragePipelineOptions): StoragePipelineOptions {\n if (!options) {\n options = {};\n }\n if (options.userAgentOptions === undefined) {\n options.userAgentOptions = {};\n }\n\n if (options.userAgentOptions.userAgentPrefix === undefined) {\n options.userAgentOptions.userAgentPrefix = \"\";\n } else if (options.userAgentOptions.userAgentPrefix !== \"\") {\n options.userAgentOptions.userAgentPrefix += \" \";\n }\n options.userAgentOptions.userAgentPrefix += `changefeed-js/${SDK_VERSION}`;\n return options;\n}\n\n/**\n * BlobChangeFeedClient.\n * @see https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blob-change-feed?tabs=azure-portal\n */\nexport class BlobChangeFeedClient {\n /**\n * blobServiceClient provided by `@azure/storage-blob` package.\n */\n private blobServiceClient: BlobServiceClient;\n private changeFeedFactory: ChangeFeedFactory;\n\n /**\n *\n * Creates an instance of BlobChangeFeedClient from connection string.\n *\n * @param connectionString - Account connection string or a SAS connection string of an Azure storage account.\n * [ Note - Account connection string can only be used in NODE.JS runtime. ]\n * Account connection string example -\n * `DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=accountKey;EndpointSuffix=core.windows.net`\n * SAS connection string example -\n * `BlobEndpoint=https://myaccount.blob.core.windows.net/;QueueEndpoint=https://myaccount.queue.core.windows.net/;FileEndpoint=https://myaccount.file.core.windows.net/;TableEndpoint=https://myaccount.table.core.windows.net/;SharedAccessSignature=sasString`\n * @param options - Optional. Options to configure the HTTP pipeline.\n */\n public static fromConnectionString(\n connectionString: string,\n // Legacy, no way to fix the eslint error without breaking. Disable the rule for this line.\n /* eslint-disable-next-line @azure/azure-sdk/ts-naming-options */\n options?: StoragePipelineOptions\n ): BlobChangeFeedClient {\n const blobServiceClient = BlobServiceClient.fromConnectionString(connectionString, options);\n return new BlobChangeFeedClient(\n blobServiceClient.url,\n blobServiceClient.credential,\n appendUserAgentPrefix(options)\n );\n }\n\n /**\n * Creates an instance of BlobChangeFeedClient.\n *\n * @param url - A Client string pointing to Azure Storage blob service, such as\n * \"https://myaccount.blob.core.windows.net\". You can append a SAS\n * if using AnonymousCredential, such as \"https://myaccount.blob.core.windows.net?sasString\".\n * @param credential - Such as AnonymousCredential, StorageSharedKeyCredential or any credential from the `@azure/identity` package to authenticate requests to the service. You can also provide an object that implements the TokenCredential interface. If not specified, AnonymousCredential is used.\n * @param options - Optional. Options to configure the HTTP pipeline.\n *\n * Example using DefaultAzureCredential from `@azure/identity`:\n *\n * ```js\n * const account = \"<storage account name>\";\n *\n * const defaultAzureCredential = new DefaultAzureCredential();\n *\n * const blobChangeFeedClient = new BlobChangeFeedClient(\n * `https://${account}.blob.core.windows.net`,\n * defaultAzureCredential\n * );\n * ```\n *\n * Example using an account name/key:\n *\n * ```js\n * const account = \"<storage account name>\"\n * const sharedKeyCredential = new StorageSharedKeyCredential(account, \"<account key>\");\n *\n * const blobChangeFeedClient = new BlobChangeFeedClient(\n * `https://${account}.blob.core.windows.net`,\n * sharedKeyCredential\n * );\n * ```\n */\n constructor(\n url: string,\n credential?: StorageSharedKeyCredential | AnonymousCredential | TokenCredential,\n // Legacy, no way to fix the eslint error without breaking. Disable the rule for this line.\n /* eslint-disable-next-line @azure/azure-sdk/ts-naming-options */\n options?: StoragePipelineOptions\n );\n\n /**\n * Creates an instance of BlobChangeFeedClient.\n *\n * @param url - A Client string pointing to Azure Storage blob service, such as\n * \"https://myaccount.blob.core.windows.net\". You can append a SAS\n * if using AnonymousCredential, such as \"https://myaccount.blob.core.windows.net?sasString\".\n * @param pipeline - Call newPipeline() to create a default\n * pipeline, or provide a customized pipeline.\n */\n constructor(url: string, pipeline: Pipeline);\n constructor(\n urlOrClient: string,\n credentialOrPipeline?:\n | StorageSharedKeyCredential\n | AnonymousCredential\n | TokenCredential\n | Pipeline,\n // Legacy, no way to fix the eslint error without breaking. Disable the rule for this line.\n /* eslint-disable-next-line @azure/azure-sdk/ts-naming-options */\n options?: StoragePipelineOptions\n ) {\n this.changeFeedFactory = new ChangeFeedFactory();\n\n if (credentialOrPipeline instanceof Pipeline) {\n this.blobServiceClient = new BlobServiceClient(urlOrClient, credentialOrPipeline);\n } else {\n this.blobServiceClient = new BlobServiceClient(\n urlOrClient,\n credentialOrPipeline,\n appendUserAgentPrefix(options)\n );\n }\n }\n\n private async *getChange(\n options: BlobChangeFeedListChangesOptions = {}\n ): AsyncIterableIterator<BlobChangeFeedEvent> {\n const changeFeed: ChangeFeed = await this.changeFeedFactory.create(\n this.blobServiceClient,\n undefined,\n options\n );\n\n while (changeFeed.hasNext()) {\n const event = await changeFeed.getChange({\n abortSignal: options.abortSignal,\n tracingOptions: options.tracingOptions,\n });\n if (event) {\n yield event;\n } else {\n return;\n }\n }\n }\n\n // start in ChangeFeedListChangesOptions will be ignored when continuationToken is specified.\n private async *getPage(\n continuationToken?: string,\n maxPageSize?: number,\n options: BlobChangeFeedListChangesOptions = {}\n ): AsyncIterableIterator<BlobChangeFeedEventPage> {\n const changeFeed: ChangeFeed = await this.changeFeedFactory.create(\n this.blobServiceClient,\n continuationToken,\n options\n );\n\n if (!maxPageSize || maxPageSize > CHANGE_FEED_MAX_PAGE_SIZE) {\n maxPageSize = CHANGE_FEED_MAX_PAGE_SIZE;\n }\n while (changeFeed.hasNext()) {\n const eventPage = new BlobChangeFeedEventPage();\n while (changeFeed.hasNext() && eventPage.events.length < maxPageSize) {\n const event = await changeFeed.getChange({\n abortSignal: options.abortSignal,\n tracingOptions: options.tracingOptions,\n });\n if (event) {\n eventPage.events.push(event);\n }\n }\n if (changeFeed.hasNext()) {\n eventPage.continuationToken = JSON.stringify(changeFeed.getCursor());\n }\n if (eventPage.events.length > 0) {\n yield eventPage;\n } else {\n return;\n }\n }\n }\n\n /**\n * Returns an async iterable iterator to list all the change feed events\n * in the specified account.\n *\n * .byPage() returns an async iterable iterator to list the change feed events in pages.\n *\n * Example using `for await` syntax:\n *\n * ```js\n * let i = 1;\n * for await (const event of blobChangeFeedClient.listChanges()) {\n * console.log(`Event ${i++}, type: ${event.eventType}`);\n * }\n * ```\n *\n * Example using `iter.next()`:\n *\n * ```js\n * let i = 1;\n * const iter = blobChangeFeedClient.listChanges();\n * let eventItem = await iter.next();\n * while (!eventItem.done) {\n * console.log(`Event ${i++}, type: ${eventItem.eventType}`);\n * eventItem = await iter.next();\n * }\n * ```\n *\n * Example using `byPage()`:\n *\n * ```js\n * // passing optional maxPageSize in the page settings\n * let i = 1;\n * for await (const eventPage of blobChangeFeedClient.listChanges().byPage({ maxPageSize: 20 })) {\n * if (eventPage.events) {\n * for (const event of eventPage.events) {\n * console.log(`Event ${i++}, type: ${event.eventType}`);\n * }\n * }\n * }\n * ```\n *\n * Example using paging with a marker:\n *\n * ```js\n * let i = 1;\n * let iterator = blobChangeFeedClient.listChanges().byPage({ maxPageSize: 2 });\n * let eventPage = (await iterator.next()).value;\n *\n * if (eventPage.events) {\n * for (const container of eventPage.events) {\n * console.log(`Event ${i++}, type: ${event.eventType}`);\n * }\n * }\n *\n * // Gets next marker\n * let marker = eventPage.continuationToken;\n * // Passing next marker as continuationToken\n * iterator = blobChangeFeedClient\n * .listChanges()\n * .byPage({ continuationToken: marker, maxPageSize: 10 });\n * eventPage = (await iterator.next()).value;\n *\n * if (eventPage.events) {\n * for (const container of eventPage.events) {\n * console.log(`Event ${i++}, type: ${event.eventType}`);\n * }\n * }\n * ```\n *\n * @param options - Options to list change feed events.\n * @returns An asyncIterableIterator that supports paging.\n */\n public listChanges(\n options: BlobChangeFeedListChangesOptions = {}\n ): PagedAsyncIterableIterator<BlobChangeFeedEvent, BlobChangeFeedEventPage> {\n const iter = this.getChange(options);\n return {\n /**\n * The next method, part of the iteration protocol\n */\n async next() {\n return iter.next();\n },\n /**\n * The connection to the async iterator, part of the iteration protocol\n */\n [Symbol.asyncIterator]() {\n return this;\n },\n /**\n * Return an AsyncIterableIterator that works a page at a time\n */\n byPage: (settings: PageSettings = {}) => {\n return this.getPage(settings.continuationToken, settings.maxPageSize, options);\n },\n };\n }\n}\n"]}
|
|
@@ -31,7 +31,7 @@ export class ChangeFeed {
|
|
|
31
31
|
if (this.segments.length > 0) {
|
|
32
32
|
this.currentSegment = await this.segmentFactory.create(this.containerClient, this.segments.shift(), undefined, {
|
|
33
33
|
abortSignal: options.abortSignal,
|
|
34
|
-
tracingOptions: updatedOptions.tracingOptions
|
|
34
|
+
tracingOptions: updatedOptions.tracingOptions,
|
|
35
35
|
});
|
|
36
36
|
}
|
|
37
37
|
// If segments is empty, refill it
|
|
@@ -39,12 +39,12 @@ export class ChangeFeed {
|
|
|
39
39
|
const year = this.years.shift();
|
|
40
40
|
this.segments = await getSegmentsInYear(this.containerClient, year, this.startTime, this.end, {
|
|
41
41
|
abortSignal: options.abortSignal,
|
|
42
|
-
tracingOptions: updatedOptions.tracingOptions
|
|
42
|
+
tracingOptions: updatedOptions.tracingOptions,
|
|
43
43
|
});
|
|
44
44
|
if (this.segments.length > 0) {
|
|
45
45
|
this.currentSegment = await this.segmentFactory.create(this.containerClient, this.segments.shift(), undefined, {
|
|
46
46
|
abortSignal: options.abortSignal,
|
|
47
|
-
tracingOptions: updatedOptions.tracingOptions
|
|
47
|
+
tracingOptions: updatedOptions.tracingOptions,
|
|
48
48
|
});
|
|
49
49
|
}
|
|
50
50
|
else {
|
|
@@ -55,7 +55,7 @@ export class ChangeFeed {
|
|
|
55
55
|
catch (e) {
|
|
56
56
|
span.setStatus({
|
|
57
57
|
code: SpanStatusCode.ERROR,
|
|
58
|
-
message: e.message
|
|
58
|
+
message: e.message,
|
|
59
59
|
});
|
|
60
60
|
throw e;
|
|
61
61
|
}
|
|
@@ -80,11 +80,11 @@ export class ChangeFeed {
|
|
|
80
80
|
while (event === undefined && this.hasNext()) {
|
|
81
81
|
event = await this.currentSegment.getChange({
|
|
82
82
|
abortSignal: options.abortSignal,
|
|
83
|
-
tracingOptions: updatedOptions.tracingOptions
|
|
83
|
+
tracingOptions: updatedOptions.tracingOptions,
|
|
84
84
|
});
|
|
85
85
|
await this.advanceSegmentIfNecessary({
|
|
86
86
|
abortSignal: options.abortSignal,
|
|
87
|
-
tracingOptions: updatedOptions.tracingOptions
|
|
87
|
+
tracingOptions: updatedOptions.tracingOptions,
|
|
88
88
|
});
|
|
89
89
|
}
|
|
90
90
|
return event;
|
|
@@ -92,7 +92,7 @@ export class ChangeFeed {
|
|
|
92
92
|
catch (e) {
|
|
93
93
|
span.setStatus({
|
|
94
94
|
code: SpanStatusCode.ERROR,
|
|
95
|
-
message: e.message
|
|
95
|
+
message: e.message,
|
|
96
96
|
});
|
|
97
97
|
throw e;
|
|
98
98
|
}
|
|
@@ -109,7 +109,7 @@ export class ChangeFeed {
|
|
|
109
109
|
CursorVersion: 1,
|
|
110
110
|
UrlHost: getHost(this.containerClient.url),
|
|
111
111
|
EndTime: (_a = this.endTime) === null || _a === void 0 ? void 0 : _a.toJSON(),
|
|
112
|
-
CurrentSegmentCursor: this.currentSegment.getCursor()
|
|
112
|
+
CurrentSegmentCursor: this.currentSegment.getCursor(),
|
|
113
113
|
};
|
|
114
114
|
}
|
|
115
115
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ChangeFeed.js","sourceRoot":"","sources":["../../../src/ChangeFeed.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAOlC,OAAO,EAAE,iBAAiB,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAE3E,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAarD,MAAM,OAAO,UAAU;IAkCrB,YACE,eAAiC,EACjC,cAA+B,EAC/B,KAAgB,EAChB,QAAmB,EACnB,cAAwB,EACxB,cAAqB,EACrB,SAAgB,EAChB,OAAc;QAEd,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,KAAK,GAAG,KAAK,IAAI,EAAE,CAAC;QACzB,IAAI,CAAC,QAAQ,GAAG,QAAQ,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,IAAI,CAAC,cAAc,EAAE;YACvB,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;SACvD;IACH,CAAC;IAEO,KAAK,CAAC,yBAAyB,CAAC,UAAsC,EAAE;QAC9E,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,UAAU,CAAC,sCAAsC,EAAE,OAAO,CAAC,CAAC;QAC7F,IAAI;YACF,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE;gBACxB,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;aACpE;YAED,wEAAwE;YACxE,IAAI,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE;gBACjC,OAAO;aACR;YAED,iDAAiD;YACjD,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE;gBAC5B,IAAI,CAAC,cAAc,GAAG,MAAM,IAAI,CAAC,cAAe,CAAC,MAAM,CACrD,IAAI,CAAC,eAAgB,EACrB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAG,EACtB,SAAS,EACT;oBACE,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;iBAC9C,CACF,CAAC;aACH;YACD,kCAAkC;iBAC7B,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE;gBAC5D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;gBAChC,IAAI,CAAC,QAAQ,GAAG,MAAM,iBAAiB,CACrC,IAAI,CAAC,eAAgB,EACrB,IAAK,EACL,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,GAAG,EACR;oBACE,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;iBAC9C,CACF,CAAC;gBAEF,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE;oBAC5B,IAAI,CAAC,cAAc,GAAG,MAAM,IAAI,CAAC,cAAe,CAAC,MAAM,CACrD,IAAI,CAAC,eAAgB,EACrB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAG,EACtB,SAAS,EACT;wBACE,WAAW,EAAE,OAAO,CAAC,WAAW;wBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;qBAC9C,CACF,CAAC;iBACH;qBAAM;oBACL,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC;iBACjC;aACF;SACF;QAAC,OAAO,CAAC,EAAE;YACV,IAAI,CAAC,SAAS,CAAC;gBACb,IAAI,EAAE,cAAc,CAAC,KAAK;gBAC1B,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC,CAAC;YACH,MAAM,CAAC,CAAC;SACT;gBAAS;YACR,IAAI,CAAC,GAAG,EAAE,CAAC;SACZ;IACH,CAAC;IAEM,OAAO;QACZ,2DAA2D;QAC3D,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE;YACxB,OAAO,KAAK,CAAC;SACd;QAED,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE;YAC3F,OAAO,KAAK,CAAC;SACd;QAED,OAAO,IAAI,CAAC,cAAc,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAI,CAAC;IAClD,CAAC;IAEM,KAAK,CAAC,SAAS,CACpB,UAAsC,EAAE;QAExC,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,UAAU,CAAC,sBAAsB,EAAE,OAAO,CAAC,CAAC;QAC7E,IAAI;YACF,IAAI,KAAK,GAAoC,SAAS,CAAC;YACvD,OAAO,KAAK,KAAK,SAAS,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE;gBAC5C,KAAK,GAAG,MAAM,IAAI,CAAC,cAAe,CAAC,SAAS,CAAC;oBAC3C,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;iBAC9C,CAAC,CAAC;gBACH,MAAM,IAAI,CAAC,yBAAyB,CAAC;oBACnC,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;iBAC9C,CAAC,CAAC;aACJ;YACD,OAAO,KAAK,CAAC;SACd;QAAC,OAAO,CAAC,EAAE;YACV,IAAI,CAAC,SAAS,CAAC;gBACb,IAAI,EAAE,cAAc,CAAC,KAAK;gBAC1B,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC,CAAC;YACH,MAAM,CAAC,CAAC;SACT;gBAAS;YACR,IAAI,CAAC,GAAG,EAAE,CAAC;SACZ;IACH,CAAC;IAEM,SAAS;;QACd,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE;YACxB,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;SACpE;QAED,OAAO;YACL,aAAa,EAAE,CAAC;YAChB,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,eAAgB,CAAC,GAAG,CAAE;YAC5C,OAAO,EAAE,MAAA,IAAI,CAAC,OAAO,0CAAE,MAAM,EAAE;YAC/B,oBAAoB,EAAE,IAAI,CAAC,cAAe,CAAC,SAAS,EAAE;SACvD,CAAC;IACJ,CAAC;CACF","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\nimport { ContainerClient, CommonOptions } from \"@azure/storage-blob\";\nimport { Segment } from \"./Segment\";\nimport { SegmentFactory } from \"./SegmentFactory\";\nimport { BlobChangeFeedEvent } from \"./models/BlobChangeFeedEvent\";\nimport { ChangeFeedCursor } from \"./models/ChangeFeedCursor\";\nimport { getSegmentsInYear, minDate, getHost } from \"./utils/utils.common\";\nimport { AbortSignalLike } from \"@azure/core-http\";\nimport { createSpan } from \"./utils/tracing\";\nimport { SpanStatusCode } from \"@azure/core-tracing\";\n\n/**\n * Options to configure {@link ChangeFeed.getChange} operation.\n */\nexport interface ChangeFeedGetChangeOptions extends CommonOptions {\n /**\n * An implementation of the `AbortSignalLike` interface to signal the request to cancel the operation.\n * For example, use the @azure/abort-controller to create an `AbortSignal`.\n */\n abortSignal?: AbortSignalLike;\n}\n\nexport class ChangeFeed {\n /**\n * BlobContainerClient for making List Blob requests and creating Segments.\n */\n private readonly containerClient?: ContainerClient;\n\n private readonly segmentFactory?: SegmentFactory;\n\n private readonly years: number[];\n\n private segments: string[];\n\n private currentSegment?: Segment;\n\n private lastConsumable?: Date;\n\n private startTime?: Date;\n\n private endTime?: Date;\n\n private end?: Date;\n\n constructor();\n constructor(\n containerClient: ContainerClient,\n segmentFactory: SegmentFactory,\n years: number[],\n segments: string[],\n currentSegment: Segment,\n lastConsumable: Date,\n startTime?: Date,\n endTime?: Date\n );\n\n constructor(\n containerClient?: ContainerClient,\n segmentFactory?: SegmentFactory,\n years?: number[],\n segments?: string[],\n currentSegment?: Segment,\n lastConsumable?: Date,\n startTime?: Date,\n endTime?: Date\n ) {\n this.containerClient = containerClient;\n this.segmentFactory = segmentFactory;\n this.years = years || [];\n this.segments = segments || [];\n this.currentSegment = currentSegment;\n this.lastConsumable = lastConsumable;\n this.startTime = startTime;\n this.endTime = endTime;\n if (this.lastConsumable) {\n this.end = minDate(this.lastConsumable, this.endTime);\n }\n }\n\n private async advanceSegmentIfNecessary(options: ChangeFeedGetChangeOptions = {}): Promise<void> {\n const { span, updatedOptions } = createSpan(\"ChangeFeed-advanceSegmentIfNecessary\", options);\n try {\n if (!this.currentSegment) {\n throw new Error(\"Empty Change Feed shouldn't call this function.\");\n }\n\n // If the current segment has more Events, we don't need to do anything.\n if (this.currentSegment.hasNext()) {\n return;\n }\n\n // If the current segment is completed, remove it\n if (this.segments.length > 0) {\n this.currentSegment = await this.segmentFactory!.create(\n this.containerClient!,\n this.segments.shift()!,\n undefined,\n {\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions
|
|
1
|
+
{"version":3,"file":"ChangeFeed.js","sourceRoot":"","sources":["../../../src/ChangeFeed.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAOlC,OAAO,EAAE,iBAAiB,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAE3E,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAarD,MAAM,OAAO,UAAU;IAkCrB,YACE,eAAiC,EACjC,cAA+B,EAC/B,KAAgB,EAChB,QAAmB,EACnB,cAAwB,EACxB,cAAqB,EACrB,SAAgB,EAChB,OAAc;QAEd,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,KAAK,GAAG,KAAK,IAAI,EAAE,CAAC;QACzB,IAAI,CAAC,QAAQ,GAAG,QAAQ,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,IAAI,CAAC,cAAc,EAAE;YACvB,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;SACvD;IACH,CAAC;IAEO,KAAK,CAAC,yBAAyB,CAAC,UAAsC,EAAE;QAC9E,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,UAAU,CAAC,sCAAsC,EAAE,OAAO,CAAC,CAAC;QAC7F,IAAI;YACF,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE;gBACxB,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;aACpE;YAED,wEAAwE;YACxE,IAAI,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE;gBACjC,OAAO;aACR;YAED,iDAAiD;YACjD,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE;gBAC5B,IAAI,CAAC,cAAc,GAAG,MAAM,IAAI,CAAC,cAAe,CAAC,MAAM,CACrD,IAAI,CAAC,eAAgB,EACrB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAG,EACtB,SAAS,EACT;oBACE,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;iBAC9C,CACF,CAAC;aACH;YACD,kCAAkC;iBAC7B,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE;gBAC5D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;gBAChC,IAAI,CAAC,QAAQ,GAAG,MAAM,iBAAiB,CACrC,IAAI,CAAC,eAAgB,EACrB,IAAK,EACL,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,GAAG,EACR;oBACE,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;iBAC9C,CACF,CAAC;gBAEF,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE;oBAC5B,IAAI,CAAC,cAAc,GAAG,MAAM,IAAI,CAAC,cAAe,CAAC,MAAM,CACrD,IAAI,CAAC,eAAgB,EACrB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAG,EACtB,SAAS,EACT;wBACE,WAAW,EAAE,OAAO,CAAC,WAAW;wBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;qBAC9C,CACF,CAAC;iBACH;qBAAM;oBACL,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC;iBACjC;aACF;SACF;QAAC,OAAO,CAAC,EAAE;YACV,IAAI,CAAC,SAAS,CAAC;gBACb,IAAI,EAAE,cAAc,CAAC,KAAK;gBAC1B,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC,CAAC;YACH,MAAM,CAAC,CAAC;SACT;gBAAS;YACR,IAAI,CAAC,GAAG,EAAE,CAAC;SACZ;IACH,CAAC;IAEM,OAAO;QACZ,2DAA2D;QAC3D,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE;YACxB,OAAO,KAAK,CAAC;SACd;QAED,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE;YAC3F,OAAO,KAAK,CAAC;SACd;QAED,OAAO,IAAI,CAAC,cAAc,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAI,CAAC;IAClD,CAAC;IAEM,KAAK,CAAC,SAAS,CACpB,UAAsC,EAAE;QAExC,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,UAAU,CAAC,sBAAsB,EAAE,OAAO,CAAC,CAAC;QAC7E,IAAI;YACF,IAAI,KAAK,GAAoC,SAAS,CAAC;YACvD,OAAO,KAAK,KAAK,SAAS,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE;gBAC5C,KAAK,GAAG,MAAM,IAAI,CAAC,cAAe,CAAC,SAAS,CAAC;oBAC3C,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;iBAC9C,CAAC,CAAC;gBACH,MAAM,IAAI,CAAC,yBAAyB,CAAC;oBACnC,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;iBAC9C,CAAC,CAAC;aACJ;YACD,OAAO,KAAK,CAAC;SACd;QAAC,OAAO,CAAC,EAAE;YACV,IAAI,CAAC,SAAS,CAAC;gBACb,IAAI,EAAE,cAAc,CAAC,KAAK;gBAC1B,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC,CAAC;YACH,MAAM,CAAC,CAAC;SACT;gBAAS;YACR,IAAI,CAAC,GAAG,EAAE,CAAC;SACZ;IACH,CAAC;IAEM,SAAS;;QACd,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE;YACxB,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;SACpE;QAED,OAAO;YACL,aAAa,EAAE,CAAC;YAChB,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,eAAgB,CAAC,GAAG,CAAE;YAC5C,OAAO,EAAE,MAAA,IAAI,CAAC,OAAO,0CAAE,MAAM,EAAE;YAC/B,oBAAoB,EAAE,IAAI,CAAC,cAAe,CAAC,SAAS,EAAE;SACvD,CAAC;IACJ,CAAC;CACF","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\nimport { ContainerClient, CommonOptions } from \"@azure/storage-blob\";\nimport { Segment } from \"./Segment\";\nimport { SegmentFactory } from \"./SegmentFactory\";\nimport { BlobChangeFeedEvent } from \"./models/BlobChangeFeedEvent\";\nimport { ChangeFeedCursor } from \"./models/ChangeFeedCursor\";\nimport { getSegmentsInYear, minDate, getHost } from \"./utils/utils.common\";\nimport { AbortSignalLike } from \"@azure/core-http\";\nimport { createSpan } from \"./utils/tracing\";\nimport { SpanStatusCode } from \"@azure/core-tracing\";\n\n/**\n * Options to configure {@link ChangeFeed.getChange} operation.\n */\nexport interface ChangeFeedGetChangeOptions extends CommonOptions {\n /**\n * An implementation of the `AbortSignalLike` interface to signal the request to cancel the operation.\n * For example, use the @azure/abort-controller to create an `AbortSignal`.\n */\n abortSignal?: AbortSignalLike;\n}\n\nexport class ChangeFeed {\n /**\n * BlobContainerClient for making List Blob requests and creating Segments.\n */\n private readonly containerClient?: ContainerClient;\n\n private readonly segmentFactory?: SegmentFactory;\n\n private readonly years: number[];\n\n private segments: string[];\n\n private currentSegment?: Segment;\n\n private lastConsumable?: Date;\n\n private startTime?: Date;\n\n private endTime?: Date;\n\n private end?: Date;\n\n constructor();\n constructor(\n containerClient: ContainerClient,\n segmentFactory: SegmentFactory,\n years: number[],\n segments: string[],\n currentSegment: Segment,\n lastConsumable: Date,\n startTime?: Date,\n endTime?: Date\n );\n\n constructor(\n containerClient?: ContainerClient,\n segmentFactory?: SegmentFactory,\n years?: number[],\n segments?: string[],\n currentSegment?: Segment,\n lastConsumable?: Date,\n startTime?: Date,\n endTime?: Date\n ) {\n this.containerClient = containerClient;\n this.segmentFactory = segmentFactory;\n this.years = years || [];\n this.segments = segments || [];\n this.currentSegment = currentSegment;\n this.lastConsumable = lastConsumable;\n this.startTime = startTime;\n this.endTime = endTime;\n if (this.lastConsumable) {\n this.end = minDate(this.lastConsumable, this.endTime);\n }\n }\n\n private async advanceSegmentIfNecessary(options: ChangeFeedGetChangeOptions = {}): Promise<void> {\n const { span, updatedOptions } = createSpan(\"ChangeFeed-advanceSegmentIfNecessary\", options);\n try {\n if (!this.currentSegment) {\n throw new Error(\"Empty Change Feed shouldn't call this function.\");\n }\n\n // If the current segment has more Events, we don't need to do anything.\n if (this.currentSegment.hasNext()) {\n return;\n }\n\n // If the current segment is completed, remove it\n if (this.segments.length > 0) {\n this.currentSegment = await this.segmentFactory!.create(\n this.containerClient!,\n this.segments.shift()!,\n undefined,\n {\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n }\n );\n }\n // If segments is empty, refill it\n else if (this.segments.length === 0 && this.years.length > 0) {\n const year = this.years.shift();\n this.segments = await getSegmentsInYear(\n this.containerClient!,\n year!,\n this.startTime,\n this.end,\n {\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n }\n );\n\n if (this.segments.length > 0) {\n this.currentSegment = await this.segmentFactory!.create(\n this.containerClient!,\n this.segments.shift()!,\n undefined,\n {\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n }\n );\n } else {\n this.currentSegment = undefined;\n }\n }\n } catch (e) {\n span.setStatus({\n code: SpanStatusCode.ERROR,\n message: e.message,\n });\n throw e;\n } finally {\n span.end();\n }\n }\n\n public hasNext(): boolean {\n // Empty ChangeFeed, using currentSegment as the indicator.\n if (!this.currentSegment) {\n return false;\n }\n\n if (this.segments.length === 0 && this.years.length === 0 && !this.currentSegment.hasNext()) {\n return false;\n }\n\n return this.currentSegment.dateTime < this.end!;\n }\n\n public async getChange(\n options: ChangeFeedGetChangeOptions = {}\n ): Promise<BlobChangeFeedEvent | undefined> {\n const { span, updatedOptions } = createSpan(\"ChangeFeed-getChange\", options);\n try {\n let event: BlobChangeFeedEvent | undefined = undefined;\n while (event === undefined && this.hasNext()) {\n event = await this.currentSegment!.getChange({\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n });\n await this.advanceSegmentIfNecessary({\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n });\n }\n return event;\n } catch (e) {\n span.setStatus({\n code: SpanStatusCode.ERROR,\n message: e.message,\n });\n throw e;\n } finally {\n span.end();\n }\n }\n\n public getCursor(): ChangeFeedCursor {\n if (!this.currentSegment) {\n throw new Error(\"Empty Change Feed shouldn't call this function.\");\n }\n\n return {\n CursorVersion: 1,\n UrlHost: getHost(this.containerClient!.url)!,\n EndTime: this.endTime?.toJSON(),\n CurrentSegmentCursor: this.currentSegment!.getCursor(),\n };\n }\n}\n"]}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Licensed under the MIT license.
|
|
3
3
|
import { ChangeFeed } from "./ChangeFeed";
|
|
4
4
|
import { CHANGE_FEED_CONTAINER_NAME, CHANGE_FEED_META_SEGMENT_PATH } from "./utils/constants";
|
|
5
|
-
import { ceilToNearestHour, floorToNearestHour, getYearsPaths, getSegmentsInYear, minDate, getHost, parseDateFromSegmentPath } from "./utils/utils.common";
|
|
5
|
+
import { ceilToNearestHour, floorToNearestHour, getYearsPaths, getSegmentsInYear, minDate, getHost, parseDateFromSegmentPath, } from "./utils/utils.common";
|
|
6
6
|
import { bodyToString } from "./utils/utils.node";
|
|
7
7
|
import { SegmentFactory } from "./SegmentFactory";
|
|
8
8
|
import { ShardFactory } from "./ShardFactory";
|
|
@@ -48,7 +48,7 @@ export class ChangeFeedFactory {
|
|
|
48
48
|
// Check if Change Feed has been enabled for this account.
|
|
49
49
|
const changeFeedContainerExists = await containerClient.exists({
|
|
50
50
|
abortSignal: options.abortSignal,
|
|
51
|
-
tracingOptions: updatedOptions.tracingOptions
|
|
51
|
+
tracingOptions: updatedOptions.tracingOptions,
|
|
52
52
|
});
|
|
53
53
|
if (!changeFeedContainerExists) {
|
|
54
54
|
throw new Error("Change Feed hasn't been enabled on this account, or is currently being enabled.");
|
|
@@ -60,13 +60,13 @@ export class ChangeFeedFactory {
|
|
|
60
60
|
const blobClient = containerClient.getBlobClient(CHANGE_FEED_META_SEGMENT_PATH);
|
|
61
61
|
const blobDownloadRes = await blobClient.download(undefined, undefined, {
|
|
62
62
|
abortSignal: options.abortSignal,
|
|
63
|
-
tracingOptions: updatedOptions.tracingOptions
|
|
63
|
+
tracingOptions: updatedOptions.tracingOptions,
|
|
64
64
|
});
|
|
65
65
|
const lastConsumable = new Date(JSON.parse(await bodyToString(blobDownloadRes)).lastConsumable);
|
|
66
66
|
// Get year paths
|
|
67
67
|
const years = await getYearsPaths(containerClient, {
|
|
68
68
|
abortSignal: options.abortSignal,
|
|
69
|
-
tracingOptions: updatedOptions.tracingOptions
|
|
69
|
+
tracingOptions: updatedOptions.tracingOptions,
|
|
70
70
|
});
|
|
71
71
|
// Dequeue any years that occur before start time.
|
|
72
72
|
if (options.start) {
|
|
@@ -82,7 +82,7 @@ export class ChangeFeedFactory {
|
|
|
82
82
|
while (segments.length === 0 && years.length !== 0) {
|
|
83
83
|
segments = await getSegmentsInYear(containerClient, years.shift(), options.start, minDate(lastConsumable, options.end), {
|
|
84
84
|
abortSignal: options.abortSignal,
|
|
85
|
-
tracingOptions: updatedOptions.tracingOptions
|
|
85
|
+
tracingOptions: updatedOptions.tracingOptions,
|
|
86
86
|
});
|
|
87
87
|
}
|
|
88
88
|
if (segments.length === 0) {
|
|
@@ -90,14 +90,14 @@ export class ChangeFeedFactory {
|
|
|
90
90
|
}
|
|
91
91
|
const currentSegment = await this.segmentFactory.create(containerClient, segments.shift(), cursor === null || cursor === void 0 ? void 0 : cursor.CurrentSegmentCursor, {
|
|
92
92
|
abortSignal: options.abortSignal,
|
|
93
|
-
tracingOptions: updatedOptions.tracingOptions
|
|
93
|
+
tracingOptions: updatedOptions.tracingOptions,
|
|
94
94
|
});
|
|
95
95
|
return new ChangeFeed(containerClient, this.segmentFactory, years, segments, currentSegment, lastConsumable, options.start, options.end);
|
|
96
96
|
}
|
|
97
97
|
catch (e) {
|
|
98
98
|
span.setStatus({
|
|
99
99
|
code: SpanStatusCode.ERROR,
|
|
100
|
-
message: e.message
|
|
100
|
+
message: e.message,
|
|
101
101
|
});
|
|
102
102
|
throw e;
|
|
103
103
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ChangeFeedFactory.js","sourceRoot":"","sources":["../../../src/ChangeFeedFactory.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAGlC,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE1C,OAAO,EAAE,0BAA0B,EAAE,6BAA6B,EAAE,MAAM,mBAAmB,CAAC;AAC9F,OAAO,EACL,iBAAiB,EACjB,kBAAkB,EAClB,aAAa,EACb,iBAAiB,EACjB,OAAO,EACP,OAAO,EACP,wBAAwB,
|
|
1
|
+
{"version":3,"file":"ChangeFeedFactory.js","sourceRoot":"","sources":["../../../src/ChangeFeedFactory.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAGlC,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE1C,OAAO,EAAE,0BAA0B,EAAE,6BAA6B,EAAE,MAAM,mBAAmB,CAAC;AAC9F,OAAO,EACL,iBAAiB,EACjB,kBAAkB,EAClB,aAAa,EACb,iBAAiB,EACjB,OAAO,EACP,OAAO,EACP,wBAAwB,GACzB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAGxD,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,4BAA4B,EAAE,MAAM,gCAAgC,CAAC;AAO9E,MAAM,OAAO,iBAAiB;IAK5B,YAAY,cAA+B;QACzC,IAAI,cAAc,EAAE;YAClB,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;SACtC;aAAM;YACL,IAAI,CAAC,cAAc,GAAG,IAAI,cAAc,CACtC,IAAI,YAAY,CACd,IAAI,YAAY,CAAC,IAAI,iBAAiB,EAAE,EAAE,IAAI,4BAA4B,EAAE,CAAC,CAC9E,CACF,CAAC;SACH;IACH,CAAC;IAEO,MAAM,CAAC,cAAc,CAAC,eAAgC,EAAE,MAAwB;QACtF,IAAI,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,MAAM,CAAC,OAAO,EAAE;YACnD,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;SACvE;QACD,IAAI,MAAM,CAAC,aAAa,KAAK,CAAC,EAAE;YAC9B,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;SAChD;IACH,CAAC;IAEM,KAAK,CAAC,MAAM,CACjB,iBAAoC,EACpC,iBAA0B,EAC1B,UAA4C,EAAE;QAE9C,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,UAAU,CAAC,0BAA0B,EAAE,OAAO,CAAC,CAAC;QAEjF,IAAI;YACF,MAAM,eAAe,GAAG,iBAAiB,CAAC,kBAAkB,CAAC,0BAA0B,CAAC,CAAC;YACzF,IAAI,MAAM,GAAiC,SAAS,CAAC;YACrD,iBAAiB;YACjB,IAAI,iBAAiB,EAAE;gBACrB,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;gBACvC,iBAAiB,CAAC,cAAc,CAAC,eAAe,EAAE,MAAO,CAAC,CAAC;gBAC3D,OAAO,CAAC,KAAK,GAAG,wBAAwB,CAAC,MAAO,CAAC,oBAAoB,CAAC,WAAY,CAAC,CAAC;gBACpF,OAAO,CAAC,GAAG,GAAG,IAAI,IAAI,CAAC,MAAO,CAAC,OAAQ,CAAC,CAAC;aAC1C;YACD,2DAA2D;iBACtD;gBACH,OAAO,CAAC,KAAK,GAAG,kBAAkB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBAClD,OAAO,CAAC,GAAG,GAAG,iBAAiB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;aAC9C;YAED,0DAA0D;YAC1D,MAAM,yBAAyB,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC;gBAC7D,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;aAC9C,CAAC,CAAC;YACH,IAAI,CAAC,yBAAyB,EAAE;gBAC9B,MAAM,IAAI,KAAK,CACb,iFAAiF,CAClF,CAAC;aACH;YAED,IAAI,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,GAAG,EAAE;gBAChE,OAAO,IAAI,UAAU,EAAE,CAAC;aACzB;YAED,uBAAuB;YACvB,MAAM,UAAU,GAAG,eAAe,CAAC,aAAa,CAAC,6BAA6B,CAAC,CAAC;YAChF,MAAM,eAAe,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,SAAS,EAAE,SAAS,EAAE;gBACtE,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;aAC9C,CAAC,CAAC;YACH,MAAM,cAAc,GAAG,IAAI,IAAI,CAC5B,IAAI,CAAC,KAAK,CAAC,MAAM,YAAY,CAAC,eAAe,CAAC,CAAkB,CAAC,cAAc,CACjF,CAAC;YAEF,iBAAiB;YACjB,MAAM,KAAK,GAAa,MAAM,aAAa,CAAC,eAAe,EAAE;gBAC3D,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;aAC9C,CAAC,CAAC;YAEH,kDAAkD;YAClD,IAAI,OAAO,CAAC,KAAK,EAAE;gBACjB,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;gBACjD,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,SAAS,EAAE;oBAC/C,KAAK,CAAC,KAAK,EAAE,CAAC;iBACf;aACF;YACD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;gBACtB,OAAO,IAAI,UAAU,EAAE,CAAC;aACzB;YAED,IAAI,QAAQ,GAAa,EAAE,CAAC;YAC5B,OAAO,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;gBAClD,QAAQ,GAAG,MAAM,iBAAiB,CAChC,eAAe,EACf,KAAK,CAAC,KAAK,EAAG,EACd,OAAO,CAAC,KAAK,EACb,OAAO,CAAC,cAAc,EAAE,OAAO,CAAC,GAAG,CAAC,EACpC;oBACE,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;iBAC9C,CACF,CAAC;aACH;YACD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE;gBACzB,OAAO,IAAI,UAAU,EAAE,CAAC;aACzB;YACD,MAAM,cAAc,GAAY,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,CAC9D,eAAe,EACf,QAAQ,CAAC,KAAK,EAAG,EACjB,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,oBAAoB,EAC5B;gBACE,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;aAC9C,CACF,CAAC;YAEF,OAAO,IAAI,UAAU,CACnB,eAAe,EACf,IAAI,CAAC,cAAc,EACnB,KAAK,EACL,QAAQ,EACR,cAAc,EACd,cAAc,EACd,OAAO,CAAC,KAAK,EACb,OAAO,CAAC,GAAG,CACZ,CAAC;SACH;QAAC,OAAO,CAAC,EAAE;YACV,IAAI,CAAC,SAAS,CAAC;gBACb,IAAI,EAAE,cAAc,CAAC,KAAK;gBAC1B,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC,CAAC;YACH,MAAM,CAAC,CAAC;SACT;gBAAS;YACR,IAAI,CAAC,GAAG,EAAE,CAAC;SACZ;IACH,CAAC;CACF","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\nimport { BlobServiceClient, ContainerClient } from \"@azure/storage-blob\";\nimport { ChangeFeed } from \"./ChangeFeed\";\nimport { ChangeFeedCursor } from \"./models/ChangeFeedCursor\";\nimport { CHANGE_FEED_CONTAINER_NAME, CHANGE_FEED_META_SEGMENT_PATH } from \"./utils/constants\";\nimport {\n ceilToNearestHour,\n floorToNearestHour,\n getYearsPaths,\n getSegmentsInYear,\n minDate,\n getHost,\n parseDateFromSegmentPath,\n} from \"./utils/utils.common\";\nimport { bodyToString } from \"./utils/utils.node\";\nimport { SegmentFactory } from \"./SegmentFactory\";\nimport { ShardFactory } from \"./ShardFactory\";\nimport { ChunkFactory } from \"./ChunkFactory\";\nimport { AvroReaderFactory } from \"./AvroReaderFactory\";\nimport { Segment } from \"./Segment\";\nimport { BlobChangeFeedListChangesOptions } from \"./models/models\";\nimport { createSpan } from \"./utils/tracing\";\nimport { SpanStatusCode } from \"@azure/core-tracing\";\nimport { LazyLoadingBlobStreamFactory } from \"./LazyLoadingBlobStreamFactory\";\n\ninterface MetaSegments {\n version?: number;\n lastConsumable: string;\n}\n\nexport class ChangeFeedFactory {\n private readonly segmentFactory: SegmentFactory;\n\n constructor();\n constructor(segmentFactory: SegmentFactory);\n constructor(segmentFactory?: SegmentFactory) {\n if (segmentFactory) {\n this.segmentFactory = segmentFactory;\n } else {\n this.segmentFactory = new SegmentFactory(\n new ShardFactory(\n new ChunkFactory(new AvroReaderFactory(), new LazyLoadingBlobStreamFactory())\n )\n );\n }\n }\n\n private static validateCursor(containerClient: ContainerClient, cursor: ChangeFeedCursor): void {\n if (getHost(containerClient.url) !== cursor.UrlHost) {\n throw new Error(\"Cursor URL host does not match container URL host.\");\n }\n if (cursor.CursorVersion !== 1) {\n throw new Error(\"Unsupported cursor version.\");\n }\n }\n\n public async create(\n blobServiceClient: BlobServiceClient,\n continuationToken?: string,\n options: BlobChangeFeedListChangesOptions = {}\n ): Promise<ChangeFeed> {\n const { span, updatedOptions } = createSpan(\"ChangeFeedFactory-create\", options);\n\n try {\n const containerClient = blobServiceClient.getContainerClient(CHANGE_FEED_CONTAINER_NAME);\n let cursor: ChangeFeedCursor | undefined = undefined;\n // Create cursor.\n if (continuationToken) {\n cursor = JSON.parse(continuationToken);\n ChangeFeedFactory.validateCursor(containerClient, cursor!);\n options.start = parseDateFromSegmentPath(cursor!.CurrentSegmentCursor.SegmentPath!);\n options.end = new Date(cursor!.EndTime!);\n }\n // Round start and end time if we are not using the cursor.\n else {\n options.start = floorToNearestHour(options.start);\n options.end = ceilToNearestHour(options.end);\n }\n\n // Check if Change Feed has been enabled for this account.\n const changeFeedContainerExists = await containerClient.exists({\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n });\n if (!changeFeedContainerExists) {\n throw new Error(\n \"Change Feed hasn't been enabled on this account, or is currently being enabled.\"\n );\n }\n\n if (options.start && options.end && options.start >= options.end) {\n return new ChangeFeed();\n }\n\n // Get last consumable.\n const blobClient = containerClient.getBlobClient(CHANGE_FEED_META_SEGMENT_PATH);\n const blobDownloadRes = await blobClient.download(undefined, undefined, {\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n });\n const lastConsumable = new Date(\n (JSON.parse(await bodyToString(blobDownloadRes)) as MetaSegments).lastConsumable\n );\n\n // Get year paths\n const years: number[] = await getYearsPaths(containerClient, {\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n });\n\n // Dequeue any years that occur before start time.\n if (options.start) {\n const startYear = options.start.getUTCFullYear();\n while (years.length > 0 && years[0] < startYear) {\n years.shift();\n }\n }\n if (years.length === 0) {\n return new ChangeFeed();\n }\n\n let segments: string[] = [];\n while (segments.length === 0 && years.length !== 0) {\n segments = await getSegmentsInYear(\n containerClient,\n years.shift()!,\n options.start,\n minDate(lastConsumable, options.end),\n {\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n }\n );\n }\n if (segments.length === 0) {\n return new ChangeFeed();\n }\n const currentSegment: Segment = await this.segmentFactory.create(\n containerClient,\n segments.shift()!,\n cursor?.CurrentSegmentCursor,\n {\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n }\n );\n\n return new ChangeFeed(\n containerClient,\n this.segmentFactory,\n years,\n segments,\n currentSegment,\n lastConsumable,\n options.start,\n options.end\n );\n } catch (e) {\n span.setStatus({\n code: SpanStatusCode.ERROR,\n message: e.message,\n });\n throw e;\n } finally {\n span.end();\n }\n }\n}\n"]}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ChunkFactory.js","sourceRoot":"","sources":["../../../src/ChunkFactory.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAIlC,OAAO,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAEhC,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAG1D,OAAO,EAAE,qCAAqC,EAAE,MAAM,mBAAmB,CAAC;AAa1E,MAAM,OAAO,YAAY;IAIvB,YACE,iBAAoC,EACpC,4BAA0D;QAE1D,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAC3C,IAAI,CAAC,4BAA4B,GAAG,4BAA4B,CAAC;IACnE,CAAC;IAEM,KAAK,CAAC,MAAM,CACjB,eAAgC,EAChC,SAAiB,EACjB,WAAoB,EACpB,UAAmB,EACnB,UAA8B,EAAE;QAEhC,MAAM,UAAU,GAAG,eAAe,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAC5D,WAAW,GAAG,WAAW,IAAI,CAAC,CAAC;QAC/B,UAAU,GAAG,UAAU,IAAI,CAAC,CAAC;QAE7B,MAAM,UAAU,GAAG,oBAAoB,CACrC,IAAI,CAAC,4BAA4B,CAAC,MAAM,CACtC,UAAU,EACV,WAAW,EACX,qCAAqC,EACrC,OAAO,CACR,CACF,CAAC;QAEF,IAAI,UAAsB,CAAC;QAC3B,IAAI,WAAW,KAAK,CAAC,EAAE;YACrB,MAAM,YAAY,GAAG,oBAAoB,CACvC,IAAI,CAAC,4BAA4B,CAAC,MAAM,CACtC,UAAU,EACV,CAAC,EACD,qCAAqC,EACrC,OAAO,CACR,CACF,CAAC;YACF,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;SAC/F;aAAM;YACL,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;SACxD;QAED,OAAO,IAAI,KAAK,CAAC,UAAU,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,EAAE;YAC/D,WAAW,EAAE,OAAO,CAAC,WAAW;SACjC,CAAC,CAAC;IACL,CAAC;CACF","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\nimport { AvroReaderFactory } from \"./AvroReaderFactory\";\nimport { ContainerClient, CommonOptions } from \"@azure/storage-blob\";\nimport { Chunk } from \"./Chunk\";\nimport { AvroReader } from \"../../storage-internal-avro/src\";\nimport { streamToAvroReadable } from \"./utils/utils.node\";\nimport { AbortSignalLike } from \"@azure/core-http\";\nimport { LazyLoadingBlobStreamFactory } from \"./LazyLoadingBlobStreamFactory\";\nimport { CHANGE_FEED_CHUNK_BLOCK_DOWNLOAD_SIZE } from \"./utils/constants\";\n\n/**\n * Options to configure {@link ChunkFactory.create} operation.\n */\nexport interface CreateChunkOptions extends CommonOptions {\n /**\n * An implementation of the `AbortSignalLike` interface to signal the request to cancel the operation.\n * For example, use the @azure/abort-controller to create an `AbortSignal`.\n */\n abortSignal?: AbortSignalLike;\n}\n\nexport class ChunkFactory {\n private readonly avroReaderFactory: AvroReaderFactory;\n private readonly lazyLoadingBlobStreamFactory: LazyLoadingBlobStreamFactory;\n\n constructor(\n avroReaderFactory: AvroReaderFactory,\n lazyLoadingBlobStreamFactory: LazyLoadingBlobStreamFactory\n ) {\n this.avroReaderFactory = avroReaderFactory;\n this.lazyLoadingBlobStreamFactory = lazyLoadingBlobStreamFactory;\n }\n\n public async create(\n containerClient: ContainerClient,\n chunkPath: string,\n blockOffset?: number,\n eventIndex?: number,\n options: CreateChunkOptions = {}\n ): Promise<Chunk> {\n const blobClient = containerClient.getBlobClient(chunkPath);\n blockOffset = blockOffset || 0;\n eventIndex = eventIndex || 0;\n\n const dataStream = streamToAvroReadable(\n this.lazyLoadingBlobStreamFactory.create(\n blobClient,\n blockOffset,\n CHANGE_FEED_CHUNK_BLOCK_DOWNLOAD_SIZE,\n options\n )\n );\n\n let avroReader: AvroReader;\n if (blockOffset !== 0) {\n const headerStream = streamToAvroReadable(\n this.lazyLoadingBlobStreamFactory.create(\n blobClient,\n 0,\n CHANGE_FEED_CHUNK_BLOCK_DOWNLOAD_SIZE,\n options\n )\n );\n avroReader = this.avroReaderFactory.create(dataStream, headerStream, blockOffset, eventIndex);\n } else {\n avroReader = this.avroReaderFactory.create(dataStream);\n }\n\n return new Chunk(avroReader, blockOffset, eventIndex, chunkPath, {\n abortSignal: options.abortSignal
|
|
1
|
+
{"version":3,"file":"ChunkFactory.js","sourceRoot":"","sources":["../../../src/ChunkFactory.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAIlC,OAAO,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAEhC,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAG1D,OAAO,EAAE,qCAAqC,EAAE,MAAM,mBAAmB,CAAC;AAa1E,MAAM,OAAO,YAAY;IAIvB,YACE,iBAAoC,EACpC,4BAA0D;QAE1D,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAC3C,IAAI,CAAC,4BAA4B,GAAG,4BAA4B,CAAC;IACnE,CAAC;IAEM,KAAK,CAAC,MAAM,CACjB,eAAgC,EAChC,SAAiB,EACjB,WAAoB,EACpB,UAAmB,EACnB,UAA8B,EAAE;QAEhC,MAAM,UAAU,GAAG,eAAe,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAC5D,WAAW,GAAG,WAAW,IAAI,CAAC,CAAC;QAC/B,UAAU,GAAG,UAAU,IAAI,CAAC,CAAC;QAE7B,MAAM,UAAU,GAAG,oBAAoB,CACrC,IAAI,CAAC,4BAA4B,CAAC,MAAM,CACtC,UAAU,EACV,WAAW,EACX,qCAAqC,EACrC,OAAO,CACR,CACF,CAAC;QAEF,IAAI,UAAsB,CAAC;QAC3B,IAAI,WAAW,KAAK,CAAC,EAAE;YACrB,MAAM,YAAY,GAAG,oBAAoB,CACvC,IAAI,CAAC,4BAA4B,CAAC,MAAM,CACtC,UAAU,EACV,CAAC,EACD,qCAAqC,EACrC,OAAO,CACR,CACF,CAAC;YACF,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;SAC/F;aAAM;YACL,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;SACxD;QAED,OAAO,IAAI,KAAK,CAAC,UAAU,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,EAAE;YAC/D,WAAW,EAAE,OAAO,CAAC,WAAW;SACjC,CAAC,CAAC;IACL,CAAC;CACF","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\nimport { AvroReaderFactory } from \"./AvroReaderFactory\";\nimport { ContainerClient, CommonOptions } from \"@azure/storage-blob\";\nimport { Chunk } from \"./Chunk\";\nimport { AvroReader } from \"../../storage-internal-avro/src\";\nimport { streamToAvroReadable } from \"./utils/utils.node\";\nimport { AbortSignalLike } from \"@azure/core-http\";\nimport { LazyLoadingBlobStreamFactory } from \"./LazyLoadingBlobStreamFactory\";\nimport { CHANGE_FEED_CHUNK_BLOCK_DOWNLOAD_SIZE } from \"./utils/constants\";\n\n/**\n * Options to configure {@link ChunkFactory.create} operation.\n */\nexport interface CreateChunkOptions extends CommonOptions {\n /**\n * An implementation of the `AbortSignalLike` interface to signal the request to cancel the operation.\n * For example, use the @azure/abort-controller to create an `AbortSignal`.\n */\n abortSignal?: AbortSignalLike;\n}\n\nexport class ChunkFactory {\n private readonly avroReaderFactory: AvroReaderFactory;\n private readonly lazyLoadingBlobStreamFactory: LazyLoadingBlobStreamFactory;\n\n constructor(\n avroReaderFactory: AvroReaderFactory,\n lazyLoadingBlobStreamFactory: LazyLoadingBlobStreamFactory\n ) {\n this.avroReaderFactory = avroReaderFactory;\n this.lazyLoadingBlobStreamFactory = lazyLoadingBlobStreamFactory;\n }\n\n public async create(\n containerClient: ContainerClient,\n chunkPath: string,\n blockOffset?: number,\n eventIndex?: number,\n options: CreateChunkOptions = {}\n ): Promise<Chunk> {\n const blobClient = containerClient.getBlobClient(chunkPath);\n blockOffset = blockOffset || 0;\n eventIndex = eventIndex || 0;\n\n const dataStream = streamToAvroReadable(\n this.lazyLoadingBlobStreamFactory.create(\n blobClient,\n blockOffset,\n CHANGE_FEED_CHUNK_BLOCK_DOWNLOAD_SIZE,\n options\n )\n );\n\n let avroReader: AvroReader;\n if (blockOffset !== 0) {\n const headerStream = streamToAvroReadable(\n this.lazyLoadingBlobStreamFactory.create(\n blobClient,\n 0,\n CHANGE_FEED_CHUNK_BLOCK_DOWNLOAD_SIZE,\n options\n )\n );\n avroReader = this.avroReaderFactory.create(dataStream, headerStream, blockOffset, eventIndex);\n } else {\n avroReader = this.avroReaderFactory.create(dataStream);\n }\n\n return new Chunk(avroReader, blockOffset, eventIndex, chunkPath, {\n abortSignal: options.abortSignal,\n });\n }\n}\n"]}
|
|
@@ -26,7 +26,7 @@ export class LazyLoadingBlobStream extends Readable {
|
|
|
26
26
|
try {
|
|
27
27
|
const properties = await this.blobClient.getProperties({
|
|
28
28
|
abortSignal: options.abortSignal,
|
|
29
|
-
tracingOptions: updatedOptions.tracingOptions
|
|
29
|
+
tracingOptions: updatedOptions.tracingOptions,
|
|
30
30
|
});
|
|
31
31
|
this.blobLength = properties.contentLength;
|
|
32
32
|
this.lastDownloadBytes = Math.min(this.blockSize, this.blobLength - this.offset);
|
|
@@ -36,14 +36,14 @@ export class LazyLoadingBlobStream extends Readable {
|
|
|
36
36
|
}
|
|
37
37
|
this.lastDownloadData = await this.blobClient.downloadToBuffer(this.offset, this.lastDownloadBytes, {
|
|
38
38
|
abortSignal: options.abortSignal,
|
|
39
|
-
tracingOptions: updatedOptions.tracingOptions
|
|
39
|
+
tracingOptions: updatedOptions.tracingOptions,
|
|
40
40
|
});
|
|
41
41
|
this.offset += this.lastDownloadBytes;
|
|
42
42
|
}
|
|
43
43
|
catch (e) {
|
|
44
44
|
span.setStatus({
|
|
45
45
|
code: SpanStatusCode.ERROR,
|
|
46
|
-
message: e.message
|
|
46
|
+
message: e.message,
|
|
47
47
|
});
|
|
48
48
|
throw e;
|
|
49
49
|
}
|
|
@@ -70,7 +70,7 @@ export class LazyLoadingBlobStream extends Readable {
|
|
|
70
70
|
if (this.lastDownloadData === undefined || ((_a = this.lastDownloadData) === null || _a === void 0 ? void 0 : _a.byteLength) === 0) {
|
|
71
71
|
await this.downloadBlock({
|
|
72
72
|
abortSignal: (_b = this.options) === null || _b === void 0 ? void 0 : _b.abortSignal,
|
|
73
|
-
tracingOptions: updatedOptions === null || updatedOptions === void 0 ? void 0 : updatedOptions.tracingOptions
|
|
73
|
+
tracingOptions: updatedOptions === null || updatedOptions === void 0 ? void 0 : updatedOptions.tracingOptions,
|
|
74
74
|
});
|
|
75
75
|
}
|
|
76
76
|
if ((_c = this.lastDownloadData) === null || _c === void 0 ? void 0 : _c.byteLength) {
|
|
@@ -91,7 +91,7 @@ export class LazyLoadingBlobStream extends Readable {
|
|
|
91
91
|
catch (e) {
|
|
92
92
|
span.setStatus({
|
|
93
93
|
code: SpanStatusCode.ERROR,
|
|
94
|
-
message: e.message
|
|
94
|
+
message: e.message,
|
|
95
95
|
});
|
|
96
96
|
this.emit("error", e);
|
|
97
97
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LazyLoadingBlobStream.js","sourceRoot":"","sources":["../../../src/LazyLoadingBlobStream.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAElC,OAAO,EAAE,QAAQ,EAAmB,MAAM,QAAQ,CAAC;AAGnD,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAqBrD;;GAEG;AACH,MAAM,OAAO,qBAAsB,SAAQ,QAAQ;IAqBjD;;;;OAIG;IACH,YACE,UAAsB,EACtB,MAAc,EACd,SAAiB,EACjB,OAAsC;QAEtC,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,iBAAiB,GAAG,CAAC,CAAC,CAAC;QAC5B,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,UAAqD,EAAE;QACjF,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,UAAU,CAAC,qCAAqC,EAAE,OAAO,CAAC,CAAC;QAC5F,IAAI;YACF,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC;gBACrD,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;aAC9C,CAAC,CAAC;YACH,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,aAAc,CAAC;YAE5C,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;YACjF,IAAI,IAAI,CAAC,iBAAiB,KAAK,CAAC,EAAE;gBAChC,IAAI,CAAC,gBAAgB,GAAG,SAAS,CAAC;gBAClC,OAAO;aACR;YAED,IAAI,CAAC,gBAAgB,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAC5D,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,iBAAiB,EACtB;gBACE,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;aAC9C,CACF,CAAC;YACF,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,iBAAiB,CAAC;SACvC;QAAC,OAAO,CAAC,EAAE;YACV,IAAI,CAAC,SAAS,CAAC;gBACb,IAAI,EAAE,cAAc,CAAC,KAAK;gBAC1B,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC,CAAC;YACH,MAAM,CAAC,CAAC;SACT;gBAAS;YACR,IAAI,CAAC,GAAG,EAAE,CAAC;SACZ;IACH,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,KAAK,CAAC,IAAa;;QAC9B,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,UAAU,CAAC,4BAA4B,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAExF,IAAI;YACF,IAAI,CAAC,IAAI,EAAE;gBACT,IAAI,GAAG,IAAI,CAAC,qBAAqB,CAAC;aACnC;YACD,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,MAAM,YAAY,GAAG,EAAE,CAAC;YACxB,GAAG;gBACD,IAAI,IAAI,CAAC,gBAAgB,KAAK,SAAS,IAAI,CAAA,MAAA,IAAI,CAAC,gBAAgB,0CAAE,UAAU,MAAK,CAAC,EAAE;oBAClF,MAAM,IAAI,CAAC,aAAa,CAAC;wBACvB,WAAW,EAAE,MAAA,IAAI,CAAC,OAAO,0CAAE,WAAW;wBACtC,cAAc,EAAE,cAAc,aAAd,cAAc,uBAAd,cAAc,CAAE,cAAc;qBAC/C,CAAC,CAAC;iBACJ;gBACD,IAAI,MAAA,IAAI,CAAC,gBAAgB,0CAAE,UAAU,EAAE;oBACrC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,KAAK,EAAE,MAAA,IAAI,CAAC,gBAAgB,0CAAE,UAAU,CAAC,CAAC;oBACtE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;oBAC7D,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;oBAC/D,KAAK,IAAI,SAAS,CAAC;iBACpB;qBAAM;oBACL,SAAS,GAAG,CAAC,CAAC;iBACf;aACF,QAAQ,SAAS,GAAG,CAAC,IAAI,KAAK,GAAG,IAAI,EAAE;YAExC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC;YAEvC,IAAI,KAAK,GAAG,IAAI,EAAE;gBAChB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aACjB;SACF;QAAC,OAAO,CAAC,EAAE;YACV,IAAI,CAAC,SAAS,CAAC;gBACb,IAAI,EAAE,cAAc,CAAC,KAAK;gBAC1B,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;SACvB;gBAAS;YACR,IAAI,CAAC,GAAG,EAAE,CAAC;SACZ;IACH,CAAC;CACF","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\nimport { Readable, ReadableOptions } from \"stream\";\nimport { BlobClient, CommonOptions } from \"@azure/storage-blob\";\nimport { AbortSignalLike } from \"@azure/core-http\";\nimport { createSpan } from \"./utils/tracing\";\nimport { SpanStatusCode } from \"@azure/core-tracing\";\n\n/**\n * Options to configure the LazyLoadingBlobStream.\n */\nexport interface LazyLoadingBlobStreamOptions extends ReadableOptions, CommonOptions {\n /**\n * An implementation of the `AbortSignalLike` interface to signal the request to cancel the operation.\n * For example, use the @azure/abort-controller to create an `AbortSignal`.\n */\n abortSignal?: AbortSignalLike;\n}\n\ninterface LazyLoadingBlobStreamDownloadBlockOptions extends CommonOptions {\n /**\n * An implementation of the `AbortSignalLike` interface to signal the request to cancel the operation.\n * For example, use the @azure/abort-controller to create an `AbortSignal`.\n */\n abortSignal?: AbortSignalLike;\n}\n\n/**\n * This class generates a readable stream from a blobClient's data.\n */\nexport class LazyLoadingBlobStream extends Readable {\n /**\n * BlobClient to make download calls with.\n */\n private readonly blobClient: BlobClient;\n\n /**\n * The offset within the blob of the next block we will download.\n */\n private offset: number;\n\n private readonly blockSize: number;\n\n private lastDownloadBytes: number;\n\n private lastDownloadData?: Buffer;\n\n private blobLength: number;\n\n private options?: LazyLoadingBlobStreamOptions;\n\n /**\n * Creates an instance of LazyLoadingBlobStream.\n *\n * @param byteLength - The total length of data contained in the buffers\n */\n constructor(\n blobClient: BlobClient,\n offset: number,\n blockSize: number,\n options?: LazyLoadingBlobStreamOptions\n ) {\n super(options);\n this.blobClient = blobClient;\n this.offset = offset;\n this.blockSize = blockSize;\n this.lastDownloadBytes = -1;\n this.blobLength = -1;\n this.options = options;\n }\n\n private async downloadBlock(options: LazyLoadingBlobStreamDownloadBlockOptions = {}) {\n const { span, updatedOptions } = createSpan(\"LazyLoadingBlobStream-downloadBlock\", options);\n try {\n const properties = await this.blobClient.getProperties({\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions
|
|
1
|
+
{"version":3,"file":"LazyLoadingBlobStream.js","sourceRoot":"","sources":["../../../src/LazyLoadingBlobStream.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAElC,OAAO,EAAE,QAAQ,EAAmB,MAAM,QAAQ,CAAC;AAGnD,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAqBrD;;GAEG;AACH,MAAM,OAAO,qBAAsB,SAAQ,QAAQ;IAqBjD;;;;OAIG;IACH,YACE,UAAsB,EACtB,MAAc,EACd,SAAiB,EACjB,OAAsC;QAEtC,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,iBAAiB,GAAG,CAAC,CAAC,CAAC;QAC5B,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,UAAqD,EAAE;QACjF,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,UAAU,CAAC,qCAAqC,EAAE,OAAO,CAAC,CAAC;QAC5F,IAAI;YACF,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC;gBACrD,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;aAC9C,CAAC,CAAC;YACH,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,aAAc,CAAC;YAE5C,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;YACjF,IAAI,IAAI,CAAC,iBAAiB,KAAK,CAAC,EAAE;gBAChC,IAAI,CAAC,gBAAgB,GAAG,SAAS,CAAC;gBAClC,OAAO;aACR;YAED,IAAI,CAAC,gBAAgB,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAC5D,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,iBAAiB,EACtB;gBACE,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;aAC9C,CACF,CAAC;YACF,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,iBAAiB,CAAC;SACvC;QAAC,OAAO,CAAC,EAAE;YACV,IAAI,CAAC,SAAS,CAAC;gBACb,IAAI,EAAE,cAAc,CAAC,KAAK;gBAC1B,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC,CAAC;YACH,MAAM,CAAC,CAAC;SACT;gBAAS;YACR,IAAI,CAAC,GAAG,EAAE,CAAC;SACZ;IACH,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,KAAK,CAAC,IAAa;;QAC9B,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,UAAU,CAAC,4BAA4B,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAExF,IAAI;YACF,IAAI,CAAC,IAAI,EAAE;gBACT,IAAI,GAAG,IAAI,CAAC,qBAAqB,CAAC;aACnC;YACD,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,MAAM,YAAY,GAAG,EAAE,CAAC;YACxB,GAAG;gBACD,IAAI,IAAI,CAAC,gBAAgB,KAAK,SAAS,IAAI,CAAA,MAAA,IAAI,CAAC,gBAAgB,0CAAE,UAAU,MAAK,CAAC,EAAE;oBAClF,MAAM,IAAI,CAAC,aAAa,CAAC;wBACvB,WAAW,EAAE,MAAA,IAAI,CAAC,OAAO,0CAAE,WAAW;wBACtC,cAAc,EAAE,cAAc,aAAd,cAAc,uBAAd,cAAc,CAAE,cAAc;qBAC/C,CAAC,CAAC;iBACJ;gBACD,IAAI,MAAA,IAAI,CAAC,gBAAgB,0CAAE,UAAU,EAAE;oBACrC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,KAAK,EAAE,MAAA,IAAI,CAAC,gBAAgB,0CAAE,UAAU,CAAC,CAAC;oBACtE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;oBAC7D,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;oBAC/D,KAAK,IAAI,SAAS,CAAC;iBACpB;qBAAM;oBACL,SAAS,GAAG,CAAC,CAAC;iBACf;aACF,QAAQ,SAAS,GAAG,CAAC,IAAI,KAAK,GAAG,IAAI,EAAE;YAExC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC;YAEvC,IAAI,KAAK,GAAG,IAAI,EAAE;gBAChB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aACjB;SACF;QAAC,OAAO,CAAC,EAAE;YACV,IAAI,CAAC,SAAS,CAAC;gBACb,IAAI,EAAE,cAAc,CAAC,KAAK;gBAC1B,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;SACvB;gBAAS;YACR,IAAI,CAAC,GAAG,EAAE,CAAC;SACZ;IACH,CAAC;CACF","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\nimport { Readable, ReadableOptions } from \"stream\";\nimport { BlobClient, CommonOptions } from \"@azure/storage-blob\";\nimport { AbortSignalLike } from \"@azure/core-http\";\nimport { createSpan } from \"./utils/tracing\";\nimport { SpanStatusCode } from \"@azure/core-tracing\";\n\n/**\n * Options to configure the LazyLoadingBlobStream.\n */\nexport interface LazyLoadingBlobStreamOptions extends ReadableOptions, CommonOptions {\n /**\n * An implementation of the `AbortSignalLike` interface to signal the request to cancel the operation.\n * For example, use the @azure/abort-controller to create an `AbortSignal`.\n */\n abortSignal?: AbortSignalLike;\n}\n\ninterface LazyLoadingBlobStreamDownloadBlockOptions extends CommonOptions {\n /**\n * An implementation of the `AbortSignalLike` interface to signal the request to cancel the operation.\n * For example, use the @azure/abort-controller to create an `AbortSignal`.\n */\n abortSignal?: AbortSignalLike;\n}\n\n/**\n * This class generates a readable stream from a blobClient's data.\n */\nexport class LazyLoadingBlobStream extends Readable {\n /**\n * BlobClient to make download calls with.\n */\n private readonly blobClient: BlobClient;\n\n /**\n * The offset within the blob of the next block we will download.\n */\n private offset: number;\n\n private readonly blockSize: number;\n\n private lastDownloadBytes: number;\n\n private lastDownloadData?: Buffer;\n\n private blobLength: number;\n\n private options?: LazyLoadingBlobStreamOptions;\n\n /**\n * Creates an instance of LazyLoadingBlobStream.\n *\n * @param byteLength - The total length of data contained in the buffers\n */\n constructor(\n blobClient: BlobClient,\n offset: number,\n blockSize: number,\n options?: LazyLoadingBlobStreamOptions\n ) {\n super(options);\n this.blobClient = blobClient;\n this.offset = offset;\n this.blockSize = blockSize;\n this.lastDownloadBytes = -1;\n this.blobLength = -1;\n this.options = options;\n }\n\n private async downloadBlock(options: LazyLoadingBlobStreamDownloadBlockOptions = {}) {\n const { span, updatedOptions } = createSpan(\"LazyLoadingBlobStream-downloadBlock\", options);\n try {\n const properties = await this.blobClient.getProperties({\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n });\n this.blobLength = properties.contentLength!;\n\n this.lastDownloadBytes = Math.min(this.blockSize, this.blobLength - this.offset);\n if (this.lastDownloadBytes === 0) {\n this.lastDownloadData = undefined;\n return;\n }\n\n this.lastDownloadData = await this.blobClient.downloadToBuffer(\n this.offset,\n this.lastDownloadBytes,\n {\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n }\n );\n this.offset += this.lastDownloadBytes;\n } catch (e) {\n span.setStatus({\n code: SpanStatusCode.ERROR,\n message: e.message,\n });\n throw e;\n } finally {\n span.end();\n }\n }\n\n /**\n * Internal _read() that will be called when the stream wants to pull more data in.\n *\n * @param size - Optional. The size of data to be read\n */\n public async _read(size?: number): Promise<void> {\n const { span, updatedOptions } = createSpan(\"LazyLoadingBlobStream-read\", this.options);\n\n try {\n if (!size) {\n size = this.readableHighWaterMark;\n }\n let count = 0;\n let chunkSize = 0;\n const chunksToPush = [];\n do {\n if (this.lastDownloadData === undefined || this.lastDownloadData?.byteLength === 0) {\n await this.downloadBlock({\n abortSignal: this.options?.abortSignal,\n tracingOptions: updatedOptions?.tracingOptions,\n });\n }\n if (this.lastDownloadData?.byteLength) {\n chunkSize = Math.min(size - count, this.lastDownloadData?.byteLength);\n chunksToPush.push(this.lastDownloadData.slice(0, chunkSize));\n this.lastDownloadData = this.lastDownloadData.slice(chunkSize);\n count += chunkSize;\n } else {\n chunkSize = 0;\n }\n } while (chunkSize > 0 && count < size);\n\n this.push(Buffer.concat(chunksToPush));\n\n if (count < size) {\n this.push(null);\n }\n } catch (e) {\n span.setStatus({\n code: SpanStatusCode.ERROR,\n message: e.message,\n });\n this.emit(\"error\", e);\n } finally {\n span.end();\n }\n }\n}\n"]}
|
|
@@ -32,7 +32,7 @@ export class Segment {
|
|
|
32
32
|
const currentShard = this.shards[this.shardIndex];
|
|
33
33
|
event = await currentShard.getChange({
|
|
34
34
|
abortSignal: options.abortSignal,
|
|
35
|
-
tracingOptions: updatedOptions.tracingOptions
|
|
35
|
+
tracingOptions: updatedOptions.tracingOptions,
|
|
36
36
|
});
|
|
37
37
|
if (!currentShard.hasNext()) {
|
|
38
38
|
this.shardDone[this.shardIndex] = true;
|
|
@@ -46,7 +46,7 @@ export class Segment {
|
|
|
46
46
|
catch (e) {
|
|
47
47
|
span.setStatus({
|
|
48
48
|
code: SpanStatusCode.ERROR,
|
|
49
|
-
message: e.message
|
|
49
|
+
message: e.message,
|
|
50
50
|
});
|
|
51
51
|
throw e;
|
|
52
52
|
}
|
|
@@ -65,7 +65,7 @@ export class Segment {
|
|
|
65
65
|
return {
|
|
66
66
|
SegmentPath: this.manifestPath,
|
|
67
67
|
ShardCursors: shardCursors,
|
|
68
|
-
CurrentShardPath: this.shards[this.shardIndex].shardPath
|
|
68
|
+
CurrentShardPath: this.shards[this.shardIndex].shardPath,
|
|
69
69
|
};
|
|
70
70
|
}
|
|
71
71
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Segment.js","sourceRoot":"","sources":["../../../src/Segment.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAOlC,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAarD,MAAM,OAAO,OAAO;IAgBlB,YACE,MAAe,EACf,UAAkB,EAClB,QAAc,EACG,YAAoB;QAApB,iBAAY,GAAZ,YAAY,CAAQ;QAErC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;QAE1B,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClD,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;IAC1B,CAAC;IAhBD,IAAW,QAAQ;QACjB,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAgBM,OAAO;QACZ,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC;IAClD,CAAC;IAEM,KAAK,CAAC,SAAS,CACpB,UAAmC,EAAE;QAErC,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,UAAU,CAAC,mBAAmB,EAAE,OAAO,CAAC,CAAC;QAE1E,IAAI;YACF,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,GAAG,CAAC,EAAE;gBAChE,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;aACxC;YAED,IAAI,KAAK,GAAoC,SAAS,CAAC;YACvD,OAAO,KAAK,KAAK,SAAS,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE;gBAC5C,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE;oBACnC,IAAI,CAAC,UAAU,GAAG,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,4BAA4B;oBAC1F,SAAS;iBACV;gBAED,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAClD,KAAK,GAAG,MAAM,YAAY,CAAC,SAAS,CAAC;oBACnC,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;iBAC9C,CAAC,CAAC;gBAEH,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,EAAE;oBAC3B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;oBACvC,IAAI,CAAC,cAAc,EAAE,CAAC;iBACvB;gBACD,0BAA0B;gBAC1B,IAAI,CAAC,UAAU,GAAG,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;aAC9D;YACD,OAAO,KAAK,CAAC;SACd;QAAC,OAAO,CAAC,EAAE;YACV,IAAI,CAAC,SAAS,CAAC;gBACb,IAAI,EAAE,cAAc,CAAC,KAAK;gBAC1B,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC,CAAC;YACH,MAAM,CAAC,CAAC;SACT;gBAAS;YACR,IAAI,CAAC,GAAG,EAAE,CAAC;SACZ;IACH,CAAC;IAEM,SAAS;QACd,MAAM,YAAY,GAAkB,EAAE,CAAC;QACvC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE;YAC/B,MAAM,WAAW,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;YACtC,IAAI,WAAW,EAAE;gBACf,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;aAChC;SACF;QAED,OAAO;YACL,WAAW,EAAE,IAAI,CAAC,YAAY;YAC9B,YAAY,EAAE,YAAY;YAC1B,gBAAgB,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,SAAS;SACzD,CAAC;IACJ,CAAC;CACF","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\nimport { BlobChangeFeedEvent } from \"./models/BlobChangeFeedEvent\";\nimport { Shard } from \"./Shard\";\nimport { SegmentCursor, ShardCursor } from \"./models/ChangeFeedCursor\";\nimport { CommonOptions } from \"@azure/storage-blob\";\nimport { AbortSignalLike } from \"@azure/core-http\";\nimport { createSpan } from \"./utils/tracing\";\nimport { SpanStatusCode } from \"@azure/core-tracing\";\n\n/**\n * Options to configure {@link Segment.getChange} operation.\n */\nexport interface SegmentGetChangeOptions extends CommonOptions {\n /**\n * An implementation of the `AbortSignalLike` interface to signal the request to cancel the operation.\n * For example, use the @azure/abort-controller to create an `AbortSignal`.\n */\n abortSignal?: AbortSignalLike;\n}\n\nexport class Segment {\n private readonly shards: Shard[];\n\n // Track shards that we have finished reading from.\n private shardDone: boolean[];\n private shardDoneCount: number;\n\n private shardIndex: number;\n\n // Assuming the dateTime of segments is rounded to hour. If not, our logic for fetching\n // change events between a time range would be incorrect.\n private _dateTime: Date;\n public get dateTime(): Date {\n return this._dateTime;\n }\n\n constructor(\n shards: Shard[],\n shardIndex: number,\n dateTime: Date,\n private readonly manifestPath: string\n ) {\n this.shards = shards;\n this.shardIndex = shardIndex;\n this._dateTime = dateTime;\n\n this.shardDone = Array(shards.length).fill(false);\n this.shardDoneCount = 0;\n }\n\n public hasNext(): boolean {\n return this.shards.length > this.shardDoneCount;\n }\n\n public async getChange(\n options: SegmentGetChangeOptions = {}\n ): Promise<BlobChangeFeedEvent | undefined> {\n const { span, updatedOptions } = createSpan(\"Segment-getChange\", options);\n\n try {\n if (this.shardIndex >= this.shards.length || this.shardIndex < 0) {\n throw new Error(\"shardIndex invalid.\");\n }\n\n let event: BlobChangeFeedEvent | undefined = undefined;\n while (event === undefined && this.hasNext()) {\n if (this.shardDone[this.shardIndex]) {\n this.shardIndex = (this.shardIndex + 1) % this.shards.length; // find next available shard\n continue;\n }\n\n const currentShard = this.shards[this.shardIndex];\n event = await currentShard.getChange({\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions
|
|
1
|
+
{"version":3,"file":"Segment.js","sourceRoot":"","sources":["../../../src/Segment.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAOlC,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAarD,MAAM,OAAO,OAAO;IAgBlB,YACE,MAAe,EACf,UAAkB,EAClB,QAAc,EACG,YAAoB;QAApB,iBAAY,GAAZ,YAAY,CAAQ;QAErC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;QAE1B,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClD,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;IAC1B,CAAC;IAhBD,IAAW,QAAQ;QACjB,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAgBM,OAAO;QACZ,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC;IAClD,CAAC;IAEM,KAAK,CAAC,SAAS,CACpB,UAAmC,EAAE;QAErC,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,UAAU,CAAC,mBAAmB,EAAE,OAAO,CAAC,CAAC;QAE1E,IAAI;YACF,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,GAAG,CAAC,EAAE;gBAChE,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;aACxC;YAED,IAAI,KAAK,GAAoC,SAAS,CAAC;YACvD,OAAO,KAAK,KAAK,SAAS,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE;gBAC5C,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE;oBACnC,IAAI,CAAC,UAAU,GAAG,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,4BAA4B;oBAC1F,SAAS;iBACV;gBAED,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAClD,KAAK,GAAG,MAAM,YAAY,CAAC,SAAS,CAAC;oBACnC,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;iBAC9C,CAAC,CAAC;gBAEH,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,EAAE;oBAC3B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;oBACvC,IAAI,CAAC,cAAc,EAAE,CAAC;iBACvB;gBACD,0BAA0B;gBAC1B,IAAI,CAAC,UAAU,GAAG,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;aAC9D;YACD,OAAO,KAAK,CAAC;SACd;QAAC,OAAO,CAAC,EAAE;YACV,IAAI,CAAC,SAAS,CAAC;gBACb,IAAI,EAAE,cAAc,CAAC,KAAK;gBAC1B,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC,CAAC;YACH,MAAM,CAAC,CAAC;SACT;gBAAS;YACR,IAAI,CAAC,GAAG,EAAE,CAAC;SACZ;IACH,CAAC;IAEM,SAAS;QACd,MAAM,YAAY,GAAkB,EAAE,CAAC;QACvC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE;YAC/B,MAAM,WAAW,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;YACtC,IAAI,WAAW,EAAE;gBACf,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;aAChC;SACF;QAED,OAAO;YACL,WAAW,EAAE,IAAI,CAAC,YAAY;YAC9B,YAAY,EAAE,YAAY;YAC1B,gBAAgB,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,SAAS;SACzD,CAAC;IACJ,CAAC;CACF","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\nimport { BlobChangeFeedEvent } from \"./models/BlobChangeFeedEvent\";\nimport { Shard } from \"./Shard\";\nimport { SegmentCursor, ShardCursor } from \"./models/ChangeFeedCursor\";\nimport { CommonOptions } from \"@azure/storage-blob\";\nimport { AbortSignalLike } from \"@azure/core-http\";\nimport { createSpan } from \"./utils/tracing\";\nimport { SpanStatusCode } from \"@azure/core-tracing\";\n\n/**\n * Options to configure {@link Segment.getChange} operation.\n */\nexport interface SegmentGetChangeOptions extends CommonOptions {\n /**\n * An implementation of the `AbortSignalLike` interface to signal the request to cancel the operation.\n * For example, use the @azure/abort-controller to create an `AbortSignal`.\n */\n abortSignal?: AbortSignalLike;\n}\n\nexport class Segment {\n private readonly shards: Shard[];\n\n // Track shards that we have finished reading from.\n private shardDone: boolean[];\n private shardDoneCount: number;\n\n private shardIndex: number;\n\n // Assuming the dateTime of segments is rounded to hour. If not, our logic for fetching\n // change events between a time range would be incorrect.\n private _dateTime: Date;\n public get dateTime(): Date {\n return this._dateTime;\n }\n\n constructor(\n shards: Shard[],\n shardIndex: number,\n dateTime: Date,\n private readonly manifestPath: string\n ) {\n this.shards = shards;\n this.shardIndex = shardIndex;\n this._dateTime = dateTime;\n\n this.shardDone = Array(shards.length).fill(false);\n this.shardDoneCount = 0;\n }\n\n public hasNext(): boolean {\n return this.shards.length > this.shardDoneCount;\n }\n\n public async getChange(\n options: SegmentGetChangeOptions = {}\n ): Promise<BlobChangeFeedEvent | undefined> {\n const { span, updatedOptions } = createSpan(\"Segment-getChange\", options);\n\n try {\n if (this.shardIndex >= this.shards.length || this.shardIndex < 0) {\n throw new Error(\"shardIndex invalid.\");\n }\n\n let event: BlobChangeFeedEvent | undefined = undefined;\n while (event === undefined && this.hasNext()) {\n if (this.shardDone[this.shardIndex]) {\n this.shardIndex = (this.shardIndex + 1) % this.shards.length; // find next available shard\n continue;\n }\n\n const currentShard = this.shards[this.shardIndex];\n event = await currentShard.getChange({\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n });\n\n if (!currentShard.hasNext()) {\n this.shardDone[this.shardIndex] = true;\n this.shardDoneCount++;\n }\n // Round robin with shards\n this.shardIndex = (this.shardIndex + 1) % this.shards.length;\n }\n return event;\n } catch (e) {\n span.setStatus({\n code: SpanStatusCode.ERROR,\n message: e.message,\n });\n throw e;\n } finally {\n span.end();\n }\n }\n\n public getCursor(): SegmentCursor {\n const shardCursors: ShardCursor[] = [];\n for (const shard of this.shards) {\n const shardCursor = shard.getCursor();\n if (shardCursor) {\n shardCursors.push(shardCursor);\n }\n }\n\n return {\n SegmentPath: this.manifestPath,\n ShardCursors: shardCursors,\n CurrentShardPath: this.shards[this.shardIndex].shardPath,\n };\n }\n}\n"]}
|
|
@@ -18,7 +18,7 @@ export class SegmentFactory {
|
|
|
18
18
|
const blobClient = containerClient.getBlobClient(manifestPath);
|
|
19
19
|
const blobDownloadRes = await blobClient.download(undefined, undefined, {
|
|
20
20
|
abortSignal: options.abortSignal,
|
|
21
|
-
tracingOptions: updatedOptions.tracingOptions
|
|
21
|
+
tracingOptions: updatedOptions.tracingOptions,
|
|
22
22
|
});
|
|
23
23
|
const blobContent = await bodyToString(blobDownloadRes);
|
|
24
24
|
const segmentManifest = JSON.parse(blobContent);
|
|
@@ -28,7 +28,7 @@ export class SegmentFactory {
|
|
|
28
28
|
const shardCursor = cursor === null || cursor === void 0 ? void 0 : cursor.ShardCursors.find((x) => x.CurrentChunkPath.startsWith(shardPathSubStr));
|
|
29
29
|
const shard = await this.shardFactory.create(containerClient, shardPathSubStr, shardCursor, {
|
|
30
30
|
abortSignal: options.abortSignal,
|
|
31
|
-
tracingOptions: updatedOptions.tracingOptions
|
|
31
|
+
tracingOptions: updatedOptions.tracingOptions,
|
|
32
32
|
});
|
|
33
33
|
if (shard.hasNext()) {
|
|
34
34
|
shards.push(shard);
|
|
@@ -46,7 +46,7 @@ export class SegmentFactory {
|
|
|
46
46
|
catch (e) {
|
|
47
47
|
span.setStatus({
|
|
48
48
|
code: SpanStatusCode.ERROR,
|
|
49
|
-
message: e.message
|
|
49
|
+
message: e.message,
|
|
50
50
|
});
|
|
51
51
|
throw e;
|
|
52
52
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SegmentFactory.js","sourceRoot":"","sources":["../../../src/SegmentFactory.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAIlC,OAAO,EAAE,0BAA0B,EAAE,MAAM,mBAAmB,CAAC;AAE/D,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAEhE,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAsBrD,MAAM,OAAO,cAAc;IAGzB,YAAY,YAA0B;QACpC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAEM,KAAK,CAAC,MAAM,CACjB,eAAgC,EAChC,YAAoB,EACpB,MAAsB,EACtB,UAAgC,EAAE;QAElC,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,UAAU,CAAC,uBAAuB,EAAE,OAAO,CAAC,CAAC;QAE9E,IAAI;YACF,MAAM,MAAM,GAAY,EAAE,CAAC;YAC3B,MAAM,QAAQ,GAAS,wBAAwB,CAAC,YAAY,CAAC,CAAC;YAE9D,MAAM,UAAU,GAAG,eAAe,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC;YAC/D,MAAM,eAAe,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,SAAS,EAAE,SAAS,EAAE;gBACtE,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;aAC9C,CAAC,CAAC;YACH,MAAM,WAAW,GAAW,MAAM,YAAY,CAAC,eAAe,CAAC,CAAC;YAEhE,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAoB,CAAC;YAEnE,MAAM,qBAAqB,GAAG,0BAA0B,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,qBAAqB;YAC1F,KAAK,MAAM,SAAS,IAAI,eAAe,CAAC,cAAc,EAAE;gBACtD,MAAM,eAAe,GAAG,SAAS,CAAC,SAAS,CAAC,qBAAqB,CAAC,CAAC;gBACnE,MAAM,WAAW,GAAG,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAClD,CAAC,CAAC,gBAAgB,CAAC,UAAU,CAAC,eAAe,CAAC,CAC/C,CAAC;gBACF,MAAM,KAAK,GAAU,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CACjD,eAAe,EACf,eAAe,EACf,WAAW,EACX;oBACE,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;iBAC9C,CACF,CAAC;gBACF,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE;oBACnB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;iBACpB;aACF;YAED,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,IAAI,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,gBAAgB,EAAE;gBAC5B,UAAU,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,MAAK,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,gBAAgB,CAAA,CAAC,CAAC;gBAC/E,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE;oBACrB,UAAU,GAAG,CAAC,CAAC;iBAChB;aACF;YACD,OAAO,IAAI,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;SAChE;QAAC,OAAO,CAAC,EAAE;YACV,IAAI,CAAC,SAAS,CAAC;gBACb,IAAI,EAAE,cAAc,CAAC,KAAK;gBAC1B,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC,CAAC;YACH,MAAM,CAAC,CAAC;SACT;gBAAS;YACR,IAAI,CAAC,GAAG,EAAE,CAAC;SACZ;IACH,CAAC;CACF","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\nimport { ShardFactory } from \"./ShardFactory\";\nimport { ContainerClient, CommonOptions } from \"@azure/storage-blob\";\nimport { CHANGE_FEED_CONTAINER_NAME } from \"./utils/constants\";\nimport { Shard } from \"./Shard\";\nimport { Segment } from \"./Segment\";\nimport { SegmentCursor } from \"./models/ChangeFeedCursor\";\nimport { bodyToString } from \"./utils/utils.node\";\nimport { parseDateFromSegmentPath } from \"./utils/utils.common\";\nimport { AbortSignalLike } from \"@azure/core-http\";\nimport { createSpan } from \"./utils/tracing\";\nimport { SpanStatusCode } from \"@azure/core-tracing\";\n\nexport interface SegmentManifest {\n version?: number;\n begin?: Date;\n intervalSecs?: number;\n status: string;\n config?: any;\n chunkFilePaths: string[];\n}\n\n/**\n * Options to configure {@link SegmentFactory.create} operation.\n */\nexport interface CreateSegmentOptions extends CommonOptions {\n /**\n * An implementation of the `AbortSignalLike` interface to signal the request to cancel the operation.\n * For example, use the @azure/abort-controller to create an `AbortSignal`.\n */\n abortSignal?: AbortSignalLike;\n}\n\nexport class SegmentFactory {\n private readonly shardFactory: ShardFactory;\n\n constructor(shardFactory: ShardFactory) {\n this.shardFactory = shardFactory;\n }\n\n public async create(\n containerClient: ContainerClient,\n manifestPath: string,\n cursor?: SegmentCursor,\n options: CreateSegmentOptions = {}\n ): Promise<Segment> {\n const { span, updatedOptions } = createSpan(\"SegmentFactory-create\", options);\n\n try {\n const shards: Shard[] = [];\n const dateTime: Date = parseDateFromSegmentPath(manifestPath);\n\n const blobClient = containerClient.getBlobClient(manifestPath);\n const blobDownloadRes = await blobClient.download(undefined, undefined, {\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions
|
|
1
|
+
{"version":3,"file":"SegmentFactory.js","sourceRoot":"","sources":["../../../src/SegmentFactory.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAIlC,OAAO,EAAE,0BAA0B,EAAE,MAAM,mBAAmB,CAAC;AAE/D,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAEhE,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAsBrD,MAAM,OAAO,cAAc;IAGzB,YAAY,YAA0B;QACpC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAEM,KAAK,CAAC,MAAM,CACjB,eAAgC,EAChC,YAAoB,EACpB,MAAsB,EACtB,UAAgC,EAAE;QAElC,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,UAAU,CAAC,uBAAuB,EAAE,OAAO,CAAC,CAAC;QAE9E,IAAI;YACF,MAAM,MAAM,GAAY,EAAE,CAAC;YAC3B,MAAM,QAAQ,GAAS,wBAAwB,CAAC,YAAY,CAAC,CAAC;YAE9D,MAAM,UAAU,GAAG,eAAe,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC;YAC/D,MAAM,eAAe,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,SAAS,EAAE,SAAS,EAAE;gBACtE,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;aAC9C,CAAC,CAAC;YACH,MAAM,WAAW,GAAW,MAAM,YAAY,CAAC,eAAe,CAAC,CAAC;YAEhE,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAoB,CAAC;YAEnE,MAAM,qBAAqB,GAAG,0BAA0B,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,qBAAqB;YAC1F,KAAK,MAAM,SAAS,IAAI,eAAe,CAAC,cAAc,EAAE;gBACtD,MAAM,eAAe,GAAG,SAAS,CAAC,SAAS,CAAC,qBAAqB,CAAC,CAAC;gBACnE,MAAM,WAAW,GAAG,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAClD,CAAC,CAAC,gBAAgB,CAAC,UAAU,CAAC,eAAe,CAAC,CAC/C,CAAC;gBACF,MAAM,KAAK,GAAU,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CACjD,eAAe,EACf,eAAe,EACf,WAAW,EACX;oBACE,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;iBAC9C,CACF,CAAC;gBACF,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE;oBACnB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;iBACpB;aACF;YAED,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,IAAI,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,gBAAgB,EAAE;gBAC5B,UAAU,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,MAAK,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,gBAAgB,CAAA,CAAC,CAAC;gBAC/E,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE;oBACrB,UAAU,GAAG,CAAC,CAAC;iBAChB;aACF;YACD,OAAO,IAAI,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;SAChE;QAAC,OAAO,CAAC,EAAE;YACV,IAAI,CAAC,SAAS,CAAC;gBACb,IAAI,EAAE,cAAc,CAAC,KAAK;gBAC1B,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC,CAAC;YACH,MAAM,CAAC,CAAC;SACT;gBAAS;YACR,IAAI,CAAC,GAAG,EAAE,CAAC;SACZ;IACH,CAAC;CACF","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\nimport { ShardFactory } from \"./ShardFactory\";\nimport { ContainerClient, CommonOptions } from \"@azure/storage-blob\";\nimport { CHANGE_FEED_CONTAINER_NAME } from \"./utils/constants\";\nimport { Shard } from \"./Shard\";\nimport { Segment } from \"./Segment\";\nimport { SegmentCursor } from \"./models/ChangeFeedCursor\";\nimport { bodyToString } from \"./utils/utils.node\";\nimport { parseDateFromSegmentPath } from \"./utils/utils.common\";\nimport { AbortSignalLike } from \"@azure/core-http\";\nimport { createSpan } from \"./utils/tracing\";\nimport { SpanStatusCode } from \"@azure/core-tracing\";\n\nexport interface SegmentManifest {\n version?: number;\n begin?: Date;\n intervalSecs?: number;\n status: string;\n config?: any;\n chunkFilePaths: string[];\n}\n\n/**\n * Options to configure {@link SegmentFactory.create} operation.\n */\nexport interface CreateSegmentOptions extends CommonOptions {\n /**\n * An implementation of the `AbortSignalLike` interface to signal the request to cancel the operation.\n * For example, use the @azure/abort-controller to create an `AbortSignal`.\n */\n abortSignal?: AbortSignalLike;\n}\n\nexport class SegmentFactory {\n private readonly shardFactory: ShardFactory;\n\n constructor(shardFactory: ShardFactory) {\n this.shardFactory = shardFactory;\n }\n\n public async create(\n containerClient: ContainerClient,\n manifestPath: string,\n cursor?: SegmentCursor,\n options: CreateSegmentOptions = {}\n ): Promise<Segment> {\n const { span, updatedOptions } = createSpan(\"SegmentFactory-create\", options);\n\n try {\n const shards: Shard[] = [];\n const dateTime: Date = parseDateFromSegmentPath(manifestPath);\n\n const blobClient = containerClient.getBlobClient(manifestPath);\n const blobDownloadRes = await blobClient.download(undefined, undefined, {\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n });\n const blobContent: string = await bodyToString(blobDownloadRes);\n\n const segmentManifest = JSON.parse(blobContent) as SegmentManifest;\n\n const containerPrefixLength = CHANGE_FEED_CONTAINER_NAME.length + 1; // \"$blobchangefeed/\"\n for (const shardPath of segmentManifest.chunkFilePaths) {\n const shardPathSubStr = shardPath.substring(containerPrefixLength);\n const shardCursor = cursor?.ShardCursors.find((x) =>\n x.CurrentChunkPath.startsWith(shardPathSubStr)\n );\n const shard: Shard = await this.shardFactory.create(\n containerClient,\n shardPathSubStr,\n shardCursor,\n {\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n }\n );\n if (shard.hasNext()) {\n shards.push(shard);\n }\n }\n\n let shardIndex = 0;\n if (cursor?.CurrentShardPath) {\n shardIndex = shards.findIndex((s) => s.shardPath === cursor?.CurrentShardPath);\n if (shardIndex === -1) {\n shardIndex = 0;\n }\n }\n return new Segment(shards, shardIndex, dateTime, manifestPath);\n } catch (e) {\n span.setStatus({\n code: SpanStatusCode.ERROR,\n message: e.message,\n });\n throw e;\n } finally {\n span.end();\n }\n }\n}\n"]}
|
|
@@ -23,7 +23,7 @@ export class Shard {
|
|
|
23
23
|
if (!this.currentChunk.hasNext() && this.chunks.length > 0) {
|
|
24
24
|
this.currentChunk = await this.chunkFactory.create(this.containerClient, this.chunks.shift(), undefined, undefined, {
|
|
25
25
|
abortSignal: options.abortSignal,
|
|
26
|
-
tracingOptions: updatedOptions.tracingOptions
|
|
26
|
+
tracingOptions: updatedOptions.tracingOptions,
|
|
27
27
|
});
|
|
28
28
|
}
|
|
29
29
|
}
|
|
@@ -32,7 +32,7 @@ export class Shard {
|
|
|
32
32
|
catch (e) {
|
|
33
33
|
span.setStatus({
|
|
34
34
|
code: SpanStatusCode.ERROR,
|
|
35
|
-
message: e.message
|
|
35
|
+
message: e.message,
|
|
36
36
|
});
|
|
37
37
|
throw e;
|
|
38
38
|
}
|
|
@@ -46,7 +46,7 @@ export class Shard {
|
|
|
46
46
|
: {
|
|
47
47
|
CurrentChunkPath: this.currentChunk.chunkPath,
|
|
48
48
|
BlockOffset: this.currentChunk.blockOffset,
|
|
49
|
-
EventIndex: this.currentChunk.eventIndex
|
|
49
|
+
EventIndex: this.currentChunk.eventIndex,
|
|
50
50
|
};
|
|
51
51
|
}
|
|
52
52
|
}
|