@azure/eventhubs-checkpointstore-blob 1.1.0-alpha.20241022.1 → 1.1.0-alpha.20241121.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/blobCheckpointStore.d.ts +2 -2
- package/dist/browser/blobCheckpointStore.d.ts.map +1 -1
- package/dist/browser/blobCheckpointStore.js.map +1 -1
- package/dist/browser/storageBlobInterfaces.d.ts +2 -2
- package/dist/browser/storageBlobInterfaces.d.ts.map +1 -1
- package/dist/browser/storageBlobInterfaces.js.map +1 -1
- package/dist/commonjs/blobCheckpointStore.d.ts +2 -2
- package/dist/commonjs/blobCheckpointStore.d.ts.map +1 -1
- package/dist/commonjs/blobCheckpointStore.js.map +1 -1
- package/dist/commonjs/storageBlobInterfaces.d.ts +2 -2
- package/dist/commonjs/storageBlobInterfaces.d.ts.map +1 -1
- package/dist/commonjs/storageBlobInterfaces.js.map +1 -1
- package/dist/esm/blobCheckpointStore.d.ts +2 -2
- package/dist/esm/blobCheckpointStore.d.ts.map +1 -1
- package/dist/esm/blobCheckpointStore.js.map +1 -1
- package/dist/esm/storageBlobInterfaces.d.ts +2 -2
- package/dist/esm/storageBlobInterfaces.d.ts.map +1 -1
- package/dist/esm/storageBlobInterfaces.js.map +1 -1
- package/dist/eventhubs-checkpointstore-blob.d.ts +14 -14
- package/package.json +4 -8
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { CheckpointStore, PartitionOwnership, Checkpoint, OperationOptions } from "@azure/event-hubs";
|
|
2
|
-
import { ContainerClientLike } from "./storageBlobInterfaces.js";
|
|
1
|
+
import type { CheckpointStore, PartitionOwnership, Checkpoint, OperationOptions } from "@azure/event-hubs";
|
|
2
|
+
import type { ContainerClientLike } from "./storageBlobInterfaces.js";
|
|
3
3
|
/**
|
|
4
4
|
* An implementation of CheckpointStore that uses Azure Blob Storage to persist checkpoint data.
|
|
5
5
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"blobCheckpointStore.d.ts","sourceRoot":"","sources":["../../src/blobCheckpointStore.ts"],"names":[],"mappings":"AAGA,OAAO,
|
|
1
|
+
{"version":3,"file":"blobCheckpointStore.d.ts","sourceRoot":"","sources":["../../src/blobCheckpointStore.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,gBAAgB,EACjB,MAAM,mBAAmB,CAAC;AAG3B,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAGtE;;GAEG;AACH,qBAAa,mBAAoB,YAAW,eAAe;IACzD,OAAO,CAAC,gBAAgB,CAAsB;IAE9C;;;OAGG;gBACS,eAAe,EAAE,mBAAmB;IAGhD;;;;;;;;;;;;;OAaG;IACG,aAAa,CACjB,uBAAuB,EAAE,MAAM,EAC/B,YAAY,EAAE,MAAM,EACpB,aAAa,EAAE,MAAM,EACrB,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC,kBAAkB,EAAE,CAAC;IAoDhC;;;;;;;;;OASG;IACG,cAAc,CAClB,kBAAkB,EAAE,kBAAkB,EAAE,EACxC,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC,kBAAkB,EAAE,CAAC;IAiDhC;;;;;;;;;;OAUG;IACG,eAAe,CACnB,uBAAuB,EAAE,MAAM,EAC/B,YAAY,EAAE,MAAM,EACpB,aAAa,EAAE,MAAM,EACrB,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC,UAAU,EAAE,CAAC;IA+CxB;;;;;;;;OAQG;IACG,gBAAgB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC;IAwC7F,OAAO,CAAC,MAAM,CAAC,aAAa;YAoBd,gBAAgB;CA+C/B;AAUD;;GAEG;AACH,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,GAAG,SAAS,GACzB,MAAM,CAcR"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"blobCheckpointStore.js","sourceRoot":"","sources":["../../src/blobCheckpointStore.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;;AASlC,OAAO,EAAE,MAAM,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAEtD,OAAO,EAAE,gCAAgC,EAAE,MAAM,iBAAiB,CAAC;AAEnE;;GAEG;AACH,MAAM,OAAO,mBAAmB;IAG9B;;;OAGG;IACH,YAAY,eAAoC;QAC9C,IAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC;IAC1C,CAAC;IACD;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,aAAa,CACjB,uBAA+B,EAC/B,YAAoB,EACpB,aAAqB,EACrB,UAA4B,EAAE;;;QAE9B,MAAM,uBAAuB,GAAyB,EAAE,CAAC;QACzD,MAAM,EAAE,WAAW,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAEhD,MAAM,UAAU,GAAG,mBAAmB,CAAC,aAAa,CAAC;YACnD,IAAI,EAAE,WAAW;YACjB,uBAAuB;YACvB,YAAY;YACZ,aAAa,EAAE,aAAa;SAC7B,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC;gBAChD,WAAW;gBACX,eAAe,EAAE,IAAI;gBACrB,MAAM,EAAE,UAAU;gBAClB,cAAc;aACf,CAAC,CAAC;;gBAEH,KAAyB,eAAA,UAAA,cAAA,KAAK,CAAA,WAAA,yEAAE,CAAC;oBAAR,qBAAK;oBAAL,WAAK;oBAAnB,MAAM,IAAI,KAAA,CAAA;oBACnB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBACtC,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBAE/C,MAAM,iBAAiB,GAAG,MAAC,IAAI,CAAC,QAA8B,mCAAI,EAAE,CAAC;oBAErE,IAAI,iBAAiB,CAAC,OAAO,IAAI,IAAI,EAAE,CAAC;wBACtC,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;oBACvE,CAAC;oBAED,MAAM,kBAAkB,GAAuB;wBAC7C,uBAAuB;wBACvB,YAAY;wBACZ,aAAa,EAAE,aAAa;wBAC5B,OAAO,EAAE,iBAAiB,CAAC,OAAO;wBAClC,WAAW,EAAE,QAAQ;wBACrB,oBAAoB,EAClB,IAAI,CAAC,UAAU,CAAC,YAAY,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,OAAO,EAAE;wBACxE,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI;qBAC3B,CAAC;oBACF,uBAAuB,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;gBACnD,CAAC;;;;;;;;;YACD,OAAO,uBAAuB,CAAC;QACjC,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,MAAM,CAAC,OAAO,CAAC,iDAAiD,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;YAC/E,kBAAkB,CAAC,GAAG,CAAC,CAAC;YAExB,IAAI,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,YAAY;gBAAE,MAAM,GAAG,CAAC;YAE1C,MAAM,IAAI,KAAK,CAAC,sDAAsD,GAAG,EAAE,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,cAAc,CAClB,kBAAwC,EACxC,UAA4B,EAAE;QAE9B,MAAM,uBAAuB,GAAyB,EAAE,CAAC;QACzD,KAAK,MAAM,SAAS,IAAI,kBAAkB,EAAE,CAAC;YAC3C,MAAM,QAAQ,GAAG,mBAAmB,CAAC,aAAa,iBAAG,IAAI,EAAE,WAAW,IAAK,SAAS,EAAG,CAAC;YACxF,IAAI,CAAC;gBACH,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,gBAAgB,CACrD,QAAQ,EACR;oBACE,OAAO,EAAE,SAAS,CAAC,OAAO;iBAC3B,EACD,SAAS,CAAC,IAAI,EACd,OAAO,CACR,CAAC;gBAEF,IAAI,mBAAmB,CAAC,YAAY,EAAE,CAAC;oBACrC,SAAS,CAAC,oBAAoB,GAAG,mBAAmB,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;gBAC9E,CAAC;gBAED,SAAS,CAAC,IAAI,GAAG,mBAAmB,CAAC,IAAI,CAAC;gBAC1C,uBAAuB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACxC,MAAM,CAAC,IAAI,CACT,IAAI,SAAS,CAAC,OAAO,mDAAmD,SAAS,CAAC,WAAW,EAAE,EAC/F,qBAAqB,SAAS,CAAC,oBAAoB,WAAW,SAAS,CAAC,IAAI,EAAE,CAC/E,CAAC;YACJ,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAClB,MAAM,SAAS,GAAG,GAAgB,CAAC;gBAEnC,IAAI,SAAS,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;oBACjC,wEAAwE;oBACxE,6EAA6E;oBAC7E,6BAA6B;oBAC7B,MAAM,CAAC,OAAO,CACZ,IAAI,SAAS,CAAC,OAAO,6BAA6B,SAAS,CAAC,WAAW,6CAA6C,CACrH,CAAC;oBACF,SAAS;gBACX,CAAC;gBAED,MAAM,CAAC,OAAO,CACZ,0DAA0D,SAAS,CAAC,WAAW,EAAE,EACjF,GAAG,CAAC,OAAO,CACZ,CAAC;gBACF,kBAAkB,CAAC,GAAG,CAAC,CAAC;gBAExB,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;QACD,OAAO,uBAAuB,CAAC;IACjC,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,eAAe,CACnB,uBAA+B,EAC/B,YAAoB,EACpB,aAAqB,EACrB,UAA4B,EAAE;;;QAE9B,MAAM,EAAE,WAAW,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAChD,MAAM,UAAU,GAAG,mBAAmB,CAAC,aAAa,CAAC;YACnD,IAAI,EAAE,YAAY;YAClB,uBAAuB;YACvB,YAAY;YACZ,aAAa;SACd,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC;YAChD,WAAW;YACX,eAAe,EAAE,IAAI;YACrB,MAAM,EAAE,UAAU;YAClB,cAAc;SACf,CAAC,CAAC;QAEH,MAAM,WAAW,GAAiB,EAAE,CAAC;;YAErC,KAAyB,eAAA,UAAA,cAAA,KAAK,CAAA,WAAA,yEAAE,CAAC;gBAAR,qBAAK;gBAAL,WAAK;gBAAnB,MAAM,IAAI,KAAA,CAAA;gBACnB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACtC,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAE/C,MAAM,kBAAkB,GAAG,MAAC,IAAI,CAAC,QAA+B,mCAAI,EAAE,CAAC;gBAEvE,MAAM,MAAM,GAAG,kBAAkB,CAAC,MAAM,CAAC;gBACzC,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;oBACnB,MAAM,IAAI,KAAK,CAAC,+CAA+C,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;gBAC/E,CAAC;gBACD,MAAM,cAAc,GAAG,eAAe,CACpC,IAAI,CAAC,IAAI,EACT,gBAAgB,EAChB,kBAAkB,CAAC,cAAc,CAClC,CAAC;gBAEF,WAAW,CAAC,IAAI,CAAC;oBACf,aAAa;oBACb,YAAY;oBACZ,uBAAuB;oBACvB,WAAW,EAAE,QAAQ;oBACrB,MAAM;oBACN,cAAc;iBACf,CAAC,CAAC;YACL,CAAC;;;;;;;;;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,gBAAgB,CAAC,UAAsB,EAAE,UAA4B,EAAE;QAC3E,gCAAgC,CAC9B,kBAAkB,EAClB,gBAAgB,EAChB,UAAU,CAAC,cAAc,CAC1B,CAAC;QACF,gCAAgC,CAAC,kBAAkB,EAAE,QAAQ,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QAElF,MAAM,QAAQ,GAAG,mBAAmB,CAAC,aAAa,iBAAG,IAAI,EAAE,YAAY,IAAK,UAAU,EAAG,CAAC;QAC1F,IAAI,CAAC;YACH,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAClD,QAAQ,EACR;gBACE,cAAc,EAAE,UAAU,CAAC,cAAc,CAAC,QAAQ,EAAE;gBACpD,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,QAAQ,EAAE;aACrC,EACD,SAAS,EACT,OAAO,CACR,CAAC;YAEF,MAAM,CAAC,OAAO,CACZ,kDAAkD,UAAU,CAAC,WAAW,EAAE,EAC1E,qBAAqB,gBAAgB,CAAC,YAAa,CAAC,WAAW,EAAE,WAC/D,gBAAgB,CAAC,IACnB,EAAE,CACH,CAAC;YACF,OAAO;QACT,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,MAAM,CAAC,OAAO,CACZ,+DAA+D,UAAU,CAAC,WAAW,GAAG,EACxF,GAAG,CAAC,OAAO,CACZ,CAAC;YACF,kBAAkB,CAAC,GAAG,CAAC,CAAC;YAExB,IAAI,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,YAAY;gBAAE,MAAM,GAAG,CAAC;YAE1C,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,MAAM,CAAC,aAAa,CAAC,MAM5B;QACC,oGAAoG;QACpG,4DAA4D;QAC5D,MAAM,iBAAiB,GAAG,MAAM,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC;QAC7D,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC;QACvD,MAAM,uBAAuB,GAAG,MAAM,CAAC,uBAAuB,CAAC,WAAW,EAAE,CAAC;QAE7E,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;YACvB,OAAO,GAAG,uBAAuB,IAAI,YAAY,IAAI,iBAAiB,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QAChH,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,uBAAuB,IAAI,YAAY,IAAI,iBAAiB,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC;QAC3F,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAC5B,QAAgB,EAChB,QAAgD,EAChD,IAAwB,EACxB,UAA4B,EAAE;QAE9B,MAAM,EAAE,WAAW,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAChD,MAAM,eAAe,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,kBAAkB,EAAE,CAAC;QAE3F,kDAAkD;QAClD,2CAA2C;QAC3C,IAAI,IAAI,EAAE,CAAC;YACT,OAAO,eAAe,CAAC,WAAW,CAAC,QAAoB,EAAE;gBACvD,WAAW;gBACX,UAAU,EAAE;oBACV,OAAO,EAAE,IAAI;iBACd;gBACD,cAAc;aACf,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,IAAI,CAAC;gBACH,qFAAqF;gBACrF,gGAAgG;gBAChG,yDAAyD;gBACzD,OAAO,MAAM,eAAe,CAAC,WAAW,CAAC,QAAoB,EAAE;oBAC7D,WAAW;oBACX,cAAc;iBACf,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAClB,0EAA0E;gBAC1E,IAAI,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,WAAW,EAAE,CAAC;oBAC9B,MAAM,GAAG,CAAC;gBACZ,CAAC;gBACD,MAAM,YAAY,GAAI,GAAiB,CAAC,OAAkD,CAAC;gBAC3F,MAAM,SAAS,GAAG,YAAY,aAAZ,YAAY,uBAAZ,YAAY,CAAE,SAAS,CAAC;gBAC1C,IAAI,CAAC,SAAS,IAAI,SAAS,KAAK,cAAc,EAAE,CAAC;oBAC/C,MAAM,GAAG,CAAC;gBACZ,CAAC;gBAED,OAAO,eAAe,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,EAAE;oBACnC,WAAW;oBACX,QAAQ,EAAE,QAAoB;oBAC9B,cAAc;iBACf,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;CACF;AAUD;;GAEG;AACH,MAAM,UAAU,eAAe,CAC7B,QAAgB,EAChB,SAAiB,EACjB,MAA0B;IAE1B,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,8BAA8B,SAAS,cAAc,QAAQ,GAAG,CAAC,CAAC;IACpF,CAAC;IAED,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAEjC,IAAI,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CACb,sCAAsC,SAAS,cAAc,QAAQ,eAAe,CACrF,CAAC;IACJ,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport {\n CheckpointStore,\n PartitionOwnership,\n Checkpoint,\n OperationOptions,\n} from \"@azure/event-hubs\";\nimport { Metadata, RestError, BlobSetMetadataResponse } from \"@azure/storage-blob\";\nimport { logger, logErrorStackTrace } from \"./log.js\";\nimport { ContainerClientLike } from \"./storageBlobInterfaces.js\";\nimport { throwTypeErrorIfParameterMissing } from \"./util/error.js\";\n\n/**\n * An implementation of CheckpointStore that uses Azure Blob Storage to persist checkpoint data.\n */\nexport class BlobCheckpointStore implements CheckpointStore {\n private _containerClient: ContainerClientLike;\n\n /**\n * Constructs a new instance of {@link BlobCheckpointStore}\n * @param containerClient - An instance of a storage blob ContainerClient.\n */\n constructor(containerClient: ContainerClientLike) {\n this._containerClient = containerClient;\n }\n /**\n * Get the list of all existing partition ownership from the underlying data store. May return empty\n * results if there are is no existing ownership information.\n * Partition Ownership contains the information on which `EventHubConsumerClient` subscribe call is currently processing the partition.\n *\n * @param fullyQualifiedNamespace - The fully qualified Event Hubs namespace. This is likely to be similar to\n * <yournamespace>.servicebus.windows.net.\n * @param eventHubName - The event hub name.\n * @param consumerGroup - The consumer group name.\n * @param options - A set of options that can be specified to influence the behavior of this method.\n * - `abortSignal`: A signal used to request operation cancellation.\n * - `tracingOptions`: Options for configuring tracing.\n * @returns Partition ownership details of all the partitions that have had an owner.\n */\n async listOwnership(\n fullyQualifiedNamespace: string,\n eventHubName: string,\n consumerGroup: string,\n options: OperationOptions = {},\n ): Promise<PartitionOwnership[]> {\n const partitionOwnershipArray: PartitionOwnership[] = [];\n const { abortSignal, tracingOptions } = options;\n\n const blobPrefix = BlobCheckpointStore.getBlobPrefix({\n type: \"ownership\",\n fullyQualifiedNamespace,\n eventHubName,\n consumerGroup: consumerGroup,\n });\n\n try {\n const blobs = this._containerClient.listBlobsFlat({\n abortSignal,\n includeMetadata: true,\n prefix: blobPrefix,\n tracingOptions,\n });\n\n for await (const blob of blobs) {\n const blobPath = blob.name.split(\"/\");\n const blobName = blobPath[blobPath.length - 1];\n\n const ownershipMetadata = (blob.metadata as OwnershipMetadata) ?? {};\n\n if (ownershipMetadata.ownerid == null) {\n throw new Error(`Missing ownerid in metadata for blob ${blob.name}`);\n }\n\n const partitionOwnership: PartitionOwnership = {\n fullyQualifiedNamespace,\n eventHubName,\n consumerGroup: consumerGroup,\n ownerId: ownershipMetadata.ownerid,\n partitionId: blobName,\n lastModifiedTimeInMs:\n blob.properties.lastModified && blob.properties.lastModified.getTime(),\n etag: blob.properties.etag,\n };\n partitionOwnershipArray.push(partitionOwnership);\n }\n return partitionOwnershipArray;\n } catch (err: any) {\n logger.warning(`Error occurred while fetching the list of blobs`, err.message);\n logErrorStackTrace(err);\n\n if (err?.name === \"AbortError\") throw err;\n\n throw new Error(`Error occurred while fetching the list of blobs. \\n${err}`);\n }\n }\n\n /**\n * Claim ownership of a list of partitions. This will return the list of partitions that were\n * successfully claimed.\n *\n * @param partitionOwnership - The list of partition ownership this instance is claiming to own.\n * @param options - A set of options that can be specified to influence the behavior of this method.\n * - `abortSignal`: A signal used to request operation cancellation.\n * - `tracingOptions`: Options for configuring tracing.\n * @returns A list partitions this instance successfully claimed ownership.\n */\n async claimOwnership(\n partitionOwnership: PartitionOwnership[],\n options: OperationOptions = {},\n ): Promise<PartitionOwnership[]> {\n const partitionOwnershipArray: PartitionOwnership[] = [];\n for (const ownership of partitionOwnership) {\n const blobName = BlobCheckpointStore.getBlobPrefix({ type: \"ownership\", ...ownership });\n try {\n const updatedBlobResponse = await this._setBlobMetadata(\n blobName,\n {\n ownerid: ownership.ownerId,\n },\n ownership.etag,\n options,\n );\n\n if (updatedBlobResponse.lastModified) {\n ownership.lastModifiedTimeInMs = updatedBlobResponse.lastModified.getTime();\n }\n\n ownership.etag = updatedBlobResponse.etag;\n partitionOwnershipArray.push(ownership);\n logger.info(\n `[${ownership.ownerId}] Claimed ownership successfully for partition: ${ownership.partitionId}`,\n `LastModifiedTime: ${ownership.lastModifiedTimeInMs}, ETag: ${ownership.etag}`,\n );\n } catch (err: any) {\n const restError = err as RestError;\n\n if (restError.statusCode === 412) {\n // etag failures (precondition not met) aren't fatal errors. They happen\n // as multiple consumers attempt to claim the same partition (first one wins)\n // and losers get this error.\n logger.verbose(\n `[${ownership.ownerId}] Did not claim partition ${ownership.partitionId}. Another processor has already claimed it.`,\n );\n continue;\n }\n\n logger.warning(\n `Error occurred while claiming ownership for partition: ${ownership.partitionId}`,\n err.message,\n );\n logErrorStackTrace(err);\n\n throw err;\n }\n }\n return partitionOwnershipArray;\n }\n\n /**\n * Lists all the checkpoints in a data store for a given namespace, eventhub and consumer group.\n *\n * @param fullyQualifiedNamespace - The fully qualified Event Hubs namespace. This is likely to be similar to\n * <yournamespace>.servicebus.windows.net.\n * @param eventHubName - The event hub name.\n * @param consumerGroup - The consumer group name.\n * @param options - A set of options that can be specified to influence the behavior of this method.\n * - `abortSignal`: A signal used to request operation cancellation.\n * - `tracingOptions`: Options for configuring tracing.\n */\n async listCheckpoints(\n fullyQualifiedNamespace: string,\n eventHubName: string,\n consumerGroup: string,\n options: OperationOptions = {},\n ): Promise<Checkpoint[]> {\n const { abortSignal, tracingOptions } = options;\n const blobPrefix = BlobCheckpointStore.getBlobPrefix({\n type: \"checkpoint\",\n fullyQualifiedNamespace,\n eventHubName,\n consumerGroup,\n });\n\n const blobs = this._containerClient.listBlobsFlat({\n abortSignal,\n includeMetadata: true,\n prefix: blobPrefix,\n tracingOptions,\n });\n\n const checkpoints: Checkpoint[] = [];\n\n for await (const blob of blobs) {\n const blobPath = blob.name.split(\"/\");\n const blobName = blobPath[blobPath.length - 1];\n\n const checkpointMetadata = (blob.metadata as CheckpointMetadata) ?? {};\n\n const offset = checkpointMetadata.offset;\n if (offset == null) {\n throw new Error(`Missing metadata property 'offset' on blob '${blob.name}'`);\n }\n const sequenceNumber = parseIntOrThrow(\n blob.name,\n \"sequencenumber\",\n checkpointMetadata.sequencenumber,\n );\n\n checkpoints.push({\n consumerGroup,\n eventHubName,\n fullyQualifiedNamespace,\n partitionId: blobName,\n offset,\n sequenceNumber,\n });\n }\n\n return checkpoints;\n }\n\n /**\n * Updates the checkpoint in the data store for a partition.\n *\n * @param checkpoint - The checkpoint.\n * @param options - A set of options that can be specified to influence the behavior of this method.\n * - `abortSignal`: A signal used to request operation cancellation.\n * - `tracingOptions`: Options for configuring tracing.\n * @returns The new etag on successful update.\n */\n async updateCheckpoint(checkpoint: Checkpoint, options: OperationOptions = {}): Promise<void> {\n throwTypeErrorIfParameterMissing(\n \"updateCheckpoint\",\n \"sequenceNumber\",\n checkpoint.sequenceNumber,\n );\n throwTypeErrorIfParameterMissing(\"updateCheckpoint\", \"offset\", checkpoint.offset);\n\n const blobName = BlobCheckpointStore.getBlobPrefix({ type: \"checkpoint\", ...checkpoint });\n try {\n const metadataResponse = await this._setBlobMetadata(\n blobName,\n {\n sequencenumber: checkpoint.sequenceNumber.toString(),\n offset: checkpoint.offset.toString(),\n },\n undefined,\n options,\n );\n\n logger.verbose(\n `Updated checkpoint successfully for partition: ${checkpoint.partitionId}`,\n `LastModifiedTime: ${metadataResponse.lastModified!.toISOString()}, ETag: ${\n metadataResponse.etag\n }`,\n );\n return;\n } catch (err: any) {\n logger.warning(\n `Error occurred while updating the checkpoint for partition: ${checkpoint.partitionId}.`,\n err.message,\n );\n logErrorStackTrace(err);\n\n if (err?.name === \"AbortError\") throw err;\n\n throw err;\n }\n }\n\n private static getBlobPrefix(params: {\n type: \"ownership\" | \"checkpoint\";\n fullyQualifiedNamespace: string;\n eventHubName: string;\n consumerGroup: string;\n partitionId?: string;\n }): string {\n // none of these are case-sensitive in eventhubs so we need to make sure we don't accidentally allow\n // the user to create a case-sensitive blob for their state!\n const consumerGroupName = params.consumerGroup.toLowerCase();\n const eventHubName = params.eventHubName.toLowerCase();\n const fullyQualifiedNamespace = params.fullyQualifiedNamespace.toLowerCase();\n\n if (params.partitionId) {\n return `${fullyQualifiedNamespace}/${eventHubName}/${consumerGroupName}/${params.type}/${params.partitionId}`;\n } else {\n return `${fullyQualifiedNamespace}/${eventHubName}/${consumerGroupName}/${params.type}/`;\n }\n }\n\n private async _setBlobMetadata(\n blobName: string,\n metadata: OwnershipMetadata | CheckpointMetadata,\n etag: string | undefined,\n options: OperationOptions = {},\n ): Promise<BlobSetMetadataResponse> {\n const { abortSignal, tracingOptions } = options;\n const blockBlobClient = this._containerClient.getBlobClient(blobName).getBlockBlobClient();\n\n // When we have an etag, we know the blob existed.\n // If we encounter an error we should fail.\n if (etag) {\n return blockBlobClient.setMetadata(metadata as Metadata, {\n abortSignal,\n conditions: {\n ifMatch: etag,\n },\n tracingOptions,\n });\n } else {\n try {\n // Attempt to set metadata, and fallback to upload if the blob doesn't already exist.\n // This avoids poor performance in storage accounts with soft-delete or blob versioning enabled.\n // https://github.com/Azure/azure-sdk-for-js/issues/10132\n return await blockBlobClient.setMetadata(metadata as Metadata, {\n abortSignal,\n tracingOptions,\n });\n } catch (err: any) {\n // Check if the error is `BlobNotFound` and fallback to `upload` if it is.\n if (err?.name !== \"RestError\") {\n throw err;\n }\n const errorDetails = (err as RestError).details as { [field: string]: string } | undefined;\n const errorCode = errorDetails?.errorCode;\n if (!errorCode || errorCode !== \"BlobNotFound\") {\n throw err;\n }\n\n return blockBlobClient.upload(\"\", 0, {\n abortSignal,\n metadata: metadata as Metadata,\n tracingOptions,\n });\n }\n }\n }\n}\n\ntype OwnershipMetadata = {\n [k in \"ownerid\"]: string | undefined;\n};\n\ntype CheckpointMetadata = {\n [k in \"sequencenumber\" | \"offset\"]: string | undefined;\n};\n\n/**\n * @internal\n */\nexport function parseIntOrThrow(\n blobName: string,\n fieldName: string,\n numStr: string | undefined,\n): number {\n if (numStr == null) {\n throw new Error(`Missing metadata property '${fieldName}' on blob '${blobName}'`);\n }\n\n const num = parseInt(numStr, 10);\n\n if (isNaN(num)) {\n throw new Error(\n `Failed to parse metadata property '${fieldName}' on blob '${blobName}' as a number`,\n );\n }\n\n return num;\n}\n"]}
|
|
1
|
+
{"version":3,"file":"blobCheckpointStore.js","sourceRoot":"","sources":["../../src/blobCheckpointStore.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;;AASlC,OAAO,EAAE,MAAM,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAEtD,OAAO,EAAE,gCAAgC,EAAE,MAAM,iBAAiB,CAAC;AAEnE;;GAEG;AACH,MAAM,OAAO,mBAAmB;IAG9B;;;OAGG;IACH,YAAY,eAAoC;QAC9C,IAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC;IAC1C,CAAC;IACD;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,aAAa,CACjB,uBAA+B,EAC/B,YAAoB,EACpB,aAAqB,EACrB,UAA4B,EAAE;;;QAE9B,MAAM,uBAAuB,GAAyB,EAAE,CAAC;QACzD,MAAM,EAAE,WAAW,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAEhD,MAAM,UAAU,GAAG,mBAAmB,CAAC,aAAa,CAAC;YACnD,IAAI,EAAE,WAAW;YACjB,uBAAuB;YACvB,YAAY;YACZ,aAAa,EAAE,aAAa;SAC7B,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC;gBAChD,WAAW;gBACX,eAAe,EAAE,IAAI;gBACrB,MAAM,EAAE,UAAU;gBAClB,cAAc;aACf,CAAC,CAAC;;gBAEH,KAAyB,eAAA,UAAA,cAAA,KAAK,CAAA,WAAA,yEAAE,CAAC;oBAAR,qBAAK;oBAAL,WAAK;oBAAnB,MAAM,IAAI,KAAA,CAAA;oBACnB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBACtC,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBAE/C,MAAM,iBAAiB,GAAG,MAAC,IAAI,CAAC,QAA8B,mCAAI,EAAE,CAAC;oBAErE,IAAI,iBAAiB,CAAC,OAAO,IAAI,IAAI,EAAE,CAAC;wBACtC,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;oBACvE,CAAC;oBAED,MAAM,kBAAkB,GAAuB;wBAC7C,uBAAuB;wBACvB,YAAY;wBACZ,aAAa,EAAE,aAAa;wBAC5B,OAAO,EAAE,iBAAiB,CAAC,OAAO;wBAClC,WAAW,EAAE,QAAQ;wBACrB,oBAAoB,EAClB,IAAI,CAAC,UAAU,CAAC,YAAY,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,OAAO,EAAE;wBACxE,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI;qBAC3B,CAAC;oBACF,uBAAuB,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;gBACnD,CAAC;;;;;;;;;YACD,OAAO,uBAAuB,CAAC;QACjC,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,MAAM,CAAC,OAAO,CAAC,iDAAiD,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;YAC/E,kBAAkB,CAAC,GAAG,CAAC,CAAC;YAExB,IAAI,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,YAAY;gBAAE,MAAM,GAAG,CAAC;YAE1C,MAAM,IAAI,KAAK,CAAC,sDAAsD,GAAG,EAAE,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,cAAc,CAClB,kBAAwC,EACxC,UAA4B,EAAE;QAE9B,MAAM,uBAAuB,GAAyB,EAAE,CAAC;QACzD,KAAK,MAAM,SAAS,IAAI,kBAAkB,EAAE,CAAC;YAC3C,MAAM,QAAQ,GAAG,mBAAmB,CAAC,aAAa,iBAAG,IAAI,EAAE,WAAW,IAAK,SAAS,EAAG,CAAC;YACxF,IAAI,CAAC;gBACH,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,gBAAgB,CACrD,QAAQ,EACR;oBACE,OAAO,EAAE,SAAS,CAAC,OAAO;iBAC3B,EACD,SAAS,CAAC,IAAI,EACd,OAAO,CACR,CAAC;gBAEF,IAAI,mBAAmB,CAAC,YAAY,EAAE,CAAC;oBACrC,SAAS,CAAC,oBAAoB,GAAG,mBAAmB,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;gBAC9E,CAAC;gBAED,SAAS,CAAC,IAAI,GAAG,mBAAmB,CAAC,IAAI,CAAC;gBAC1C,uBAAuB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACxC,MAAM,CAAC,IAAI,CACT,IAAI,SAAS,CAAC,OAAO,mDAAmD,SAAS,CAAC,WAAW,EAAE,EAC/F,qBAAqB,SAAS,CAAC,oBAAoB,WAAW,SAAS,CAAC,IAAI,EAAE,CAC/E,CAAC;YACJ,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAClB,MAAM,SAAS,GAAG,GAAgB,CAAC;gBAEnC,IAAI,SAAS,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;oBACjC,wEAAwE;oBACxE,6EAA6E;oBAC7E,6BAA6B;oBAC7B,MAAM,CAAC,OAAO,CACZ,IAAI,SAAS,CAAC,OAAO,6BAA6B,SAAS,CAAC,WAAW,6CAA6C,CACrH,CAAC;oBACF,SAAS;gBACX,CAAC;gBAED,MAAM,CAAC,OAAO,CACZ,0DAA0D,SAAS,CAAC,WAAW,EAAE,EACjF,GAAG,CAAC,OAAO,CACZ,CAAC;gBACF,kBAAkB,CAAC,GAAG,CAAC,CAAC;gBAExB,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;QACD,OAAO,uBAAuB,CAAC;IACjC,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,eAAe,CACnB,uBAA+B,EAC/B,YAAoB,EACpB,aAAqB,EACrB,UAA4B,EAAE;;;QAE9B,MAAM,EAAE,WAAW,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAChD,MAAM,UAAU,GAAG,mBAAmB,CAAC,aAAa,CAAC;YACnD,IAAI,EAAE,YAAY;YAClB,uBAAuB;YACvB,YAAY;YACZ,aAAa;SACd,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC;YAChD,WAAW;YACX,eAAe,EAAE,IAAI;YACrB,MAAM,EAAE,UAAU;YAClB,cAAc;SACf,CAAC,CAAC;QAEH,MAAM,WAAW,GAAiB,EAAE,CAAC;;YAErC,KAAyB,eAAA,UAAA,cAAA,KAAK,CAAA,WAAA,yEAAE,CAAC;gBAAR,qBAAK;gBAAL,WAAK;gBAAnB,MAAM,IAAI,KAAA,CAAA;gBACnB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACtC,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAE/C,MAAM,kBAAkB,GAAG,MAAC,IAAI,CAAC,QAA+B,mCAAI,EAAE,CAAC;gBAEvE,MAAM,MAAM,GAAG,kBAAkB,CAAC,MAAM,CAAC;gBACzC,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;oBACnB,MAAM,IAAI,KAAK,CAAC,+CAA+C,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;gBAC/E,CAAC;gBACD,MAAM,cAAc,GAAG,eAAe,CACpC,IAAI,CAAC,IAAI,EACT,gBAAgB,EAChB,kBAAkB,CAAC,cAAc,CAClC,CAAC;gBAEF,WAAW,CAAC,IAAI,CAAC;oBACf,aAAa;oBACb,YAAY;oBACZ,uBAAuB;oBACvB,WAAW,EAAE,QAAQ;oBACrB,MAAM;oBACN,cAAc;iBACf,CAAC,CAAC;YACL,CAAC;;;;;;;;;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,gBAAgB,CAAC,UAAsB,EAAE,UAA4B,EAAE;QAC3E,gCAAgC,CAC9B,kBAAkB,EAClB,gBAAgB,EAChB,UAAU,CAAC,cAAc,CAC1B,CAAC;QACF,gCAAgC,CAAC,kBAAkB,EAAE,QAAQ,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QAElF,MAAM,QAAQ,GAAG,mBAAmB,CAAC,aAAa,iBAAG,IAAI,EAAE,YAAY,IAAK,UAAU,EAAG,CAAC;QAC1F,IAAI,CAAC;YACH,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAClD,QAAQ,EACR;gBACE,cAAc,EAAE,UAAU,CAAC,cAAc,CAAC,QAAQ,EAAE;gBACpD,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,QAAQ,EAAE;aACrC,EACD,SAAS,EACT,OAAO,CACR,CAAC;YAEF,MAAM,CAAC,OAAO,CACZ,kDAAkD,UAAU,CAAC,WAAW,EAAE,EAC1E,qBAAqB,gBAAgB,CAAC,YAAa,CAAC,WAAW,EAAE,WAC/D,gBAAgB,CAAC,IACnB,EAAE,CACH,CAAC;YACF,OAAO;QACT,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,MAAM,CAAC,OAAO,CACZ,+DAA+D,UAAU,CAAC,WAAW,GAAG,EACxF,GAAG,CAAC,OAAO,CACZ,CAAC;YACF,kBAAkB,CAAC,GAAG,CAAC,CAAC;YAExB,IAAI,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,YAAY;gBAAE,MAAM,GAAG,CAAC;YAE1C,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,MAAM,CAAC,aAAa,CAAC,MAM5B;QACC,oGAAoG;QACpG,4DAA4D;QAC5D,MAAM,iBAAiB,GAAG,MAAM,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC;QAC7D,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC;QACvD,MAAM,uBAAuB,GAAG,MAAM,CAAC,uBAAuB,CAAC,WAAW,EAAE,CAAC;QAE7E,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;YACvB,OAAO,GAAG,uBAAuB,IAAI,YAAY,IAAI,iBAAiB,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QAChH,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,uBAAuB,IAAI,YAAY,IAAI,iBAAiB,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC;QAC3F,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAC5B,QAAgB,EAChB,QAAgD,EAChD,IAAwB,EACxB,UAA4B,EAAE;QAE9B,MAAM,EAAE,WAAW,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAChD,MAAM,eAAe,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,kBAAkB,EAAE,CAAC;QAE3F,kDAAkD;QAClD,2CAA2C;QAC3C,IAAI,IAAI,EAAE,CAAC;YACT,OAAO,eAAe,CAAC,WAAW,CAAC,QAAoB,EAAE;gBACvD,WAAW;gBACX,UAAU,EAAE;oBACV,OAAO,EAAE,IAAI;iBACd;gBACD,cAAc;aACf,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,IAAI,CAAC;gBACH,qFAAqF;gBACrF,gGAAgG;gBAChG,yDAAyD;gBACzD,OAAO,MAAM,eAAe,CAAC,WAAW,CAAC,QAAoB,EAAE;oBAC7D,WAAW;oBACX,cAAc;iBACf,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAClB,0EAA0E;gBAC1E,IAAI,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,WAAW,EAAE,CAAC;oBAC9B,MAAM,GAAG,CAAC;gBACZ,CAAC;gBACD,MAAM,YAAY,GAAI,GAAiB,CAAC,OAAkD,CAAC;gBAC3F,MAAM,SAAS,GAAG,YAAY,aAAZ,YAAY,uBAAZ,YAAY,CAAE,SAAS,CAAC;gBAC1C,IAAI,CAAC,SAAS,IAAI,SAAS,KAAK,cAAc,EAAE,CAAC;oBAC/C,MAAM,GAAG,CAAC;gBACZ,CAAC;gBAED,OAAO,eAAe,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,EAAE;oBACnC,WAAW;oBACX,QAAQ,EAAE,QAAoB;oBAC9B,cAAc;iBACf,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;CACF;AAUD;;GAEG;AACH,MAAM,UAAU,eAAe,CAC7B,QAAgB,EAChB,SAAiB,EACjB,MAA0B;IAE1B,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,8BAA8B,SAAS,cAAc,QAAQ,GAAG,CAAC,CAAC;IACpF,CAAC;IAED,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAEjC,IAAI,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CACb,sCAAsC,SAAS,cAAc,QAAQ,eAAe,CACrF,CAAC;IACJ,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport type {\n CheckpointStore,\n PartitionOwnership,\n Checkpoint,\n OperationOptions,\n} from \"@azure/event-hubs\";\nimport type { Metadata, RestError, BlobSetMetadataResponse } from \"@azure/storage-blob\";\nimport { logger, logErrorStackTrace } from \"./log.js\";\nimport type { ContainerClientLike } from \"./storageBlobInterfaces.js\";\nimport { throwTypeErrorIfParameterMissing } from \"./util/error.js\";\n\n/**\n * An implementation of CheckpointStore that uses Azure Blob Storage to persist checkpoint data.\n */\nexport class BlobCheckpointStore implements CheckpointStore {\n private _containerClient: ContainerClientLike;\n\n /**\n * Constructs a new instance of {@link BlobCheckpointStore}\n * @param containerClient - An instance of a storage blob ContainerClient.\n */\n constructor(containerClient: ContainerClientLike) {\n this._containerClient = containerClient;\n }\n /**\n * Get the list of all existing partition ownership from the underlying data store. May return empty\n * results if there are is no existing ownership information.\n * Partition Ownership contains the information on which `EventHubConsumerClient` subscribe call is currently processing the partition.\n *\n * @param fullyQualifiedNamespace - The fully qualified Event Hubs namespace. This is likely to be similar to\n * <yournamespace>.servicebus.windows.net.\n * @param eventHubName - The event hub name.\n * @param consumerGroup - The consumer group name.\n * @param options - A set of options that can be specified to influence the behavior of this method.\n * - `abortSignal`: A signal used to request operation cancellation.\n * - `tracingOptions`: Options for configuring tracing.\n * @returns Partition ownership details of all the partitions that have had an owner.\n */\n async listOwnership(\n fullyQualifiedNamespace: string,\n eventHubName: string,\n consumerGroup: string,\n options: OperationOptions = {},\n ): Promise<PartitionOwnership[]> {\n const partitionOwnershipArray: PartitionOwnership[] = [];\n const { abortSignal, tracingOptions } = options;\n\n const blobPrefix = BlobCheckpointStore.getBlobPrefix({\n type: \"ownership\",\n fullyQualifiedNamespace,\n eventHubName,\n consumerGroup: consumerGroup,\n });\n\n try {\n const blobs = this._containerClient.listBlobsFlat({\n abortSignal,\n includeMetadata: true,\n prefix: blobPrefix,\n tracingOptions,\n });\n\n for await (const blob of blobs) {\n const blobPath = blob.name.split(\"/\");\n const blobName = blobPath[blobPath.length - 1];\n\n const ownershipMetadata = (blob.metadata as OwnershipMetadata) ?? {};\n\n if (ownershipMetadata.ownerid == null) {\n throw new Error(`Missing ownerid in metadata for blob ${blob.name}`);\n }\n\n const partitionOwnership: PartitionOwnership = {\n fullyQualifiedNamespace,\n eventHubName,\n consumerGroup: consumerGroup,\n ownerId: ownershipMetadata.ownerid,\n partitionId: blobName,\n lastModifiedTimeInMs:\n blob.properties.lastModified && blob.properties.lastModified.getTime(),\n etag: blob.properties.etag,\n };\n partitionOwnershipArray.push(partitionOwnership);\n }\n return partitionOwnershipArray;\n } catch (err: any) {\n logger.warning(`Error occurred while fetching the list of blobs`, err.message);\n logErrorStackTrace(err);\n\n if (err?.name === \"AbortError\") throw err;\n\n throw new Error(`Error occurred while fetching the list of blobs. \\n${err}`);\n }\n }\n\n /**\n * Claim ownership of a list of partitions. This will return the list of partitions that were\n * successfully claimed.\n *\n * @param partitionOwnership - The list of partition ownership this instance is claiming to own.\n * @param options - A set of options that can be specified to influence the behavior of this method.\n * - `abortSignal`: A signal used to request operation cancellation.\n * - `tracingOptions`: Options for configuring tracing.\n * @returns A list partitions this instance successfully claimed ownership.\n */\n async claimOwnership(\n partitionOwnership: PartitionOwnership[],\n options: OperationOptions = {},\n ): Promise<PartitionOwnership[]> {\n const partitionOwnershipArray: PartitionOwnership[] = [];\n for (const ownership of partitionOwnership) {\n const blobName = BlobCheckpointStore.getBlobPrefix({ type: \"ownership\", ...ownership });\n try {\n const updatedBlobResponse = await this._setBlobMetadata(\n blobName,\n {\n ownerid: ownership.ownerId,\n },\n ownership.etag,\n options,\n );\n\n if (updatedBlobResponse.lastModified) {\n ownership.lastModifiedTimeInMs = updatedBlobResponse.lastModified.getTime();\n }\n\n ownership.etag = updatedBlobResponse.etag;\n partitionOwnershipArray.push(ownership);\n logger.info(\n `[${ownership.ownerId}] Claimed ownership successfully for partition: ${ownership.partitionId}`,\n `LastModifiedTime: ${ownership.lastModifiedTimeInMs}, ETag: ${ownership.etag}`,\n );\n } catch (err: any) {\n const restError = err as RestError;\n\n if (restError.statusCode === 412) {\n // etag failures (precondition not met) aren't fatal errors. They happen\n // as multiple consumers attempt to claim the same partition (first one wins)\n // and losers get this error.\n logger.verbose(\n `[${ownership.ownerId}] Did not claim partition ${ownership.partitionId}. Another processor has already claimed it.`,\n );\n continue;\n }\n\n logger.warning(\n `Error occurred while claiming ownership for partition: ${ownership.partitionId}`,\n err.message,\n );\n logErrorStackTrace(err);\n\n throw err;\n }\n }\n return partitionOwnershipArray;\n }\n\n /**\n * Lists all the checkpoints in a data store for a given namespace, eventhub and consumer group.\n *\n * @param fullyQualifiedNamespace - The fully qualified Event Hubs namespace. This is likely to be similar to\n * <yournamespace>.servicebus.windows.net.\n * @param eventHubName - The event hub name.\n * @param consumerGroup - The consumer group name.\n * @param options - A set of options that can be specified to influence the behavior of this method.\n * - `abortSignal`: A signal used to request operation cancellation.\n * - `tracingOptions`: Options for configuring tracing.\n */\n async listCheckpoints(\n fullyQualifiedNamespace: string,\n eventHubName: string,\n consumerGroup: string,\n options: OperationOptions = {},\n ): Promise<Checkpoint[]> {\n const { abortSignal, tracingOptions } = options;\n const blobPrefix = BlobCheckpointStore.getBlobPrefix({\n type: \"checkpoint\",\n fullyQualifiedNamespace,\n eventHubName,\n consumerGroup,\n });\n\n const blobs = this._containerClient.listBlobsFlat({\n abortSignal,\n includeMetadata: true,\n prefix: blobPrefix,\n tracingOptions,\n });\n\n const checkpoints: Checkpoint[] = [];\n\n for await (const blob of blobs) {\n const blobPath = blob.name.split(\"/\");\n const blobName = blobPath[blobPath.length - 1];\n\n const checkpointMetadata = (blob.metadata as CheckpointMetadata) ?? {};\n\n const offset = checkpointMetadata.offset;\n if (offset == null) {\n throw new Error(`Missing metadata property 'offset' on blob '${blob.name}'`);\n }\n const sequenceNumber = parseIntOrThrow(\n blob.name,\n \"sequencenumber\",\n checkpointMetadata.sequencenumber,\n );\n\n checkpoints.push({\n consumerGroup,\n eventHubName,\n fullyQualifiedNamespace,\n partitionId: blobName,\n offset,\n sequenceNumber,\n });\n }\n\n return checkpoints;\n }\n\n /**\n * Updates the checkpoint in the data store for a partition.\n *\n * @param checkpoint - The checkpoint.\n * @param options - A set of options that can be specified to influence the behavior of this method.\n * - `abortSignal`: A signal used to request operation cancellation.\n * - `tracingOptions`: Options for configuring tracing.\n * @returns The new etag on successful update.\n */\n async updateCheckpoint(checkpoint: Checkpoint, options: OperationOptions = {}): Promise<void> {\n throwTypeErrorIfParameterMissing(\n \"updateCheckpoint\",\n \"sequenceNumber\",\n checkpoint.sequenceNumber,\n );\n throwTypeErrorIfParameterMissing(\"updateCheckpoint\", \"offset\", checkpoint.offset);\n\n const blobName = BlobCheckpointStore.getBlobPrefix({ type: \"checkpoint\", ...checkpoint });\n try {\n const metadataResponse = await this._setBlobMetadata(\n blobName,\n {\n sequencenumber: checkpoint.sequenceNumber.toString(),\n offset: checkpoint.offset.toString(),\n },\n undefined,\n options,\n );\n\n logger.verbose(\n `Updated checkpoint successfully for partition: ${checkpoint.partitionId}`,\n `LastModifiedTime: ${metadataResponse.lastModified!.toISOString()}, ETag: ${\n metadataResponse.etag\n }`,\n );\n return;\n } catch (err: any) {\n logger.warning(\n `Error occurred while updating the checkpoint for partition: ${checkpoint.partitionId}.`,\n err.message,\n );\n logErrorStackTrace(err);\n\n if (err?.name === \"AbortError\") throw err;\n\n throw err;\n }\n }\n\n private static getBlobPrefix(params: {\n type: \"ownership\" | \"checkpoint\";\n fullyQualifiedNamespace: string;\n eventHubName: string;\n consumerGroup: string;\n partitionId?: string;\n }): string {\n // none of these are case-sensitive in eventhubs so we need to make sure we don't accidentally allow\n // the user to create a case-sensitive blob for their state!\n const consumerGroupName = params.consumerGroup.toLowerCase();\n const eventHubName = params.eventHubName.toLowerCase();\n const fullyQualifiedNamespace = params.fullyQualifiedNamespace.toLowerCase();\n\n if (params.partitionId) {\n return `${fullyQualifiedNamespace}/${eventHubName}/${consumerGroupName}/${params.type}/${params.partitionId}`;\n } else {\n return `${fullyQualifiedNamespace}/${eventHubName}/${consumerGroupName}/${params.type}/`;\n }\n }\n\n private async _setBlobMetadata(\n blobName: string,\n metadata: OwnershipMetadata | CheckpointMetadata,\n etag: string | undefined,\n options: OperationOptions = {},\n ): Promise<BlobSetMetadataResponse> {\n const { abortSignal, tracingOptions } = options;\n const blockBlobClient = this._containerClient.getBlobClient(blobName).getBlockBlobClient();\n\n // When we have an etag, we know the blob existed.\n // If we encounter an error we should fail.\n if (etag) {\n return blockBlobClient.setMetadata(metadata as Metadata, {\n abortSignal,\n conditions: {\n ifMatch: etag,\n },\n tracingOptions,\n });\n } else {\n try {\n // Attempt to set metadata, and fallback to upload if the blob doesn't already exist.\n // This avoids poor performance in storage accounts with soft-delete or blob versioning enabled.\n // https://github.com/Azure/azure-sdk-for-js/issues/10132\n return await blockBlobClient.setMetadata(metadata as Metadata, {\n abortSignal,\n tracingOptions,\n });\n } catch (err: any) {\n // Check if the error is `BlobNotFound` and fallback to `upload` if it is.\n if (err?.name !== \"RestError\") {\n throw err;\n }\n const errorDetails = (err as RestError).details as { [field: string]: string } | undefined;\n const errorCode = errorDetails?.errorCode;\n if (!errorCode || errorCode !== \"BlobNotFound\") {\n throw err;\n }\n\n return blockBlobClient.upload(\"\", 0, {\n abortSignal,\n metadata: metadata as Metadata,\n tracingOptions,\n });\n }\n }\n }\n}\n\ntype OwnershipMetadata = {\n [k in \"ownerid\"]: string | undefined;\n};\n\ntype CheckpointMetadata = {\n [k in \"sequencenumber\" | \"offset\"]: string | undefined;\n};\n\n/**\n * @internal\n */\nexport function parseIntOrThrow(\n blobName: string,\n fieldName: string,\n numStr: string | undefined,\n): number {\n if (numStr == null) {\n throw new Error(`Missing metadata property '${fieldName}' on blob '${blobName}'`);\n }\n\n const num = parseInt(numStr, 10);\n\n if (isNaN(num)) {\n throw new Error(\n `Failed to parse metadata property '${fieldName}' on blob '${blobName}' as a number`,\n );\n }\n\n return num;\n}\n"]}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { Metadata, BlobItem, ContainerListBlobFlatSegmentResponse, ContainerListBlobsOptions, BlockBlobUploadOptions, BlockBlobUploadResponse, HttpRequestBody, BlobSetMetadataOptions, ContainerSetMetadataResponse } from "@azure/storage-blob";
|
|
2
|
-
import { PagedAsyncIterableIterator } from "@azure/core-paging";
|
|
1
|
+
import type { Metadata, BlobItem, ContainerListBlobFlatSegmentResponse, ContainerListBlobsOptions, BlockBlobUploadOptions, BlockBlobUploadResponse, HttpRequestBody, BlobSetMetadataOptions, ContainerSetMetadataResponse } from "@azure/storage-blob";
|
|
2
|
+
import type { PagedAsyncIterableIterator } from "@azure/core-paging";
|
|
3
3
|
/**
|
|
4
4
|
* An interface compatible with an instance of {@link BlobClient}.
|
|
5
5
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"storageBlobInterfaces.d.ts","sourceRoot":"","sources":["../../src/storageBlobInterfaces.ts"],"names":[],"mappings":"AAGA,OAAO,
|
|
1
|
+
{"version":3,"file":"storageBlobInterfaces.d.ts","sourceRoot":"","sources":["../../src/storageBlobInterfaces.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,QAAQ,EACR,QAAQ,EACR,oCAAoC,EACpC,yBAAyB,EACzB,sBAAsB,EACtB,uBAAuB,EACvB,eAAe,EACf,sBAAsB,EACtB,4BAA4B,EAC7B,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AAErE;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;OAEG;IACH,kBAAkB,IAAI,mBAAmB,CAAC;CAC3C;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC;;OAEG;IACH,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,cAAc,CAAC;IAChD;;;OAGG;IACH,aAAa,CACX,OAAO,CAAC,EAAE,yBAAyB,GAClC,0BAA0B,CAAC,QAAQ,EAAE,oCAAoC,CAAC,CAAC;CAC/E;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC;;OAEG;IACH,MAAM,CACJ,IAAI,EAAE,eAAe,EACrB,aAAa,EAAE,MAAM,EACrB,OAAO,CAAC,EAAE,sBAAsB,GAC/B,OAAO,CAAC,uBAAuB,CAAC,CAAC;IACpC;;OAEG;IACH,WAAW,CACT,QAAQ,CAAC,EAAE,QAAQ,EACnB,OAAO,CAAC,EAAE,sBAAsB,GAC/B,OAAO,CAAC,4BAA4B,CAAC,CAAC;CAC1C"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"storageBlobInterfaces.js","sourceRoot":"","sources":["../../src/storageBlobInterfaces.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport {\n Metadata,\n BlobItem,\n ContainerListBlobFlatSegmentResponse,\n ContainerListBlobsOptions,\n BlockBlobUploadOptions,\n BlockBlobUploadResponse,\n HttpRequestBody,\n BlobSetMetadataOptions,\n ContainerSetMetadataResponse,\n} from \"@azure/storage-blob\";\nimport { PagedAsyncIterableIterator } from \"@azure/core-paging\";\n\n/**\n * An interface compatible with an instance of {@link BlobClient}.\n */\nexport interface BlobClientLike {\n /**\n * Creates a BlockBlobClient object.\n */\n getBlockBlobClient(): BlockBlobClientLike;\n}\n\n/**\n * An interface compatible with Storage Blob's ContainerClient class.\n */\nexport interface ContainerClientLike {\n /**\n * Creates a {@link BlobClient}\n */\n getBlobClient(blobName: string): BlobClientLike;\n /**\n * Returns an async iterable iterator to list all the blobs\n * under the specified account.\n */\n listBlobsFlat(\n options?: ContainerListBlobsOptions,\n ): PagedAsyncIterableIterator<BlobItem, ContainerListBlobFlatSegmentResponse>;\n}\n\n/**\n * An interface compatible with Storage Blob's BlockBlobClient class.\n */\nexport interface BlockBlobClientLike {\n /**\n * Creates a new block blob, or updated the content of an existing block blob.\n */\n upload(\n body: HttpRequestBody,\n contentLength: number,\n options?: BlockBlobUploadOptions,\n ): Promise<BlockBlobUploadResponse>;\n /**\n * Sets user-defined metadata for the specified blob as one or more name-value pairs.\n */\n setMetadata(\n metadata?: Metadata,\n options?: BlobSetMetadataOptions,\n ): Promise<ContainerSetMetadataResponse>;\n}\n"]}
|
|
1
|
+
{"version":3,"file":"storageBlobInterfaces.js","sourceRoot":"","sources":["../../src/storageBlobInterfaces.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport type {\n Metadata,\n BlobItem,\n ContainerListBlobFlatSegmentResponse,\n ContainerListBlobsOptions,\n BlockBlobUploadOptions,\n BlockBlobUploadResponse,\n HttpRequestBody,\n BlobSetMetadataOptions,\n ContainerSetMetadataResponse,\n} from \"@azure/storage-blob\";\nimport type { PagedAsyncIterableIterator } from \"@azure/core-paging\";\n\n/**\n * An interface compatible with an instance of {@link BlobClient}.\n */\nexport interface BlobClientLike {\n /**\n * Creates a BlockBlobClient object.\n */\n getBlockBlobClient(): BlockBlobClientLike;\n}\n\n/**\n * An interface compatible with Storage Blob's ContainerClient class.\n */\nexport interface ContainerClientLike {\n /**\n * Creates a {@link BlobClient}\n */\n getBlobClient(blobName: string): BlobClientLike;\n /**\n * Returns an async iterable iterator to list all the blobs\n * under the specified account.\n */\n listBlobsFlat(\n options?: ContainerListBlobsOptions,\n ): PagedAsyncIterableIterator<BlobItem, ContainerListBlobFlatSegmentResponse>;\n}\n\n/**\n * An interface compatible with Storage Blob's BlockBlobClient class.\n */\nexport interface BlockBlobClientLike {\n /**\n * Creates a new block blob, or updated the content of an existing block blob.\n */\n upload(\n body: HttpRequestBody,\n contentLength: number,\n options?: BlockBlobUploadOptions,\n ): Promise<BlockBlobUploadResponse>;\n /**\n * Sets user-defined metadata for the specified blob as one or more name-value pairs.\n */\n setMetadata(\n metadata?: Metadata,\n options?: BlobSetMetadataOptions,\n ): Promise<ContainerSetMetadataResponse>;\n}\n"]}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { CheckpointStore, PartitionOwnership, Checkpoint, OperationOptions } from "@azure/event-hubs";
|
|
2
|
-
import { ContainerClientLike } from "./storageBlobInterfaces.js";
|
|
1
|
+
import type { CheckpointStore, PartitionOwnership, Checkpoint, OperationOptions } from "@azure/event-hubs";
|
|
2
|
+
import type { ContainerClientLike } from "./storageBlobInterfaces.js";
|
|
3
3
|
/**
|
|
4
4
|
* An implementation of CheckpointStore that uses Azure Blob Storage to persist checkpoint data.
|
|
5
5
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"blobCheckpointStore.d.ts","sourceRoot":"","sources":["../../src/blobCheckpointStore.ts"],"names":[],"mappings":"AAGA,OAAO,
|
|
1
|
+
{"version":3,"file":"blobCheckpointStore.d.ts","sourceRoot":"","sources":["../../src/blobCheckpointStore.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,gBAAgB,EACjB,MAAM,mBAAmB,CAAC;AAG3B,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAGtE;;GAEG;AACH,qBAAa,mBAAoB,YAAW,eAAe;IACzD,OAAO,CAAC,gBAAgB,CAAsB;IAE9C;;;OAGG;gBACS,eAAe,EAAE,mBAAmB;IAGhD;;;;;;;;;;;;;OAaG;IACG,aAAa,CACjB,uBAAuB,EAAE,MAAM,EAC/B,YAAY,EAAE,MAAM,EACpB,aAAa,EAAE,MAAM,EACrB,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC,kBAAkB,EAAE,CAAC;IAoDhC;;;;;;;;;OASG;IACG,cAAc,CAClB,kBAAkB,EAAE,kBAAkB,EAAE,EACxC,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC,kBAAkB,EAAE,CAAC;IAiDhC;;;;;;;;;;OAUG;IACG,eAAe,CACnB,uBAAuB,EAAE,MAAM,EAC/B,YAAY,EAAE,MAAM,EACpB,aAAa,EAAE,MAAM,EACrB,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC,UAAU,EAAE,CAAC;IA+CxB;;;;;;;;OAQG;IACG,gBAAgB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC;IAwC7F,OAAO,CAAC,MAAM,CAAC,aAAa;YAoBd,gBAAgB;CA+C/B;AAUD;;GAEG;AACH,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,GAAG,SAAS,GACzB,MAAM,CAcR"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"blobCheckpointStore.js","sourceRoot":"","sources":["../../src/blobCheckpointStore.ts"],"names":[],"mappings":";AAAA,uCAAuC;AACvC,kCAAkC;;;AA+VlC,0CAkBC;;AAxWD,qCAAsD;AAEtD,8CAAmE;AAEnE;;GAEG;AACH,MAAa,mBAAmB;IAG9B;;;OAGG;IACH,YAAY,eAAoC;QAC9C,IAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC;IAC1C,CAAC;IACD;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,aAAa,CACjB,uBAA+B,EAC/B,YAAoB,EACpB,aAAqB,EACrB,UAA4B,EAAE;;;QAE9B,MAAM,uBAAuB,GAAyB,EAAE,CAAC;QACzD,MAAM,EAAE,WAAW,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAEhD,MAAM,UAAU,GAAG,mBAAmB,CAAC,aAAa,CAAC;YACnD,IAAI,EAAE,WAAW;YACjB,uBAAuB;YACvB,YAAY;YACZ,aAAa,EAAE,aAAa;SAC7B,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC;gBAChD,WAAW;gBACX,eAAe,EAAE,IAAI;gBACrB,MAAM,EAAE,UAAU;gBAClB,cAAc;aACf,CAAC,CAAC;;gBAEH,KAAyB,eAAA,UAAA,sBAAA,KAAK,CAAA,WAAA,yEAAE,CAAC;oBAAR,qBAAK;oBAAL,WAAK;oBAAnB,MAAM,IAAI,KAAA,CAAA;oBACnB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBACtC,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBAE/C,MAAM,iBAAiB,GAAG,MAAC,IAAI,CAAC,QAA8B,mCAAI,EAAE,CAAC;oBAErE,IAAI,iBAAiB,CAAC,OAAO,IAAI,IAAI,EAAE,CAAC;wBACtC,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;oBACvE,CAAC;oBAED,MAAM,kBAAkB,GAAuB;wBAC7C,uBAAuB;wBACvB,YAAY;wBACZ,aAAa,EAAE,aAAa;wBAC5B,OAAO,EAAE,iBAAiB,CAAC,OAAO;wBAClC,WAAW,EAAE,QAAQ;wBACrB,oBAAoB,EAClB,IAAI,CAAC,UAAU,CAAC,YAAY,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,OAAO,EAAE;wBACxE,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI;qBAC3B,CAAC;oBACF,uBAAuB,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;gBACnD,CAAC;;;;;;;;;YACD,OAAO,uBAAuB,CAAC;QACjC,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,eAAM,CAAC,OAAO,CAAC,iDAAiD,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;YAC/E,IAAA,2BAAkB,EAAC,GAAG,CAAC,CAAC;YAExB,IAAI,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,YAAY;gBAAE,MAAM,GAAG,CAAC;YAE1C,MAAM,IAAI,KAAK,CAAC,sDAAsD,GAAG,EAAE,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,cAAc,CAClB,kBAAwC,EACxC,UAA4B,EAAE;QAE9B,MAAM,uBAAuB,GAAyB,EAAE,CAAC;QACzD,KAAK,MAAM,SAAS,IAAI,kBAAkB,EAAE,CAAC;YAC3C,MAAM,QAAQ,GAAG,mBAAmB,CAAC,aAAa,iBAAG,IAAI,EAAE,WAAW,IAAK,SAAS,EAAG,CAAC;YACxF,IAAI,CAAC;gBACH,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,gBAAgB,CACrD,QAAQ,EACR;oBACE,OAAO,EAAE,SAAS,CAAC,OAAO;iBAC3B,EACD,SAAS,CAAC,IAAI,EACd,OAAO,CACR,CAAC;gBAEF,IAAI,mBAAmB,CAAC,YAAY,EAAE,CAAC;oBACrC,SAAS,CAAC,oBAAoB,GAAG,mBAAmB,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;gBAC9E,CAAC;gBAED,SAAS,CAAC,IAAI,GAAG,mBAAmB,CAAC,IAAI,CAAC;gBAC1C,uBAAuB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACxC,eAAM,CAAC,IAAI,CACT,IAAI,SAAS,CAAC,OAAO,mDAAmD,SAAS,CAAC,WAAW,EAAE,EAC/F,qBAAqB,SAAS,CAAC,oBAAoB,WAAW,SAAS,CAAC,IAAI,EAAE,CAC/E,CAAC;YACJ,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAClB,MAAM,SAAS,GAAG,GAAgB,CAAC;gBAEnC,IAAI,SAAS,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;oBACjC,wEAAwE;oBACxE,6EAA6E;oBAC7E,6BAA6B;oBAC7B,eAAM,CAAC,OAAO,CACZ,IAAI,SAAS,CAAC,OAAO,6BAA6B,SAAS,CAAC,WAAW,6CAA6C,CACrH,CAAC;oBACF,SAAS;gBACX,CAAC;gBAED,eAAM,CAAC,OAAO,CACZ,0DAA0D,SAAS,CAAC,WAAW,EAAE,EACjF,GAAG,CAAC,OAAO,CACZ,CAAC;gBACF,IAAA,2BAAkB,EAAC,GAAG,CAAC,CAAC;gBAExB,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;QACD,OAAO,uBAAuB,CAAC;IACjC,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,eAAe,CACnB,uBAA+B,EAC/B,YAAoB,EACpB,aAAqB,EACrB,UAA4B,EAAE;;;QAE9B,MAAM,EAAE,WAAW,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAChD,MAAM,UAAU,GAAG,mBAAmB,CAAC,aAAa,CAAC;YACnD,IAAI,EAAE,YAAY;YAClB,uBAAuB;YACvB,YAAY;YACZ,aAAa;SACd,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC;YAChD,WAAW;YACX,eAAe,EAAE,IAAI;YACrB,MAAM,EAAE,UAAU;YAClB,cAAc;SACf,CAAC,CAAC;QAEH,MAAM,WAAW,GAAiB,EAAE,CAAC;;YAErC,KAAyB,eAAA,UAAA,sBAAA,KAAK,CAAA,WAAA,yEAAE,CAAC;gBAAR,qBAAK;gBAAL,WAAK;gBAAnB,MAAM,IAAI,KAAA,CAAA;gBACnB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACtC,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAE/C,MAAM,kBAAkB,GAAG,MAAC,IAAI,CAAC,QAA+B,mCAAI,EAAE,CAAC;gBAEvE,MAAM,MAAM,GAAG,kBAAkB,CAAC,MAAM,CAAC;gBACzC,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;oBACnB,MAAM,IAAI,KAAK,CAAC,+CAA+C,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;gBAC/E,CAAC;gBACD,MAAM,cAAc,GAAG,eAAe,CACpC,IAAI,CAAC,IAAI,EACT,gBAAgB,EAChB,kBAAkB,CAAC,cAAc,CAClC,CAAC;gBAEF,WAAW,CAAC,IAAI,CAAC;oBACf,aAAa;oBACb,YAAY;oBACZ,uBAAuB;oBACvB,WAAW,EAAE,QAAQ;oBACrB,MAAM;oBACN,cAAc;iBACf,CAAC,CAAC;YACL,CAAC;;;;;;;;;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,gBAAgB,CAAC,UAAsB,EAAE,UAA4B,EAAE;QAC3E,IAAA,2CAAgC,EAC9B,kBAAkB,EAClB,gBAAgB,EAChB,UAAU,CAAC,cAAc,CAC1B,CAAC;QACF,IAAA,2CAAgC,EAAC,kBAAkB,EAAE,QAAQ,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QAElF,MAAM,QAAQ,GAAG,mBAAmB,CAAC,aAAa,iBAAG,IAAI,EAAE,YAAY,IAAK,UAAU,EAAG,CAAC;QAC1F,IAAI,CAAC;YACH,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAClD,QAAQ,EACR;gBACE,cAAc,EAAE,UAAU,CAAC,cAAc,CAAC,QAAQ,EAAE;gBACpD,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,QAAQ,EAAE;aACrC,EACD,SAAS,EACT,OAAO,CACR,CAAC;YAEF,eAAM,CAAC,OAAO,CACZ,kDAAkD,UAAU,CAAC,WAAW,EAAE,EAC1E,qBAAqB,gBAAgB,CAAC,YAAa,CAAC,WAAW,EAAE,WAC/D,gBAAgB,CAAC,IACnB,EAAE,CACH,CAAC;YACF,OAAO;QACT,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,eAAM,CAAC,OAAO,CACZ,+DAA+D,UAAU,CAAC,WAAW,GAAG,EACxF,GAAG,CAAC,OAAO,CACZ,CAAC;YACF,IAAA,2BAAkB,EAAC,GAAG,CAAC,CAAC;YAExB,IAAI,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,YAAY;gBAAE,MAAM,GAAG,CAAC;YAE1C,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,MAAM,CAAC,aAAa,CAAC,MAM5B;QACC,oGAAoG;QACpG,4DAA4D;QAC5D,MAAM,iBAAiB,GAAG,MAAM,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC;QAC7D,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC;QACvD,MAAM,uBAAuB,GAAG,MAAM,CAAC,uBAAuB,CAAC,WAAW,EAAE,CAAC;QAE7E,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;YACvB,OAAO,GAAG,uBAAuB,IAAI,YAAY,IAAI,iBAAiB,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QAChH,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,uBAAuB,IAAI,YAAY,IAAI,iBAAiB,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC;QAC3F,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAC5B,QAAgB,EAChB,QAAgD,EAChD,IAAwB,EACxB,UAA4B,EAAE;QAE9B,MAAM,EAAE,WAAW,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAChD,MAAM,eAAe,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,kBAAkB,EAAE,CAAC;QAE3F,kDAAkD;QAClD,2CAA2C;QAC3C,IAAI,IAAI,EAAE,CAAC;YACT,OAAO,eAAe,CAAC,WAAW,CAAC,QAAoB,EAAE;gBACvD,WAAW;gBACX,UAAU,EAAE;oBACV,OAAO,EAAE,IAAI;iBACd;gBACD,cAAc;aACf,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,IAAI,CAAC;gBACH,qFAAqF;gBACrF,gGAAgG;gBAChG,yDAAyD;gBACzD,OAAO,MAAM,eAAe,CAAC,WAAW,CAAC,QAAoB,EAAE;oBAC7D,WAAW;oBACX,cAAc;iBACf,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAClB,0EAA0E;gBAC1E,IAAI,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,WAAW,EAAE,CAAC;oBAC9B,MAAM,GAAG,CAAC;gBACZ,CAAC;gBACD,MAAM,YAAY,GAAI,GAAiB,CAAC,OAAkD,CAAC;gBAC3F,MAAM,SAAS,GAAG,YAAY,aAAZ,YAAY,uBAAZ,YAAY,CAAE,SAAS,CAAC;gBAC1C,IAAI,CAAC,SAAS,IAAI,SAAS,KAAK,cAAc,EAAE,CAAC;oBAC/C,MAAM,GAAG,CAAC;gBACZ,CAAC;gBAED,OAAO,eAAe,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,EAAE;oBACnC,WAAW;oBACX,QAAQ,EAAE,QAAoB;oBAC9B,cAAc;iBACf,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;CACF;AAlUD,kDAkUC;AAUD;;GAEG;AACH,SAAgB,eAAe,CAC7B,QAAgB,EAChB,SAAiB,EACjB,MAA0B;IAE1B,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,8BAA8B,SAAS,cAAc,QAAQ,GAAG,CAAC,CAAC;IACpF,CAAC;IAED,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAEjC,IAAI,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CACb,sCAAsC,SAAS,cAAc,QAAQ,eAAe,CACrF,CAAC;IACJ,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport {\n CheckpointStore,\n PartitionOwnership,\n Checkpoint,\n OperationOptions,\n} from \"@azure/event-hubs\";\nimport { Metadata, RestError, BlobSetMetadataResponse } from \"@azure/storage-blob\";\nimport { logger, logErrorStackTrace } from \"./log.js\";\nimport { ContainerClientLike } from \"./storageBlobInterfaces.js\";\nimport { throwTypeErrorIfParameterMissing } from \"./util/error.js\";\n\n/**\n * An implementation of CheckpointStore that uses Azure Blob Storage to persist checkpoint data.\n */\nexport class BlobCheckpointStore implements CheckpointStore {\n private _containerClient: ContainerClientLike;\n\n /**\n * Constructs a new instance of {@link BlobCheckpointStore}\n * @param containerClient - An instance of a storage blob ContainerClient.\n */\n constructor(containerClient: ContainerClientLike) {\n this._containerClient = containerClient;\n }\n /**\n * Get the list of all existing partition ownership from the underlying data store. May return empty\n * results if there are is no existing ownership information.\n * Partition Ownership contains the information on which `EventHubConsumerClient` subscribe call is currently processing the partition.\n *\n * @param fullyQualifiedNamespace - The fully qualified Event Hubs namespace. This is likely to be similar to\n * <yournamespace>.servicebus.windows.net.\n * @param eventHubName - The event hub name.\n * @param consumerGroup - The consumer group name.\n * @param options - A set of options that can be specified to influence the behavior of this method.\n * - `abortSignal`: A signal used to request operation cancellation.\n * - `tracingOptions`: Options for configuring tracing.\n * @returns Partition ownership details of all the partitions that have had an owner.\n */\n async listOwnership(\n fullyQualifiedNamespace: string,\n eventHubName: string,\n consumerGroup: string,\n options: OperationOptions = {},\n ): Promise<PartitionOwnership[]> {\n const partitionOwnershipArray: PartitionOwnership[] = [];\n const { abortSignal, tracingOptions } = options;\n\n const blobPrefix = BlobCheckpointStore.getBlobPrefix({\n type: \"ownership\",\n fullyQualifiedNamespace,\n eventHubName,\n consumerGroup: consumerGroup,\n });\n\n try {\n const blobs = this._containerClient.listBlobsFlat({\n abortSignal,\n includeMetadata: true,\n prefix: blobPrefix,\n tracingOptions,\n });\n\n for await (const blob of blobs) {\n const blobPath = blob.name.split(\"/\");\n const blobName = blobPath[blobPath.length - 1];\n\n const ownershipMetadata = (blob.metadata as OwnershipMetadata) ?? {};\n\n if (ownershipMetadata.ownerid == null) {\n throw new Error(`Missing ownerid in metadata for blob ${blob.name}`);\n }\n\n const partitionOwnership: PartitionOwnership = {\n fullyQualifiedNamespace,\n eventHubName,\n consumerGroup: consumerGroup,\n ownerId: ownershipMetadata.ownerid,\n partitionId: blobName,\n lastModifiedTimeInMs:\n blob.properties.lastModified && blob.properties.lastModified.getTime(),\n etag: blob.properties.etag,\n };\n partitionOwnershipArray.push(partitionOwnership);\n }\n return partitionOwnershipArray;\n } catch (err: any) {\n logger.warning(`Error occurred while fetching the list of blobs`, err.message);\n logErrorStackTrace(err);\n\n if (err?.name === \"AbortError\") throw err;\n\n throw new Error(`Error occurred while fetching the list of blobs. \\n${err}`);\n }\n }\n\n /**\n * Claim ownership of a list of partitions. This will return the list of partitions that were\n * successfully claimed.\n *\n * @param partitionOwnership - The list of partition ownership this instance is claiming to own.\n * @param options - A set of options that can be specified to influence the behavior of this method.\n * - `abortSignal`: A signal used to request operation cancellation.\n * - `tracingOptions`: Options for configuring tracing.\n * @returns A list partitions this instance successfully claimed ownership.\n */\n async claimOwnership(\n partitionOwnership: PartitionOwnership[],\n options: OperationOptions = {},\n ): Promise<PartitionOwnership[]> {\n const partitionOwnershipArray: PartitionOwnership[] = [];\n for (const ownership of partitionOwnership) {\n const blobName = BlobCheckpointStore.getBlobPrefix({ type: \"ownership\", ...ownership });\n try {\n const updatedBlobResponse = await this._setBlobMetadata(\n blobName,\n {\n ownerid: ownership.ownerId,\n },\n ownership.etag,\n options,\n );\n\n if (updatedBlobResponse.lastModified) {\n ownership.lastModifiedTimeInMs = updatedBlobResponse.lastModified.getTime();\n }\n\n ownership.etag = updatedBlobResponse.etag;\n partitionOwnershipArray.push(ownership);\n logger.info(\n `[${ownership.ownerId}] Claimed ownership successfully for partition: ${ownership.partitionId}`,\n `LastModifiedTime: ${ownership.lastModifiedTimeInMs}, ETag: ${ownership.etag}`,\n );\n } catch (err: any) {\n const restError = err as RestError;\n\n if (restError.statusCode === 412) {\n // etag failures (precondition not met) aren't fatal errors. They happen\n // as multiple consumers attempt to claim the same partition (first one wins)\n // and losers get this error.\n logger.verbose(\n `[${ownership.ownerId}] Did not claim partition ${ownership.partitionId}. Another processor has already claimed it.`,\n );\n continue;\n }\n\n logger.warning(\n `Error occurred while claiming ownership for partition: ${ownership.partitionId}`,\n err.message,\n );\n logErrorStackTrace(err);\n\n throw err;\n }\n }\n return partitionOwnershipArray;\n }\n\n /**\n * Lists all the checkpoints in a data store for a given namespace, eventhub and consumer group.\n *\n * @param fullyQualifiedNamespace - The fully qualified Event Hubs namespace. This is likely to be similar to\n * <yournamespace>.servicebus.windows.net.\n * @param eventHubName - The event hub name.\n * @param consumerGroup - The consumer group name.\n * @param options - A set of options that can be specified to influence the behavior of this method.\n * - `abortSignal`: A signal used to request operation cancellation.\n * - `tracingOptions`: Options for configuring tracing.\n */\n async listCheckpoints(\n fullyQualifiedNamespace: string,\n eventHubName: string,\n consumerGroup: string,\n options: OperationOptions = {},\n ): Promise<Checkpoint[]> {\n const { abortSignal, tracingOptions } = options;\n const blobPrefix = BlobCheckpointStore.getBlobPrefix({\n type: \"checkpoint\",\n fullyQualifiedNamespace,\n eventHubName,\n consumerGroup,\n });\n\n const blobs = this._containerClient.listBlobsFlat({\n abortSignal,\n includeMetadata: true,\n prefix: blobPrefix,\n tracingOptions,\n });\n\n const checkpoints: Checkpoint[] = [];\n\n for await (const blob of blobs) {\n const blobPath = blob.name.split(\"/\");\n const blobName = blobPath[blobPath.length - 1];\n\n const checkpointMetadata = (blob.metadata as CheckpointMetadata) ?? {};\n\n const offset = checkpointMetadata.offset;\n if (offset == null) {\n throw new Error(`Missing metadata property 'offset' on blob '${blob.name}'`);\n }\n const sequenceNumber = parseIntOrThrow(\n blob.name,\n \"sequencenumber\",\n checkpointMetadata.sequencenumber,\n );\n\n checkpoints.push({\n consumerGroup,\n eventHubName,\n fullyQualifiedNamespace,\n partitionId: blobName,\n offset,\n sequenceNumber,\n });\n }\n\n return checkpoints;\n }\n\n /**\n * Updates the checkpoint in the data store for a partition.\n *\n * @param checkpoint - The checkpoint.\n * @param options - A set of options that can be specified to influence the behavior of this method.\n * - `abortSignal`: A signal used to request operation cancellation.\n * - `tracingOptions`: Options for configuring tracing.\n * @returns The new etag on successful update.\n */\n async updateCheckpoint(checkpoint: Checkpoint, options: OperationOptions = {}): Promise<void> {\n throwTypeErrorIfParameterMissing(\n \"updateCheckpoint\",\n \"sequenceNumber\",\n checkpoint.sequenceNumber,\n );\n throwTypeErrorIfParameterMissing(\"updateCheckpoint\", \"offset\", checkpoint.offset);\n\n const blobName = BlobCheckpointStore.getBlobPrefix({ type: \"checkpoint\", ...checkpoint });\n try {\n const metadataResponse = await this._setBlobMetadata(\n blobName,\n {\n sequencenumber: checkpoint.sequenceNumber.toString(),\n offset: checkpoint.offset.toString(),\n },\n undefined,\n options,\n );\n\n logger.verbose(\n `Updated checkpoint successfully for partition: ${checkpoint.partitionId}`,\n `LastModifiedTime: ${metadataResponse.lastModified!.toISOString()}, ETag: ${\n metadataResponse.etag\n }`,\n );\n return;\n } catch (err: any) {\n logger.warning(\n `Error occurred while updating the checkpoint for partition: ${checkpoint.partitionId}.`,\n err.message,\n );\n logErrorStackTrace(err);\n\n if (err?.name === \"AbortError\") throw err;\n\n throw err;\n }\n }\n\n private static getBlobPrefix(params: {\n type: \"ownership\" | \"checkpoint\";\n fullyQualifiedNamespace: string;\n eventHubName: string;\n consumerGroup: string;\n partitionId?: string;\n }): string {\n // none of these are case-sensitive in eventhubs so we need to make sure we don't accidentally allow\n // the user to create a case-sensitive blob for their state!\n const consumerGroupName = params.consumerGroup.toLowerCase();\n const eventHubName = params.eventHubName.toLowerCase();\n const fullyQualifiedNamespace = params.fullyQualifiedNamespace.toLowerCase();\n\n if (params.partitionId) {\n return `${fullyQualifiedNamespace}/${eventHubName}/${consumerGroupName}/${params.type}/${params.partitionId}`;\n } else {\n return `${fullyQualifiedNamespace}/${eventHubName}/${consumerGroupName}/${params.type}/`;\n }\n }\n\n private async _setBlobMetadata(\n blobName: string,\n metadata: OwnershipMetadata | CheckpointMetadata,\n etag: string | undefined,\n options: OperationOptions = {},\n ): Promise<BlobSetMetadataResponse> {\n const { abortSignal, tracingOptions } = options;\n const blockBlobClient = this._containerClient.getBlobClient(blobName).getBlockBlobClient();\n\n // When we have an etag, we know the blob existed.\n // If we encounter an error we should fail.\n if (etag) {\n return blockBlobClient.setMetadata(metadata as Metadata, {\n abortSignal,\n conditions: {\n ifMatch: etag,\n },\n tracingOptions,\n });\n } else {\n try {\n // Attempt to set metadata, and fallback to upload if the blob doesn't already exist.\n // This avoids poor performance in storage accounts with soft-delete or blob versioning enabled.\n // https://github.com/Azure/azure-sdk-for-js/issues/10132\n return await blockBlobClient.setMetadata(metadata as Metadata, {\n abortSignal,\n tracingOptions,\n });\n } catch (err: any) {\n // Check if the error is `BlobNotFound` and fallback to `upload` if it is.\n if (err?.name !== \"RestError\") {\n throw err;\n }\n const errorDetails = (err as RestError).details as { [field: string]: string } | undefined;\n const errorCode = errorDetails?.errorCode;\n if (!errorCode || errorCode !== \"BlobNotFound\") {\n throw err;\n }\n\n return blockBlobClient.upload(\"\", 0, {\n abortSignal,\n metadata: metadata as Metadata,\n tracingOptions,\n });\n }\n }\n }\n}\n\ntype OwnershipMetadata = {\n [k in \"ownerid\"]: string | undefined;\n};\n\ntype CheckpointMetadata = {\n [k in \"sequencenumber\" | \"offset\"]: string | undefined;\n};\n\n/**\n * @internal\n */\nexport function parseIntOrThrow(\n blobName: string,\n fieldName: string,\n numStr: string | undefined,\n): number {\n if (numStr == null) {\n throw new Error(`Missing metadata property '${fieldName}' on blob '${blobName}'`);\n }\n\n const num = parseInt(numStr, 10);\n\n if (isNaN(num)) {\n throw new Error(\n `Failed to parse metadata property '${fieldName}' on blob '${blobName}' as a number`,\n );\n }\n\n return num;\n}\n"]}
|
|
1
|
+
{"version":3,"file":"blobCheckpointStore.js","sourceRoot":"","sources":["../../src/blobCheckpointStore.ts"],"names":[],"mappings":";AAAA,uCAAuC;AACvC,kCAAkC;;;AA+VlC,0CAkBC;;AAxWD,qCAAsD;AAEtD,8CAAmE;AAEnE;;GAEG;AACH,MAAa,mBAAmB;IAG9B;;;OAGG;IACH,YAAY,eAAoC;QAC9C,IAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC;IAC1C,CAAC;IACD;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,aAAa,CACjB,uBAA+B,EAC/B,YAAoB,EACpB,aAAqB,EACrB,UAA4B,EAAE;;;QAE9B,MAAM,uBAAuB,GAAyB,EAAE,CAAC;QACzD,MAAM,EAAE,WAAW,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAEhD,MAAM,UAAU,GAAG,mBAAmB,CAAC,aAAa,CAAC;YACnD,IAAI,EAAE,WAAW;YACjB,uBAAuB;YACvB,YAAY;YACZ,aAAa,EAAE,aAAa;SAC7B,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC;gBAChD,WAAW;gBACX,eAAe,EAAE,IAAI;gBACrB,MAAM,EAAE,UAAU;gBAClB,cAAc;aACf,CAAC,CAAC;;gBAEH,KAAyB,eAAA,UAAA,sBAAA,KAAK,CAAA,WAAA,yEAAE,CAAC;oBAAR,qBAAK;oBAAL,WAAK;oBAAnB,MAAM,IAAI,KAAA,CAAA;oBACnB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBACtC,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBAE/C,MAAM,iBAAiB,GAAG,MAAC,IAAI,CAAC,QAA8B,mCAAI,EAAE,CAAC;oBAErE,IAAI,iBAAiB,CAAC,OAAO,IAAI,IAAI,EAAE,CAAC;wBACtC,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;oBACvE,CAAC;oBAED,MAAM,kBAAkB,GAAuB;wBAC7C,uBAAuB;wBACvB,YAAY;wBACZ,aAAa,EAAE,aAAa;wBAC5B,OAAO,EAAE,iBAAiB,CAAC,OAAO;wBAClC,WAAW,EAAE,QAAQ;wBACrB,oBAAoB,EAClB,IAAI,CAAC,UAAU,CAAC,YAAY,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,OAAO,EAAE;wBACxE,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI;qBAC3B,CAAC;oBACF,uBAAuB,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;gBACnD,CAAC;;;;;;;;;YACD,OAAO,uBAAuB,CAAC;QACjC,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,eAAM,CAAC,OAAO,CAAC,iDAAiD,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;YAC/E,IAAA,2BAAkB,EAAC,GAAG,CAAC,CAAC;YAExB,IAAI,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,YAAY;gBAAE,MAAM,GAAG,CAAC;YAE1C,MAAM,IAAI,KAAK,CAAC,sDAAsD,GAAG,EAAE,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,cAAc,CAClB,kBAAwC,EACxC,UAA4B,EAAE;QAE9B,MAAM,uBAAuB,GAAyB,EAAE,CAAC;QACzD,KAAK,MAAM,SAAS,IAAI,kBAAkB,EAAE,CAAC;YAC3C,MAAM,QAAQ,GAAG,mBAAmB,CAAC,aAAa,iBAAG,IAAI,EAAE,WAAW,IAAK,SAAS,EAAG,CAAC;YACxF,IAAI,CAAC;gBACH,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,gBAAgB,CACrD,QAAQ,EACR;oBACE,OAAO,EAAE,SAAS,CAAC,OAAO;iBAC3B,EACD,SAAS,CAAC,IAAI,EACd,OAAO,CACR,CAAC;gBAEF,IAAI,mBAAmB,CAAC,YAAY,EAAE,CAAC;oBACrC,SAAS,CAAC,oBAAoB,GAAG,mBAAmB,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;gBAC9E,CAAC;gBAED,SAAS,CAAC,IAAI,GAAG,mBAAmB,CAAC,IAAI,CAAC;gBAC1C,uBAAuB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACxC,eAAM,CAAC,IAAI,CACT,IAAI,SAAS,CAAC,OAAO,mDAAmD,SAAS,CAAC,WAAW,EAAE,EAC/F,qBAAqB,SAAS,CAAC,oBAAoB,WAAW,SAAS,CAAC,IAAI,EAAE,CAC/E,CAAC;YACJ,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAClB,MAAM,SAAS,GAAG,GAAgB,CAAC;gBAEnC,IAAI,SAAS,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;oBACjC,wEAAwE;oBACxE,6EAA6E;oBAC7E,6BAA6B;oBAC7B,eAAM,CAAC,OAAO,CACZ,IAAI,SAAS,CAAC,OAAO,6BAA6B,SAAS,CAAC,WAAW,6CAA6C,CACrH,CAAC;oBACF,SAAS;gBACX,CAAC;gBAED,eAAM,CAAC,OAAO,CACZ,0DAA0D,SAAS,CAAC,WAAW,EAAE,EACjF,GAAG,CAAC,OAAO,CACZ,CAAC;gBACF,IAAA,2BAAkB,EAAC,GAAG,CAAC,CAAC;gBAExB,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;QACD,OAAO,uBAAuB,CAAC;IACjC,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,eAAe,CACnB,uBAA+B,EAC/B,YAAoB,EACpB,aAAqB,EACrB,UAA4B,EAAE;;;QAE9B,MAAM,EAAE,WAAW,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAChD,MAAM,UAAU,GAAG,mBAAmB,CAAC,aAAa,CAAC;YACnD,IAAI,EAAE,YAAY;YAClB,uBAAuB;YACvB,YAAY;YACZ,aAAa;SACd,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC;YAChD,WAAW;YACX,eAAe,EAAE,IAAI;YACrB,MAAM,EAAE,UAAU;YAClB,cAAc;SACf,CAAC,CAAC;QAEH,MAAM,WAAW,GAAiB,EAAE,CAAC;;YAErC,KAAyB,eAAA,UAAA,sBAAA,KAAK,CAAA,WAAA,yEAAE,CAAC;gBAAR,qBAAK;gBAAL,WAAK;gBAAnB,MAAM,IAAI,KAAA,CAAA;gBACnB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACtC,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAE/C,MAAM,kBAAkB,GAAG,MAAC,IAAI,CAAC,QAA+B,mCAAI,EAAE,CAAC;gBAEvE,MAAM,MAAM,GAAG,kBAAkB,CAAC,MAAM,CAAC;gBACzC,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;oBACnB,MAAM,IAAI,KAAK,CAAC,+CAA+C,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;gBAC/E,CAAC;gBACD,MAAM,cAAc,GAAG,eAAe,CACpC,IAAI,CAAC,IAAI,EACT,gBAAgB,EAChB,kBAAkB,CAAC,cAAc,CAClC,CAAC;gBAEF,WAAW,CAAC,IAAI,CAAC;oBACf,aAAa;oBACb,YAAY;oBACZ,uBAAuB;oBACvB,WAAW,EAAE,QAAQ;oBACrB,MAAM;oBACN,cAAc;iBACf,CAAC,CAAC;YACL,CAAC;;;;;;;;;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,gBAAgB,CAAC,UAAsB,EAAE,UAA4B,EAAE;QAC3E,IAAA,2CAAgC,EAC9B,kBAAkB,EAClB,gBAAgB,EAChB,UAAU,CAAC,cAAc,CAC1B,CAAC;QACF,IAAA,2CAAgC,EAAC,kBAAkB,EAAE,QAAQ,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QAElF,MAAM,QAAQ,GAAG,mBAAmB,CAAC,aAAa,iBAAG,IAAI,EAAE,YAAY,IAAK,UAAU,EAAG,CAAC;QAC1F,IAAI,CAAC;YACH,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAClD,QAAQ,EACR;gBACE,cAAc,EAAE,UAAU,CAAC,cAAc,CAAC,QAAQ,EAAE;gBACpD,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,QAAQ,EAAE;aACrC,EACD,SAAS,EACT,OAAO,CACR,CAAC;YAEF,eAAM,CAAC,OAAO,CACZ,kDAAkD,UAAU,CAAC,WAAW,EAAE,EAC1E,qBAAqB,gBAAgB,CAAC,YAAa,CAAC,WAAW,EAAE,WAC/D,gBAAgB,CAAC,IACnB,EAAE,CACH,CAAC;YACF,OAAO;QACT,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,eAAM,CAAC,OAAO,CACZ,+DAA+D,UAAU,CAAC,WAAW,GAAG,EACxF,GAAG,CAAC,OAAO,CACZ,CAAC;YACF,IAAA,2BAAkB,EAAC,GAAG,CAAC,CAAC;YAExB,IAAI,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,YAAY;gBAAE,MAAM,GAAG,CAAC;YAE1C,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,MAAM,CAAC,aAAa,CAAC,MAM5B;QACC,oGAAoG;QACpG,4DAA4D;QAC5D,MAAM,iBAAiB,GAAG,MAAM,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC;QAC7D,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC;QACvD,MAAM,uBAAuB,GAAG,MAAM,CAAC,uBAAuB,CAAC,WAAW,EAAE,CAAC;QAE7E,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;YACvB,OAAO,GAAG,uBAAuB,IAAI,YAAY,IAAI,iBAAiB,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QAChH,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,uBAAuB,IAAI,YAAY,IAAI,iBAAiB,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC;QAC3F,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAC5B,QAAgB,EAChB,QAAgD,EAChD,IAAwB,EACxB,UAA4B,EAAE;QAE9B,MAAM,EAAE,WAAW,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAChD,MAAM,eAAe,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,kBAAkB,EAAE,CAAC;QAE3F,kDAAkD;QAClD,2CAA2C;QAC3C,IAAI,IAAI,EAAE,CAAC;YACT,OAAO,eAAe,CAAC,WAAW,CAAC,QAAoB,EAAE;gBACvD,WAAW;gBACX,UAAU,EAAE;oBACV,OAAO,EAAE,IAAI;iBACd;gBACD,cAAc;aACf,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,IAAI,CAAC;gBACH,qFAAqF;gBACrF,gGAAgG;gBAChG,yDAAyD;gBACzD,OAAO,MAAM,eAAe,CAAC,WAAW,CAAC,QAAoB,EAAE;oBAC7D,WAAW;oBACX,cAAc;iBACf,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAClB,0EAA0E;gBAC1E,IAAI,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,WAAW,EAAE,CAAC;oBAC9B,MAAM,GAAG,CAAC;gBACZ,CAAC;gBACD,MAAM,YAAY,GAAI,GAAiB,CAAC,OAAkD,CAAC;gBAC3F,MAAM,SAAS,GAAG,YAAY,aAAZ,YAAY,uBAAZ,YAAY,CAAE,SAAS,CAAC;gBAC1C,IAAI,CAAC,SAAS,IAAI,SAAS,KAAK,cAAc,EAAE,CAAC;oBAC/C,MAAM,GAAG,CAAC;gBACZ,CAAC;gBAED,OAAO,eAAe,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,EAAE;oBACnC,WAAW;oBACX,QAAQ,EAAE,QAAoB;oBAC9B,cAAc;iBACf,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;CACF;AAlUD,kDAkUC;AAUD;;GAEG;AACH,SAAgB,eAAe,CAC7B,QAAgB,EAChB,SAAiB,EACjB,MAA0B;IAE1B,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,8BAA8B,SAAS,cAAc,QAAQ,GAAG,CAAC,CAAC;IACpF,CAAC;IAED,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAEjC,IAAI,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CACb,sCAAsC,SAAS,cAAc,QAAQ,eAAe,CACrF,CAAC;IACJ,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport type {\n CheckpointStore,\n PartitionOwnership,\n Checkpoint,\n OperationOptions,\n} from \"@azure/event-hubs\";\nimport type { Metadata, RestError, BlobSetMetadataResponse } from \"@azure/storage-blob\";\nimport { logger, logErrorStackTrace } from \"./log.js\";\nimport type { ContainerClientLike } from \"./storageBlobInterfaces.js\";\nimport { throwTypeErrorIfParameterMissing } from \"./util/error.js\";\n\n/**\n * An implementation of CheckpointStore that uses Azure Blob Storage to persist checkpoint data.\n */\nexport class BlobCheckpointStore implements CheckpointStore {\n private _containerClient: ContainerClientLike;\n\n /**\n * Constructs a new instance of {@link BlobCheckpointStore}\n * @param containerClient - An instance of a storage blob ContainerClient.\n */\n constructor(containerClient: ContainerClientLike) {\n this._containerClient = containerClient;\n }\n /**\n * Get the list of all existing partition ownership from the underlying data store. May return empty\n * results if there are is no existing ownership information.\n * Partition Ownership contains the information on which `EventHubConsumerClient` subscribe call is currently processing the partition.\n *\n * @param fullyQualifiedNamespace - The fully qualified Event Hubs namespace. This is likely to be similar to\n * <yournamespace>.servicebus.windows.net.\n * @param eventHubName - The event hub name.\n * @param consumerGroup - The consumer group name.\n * @param options - A set of options that can be specified to influence the behavior of this method.\n * - `abortSignal`: A signal used to request operation cancellation.\n * - `tracingOptions`: Options for configuring tracing.\n * @returns Partition ownership details of all the partitions that have had an owner.\n */\n async listOwnership(\n fullyQualifiedNamespace: string,\n eventHubName: string,\n consumerGroup: string,\n options: OperationOptions = {},\n ): Promise<PartitionOwnership[]> {\n const partitionOwnershipArray: PartitionOwnership[] = [];\n const { abortSignal, tracingOptions } = options;\n\n const blobPrefix = BlobCheckpointStore.getBlobPrefix({\n type: \"ownership\",\n fullyQualifiedNamespace,\n eventHubName,\n consumerGroup: consumerGroup,\n });\n\n try {\n const blobs = this._containerClient.listBlobsFlat({\n abortSignal,\n includeMetadata: true,\n prefix: blobPrefix,\n tracingOptions,\n });\n\n for await (const blob of blobs) {\n const blobPath = blob.name.split(\"/\");\n const blobName = blobPath[blobPath.length - 1];\n\n const ownershipMetadata = (blob.metadata as OwnershipMetadata) ?? {};\n\n if (ownershipMetadata.ownerid == null) {\n throw new Error(`Missing ownerid in metadata for blob ${blob.name}`);\n }\n\n const partitionOwnership: PartitionOwnership = {\n fullyQualifiedNamespace,\n eventHubName,\n consumerGroup: consumerGroup,\n ownerId: ownershipMetadata.ownerid,\n partitionId: blobName,\n lastModifiedTimeInMs:\n blob.properties.lastModified && blob.properties.lastModified.getTime(),\n etag: blob.properties.etag,\n };\n partitionOwnershipArray.push(partitionOwnership);\n }\n return partitionOwnershipArray;\n } catch (err: any) {\n logger.warning(`Error occurred while fetching the list of blobs`, err.message);\n logErrorStackTrace(err);\n\n if (err?.name === \"AbortError\") throw err;\n\n throw new Error(`Error occurred while fetching the list of blobs. \\n${err}`);\n }\n }\n\n /**\n * Claim ownership of a list of partitions. This will return the list of partitions that were\n * successfully claimed.\n *\n * @param partitionOwnership - The list of partition ownership this instance is claiming to own.\n * @param options - A set of options that can be specified to influence the behavior of this method.\n * - `abortSignal`: A signal used to request operation cancellation.\n * - `tracingOptions`: Options for configuring tracing.\n * @returns A list partitions this instance successfully claimed ownership.\n */\n async claimOwnership(\n partitionOwnership: PartitionOwnership[],\n options: OperationOptions = {},\n ): Promise<PartitionOwnership[]> {\n const partitionOwnershipArray: PartitionOwnership[] = [];\n for (const ownership of partitionOwnership) {\n const blobName = BlobCheckpointStore.getBlobPrefix({ type: \"ownership\", ...ownership });\n try {\n const updatedBlobResponse = await this._setBlobMetadata(\n blobName,\n {\n ownerid: ownership.ownerId,\n },\n ownership.etag,\n options,\n );\n\n if (updatedBlobResponse.lastModified) {\n ownership.lastModifiedTimeInMs = updatedBlobResponse.lastModified.getTime();\n }\n\n ownership.etag = updatedBlobResponse.etag;\n partitionOwnershipArray.push(ownership);\n logger.info(\n `[${ownership.ownerId}] Claimed ownership successfully for partition: ${ownership.partitionId}`,\n `LastModifiedTime: ${ownership.lastModifiedTimeInMs}, ETag: ${ownership.etag}`,\n );\n } catch (err: any) {\n const restError = err as RestError;\n\n if (restError.statusCode === 412) {\n // etag failures (precondition not met) aren't fatal errors. They happen\n // as multiple consumers attempt to claim the same partition (first one wins)\n // and losers get this error.\n logger.verbose(\n `[${ownership.ownerId}] Did not claim partition ${ownership.partitionId}. Another processor has already claimed it.`,\n );\n continue;\n }\n\n logger.warning(\n `Error occurred while claiming ownership for partition: ${ownership.partitionId}`,\n err.message,\n );\n logErrorStackTrace(err);\n\n throw err;\n }\n }\n return partitionOwnershipArray;\n }\n\n /**\n * Lists all the checkpoints in a data store for a given namespace, eventhub and consumer group.\n *\n * @param fullyQualifiedNamespace - The fully qualified Event Hubs namespace. This is likely to be similar to\n * <yournamespace>.servicebus.windows.net.\n * @param eventHubName - The event hub name.\n * @param consumerGroup - The consumer group name.\n * @param options - A set of options that can be specified to influence the behavior of this method.\n * - `abortSignal`: A signal used to request operation cancellation.\n * - `tracingOptions`: Options for configuring tracing.\n */\n async listCheckpoints(\n fullyQualifiedNamespace: string,\n eventHubName: string,\n consumerGroup: string,\n options: OperationOptions = {},\n ): Promise<Checkpoint[]> {\n const { abortSignal, tracingOptions } = options;\n const blobPrefix = BlobCheckpointStore.getBlobPrefix({\n type: \"checkpoint\",\n fullyQualifiedNamespace,\n eventHubName,\n consumerGroup,\n });\n\n const blobs = this._containerClient.listBlobsFlat({\n abortSignal,\n includeMetadata: true,\n prefix: blobPrefix,\n tracingOptions,\n });\n\n const checkpoints: Checkpoint[] = [];\n\n for await (const blob of blobs) {\n const blobPath = blob.name.split(\"/\");\n const blobName = blobPath[blobPath.length - 1];\n\n const checkpointMetadata = (blob.metadata as CheckpointMetadata) ?? {};\n\n const offset = checkpointMetadata.offset;\n if (offset == null) {\n throw new Error(`Missing metadata property 'offset' on blob '${blob.name}'`);\n }\n const sequenceNumber = parseIntOrThrow(\n blob.name,\n \"sequencenumber\",\n checkpointMetadata.sequencenumber,\n );\n\n checkpoints.push({\n consumerGroup,\n eventHubName,\n fullyQualifiedNamespace,\n partitionId: blobName,\n offset,\n sequenceNumber,\n });\n }\n\n return checkpoints;\n }\n\n /**\n * Updates the checkpoint in the data store for a partition.\n *\n * @param checkpoint - The checkpoint.\n * @param options - A set of options that can be specified to influence the behavior of this method.\n * - `abortSignal`: A signal used to request operation cancellation.\n * - `tracingOptions`: Options for configuring tracing.\n * @returns The new etag on successful update.\n */\n async updateCheckpoint(checkpoint: Checkpoint, options: OperationOptions = {}): Promise<void> {\n throwTypeErrorIfParameterMissing(\n \"updateCheckpoint\",\n \"sequenceNumber\",\n checkpoint.sequenceNumber,\n );\n throwTypeErrorIfParameterMissing(\"updateCheckpoint\", \"offset\", checkpoint.offset);\n\n const blobName = BlobCheckpointStore.getBlobPrefix({ type: \"checkpoint\", ...checkpoint });\n try {\n const metadataResponse = await this._setBlobMetadata(\n blobName,\n {\n sequencenumber: checkpoint.sequenceNumber.toString(),\n offset: checkpoint.offset.toString(),\n },\n undefined,\n options,\n );\n\n logger.verbose(\n `Updated checkpoint successfully for partition: ${checkpoint.partitionId}`,\n `LastModifiedTime: ${metadataResponse.lastModified!.toISOString()}, ETag: ${\n metadataResponse.etag\n }`,\n );\n return;\n } catch (err: any) {\n logger.warning(\n `Error occurred while updating the checkpoint for partition: ${checkpoint.partitionId}.`,\n err.message,\n );\n logErrorStackTrace(err);\n\n if (err?.name === \"AbortError\") throw err;\n\n throw err;\n }\n }\n\n private static getBlobPrefix(params: {\n type: \"ownership\" | \"checkpoint\";\n fullyQualifiedNamespace: string;\n eventHubName: string;\n consumerGroup: string;\n partitionId?: string;\n }): string {\n // none of these are case-sensitive in eventhubs so we need to make sure we don't accidentally allow\n // the user to create a case-sensitive blob for their state!\n const consumerGroupName = params.consumerGroup.toLowerCase();\n const eventHubName = params.eventHubName.toLowerCase();\n const fullyQualifiedNamespace = params.fullyQualifiedNamespace.toLowerCase();\n\n if (params.partitionId) {\n return `${fullyQualifiedNamespace}/${eventHubName}/${consumerGroupName}/${params.type}/${params.partitionId}`;\n } else {\n return `${fullyQualifiedNamespace}/${eventHubName}/${consumerGroupName}/${params.type}/`;\n }\n }\n\n private async _setBlobMetadata(\n blobName: string,\n metadata: OwnershipMetadata | CheckpointMetadata,\n etag: string | undefined,\n options: OperationOptions = {},\n ): Promise<BlobSetMetadataResponse> {\n const { abortSignal, tracingOptions } = options;\n const blockBlobClient = this._containerClient.getBlobClient(blobName).getBlockBlobClient();\n\n // When we have an etag, we know the blob existed.\n // If we encounter an error we should fail.\n if (etag) {\n return blockBlobClient.setMetadata(metadata as Metadata, {\n abortSignal,\n conditions: {\n ifMatch: etag,\n },\n tracingOptions,\n });\n } else {\n try {\n // Attempt to set metadata, and fallback to upload if the blob doesn't already exist.\n // This avoids poor performance in storage accounts with soft-delete or blob versioning enabled.\n // https://github.com/Azure/azure-sdk-for-js/issues/10132\n return await blockBlobClient.setMetadata(metadata as Metadata, {\n abortSignal,\n tracingOptions,\n });\n } catch (err: any) {\n // Check if the error is `BlobNotFound` and fallback to `upload` if it is.\n if (err?.name !== \"RestError\") {\n throw err;\n }\n const errorDetails = (err as RestError).details as { [field: string]: string } | undefined;\n const errorCode = errorDetails?.errorCode;\n if (!errorCode || errorCode !== \"BlobNotFound\") {\n throw err;\n }\n\n return blockBlobClient.upload(\"\", 0, {\n abortSignal,\n metadata: metadata as Metadata,\n tracingOptions,\n });\n }\n }\n }\n}\n\ntype OwnershipMetadata = {\n [k in \"ownerid\"]: string | undefined;\n};\n\ntype CheckpointMetadata = {\n [k in \"sequencenumber\" | \"offset\"]: string | undefined;\n};\n\n/**\n * @internal\n */\nexport function parseIntOrThrow(\n blobName: string,\n fieldName: string,\n numStr: string | undefined,\n): number {\n if (numStr == null) {\n throw new Error(`Missing metadata property '${fieldName}' on blob '${blobName}'`);\n }\n\n const num = parseInt(numStr, 10);\n\n if (isNaN(num)) {\n throw new Error(\n `Failed to parse metadata property '${fieldName}' on blob '${blobName}' as a number`,\n );\n }\n\n return num;\n}\n"]}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { Metadata, BlobItem, ContainerListBlobFlatSegmentResponse, ContainerListBlobsOptions, BlockBlobUploadOptions, BlockBlobUploadResponse, HttpRequestBody, BlobSetMetadataOptions, ContainerSetMetadataResponse } from "@azure/storage-blob";
|
|
2
|
-
import { PagedAsyncIterableIterator } from "@azure/core-paging";
|
|
1
|
+
import type { Metadata, BlobItem, ContainerListBlobFlatSegmentResponse, ContainerListBlobsOptions, BlockBlobUploadOptions, BlockBlobUploadResponse, HttpRequestBody, BlobSetMetadataOptions, ContainerSetMetadataResponse } from "@azure/storage-blob";
|
|
2
|
+
import type { PagedAsyncIterableIterator } from "@azure/core-paging";
|
|
3
3
|
/**
|
|
4
4
|
* An interface compatible with an instance of {@link BlobClient}.
|
|
5
5
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"storageBlobInterfaces.d.ts","sourceRoot":"","sources":["../../src/storageBlobInterfaces.ts"],"names":[],"mappings":"AAGA,OAAO,
|
|
1
|
+
{"version":3,"file":"storageBlobInterfaces.d.ts","sourceRoot":"","sources":["../../src/storageBlobInterfaces.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,QAAQ,EACR,QAAQ,EACR,oCAAoC,EACpC,yBAAyB,EACzB,sBAAsB,EACtB,uBAAuB,EACvB,eAAe,EACf,sBAAsB,EACtB,4BAA4B,EAC7B,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AAErE;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;OAEG;IACH,kBAAkB,IAAI,mBAAmB,CAAC;CAC3C;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC;;OAEG;IACH,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,cAAc,CAAC;IAChD;;;OAGG;IACH,aAAa,CACX,OAAO,CAAC,EAAE,yBAAyB,GAClC,0BAA0B,CAAC,QAAQ,EAAE,oCAAoC,CAAC,CAAC;CAC/E;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC;;OAEG;IACH,MAAM,CACJ,IAAI,EAAE,eAAe,EACrB,aAAa,EAAE,MAAM,EACrB,OAAO,CAAC,EAAE,sBAAsB,GAC/B,OAAO,CAAC,uBAAuB,CAAC,CAAC;IACpC;;OAEG;IACH,WAAW,CACT,QAAQ,CAAC,EAAE,QAAQ,EACnB,OAAO,CAAC,EAAE,sBAAsB,GAC/B,OAAO,CAAC,4BAA4B,CAAC,CAAC;CAC1C"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"storageBlobInterfaces.js","sourceRoot":"","sources":["../../src/storageBlobInterfaces.ts"],"names":[],"mappings":";AAAA,uCAAuC;AACvC,kCAAkC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport {\n Metadata,\n BlobItem,\n ContainerListBlobFlatSegmentResponse,\n ContainerListBlobsOptions,\n BlockBlobUploadOptions,\n BlockBlobUploadResponse,\n HttpRequestBody,\n BlobSetMetadataOptions,\n ContainerSetMetadataResponse,\n} from \"@azure/storage-blob\";\nimport { PagedAsyncIterableIterator } from \"@azure/core-paging\";\n\n/**\n * An interface compatible with an instance of {@link BlobClient}.\n */\nexport interface BlobClientLike {\n /**\n * Creates a BlockBlobClient object.\n */\n getBlockBlobClient(): BlockBlobClientLike;\n}\n\n/**\n * An interface compatible with Storage Blob's ContainerClient class.\n */\nexport interface ContainerClientLike {\n /**\n * Creates a {@link BlobClient}\n */\n getBlobClient(blobName: string): BlobClientLike;\n /**\n * Returns an async iterable iterator to list all the blobs\n * under the specified account.\n */\n listBlobsFlat(\n options?: ContainerListBlobsOptions,\n ): PagedAsyncIterableIterator<BlobItem, ContainerListBlobFlatSegmentResponse>;\n}\n\n/**\n * An interface compatible with Storage Blob's BlockBlobClient class.\n */\nexport interface BlockBlobClientLike {\n /**\n * Creates a new block blob, or updated the content of an existing block blob.\n */\n upload(\n body: HttpRequestBody,\n contentLength: number,\n options?: BlockBlobUploadOptions,\n ): Promise<BlockBlobUploadResponse>;\n /**\n * Sets user-defined metadata for the specified blob as one or more name-value pairs.\n */\n setMetadata(\n metadata?: Metadata,\n options?: BlobSetMetadataOptions,\n ): Promise<ContainerSetMetadataResponse>;\n}\n"]}
|
|
1
|
+
{"version":3,"file":"storageBlobInterfaces.js","sourceRoot":"","sources":["../../src/storageBlobInterfaces.ts"],"names":[],"mappings":";AAAA,uCAAuC;AACvC,kCAAkC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport type {\n Metadata,\n BlobItem,\n ContainerListBlobFlatSegmentResponse,\n ContainerListBlobsOptions,\n BlockBlobUploadOptions,\n BlockBlobUploadResponse,\n HttpRequestBody,\n BlobSetMetadataOptions,\n ContainerSetMetadataResponse,\n} from \"@azure/storage-blob\";\nimport type { PagedAsyncIterableIterator } from \"@azure/core-paging\";\n\n/**\n * An interface compatible with an instance of {@link BlobClient}.\n */\nexport interface BlobClientLike {\n /**\n * Creates a BlockBlobClient object.\n */\n getBlockBlobClient(): BlockBlobClientLike;\n}\n\n/**\n * An interface compatible with Storage Blob's ContainerClient class.\n */\nexport interface ContainerClientLike {\n /**\n * Creates a {@link BlobClient}\n */\n getBlobClient(blobName: string): BlobClientLike;\n /**\n * Returns an async iterable iterator to list all the blobs\n * under the specified account.\n */\n listBlobsFlat(\n options?: ContainerListBlobsOptions,\n ): PagedAsyncIterableIterator<BlobItem, ContainerListBlobFlatSegmentResponse>;\n}\n\n/**\n * An interface compatible with Storage Blob's BlockBlobClient class.\n */\nexport interface BlockBlobClientLike {\n /**\n * Creates a new block blob, or updated the content of an existing block blob.\n */\n upload(\n body: HttpRequestBody,\n contentLength: number,\n options?: BlockBlobUploadOptions,\n ): Promise<BlockBlobUploadResponse>;\n /**\n * Sets user-defined metadata for the specified blob as one or more name-value pairs.\n */\n setMetadata(\n metadata?: Metadata,\n options?: BlobSetMetadataOptions,\n ): Promise<ContainerSetMetadataResponse>;\n}\n"]}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { CheckpointStore, PartitionOwnership, Checkpoint, OperationOptions } from "@azure/event-hubs";
|
|
2
|
-
import { ContainerClientLike } from "./storageBlobInterfaces.js";
|
|
1
|
+
import type { CheckpointStore, PartitionOwnership, Checkpoint, OperationOptions } from "@azure/event-hubs";
|
|
2
|
+
import type { ContainerClientLike } from "./storageBlobInterfaces.js";
|
|
3
3
|
/**
|
|
4
4
|
* An implementation of CheckpointStore that uses Azure Blob Storage to persist checkpoint data.
|
|
5
5
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"blobCheckpointStore.d.ts","sourceRoot":"","sources":["../../src/blobCheckpointStore.ts"],"names":[],"mappings":"AAGA,OAAO,
|
|
1
|
+
{"version":3,"file":"blobCheckpointStore.d.ts","sourceRoot":"","sources":["../../src/blobCheckpointStore.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,gBAAgB,EACjB,MAAM,mBAAmB,CAAC;AAG3B,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAGtE;;GAEG;AACH,qBAAa,mBAAoB,YAAW,eAAe;IACzD,OAAO,CAAC,gBAAgB,CAAsB;IAE9C;;;OAGG;gBACS,eAAe,EAAE,mBAAmB;IAGhD;;;;;;;;;;;;;OAaG;IACG,aAAa,CACjB,uBAAuB,EAAE,MAAM,EAC/B,YAAY,EAAE,MAAM,EACpB,aAAa,EAAE,MAAM,EACrB,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC,kBAAkB,EAAE,CAAC;IAoDhC;;;;;;;;;OASG;IACG,cAAc,CAClB,kBAAkB,EAAE,kBAAkB,EAAE,EACxC,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC,kBAAkB,EAAE,CAAC;IAiDhC;;;;;;;;;;OAUG;IACG,eAAe,CACnB,uBAAuB,EAAE,MAAM,EAC/B,YAAY,EAAE,MAAM,EACpB,aAAa,EAAE,MAAM,EACrB,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC,UAAU,EAAE,CAAC;IA+CxB;;;;;;;;OAQG;IACG,gBAAgB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC;IAwC7F,OAAO,CAAC,MAAM,CAAC,aAAa;YAoBd,gBAAgB;CA+C/B;AAUD;;GAEG;AACH,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,GAAG,SAAS,GACzB,MAAM,CAcR"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"blobCheckpointStore.js","sourceRoot":"","sources":["../../src/blobCheckpointStore.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;;AASlC,OAAO,EAAE,MAAM,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAEtD,OAAO,EAAE,gCAAgC,EAAE,MAAM,iBAAiB,CAAC;AAEnE;;GAEG;AACH,MAAM,OAAO,mBAAmB;IAG9B;;;OAGG;IACH,YAAY,eAAoC;QAC9C,IAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC;IAC1C,CAAC;IACD;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,aAAa,CACjB,uBAA+B,EAC/B,YAAoB,EACpB,aAAqB,EACrB,UAA4B,EAAE;;;QAE9B,MAAM,uBAAuB,GAAyB,EAAE,CAAC;QACzD,MAAM,EAAE,WAAW,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAEhD,MAAM,UAAU,GAAG,mBAAmB,CAAC,aAAa,CAAC;YACnD,IAAI,EAAE,WAAW;YACjB,uBAAuB;YACvB,YAAY;YACZ,aAAa,EAAE,aAAa;SAC7B,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC;gBAChD,WAAW;gBACX,eAAe,EAAE,IAAI;gBACrB,MAAM,EAAE,UAAU;gBAClB,cAAc;aACf,CAAC,CAAC;;gBAEH,KAAyB,eAAA,UAAA,cAAA,KAAK,CAAA,WAAA,yEAAE,CAAC;oBAAR,qBAAK;oBAAL,WAAK;oBAAnB,MAAM,IAAI,KAAA,CAAA;oBACnB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBACtC,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBAE/C,MAAM,iBAAiB,GAAG,MAAC,IAAI,CAAC,QAA8B,mCAAI,EAAE,CAAC;oBAErE,IAAI,iBAAiB,CAAC,OAAO,IAAI,IAAI,EAAE,CAAC;wBACtC,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;oBACvE,CAAC;oBAED,MAAM,kBAAkB,GAAuB;wBAC7C,uBAAuB;wBACvB,YAAY;wBACZ,aAAa,EAAE,aAAa;wBAC5B,OAAO,EAAE,iBAAiB,CAAC,OAAO;wBAClC,WAAW,EAAE,QAAQ;wBACrB,oBAAoB,EAClB,IAAI,CAAC,UAAU,CAAC,YAAY,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,OAAO,EAAE;wBACxE,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI;qBAC3B,CAAC;oBACF,uBAAuB,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;gBACnD,CAAC;;;;;;;;;YACD,OAAO,uBAAuB,CAAC;QACjC,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,MAAM,CAAC,OAAO,CAAC,iDAAiD,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;YAC/E,kBAAkB,CAAC,GAAG,CAAC,CAAC;YAExB,IAAI,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,YAAY;gBAAE,MAAM,GAAG,CAAC;YAE1C,MAAM,IAAI,KAAK,CAAC,sDAAsD,GAAG,EAAE,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,cAAc,CAClB,kBAAwC,EACxC,UAA4B,EAAE;QAE9B,MAAM,uBAAuB,GAAyB,EAAE,CAAC;QACzD,KAAK,MAAM,SAAS,IAAI,kBAAkB,EAAE,CAAC;YAC3C,MAAM,QAAQ,GAAG,mBAAmB,CAAC,aAAa,iBAAG,IAAI,EAAE,WAAW,IAAK,SAAS,EAAG,CAAC;YACxF,IAAI,CAAC;gBACH,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,gBAAgB,CACrD,QAAQ,EACR;oBACE,OAAO,EAAE,SAAS,CAAC,OAAO;iBAC3B,EACD,SAAS,CAAC,IAAI,EACd,OAAO,CACR,CAAC;gBAEF,IAAI,mBAAmB,CAAC,YAAY,EAAE,CAAC;oBACrC,SAAS,CAAC,oBAAoB,GAAG,mBAAmB,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;gBAC9E,CAAC;gBAED,SAAS,CAAC,IAAI,GAAG,mBAAmB,CAAC,IAAI,CAAC;gBAC1C,uBAAuB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACxC,MAAM,CAAC,IAAI,CACT,IAAI,SAAS,CAAC,OAAO,mDAAmD,SAAS,CAAC,WAAW,EAAE,EAC/F,qBAAqB,SAAS,CAAC,oBAAoB,WAAW,SAAS,CAAC,IAAI,EAAE,CAC/E,CAAC;YACJ,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAClB,MAAM,SAAS,GAAG,GAAgB,CAAC;gBAEnC,IAAI,SAAS,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;oBACjC,wEAAwE;oBACxE,6EAA6E;oBAC7E,6BAA6B;oBAC7B,MAAM,CAAC,OAAO,CACZ,IAAI,SAAS,CAAC,OAAO,6BAA6B,SAAS,CAAC,WAAW,6CAA6C,CACrH,CAAC;oBACF,SAAS;gBACX,CAAC;gBAED,MAAM,CAAC,OAAO,CACZ,0DAA0D,SAAS,CAAC,WAAW,EAAE,EACjF,GAAG,CAAC,OAAO,CACZ,CAAC;gBACF,kBAAkB,CAAC,GAAG,CAAC,CAAC;gBAExB,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;QACD,OAAO,uBAAuB,CAAC;IACjC,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,eAAe,CACnB,uBAA+B,EAC/B,YAAoB,EACpB,aAAqB,EACrB,UAA4B,EAAE;;;QAE9B,MAAM,EAAE,WAAW,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAChD,MAAM,UAAU,GAAG,mBAAmB,CAAC,aAAa,CAAC;YACnD,IAAI,EAAE,YAAY;YAClB,uBAAuB;YACvB,YAAY;YACZ,aAAa;SACd,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC;YAChD,WAAW;YACX,eAAe,EAAE,IAAI;YACrB,MAAM,EAAE,UAAU;YAClB,cAAc;SACf,CAAC,CAAC;QAEH,MAAM,WAAW,GAAiB,EAAE,CAAC;;YAErC,KAAyB,eAAA,UAAA,cAAA,KAAK,CAAA,WAAA,yEAAE,CAAC;gBAAR,qBAAK;gBAAL,WAAK;gBAAnB,MAAM,IAAI,KAAA,CAAA;gBACnB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACtC,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAE/C,MAAM,kBAAkB,GAAG,MAAC,IAAI,CAAC,QAA+B,mCAAI,EAAE,CAAC;gBAEvE,MAAM,MAAM,GAAG,kBAAkB,CAAC,MAAM,CAAC;gBACzC,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;oBACnB,MAAM,IAAI,KAAK,CAAC,+CAA+C,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;gBAC/E,CAAC;gBACD,MAAM,cAAc,GAAG,eAAe,CACpC,IAAI,CAAC,IAAI,EACT,gBAAgB,EAChB,kBAAkB,CAAC,cAAc,CAClC,CAAC;gBAEF,WAAW,CAAC,IAAI,CAAC;oBACf,aAAa;oBACb,YAAY;oBACZ,uBAAuB;oBACvB,WAAW,EAAE,QAAQ;oBACrB,MAAM;oBACN,cAAc;iBACf,CAAC,CAAC;YACL,CAAC;;;;;;;;;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,gBAAgB,CAAC,UAAsB,EAAE,UAA4B,EAAE;QAC3E,gCAAgC,CAC9B,kBAAkB,EAClB,gBAAgB,EAChB,UAAU,CAAC,cAAc,CAC1B,CAAC;QACF,gCAAgC,CAAC,kBAAkB,EAAE,QAAQ,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QAElF,MAAM,QAAQ,GAAG,mBAAmB,CAAC,aAAa,iBAAG,IAAI,EAAE,YAAY,IAAK,UAAU,EAAG,CAAC;QAC1F,IAAI,CAAC;YACH,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAClD,QAAQ,EACR;gBACE,cAAc,EAAE,UAAU,CAAC,cAAc,CAAC,QAAQ,EAAE;gBACpD,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,QAAQ,EAAE;aACrC,EACD,SAAS,EACT,OAAO,CACR,CAAC;YAEF,MAAM,CAAC,OAAO,CACZ,kDAAkD,UAAU,CAAC,WAAW,EAAE,EAC1E,qBAAqB,gBAAgB,CAAC,YAAa,CAAC,WAAW,EAAE,WAC/D,gBAAgB,CAAC,IACnB,EAAE,CACH,CAAC;YACF,OAAO;QACT,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,MAAM,CAAC,OAAO,CACZ,+DAA+D,UAAU,CAAC,WAAW,GAAG,EACxF,GAAG,CAAC,OAAO,CACZ,CAAC;YACF,kBAAkB,CAAC,GAAG,CAAC,CAAC;YAExB,IAAI,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,YAAY;gBAAE,MAAM,GAAG,CAAC;YAE1C,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,MAAM,CAAC,aAAa,CAAC,MAM5B;QACC,oGAAoG;QACpG,4DAA4D;QAC5D,MAAM,iBAAiB,GAAG,MAAM,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC;QAC7D,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC;QACvD,MAAM,uBAAuB,GAAG,MAAM,CAAC,uBAAuB,CAAC,WAAW,EAAE,CAAC;QAE7E,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;YACvB,OAAO,GAAG,uBAAuB,IAAI,YAAY,IAAI,iBAAiB,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QAChH,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,uBAAuB,IAAI,YAAY,IAAI,iBAAiB,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC;QAC3F,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAC5B,QAAgB,EAChB,QAAgD,EAChD,IAAwB,EACxB,UAA4B,EAAE;QAE9B,MAAM,EAAE,WAAW,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAChD,MAAM,eAAe,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,kBAAkB,EAAE,CAAC;QAE3F,kDAAkD;QAClD,2CAA2C;QAC3C,IAAI,IAAI,EAAE,CAAC;YACT,OAAO,eAAe,CAAC,WAAW,CAAC,QAAoB,EAAE;gBACvD,WAAW;gBACX,UAAU,EAAE;oBACV,OAAO,EAAE,IAAI;iBACd;gBACD,cAAc;aACf,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,IAAI,CAAC;gBACH,qFAAqF;gBACrF,gGAAgG;gBAChG,yDAAyD;gBACzD,OAAO,MAAM,eAAe,CAAC,WAAW,CAAC,QAAoB,EAAE;oBAC7D,WAAW;oBACX,cAAc;iBACf,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAClB,0EAA0E;gBAC1E,IAAI,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,WAAW,EAAE,CAAC;oBAC9B,MAAM,GAAG,CAAC;gBACZ,CAAC;gBACD,MAAM,YAAY,GAAI,GAAiB,CAAC,OAAkD,CAAC;gBAC3F,MAAM,SAAS,GAAG,YAAY,aAAZ,YAAY,uBAAZ,YAAY,CAAE,SAAS,CAAC;gBAC1C,IAAI,CAAC,SAAS,IAAI,SAAS,KAAK,cAAc,EAAE,CAAC;oBAC/C,MAAM,GAAG,CAAC;gBACZ,CAAC;gBAED,OAAO,eAAe,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,EAAE;oBACnC,WAAW;oBACX,QAAQ,EAAE,QAAoB;oBAC9B,cAAc;iBACf,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;CACF;AAUD;;GAEG;AACH,MAAM,UAAU,eAAe,CAC7B,QAAgB,EAChB,SAAiB,EACjB,MAA0B;IAE1B,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,8BAA8B,SAAS,cAAc,QAAQ,GAAG,CAAC,CAAC;IACpF,CAAC;IAED,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAEjC,IAAI,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CACb,sCAAsC,SAAS,cAAc,QAAQ,eAAe,CACrF,CAAC;IACJ,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport {\n CheckpointStore,\n PartitionOwnership,\n Checkpoint,\n OperationOptions,\n} from \"@azure/event-hubs\";\nimport { Metadata, RestError, BlobSetMetadataResponse } from \"@azure/storage-blob\";\nimport { logger, logErrorStackTrace } from \"./log.js\";\nimport { ContainerClientLike } from \"./storageBlobInterfaces.js\";\nimport { throwTypeErrorIfParameterMissing } from \"./util/error.js\";\n\n/**\n * An implementation of CheckpointStore that uses Azure Blob Storage to persist checkpoint data.\n */\nexport class BlobCheckpointStore implements CheckpointStore {\n private _containerClient: ContainerClientLike;\n\n /**\n * Constructs a new instance of {@link BlobCheckpointStore}\n * @param containerClient - An instance of a storage blob ContainerClient.\n */\n constructor(containerClient: ContainerClientLike) {\n this._containerClient = containerClient;\n }\n /**\n * Get the list of all existing partition ownership from the underlying data store. May return empty\n * results if there are is no existing ownership information.\n * Partition Ownership contains the information on which `EventHubConsumerClient` subscribe call is currently processing the partition.\n *\n * @param fullyQualifiedNamespace - The fully qualified Event Hubs namespace. This is likely to be similar to\n * <yournamespace>.servicebus.windows.net.\n * @param eventHubName - The event hub name.\n * @param consumerGroup - The consumer group name.\n * @param options - A set of options that can be specified to influence the behavior of this method.\n * - `abortSignal`: A signal used to request operation cancellation.\n * - `tracingOptions`: Options for configuring tracing.\n * @returns Partition ownership details of all the partitions that have had an owner.\n */\n async listOwnership(\n fullyQualifiedNamespace: string,\n eventHubName: string,\n consumerGroup: string,\n options: OperationOptions = {},\n ): Promise<PartitionOwnership[]> {\n const partitionOwnershipArray: PartitionOwnership[] = [];\n const { abortSignal, tracingOptions } = options;\n\n const blobPrefix = BlobCheckpointStore.getBlobPrefix({\n type: \"ownership\",\n fullyQualifiedNamespace,\n eventHubName,\n consumerGroup: consumerGroup,\n });\n\n try {\n const blobs = this._containerClient.listBlobsFlat({\n abortSignal,\n includeMetadata: true,\n prefix: blobPrefix,\n tracingOptions,\n });\n\n for await (const blob of blobs) {\n const blobPath = blob.name.split(\"/\");\n const blobName = blobPath[blobPath.length - 1];\n\n const ownershipMetadata = (blob.metadata as OwnershipMetadata) ?? {};\n\n if (ownershipMetadata.ownerid == null) {\n throw new Error(`Missing ownerid in metadata for blob ${blob.name}`);\n }\n\n const partitionOwnership: PartitionOwnership = {\n fullyQualifiedNamespace,\n eventHubName,\n consumerGroup: consumerGroup,\n ownerId: ownershipMetadata.ownerid,\n partitionId: blobName,\n lastModifiedTimeInMs:\n blob.properties.lastModified && blob.properties.lastModified.getTime(),\n etag: blob.properties.etag,\n };\n partitionOwnershipArray.push(partitionOwnership);\n }\n return partitionOwnershipArray;\n } catch (err: any) {\n logger.warning(`Error occurred while fetching the list of blobs`, err.message);\n logErrorStackTrace(err);\n\n if (err?.name === \"AbortError\") throw err;\n\n throw new Error(`Error occurred while fetching the list of blobs. \\n${err}`);\n }\n }\n\n /**\n * Claim ownership of a list of partitions. This will return the list of partitions that were\n * successfully claimed.\n *\n * @param partitionOwnership - The list of partition ownership this instance is claiming to own.\n * @param options - A set of options that can be specified to influence the behavior of this method.\n * - `abortSignal`: A signal used to request operation cancellation.\n * - `tracingOptions`: Options for configuring tracing.\n * @returns A list partitions this instance successfully claimed ownership.\n */\n async claimOwnership(\n partitionOwnership: PartitionOwnership[],\n options: OperationOptions = {},\n ): Promise<PartitionOwnership[]> {\n const partitionOwnershipArray: PartitionOwnership[] = [];\n for (const ownership of partitionOwnership) {\n const blobName = BlobCheckpointStore.getBlobPrefix({ type: \"ownership\", ...ownership });\n try {\n const updatedBlobResponse = await this._setBlobMetadata(\n blobName,\n {\n ownerid: ownership.ownerId,\n },\n ownership.etag,\n options,\n );\n\n if (updatedBlobResponse.lastModified) {\n ownership.lastModifiedTimeInMs = updatedBlobResponse.lastModified.getTime();\n }\n\n ownership.etag = updatedBlobResponse.etag;\n partitionOwnershipArray.push(ownership);\n logger.info(\n `[${ownership.ownerId}] Claimed ownership successfully for partition: ${ownership.partitionId}`,\n `LastModifiedTime: ${ownership.lastModifiedTimeInMs}, ETag: ${ownership.etag}`,\n );\n } catch (err: any) {\n const restError = err as RestError;\n\n if (restError.statusCode === 412) {\n // etag failures (precondition not met) aren't fatal errors. They happen\n // as multiple consumers attempt to claim the same partition (first one wins)\n // and losers get this error.\n logger.verbose(\n `[${ownership.ownerId}] Did not claim partition ${ownership.partitionId}. Another processor has already claimed it.`,\n );\n continue;\n }\n\n logger.warning(\n `Error occurred while claiming ownership for partition: ${ownership.partitionId}`,\n err.message,\n );\n logErrorStackTrace(err);\n\n throw err;\n }\n }\n return partitionOwnershipArray;\n }\n\n /**\n * Lists all the checkpoints in a data store for a given namespace, eventhub and consumer group.\n *\n * @param fullyQualifiedNamespace - The fully qualified Event Hubs namespace. This is likely to be similar to\n * <yournamespace>.servicebus.windows.net.\n * @param eventHubName - The event hub name.\n * @param consumerGroup - The consumer group name.\n * @param options - A set of options that can be specified to influence the behavior of this method.\n * - `abortSignal`: A signal used to request operation cancellation.\n * - `tracingOptions`: Options for configuring tracing.\n */\n async listCheckpoints(\n fullyQualifiedNamespace: string,\n eventHubName: string,\n consumerGroup: string,\n options: OperationOptions = {},\n ): Promise<Checkpoint[]> {\n const { abortSignal, tracingOptions } = options;\n const blobPrefix = BlobCheckpointStore.getBlobPrefix({\n type: \"checkpoint\",\n fullyQualifiedNamespace,\n eventHubName,\n consumerGroup,\n });\n\n const blobs = this._containerClient.listBlobsFlat({\n abortSignal,\n includeMetadata: true,\n prefix: blobPrefix,\n tracingOptions,\n });\n\n const checkpoints: Checkpoint[] = [];\n\n for await (const blob of blobs) {\n const blobPath = blob.name.split(\"/\");\n const blobName = blobPath[blobPath.length - 1];\n\n const checkpointMetadata = (blob.metadata as CheckpointMetadata) ?? {};\n\n const offset = checkpointMetadata.offset;\n if (offset == null) {\n throw new Error(`Missing metadata property 'offset' on blob '${blob.name}'`);\n }\n const sequenceNumber = parseIntOrThrow(\n blob.name,\n \"sequencenumber\",\n checkpointMetadata.sequencenumber,\n );\n\n checkpoints.push({\n consumerGroup,\n eventHubName,\n fullyQualifiedNamespace,\n partitionId: blobName,\n offset,\n sequenceNumber,\n });\n }\n\n return checkpoints;\n }\n\n /**\n * Updates the checkpoint in the data store for a partition.\n *\n * @param checkpoint - The checkpoint.\n * @param options - A set of options that can be specified to influence the behavior of this method.\n * - `abortSignal`: A signal used to request operation cancellation.\n * - `tracingOptions`: Options for configuring tracing.\n * @returns The new etag on successful update.\n */\n async updateCheckpoint(checkpoint: Checkpoint, options: OperationOptions = {}): Promise<void> {\n throwTypeErrorIfParameterMissing(\n \"updateCheckpoint\",\n \"sequenceNumber\",\n checkpoint.sequenceNumber,\n );\n throwTypeErrorIfParameterMissing(\"updateCheckpoint\", \"offset\", checkpoint.offset);\n\n const blobName = BlobCheckpointStore.getBlobPrefix({ type: \"checkpoint\", ...checkpoint });\n try {\n const metadataResponse = await this._setBlobMetadata(\n blobName,\n {\n sequencenumber: checkpoint.sequenceNumber.toString(),\n offset: checkpoint.offset.toString(),\n },\n undefined,\n options,\n );\n\n logger.verbose(\n `Updated checkpoint successfully for partition: ${checkpoint.partitionId}`,\n `LastModifiedTime: ${metadataResponse.lastModified!.toISOString()}, ETag: ${\n metadataResponse.etag\n }`,\n );\n return;\n } catch (err: any) {\n logger.warning(\n `Error occurred while updating the checkpoint for partition: ${checkpoint.partitionId}.`,\n err.message,\n );\n logErrorStackTrace(err);\n\n if (err?.name === \"AbortError\") throw err;\n\n throw err;\n }\n }\n\n private static getBlobPrefix(params: {\n type: \"ownership\" | \"checkpoint\";\n fullyQualifiedNamespace: string;\n eventHubName: string;\n consumerGroup: string;\n partitionId?: string;\n }): string {\n // none of these are case-sensitive in eventhubs so we need to make sure we don't accidentally allow\n // the user to create a case-sensitive blob for their state!\n const consumerGroupName = params.consumerGroup.toLowerCase();\n const eventHubName = params.eventHubName.toLowerCase();\n const fullyQualifiedNamespace = params.fullyQualifiedNamespace.toLowerCase();\n\n if (params.partitionId) {\n return `${fullyQualifiedNamespace}/${eventHubName}/${consumerGroupName}/${params.type}/${params.partitionId}`;\n } else {\n return `${fullyQualifiedNamespace}/${eventHubName}/${consumerGroupName}/${params.type}/`;\n }\n }\n\n private async _setBlobMetadata(\n blobName: string,\n metadata: OwnershipMetadata | CheckpointMetadata,\n etag: string | undefined,\n options: OperationOptions = {},\n ): Promise<BlobSetMetadataResponse> {\n const { abortSignal, tracingOptions } = options;\n const blockBlobClient = this._containerClient.getBlobClient(blobName).getBlockBlobClient();\n\n // When we have an etag, we know the blob existed.\n // If we encounter an error we should fail.\n if (etag) {\n return blockBlobClient.setMetadata(metadata as Metadata, {\n abortSignal,\n conditions: {\n ifMatch: etag,\n },\n tracingOptions,\n });\n } else {\n try {\n // Attempt to set metadata, and fallback to upload if the blob doesn't already exist.\n // This avoids poor performance in storage accounts with soft-delete or blob versioning enabled.\n // https://github.com/Azure/azure-sdk-for-js/issues/10132\n return await blockBlobClient.setMetadata(metadata as Metadata, {\n abortSignal,\n tracingOptions,\n });\n } catch (err: any) {\n // Check if the error is `BlobNotFound` and fallback to `upload` if it is.\n if (err?.name !== \"RestError\") {\n throw err;\n }\n const errorDetails = (err as RestError).details as { [field: string]: string } | undefined;\n const errorCode = errorDetails?.errorCode;\n if (!errorCode || errorCode !== \"BlobNotFound\") {\n throw err;\n }\n\n return blockBlobClient.upload(\"\", 0, {\n abortSignal,\n metadata: metadata as Metadata,\n tracingOptions,\n });\n }\n }\n }\n}\n\ntype OwnershipMetadata = {\n [k in \"ownerid\"]: string | undefined;\n};\n\ntype CheckpointMetadata = {\n [k in \"sequencenumber\" | \"offset\"]: string | undefined;\n};\n\n/**\n * @internal\n */\nexport function parseIntOrThrow(\n blobName: string,\n fieldName: string,\n numStr: string | undefined,\n): number {\n if (numStr == null) {\n throw new Error(`Missing metadata property '${fieldName}' on blob '${blobName}'`);\n }\n\n const num = parseInt(numStr, 10);\n\n if (isNaN(num)) {\n throw new Error(\n `Failed to parse metadata property '${fieldName}' on blob '${blobName}' as a number`,\n );\n }\n\n return num;\n}\n"]}
|
|
1
|
+
{"version":3,"file":"blobCheckpointStore.js","sourceRoot":"","sources":["../../src/blobCheckpointStore.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;;AASlC,OAAO,EAAE,MAAM,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAEtD,OAAO,EAAE,gCAAgC,EAAE,MAAM,iBAAiB,CAAC;AAEnE;;GAEG;AACH,MAAM,OAAO,mBAAmB;IAG9B;;;OAGG;IACH,YAAY,eAAoC;QAC9C,IAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC;IAC1C,CAAC;IACD;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,aAAa,CACjB,uBAA+B,EAC/B,YAAoB,EACpB,aAAqB,EACrB,UAA4B,EAAE;;;QAE9B,MAAM,uBAAuB,GAAyB,EAAE,CAAC;QACzD,MAAM,EAAE,WAAW,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAEhD,MAAM,UAAU,GAAG,mBAAmB,CAAC,aAAa,CAAC;YACnD,IAAI,EAAE,WAAW;YACjB,uBAAuB;YACvB,YAAY;YACZ,aAAa,EAAE,aAAa;SAC7B,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC;gBAChD,WAAW;gBACX,eAAe,EAAE,IAAI;gBACrB,MAAM,EAAE,UAAU;gBAClB,cAAc;aACf,CAAC,CAAC;;gBAEH,KAAyB,eAAA,UAAA,cAAA,KAAK,CAAA,WAAA,yEAAE,CAAC;oBAAR,qBAAK;oBAAL,WAAK;oBAAnB,MAAM,IAAI,KAAA,CAAA;oBACnB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBACtC,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBAE/C,MAAM,iBAAiB,GAAG,MAAC,IAAI,CAAC,QAA8B,mCAAI,EAAE,CAAC;oBAErE,IAAI,iBAAiB,CAAC,OAAO,IAAI,IAAI,EAAE,CAAC;wBACtC,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;oBACvE,CAAC;oBAED,MAAM,kBAAkB,GAAuB;wBAC7C,uBAAuB;wBACvB,YAAY;wBACZ,aAAa,EAAE,aAAa;wBAC5B,OAAO,EAAE,iBAAiB,CAAC,OAAO;wBAClC,WAAW,EAAE,QAAQ;wBACrB,oBAAoB,EAClB,IAAI,CAAC,UAAU,CAAC,YAAY,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,OAAO,EAAE;wBACxE,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI;qBAC3B,CAAC;oBACF,uBAAuB,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;gBACnD,CAAC;;;;;;;;;YACD,OAAO,uBAAuB,CAAC;QACjC,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,MAAM,CAAC,OAAO,CAAC,iDAAiD,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;YAC/E,kBAAkB,CAAC,GAAG,CAAC,CAAC;YAExB,IAAI,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,YAAY;gBAAE,MAAM,GAAG,CAAC;YAE1C,MAAM,IAAI,KAAK,CAAC,sDAAsD,GAAG,EAAE,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,cAAc,CAClB,kBAAwC,EACxC,UAA4B,EAAE;QAE9B,MAAM,uBAAuB,GAAyB,EAAE,CAAC;QACzD,KAAK,MAAM,SAAS,IAAI,kBAAkB,EAAE,CAAC;YAC3C,MAAM,QAAQ,GAAG,mBAAmB,CAAC,aAAa,iBAAG,IAAI,EAAE,WAAW,IAAK,SAAS,EAAG,CAAC;YACxF,IAAI,CAAC;gBACH,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,gBAAgB,CACrD,QAAQ,EACR;oBACE,OAAO,EAAE,SAAS,CAAC,OAAO;iBAC3B,EACD,SAAS,CAAC,IAAI,EACd,OAAO,CACR,CAAC;gBAEF,IAAI,mBAAmB,CAAC,YAAY,EAAE,CAAC;oBACrC,SAAS,CAAC,oBAAoB,GAAG,mBAAmB,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;gBAC9E,CAAC;gBAED,SAAS,CAAC,IAAI,GAAG,mBAAmB,CAAC,IAAI,CAAC;gBAC1C,uBAAuB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACxC,MAAM,CAAC,IAAI,CACT,IAAI,SAAS,CAAC,OAAO,mDAAmD,SAAS,CAAC,WAAW,EAAE,EAC/F,qBAAqB,SAAS,CAAC,oBAAoB,WAAW,SAAS,CAAC,IAAI,EAAE,CAC/E,CAAC;YACJ,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAClB,MAAM,SAAS,GAAG,GAAgB,CAAC;gBAEnC,IAAI,SAAS,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;oBACjC,wEAAwE;oBACxE,6EAA6E;oBAC7E,6BAA6B;oBAC7B,MAAM,CAAC,OAAO,CACZ,IAAI,SAAS,CAAC,OAAO,6BAA6B,SAAS,CAAC,WAAW,6CAA6C,CACrH,CAAC;oBACF,SAAS;gBACX,CAAC;gBAED,MAAM,CAAC,OAAO,CACZ,0DAA0D,SAAS,CAAC,WAAW,EAAE,EACjF,GAAG,CAAC,OAAO,CACZ,CAAC;gBACF,kBAAkB,CAAC,GAAG,CAAC,CAAC;gBAExB,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;QACD,OAAO,uBAAuB,CAAC;IACjC,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,eAAe,CACnB,uBAA+B,EAC/B,YAAoB,EACpB,aAAqB,EACrB,UAA4B,EAAE;;;QAE9B,MAAM,EAAE,WAAW,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAChD,MAAM,UAAU,GAAG,mBAAmB,CAAC,aAAa,CAAC;YACnD,IAAI,EAAE,YAAY;YAClB,uBAAuB;YACvB,YAAY;YACZ,aAAa;SACd,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC;YAChD,WAAW;YACX,eAAe,EAAE,IAAI;YACrB,MAAM,EAAE,UAAU;YAClB,cAAc;SACf,CAAC,CAAC;QAEH,MAAM,WAAW,GAAiB,EAAE,CAAC;;YAErC,KAAyB,eAAA,UAAA,cAAA,KAAK,CAAA,WAAA,yEAAE,CAAC;gBAAR,qBAAK;gBAAL,WAAK;gBAAnB,MAAM,IAAI,KAAA,CAAA;gBACnB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACtC,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAE/C,MAAM,kBAAkB,GAAG,MAAC,IAAI,CAAC,QAA+B,mCAAI,EAAE,CAAC;gBAEvE,MAAM,MAAM,GAAG,kBAAkB,CAAC,MAAM,CAAC;gBACzC,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;oBACnB,MAAM,IAAI,KAAK,CAAC,+CAA+C,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;gBAC/E,CAAC;gBACD,MAAM,cAAc,GAAG,eAAe,CACpC,IAAI,CAAC,IAAI,EACT,gBAAgB,EAChB,kBAAkB,CAAC,cAAc,CAClC,CAAC;gBAEF,WAAW,CAAC,IAAI,CAAC;oBACf,aAAa;oBACb,YAAY;oBACZ,uBAAuB;oBACvB,WAAW,EAAE,QAAQ;oBACrB,MAAM;oBACN,cAAc;iBACf,CAAC,CAAC;YACL,CAAC;;;;;;;;;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,gBAAgB,CAAC,UAAsB,EAAE,UAA4B,EAAE;QAC3E,gCAAgC,CAC9B,kBAAkB,EAClB,gBAAgB,EAChB,UAAU,CAAC,cAAc,CAC1B,CAAC;QACF,gCAAgC,CAAC,kBAAkB,EAAE,QAAQ,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QAElF,MAAM,QAAQ,GAAG,mBAAmB,CAAC,aAAa,iBAAG,IAAI,EAAE,YAAY,IAAK,UAAU,EAAG,CAAC;QAC1F,IAAI,CAAC;YACH,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAClD,QAAQ,EACR;gBACE,cAAc,EAAE,UAAU,CAAC,cAAc,CAAC,QAAQ,EAAE;gBACpD,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,QAAQ,EAAE;aACrC,EACD,SAAS,EACT,OAAO,CACR,CAAC;YAEF,MAAM,CAAC,OAAO,CACZ,kDAAkD,UAAU,CAAC,WAAW,EAAE,EAC1E,qBAAqB,gBAAgB,CAAC,YAAa,CAAC,WAAW,EAAE,WAC/D,gBAAgB,CAAC,IACnB,EAAE,CACH,CAAC;YACF,OAAO;QACT,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,MAAM,CAAC,OAAO,CACZ,+DAA+D,UAAU,CAAC,WAAW,GAAG,EACxF,GAAG,CAAC,OAAO,CACZ,CAAC;YACF,kBAAkB,CAAC,GAAG,CAAC,CAAC;YAExB,IAAI,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,YAAY;gBAAE,MAAM,GAAG,CAAC;YAE1C,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,MAAM,CAAC,aAAa,CAAC,MAM5B;QACC,oGAAoG;QACpG,4DAA4D;QAC5D,MAAM,iBAAiB,GAAG,MAAM,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC;QAC7D,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC;QACvD,MAAM,uBAAuB,GAAG,MAAM,CAAC,uBAAuB,CAAC,WAAW,EAAE,CAAC;QAE7E,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;YACvB,OAAO,GAAG,uBAAuB,IAAI,YAAY,IAAI,iBAAiB,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QAChH,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,uBAAuB,IAAI,YAAY,IAAI,iBAAiB,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC;QAC3F,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAC5B,QAAgB,EAChB,QAAgD,EAChD,IAAwB,EACxB,UAA4B,EAAE;QAE9B,MAAM,EAAE,WAAW,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAChD,MAAM,eAAe,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,kBAAkB,EAAE,CAAC;QAE3F,kDAAkD;QAClD,2CAA2C;QAC3C,IAAI,IAAI,EAAE,CAAC;YACT,OAAO,eAAe,CAAC,WAAW,CAAC,QAAoB,EAAE;gBACvD,WAAW;gBACX,UAAU,EAAE;oBACV,OAAO,EAAE,IAAI;iBACd;gBACD,cAAc;aACf,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,IAAI,CAAC;gBACH,qFAAqF;gBACrF,gGAAgG;gBAChG,yDAAyD;gBACzD,OAAO,MAAM,eAAe,CAAC,WAAW,CAAC,QAAoB,EAAE;oBAC7D,WAAW;oBACX,cAAc;iBACf,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAClB,0EAA0E;gBAC1E,IAAI,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,WAAW,EAAE,CAAC;oBAC9B,MAAM,GAAG,CAAC;gBACZ,CAAC;gBACD,MAAM,YAAY,GAAI,GAAiB,CAAC,OAAkD,CAAC;gBAC3F,MAAM,SAAS,GAAG,YAAY,aAAZ,YAAY,uBAAZ,YAAY,CAAE,SAAS,CAAC;gBAC1C,IAAI,CAAC,SAAS,IAAI,SAAS,KAAK,cAAc,EAAE,CAAC;oBAC/C,MAAM,GAAG,CAAC;gBACZ,CAAC;gBAED,OAAO,eAAe,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,EAAE;oBACnC,WAAW;oBACX,QAAQ,EAAE,QAAoB;oBAC9B,cAAc;iBACf,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;CACF;AAUD;;GAEG;AACH,MAAM,UAAU,eAAe,CAC7B,QAAgB,EAChB,SAAiB,EACjB,MAA0B;IAE1B,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,8BAA8B,SAAS,cAAc,QAAQ,GAAG,CAAC,CAAC;IACpF,CAAC;IAED,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAEjC,IAAI,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CACb,sCAAsC,SAAS,cAAc,QAAQ,eAAe,CACrF,CAAC;IACJ,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport type {\n CheckpointStore,\n PartitionOwnership,\n Checkpoint,\n OperationOptions,\n} from \"@azure/event-hubs\";\nimport type { Metadata, RestError, BlobSetMetadataResponse } from \"@azure/storage-blob\";\nimport { logger, logErrorStackTrace } from \"./log.js\";\nimport type { ContainerClientLike } from \"./storageBlobInterfaces.js\";\nimport { throwTypeErrorIfParameterMissing } from \"./util/error.js\";\n\n/**\n * An implementation of CheckpointStore that uses Azure Blob Storage to persist checkpoint data.\n */\nexport class BlobCheckpointStore implements CheckpointStore {\n private _containerClient: ContainerClientLike;\n\n /**\n * Constructs a new instance of {@link BlobCheckpointStore}\n * @param containerClient - An instance of a storage blob ContainerClient.\n */\n constructor(containerClient: ContainerClientLike) {\n this._containerClient = containerClient;\n }\n /**\n * Get the list of all existing partition ownership from the underlying data store. May return empty\n * results if there are is no existing ownership information.\n * Partition Ownership contains the information on which `EventHubConsumerClient` subscribe call is currently processing the partition.\n *\n * @param fullyQualifiedNamespace - The fully qualified Event Hubs namespace. This is likely to be similar to\n * <yournamespace>.servicebus.windows.net.\n * @param eventHubName - The event hub name.\n * @param consumerGroup - The consumer group name.\n * @param options - A set of options that can be specified to influence the behavior of this method.\n * - `abortSignal`: A signal used to request operation cancellation.\n * - `tracingOptions`: Options for configuring tracing.\n * @returns Partition ownership details of all the partitions that have had an owner.\n */\n async listOwnership(\n fullyQualifiedNamespace: string,\n eventHubName: string,\n consumerGroup: string,\n options: OperationOptions = {},\n ): Promise<PartitionOwnership[]> {\n const partitionOwnershipArray: PartitionOwnership[] = [];\n const { abortSignal, tracingOptions } = options;\n\n const blobPrefix = BlobCheckpointStore.getBlobPrefix({\n type: \"ownership\",\n fullyQualifiedNamespace,\n eventHubName,\n consumerGroup: consumerGroup,\n });\n\n try {\n const blobs = this._containerClient.listBlobsFlat({\n abortSignal,\n includeMetadata: true,\n prefix: blobPrefix,\n tracingOptions,\n });\n\n for await (const blob of blobs) {\n const blobPath = blob.name.split(\"/\");\n const blobName = blobPath[blobPath.length - 1];\n\n const ownershipMetadata = (blob.metadata as OwnershipMetadata) ?? {};\n\n if (ownershipMetadata.ownerid == null) {\n throw new Error(`Missing ownerid in metadata for blob ${blob.name}`);\n }\n\n const partitionOwnership: PartitionOwnership = {\n fullyQualifiedNamespace,\n eventHubName,\n consumerGroup: consumerGroup,\n ownerId: ownershipMetadata.ownerid,\n partitionId: blobName,\n lastModifiedTimeInMs:\n blob.properties.lastModified && blob.properties.lastModified.getTime(),\n etag: blob.properties.etag,\n };\n partitionOwnershipArray.push(partitionOwnership);\n }\n return partitionOwnershipArray;\n } catch (err: any) {\n logger.warning(`Error occurred while fetching the list of blobs`, err.message);\n logErrorStackTrace(err);\n\n if (err?.name === \"AbortError\") throw err;\n\n throw new Error(`Error occurred while fetching the list of blobs. \\n${err}`);\n }\n }\n\n /**\n * Claim ownership of a list of partitions. This will return the list of partitions that were\n * successfully claimed.\n *\n * @param partitionOwnership - The list of partition ownership this instance is claiming to own.\n * @param options - A set of options that can be specified to influence the behavior of this method.\n * - `abortSignal`: A signal used to request operation cancellation.\n * - `tracingOptions`: Options for configuring tracing.\n * @returns A list partitions this instance successfully claimed ownership.\n */\n async claimOwnership(\n partitionOwnership: PartitionOwnership[],\n options: OperationOptions = {},\n ): Promise<PartitionOwnership[]> {\n const partitionOwnershipArray: PartitionOwnership[] = [];\n for (const ownership of partitionOwnership) {\n const blobName = BlobCheckpointStore.getBlobPrefix({ type: \"ownership\", ...ownership });\n try {\n const updatedBlobResponse = await this._setBlobMetadata(\n blobName,\n {\n ownerid: ownership.ownerId,\n },\n ownership.etag,\n options,\n );\n\n if (updatedBlobResponse.lastModified) {\n ownership.lastModifiedTimeInMs = updatedBlobResponse.lastModified.getTime();\n }\n\n ownership.etag = updatedBlobResponse.etag;\n partitionOwnershipArray.push(ownership);\n logger.info(\n `[${ownership.ownerId}] Claimed ownership successfully for partition: ${ownership.partitionId}`,\n `LastModifiedTime: ${ownership.lastModifiedTimeInMs}, ETag: ${ownership.etag}`,\n );\n } catch (err: any) {\n const restError = err as RestError;\n\n if (restError.statusCode === 412) {\n // etag failures (precondition not met) aren't fatal errors. They happen\n // as multiple consumers attempt to claim the same partition (first one wins)\n // and losers get this error.\n logger.verbose(\n `[${ownership.ownerId}] Did not claim partition ${ownership.partitionId}. Another processor has already claimed it.`,\n );\n continue;\n }\n\n logger.warning(\n `Error occurred while claiming ownership for partition: ${ownership.partitionId}`,\n err.message,\n );\n logErrorStackTrace(err);\n\n throw err;\n }\n }\n return partitionOwnershipArray;\n }\n\n /**\n * Lists all the checkpoints in a data store for a given namespace, eventhub and consumer group.\n *\n * @param fullyQualifiedNamespace - The fully qualified Event Hubs namespace. This is likely to be similar to\n * <yournamespace>.servicebus.windows.net.\n * @param eventHubName - The event hub name.\n * @param consumerGroup - The consumer group name.\n * @param options - A set of options that can be specified to influence the behavior of this method.\n * - `abortSignal`: A signal used to request operation cancellation.\n * - `tracingOptions`: Options for configuring tracing.\n */\n async listCheckpoints(\n fullyQualifiedNamespace: string,\n eventHubName: string,\n consumerGroup: string,\n options: OperationOptions = {},\n ): Promise<Checkpoint[]> {\n const { abortSignal, tracingOptions } = options;\n const blobPrefix = BlobCheckpointStore.getBlobPrefix({\n type: \"checkpoint\",\n fullyQualifiedNamespace,\n eventHubName,\n consumerGroup,\n });\n\n const blobs = this._containerClient.listBlobsFlat({\n abortSignal,\n includeMetadata: true,\n prefix: blobPrefix,\n tracingOptions,\n });\n\n const checkpoints: Checkpoint[] = [];\n\n for await (const blob of blobs) {\n const blobPath = blob.name.split(\"/\");\n const blobName = blobPath[blobPath.length - 1];\n\n const checkpointMetadata = (blob.metadata as CheckpointMetadata) ?? {};\n\n const offset = checkpointMetadata.offset;\n if (offset == null) {\n throw new Error(`Missing metadata property 'offset' on blob '${blob.name}'`);\n }\n const sequenceNumber = parseIntOrThrow(\n blob.name,\n \"sequencenumber\",\n checkpointMetadata.sequencenumber,\n );\n\n checkpoints.push({\n consumerGroup,\n eventHubName,\n fullyQualifiedNamespace,\n partitionId: blobName,\n offset,\n sequenceNumber,\n });\n }\n\n return checkpoints;\n }\n\n /**\n * Updates the checkpoint in the data store for a partition.\n *\n * @param checkpoint - The checkpoint.\n * @param options - A set of options that can be specified to influence the behavior of this method.\n * - `abortSignal`: A signal used to request operation cancellation.\n * - `tracingOptions`: Options for configuring tracing.\n * @returns The new etag on successful update.\n */\n async updateCheckpoint(checkpoint: Checkpoint, options: OperationOptions = {}): Promise<void> {\n throwTypeErrorIfParameterMissing(\n \"updateCheckpoint\",\n \"sequenceNumber\",\n checkpoint.sequenceNumber,\n );\n throwTypeErrorIfParameterMissing(\"updateCheckpoint\", \"offset\", checkpoint.offset);\n\n const blobName = BlobCheckpointStore.getBlobPrefix({ type: \"checkpoint\", ...checkpoint });\n try {\n const metadataResponse = await this._setBlobMetadata(\n blobName,\n {\n sequencenumber: checkpoint.sequenceNumber.toString(),\n offset: checkpoint.offset.toString(),\n },\n undefined,\n options,\n );\n\n logger.verbose(\n `Updated checkpoint successfully for partition: ${checkpoint.partitionId}`,\n `LastModifiedTime: ${metadataResponse.lastModified!.toISOString()}, ETag: ${\n metadataResponse.etag\n }`,\n );\n return;\n } catch (err: any) {\n logger.warning(\n `Error occurred while updating the checkpoint for partition: ${checkpoint.partitionId}.`,\n err.message,\n );\n logErrorStackTrace(err);\n\n if (err?.name === \"AbortError\") throw err;\n\n throw err;\n }\n }\n\n private static getBlobPrefix(params: {\n type: \"ownership\" | \"checkpoint\";\n fullyQualifiedNamespace: string;\n eventHubName: string;\n consumerGroup: string;\n partitionId?: string;\n }): string {\n // none of these are case-sensitive in eventhubs so we need to make sure we don't accidentally allow\n // the user to create a case-sensitive blob for their state!\n const consumerGroupName = params.consumerGroup.toLowerCase();\n const eventHubName = params.eventHubName.toLowerCase();\n const fullyQualifiedNamespace = params.fullyQualifiedNamespace.toLowerCase();\n\n if (params.partitionId) {\n return `${fullyQualifiedNamespace}/${eventHubName}/${consumerGroupName}/${params.type}/${params.partitionId}`;\n } else {\n return `${fullyQualifiedNamespace}/${eventHubName}/${consumerGroupName}/${params.type}/`;\n }\n }\n\n private async _setBlobMetadata(\n blobName: string,\n metadata: OwnershipMetadata | CheckpointMetadata,\n etag: string | undefined,\n options: OperationOptions = {},\n ): Promise<BlobSetMetadataResponse> {\n const { abortSignal, tracingOptions } = options;\n const blockBlobClient = this._containerClient.getBlobClient(blobName).getBlockBlobClient();\n\n // When we have an etag, we know the blob existed.\n // If we encounter an error we should fail.\n if (etag) {\n return blockBlobClient.setMetadata(metadata as Metadata, {\n abortSignal,\n conditions: {\n ifMatch: etag,\n },\n tracingOptions,\n });\n } else {\n try {\n // Attempt to set metadata, and fallback to upload if the blob doesn't already exist.\n // This avoids poor performance in storage accounts with soft-delete or blob versioning enabled.\n // https://github.com/Azure/azure-sdk-for-js/issues/10132\n return await blockBlobClient.setMetadata(metadata as Metadata, {\n abortSignal,\n tracingOptions,\n });\n } catch (err: any) {\n // Check if the error is `BlobNotFound` and fallback to `upload` if it is.\n if (err?.name !== \"RestError\") {\n throw err;\n }\n const errorDetails = (err as RestError).details as { [field: string]: string } | undefined;\n const errorCode = errorDetails?.errorCode;\n if (!errorCode || errorCode !== \"BlobNotFound\") {\n throw err;\n }\n\n return blockBlobClient.upload(\"\", 0, {\n abortSignal,\n metadata: metadata as Metadata,\n tracingOptions,\n });\n }\n }\n }\n}\n\ntype OwnershipMetadata = {\n [k in \"ownerid\"]: string | undefined;\n};\n\ntype CheckpointMetadata = {\n [k in \"sequencenumber\" | \"offset\"]: string | undefined;\n};\n\n/**\n * @internal\n */\nexport function parseIntOrThrow(\n blobName: string,\n fieldName: string,\n numStr: string | undefined,\n): number {\n if (numStr == null) {\n throw new Error(`Missing metadata property '${fieldName}' on blob '${blobName}'`);\n }\n\n const num = parseInt(numStr, 10);\n\n if (isNaN(num)) {\n throw new Error(\n `Failed to parse metadata property '${fieldName}' on blob '${blobName}' as a number`,\n );\n }\n\n return num;\n}\n"]}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { Metadata, BlobItem, ContainerListBlobFlatSegmentResponse, ContainerListBlobsOptions, BlockBlobUploadOptions, BlockBlobUploadResponse, HttpRequestBody, BlobSetMetadataOptions, ContainerSetMetadataResponse } from "@azure/storage-blob";
|
|
2
|
-
import { PagedAsyncIterableIterator } from "@azure/core-paging";
|
|
1
|
+
import type { Metadata, BlobItem, ContainerListBlobFlatSegmentResponse, ContainerListBlobsOptions, BlockBlobUploadOptions, BlockBlobUploadResponse, HttpRequestBody, BlobSetMetadataOptions, ContainerSetMetadataResponse } from "@azure/storage-blob";
|
|
2
|
+
import type { PagedAsyncIterableIterator } from "@azure/core-paging";
|
|
3
3
|
/**
|
|
4
4
|
* An interface compatible with an instance of {@link BlobClient}.
|
|
5
5
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"storageBlobInterfaces.d.ts","sourceRoot":"","sources":["../../src/storageBlobInterfaces.ts"],"names":[],"mappings":"AAGA,OAAO,
|
|
1
|
+
{"version":3,"file":"storageBlobInterfaces.d.ts","sourceRoot":"","sources":["../../src/storageBlobInterfaces.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,QAAQ,EACR,QAAQ,EACR,oCAAoC,EACpC,yBAAyB,EACzB,sBAAsB,EACtB,uBAAuB,EACvB,eAAe,EACf,sBAAsB,EACtB,4BAA4B,EAC7B,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AAErE;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;OAEG;IACH,kBAAkB,IAAI,mBAAmB,CAAC;CAC3C;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC;;OAEG;IACH,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,cAAc,CAAC;IAChD;;;OAGG;IACH,aAAa,CACX,OAAO,CAAC,EAAE,yBAAyB,GAClC,0BAA0B,CAAC,QAAQ,EAAE,oCAAoC,CAAC,CAAC;CAC/E;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC;;OAEG;IACH,MAAM,CACJ,IAAI,EAAE,eAAe,EACrB,aAAa,EAAE,MAAM,EACrB,OAAO,CAAC,EAAE,sBAAsB,GAC/B,OAAO,CAAC,uBAAuB,CAAC,CAAC;IACpC;;OAEG;IACH,WAAW,CACT,QAAQ,CAAC,EAAE,QAAQ,EACnB,OAAO,CAAC,EAAE,sBAAsB,GAC/B,OAAO,CAAC,4BAA4B,CAAC,CAAC;CAC1C"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"storageBlobInterfaces.js","sourceRoot":"","sources":["../../src/storageBlobInterfaces.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport {\n Metadata,\n BlobItem,\n ContainerListBlobFlatSegmentResponse,\n ContainerListBlobsOptions,\n BlockBlobUploadOptions,\n BlockBlobUploadResponse,\n HttpRequestBody,\n BlobSetMetadataOptions,\n ContainerSetMetadataResponse,\n} from \"@azure/storage-blob\";\nimport { PagedAsyncIterableIterator } from \"@azure/core-paging\";\n\n/**\n * An interface compatible with an instance of {@link BlobClient}.\n */\nexport interface BlobClientLike {\n /**\n * Creates a BlockBlobClient object.\n */\n getBlockBlobClient(): BlockBlobClientLike;\n}\n\n/**\n * An interface compatible with Storage Blob's ContainerClient class.\n */\nexport interface ContainerClientLike {\n /**\n * Creates a {@link BlobClient}\n */\n getBlobClient(blobName: string): BlobClientLike;\n /**\n * Returns an async iterable iterator to list all the blobs\n * under the specified account.\n */\n listBlobsFlat(\n options?: ContainerListBlobsOptions,\n ): PagedAsyncIterableIterator<BlobItem, ContainerListBlobFlatSegmentResponse>;\n}\n\n/**\n * An interface compatible with Storage Blob's BlockBlobClient class.\n */\nexport interface BlockBlobClientLike {\n /**\n * Creates a new block blob, or updated the content of an existing block blob.\n */\n upload(\n body: HttpRequestBody,\n contentLength: number,\n options?: BlockBlobUploadOptions,\n ): Promise<BlockBlobUploadResponse>;\n /**\n * Sets user-defined metadata for the specified blob as one or more name-value pairs.\n */\n setMetadata(\n metadata?: Metadata,\n options?: BlobSetMetadataOptions,\n ): Promise<ContainerSetMetadataResponse>;\n}\n"]}
|
|
1
|
+
{"version":3,"file":"storageBlobInterfaces.js","sourceRoot":"","sources":["../../src/storageBlobInterfaces.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport type {\n Metadata,\n BlobItem,\n ContainerListBlobFlatSegmentResponse,\n ContainerListBlobsOptions,\n BlockBlobUploadOptions,\n BlockBlobUploadResponse,\n HttpRequestBody,\n BlobSetMetadataOptions,\n ContainerSetMetadataResponse,\n} from \"@azure/storage-blob\";\nimport type { PagedAsyncIterableIterator } from \"@azure/core-paging\";\n\n/**\n * An interface compatible with an instance of {@link BlobClient}.\n */\nexport interface BlobClientLike {\n /**\n * Creates a BlockBlobClient object.\n */\n getBlockBlobClient(): BlockBlobClientLike;\n}\n\n/**\n * An interface compatible with Storage Blob's ContainerClient class.\n */\nexport interface ContainerClientLike {\n /**\n * Creates a {@link BlobClient}\n */\n getBlobClient(blobName: string): BlobClientLike;\n /**\n * Returns an async iterable iterator to list all the blobs\n * under the specified account.\n */\n listBlobsFlat(\n options?: ContainerListBlobsOptions,\n ): PagedAsyncIterableIterator<BlobItem, ContainerListBlobFlatSegmentResponse>;\n}\n\n/**\n * An interface compatible with Storage Blob's BlockBlobClient class.\n */\nexport interface BlockBlobClientLike {\n /**\n * Creates a new block blob, or updated the content of an existing block blob.\n */\n upload(\n body: HttpRequestBody,\n contentLength: number,\n options?: BlockBlobUploadOptions,\n ): Promise<BlockBlobUploadResponse>;\n /**\n * Sets user-defined metadata for the specified blob as one or more name-value pairs.\n */\n setMetadata(\n metadata?: Metadata,\n options?: BlobSetMetadataOptions,\n ): Promise<ContainerSetMetadataResponse>;\n}\n"]}
|
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
import { AzureLogger } from '@azure/logger';
|
|
2
|
-
import { BlobItem } from '@azure/storage-blob';
|
|
3
|
-
import { BlobSetMetadataOptions } from '@azure/storage-blob';
|
|
4
|
-
import { BlockBlobUploadOptions } from '@azure/storage-blob';
|
|
5
|
-
import { BlockBlobUploadResponse } from '@azure/storage-blob';
|
|
6
|
-
import { Checkpoint } from '@azure/event-hubs';
|
|
7
|
-
import { CheckpointStore } from '@azure/event-hubs';
|
|
8
|
-
import { ContainerListBlobFlatSegmentResponse } from '@azure/storage-blob';
|
|
9
|
-
import { ContainerListBlobsOptions } from '@azure/storage-blob';
|
|
10
|
-
import { ContainerSetMetadataResponse } from '@azure/storage-blob';
|
|
11
|
-
import { HttpRequestBody } from '@azure/storage-blob';
|
|
12
|
-
import { Metadata } from '@azure/storage-blob';
|
|
13
|
-
import { OperationOptions } from '@azure/event-hubs';
|
|
14
|
-
import { PagedAsyncIterableIterator } from '@azure/core-paging';
|
|
15
|
-
import { PartitionOwnership } from '@azure/event-hubs';
|
|
2
|
+
import type { BlobItem } from '@azure/storage-blob';
|
|
3
|
+
import type { BlobSetMetadataOptions } from '@azure/storage-blob';
|
|
4
|
+
import type { BlockBlobUploadOptions } from '@azure/storage-blob';
|
|
5
|
+
import type { BlockBlobUploadResponse } from '@azure/storage-blob';
|
|
6
|
+
import type { Checkpoint } from '@azure/event-hubs';
|
|
7
|
+
import type { CheckpointStore } from '@azure/event-hubs';
|
|
8
|
+
import type { ContainerListBlobFlatSegmentResponse } from '@azure/storage-blob';
|
|
9
|
+
import type { ContainerListBlobsOptions } from '@azure/storage-blob';
|
|
10
|
+
import type { ContainerSetMetadataResponse } from '@azure/storage-blob';
|
|
11
|
+
import type { HttpRequestBody } from '@azure/storage-blob';
|
|
12
|
+
import type { Metadata } from '@azure/storage-blob';
|
|
13
|
+
import type { OperationOptions } from '@azure/event-hubs';
|
|
14
|
+
import type { PagedAsyncIterableIterator } from '@azure/core-paging';
|
|
15
|
+
import type { PartitionOwnership } from '@azure/event-hubs';
|
|
16
16
|
|
|
17
17
|
/**
|
|
18
18
|
* An implementation of CheckpointStore that uses Azure Blob Storage to persist checkpoint data.
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@azure/eventhubs-checkpointstore-blob",
|
|
3
3
|
"sdk-type": "client",
|
|
4
|
-
"version": "1.1.0-alpha.
|
|
4
|
+
"version": "1.1.0-alpha.20241121.1",
|
|
5
5
|
"description": "An Azure Storage Blob solution to store checkpoints when using Event Hubs.",
|
|
6
6
|
"author": "Microsoft Corporation",
|
|
7
7
|
"license": "MIT",
|
|
@@ -29,20 +29,19 @@
|
|
|
29
29
|
"LICENSE"
|
|
30
30
|
],
|
|
31
31
|
"scripts": {
|
|
32
|
-
"audit": "node ../../../common/scripts/rush-audit.js && rimraf node_modules package-lock.json && npm i --package-lock-only 2>&1 && npm audit",
|
|
33
32
|
"build": "npm run clean && dev-tool run build-package && dev-tool run extract-api",
|
|
34
33
|
"build:samples": "echo Obsolete.",
|
|
35
34
|
"build:test": "npm run build:test:node",
|
|
36
35
|
"build:test:browser": "dev-tool run build-package && dev-tool run build-test",
|
|
37
36
|
"build:test:node": "dev-tool run build-package && dev-tool run build-test",
|
|
38
37
|
"check-format": "dev-tool run vendored prettier --list-different --config ../../../.prettierrc.json --ignore-path ../../../.prettierignore \"src/**/*.{ts,cts,mts}\" \"test/**/*.{ts,cts,mts}\" \"*.{js,cjs,mjs,json}\"",
|
|
39
|
-
"clean": "rimraf --glob dist dist-* temp *.tgz *.log .tshy *.xml",
|
|
38
|
+
"clean": "dev-tool run vendored rimraf --glob dist dist-* temp *.tgz *.log .tshy *.xml",
|
|
40
39
|
"execute:samples": "dev-tool samples run samples-dev",
|
|
41
40
|
"extract-api": "dev-tool run build-package && dev-tool run extract-api",
|
|
42
41
|
"format": "dev-tool run vendored prettier --write --config ../../../.prettierrc.json --ignore-path ../../../.prettierignore \"src/**/*.{ts,cts,mts}\" \"test/**/*.{ts,cts,mts}\" \"samples-dev/**/*.{ts,cts,mts}\" \"*.{js,cjs,mjs,json}\"",
|
|
43
42
|
"integration-test": "npm run integration-test:node && npm run integration-test:browser",
|
|
44
|
-
"integration-test:browser": "dev-tool run build-package && dev-tool run build-test && cross-env TEST_MODE=live dev-tool run test:vitest --browser --no-test-proxy",
|
|
45
|
-
"integration-test:node": "cross-env TEST_MODE=live npm run vitest:node",
|
|
43
|
+
"integration-test:browser": "dev-tool run build-package && dev-tool run build-test && dev-tool run vendored cross-env TEST_MODE=live dev-tool run test:vitest --browser --no-test-proxy",
|
|
44
|
+
"integration-test:node": "dev-tool run vendored cross-env TEST_MODE=live npm run vitest:node",
|
|
46
45
|
"lint": "eslint package.json api-extractor.json src test README.md",
|
|
47
46
|
"lint:fix": "eslint package.json api-extractor.json src test README.md --fix --fix-type [problem,suggestion]",
|
|
48
47
|
"pack": "npm pack 2>&1",
|
|
@@ -86,7 +85,6 @@
|
|
|
86
85
|
"@azure/dev-tool": ">=1.0.0-alpha <1.0.0-alphb",
|
|
87
86
|
"@azure/eslint-plugin-azure-sdk": ">=3.0.0-alpha <3.0.0-alphb",
|
|
88
87
|
"@azure/identity": "^4.4.1",
|
|
89
|
-
"@microsoft/api-extractor": "^7.31.1",
|
|
90
88
|
"@rollup/plugin-inject": "^5.0.5",
|
|
91
89
|
"@types/chai-as-promised": "^7.1.8",
|
|
92
90
|
"@types/debug": "^4.1.12",
|
|
@@ -95,13 +93,11 @@
|
|
|
95
93
|
"@vitest/coverage-istanbul": "^2.0.5",
|
|
96
94
|
"buffer": "^6.0.3",
|
|
97
95
|
"chai-as-promised": "^8.0.0",
|
|
98
|
-
"cross-env": "^7.0.3",
|
|
99
96
|
"debug": "^4.3.6",
|
|
100
97
|
"dotenv": "^16.4.5",
|
|
101
98
|
"eslint": "^9.9.0",
|
|
102
99
|
"playwright": "^1.45.3",
|
|
103
100
|
"process": "^0.11.10",
|
|
104
|
-
"rimraf": "^6.0.1",
|
|
105
101
|
"stream": "^0.0.3",
|
|
106
102
|
"tsx": "^4.16.2",
|
|
107
103
|
"typescript": "~5.6.2",
|