@azure/storage-blob-changefeed 12.0.0-alpha.20230213.2 → 12.0.0-alpha.20230216.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +30 -149
- package/dist/index.js.map +1 -1
- package/dist-esm/storage-blob-changefeed/src/ChangeFeed.js +5 -28
- package/dist-esm/storage-blob-changefeed/src/ChangeFeed.js.map +1 -1
- package/dist-esm/storage-blob-changefeed/src/ChangeFeedFactory.js +3 -15
- package/dist-esm/storage-blob-changefeed/src/ChangeFeedFactory.js.map +1 -1
- package/dist-esm/storage-blob-changefeed/src/LazyLoadingBlobStream.js +7 -29
- package/dist-esm/storage-blob-changefeed/src/LazyLoadingBlobStream.js.map +1 -1
- package/dist-esm/storage-blob-changefeed/src/Segment.js +3 -15
- package/dist-esm/storage-blob-changefeed/src/Segment.js.map +1 -1
- package/dist-esm/storage-blob-changefeed/src/SegmentFactory.js +3 -15
- package/dist-esm/storage-blob-changefeed/src/SegmentFactory.js.map +1 -1
- package/dist-esm/storage-blob-changefeed/src/Shard.js +3 -15
- package/dist-esm/storage-blob-changefeed/src/Shard.js.map +1 -1
- package/dist-esm/storage-blob-changefeed/src/ShardFactory.js +4 -16
- package/dist-esm/storage-blob-changefeed/src/ShardFactory.js.map +1 -1
- package/dist-esm/storage-blob-changefeed/src/utils/tracing.js +5 -3
- package/dist-esm/storage-blob-changefeed/src/utils/tracing.js.map +1 -1
- package/dist-esm/storage-blob-changefeed/src/utils/utils.common.js +7 -30
- package/dist-esm/storage-blob-changefeed/src/utils/utils.common.js.map +1 -1
- package/package.json +2 -2
- package/types/3.1/storage-blob-changefeed/src/utils/tracing.d.ts +1 -6
- package/types/latest/storage-blob-changefeed/src/ChangeFeed.d.ts.map +1 -1
- package/types/latest/storage-blob-changefeed/src/ChangeFeedFactory.d.ts.map +1 -1
- package/types/latest/storage-blob-changefeed/src/LazyLoadingBlobStream.d.ts.map +1 -1
- package/types/latest/storage-blob-changefeed/src/Segment.d.ts.map +1 -1
- package/types/latest/storage-blob-changefeed/src/SegmentFactory.d.ts.map +1 -1
- package/types/latest/storage-blob-changefeed/src/Shard.d.ts.map +1 -1
- package/types/latest/storage-blob-changefeed/src/ShardFactory.d.ts.map +1 -1
- package/types/latest/storage-blob-changefeed/src/utils/tracing.d.ts +1 -6
- package/types/latest/storage-blob-changefeed/src/utils/tracing.d.ts.map +1 -1
- package/types/latest/storage-blob-changefeed/src/utils/utils.common.d.ts.map +1 -1
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
// Copyright (c) Microsoft Corporation.
|
|
2
2
|
// Licensed under the MIT license.
|
|
3
3
|
import { getSegmentsInYear, minDate, getHost } from "./utils/utils.common";
|
|
4
|
-
import {
|
|
5
|
-
import { SpanStatusCode } from "@azure/core-tracing";
|
|
4
|
+
import { tracingClient } from "./utils/tracing";
|
|
6
5
|
export class ChangeFeed {
|
|
7
6
|
constructor(containerClient, segmentFactory, years, segments, currentSegment, lastConsumable, startTime, endTime) {
|
|
8
7
|
this.containerClient = containerClient;
|
|
@@ -18,8 +17,7 @@ export class ChangeFeed {
|
|
|
18
17
|
}
|
|
19
18
|
}
|
|
20
19
|
async advanceSegmentIfNecessary(options = {}) {
|
|
21
|
-
|
|
22
|
-
try {
|
|
20
|
+
return tracingClient.withSpan("ChangeFeed-advanceSegmentIfNecessary", options, async (updatedOptions) => {
|
|
23
21
|
if (!this.currentSegment) {
|
|
24
22
|
throw new Error("Empty Change Feed shouldn't call this function.");
|
|
25
23
|
}
|
|
@@ -51,17 +49,7 @@ export class ChangeFeed {
|
|
|
51
49
|
this.currentSegment = undefined;
|
|
52
50
|
}
|
|
53
51
|
}
|
|
54
|
-
}
|
|
55
|
-
catch (e) {
|
|
56
|
-
span.setStatus({
|
|
57
|
-
code: SpanStatusCode.ERROR,
|
|
58
|
-
message: e.message,
|
|
59
|
-
});
|
|
60
|
-
throw e;
|
|
61
|
-
}
|
|
62
|
-
finally {
|
|
63
|
-
span.end();
|
|
64
|
-
}
|
|
52
|
+
});
|
|
65
53
|
}
|
|
66
54
|
hasNext() {
|
|
67
55
|
// Empty ChangeFeed, using currentSegment as the indicator.
|
|
@@ -74,8 +62,7 @@ export class ChangeFeed {
|
|
|
74
62
|
return this.currentSegment.dateTime < this.end;
|
|
75
63
|
}
|
|
76
64
|
async getChange(options = {}) {
|
|
77
|
-
|
|
78
|
-
try {
|
|
65
|
+
return tracingClient.withSpan("ChangeFeed-getChange", options, async (updatedOptions) => {
|
|
79
66
|
let event = undefined;
|
|
80
67
|
while (event === undefined && this.hasNext()) {
|
|
81
68
|
event = await this.currentSegment.getChange({
|
|
@@ -88,17 +75,7 @@ export class ChangeFeed {
|
|
|
88
75
|
});
|
|
89
76
|
}
|
|
90
77
|
return event;
|
|
91
|
-
}
|
|
92
|
-
catch (e) {
|
|
93
|
-
span.setStatus({
|
|
94
|
-
code: SpanStatusCode.ERROR,
|
|
95
|
-
message: e.message,
|
|
96
|
-
});
|
|
97
|
-
throw e;
|
|
98
|
-
}
|
|
99
|
-
finally {
|
|
100
|
-
span.end();
|
|
101
|
-
}
|
|
78
|
+
});
|
|
102
79
|
}
|
|
103
80
|
getCursor() {
|
|
104
81
|
var _a;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ChangeFeed.js","sourceRoot":"","sources":["../../../src/ChangeFeed.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAOlC,OAAO,EAAE,iBAAiB,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAE3E,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"ChangeFeed.js","sourceRoot":"","sources":["../../../src/ChangeFeed.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAOlC,OAAO,EAAE,iBAAiB,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAE3E,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAahD,MAAM,OAAO,UAAU;IAkCrB,YACE,eAAiC,EACjC,cAA+B,EAC/B,KAAgB,EAChB,QAAmB,EACnB,cAAwB,EACxB,cAAqB,EACrB,SAAgB,EAChB,OAAc;QAEd,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,KAAK,GAAG,KAAK,IAAI,EAAE,CAAC;QACzB,IAAI,CAAC,QAAQ,GAAG,QAAQ,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,IAAI,CAAC,cAAc,EAAE;YACvB,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;SACvD;IACH,CAAC;IAEO,KAAK,CAAC,yBAAyB,CAAC,UAAsC,EAAE;QAC9E,OAAO,aAAa,CAAC,QAAQ,CAC3B,sCAAsC,EACtC,OAAO,EACP,KAAK,EAAE,cAAc,EAAE,EAAE;YACvB,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE;gBACxB,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;aACpE;YAED,wEAAwE;YACxE,IAAI,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE;gBACjC,OAAO;aACR;YAED,iDAAiD;YACjD,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE;gBAC5B,IAAI,CAAC,cAAc,GAAG,MAAM,IAAI,CAAC,cAAe,CAAC,MAAM,CACrD,IAAI,CAAC,eAAgB,EACrB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAG,EACtB,SAAS,EACT;oBACE,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;iBAC9C,CACF,CAAC;aACH;YACD,kCAAkC;iBAC7B,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE;gBAC5D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;gBAChC,IAAI,CAAC,QAAQ,GAAG,MAAM,iBAAiB,CACrC,IAAI,CAAC,eAAgB,EACrB,IAAK,EACL,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,GAAG,EACR;oBACE,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;iBAC9C,CACF,CAAC;gBAEF,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE;oBAC5B,IAAI,CAAC,cAAc,GAAG,MAAM,IAAI,CAAC,cAAe,CAAC,MAAM,CACrD,IAAI,CAAC,eAAgB,EACrB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAG,EACtB,SAAS,EACT;wBACE,WAAW,EAAE,OAAO,CAAC,WAAW;wBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;qBAC9C,CACF,CAAC;iBACH;qBAAM;oBACL,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC;iBACjC;aACF;QACH,CAAC,CACF,CAAC;IACJ,CAAC;IAEM,OAAO;QACZ,2DAA2D;QAC3D,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE;YACxB,OAAO,KAAK,CAAC;SACd;QAED,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE;YAC3F,OAAO,KAAK,CAAC;SACd;QAED,OAAO,IAAI,CAAC,cAAc,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAI,CAAC;IAClD,CAAC;IAEM,KAAK,CAAC,SAAS,CACpB,UAAsC,EAAE;QAExC,OAAO,aAAa,CAAC,QAAQ,CAAC,sBAAsB,EAAE,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,EAAE;YACtF,IAAI,KAAK,GAAoC,SAAS,CAAC;YACvD,OAAO,KAAK,KAAK,SAAS,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE;gBAC5C,KAAK,GAAG,MAAM,IAAI,CAAC,cAAe,CAAC,SAAS,CAAC;oBAC3C,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;iBAC9C,CAAC,CAAC;gBACH,MAAM,IAAI,CAAC,yBAAyB,CAAC;oBACnC,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;iBAC9C,CAAC,CAAC;aACJ;YACD,OAAO,KAAK,CAAC;QACf,CAAC,CAAC,CAAC;IACL,CAAC;IAEM,SAAS;;QACd,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE;YACxB,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;SACpE;QAED,OAAO;YACL,aAAa,EAAE,CAAC;YAChB,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,eAAgB,CAAC,GAAG,CAAE;YAC5C,OAAO,EAAE,MAAA,IAAI,CAAC,OAAO,0CAAE,MAAM,EAAE;YAC/B,oBAAoB,EAAE,IAAI,CAAC,cAAe,CAAC,SAAS,EAAE;SACvD,CAAC;IACJ,CAAC;CACF","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\nimport { ContainerClient, CommonOptions } from \"@azure/storage-blob\";\nimport { Segment } from \"./Segment\";\nimport { SegmentFactory } from \"./SegmentFactory\";\nimport { BlobChangeFeedEvent } from \"./models/BlobChangeFeedEvent\";\nimport { ChangeFeedCursor } from \"./models/ChangeFeedCursor\";\nimport { getSegmentsInYear, minDate, getHost } from \"./utils/utils.common\";\nimport { AbortSignalLike } from \"@azure/abort-controller\";\nimport { tracingClient } from \"./utils/tracing\";\n\n/**\n * Options to configure {@link ChangeFeed.getChange} operation.\n */\nexport interface ChangeFeedGetChangeOptions extends CommonOptions {\n /**\n * An implementation of the `AbortSignalLike` interface to signal the request to cancel the operation.\n * For example, use the @azure/abort-controller to create an `AbortSignal`.\n */\n abortSignal?: AbortSignalLike;\n}\n\nexport class ChangeFeed {\n /**\n * BlobContainerClient for making List Blob requests and creating Segments.\n */\n private readonly containerClient?: ContainerClient;\n\n private readonly segmentFactory?: SegmentFactory;\n\n private readonly years: number[];\n\n private segments: string[];\n\n private currentSegment?: Segment;\n\n private lastConsumable?: Date;\n\n private startTime?: Date;\n\n private endTime?: Date;\n\n private end?: Date;\n\n constructor();\n constructor(\n containerClient: ContainerClient,\n segmentFactory: SegmentFactory,\n years: number[],\n segments: string[],\n currentSegment: Segment,\n lastConsumable: Date,\n startTime?: Date,\n endTime?: Date\n );\n\n constructor(\n containerClient?: ContainerClient,\n segmentFactory?: SegmentFactory,\n years?: number[],\n segments?: string[],\n currentSegment?: Segment,\n lastConsumable?: Date,\n startTime?: Date,\n endTime?: Date\n ) {\n this.containerClient = containerClient;\n this.segmentFactory = segmentFactory;\n this.years = years || [];\n this.segments = segments || [];\n this.currentSegment = currentSegment;\n this.lastConsumable = lastConsumable;\n this.startTime = startTime;\n this.endTime = endTime;\n if (this.lastConsumable) {\n this.end = minDate(this.lastConsumable, this.endTime);\n }\n }\n\n private async advanceSegmentIfNecessary(options: ChangeFeedGetChangeOptions = {}): Promise<void> {\n return tracingClient.withSpan(\n \"ChangeFeed-advanceSegmentIfNecessary\",\n options,\n async (updatedOptions) => {\n if (!this.currentSegment) {\n throw new Error(\"Empty Change Feed shouldn't call this function.\");\n }\n\n // If the current segment has more Events, we don't need to do anything.\n if (this.currentSegment.hasNext()) {\n return;\n }\n\n // If the current segment is completed, remove it\n if (this.segments.length > 0) {\n this.currentSegment = await this.segmentFactory!.create(\n this.containerClient!,\n this.segments.shift()!,\n undefined,\n {\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n }\n );\n }\n // If segments is empty, refill it\n else if (this.segments.length === 0 && this.years.length > 0) {\n const year = this.years.shift();\n this.segments = await getSegmentsInYear(\n this.containerClient!,\n year!,\n this.startTime,\n this.end,\n {\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n }\n );\n\n if (this.segments.length > 0) {\n this.currentSegment = await this.segmentFactory!.create(\n this.containerClient!,\n this.segments.shift()!,\n undefined,\n {\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n }\n );\n } else {\n this.currentSegment = undefined;\n }\n }\n }\n );\n }\n\n public hasNext(): boolean {\n // Empty ChangeFeed, using currentSegment as the indicator.\n if (!this.currentSegment) {\n return false;\n }\n\n if (this.segments.length === 0 && this.years.length === 0 && !this.currentSegment.hasNext()) {\n return false;\n }\n\n return this.currentSegment.dateTime < this.end!;\n }\n\n public async getChange(\n options: ChangeFeedGetChangeOptions = {}\n ): Promise<BlobChangeFeedEvent | undefined> {\n return tracingClient.withSpan(\"ChangeFeed-getChange\", options, async (updatedOptions) => {\n let event: BlobChangeFeedEvent | undefined = undefined;\n while (event === undefined && this.hasNext()) {\n event = await this.currentSegment!.getChange({\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n });\n await this.advanceSegmentIfNecessary({\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n });\n }\n return event;\n });\n }\n\n public getCursor(): ChangeFeedCursor {\n if (!this.currentSegment) {\n throw new Error(\"Empty Change Feed shouldn't call this function.\");\n }\n\n return {\n CursorVersion: 1,\n UrlHost: getHost(this.containerClient!.url)!,\n EndTime: this.endTime?.toJSON(),\n CurrentSegmentCursor: this.currentSegment!.getCursor(),\n };\n }\n}\n"]}
|
|
@@ -8,8 +8,7 @@ import { SegmentFactory } from "./SegmentFactory";
|
|
|
8
8
|
import { ShardFactory } from "./ShardFactory";
|
|
9
9
|
import { ChunkFactory } from "./ChunkFactory";
|
|
10
10
|
import { AvroReaderFactory } from "./AvroReaderFactory";
|
|
11
|
-
import {
|
|
12
|
-
import { SpanStatusCode } from "@azure/core-tracing";
|
|
11
|
+
import { tracingClient } from "./utils/tracing";
|
|
13
12
|
import { LazyLoadingBlobStreamFactory } from "./LazyLoadingBlobStreamFactory";
|
|
14
13
|
export class ChangeFeedFactory {
|
|
15
14
|
constructor(segmentFactoryOrMaxTransferSize) {
|
|
@@ -38,8 +37,7 @@ export class ChangeFeedFactory {
|
|
|
38
37
|
}
|
|
39
38
|
}
|
|
40
39
|
async create(blobServiceClient, continuationToken, options = {}) {
|
|
41
|
-
|
|
42
|
-
try {
|
|
40
|
+
return tracingClient.withSpan("ChangeFeedFactory-create", options, async (updatedOptions) => {
|
|
43
41
|
const containerClient = blobServiceClient.getContainerClient(CHANGE_FEED_CONTAINER_NAME);
|
|
44
42
|
let cursor = undefined;
|
|
45
43
|
// Create cursor.
|
|
@@ -113,17 +111,7 @@ export class ChangeFeedFactory {
|
|
|
113
111
|
tracingOptions: updatedOptions.tracingOptions,
|
|
114
112
|
});
|
|
115
113
|
return new ChangeFeed(containerClient, this.segmentFactory, years, segments, currentSegment, lastConsumable, options.start, options.end);
|
|
116
|
-
}
|
|
117
|
-
catch (e) {
|
|
118
|
-
span.setStatus({
|
|
119
|
-
code: SpanStatusCode.ERROR,
|
|
120
|
-
message: e.message,
|
|
121
|
-
});
|
|
122
|
-
throw e;
|
|
123
|
-
}
|
|
124
|
-
finally {
|
|
125
|
-
span.end();
|
|
126
|
-
}
|
|
114
|
+
});
|
|
127
115
|
}
|
|
128
116
|
}
|
|
129
117
|
//# sourceMappingURL=ChangeFeedFactory.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ChangeFeedFactory.js","sourceRoot":"","sources":["../../../src/ChangeFeedFactory.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAGlC,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE1C,OAAO,EAAE,0BAA0B,EAAE,6BAA6B,EAAE,MAAM,mBAAmB,CAAC;AAC9F,OAAO,EACL,iBAAiB,EACjB,kBAAkB,EAClB,aAAa,EACb,iBAAiB,EACjB,OAAO,EACP,OAAO,EACP,wBAAwB,GACzB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAGxD,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,4BAA4B,EAAE,MAAM,gCAAgC,CAAC;AAO9E,MAAM,OAAO,iBAAiB;IAM5B,YAAY,+BAAyD;QACnE,IAAI,cAA0C,CAAC;QAC/C,IAAI,+BAA+B,EAAE;YACnC,IAAI,MAAM,CAAC,QAAQ,CAAC,+BAA+B,CAAC,EAAE;gBACpD,IAAI,CAAC,eAAe,GAAG,+BAAyC,CAAC;aAClE;iBAAM,IAAI,+BAA+B,YAAY,cAAc,EAAE;gBACpE,cAAc,GAAG,+BAAiD,CAAC;aACpE;SACF;QAED,IAAI,cAAc,EAAE;YAClB,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;SACtC;aAAM;YACL,IAAI,CAAC,cAAc,GAAG,IAAI,cAAc,CACtC,IAAI,YAAY,CACd,IAAI,YAAY,CACd,IAAI,iBAAiB,EAAE,EACvB,IAAI,4BAA4B,EAAE,EAClC,IAAI,CAAC,eAAe,CACrB,CACF,CACF,CAAC;SACH;IACH,CAAC;IAEO,MAAM,CAAC,cAAc,CAAC,eAAgC,EAAE,MAAwB;QACtF,IAAI,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,MAAM,CAAC,OAAO,EAAE;YACnD,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;SACvE;QACD,IAAI,MAAM,CAAC,aAAa,KAAK,CAAC,EAAE;YAC9B,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;SAChD;IACH,CAAC;IAEM,KAAK,CAAC,MAAM,CACjB,iBAAoC,EACpC,iBAA0B,EAC1B,UAA4C,EAAE;QAE9C,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,UAAU,CAAC,0BAA0B,EAAE,OAAO,CAAC,CAAC;QAEjF,IAAI;YACF,MAAM,eAAe,GAAG,iBAAiB,CAAC,kBAAkB,CAAC,0BAA0B,CAAC,CAAC;YACzF,IAAI,MAAM,GAAiC,SAAS,CAAC;YACrD,iBAAiB;YACjB,IAAI,iBAAiB,EAAE;gBACrB,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;gBACvC,iBAAiB,CAAC,cAAc,CAAC,eAAe,EAAE,MAAO,CAAC,CAAC;gBAC3D,OAAO,CAAC,KAAK,GAAG,wBAAwB,CAAC,MAAO,CAAC,oBAAoB,CAAC,WAAY,CAAC,CAAC;gBACpF,OAAO,CAAC,GAAG,GAAG,IAAI,IAAI,CAAC,MAAO,CAAC,OAAQ,CAAC,CAAC;aAC1C;YACD,2DAA2D;iBACtD;gBACH,OAAO,CAAC,KAAK,GAAG,kBAAkB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBAClD,OAAO,CAAC,GAAG,GAAG,iBAAiB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;aAC9C;YAED,0DAA0D;YAC1D,MAAM,yBAAyB,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC;gBAC7D,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;aAC9C,CAAC,CAAC;YACH,IAAI,CAAC,yBAAyB,EAAE;gBAC9B,MAAM,IAAI,KAAK,CACb,iFAAiF,CAClF,CAAC;aACH;YAED,IAAI,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,GAAG,EAAE;gBAChE,OAAO,IAAI,UAAU,EAAE,CAAC;aACzB;YAED,uBAAuB;YACvB,MAAM,UAAU,GAAG,eAAe,CAAC,aAAa,CAAC,6BAA6B,CAAC,CAAC;YAChF,IAAI,eAAe,CAAC;YACpB,IAAI;gBACF,eAAe,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,SAAS,EAAE,SAAS,EAAE;oBAChE,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;iBAC9C,CAAC,CAAC;aACJ;YAAC,OAAO,GAAQ,EAAE;gBACjB,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG,EAAE;oBAC1B,OAAO,IAAI,UAAU,EAAE,CAAC;iBACzB;qBAAM;oBACL,MAAM,GAAG,CAAC;iBACX;aACF;YACD,MAAM,cAAc,GAAG,IAAI,IAAI,CAC5B,IAAI,CAAC,KAAK,CAAC,MAAM,YAAY,CAAC,eAAe,CAAC,CAAkB,CAAC,cAAc,CACjF,CAAC;YAEF,iBAAiB;YACjB,MAAM,KAAK,GAAa,MAAM,aAAa,CAAC,eAAe,EAAE;gBAC3D,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;aAC9C,CAAC,CAAC;YAEH,kDAAkD;YAClD,IAAI,OAAO,CAAC,KAAK,EAAE;gBACjB,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;gBACjD,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,SAAS,EAAE;oBAC/C,KAAK,CAAC,KAAK,EAAE,CAAC;iBACf;aACF;YACD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;gBACtB,OAAO,IAAI,UAAU,EAAE,CAAC;aACzB;YAED,IAAI,QAAQ,GAAa,EAAE,CAAC;YAC5B,OAAO,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;gBAClD,QAAQ,GAAG,MAAM,iBAAiB,CAChC,eAAe,EACf,KAAK,CAAC,KAAK,EAAG,EACd,OAAO,CAAC,KAAK,EACb,OAAO,CAAC,cAAc,EAAE,OAAO,CAAC,GAAG,CAAC,EACpC;oBACE,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;iBAC9C,CACF,CAAC;aACH;YACD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE;gBACzB,OAAO,IAAI,UAAU,EAAE,CAAC;aACzB;YACD,MAAM,cAAc,GAAY,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,CAC9D,eAAe,EACf,QAAQ,CAAC,KAAK,EAAG,EACjB,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,oBAAoB,EAC5B;gBACE,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;aAC9C,CACF,CAAC;YAEF,OAAO,IAAI,UAAU,CACnB,eAAe,EACf,IAAI,CAAC,cAAc,EACnB,KAAK,EACL,QAAQ,EACR,cAAc,EACd,cAAc,EACd,OAAO,CAAC,KAAK,EACb,OAAO,CAAC,GAAG,CACZ,CAAC;SACH;QAAC,OAAO,CAAM,EAAE;YACf,IAAI,CAAC,SAAS,CAAC;gBACb,IAAI,EAAE,cAAc,CAAC,KAAK;gBAC1B,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC,CAAC;YACH,MAAM,CAAC,CAAC;SACT;gBAAS;YACR,IAAI,CAAC,GAAG,EAAE,CAAC;SACZ;IACH,CAAC;CACF","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\nimport { BlobServiceClient, ContainerClient } from \"@azure/storage-blob\";\nimport { ChangeFeed } from \"./ChangeFeed\";\nimport { ChangeFeedCursor } from \"./models/ChangeFeedCursor\";\nimport { CHANGE_FEED_CONTAINER_NAME, CHANGE_FEED_META_SEGMENT_PATH } from \"./utils/constants\";\nimport {\n ceilToNearestHour,\n floorToNearestHour,\n getYearsPaths,\n getSegmentsInYear,\n minDate,\n getHost,\n parseDateFromSegmentPath,\n} from \"./utils/utils.common\";\nimport { bodyToString } from \"./utils/utils.node\";\nimport { SegmentFactory } from \"./SegmentFactory\";\nimport { ShardFactory } from \"./ShardFactory\";\nimport { ChunkFactory } from \"./ChunkFactory\";\nimport { AvroReaderFactory } from \"./AvroReaderFactory\";\nimport { Segment } from \"./Segment\";\nimport { BlobChangeFeedListChangesOptions } from \"./models/models\";\nimport { createSpan } from \"./utils/tracing\";\nimport { SpanStatusCode } from \"@azure/core-tracing\";\nimport { LazyLoadingBlobStreamFactory } from \"./LazyLoadingBlobStreamFactory\";\n\ninterface MetaSegments {\n version?: number;\n lastConsumable: string;\n}\n\nexport class ChangeFeedFactory {\n private readonly segmentFactory: SegmentFactory;\n private readonly maxTransferSize?: number;\n\n constructor(maxTransferSize?: number);\n constructor(segmentFactory: SegmentFactory);\n constructor(segmentFactoryOrMaxTransferSize?: SegmentFactory | number) {\n let segmentFactory: SegmentFactory | undefined;\n if (segmentFactoryOrMaxTransferSize) {\n if (Number.isFinite(segmentFactoryOrMaxTransferSize)) {\n this.maxTransferSize = segmentFactoryOrMaxTransferSize as number;\n } else if (segmentFactoryOrMaxTransferSize instanceof SegmentFactory) {\n segmentFactory = segmentFactoryOrMaxTransferSize as SegmentFactory;\n }\n }\n\n if (segmentFactory) {\n this.segmentFactory = segmentFactory;\n } else {\n this.segmentFactory = new SegmentFactory(\n new ShardFactory(\n new ChunkFactory(\n new AvroReaderFactory(),\n new LazyLoadingBlobStreamFactory(),\n this.maxTransferSize\n )\n )\n );\n }\n }\n\n private static validateCursor(containerClient: ContainerClient, cursor: ChangeFeedCursor): void {\n if (getHost(containerClient.url) !== cursor.UrlHost) {\n throw new Error(\"Cursor URL host does not match container URL host.\");\n }\n if (cursor.CursorVersion !== 1) {\n throw new Error(\"Unsupported cursor version.\");\n }\n }\n\n public async create(\n blobServiceClient: BlobServiceClient,\n continuationToken?: string,\n options: BlobChangeFeedListChangesOptions = {}\n ): Promise<ChangeFeed> {\n const { span, updatedOptions } = createSpan(\"ChangeFeedFactory-create\", options);\n\n try {\n const containerClient = blobServiceClient.getContainerClient(CHANGE_FEED_CONTAINER_NAME);\n let cursor: ChangeFeedCursor | undefined = undefined;\n // Create cursor.\n if (continuationToken) {\n cursor = JSON.parse(continuationToken);\n ChangeFeedFactory.validateCursor(containerClient, cursor!);\n options.start = parseDateFromSegmentPath(cursor!.CurrentSegmentCursor.SegmentPath!);\n options.end = new Date(cursor!.EndTime!);\n }\n // Round start and end time if we are not using the cursor.\n else {\n options.start = floorToNearestHour(options.start);\n options.end = ceilToNearestHour(options.end);\n }\n\n // Check if Change Feed has been enabled for this account.\n const changeFeedContainerExists = await containerClient.exists({\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n });\n if (!changeFeedContainerExists) {\n throw new Error(\n \"Change Feed hasn't been enabled on this account, or is currently being enabled.\"\n );\n }\n\n if (options.start && options.end && options.start >= options.end) {\n return new ChangeFeed();\n }\n\n // Get last consumable.\n const blobClient = containerClient.getBlobClient(CHANGE_FEED_META_SEGMENT_PATH);\n let blobDownloadRes;\n try {\n blobDownloadRes = await blobClient.download(undefined, undefined, {\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n });\n } catch (err: any) {\n if (err.statusCode === 404) {\n return new ChangeFeed();\n } else {\n throw err;\n }\n }\n const lastConsumable = new Date(\n (JSON.parse(await bodyToString(blobDownloadRes)) as MetaSegments).lastConsumable\n );\n\n // Get year paths\n const years: number[] = await getYearsPaths(containerClient, {\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n });\n\n // Dequeue any years that occur before start time.\n if (options.start) {\n const startYear = options.start.getUTCFullYear();\n while (years.length > 0 && years[0] < startYear) {\n years.shift();\n }\n }\n if (years.length === 0) {\n return new ChangeFeed();\n }\n\n let segments: string[] = [];\n while (segments.length === 0 && years.length !== 0) {\n segments = await getSegmentsInYear(\n containerClient,\n years.shift()!,\n options.start,\n minDate(lastConsumable, options.end),\n {\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n }\n );\n }\n if (segments.length === 0) {\n return new ChangeFeed();\n }\n const currentSegment: Segment = await this.segmentFactory.create(\n containerClient,\n segments.shift()!,\n cursor?.CurrentSegmentCursor,\n {\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n }\n );\n\n return new ChangeFeed(\n containerClient,\n this.segmentFactory,\n years,\n segments,\n currentSegment,\n lastConsumable,\n options.start,\n options.end\n );\n } catch (e: any) {\n span.setStatus({\n code: SpanStatusCode.ERROR,\n message: e.message,\n });\n throw e;\n } finally {\n span.end();\n }\n }\n}\n"]}
|
|
1
|
+
{"version":3,"file":"ChangeFeedFactory.js","sourceRoot":"","sources":["../../../src/ChangeFeedFactory.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAGlC,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE1C,OAAO,EAAE,0BAA0B,EAAE,6BAA6B,EAAE,MAAM,mBAAmB,CAAC;AAC9F,OAAO,EACL,iBAAiB,EACjB,kBAAkB,EAClB,aAAa,EACb,iBAAiB,EACjB,OAAO,EACP,OAAO,EACP,wBAAwB,GACzB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAGxD,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,4BAA4B,EAAE,MAAM,gCAAgC,CAAC;AAO9E,MAAM,OAAO,iBAAiB;IAM5B,YAAY,+BAAyD;QACnE,IAAI,cAA0C,CAAC;QAC/C,IAAI,+BAA+B,EAAE;YACnC,IAAI,MAAM,CAAC,QAAQ,CAAC,+BAA+B,CAAC,EAAE;gBACpD,IAAI,CAAC,eAAe,GAAG,+BAAyC,CAAC;aAClE;iBAAM,IAAI,+BAA+B,YAAY,cAAc,EAAE;gBACpE,cAAc,GAAG,+BAAiD,CAAC;aACpE;SACF;QAED,IAAI,cAAc,EAAE;YAClB,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;SACtC;aAAM;YACL,IAAI,CAAC,cAAc,GAAG,IAAI,cAAc,CACtC,IAAI,YAAY,CACd,IAAI,YAAY,CACd,IAAI,iBAAiB,EAAE,EACvB,IAAI,4BAA4B,EAAE,EAClC,IAAI,CAAC,eAAe,CACrB,CACF,CACF,CAAC;SACH;IACH,CAAC;IAEO,MAAM,CAAC,cAAc,CAAC,eAAgC,EAAE,MAAwB;QACtF,IAAI,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,MAAM,CAAC,OAAO,EAAE;YACnD,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;SACvE;QACD,IAAI,MAAM,CAAC,aAAa,KAAK,CAAC,EAAE;YAC9B,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;SAChD;IACH,CAAC;IAEM,KAAK,CAAC,MAAM,CACjB,iBAAoC,EACpC,iBAA0B,EAC1B,UAA4C,EAAE;QAE9C,OAAO,aAAa,CAAC,QAAQ,CAAC,0BAA0B,EAAE,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,EAAE;YAC1F,MAAM,eAAe,GAAG,iBAAiB,CAAC,kBAAkB,CAAC,0BAA0B,CAAC,CAAC;YACzF,IAAI,MAAM,GAAiC,SAAS,CAAC;YACrD,iBAAiB;YACjB,IAAI,iBAAiB,EAAE;gBACrB,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;gBACvC,iBAAiB,CAAC,cAAc,CAAC,eAAe,EAAE,MAAO,CAAC,CAAC;gBAC3D,OAAO,CAAC,KAAK,GAAG,wBAAwB,CAAC,MAAO,CAAC,oBAAoB,CAAC,WAAY,CAAC,CAAC;gBACpF,OAAO,CAAC,GAAG,GAAG,IAAI,IAAI,CAAC,MAAO,CAAC,OAAQ,CAAC,CAAC;aAC1C;YACD,2DAA2D;iBACtD;gBACH,OAAO,CAAC,KAAK,GAAG,kBAAkB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBAClD,OAAO,CAAC,GAAG,GAAG,iBAAiB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;aAC9C;YAED,0DAA0D;YAC1D,MAAM,yBAAyB,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC;gBAC7D,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;aAC9C,CAAC,CAAC;YACH,IAAI,CAAC,yBAAyB,EAAE;gBAC9B,MAAM,IAAI,KAAK,CACb,iFAAiF,CAClF,CAAC;aACH;YAED,IAAI,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,GAAG,EAAE;gBAChE,OAAO,IAAI,UAAU,EAAE,CAAC;aACzB;YAED,uBAAuB;YACvB,MAAM,UAAU,GAAG,eAAe,CAAC,aAAa,CAAC,6BAA6B,CAAC,CAAC;YAChF,IAAI,eAAe,CAAC;YACpB,IAAI;gBACF,eAAe,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,SAAS,EAAE,SAAS,EAAE;oBAChE,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;iBAC9C,CAAC,CAAC;aACJ;YAAC,OAAO,GAAQ,EAAE;gBACjB,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG,EAAE;oBAC1B,OAAO,IAAI,UAAU,EAAE,CAAC;iBACzB;qBAAM;oBACL,MAAM,GAAG,CAAC;iBACX;aACF;YACD,MAAM,cAAc,GAAG,IAAI,IAAI,CAC5B,IAAI,CAAC,KAAK,CAAC,MAAM,YAAY,CAAC,eAAe,CAAC,CAAkB,CAAC,cAAc,CACjF,CAAC;YAEF,iBAAiB;YACjB,MAAM,KAAK,GAAa,MAAM,aAAa,CAAC,eAAe,EAAE;gBAC3D,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;aAC9C,CAAC,CAAC;YAEH,kDAAkD;YAClD,IAAI,OAAO,CAAC,KAAK,EAAE;gBACjB,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;gBACjD,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,SAAS,EAAE;oBAC/C,KAAK,CAAC,KAAK,EAAE,CAAC;iBACf;aACF;YACD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;gBACtB,OAAO,IAAI,UAAU,EAAE,CAAC;aACzB;YAED,IAAI,QAAQ,GAAa,EAAE,CAAC;YAC5B,OAAO,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;gBAClD,QAAQ,GAAG,MAAM,iBAAiB,CAChC,eAAe,EACf,KAAK,CAAC,KAAK,EAAG,EACd,OAAO,CAAC,KAAK,EACb,OAAO,CAAC,cAAc,EAAE,OAAO,CAAC,GAAG,CAAC,EACpC;oBACE,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;iBAC9C,CACF,CAAC;aACH;YACD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE;gBACzB,OAAO,IAAI,UAAU,EAAE,CAAC;aACzB;YACD,MAAM,cAAc,GAAY,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,CAC9D,eAAe,EACf,QAAQ,CAAC,KAAK,EAAG,EACjB,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,oBAAoB,EAC5B;gBACE,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;aAC9C,CACF,CAAC;YAEF,OAAO,IAAI,UAAU,CACnB,eAAe,EACf,IAAI,CAAC,cAAc,EACnB,KAAK,EACL,QAAQ,EACR,cAAc,EACd,cAAc,EACd,OAAO,CAAC,KAAK,EACb,OAAO,CAAC,GAAG,CACZ,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;CACF","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\nimport { BlobServiceClient, ContainerClient } from \"@azure/storage-blob\";\nimport { ChangeFeed } from \"./ChangeFeed\";\nimport { ChangeFeedCursor } from \"./models/ChangeFeedCursor\";\nimport { CHANGE_FEED_CONTAINER_NAME, CHANGE_FEED_META_SEGMENT_PATH } from \"./utils/constants\";\nimport {\n ceilToNearestHour,\n floorToNearestHour,\n getYearsPaths,\n getSegmentsInYear,\n minDate,\n getHost,\n parseDateFromSegmentPath,\n} from \"./utils/utils.common\";\nimport { bodyToString } from \"./utils/utils.node\";\nimport { SegmentFactory } from \"./SegmentFactory\";\nimport { ShardFactory } from \"./ShardFactory\";\nimport { ChunkFactory } from \"./ChunkFactory\";\nimport { AvroReaderFactory } from \"./AvroReaderFactory\";\nimport { Segment } from \"./Segment\";\nimport { BlobChangeFeedListChangesOptions } from \"./models/models\";\nimport { tracingClient } from \"./utils/tracing\";\nimport { LazyLoadingBlobStreamFactory } from \"./LazyLoadingBlobStreamFactory\";\n\ninterface MetaSegments {\n version?: number;\n lastConsumable: string;\n}\n\nexport class ChangeFeedFactory {\n private readonly segmentFactory: SegmentFactory;\n private readonly maxTransferSize?: number;\n\n constructor(maxTransferSize?: number);\n constructor(segmentFactory: SegmentFactory);\n constructor(segmentFactoryOrMaxTransferSize?: SegmentFactory | number) {\n let segmentFactory: SegmentFactory | undefined;\n if (segmentFactoryOrMaxTransferSize) {\n if (Number.isFinite(segmentFactoryOrMaxTransferSize)) {\n this.maxTransferSize = segmentFactoryOrMaxTransferSize as number;\n } else if (segmentFactoryOrMaxTransferSize instanceof SegmentFactory) {\n segmentFactory = segmentFactoryOrMaxTransferSize as SegmentFactory;\n }\n }\n\n if (segmentFactory) {\n this.segmentFactory = segmentFactory;\n } else {\n this.segmentFactory = new SegmentFactory(\n new ShardFactory(\n new ChunkFactory(\n new AvroReaderFactory(),\n new LazyLoadingBlobStreamFactory(),\n this.maxTransferSize\n )\n )\n );\n }\n }\n\n private static validateCursor(containerClient: ContainerClient, cursor: ChangeFeedCursor): void {\n if (getHost(containerClient.url) !== cursor.UrlHost) {\n throw new Error(\"Cursor URL host does not match container URL host.\");\n }\n if (cursor.CursorVersion !== 1) {\n throw new Error(\"Unsupported cursor version.\");\n }\n }\n\n public async create(\n blobServiceClient: BlobServiceClient,\n continuationToken?: string,\n options: BlobChangeFeedListChangesOptions = {}\n ): Promise<ChangeFeed> {\n return tracingClient.withSpan(\"ChangeFeedFactory-create\", options, async (updatedOptions) => {\n const containerClient = blobServiceClient.getContainerClient(CHANGE_FEED_CONTAINER_NAME);\n let cursor: ChangeFeedCursor | undefined = undefined;\n // Create cursor.\n if (continuationToken) {\n cursor = JSON.parse(continuationToken);\n ChangeFeedFactory.validateCursor(containerClient, cursor!);\n options.start = parseDateFromSegmentPath(cursor!.CurrentSegmentCursor.SegmentPath!);\n options.end = new Date(cursor!.EndTime!);\n }\n // Round start and end time if we are not using the cursor.\n else {\n options.start = floorToNearestHour(options.start);\n options.end = ceilToNearestHour(options.end);\n }\n\n // Check if Change Feed has been enabled for this account.\n const changeFeedContainerExists = await containerClient.exists({\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n });\n if (!changeFeedContainerExists) {\n throw new Error(\n \"Change Feed hasn't been enabled on this account, or is currently being enabled.\"\n );\n }\n\n if (options.start && options.end && options.start >= options.end) {\n return new ChangeFeed();\n }\n\n // Get last consumable.\n const blobClient = containerClient.getBlobClient(CHANGE_FEED_META_SEGMENT_PATH);\n let blobDownloadRes;\n try {\n blobDownloadRes = await blobClient.download(undefined, undefined, {\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n });\n } catch (err: any) {\n if (err.statusCode === 404) {\n return new ChangeFeed();\n } else {\n throw err;\n }\n }\n const lastConsumable = new Date(\n (JSON.parse(await bodyToString(blobDownloadRes)) as MetaSegments).lastConsumable\n );\n\n // Get year paths\n const years: number[] = await getYearsPaths(containerClient, {\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n });\n\n // Dequeue any years that occur before start time.\n if (options.start) {\n const startYear = options.start.getUTCFullYear();\n while (years.length > 0 && years[0] < startYear) {\n years.shift();\n }\n }\n if (years.length === 0) {\n return new ChangeFeed();\n }\n\n let segments: string[] = [];\n while (segments.length === 0 && years.length !== 0) {\n segments = await getSegmentsInYear(\n containerClient,\n years.shift()!,\n options.start,\n minDate(lastConsumable, options.end),\n {\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n }\n );\n }\n if (segments.length === 0) {\n return new ChangeFeed();\n }\n const currentSegment: Segment = await this.segmentFactory.create(\n containerClient,\n segments.shift()!,\n cursor?.CurrentSegmentCursor,\n {\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n }\n );\n\n return new ChangeFeed(\n containerClient,\n this.segmentFactory,\n years,\n segments,\n currentSegment,\n lastConsumable,\n options.start,\n options.end\n );\n });\n }\n}\n"]}
|
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
// Copyright (c) Microsoft Corporation.
|
|
2
2
|
// Licensed under the MIT license.
|
|
3
3
|
import { Readable } from "stream";
|
|
4
|
-
import {
|
|
5
|
-
import { SpanStatusCode } from "@azure/core-tracing";
|
|
4
|
+
import { tracingClient } from "./utils/tracing";
|
|
6
5
|
/**
|
|
7
6
|
* This class generates a readable stream from a blobClient's data.
|
|
8
7
|
*/
|
|
@@ -22,8 +21,7 @@ export class LazyLoadingBlobStream extends Readable {
|
|
|
22
21
|
this.options = options;
|
|
23
22
|
}
|
|
24
23
|
async downloadBlock(options = {}) {
|
|
25
|
-
|
|
26
|
-
try {
|
|
24
|
+
return tracingClient.withSpan("LazyLoadingBlobStream-downloadBlock", options, async (updatedOptions) => {
|
|
27
25
|
const properties = await this.blobClient.getProperties({
|
|
28
26
|
abortSignal: options.abortSignal,
|
|
29
27
|
tracingOptions: updatedOptions.tracingOptions,
|
|
@@ -39,17 +37,7 @@ export class LazyLoadingBlobStream extends Readable {
|
|
|
39
37
|
tracingOptions: updatedOptions.tracingOptions,
|
|
40
38
|
});
|
|
41
39
|
this.offset += this.lastDownloadBytes;
|
|
42
|
-
}
|
|
43
|
-
catch (e) {
|
|
44
|
-
span.setStatus({
|
|
45
|
-
code: SpanStatusCode.ERROR,
|
|
46
|
-
message: e.message,
|
|
47
|
-
});
|
|
48
|
-
throw e;
|
|
49
|
-
}
|
|
50
|
-
finally {
|
|
51
|
-
span.end();
|
|
52
|
-
}
|
|
40
|
+
});
|
|
53
41
|
}
|
|
54
42
|
/**
|
|
55
43
|
* Internal _read() that will be called when the stream wants to pull more data in.
|
|
@@ -57,9 +45,9 @@ export class LazyLoadingBlobStream extends Readable {
|
|
|
57
45
|
* @param size - Optional. The size of data to be read
|
|
58
46
|
*/
|
|
59
47
|
async _read(size) {
|
|
60
|
-
var _a
|
|
61
|
-
|
|
62
|
-
|
|
48
|
+
var _a;
|
|
49
|
+
return tracingClient.withSpan("LazyLoadingBlobStream-read", (_a = this.options) !== null && _a !== void 0 ? _a : {}, async (updatedOptions) => {
|
|
50
|
+
var _a, _b, _c, _d;
|
|
63
51
|
if (!size) {
|
|
64
52
|
size = this.readableHighWaterMark;
|
|
65
53
|
}
|
|
@@ -87,17 +75,7 @@ export class LazyLoadingBlobStream extends Readable {
|
|
|
87
75
|
if (count < size) {
|
|
88
76
|
this.push(null);
|
|
89
77
|
}
|
|
90
|
-
}
|
|
91
|
-
catch (e) {
|
|
92
|
-
span.setStatus({
|
|
93
|
-
code: SpanStatusCode.ERROR,
|
|
94
|
-
message: e.message,
|
|
95
|
-
});
|
|
96
|
-
this.emit("error", e);
|
|
97
|
-
}
|
|
98
|
-
finally {
|
|
99
|
-
span.end();
|
|
100
|
-
}
|
|
78
|
+
});
|
|
101
79
|
}
|
|
102
80
|
}
|
|
103
81
|
//# sourceMappingURL=LazyLoadingBlobStream.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LazyLoadingBlobStream.js","sourceRoot":"","sources":["../../../src/LazyLoadingBlobStream.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAElC,OAAO,EAAE,QAAQ,EAAmB,MAAM,QAAQ,CAAC;AAGnD,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"LazyLoadingBlobStream.js","sourceRoot":"","sources":["../../../src/LazyLoadingBlobStream.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAElC,OAAO,EAAE,QAAQ,EAAmB,MAAM,QAAQ,CAAC;AAGnD,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAqBhD;;GAEG;AACH,MAAM,OAAO,qBAAsB,SAAQ,QAAQ;IAqBjD;;;;OAIG;IACH,YACE,UAAsB,EACtB,MAAc,EACd,SAAiB,EACjB,OAAsC;QAEtC,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,iBAAiB,GAAG,CAAC,CAAC,CAAC;QAC5B,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,UAAqD,EAAE;QACjF,OAAO,aAAa,CAAC,QAAQ,CAC3B,qCAAqC,EACrC,OAAO,EACP,KAAK,EAAE,cAAc,EAAE,EAAE;YACvB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC;gBACrD,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;aAC9C,CAAC,CAAC;YACH,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,aAAc,CAAC;YAE5C,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;YACjF,IAAI,IAAI,CAAC,iBAAiB,KAAK,CAAC,EAAE;gBAChC,IAAI,CAAC,gBAAgB,GAAG,SAAS,CAAC;gBAClC,OAAO;aACR;YAED,IAAI,CAAC,gBAAgB,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAC5D,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,iBAAiB,EACtB;gBACE,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;aAC9C,CACF,CAAC;YACF,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,iBAAiB,CAAC;QACxC,CAAC,CACF,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,KAAK,CAAC,IAAa;;QAC9B,OAAO,aAAa,CAAC,QAAQ,CAC3B,4BAA4B,EAC5B,MAAA,IAAI,CAAC,OAAO,mCAAI,EAAE,EAClB,KAAK,EAAE,cAAc,EAAE,EAAE;;YACvB,IAAI,CAAC,IAAI,EAAE;gBACT,IAAI,GAAG,IAAI,CAAC,qBAAqB,CAAC;aACnC;YACD,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,MAAM,YAAY,GAAG,EAAE,CAAC;YACxB,GAAG;gBACD,IAAI,IAAI,CAAC,gBAAgB,KAAK,SAAS,IAAI,CAAA,MAAA,IAAI,CAAC,gBAAgB,0CAAE,UAAU,MAAK,CAAC,EAAE;oBAClF,MAAM,IAAI,CAAC,aAAa,CAAC;wBACvB,WAAW,EAAE,MAAA,IAAI,CAAC,OAAO,0CAAE,WAAW;wBACtC,cAAc,EAAE,cAAc,aAAd,cAAc,uBAAd,cAAc,CAAE,cAAc;qBAC/C,CAAC,CAAC;iBACJ;gBACD,IAAI,MAAA,IAAI,CAAC,gBAAgB,0CAAE,UAAU,EAAE;oBACrC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,KAAK,EAAE,MAAA,IAAI,CAAC,gBAAgB,0CAAE,UAAU,CAAC,CAAC;oBACtE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;oBAC7D,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;oBAC/D,KAAK,IAAI,SAAS,CAAC;iBACpB;qBAAM;oBACL,SAAS,GAAG,CAAC,CAAC;iBACf;aACF,QAAQ,SAAS,GAAG,CAAC,IAAI,KAAK,GAAG,IAAI,EAAE;YAExC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC;YAEvC,IAAI,KAAK,GAAG,IAAI,EAAE;gBAChB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aACjB;QACH,CAAC,CACF,CAAC;IACJ,CAAC;CACF","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\nimport { Readable, ReadableOptions } from \"stream\";\nimport { BlobClient, CommonOptions } from \"@azure/storage-blob\";\nimport { AbortSignalLike } from \"@azure/abort-controller\";\nimport { tracingClient } from \"./utils/tracing\";\n\n/**\n * Options to configure the LazyLoadingBlobStream.\n */\nexport interface LazyLoadingBlobStreamOptions extends ReadableOptions, CommonOptions {\n /**\n * An implementation of the `AbortSignalLike` interface to signal the request to cancel the operation.\n * For example, use the @azure/abort-controller to create an `AbortSignal`.\n */\n abortSignal?: AbortSignalLike;\n}\n\ninterface LazyLoadingBlobStreamDownloadBlockOptions extends CommonOptions {\n /**\n * An implementation of the `AbortSignalLike` interface to signal the request to cancel the operation.\n * For example, use the @azure/abort-controller to create an `AbortSignal`.\n */\n abortSignal?: AbortSignalLike;\n}\n\n/**\n * This class generates a readable stream from a blobClient's data.\n */\nexport class LazyLoadingBlobStream extends Readable {\n /**\n * BlobClient to make download calls with.\n */\n private readonly blobClient: BlobClient;\n\n /**\n * The offset within the blob of the next block we will download.\n */\n private offset: number;\n\n private readonly blockSize: number;\n\n private lastDownloadBytes: number;\n\n private lastDownloadData?: Buffer;\n\n private blobLength: number;\n\n private options?: LazyLoadingBlobStreamOptions;\n\n /**\n * Creates an instance of LazyLoadingBlobStream.\n *\n * @param byteLength - The total length of data contained in the buffers\n */\n constructor(\n blobClient: BlobClient,\n offset: number,\n blockSize: number,\n options?: LazyLoadingBlobStreamOptions\n ) {\n super(options);\n this.blobClient = blobClient;\n this.offset = offset;\n this.blockSize = blockSize;\n this.lastDownloadBytes = -1;\n this.blobLength = -1;\n this.options = options;\n }\n\n private async downloadBlock(options: LazyLoadingBlobStreamDownloadBlockOptions = {}) {\n return tracingClient.withSpan(\n \"LazyLoadingBlobStream-downloadBlock\",\n options,\n async (updatedOptions) => {\n const properties = await this.blobClient.getProperties({\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n });\n this.blobLength = properties.contentLength!;\n\n this.lastDownloadBytes = Math.min(this.blockSize, this.blobLength - this.offset);\n if (this.lastDownloadBytes === 0) {\n this.lastDownloadData = undefined;\n return;\n }\n\n this.lastDownloadData = await this.blobClient.downloadToBuffer(\n this.offset,\n this.lastDownloadBytes,\n {\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n }\n );\n this.offset += this.lastDownloadBytes;\n }\n );\n }\n\n /**\n * Internal _read() that will be called when the stream wants to pull more data in.\n *\n * @param size - Optional. The size of data to be read\n */\n public async _read(size?: number): Promise<void> {\n return tracingClient.withSpan(\n \"LazyLoadingBlobStream-read\",\n this.options ?? {},\n async (updatedOptions) => {\n if (!size) {\n size = this.readableHighWaterMark;\n }\n let count = 0;\n let chunkSize = 0;\n const chunksToPush = [];\n do {\n if (this.lastDownloadData === undefined || this.lastDownloadData?.byteLength === 0) {\n await this.downloadBlock({\n abortSignal: this.options?.abortSignal,\n tracingOptions: updatedOptions?.tracingOptions,\n });\n }\n if (this.lastDownloadData?.byteLength) {\n chunkSize = Math.min(size - count, this.lastDownloadData?.byteLength);\n chunksToPush.push(this.lastDownloadData.slice(0, chunkSize));\n this.lastDownloadData = this.lastDownloadData.slice(chunkSize);\n count += chunkSize;\n } else {\n chunkSize = 0;\n }\n } while (chunkSize > 0 && count < size);\n\n this.push(Buffer.concat(chunksToPush));\n\n if (count < size) {\n this.push(null);\n }\n }\n );\n }\n}\n"]}
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
// Copyright (c) Microsoft Corporation.
|
|
2
2
|
// Licensed under the MIT license.
|
|
3
|
-
import {
|
|
4
|
-
import { SpanStatusCode } from "@azure/core-tracing";
|
|
3
|
+
import { tracingClient } from "./utils/tracing";
|
|
5
4
|
export class Segment {
|
|
6
5
|
constructor(shards, shardIndex, dateTime, manifestPath) {
|
|
7
6
|
this.manifestPath = manifestPath;
|
|
@@ -18,8 +17,7 @@ export class Segment {
|
|
|
18
17
|
return this.shards.length > this.shardDoneCount;
|
|
19
18
|
}
|
|
20
19
|
async getChange(options = {}) {
|
|
21
|
-
|
|
22
|
-
try {
|
|
20
|
+
return tracingClient.withSpan("Segment-getChange", options, async (updatedOptions) => {
|
|
23
21
|
if (this.shardIndex >= this.shards.length || this.shardIndex < 0) {
|
|
24
22
|
throw new Error("shardIndex invalid.");
|
|
25
23
|
}
|
|
@@ -42,17 +40,7 @@ export class Segment {
|
|
|
42
40
|
this.shardIndex = (this.shardIndex + 1) % this.shards.length;
|
|
43
41
|
}
|
|
44
42
|
return event;
|
|
45
|
-
}
|
|
46
|
-
catch (e) {
|
|
47
|
-
span.setStatus({
|
|
48
|
-
code: SpanStatusCode.ERROR,
|
|
49
|
-
message: e.message,
|
|
50
|
-
});
|
|
51
|
-
throw e;
|
|
52
|
-
}
|
|
53
|
-
finally {
|
|
54
|
-
span.end();
|
|
55
|
-
}
|
|
43
|
+
});
|
|
56
44
|
}
|
|
57
45
|
getCursor() {
|
|
58
46
|
const shardCursors = [];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Segment.js","sourceRoot":"","sources":["../../../src/Segment.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAOlC,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"Segment.js","sourceRoot":"","sources":["../../../src/Segment.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAOlC,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAahD,MAAM,OAAO,OAAO;IAgBlB,YACE,MAAe,EACf,UAAkB,EAClB,QAAc,EACG,YAAoB;QAApB,iBAAY,GAAZ,YAAY,CAAQ;QAErC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;QAE1B,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClD,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;IAC1B,CAAC;IAhBD,IAAW,QAAQ;QACjB,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAgBM,OAAO;QACZ,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC;IAClD,CAAC;IAEM,KAAK,CAAC,SAAS,CACpB,UAAmC,EAAE;QAErC,OAAO,aAAa,CAAC,QAAQ,CAAC,mBAAmB,EAAE,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,EAAE;YACnF,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,GAAG,CAAC,EAAE;gBAChE,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;aACxC;YAED,IAAI,KAAK,GAAoC,SAAS,CAAC;YACvD,OAAO,KAAK,KAAK,SAAS,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE;gBAC5C,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE;oBACnC,IAAI,CAAC,UAAU,GAAG,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,4BAA4B;oBAC1F,SAAS;iBACV;gBAED,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAClD,KAAK,GAAG,MAAM,YAAY,CAAC,SAAS,CAAC;oBACnC,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;iBAC9C,CAAC,CAAC;gBAEH,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,EAAE;oBAC3B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;oBACvC,IAAI,CAAC,cAAc,EAAE,CAAC;iBACvB;gBACD,0BAA0B;gBAC1B,IAAI,CAAC,UAAU,GAAG,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;aAC9D;YACD,OAAO,KAAK,CAAC;QACf,CAAC,CAAC,CAAC;IACL,CAAC;IAEM,SAAS;QACd,MAAM,YAAY,GAAkB,EAAE,CAAC;QACvC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE;YAC/B,MAAM,WAAW,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;YACtC,IAAI,WAAW,EAAE;gBACf,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;aAChC;SACF;QAED,OAAO;YACL,WAAW,EAAE,IAAI,CAAC,YAAY;YAC9B,YAAY,EAAE,YAAY;YAC1B,gBAAgB,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,SAAS;SACzD,CAAC;IACJ,CAAC;CACF","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\nimport { BlobChangeFeedEvent } from \"./models/BlobChangeFeedEvent\";\nimport { Shard } from \"./Shard\";\nimport { SegmentCursor, ShardCursor } from \"./models/ChangeFeedCursor\";\nimport { CommonOptions } from \"@azure/storage-blob\";\nimport { AbortSignalLike } from \"@azure/abort-controller\";\nimport { tracingClient } from \"./utils/tracing\";\n\n/**\n * Options to configure {@link Segment.getChange} operation.\n */\nexport interface SegmentGetChangeOptions extends CommonOptions {\n /**\n * An implementation of the `AbortSignalLike` interface to signal the request to cancel the operation.\n * For example, use the @azure/abort-controller to create an `AbortSignal`.\n */\n abortSignal?: AbortSignalLike;\n}\n\nexport class Segment {\n private readonly shards: Shard[];\n\n // Track shards that we have finished reading from.\n private shardDone: boolean[];\n private shardDoneCount: number;\n\n private shardIndex: number;\n\n // Assuming the dateTime of segments is rounded to hour. If not, our logic for fetching\n // change events between a time range would be incorrect.\n private _dateTime: Date;\n public get dateTime(): Date {\n return this._dateTime;\n }\n\n constructor(\n shards: Shard[],\n shardIndex: number,\n dateTime: Date,\n private readonly manifestPath: string\n ) {\n this.shards = shards;\n this.shardIndex = shardIndex;\n this._dateTime = dateTime;\n\n this.shardDone = Array(shards.length).fill(false);\n this.shardDoneCount = 0;\n }\n\n public hasNext(): boolean {\n return this.shards.length > this.shardDoneCount;\n }\n\n public async getChange(\n options: SegmentGetChangeOptions = {}\n ): Promise<BlobChangeFeedEvent | undefined> {\n return tracingClient.withSpan(\"Segment-getChange\", options, async (updatedOptions) => {\n if (this.shardIndex >= this.shards.length || this.shardIndex < 0) {\n throw new Error(\"shardIndex invalid.\");\n }\n\n let event: BlobChangeFeedEvent | undefined = undefined;\n while (event === undefined && this.hasNext()) {\n if (this.shardDone[this.shardIndex]) {\n this.shardIndex = (this.shardIndex + 1) % this.shards.length; // find next available shard\n continue;\n }\n\n const currentShard = this.shards[this.shardIndex];\n event = await currentShard.getChange({\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n });\n\n if (!currentShard.hasNext()) {\n this.shardDone[this.shardIndex] = true;\n this.shardDoneCount++;\n }\n // Round robin with shards\n this.shardIndex = (this.shardIndex + 1) % this.shards.length;\n }\n return event;\n });\n }\n\n public getCursor(): SegmentCursor {\n const shardCursors: ShardCursor[] = [];\n for (const shard of this.shards) {\n const shardCursor = shard.getCursor();\n if (shardCursor) {\n shardCursors.push(shardCursor);\n }\n }\n\n return {\n SegmentPath: this.manifestPath,\n ShardCursors: shardCursors,\n CurrentShardPath: this.shards[this.shardIndex].shardPath,\n };\n }\n}\n"]}
|
|
@@ -4,15 +4,13 @@ import { CHANGE_FEED_CONTAINER_NAME } from "./utils/constants";
|
|
|
4
4
|
import { Segment } from "./Segment";
|
|
5
5
|
import { bodyToString } from "./utils/utils.node";
|
|
6
6
|
import { parseDateFromSegmentPath } from "./utils/utils.common";
|
|
7
|
-
import {
|
|
8
|
-
import { SpanStatusCode } from "@azure/core-tracing";
|
|
7
|
+
import { tracingClient } from "./utils/tracing";
|
|
9
8
|
export class SegmentFactory {
|
|
10
9
|
constructor(shardFactory) {
|
|
11
10
|
this.shardFactory = shardFactory;
|
|
12
11
|
}
|
|
13
12
|
async create(containerClient, manifestPath, cursor, options = {}) {
|
|
14
|
-
|
|
15
|
-
try {
|
|
13
|
+
return tracingClient.withSpan("SegmentFactory-create", options, async (updatedOptions) => {
|
|
16
14
|
const shards = [];
|
|
17
15
|
const dateTime = parseDateFromSegmentPath(manifestPath);
|
|
18
16
|
const blobClient = containerClient.getBlobClient(manifestPath);
|
|
@@ -42,17 +40,7 @@ export class SegmentFactory {
|
|
|
42
40
|
}
|
|
43
41
|
}
|
|
44
42
|
return new Segment(shards, shardIndex, dateTime, manifestPath);
|
|
45
|
-
}
|
|
46
|
-
catch (e) {
|
|
47
|
-
span.setStatus({
|
|
48
|
-
code: SpanStatusCode.ERROR,
|
|
49
|
-
message: e.message,
|
|
50
|
-
});
|
|
51
|
-
throw e;
|
|
52
|
-
}
|
|
53
|
-
finally {
|
|
54
|
-
span.end();
|
|
55
|
-
}
|
|
43
|
+
});
|
|
56
44
|
}
|
|
57
45
|
}
|
|
58
46
|
//# sourceMappingURL=SegmentFactory.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SegmentFactory.js","sourceRoot":"","sources":["../../../src/SegmentFactory.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAIlC,OAAO,EAAE,0BAA0B,EAAE,MAAM,mBAAmB,CAAC;AAE/D,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAEhE,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"SegmentFactory.js","sourceRoot":"","sources":["../../../src/SegmentFactory.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAIlC,OAAO,EAAE,0BAA0B,EAAE,MAAM,mBAAmB,CAAC;AAE/D,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAEhE,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAsBhD,MAAM,OAAO,cAAc;IAGzB,YAAY,YAA0B;QACpC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAEM,KAAK,CAAC,MAAM,CACjB,eAAgC,EAChC,YAAoB,EACpB,MAAsB,EACtB,UAAgC,EAAE;QAElC,OAAO,aAAa,CAAC,QAAQ,CAAC,uBAAuB,EAAE,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,EAAE;YACvF,MAAM,MAAM,GAAY,EAAE,CAAC;YAC3B,MAAM,QAAQ,GAAS,wBAAwB,CAAC,YAAY,CAAC,CAAC;YAE9D,MAAM,UAAU,GAAG,eAAe,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC;YAC/D,MAAM,eAAe,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,SAAS,EAAE,SAAS,EAAE;gBACtE,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;aAC9C,CAAC,CAAC;YACH,MAAM,WAAW,GAAW,MAAM,YAAY,CAAC,eAAe,CAAC,CAAC;YAEhE,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAoB,CAAC;YAEnE,MAAM,qBAAqB,GAAG,0BAA0B,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,qBAAqB;YAC1F,KAAK,MAAM,SAAS,IAAI,eAAe,CAAC,cAAc,EAAE;gBACtD,MAAM,eAAe,GAAG,SAAS,CAAC,SAAS,CAAC,qBAAqB,CAAC,CAAC;gBACnE,MAAM,WAAW,GAAG,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAClD,CAAC,CAAC,gBAAgB,CAAC,UAAU,CAAC,eAAe,CAAC,CAC/C,CAAC;gBACF,MAAM,KAAK,GAAU,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CACjD,eAAe,EACf,eAAe,EACf,WAAW,EACX;oBACE,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;iBAC9C,CACF,CAAC;gBACF,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE;oBACnB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;iBACpB;aACF;YAED,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,IAAI,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,gBAAgB,EAAE;gBAC5B,UAAU,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,MAAK,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,gBAAgB,CAAA,CAAC,CAAC;gBAC/E,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE;oBACrB,UAAU,GAAG,CAAC,CAAC;iBAChB;aACF;YACD,OAAO,IAAI,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;IACL,CAAC;CACF","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\nimport { ShardFactory } from \"./ShardFactory\";\nimport { ContainerClient, CommonOptions } from \"@azure/storage-blob\";\nimport { CHANGE_FEED_CONTAINER_NAME } from \"./utils/constants\";\nimport { Shard } from \"./Shard\";\nimport { Segment } from \"./Segment\";\nimport { SegmentCursor } from \"./models/ChangeFeedCursor\";\nimport { bodyToString } from \"./utils/utils.node\";\nimport { parseDateFromSegmentPath } from \"./utils/utils.common\";\nimport { AbortSignalLike } from \"@azure/abort-controller\";\nimport { tracingClient } from \"./utils/tracing\";\n\nexport interface SegmentManifest {\n version?: number;\n begin?: Date;\n intervalSecs?: number;\n status: string;\n config?: any;\n chunkFilePaths: string[];\n}\n\n/**\n * Options to configure {@link SegmentFactory.create} operation.\n */\nexport interface CreateSegmentOptions extends CommonOptions {\n /**\n * An implementation of the `AbortSignalLike` interface to signal the request to cancel the operation.\n * For example, use the @azure/abort-controller to create an `AbortSignal`.\n */\n abortSignal?: AbortSignalLike;\n}\n\nexport class SegmentFactory {\n private readonly shardFactory: ShardFactory;\n\n constructor(shardFactory: ShardFactory) {\n this.shardFactory = shardFactory;\n }\n\n public async create(\n containerClient: ContainerClient,\n manifestPath: string,\n cursor?: SegmentCursor,\n options: CreateSegmentOptions = {}\n ): Promise<Segment> {\n return tracingClient.withSpan(\"SegmentFactory-create\", options, async (updatedOptions) => {\n const shards: Shard[] = [];\n const dateTime: Date = parseDateFromSegmentPath(manifestPath);\n\n const blobClient = containerClient.getBlobClient(manifestPath);\n const blobDownloadRes = await blobClient.download(undefined, undefined, {\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n });\n const blobContent: string = await bodyToString(blobDownloadRes);\n\n const segmentManifest = JSON.parse(blobContent) as SegmentManifest;\n\n const containerPrefixLength = CHANGE_FEED_CONTAINER_NAME.length + 1; // \"$blobchangefeed/\"\n for (const shardPath of segmentManifest.chunkFilePaths) {\n const shardPathSubStr = shardPath.substring(containerPrefixLength);\n const shardCursor = cursor?.ShardCursors.find((x) =>\n x.CurrentChunkPath.startsWith(shardPathSubStr)\n );\n const shard: Shard = await this.shardFactory.create(\n containerClient,\n shardPathSubStr,\n shardCursor,\n {\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n }\n );\n if (shard.hasNext()) {\n shards.push(shard);\n }\n }\n\n let shardIndex = 0;\n if (cursor?.CurrentShardPath) {\n shardIndex = shards.findIndex((s) => s.shardPath === cursor?.CurrentShardPath);\n if (shardIndex === -1) {\n shardIndex = 0;\n }\n }\n return new Segment(shards, shardIndex, dateTime, manifestPath);\n });\n }\n}\n"]}
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
// Copyright (c) Microsoft Corporation.
|
|
2
2
|
// Licensed under the MIT license.
|
|
3
|
-
import {
|
|
4
|
-
import { SpanStatusCode } from "@azure/core-tracing";
|
|
3
|
+
import { tracingClient } from "./utils/tracing";
|
|
5
4
|
export class Shard {
|
|
6
5
|
constructor(containerClient, chunkFactory, chunks, currentChunk, shardPath) {
|
|
7
6
|
this.shardPath = shardPath;
|
|
@@ -14,8 +13,7 @@ export class Shard {
|
|
|
14
13
|
return (this.chunks.length > 0 || (this.currentChunk !== undefined && this.currentChunk.hasNext()));
|
|
15
14
|
}
|
|
16
15
|
async getChange(options = {}) {
|
|
17
|
-
|
|
18
|
-
try {
|
|
16
|
+
return tracingClient.withSpan("Shard-getChange", options, async (updatedOptions) => {
|
|
19
17
|
let event = undefined;
|
|
20
18
|
while (event === undefined && this.hasNext()) {
|
|
21
19
|
event = await this.currentChunk.getChange();
|
|
@@ -28,17 +26,7 @@ export class Shard {
|
|
|
28
26
|
}
|
|
29
27
|
}
|
|
30
28
|
return event;
|
|
31
|
-
}
|
|
32
|
-
catch (e) {
|
|
33
|
-
span.setStatus({
|
|
34
|
-
code: SpanStatusCode.ERROR,
|
|
35
|
-
message: e.message,
|
|
36
|
-
});
|
|
37
|
-
throw e;
|
|
38
|
-
}
|
|
39
|
-
finally {
|
|
40
|
-
span.end();
|
|
41
|
-
}
|
|
29
|
+
});
|
|
42
30
|
}
|
|
43
31
|
getCursor() {
|
|
44
32
|
return this.currentChunk === undefined
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Shard.js","sourceRoot":"","sources":["../../../src/Shard.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAQlC,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"Shard.js","sourceRoot":"","sources":["../../../src/Shard.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAQlC,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAahD,MAAM,OAAO,KAAK;IAShB,YACE,eAAgC,EAChC,YAA0B,EAC1B,MAAgB,EAChB,YAA+B,EACf,SAAiB;QAAjB,cAAS,GAAT,SAAS,CAAQ;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAEM,OAAO;QACZ,OAAO,CACL,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,KAAK,SAAS,IAAI,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAC3F,CAAC;IACJ,CAAC;IAEM,KAAK,CAAC,SAAS,CACpB,UAAiC,EAAE;QAEnC,OAAO,aAAa,CAAC,QAAQ,CAAC,iBAAiB,EAAE,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,EAAE;YACjF,IAAI,KAAK,GAAoC,SAAS,CAAC;YACvD,OAAO,KAAK,KAAK,SAAS,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE;gBAC5C,KAAK,GAAG,MAAM,IAAI,CAAC,YAAa,CAAC,SAAS,EAAE,CAAC;gBAE7C,sDAAsD;gBACtD,IAAI,CAAC,IAAI,CAAC,YAAa,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;oBAC3D,IAAI,CAAC,YAAY,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAChD,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAG,EACpB,SAAS,EACT,SAAS,EACT;wBACE,WAAW,EAAE,OAAO,CAAC,WAAW;wBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;qBAC9C,CACF,CAAC;iBACH;aACF;YACD,OAAO,KAAK,CAAC;QACf,CAAC,CAAC,CAAC;IACL,CAAC;IAEM,SAAS;QACd,OAAO,IAAI,CAAC,YAAY,KAAK,SAAS;YACpC,CAAC,CAAC,SAAS;YACX,CAAC,CAAC;gBACE,gBAAgB,EAAE,IAAI,CAAC,YAAY,CAAC,SAAS;gBAC7C,WAAW,EAAE,IAAI,CAAC,YAAY,CAAC,WAAW;gBAC1C,UAAU,EAAE,IAAI,CAAC,YAAY,CAAC,UAAU;aACzC,CAAC;IACR,CAAC;CACF","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\nimport { ContainerClient, CommonOptions } from \"@azure/storage-blob\";\nimport { ChunkFactory } from \"./ChunkFactory\";\nimport { Chunk } from \"./Chunk\";\nimport { BlobChangeFeedEvent } from \"./models/BlobChangeFeedEvent\";\nimport { ShardCursor } from \"./models/ChangeFeedCursor\";\nimport { AbortSignalLike } from \"@azure/abort-controller\";\nimport { tracingClient } from \"./utils/tracing\";\n\n/**\n * Options to configure {@link Shard.getChange} operation.\n */\nexport interface ShardGetChangeOptions extends CommonOptions {\n /**\n * An implementation of the `AbortSignalLike` interface to signal the request to cancel the operation.\n * For example, use the @azure/abort-controller to create an `AbortSignal`.\n */\n abortSignal?: AbortSignalLike;\n}\n\nexport class Shard {\n private readonly containerClient: ContainerClient;\n\n private readonly chunkFactory: ChunkFactory;\n\n private readonly chunks: string[];\n\n private currentChunk: Chunk | undefined;\n\n constructor(\n containerClient: ContainerClient,\n chunkFactory: ChunkFactory,\n chunks: string[],\n currentChunk: Chunk | undefined,\n public readonly shardPath: string\n ) {\n this.containerClient = containerClient;\n this.chunkFactory = chunkFactory;\n this.chunks = chunks;\n this.currentChunk = currentChunk;\n }\n\n public hasNext(): boolean {\n return (\n this.chunks.length > 0 || (this.currentChunk !== undefined && this.currentChunk.hasNext())\n );\n }\n\n public async getChange(\n options: ShardGetChangeOptions = {}\n ): Promise<BlobChangeFeedEvent | undefined> {\n return tracingClient.withSpan(\"Shard-getChange\", options, async (updatedOptions) => {\n let event: BlobChangeFeedEvent | undefined = undefined;\n while (event === undefined && this.hasNext()) {\n event = await this.currentChunk!.getChange();\n\n // Remove currentChunk if it doesn't have more events.\n if (!this.currentChunk!.hasNext() && this.chunks.length > 0) {\n this.currentChunk = await this.chunkFactory.create(\n this.containerClient,\n this.chunks.shift()!,\n undefined,\n undefined,\n {\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n }\n );\n }\n }\n return event;\n });\n }\n\n public getCursor(): ShardCursor | undefined {\n return this.currentChunk === undefined\n ? undefined\n : {\n CurrentChunkPath: this.currentChunk.chunkPath,\n BlockOffset: this.currentChunk.blockOffset,\n EventIndex: this.currentChunk.eventIndex,\n };\n }\n}\n"]}
|
|
@@ -2,16 +2,14 @@
|
|
|
2
2
|
// Licensed under the MIT license.
|
|
3
3
|
import { __asyncValues } from "tslib";
|
|
4
4
|
import { Shard } from "./Shard";
|
|
5
|
-
import {
|
|
6
|
-
import { createSpan } from "./utils/tracing";
|
|
5
|
+
import { tracingClient } from "./utils/tracing";
|
|
7
6
|
export class ShardFactory {
|
|
8
7
|
constructor(chunkFactory) {
|
|
9
8
|
this.chunkFactory = chunkFactory;
|
|
10
9
|
}
|
|
11
10
|
async create(containerClient, shardPath, shardCursor, options = {}) {
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
try {
|
|
11
|
+
return tracingClient.withSpan("ShardFactory-create", options, async (updatedOptions) => {
|
|
12
|
+
var e_1, _a;
|
|
15
13
|
const chunks = [];
|
|
16
14
|
const blockOffset = (shardCursor === null || shardCursor === void 0 ? void 0 : shardCursor.BlockOffset) || 0;
|
|
17
15
|
const eventIndex = (shardCursor === null || shardCursor === void 0 ? void 0 : shardCursor.EventIndex) || 0;
|
|
@@ -62,17 +60,7 @@ export class ShardFactory {
|
|
|
62
60
|
});
|
|
63
61
|
}
|
|
64
62
|
return new Shard(containerClient, this.chunkFactory, chunks, currentChunk, shardPath);
|
|
65
|
-
}
|
|
66
|
-
catch (e) {
|
|
67
|
-
span.setStatus({
|
|
68
|
-
code: SpanStatusCode.ERROR,
|
|
69
|
-
message: e.message,
|
|
70
|
-
});
|
|
71
|
-
throw e;
|
|
72
|
-
}
|
|
73
|
-
finally {
|
|
74
|
-
span.end();
|
|
75
|
-
}
|
|
63
|
+
});
|
|
76
64
|
}
|
|
77
65
|
}
|
|
78
66
|
//# sourceMappingURL=ShardFactory.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ShardFactory.js","sourceRoot":"","sources":["../../../src/ShardFactory.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;;AAIlC,OAAO,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAIhC,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"ShardFactory.js","sourceRoot":"","sources":["../../../src/ShardFactory.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;;AAIlC,OAAO,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAIhC,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAahD,MAAM,OAAO,YAAY;IAGvB,YAAY,YAA0B;QACpC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAEM,KAAK,CAAC,MAAM,CACjB,eAAgC,EAChC,SAAiB,EACjB,WAAyB,EACzB,UAA8B,EAAE;QAEhC,OAAO,aAAa,CAAC,QAAQ,CAAC,qBAAqB,EAAE,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,EAAE;;YACrF,MAAM,MAAM,GAAa,EAAE,CAAC;YAC5B,MAAM,WAAW,GAAW,CAAA,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,WAAW,KAAI,CAAC,CAAC;YAC1D,MAAM,UAAU,GAAW,CAAA,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,UAAU,KAAI,CAAC,CAAC;;gBAExD,KAA6B,IAAA,KAAA,cAAA,eAAe,CAAC,aAAa,CAAC;oBACzD,MAAM,EAAE,SAAS;oBACjB,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;iBAC9C,CAAC,CAAA,IAAA;oBAJS,MAAM,QAAQ,WAAA,CAAA;oBAKvB,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;iBAC5B;;;;;;;;;YAED,MAAM,gBAAgB,GAAG,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,gBAAgB,CAAC;YACvD,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC;YACpB,IAAI,YAAY,GAAsB,SAAS,CAAC;YAChD,8CAA8C;YAC9C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE;gBACvB,gCAAgC;gBAChC,IAAI,gBAAgB,EAAE;oBACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;wBACtC,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,gBAAgB,EAAE;4BAClC,UAAU,GAAG,CAAC,CAAC;4BACf,MAAM;yBACP;qBACF;oBACD,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE;wBACrB,MAAM,IAAI,KAAK,CAAC,SAAS,gBAAgB,aAAa,CAAC,CAAC;qBACzD;iBACF;qBAAM;oBACL,UAAU,GAAG,CAAC,CAAC;iBAChB;gBAED,iCAAiC;gBACjC,IAAI,UAAU,GAAG,CAAC,EAAE;oBAClB,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;iBAC9B;gBAED,YAAY,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAC3C,eAAe,EACf,MAAM,CAAC,KAAK,EAAG,EACf,WAAW,EACX,UAAU,EACV;oBACE,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,cAAc,EAAE,cAAc,CAAC,cAAc;iBAC9C,CACF,CAAC;aACH;YAED,OAAO,IAAI,KAAK,CAAC,eAAe,EAAE,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,YAAY,EAAE,SAAS,CAAC,CAAC;QACxF,CAAC,CAAC,CAAC;IACL,CAAC;CACF","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\nimport { ChunkFactory } from \"./ChunkFactory\";\nimport { ShardCursor } from \"./models/ChangeFeedCursor\";\nimport { Shard } from \"./Shard\";\nimport { ContainerClient, CommonOptions } from \"@azure/storage-blob\";\nimport { Chunk } from \"./Chunk\";\nimport { AbortSignalLike } from \"@azure/abort-controller\";\nimport { tracingClient } from \"./utils/tracing\";\n\n/**\n * Options to configure {@link ShardFactory.create} operation.\n */\nexport interface CreateShardOptions extends CommonOptions {\n /**\n * An implementation of the `AbortSignalLike` interface to signal the request to cancel the operation.\n * For example, use the @azure/abort-controller to create an `AbortSignal`.\n */\n abortSignal?: AbortSignalLike;\n}\n\nexport class ShardFactory {\n private readonly chunkFactory: ChunkFactory;\n\n constructor(chunkFactory: ChunkFactory) {\n this.chunkFactory = chunkFactory;\n }\n\n public async create(\n containerClient: ContainerClient,\n shardPath: string,\n shardCursor?: ShardCursor,\n options: CreateShardOptions = {}\n ): Promise<Shard> {\n return tracingClient.withSpan(\"ShardFactory-create\", options, async (updatedOptions) => {\n const chunks: string[] = [];\n const blockOffset: number = shardCursor?.BlockOffset || 0;\n const eventIndex: number = shardCursor?.EventIndex || 0;\n\n for await (const blobItem of containerClient.listBlobsFlat({\n prefix: shardPath,\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n })) {\n chunks.push(blobItem.name);\n }\n\n const currentChunkPath = shardCursor?.CurrentChunkPath;\n let chunkIndex = -1;\n let currentChunk: Chunk | undefined = undefined;\n // Chunks can be empty right after hour flips.\n if (chunks.length !== 0) {\n // Fast forward to current Chunk\n if (currentChunkPath) {\n for (let i = 0; i < chunks.length; i++) {\n if (chunks[i] === currentChunkPath) {\n chunkIndex = i;\n break;\n }\n }\n if (chunkIndex === -1) {\n throw new Error(`Chunk ${currentChunkPath} not found.`);\n }\n } else {\n chunkIndex = 0;\n }\n\n // Fast forward to current Chunk.\n if (chunkIndex > 0) {\n chunks.splice(0, chunkIndex);\n }\n\n currentChunk = await this.chunkFactory.create(\n containerClient,\n chunks.shift()!,\n blockOffset,\n eventIndex,\n {\n abortSignal: options.abortSignal,\n tracingOptions: updatedOptions.tracingOptions,\n }\n );\n }\n\n return new Shard(containerClient, this.chunkFactory, chunks, currentChunk, shardPath);\n });\n }\n}\n"]}
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
// Copyright (c) Microsoft Corporation.
|
|
2
2
|
// Licensed under the MIT license.
|
|
3
|
-
import {
|
|
3
|
+
import { createTracingClient } from "@azure/core-tracing";
|
|
4
|
+
import { SDK_VERSION } from "./constants";
|
|
4
5
|
/**
|
|
5
6
|
* Creates a span using the global tracer.
|
|
6
7
|
* @internal
|
|
7
8
|
*/
|
|
8
|
-
export const
|
|
9
|
-
|
|
9
|
+
export const tracingClient = createTracingClient({
|
|
10
|
+
packageName: "@azure/storage-blob-changefeed",
|
|
11
|
+
packageVersion: SDK_VERSION,
|
|
10
12
|
namespace: "Microsoft.Storage",
|
|
11
13
|
});
|
|
12
14
|
//# sourceMappingURL=tracing.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tracing.js","sourceRoot":"","sources":["../../../../src/utils/tracing.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAElC,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"tracing.js","sourceRoot":"","sources":["../../../../src/utils/tracing.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;AAElC,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE1C;;;GAGG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,mBAAmB,CAAC;IAC/C,WAAW,EAAE,gCAAgC;IAC7C,cAAc,EAAE,WAAW;IAC3B,SAAS,EAAE,mBAAmB;CAC/B,CAAC,CAAC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\nimport { createTracingClient } from \"@azure/core-tracing\";\nimport { SDK_VERSION } from \"./constants\";\n\n/**\n * Creates a span using the global tracer.\n * @internal\n */\nexport const tracingClient = createTracingClient({\n packageName: \"@azure/storage-blob-changefeed\",\n packageVersion: SDK_VERSION,\n namespace: \"Microsoft.Storage\",\n});\n"]}
|