@fluidframework/odsp-driver 1.2.6 → 1.3.0-100520

package/lib/odspDeltaStorageService.js.map

@@ -1 +1 @@
-{"version":3,"file":"odspDeltaStorageService.js","sourceRoot":"","sources":["../src/odspDeltaStorageService.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,OAAO,IAAI,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EAAE,EAAE,IAAI,IAAI,EAAE,MAAM,MAAM,CAAC;AAElC,OAAO,EAAE,MAAM,EAAE,MAAM,8BAA8B,CAAC;AAItD,OAAO,EACH,UAAU,EACV,cAAc,GACjB,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EAAE,2BAA2B,EAAE,MAAM,aAAa,CAAC;AAE1D;;GAEG;AACH,MAAM,OAAO,uBAAuB;IAChC,YACqB,YAAoB,EACpB,eAAgD,EAChD,YAA0B,EAC1B,MAAwB;QAHxB,iBAAY,GAAZ,YAAY,CAAQ;QACpB,oBAAe,GAAf,eAAe,CAAiC;QAChD,iBAAY,GAAZ,YAAY,CAAc;QAC1B,WAAM,GAAN,MAAM,CAAkB;IAE7C,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,GAAG,CACb,IAAY,EACZ,EAAU,EACV,cAAoC,EACpC,WAAoB;QAEpB,OAAO,2BAA2B,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;YACjD,gFAAgF;YAChF,kFAAkF;YAClF,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACxC,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;YAEzE,MAAM,YAAY,GAAG,IAAI,EAAE,CAAC;YAC5B,IAAI,QAAQ,GAAG,KAAK,YAAY,MAAM,CAAC;YACvC,QAAQ,IAAI,yBAAyB,YAAY,MAAM,CAAC;YACxD,QAAQ,IAAI,iCAAiC,CAAC;YAE9C,QAAQ,IAAI,cAAc,CAAC;YAC3B,QAAQ,IAAI,SAAS,YAAY,IAAI,CAAC;YACtC,MAAM,OAAO,GAA8B;gBACvC,cAAc,EAAE,gCAAgC,YAAY,EAAE;aACjE,CAAC;YAEF,mGAAmG;YACnG,oGAAoG;YACpG,0GAA0G;YAC1G,6EAA6E;YAC7E,kFAAkF;YAClF,MAAM,KAAK,GAAG,IAAI,eAAe,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,KAAK,CAAC,CAAC;YAErD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,mBAAmB,CACxD,OAAO,EACP;gBACI,OAAO;gBACP,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,MAAM;gBACd,MAAM,EAAE,KAAK,CAAC,MAAM;aACvB,EACD,KAAK,EACL,IAAI,EACJ,WAAW,CACd,CAAC;YACF,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,oBAAoB,GAAG,QAAQ,CAAC,OAAO,CAAC;YAC9C,IAAI,QAAqC,CAAC;YAC1C,IAAI,oBAAoB,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,IAAI,oBAAoB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;gBAChF,QAAQ,GAAI,oBAAoB,CAAC,KAAoC,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;aAC1G;iBAAM;gBACH,QAAQ,GAAG,oBAAoB,CAAC,KAAoC,CAAC;aACxE;YAED,IAAI,CAAC,MAAM,CAAC,oBAAoB,6CAC5B,SAAS,EAAE,UAAU,EACrB,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,EAC7D,MAAM,EAAE,QAAQ,CAAC,MAAM,EACvB,QAAQ,EAAE,QAAQ,CAAC,QAAQ,IACxB,QAAQ,CAAC,UAAU,KACtB,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACjC,IAAI;gBACJ,EAAE,KACC,cAAc,EACnB,CAAC;YAEH,oGAAoG;YACpG,4GAA4G;YAC5G,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;QAC9C,CAAC,CAAC,CAAC;IACP,CAAC;IAEM,QAAQ,CAAC,IAAY,EAAE,EAAU;QACpC,MAAM,MAAM,GAAG,kBAAkB,CAAC,qBAAqB,IAAI,0BAA0B,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAC/F,MAAM,WAAW,GAAG,iBAAiB,MAAM,EAAE,CAAC;QAC9C,OAAO,GAAG,IAAI,CAAC,YAAY,GAAG,WAAW,EAAE,CAAC;IAChD,CAAC;CACJ;AAED,MAAM,OAAO,yBAAyB;IAGlC,YACY,WAAoD,EAC3C,MAAwB,EACxB,SAAiB,EACjB,WAAmB,EACnB,cAIuC,EACvC,SAA6E,EAC7E,iBAAqD,EACrD,WAAuD;QAXhE,gBAAW,GAAX,WAAW,CAAyC;QAC3C,WAAM,GAAN,MAAM,CAAkB;QACxB,cAAS,GAAT,SAAS,CAAQ;QACjB,gBAAW,GAAX,WAAW,CAAQ;QACnB,mBAAc,GAAd,cAAc,CAIyB;QACvC,cAAS,GAAT,SAAS,CAAoE;QAC7E,sBAAiB,GAAjB,iBAAiB,CAAoC;QACrD,gBAAW,GAAX,WAAW,CAA4C;QAdpE,mBAAc,GAAG,MAAM,CAAC,gBAAgB,CAAC;IAgBjD,CAAC;IAES,gBAAgB,CAAC,MAAc,EAAE,QAAqC,EAAE,IAAY;QAC1F,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE;YACvB,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC;YACzC,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;YAC/B,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC;YACjD,IAAI,KAAK,KAAK,IAAI,EAAE;gBAChB,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,EAAE,mBAAmB,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;gBAClG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;aACvB;YACD,IAAI,IAAI,GAAG,CAAC,KAAK,IAAI,GAAG,MAAM,EAAE;gBAC5B,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,EAAE,mBAAmB,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;gBAClG,uEAAuE;gBACvE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;aACvB;SACJ;IACL,CAAC;IAEM,aAAa,CAChB,SAAiB,EACjB,OAA2B,EAC3B,WAAyB,EACzB,UAAoB,EACpB,WAAoB;QACpB,mGAAmG;QACnG,yGAAyG;QACzG,4BAA4B;QAC5B,gGAAgG;QAChG,MAAM,CAAC,CAAC,UAAU,IAAI,OAAO,KAAK,SAAS,EAAE,KAAK,CAAC,CAAC;QAEpD,IAAI,eAAe,GAAG,CAAC,CAAC;QACxB,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,cAAc,GAAG,CAAC,CAAC;QAEvB,MAAM,eAAe,GAAG,KAAK,EAAE,IAAY,EAAE,EAAU,EAAE,cAAoC,EAAE,EAAE;YAC7F,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE;gBACjE,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAC5C,EAAE,CAAC,cAAc,IAAI,IAAI,IAAI,EAAE,CAAC,cAAc,GAAG,EAAE,CAAC,CAAC;gBACzD,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;gBAChD,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC,cAAc,KAAK,IAAI,EAAE;oBAC5D,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC;oBAC5E,eAAe,GAAG,QAAQ,CAAC,MAAM,CAAC;oBAClC,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;iBAC5C;gBACD,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC;aAChC;YAED,kDAAkD;YAClD,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAEjC,mFAAmF;YACnF,sCAAsC;YACtC,IAAI,IAAI,GAAG,IAAI,CAAC,cAAc,EAAE;gBAC5B,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBACzD,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,iBAAiB,EAAE,IAAI,CAAC,CAAC;gBACzD,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE;oBAChC,YAAY,IAAI,iBAAiB,CAAC,MAAM,CAAC;oBACzC,OAAO;wBACH,QAAQ,EAAE,iBAAiB;wBAC3B,aAAa,EAAE,IAAI;qBACtB,CAAC;iBACL;gBACD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;aAC7D;YAED,IAAI,UAAU,EAAE;gBACZ,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;aACjD;YAED,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,EAAE,EAAE,cAAc,EAAE,WAAW,CAAC,CAAC;YAC7E,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YACrD,cAAc,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC;YACtC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC/B,OAAO,GAAG,CAAC;QACf,CAAC,CAAC;QAEF,MAAM,MAAM,GAAG,UAAU,CACrB,KAAK,EAAE,IAAY,EAAE,EAAU,EAAE,cAAoC,EAAE,EAAE;YACrE,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,IAAI,EAAE,EAAE,EAAE,cAAc,CAAC,CAAC;YAC/D,+BAA+B;YAC/B,IAAI,CAAC,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAC1D,OAAO,MAAM,CAAC;QAClB,CAAC;QACD,uEAAuE;QACvE,0DAA0D;QAC1D,IAAI,CAAC,WAAW,EAChB,SAAS,EAAE,YAAY;QACvB,OAAO,EAAE,YAAY;QACrB,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,MAAM,EACX,WAAW,EACX,WAAW,CACd,CAAC;QAEF,OAAO,cAAc,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE;YACrC,IAAI,MAAM,CAAC,IAAI,IAAI,eAAe,GAAG,YAAY,GAAG,cAAc,KAAK,CAAC,EAAE;gBACtE,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;oBAC7B,SAAS,EAAE,mBAAmB;oBAC9B,eAAe;oBACf,YAAY;oBACZ,cAAc;iBACjB,CAAC,CAAC;aACN;QACL,CAAC,CAAC,CAAC;IACP,CAAC;CACJ","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { default as AbortController } from \"abort-controller\";\nimport { v4 as uuid } from \"uuid\";\nimport { ITelemetryLogger, ITelemetryProperties } from \"@fluidframework/common-definitions\";\nimport { assert } from \"@fluidframework/common-utils\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/protocol-definitions\";\nimport { InstrumentedStorageTokenFetcher } from \"@fluidframework/odsp-driver-definitions\";\nimport { IDeltasFetchResult, IDocumentDeltaStorageService } from \"@fluidframework/driver-definitions\";\nimport {\n    requestOps,\n    streamObserver,\n} from \"@fluidframework/driver-utils\";\nimport { IDeltaStorageGetResponse, ISequencedDeltaOpMessage } from \"./contracts\";\nimport { EpochTracker } from \"./epochTracker\";\nimport { getWithRetryForTokenRefresh } from \"./odspUtils\";\n\n/**\n * Provides access to the underlying delta storage on the server for sharepoint driver.\n */\nexport class OdspDeltaStorageService {\n    constructor(\n        private readonly deltaFeedUrl: string,\n        private readonly getStorageToken: InstrumentedStorageTokenFetcher,\n        private readonly epochTracker: EpochTracker,\n        private readonly logger: ITelemetryLogger,\n    ) {\n    }\n\n    /**\n     * Retrieves ops from cache\n     * @param from - inclusive\n     * @param to - exclusive\n     * @param telemetryProps - properties to add when issuing telemetry events\n     * @returns ops retrieved & info if result was partial (i.e. more is available)\n     */\n     public async get(\n        from: number,\n        to: number,\n        telemetryProps: ITelemetryProperties,\n        fetchReason?: string,\n    ): Promise<IDeltasFetchResult> {\n        return getWithRetryForTokenRefresh(async (options) => {\n            // Note - this call ends up in getSocketStorageDiscovery() and can refresh token\n            // Thus it needs to be done before we call getStorageToken() to reduce extra calls\n            const baseUrl = this.buildUrl(from, to);\n            const storageToken = await this.getStorageToken(options, \"DeltaStorage\");\n\n            const formBoundary = uuid();\n            let postBody = `--${formBoundary}\\r\\n`;\n            postBody += `Authorization: Bearer ${storageToken}\\r\\n`;\n            postBody += `X-HTTP-Method-Override: GET\\r\\n`;\n\n            postBody += `_post: 1\\r\\n`;\n            postBody += `\\r\\n--${formBoundary}--`;\n            const headers: { [index: string]: any; } = {\n                \"Content-Type\": `multipart/form-data;boundary=${formBoundary}`,\n            };\n\n            // Some request take a long time (1-2 minutes) to complete, where telemetry shows very small amount\n            // of time spent on server, and usually small payload sizes. I.e. all the time is spent somewhere in\n            // networking. Even bigger problem - a lot of requests timeout (based on cursory look - after 1-2 minutes)\n            // So adding some timeout to ensure we retry again in hope of faster success.\n            // Please see https://github.com/microsoft/FluidFramework/issues/6997 for details.\n            const abort = new AbortController();\n            const timer = setTimeout(() => abort.abort(), 30000);\n\n            const response = await this.epochTracker.fetchAndParseAsJSON<IDeltaStorageGetResponse>(\n                baseUrl,\n                {\n                    headers,\n                    body: postBody,\n                    method: \"POST\",\n                    signal: abort.signal,\n                },\n                \"ops\",\n                true,\n                fetchReason,\n            );\n            clearTimeout(timer);\n            const deltaStorageResponse = response.content;\n            let messages: ISequencedDocumentMessage[];\n            if (deltaStorageResponse.value.length > 0 && \"op\" in deltaStorageResponse.value[0]) {\n                messages = (deltaStorageResponse.value as ISequencedDeltaOpMessage[]).map((operation) => operation.op);\n            } else {\n                messages = deltaStorageResponse.value as ISequencedDocumentMessage[];\n            }\n\n            this.logger.sendPerformanceEvent({\n                eventName: \"OpsFetch\",\n                headers: Object.keys(headers).length !== 0 ? true : undefined,\n                length: messages.length,\n                duration: response.duration, // this duration for single attempt!\n                ...response.propsToLog,\n                attempts: options.refresh ? 2 : 1,\n                from,\n                to,\n                ...telemetryProps,\n            });\n\n            // It is assumed that server always returns all the ops that it has in the range that was requested.\n            // This may change in the future, if so, we need to adjust and receive \"end\" value from server in such case.\n            return { messages, partialResult: false };\n        });\n    }\n\n    public buildUrl(from: number, to: number) {\n        const filter = encodeURIComponent(`sequenceNumber ge ${from} and sequenceNumber le ${to - 1}`);\n        const queryString = `?ump=1&filter=${filter}`;\n        return `${this.deltaFeedUrl}${queryString}`;\n    }\n}\n\nexport class OdspDeltaStorageWithCache implements IDocumentDeltaStorageService {\n    private firstCacheMiss = Number.MAX_SAFE_INTEGER;\n\n    public constructor(\n        private snapshotOps: ISequencedDocumentMessage[] | undefined,\n        private readonly logger: ITelemetryLogger,\n        private readonly batchSize: number,\n        private readonly concurrency: number,\n        private readonly getFromStorage: (\n            from: number,\n            to: number,\n            telemetryProps: ITelemetryProperties,\n            fetchReason?: string) => Promise<IDeltasFetchResult>,\n        private readonly getCached: (from: number, to: number) => Promise<ISequencedDocumentMessage[]>,\n        private readonly requestFromSocket: (from: number, to: number) => void,\n        private readonly opsReceived: (ops: ISequencedDocumentMessage[]) => void,\n    ) {\n    }\n\n    protected validateMessages(reason: string, messages: ISequencedDocumentMessage[], from: number) {\n        if (messages.length !== 0) {\n            const start = messages[0].sequenceNumber;\n            const length = messages.length;\n            const last = messages[length - 1].sequenceNumber;\n            if (start !== from) {\n                this.logger.sendErrorEvent({ eventName: \"OpsFetchViolation\", reason, from, start, last, length });\n                messages.length = 0;\n            }\n            if (last + 1 !== from + length) {\n                this.logger.sendErrorEvent({ eventName: \"OpsFetchViolation\", reason, from, start, last, length });\n                // we can do better here by finding consecutive sub-block and return it\n                messages.length = 0;\n            }\n        }\n    }\n\n    public fetchMessages(\n        fromTotal: number,\n        toTotal: number | undefined,\n        abortSignal?: AbortSignal,\n        cachedOnly?: boolean,\n        fetchReason?: string) {\n        // We do not control what's in the cache. Current API assumes that fetchMessages() keeps banging on\n        // storage / cache until it gets ops it needs. This would result in deadlock if fixed range is asked from\n        // cache and it's not there.\n        // Better implementation would be to return only what we have in cache, but that also breaks API\n        assert(!cachedOnly || toTotal === undefined, 0x1e3);\n\n        let opsFromSnapshot = 0;\n        let opsFromCache = 0;\n        let opsFromStorage = 0;\n\n        const requestCallback = async (from: number, to: number, telemetryProps: ITelemetryProperties) => {\n            if (this.snapshotOps !== undefined && this.snapshotOps.length !== 0) {\n                const messages = this.snapshotOps.filter((op) =>\n                    op.sequenceNumber >= from && op.sequenceNumber < to);\n                this.validateMessages(\"cached\", messages, from);\n                if (messages.length > 0 && messages[0].sequenceNumber === from) {\n                    this.snapshotOps = this.snapshotOps.filter((op) => op.sequenceNumber >= to);\n                    opsFromSnapshot = messages.length;\n                    return { messages, partialResult: true };\n                }\n                this.snapshotOps = undefined;\n            }\n\n            // Kick out request to PUSH for ops if it has them\n            this.requestFromSocket(from, to);\n\n            // Cache in normal flow is continuous. Once there is a miss, stop consulting cache.\n            // This saves a bit of processing time\n            if (from < this.firstCacheMiss) {\n                const messagesFromCache = await this.getCached(from, to);\n                this.validateMessages(\"cached\", messagesFromCache, from);\n                if (messagesFromCache.length !== 0) {\n                    opsFromCache += messagesFromCache.length;\n                    return {\n                        messages: messagesFromCache,\n                        partialResult: true,\n                    };\n                }\n                this.firstCacheMiss = Math.min(this.firstCacheMiss, from);\n            }\n\n            if (cachedOnly) {\n                return { messages: [], partialResult: false };\n            }\n\n            const ops = await this.getFromStorage(from, to, telemetryProps, fetchReason);\n            this.validateMessages(\"storage\", ops.messages, from);\n            opsFromStorage += ops.messages.length;\n            this.opsReceived(ops.messages);\n            return ops;\n        };\n\n        const stream = requestOps(\n            async (from: number, to: number, telemetryProps: ITelemetryProperties) => {\n                const result = await requestCallback(from, to, telemetryProps);\n                // Catch all case, just in case\n                this.validateMessages(\"catch all\", result.messages, from);\n                return result;\n            },\n            // Staging: starting with no concurrency, listening for feedback first.\n            // In future releases we will switch to actual concurrency\n            this.concurrency,\n            fromTotal, // inclusive\n            toTotal, // exclusive\n            this.batchSize,\n            this.logger,\n            abortSignal,\n            fetchReason,\n        );\n\n        return streamObserver(stream, (result) => {\n            if (result.done && opsFromSnapshot + opsFromCache + opsFromStorage !== 0) {\n                this.logger.sendPerformanceEvent({\n                    eventName: \"CacheOpsRetrieved\",\n                    opsFromSnapshot,\n                    opsFromCache,\n                    opsFromStorage,\n                });\n            }\n        });\n    }\n}\n"]}
+{"version":3,"file":"odspDeltaStorageService.js","sourceRoot":"","sources":["../src/odspDeltaStorageService.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,OAAO,IAAI,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EAAE,EAAE,IAAI,IAAI,EAAE,MAAM,MAAM,CAAC;AAElC,OAAO,EAAE,MAAM,EAAE,MAAM,8BAA8B,CAAC;AAItD,OAAO,EACH,UAAU,EACV,cAAc,GACjB,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EAAE,2BAA2B,EAAE,MAAM,aAAa,CAAC;AAE1D;;GAEG;AACH,MAAM,OAAO,uBAAuB;IAChC,YACqB,YAAoB,EACpB,eAAgD,EAChD,YAA0B,EAC1B,MAAwB;QAHxB,iBAAY,GAAZ,YAAY,CAAQ;QACpB,oBAAe,GAAf,eAAe,CAAiC;QAChD,iBAAY,GAAZ,YAAY,CAAc;QAC1B,WAAM,GAAN,MAAM,CAAkB;IAE7C,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,GAAG,CACb,IAAY,EACZ,EAAU,EACV,cAAoC,EACpC,YAAqB;QAErB,OAAO,2BAA2B,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;YACjD,gFAAgF;YAChF,kFAAkF;YAClF,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACxC,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;YAEzE,MAAM,YAAY,GAAG,IAAI,EAAE,CAAC;YAC5B,IAAI,QAAQ,GAAG,KAAK,YAAY,MAAM,CAAC;YACvC,QAAQ,IAAI,yBAAyB,YAAY,MAAM,CAAC;YACxD,QAAQ,IAAI,iCAAiC,CAAC;YAE9C,QAAQ,IAAI,cAAc,CAAC;YAC3B,QAAQ,IAAI,SAAS,YAAY,IAAI,CAAC;YACtC,MAAM,OAAO,GAA8B;gBACvC,cAAc,EAAE,gCAAgC,YAAY,EAAE;aACjE,CAAC;YAEF,mGAAmG;YACnG,oGAAoG;YACpG,0GAA0G;YAC1G,6EAA6E;YAC7E,kFAAkF;YAClF,MAAM,KAAK,GAAG,IAAI,eAAe,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,KAAK,CAAC,CAAC;YAErD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,mBAAmB,CACxD,OAAO,EACP;gBACI,OAAO;gBACP,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,MAAM;gBACd,MAAM,EAAE,KAAK,CAAC,MAAM;aACvB,EACD,KAAK,EACL,IAAI,EACJ,YAAY,CACf,CAAC;YACF,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,oBAAoB,GAAG,QAAQ,CAAC,OAAO,CAAC;YAC9C,IAAI,QAAqC,CAAC;YAC1C,IAAI,oBAAoB,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,IAAI,oBAAoB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;gBAChF,QAAQ,GAAI,oBAAoB,CAAC,KAAoC,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;aAC1G;iBAAM;gBACH,QAAQ,GAAG,oBAAoB,CAAC,KAAoC,CAAC;aACxE;YAED,IAAI,CAAC,MAAM,CAAC,oBAAoB,2DAC5B,SAAS,EAAE,UAAU,EACrB,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,EAC7D,MAAM,EAAE,QAAQ,CAAC,MAAM,EACvB,QAAQ,EAAE,QAAQ,CAAC,QAAQ,IACxB,QAAQ,CAAC,UAAU,KACtB,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACjC,IAAI;gBACJ,EAAE,KACC,cAAc,KACjB,MAAM,EAAE,YAAY,IACtB,CAAC;YAEH,oGAAoG;YACpG,4GAA4G;YAC5G,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;QAC9C,CAAC,CAAC,CAAC;IACP,CAAC;IAEM,QAAQ,CAAC,IAAY,EAAE,EAAU;QACpC,MAAM,MAAM,GAAG,kBAAkB,CAAC,qBAAqB,IAAI,0BAA0B,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAC/F,MAAM,WAAW,GAAG,iBAAiB,MAAM,EAAE,CAAC;QAC9C,OAAO,GAAG,IAAI,CAAC,YAAY,GAAG,WAAW,EAAE,CAAC;IAChD,CAAC;CACJ;AAED,MAAM,OAAO,yBAAyB;IAGlC,YACY,WAAoD,EAC3C,MAAwB,EACxB,SAAiB,EACjB,WAAmB,EACnB,cAIuC,EACvC,SAA6E,EAC7E,iBAAqD,EACrD,WAAuD;QAXhE,gBAAW,GAAX,WAAW,CAAyC;QAC3C,WAAM,GAAN,MAAM,CAAkB;QACxB,cAAS,GAAT,SAAS,CAAQ;QACjB,gBAAW,GAAX,WAAW,CAAQ;QACnB,mBAAc,GAAd,cAAc,CAIyB;QACvC,cAAS,GAAT,SAAS,CAAoE;QAC7E,sBAAiB,GAAjB,iBAAiB,CAAoC;QACrD,gBAAW,GAAX,WAAW,CAA4C;QAdpE,mBAAc,GAAG,MAAM,CAAC,gBAAgB,CAAC;IAgBjD,CAAC;IAES,gBAAgB,CAAC,MAAc,EAAE,QAAqC,EAAE,IAAY;QAC1F,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE;YACvB,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC;YACzC,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;YAC/B,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC;YACjD,IAAI,KAAK,KAAK,IAAI,EAAE;gBAChB,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,EAAE,mBAAmB,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;gBAClG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;aACvB;YACD,IAAI,IAAI,GAAG,CAAC,KAAK,IAAI,GAAG,MAAM,EAAE;gBAC5B,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,EAAE,mBAAmB,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;gBAClG,uEAAuE;gBACvE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;aACvB;SACJ;IACL,CAAC;IAEM,aAAa,CAChB,SAAiB,EACjB,OAA2B,EAC3B,WAAyB,EACzB,UAAoB,EACpB,WAAoB;QACpB,mGAAmG;QACnG,yGAAyG;QACzG,4BAA4B;QAC5B,gGAAgG;QAChG,MAAM,CAAC,CAAC,UAAU,IAAI,OAAO,KAAK,SAAS,EAAE,KAAK,CAAC,CAAC;QAEpD,IAAI,eAAe,GAAG,CAAC,CAAC;QACxB,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,cAAc,GAAG,CAAC,CAAC;QAEvB,MAAM,eAAe,GAAG,KAAK,EAAE,IAAY,EAAE,EAAU,EAAE,cAAoC,EAAE,EAAE;YAC7F,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE;gBACjE,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAC5C,EAAE,CAAC,cAAc,IAAI,IAAI,IAAI,EAAE,CAAC,cAAc,GAAG,EAAE,CAAC,CAAC;gBACzD,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;gBAChD,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC,cAAc,KAAK,IAAI,EAAE;oBAC5D,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC;oBAC5E,eAAe,GAAG,QAAQ,CAAC,MAAM,CAAC;oBAClC,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;iBAC5C;gBACD,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC;aAChC;YAED,kDAAkD;YAClD,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAEjC,mFAAmF;YACnF,sCAAsC;YACtC,IAAI,IAAI,GAAG,IAAI,CAAC,cAAc,EAAE;gBAC5B,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBACzD,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,iBAAiB,EAAE,IAAI,CAAC,CAAC;gBACzD,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE;oBAChC,YAAY,IAAI,iBAAiB,CAAC,MAAM,CAAC;oBACzC,OAAO;wBACH,QAAQ,EAAE,iBAAiB;wBAC3B,aAAa,EAAE,IAAI;qBACtB,CAAC;iBACL;gBACD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;aAC7D;YAED,IAAI,UAAU,EAAE;gBACZ,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;aACjD;YAED,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,EAAE,EAAE,cAAc,EAAE,WAAW,CAAC,CAAC;YAC7E,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YACrD,cAAc,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC;YACtC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC/B,OAAO,GAAG,CAAC;QACf,CAAC,CAAC;QAEF,MAAM,MAAM,GAAG,UAAU,CACrB,KAAK,EAAE,IAAY,EAAE,EAAU,EAAE,cAAoC,EAAE,EAAE;YACrE,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,IAAI,EAAE,EAAE,EAAE,cAAc,CAAC,CAAC;YAC/D,+BAA+B;YAC/B,IAAI,CAAC,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAC1D,OAAO,MAAM,CAAC;QAClB,CAAC;QACD,uEAAuE;QACvE,0DAA0D;QAC1D,IAAI,CAAC,WAAW,EAChB,SAAS,EAAE,YAAY;QACvB,OAAO,EAAE,YAAY;QACrB,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,MAAM,EACX,WAAW,EACX,WAAW,CACd,CAAC;QAEF,OAAO,cAAc,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE;YACrC,IAAI,MAAM,CAAC,IAAI,IAAI,eAAe,GAAG,YAAY,GAAG,cAAc,KAAK,CAAC,EAAE;gBACtE,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;oBAC7B,SAAS,EAAE,mBAAmB;oBAC9B,eAAe;oBACf,YAAY;oBACZ,cAAc;iBACjB,CAAC,CAAC;aACN;QACL,CAAC,CAAC,CAAC;IACP,CAAC;CACJ","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { default as AbortController } from \"abort-controller\";\nimport { v4 as uuid } from \"uuid\";\nimport { ITelemetryLogger, ITelemetryProperties } from \"@fluidframework/common-definitions\";\nimport { assert } from \"@fluidframework/common-utils\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/protocol-definitions\";\nimport { InstrumentedStorageTokenFetcher } from \"@fluidframework/odsp-driver-definitions\";\nimport { IDeltasFetchResult, IDocumentDeltaStorageService } from \"@fluidframework/driver-definitions\";\nimport {\n    requestOps,\n    streamObserver,\n} from \"@fluidframework/driver-utils\";\nimport { IDeltaStorageGetResponse, ISequencedDeltaOpMessage } from \"./contracts\";\nimport { EpochTracker } from \"./epochTracker\";\nimport { getWithRetryForTokenRefresh } from \"./odspUtils\";\n\n/**\n * Provides access to the underlying delta storage on the server for sharepoint driver.\n */\nexport class OdspDeltaStorageService {\n    constructor(\n        private readonly deltaFeedUrl: string,\n        private readonly getStorageToken: InstrumentedStorageTokenFetcher,\n        private readonly epochTracker: EpochTracker,\n        private readonly logger: ITelemetryLogger,\n    ) {\n    }\n\n    /**\n     * Retrieves ops from cache\n     * @param from - inclusive\n     * @param to - exclusive\n     * @param telemetryProps - properties to add when issuing telemetry events\n     * @param scenarioName - reason for fetching ops\n     * @returns ops retrieved & info if result was partial (i.e. more is available)\n     */\n     public async get(\n        from: number,\n        to: number,\n        telemetryProps: ITelemetryProperties,\n        scenarioName?: string,\n    ): Promise<IDeltasFetchResult> {\n        return getWithRetryForTokenRefresh(async (options) => {\n            // Note - this call ends up in getSocketStorageDiscovery() and can refresh token\n            // Thus it needs to be done before we call getStorageToken() to reduce extra calls\n            const baseUrl = this.buildUrl(from, to);\n            const storageToken = await this.getStorageToken(options, \"DeltaStorage\");\n\n            const formBoundary = uuid();\n            let postBody = `--${formBoundary}\\r\\n`;\n            postBody += `Authorization: Bearer ${storageToken}\\r\\n`;\n            postBody += `X-HTTP-Method-Override: GET\\r\\n`;\n\n            postBody += `_post: 1\\r\\n`;\n            postBody += `\\r\\n--${formBoundary}--`;\n            const headers: { [index: string]: any; } = {\n                \"Content-Type\": `multipart/form-data;boundary=${formBoundary}`,\n            };\n\n            // Some request take a long time (1-2 minutes) to complete, where telemetry shows very small amount\n            // of time spent on server, and usually small payload sizes. I.e. all the time is spent somewhere in\n            // networking. Even bigger problem - a lot of requests timeout (based on cursory look - after 1-2 minutes)\n            // So adding some timeout to ensure we retry again in hope of faster success.\n            // Please see https://github.com/microsoft/FluidFramework/issues/6997 for details.\n            const abort = new AbortController();\n            const timer = setTimeout(() => abort.abort(), 30000);\n\n            const response = await this.epochTracker.fetchAndParseAsJSON<IDeltaStorageGetResponse>(\n                baseUrl,\n                {\n                    headers,\n                    body: postBody,\n                    method: \"POST\",\n                    signal: abort.signal,\n                },\n                \"ops\",\n                true,\n                scenarioName,\n            );\n            clearTimeout(timer);\n            const deltaStorageResponse = response.content;\n            let messages: ISequencedDocumentMessage[];\n            if (deltaStorageResponse.value.length > 0 && \"op\" in deltaStorageResponse.value[0]) {\n                messages = (deltaStorageResponse.value as ISequencedDeltaOpMessage[]).map((operation) => operation.op);\n            } else {\n                messages = deltaStorageResponse.value as ISequencedDocumentMessage[];\n            }\n\n            this.logger.sendPerformanceEvent({\n                eventName: \"OpsFetch\",\n                headers: Object.keys(headers).length !== 0 ? true : undefined,\n                length: messages.length,\n                duration: response.duration, // this duration for single attempt!\n                ...response.propsToLog,\n                attempts: options.refresh ? 2 : 1,\n                from,\n                to,\n                ...telemetryProps,\n                reason: scenarioName,\n            });\n\n            // It is assumed that server always returns all the ops that it has in the range that was requested.\n            // This may change in the future, if so, we need to adjust and receive \"end\" value from server in such case.\n            return { messages, partialResult: false };\n        });\n    }\n\n    public buildUrl(from: number, to: number) {\n        const filter = encodeURIComponent(`sequenceNumber ge ${from} and sequenceNumber le ${to - 1}`);\n        const queryString = `?ump=1&filter=${filter}`;\n        return `${this.deltaFeedUrl}${queryString}`;\n    }\n}\n\nexport class OdspDeltaStorageWithCache implements IDocumentDeltaStorageService {\n    private firstCacheMiss = Number.MAX_SAFE_INTEGER;\n\n    public constructor(\n        private snapshotOps: ISequencedDocumentMessage[] | undefined,\n        private readonly logger: ITelemetryLogger,\n        private readonly batchSize: number,\n        private readonly concurrency: number,\n        private readonly getFromStorage: (\n            from: number,\n            to: number,\n            telemetryProps: ITelemetryProperties,\n            fetchReason?: string) => Promise<IDeltasFetchResult>,\n        private readonly getCached: (from: number, to: number) => Promise<ISequencedDocumentMessage[]>,\n        private readonly requestFromSocket: (from: number, to: number) => void,\n        private readonly opsReceived: (ops: ISequencedDocumentMessage[]) => void,\n    ) {\n    }\n\n    protected validateMessages(reason: string, messages: ISequencedDocumentMessage[], from: number) {\n        if (messages.length !== 0) {\n            const start = messages[0].sequenceNumber;\n            const length = messages.length;\n            const last = messages[length - 1].sequenceNumber;\n            if (start !== from) {\n                this.logger.sendErrorEvent({ eventName: \"OpsFetchViolation\", reason, from, start, last, length });\n                messages.length = 0;\n            }\n            if (last + 1 !== from + length) {\n                this.logger.sendErrorEvent({ eventName: \"OpsFetchViolation\", reason, from, start, last, length });\n                // we can do better here by finding consecutive sub-block and return it\n                messages.length = 0;\n            }\n        }\n    }\n\n    public fetchMessages(\n        fromTotal: number,\n        toTotal: number | undefined,\n        abortSignal?: AbortSignal,\n        cachedOnly?: boolean,\n        fetchReason?: string) {\n        // We do not control what's in the cache. Current API assumes that fetchMessages() keeps banging on\n        // storage / cache until it gets ops it needs. This would result in deadlock if fixed range is asked from\n        // cache and it's not there.\n        // Better implementation would be to return only what we have in cache, but that also breaks API\n        assert(!cachedOnly || toTotal === undefined, 0x1e3);\n\n        let opsFromSnapshot = 0;\n        let opsFromCache = 0;\n        let opsFromStorage = 0;\n\n        const requestCallback = async (from: number, to: number, telemetryProps: ITelemetryProperties) => {\n            if (this.snapshotOps !== undefined && this.snapshotOps.length !== 0) {\n                const messages = this.snapshotOps.filter((op) =>\n                    op.sequenceNumber >= from && op.sequenceNumber < to);\n                this.validateMessages(\"cached\", messages, from);\n                if (messages.length > 0 && messages[0].sequenceNumber === from) {\n                    this.snapshotOps = this.snapshotOps.filter((op) => op.sequenceNumber >= to);\n                    opsFromSnapshot = messages.length;\n                    return { messages, partialResult: true };\n                }\n                this.snapshotOps = undefined;\n            }\n\n            // Kick out request to PUSH for ops if it has them\n            this.requestFromSocket(from, to);\n\n            // Cache in normal flow is continuous. Once there is a miss, stop consulting cache.\n            // This saves a bit of processing time\n            if (from < this.firstCacheMiss) {\n                const messagesFromCache = await this.getCached(from, to);\n                this.validateMessages(\"cached\", messagesFromCache, from);\n                if (messagesFromCache.length !== 0) {\n                    opsFromCache += messagesFromCache.length;\n                    return {\n                        messages: messagesFromCache,\n                        partialResult: true,\n                    };\n                }\n                this.firstCacheMiss = Math.min(this.firstCacheMiss, from);\n            }\n\n            if (cachedOnly) {\n                return { messages: [], partialResult: false };\n            }\n\n            const ops = await this.getFromStorage(from, to, telemetryProps, fetchReason);\n            this.validateMessages(\"storage\", ops.messages, from);\n            opsFromStorage += ops.messages.length;\n            this.opsReceived(ops.messages);\n            return ops;\n        };\n\n        const stream = requestOps(\n            async (from: number, to: number, telemetryProps: ITelemetryProperties) => {\n                const result = await requestCallback(from, to, telemetryProps);\n                // Catch all case, just in case\n                this.validateMessages(\"catch all\", result.messages, from);\n                return result;\n            },\n            // Staging: starting with no concurrency, listening for feedback first.\n            // In future releases we will switch to actual concurrency\n            this.concurrency,\n            fromTotal, // inclusive\n            toTotal, // exclusive\n            this.batchSize,\n            this.logger,\n            abortSignal,\n            fetchReason,\n        );\n\n        return streamObserver(stream, (result) => {\n            if (result.done && opsFromSnapshot + opsFromCache + opsFromStorage !== 0) {\n                this.logger.sendPerformanceEvent({\n                    eventName: \"CacheOpsRetrieved\",\n                    opsFromSnapshot,\n                    opsFromCache,\n                    opsFromStorage,\n                });\n            }\n        });\n    }\n}\n"]}

package/lib/packageVersion.d.ts

@@ -5,5 +5,5 @@
  * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY
  */
 export declare const pkgName = "@fluidframework/odsp-driver";
-export declare const pkgVersion = "1.2.6";
+export declare const pkgVersion = "1.3.0-100520";
 //# sourceMappingURL=packageVersion.d.ts.map

package/lib/packageVersion.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"packageVersion.d.ts","sourceRoot":"","sources":["../src/packageVersion.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,eAAO,MAAM,OAAO,gCAAgC,CAAC;AACrD,eAAO,MAAM,UAAU,UAAU,CAAC"}
+{"version":3,"file":"packageVersion.d.ts","sourceRoot":"","sources":["../src/packageVersion.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,eAAO,MAAM,OAAO,gCAAgC,CAAC;AACrD,eAAO,MAAM,UAAU,iBAAiB,CAAC"}

package/lib/packageVersion.js

@@ -5,5 +5,5 @@
  * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY
  */
 export const pkgName = "@fluidframework/odsp-driver";
-export const pkgVersion = "1.2.6";
+export const pkgVersion = "1.3.0-100520";
 //# sourceMappingURL=packageVersion.js.map

package/lib/packageVersion.js.map

@@ -1 +1 @@
-{"version":3,"file":"packageVersion.js","sourceRoot":"","sources":["../src/packageVersion.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,CAAC,MAAM,OAAO,GAAG,6BAA6B,CAAC;AACrD,MAAM,CAAC,MAAM,UAAU,GAAG,OAAO,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n *\n * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY\n */\n\nexport const pkgName = \"@fluidframework/odsp-driver\";\nexport const pkgVersion = \"1.2.6\";\n"]}
+{"version":3,"file":"packageVersion.js","sourceRoot":"","sources":["../src/packageVersion.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,CAAC,MAAM,OAAO,GAAG,6BAA6B,CAAC;AACrD,MAAM,CAAC,MAAM,UAAU,GAAG,cAAc,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n *\n * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY\n */\n\nexport const pkgName = \"@fluidframework/odsp-driver\";\nexport const pkgVersion = \"1.3.0-100520\";\n"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fluidframework/odsp-driver",
-  "version": "1.2.6",
+  "version": "1.3.0-100520",
   "description": "Socket storage implementation for SPO and ODC",
   "homepage": "https://fluidframework.com",
   "repository": {
@@ -62,16 +62,16 @@
   "dependencies": {
     "@fluidframework/common-definitions": "^0.20.1",
     "@fluidframework/common-utils": "^0.32.1",
-    "@fluidframework/core-interfaces": "^1.2.6",
-    "@fluidframework/driver-base": "^1.2.6",
-    "@fluidframework/driver-definitions": "^1.2.6",
-    "@fluidframework/driver-utils": "^1.2.6",
+    "@fluidframework/core-interfaces": "1.3.0-100520",
+    "@fluidframework/driver-base": "1.3.0-100520",
+    "@fluidframework/driver-definitions": "1.3.0-100520",
+    "@fluidframework/driver-utils": "1.3.0-100520",
     "@fluidframework/gitresources": "^0.1036.5000",
-    "@fluidframework/odsp-doclib-utils": "^1.2.6",
-    "@fluidframework/odsp-driver-definitions": "^1.2.6",
+    "@fluidframework/odsp-doclib-utils": "1.3.0-100520",
+    "@fluidframework/odsp-driver-definitions": "1.3.0-100520",
     "@fluidframework/protocol-base": "^0.1036.5000",
     "@fluidframework/protocol-definitions": "^0.1028.2000",
-    "@fluidframework/telemetry-utils": "^1.2.6",
+    "@fluidframework/telemetry-utils": "1.3.0-100520",
     "abort-controller": "^3.0.0",
     "node-fetch": "^2.6.1",
     "socket.io-client": "^4.4.1",
@@ -81,8 +81,8 @@
     "@fluidframework/build-common": "^0.24.0",
     "@fluidframework/build-tools": "^0.2.74327",
     "@fluidframework/eslint-config-fluid": "^0.28.2000",
-    "@fluidframework/mocha-test-setup": "^1.2.6",
-    "@fluidframework/odsp-driver-previous": "npm:@fluidframework/odsp-driver@1.2.1",
+    "@fluidframework/mocha-test-setup": "1.3.0-100520",
+    "@fluidframework/odsp-driver-previous": "npm:@fluidframework/odsp-driver@^1.2.0",
     "@microsoft/api-extractor": "^7.22.2",
     "@rushstack/eslint-config": "^2.5.1",
     "@types/mocha": "^9.1.1",
@@ -100,7 +100,7 @@
     "typescript-formatter": "7.1.0"
   },
   "typeValidation": {
-    "version": "1.2.2",
+    "version": "1.3.0",
     "broken": {}
   }
 }

package/src/compactSnapshotParser.ts

@@ -81,11 +81,9 @@ function readTreeSection(node: NodeCore) {
         if (records.value !== undefined) {
             assertBlobCoreInstance(records.value, "Blob value should be BlobCore");
             snapshotTree.blobs[path] = records.value.toString();
-        } else if (records.children !== undefined) {
+        } else {
             assertNodeCoreInstance(records.children, "Trees should be of type NodeCore");
             snapshotTree.trees[path] = readTreeSection(records.children);
-        } else {
-            snapshotTree.trees[path] = { blobs: {}, commits: {}, trees: {} };
         }
         if (records.unreferenced !== undefined) {
             assertBoolInstance(records.unreferenced, "Unreferenced flag should be bool");

package/src/compactSnapshotWriter.ts

@@ -60,12 +60,9 @@ function writeTreeSectionCore(treesNode: NodeCore, snapshotTree: ISnapshotTree)
         if (snapshotTree.unreferenced) {
             addBoolProperty(treeNode, "unreferenced", snapshotTree.unreferenced);
         }
-        // Only write children prop if either blobs or trees are present.
-        if (Object.keys(value.blobs).length > 0 || Object.keys(value.trees).length > 0) {
-            treeNode.addString("children", true);
-            const childNode = treeNode.addNode("list");
-            writeTreeSectionCore(childNode, value);
-        }
+        treeNode.addString("children", true);
+        const childNode = treeNode.addNode("list");
+        writeTreeSectionCore(childNode, value);
     }
 
     if (snapshotTree.blobs) {

package/src/fetchSnapshot.ts

@@ -16,7 +16,7 @@ import {
     InstrumentedStorageTokenFetcher,
 } from "@fluidframework/odsp-driver-definitions";
 import { ISnapshotTree } from "@fluidframework/protocol-definitions";
-import { isRuntimeMessage, NonRetryableError } from "@fluidframework/driver-utils";
+import { DriverErrorTelemetryProps, isRuntimeMessage, NonRetryableError } from "@fluidframework/driver-utils";
 import { IOdspSnapshot, ISnapshotCachedEntry, IVersionedValueWithEpoch, persistedCacheValueVersion } from "./contracts";
 import { getQueryString } from "./getQueryString";
 import { getUrlAndHeadersWithAuth } from "./getUrlAndHeadersWithAuth";
@@ -234,26 +234,30 @@ async function fetchLatestSnapshotCore(
                     snapshotOptions,
                     controller,
                 );
+
                 const odspResponse = response.odspResponse;
                 const contentType = odspResponse.headers.get("content-type");
-                odspResponse.propsToLog = {
+
+                // Measure how much time we spend processing payload
+                const snapshotParseEvent = PerformanceEvent.start(logger, {
+                    eventName: "SnapshotParse",
+                    driverVersion: pkgVersion,
+                    contentType,
+                });
+
+                const propsToLog: DriverErrorTelemetryProps = {
                     ...odspResponse.propsToLog,
                     contentType,
                     accept: response.requestHeaders.accept,
+                    driverVersion: pkgVersion,
                 };
                 let parsedSnapshotContents: IOdspResponse<ISnapshotContents> | undefined;
-                let contentTypeToRead: string | undefined;
-                if (contentType?.indexOf("application/ms-fluid") !== -1) {
-                    contentTypeToRead = "application/ms-fluid";
-                } else if (contentType?.indexOf("application/json") !== -1) {
-                    contentTypeToRead = "application/json";
-                }
 
                 try {
-                    switch (contentTypeToRead) {
+                    switch (contentType) {
                         case "application/json": {
                             const text = await odspResponse.content.text();
-                            odspResponse.propsToLog.bodySize = text.length;
+                            propsToLog.bodySize = text.length;
                             const content: IOdspSnapshot = JSON.parse(text);
                             validateBlobsAndTrees(content);
                             const snapshotContents: ISnapshotContents =
@@ -263,7 +267,7 @@ async function fetchLatestSnapshotCore(
                         }
                         case "application/ms-fluid": {
                             const content = await odspResponse.content.arrayBuffer();
-                            odspResponse.propsToLog.bodySize = content.byteLength;
+                            propsToLog.bodySize = content.byteLength;
                             const snapshotContents: ISnapshotContents = parseCompactSnapshotResponse(
                                 new ReadBuffer(new Uint8Array(content)));
                             if (snapshotContents.snapshotTree.trees === undefined ||
@@ -271,7 +275,7 @@ async function fetchLatestSnapshotCore(
                                     throw new NonRetryableError(
                                         "Returned odsp snapshot is malformed. No trees or blobs!",
                                         DriverErrorType.incorrectServerResponse,
-                                        { driverVersion: pkgVersion, ...odspResponse.propsToLog },
+                                        propsToLog,
                                     );
                                 }
                             parsedSnapshotContents = { ...odspResponse, content: snapshotContents };
@@ -281,12 +285,12 @@ async function fetchLatestSnapshotCore(
                             throw new NonRetryableError(
                                 "Unknown snapshot content type",
                                 DriverErrorType.incorrectServerResponse,
-                                { driverVersion: pkgVersion, ...odspResponse.propsToLog },
+                                propsToLog,
                             );
                     }
                 } catch (error) {
                     if (isFluidError(error)) {
-                        error.addTelemetryProperties({ driverVersion: pkgVersion, ...odspResponse.propsToLog });
+                        error.addTelemetryProperties(propsToLog);
                         throw error;
                     }
                     const enhancedError = wrapError(
@@ -294,11 +298,14 @@ async function fetchLatestSnapshotCore(
                         (errorMessage) => new NonRetryableError(
                             `Error parsing snapshot response: ${errorMessage}`,
                             DriverErrorType.genericError,
-                            { driverVersion: pkgVersion, ...odspResponse.propsToLog }));
+                            propsToLog));
                     throw enhancedError;
                 }
+
                 assert(parsedSnapshotContents !== undefined, 0x312 /* snapshot should be parsed */);
                 const snapshot = parsedSnapshotContents.content;
+                const { trees, numBlobs, encodedBlobsSize } = evalBlobsAndTrees(snapshot);
+
                 // From: https://developer.mozilla.org/en-US/docs/Web/API/PerformanceResourceTiming
                 // fetchStart: immediately before the browser starts to fetch the resource.
                 // requestStart: immediately before the browser starts requesting the resource from the server
@@ -351,9 +358,6 @@ async function fetchLatestSnapshotCore(
                     }
                 }
 
-                const { numTrees, numBlobs, encodedBlobsSize } =
-                    evalBlobsAndTrees(parsedSnapshotContents.content);
-
                 // There are some scenarios in ODSP where we cannot cache, trees/latest will explicitly tell us when we
                 // cannot cache using an HTTP response header.
                 const canCache =
@@ -382,8 +386,11 @@ async function fetchLatestSnapshotCore(
                     // eslint-disable-next-line @typescript-eslint/no-floating-promises
                     putInCache(valueWithEpoch);
                 }
+
+                snapshotParseEvent.end();
+
                 event.end({
-                    trees: numTrees,
+                    trees,
                     blobs: snapshot.blobs?.size ?? 0,
                     leafNodes: numBlobs,
                     encodedBlobsSize,
@@ -413,7 +420,7 @@ async function fetchLatestSnapshotCore(
                     // Azure Fluid Relay service is the redeem status (S means success), and FRP is a flag to indicate
                     // if the permission has changed.
                     sltelemetry: odspResponse.headers.get("x-fluid-sltelemetry"),
-                    ...odspResponse.propsToLog,
+                    ...propsToLog,
                 });
                 return snapshot;
             },
@@ -473,13 +480,13 @@ function getFormBodyAndHeaders(
 }
 
 function evalBlobsAndTrees(snapshot: ISnapshotContents) {
-    const numTrees = countTreesInSnapshotTree(snapshot.snapshotTree);
+    const trees = countTreesInSnapshotTree(snapshot.snapshotTree);
     const numBlobs = snapshot.blobs.size;
     let encodedBlobsSize = 0;
     for (const [_, blobContent] of snapshot.blobs) {
         encodedBlobsSize += blobContent.byteLength;
     }
-    return { numTrees, numBlobs, encodedBlobsSize };
+    return { trees, numBlobs, encodedBlobsSize };
 }
 
 export function validateBlobsAndTrees(snapshot: IOdspSnapshot) {
@@ -542,12 +549,14 @@ export async function downloadSnapshot(
     };
     // Decide what snapshot format to fetch as per the feature gate.
     switch (snapshotFormatFetchType) {
+        case SnapshotFormatSupportType.JsonAndBinary:
+            headers.accept = `application/json, application/ms-fluid; v=${currentReadVersion}`;
+            break;
         case SnapshotFormatSupportType.Binary:
             headers.accept = `application/ms-fluid; v=${currentReadVersion}`;
             break;
         default:
-            // By default ask both versions and let the server decide the format.
-            headers.accept = `application/json, application/ms-fluid; v=${currentReadVersion}`;
+            headers.accept = "application/json";
     }
 
     const odspResponse = await (epochTracker?.fetch(url, fetchOptions, "treesLatest", true, scenarioName) ??

package/src/getFileLink.ts

@@ -4,8 +4,8 @@
  */
 
 import { ITelemetryLogger } from "@fluidframework/common-definitions";
-import { assert, delay } from "@fluidframework/common-utils";
-import { canRetryOnError, getRetryDelayFromError, NonRetryableError } from "@fluidframework/driver-utils";
+import { assert } from "@fluidframework/common-utils";
+import { canRetryOnError, NonRetryableError } from "@fluidframework/driver-utils";
 import { PerformanceEvent } from "@fluidframework/telemetry-utils";
 import { DriverErrorType } from "@fluidframework/driver-definitions";
 import {
@@ -17,6 +17,7 @@ import {
 import { getUrlAndHeadersWithAuth } from "./getUrlAndHeadersWithAuth";
 import { fetchHelper, getWithRetryForTokenRefresh, toInstrumentedOdspTokenFetcher } from "./odspUtils";
 import { pkgVersion as driverVersion } from "./packageVersion";
+import { runWithRetry } from "./retryUtils";
 
 // Store cached responses for the lifetime of web session as file link remains the same for given file item
 const fileLinkCache = new Map<string, Promise<string>>();
@@ -47,33 +48,29 @@ export async function getFileLink(
         return maybeFileLinkCacheEntry;
     }
 
-    const valueGenerator = async function() {
-        let result: string | undefined;
-        let success = false;
-        let retryAfterMs = 1000;
-        do {
-            try {
-                result = await getFileLinkCore(getToken, odspUrlParts, identityType, logger);
-                success = true;
-            } catch (err) {
-                // If it is not retriable, then just throw
-                if (!canRetryOnError(err)) {
-                    // Delete from the cache to permit retrying later.
-                    fileLinkCache.delete(cacheKey);
-                    throw err;
-                }
-                // If the error is throttling error, then wait for the specified time before retrying.
-                // If the waitTime is not specified, then we start with retrying immediately to max of 8s.
-                retryAfterMs = getRetryDelayFromError(err) ?? Math.min(retryAfterMs * 2, 8000);
-                await delay(retryAfterMs);
+    const fileLinkGenerator = async function() {
+        let fileLinkCore: string;
+        try {
+            fileLinkCore = await runWithRetry(
+                async () => getFileLinkCore(getToken, odspUrlParts, identityType, logger),
+                "getFileLinkCore",
+                logger,
+            );
+        } catch (err) {
+            // runWithRetry throws a non retriable error after it hits the max # of attempts
+            // or encounters an unexpected error type
+            if (!canRetryOnError(err)) {
+                // Delete from the cache to permit retrying later.
+                fileLinkCache.delete(cacheKey);
             }
-        } while (!success);
+            throw err;
+        }
 
         // We are guaranteed to run the getFileLinkCore at least once with successful result (which must be a string)
-        assert(result !== undefined, 0x292 /* "Unexpected undefined result from getFileLinkCore" */);
-        return result;
+        assert(fileLinkCore !== undefined, 0x292 /* "Unexpected undefined result from getFileLinkCore" */);
+        return fileLinkCore;
     };
-    const fileLink = valueGenerator();
+    const fileLink = fileLinkGenerator();
     fileLinkCache.set(cacheKey, fileLink);
     return fileLink;
 }
@@ -89,7 +86,7 @@ async function getFileLinkCore(
     // ODSP link requires extra call to return link that is resistant to file being renamed or moved to different folder
     return PerformanceEvent.timedExecAsync(
         logger,
-        { eventName: "odspFileLink", requestName: "getSharingInformation" },
+        { eventName: "odspFileLink", requestName: "getSharingLink" },
         async (event) => {
             let attempts = 0;
             let additionalProps;
@@ -106,8 +103,8 @@ async function getFileLinkCore(
                     0x2bb /* "Instrumented token fetcher with throwOnNullToken = true should never return null" */);
 
                 const { url, headers } = getUrlAndHeadersWithAuth(
-                    `${odspUrlParts.siteUrl}/_api/web/GetFileByUrl(@a1)/ListItemAllFields/GetSharingInformation?@a1=${
-                        encodeURIComponent(`'${fileItem.webDavUrl}'`)
+                    `${odspUrlParts.siteUrl}/_api/web/GetFileByServerRelativeUrl(@a1)/Linkingurl?@a1=${
+                        encodeURIComponent(`'${new URL(fileItem.webDavUrl).pathname}'`)
                     }`,
                     storageToken,
                     false,
@@ -124,15 +121,15 @@ async function getFileLinkCore(
                 additionalProps = response.propsToLog;
 
                 const sharingInfo = await response.content.json();
-                const directUrl = sharingInfo?.d?.directUrl;
-                if (typeof directUrl !== "string") {
+                const linkingUrl = sharingInfo?.d?.LinkingUrl;
+                if (typeof linkingUrl !== "string") {
                     // This will retry once in getWithRetryForTokenRefresh
                     throw new NonRetryableError(
-                        "Malformed GetSharingInformation response",
+                        "Malformed GetSharingLink response",
                         DriverErrorType.incorrectServerResponse,
                         { driverVersion });
                 }
-                return directUrl;
+                return linkingUrl;
             });
             event.end({ ...additionalProps, attempts });
             return fileLink;

package/src/odspDeltaStorageService.ts

@@ -35,13 +35,14 @@ export class OdspDeltaStorageService {
      * @param from - inclusive
      * @param to - exclusive
      * @param telemetryProps - properties to add when issuing telemetry events
+     * @param scenarioName - reason for fetching ops
      * @returns ops retrieved & info if result was partial (i.e. more is available)
      */
      public async get(
         from: number,
         to: number,
         telemetryProps: ITelemetryProperties,
-        fetchReason?: string,
+        scenarioName?: string,
     ): Promise<IDeltasFetchResult> {
         return getWithRetryForTokenRefresh(async (options) => {
             // Note - this call ends up in getSocketStorageDiscovery() and can refresh token
@@ -78,7 +79,7 @@ export class OdspDeltaStorageService {
                 },
                 "ops",
                 true,
-                fetchReason,
+                scenarioName,
             );
             clearTimeout(timer);
             const deltaStorageResponse = response.content;
@@ -99,6 +100,7 @@ export class OdspDeltaStorageService {
                 from,
                 to,
                 ...telemetryProps,
+                reason: scenarioName,
             });
 
             // It is assumed that server always returns all the ops that it has in the range that was requested.

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/odsp-driver";
-export const pkgVersion = "1.2.6";
+export const pkgVersion = "1.3.0-100520";