@fluidframework/odsp-driver 1.2.7 → 2.0.0-dev.1.3.0.96595

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (254) hide show
  1. package/.mocharc.js +12 -0
  2. package/README.md +19 -0
  3. package/dist/ReadBufferUtils.d.ts.map +1 -1
  4. package/dist/ReadBufferUtils.js +1 -0
  5. package/dist/ReadBufferUtils.js.map +1 -1
  6. package/dist/WriteBufferUtils.d.ts +3 -5
  7. package/dist/WriteBufferUtils.d.ts.map +1 -1
  8. package/dist/WriteBufferUtils.js +59 -55
  9. package/dist/WriteBufferUtils.js.map +1 -1
  10. package/dist/compactSnapshotParser.d.ts +2 -2
  11. package/dist/compactSnapshotParser.d.ts.map +1 -1
  12. package/dist/compactSnapshotParser.js +91 -34
  13. package/dist/compactSnapshotParser.js.map +1 -1
  14. package/dist/compactSnapshotWriter.d.ts +1 -2
  15. package/dist/compactSnapshotWriter.d.ts.map +1 -1
  16. package/dist/compactSnapshotWriter.js +17 -14
  17. package/dist/compactSnapshotWriter.js.map +1 -1
  18. package/dist/contracts.d.ts +1 -18
  19. package/dist/contracts.d.ts.map +1 -1
  20. package/dist/contracts.js.map +1 -1
  21. package/dist/createFile.d.ts +1 -1
  22. package/dist/createFile.d.ts.map +1 -1
  23. package/dist/createFile.js +53 -16
  24. package/dist/createFile.js.map +1 -1
  25. package/dist/createNewUtils.d.ts.map +1 -1
  26. package/dist/createNewUtils.js +0 -1
  27. package/dist/createNewUtils.js.map +1 -1
  28. package/dist/createOdspCreateContainerRequest.d.ts +4 -3
  29. package/dist/createOdspCreateContainerRequest.d.ts.map +1 -1
  30. package/dist/createOdspCreateContainerRequest.js +6 -3
  31. package/dist/createOdspCreateContainerRequest.js.map +1 -1
  32. package/dist/epochTracker.d.ts +1 -0
  33. package/dist/epochTracker.d.ts.map +1 -1
  34. package/dist/epochTracker.js +24 -5
  35. package/dist/epochTracker.js.map +1 -1
  36. package/dist/fetchSnapshot.d.ts +1 -2
  37. package/dist/fetchSnapshot.d.ts.map +1 -1
  38. package/dist/fetchSnapshot.js +22 -27
  39. package/dist/fetchSnapshot.js.map +1 -1
  40. package/dist/getFileLink.d.ts +3 -6
  41. package/dist/getFileLink.d.ts.map +1 -1
  42. package/dist/getFileLink.js +22 -33
  43. package/dist/getFileLink.js.map +1 -1
  44. package/dist/index.d.ts +1 -2
  45. package/dist/index.d.ts.map +1 -1
  46. package/dist/index.js +3 -3
  47. package/dist/index.js.map +1 -1
  48. package/dist/localOdspDriver/localOdspDocumentStorageManager.d.ts.map +1 -1
  49. package/dist/localOdspDriver/localOdspDocumentStorageManager.js +1 -2
  50. package/dist/localOdspDriver/localOdspDocumentStorageManager.js.map +1 -1
  51. package/dist/odspDeltaStorageService.d.ts +3 -2
  52. package/dist/odspDeltaStorageService.d.ts.map +1 -1
  53. package/dist/odspDeltaStorageService.js +9 -12
  54. package/dist/odspDeltaStorageService.js.map +1 -1
  55. package/dist/odspDocumentDeltaConnection.d.ts.map +1 -1
  56. package/dist/odspDocumentDeltaConnection.js +3 -6
  57. package/dist/odspDocumentDeltaConnection.js.map +1 -1
  58. package/dist/odspDocumentService.d.ts +1 -0
  59. package/dist/odspDocumentService.d.ts.map +1 -1
  60. package/dist/odspDocumentService.js +12 -6
  61. package/dist/odspDocumentService.js.map +1 -1
  62. package/dist/odspDocumentServiceFactoryCore.d.ts.map +1 -1
  63. package/dist/odspDocumentServiceFactoryCore.js +29 -6
  64. package/dist/odspDocumentServiceFactoryCore.js.map +1 -1
  65. package/dist/odspDocumentStorageManager.d.ts +4 -4
  66. package/dist/odspDocumentStorageManager.d.ts.map +1 -1
  67. package/dist/odspDocumentStorageManager.js +82 -64
  68. package/dist/odspDocumentStorageManager.js.map +1 -1
  69. package/dist/odspDocumentStorageServiceBase.d.ts +1 -1
  70. package/dist/odspDocumentStorageServiceBase.d.ts.map +1 -1
  71. package/dist/odspDocumentStorageServiceBase.js +4 -2
  72. package/dist/odspDocumentStorageServiceBase.js.map +1 -1
  73. package/dist/odspDriverUrlResolverForShareLink.d.ts.map +1 -1
  74. package/dist/odspDriverUrlResolverForShareLink.js +4 -4
  75. package/dist/odspDriverUrlResolverForShareLink.js.map +1 -1
  76. package/dist/odspFluidFileLink.js +1 -1
  77. package/dist/odspFluidFileLink.js.map +1 -1
  78. package/dist/odspLocationRedirection.d.ts +14 -0
  79. package/dist/odspLocationRedirection.d.ts.map +1 -0
  80. package/dist/odspLocationRedirection.js +24 -0
  81. package/dist/odspLocationRedirection.js.map +1 -0
  82. package/dist/odspSnapshotParser.d.ts.map +1 -1
  83. package/dist/odspSnapshotParser.js +1 -2
  84. package/dist/odspSnapshotParser.js.map +1 -1
  85. package/dist/odspSummaryUploadManager.d.ts +6 -3
  86. package/dist/odspSummaryUploadManager.d.ts.map +1 -1
  87. package/dist/odspSummaryUploadManager.js +14 -17
  88. package/dist/odspSummaryUploadManager.js.map +1 -1
  89. package/dist/odspUrlHelper.js +2 -1
  90. package/dist/odspUrlHelper.js.map +1 -1
  91. package/dist/odspUtils.d.ts +11 -2
  92. package/dist/odspUtils.d.ts.map +1 -1
  93. package/dist/odspUtils.js +32 -5
  94. package/dist/odspUtils.js.map +1 -1
  95. package/dist/opsCaching.d.ts.map +1 -1
  96. package/dist/opsCaching.js +3 -2
  97. package/dist/opsCaching.js.map +1 -1
  98. package/dist/packageVersion.d.ts +1 -1
  99. package/dist/packageVersion.d.ts.map +1 -1
  100. package/dist/packageVersion.js +1 -1
  101. package/dist/packageVersion.js.map +1 -1
  102. package/dist/prefetchLatestSnapshot.d.ts +6 -4
  103. package/dist/prefetchLatestSnapshot.d.ts.map +1 -1
  104. package/dist/prefetchLatestSnapshot.js +6 -4
  105. package/dist/prefetchLatestSnapshot.js.map +1 -1
  106. package/dist/retryUtils.d.ts.map +1 -1
  107. package/dist/retryUtils.js +8 -4
  108. package/dist/retryUtils.js.map +1 -1
  109. package/dist/zipItDataRepresentationUtils.d.ts +30 -18
  110. package/dist/zipItDataRepresentationUtils.d.ts.map +1 -1
  111. package/dist/zipItDataRepresentationUtils.js +170 -76
  112. package/dist/zipItDataRepresentationUtils.js.map +1 -1
  113. package/lib/ReadBufferUtils.d.ts.map +1 -1
  114. package/lib/ReadBufferUtils.js +1 -0
  115. package/lib/ReadBufferUtils.js.map +1 -1
  116. package/lib/WriteBufferUtils.d.ts +3 -5
  117. package/lib/WriteBufferUtils.d.ts.map +1 -1
  118. package/lib/WriteBufferUtils.js +61 -57
  119. package/lib/WriteBufferUtils.js.map +1 -1
  120. package/lib/compactSnapshotParser.d.ts +2 -2
  121. package/lib/compactSnapshotParser.d.ts.map +1 -1
  122. package/lib/compactSnapshotParser.js +92 -35
  123. package/lib/compactSnapshotParser.js.map +1 -1
  124. package/lib/compactSnapshotWriter.d.ts +1 -2
  125. package/lib/compactSnapshotWriter.d.ts.map +1 -1
  126. package/lib/compactSnapshotWriter.js +18 -15
  127. package/lib/compactSnapshotWriter.js.map +1 -1
  128. package/lib/contracts.d.ts +1 -18
  129. package/lib/contracts.d.ts.map +1 -1
  130. package/lib/contracts.js.map +1 -1
  131. package/lib/createFile.d.ts +1 -1
  132. package/lib/createFile.d.ts.map +1 -1
  133. package/lib/createFile.js +54 -17
  134. package/lib/createFile.js.map +1 -1
  135. package/lib/createNewUtils.d.ts.map +1 -1
  136. package/lib/createNewUtils.js +0 -1
  137. package/lib/createNewUtils.js.map +1 -1
  138. package/lib/createOdspCreateContainerRequest.d.ts +4 -3
  139. package/lib/createOdspCreateContainerRequest.d.ts.map +1 -1
  140. package/lib/createOdspCreateContainerRequest.js +6 -3
  141. package/lib/createOdspCreateContainerRequest.js.map +1 -1
  142. package/lib/epochTracker.d.ts +1 -0
  143. package/lib/epochTracker.d.ts.map +1 -1
  144. package/lib/epochTracker.js +26 -7
  145. package/lib/epochTracker.js.map +1 -1
  146. package/lib/fetchSnapshot.d.ts +1 -2
  147. package/lib/fetchSnapshot.d.ts.map +1 -1
  148. package/lib/fetchSnapshot.js +22 -27
  149. package/lib/fetchSnapshot.js.map +1 -1
  150. package/lib/getFileLink.d.ts +3 -6
  151. package/lib/getFileLink.d.ts.map +1 -1
  152. package/lib/getFileLink.js +24 -35
  153. package/lib/getFileLink.js.map +1 -1
  154. package/lib/index.d.ts +1 -2
  155. package/lib/index.d.ts.map +1 -1
  156. package/lib/index.js +1 -3
  157. package/lib/index.js.map +1 -1
  158. package/lib/localOdspDriver/localOdspDocumentStorageManager.d.ts.map +1 -1
  159. package/lib/localOdspDriver/localOdspDocumentStorageManager.js +1 -2
  160. package/lib/localOdspDriver/localOdspDocumentStorageManager.js.map +1 -1
  161. package/lib/odspDeltaStorageService.d.ts +3 -2
  162. package/lib/odspDeltaStorageService.d.ts.map +1 -1
  163. package/lib/odspDeltaStorageService.js +9 -12
  164. package/lib/odspDeltaStorageService.js.map +1 -1
  165. package/lib/odspDocumentDeltaConnection.d.ts.map +1 -1
  166. package/lib/odspDocumentDeltaConnection.js +3 -6
  167. package/lib/odspDocumentDeltaConnection.js.map +1 -1
  168. package/lib/odspDocumentService.d.ts +1 -0
  169. package/lib/odspDocumentService.d.ts.map +1 -1
  170. package/lib/odspDocumentService.js +14 -8
  171. package/lib/odspDocumentService.js.map +1 -1
  172. package/lib/odspDocumentServiceFactoryCore.d.ts.map +1 -1
  173. package/lib/odspDocumentServiceFactoryCore.js +29 -6
  174. package/lib/odspDocumentServiceFactoryCore.js.map +1 -1
  175. package/lib/odspDocumentStorageManager.d.ts +4 -4
  176. package/lib/odspDocumentStorageManager.d.ts.map +1 -1
  177. package/lib/odspDocumentStorageManager.js +83 -65
  178. package/lib/odspDocumentStorageManager.js.map +1 -1
  179. package/lib/odspDocumentStorageServiceBase.d.ts +1 -1
  180. package/lib/odspDocumentStorageServiceBase.d.ts.map +1 -1
  181. package/lib/odspDocumentStorageServiceBase.js +4 -2
  182. package/lib/odspDocumentStorageServiceBase.js.map +1 -1
  183. package/lib/odspDriverUrlResolverForShareLink.d.ts.map +1 -1
  184. package/lib/odspDriverUrlResolverForShareLink.js +4 -4
  185. package/lib/odspDriverUrlResolverForShareLink.js.map +1 -1
  186. package/lib/odspFluidFileLink.js +1 -1
  187. package/lib/odspFluidFileLink.js.map +1 -1
  188. package/lib/odspLocationRedirection.d.ts +14 -0
  189. package/lib/odspLocationRedirection.d.ts.map +1 -0
  190. package/lib/odspLocationRedirection.js +20 -0
  191. package/lib/odspLocationRedirection.js.map +1 -0
  192. package/lib/odspSnapshotParser.d.ts.map +1 -1
  193. package/lib/odspSnapshotParser.js +1 -2
  194. package/lib/odspSnapshotParser.js.map +1 -1
  195. package/lib/odspSummaryUploadManager.d.ts +6 -3
  196. package/lib/odspSummaryUploadManager.d.ts.map +1 -1
  197. package/lib/odspSummaryUploadManager.js +14 -17
  198. package/lib/odspSummaryUploadManager.js.map +1 -1
  199. package/lib/odspUrlHelper.js +2 -1
  200. package/lib/odspUrlHelper.js.map +1 -1
  201. package/lib/odspUtils.d.ts +11 -2
  202. package/lib/odspUtils.d.ts.map +1 -1
  203. package/lib/odspUtils.js +31 -5
  204. package/lib/odspUtils.js.map +1 -1
  205. package/lib/opsCaching.d.ts.map +1 -1
  206. package/lib/opsCaching.js +3 -2
  207. package/lib/opsCaching.js.map +1 -1
  208. package/lib/packageVersion.d.ts +1 -1
  209. package/lib/packageVersion.d.ts.map +1 -1
  210. package/lib/packageVersion.js +1 -1
  211. package/lib/packageVersion.js.map +1 -1
  212. package/lib/prefetchLatestSnapshot.d.ts +6 -4
  213. package/lib/prefetchLatestSnapshot.d.ts.map +1 -1
  214. package/lib/prefetchLatestSnapshot.js +6 -4
  215. package/lib/prefetchLatestSnapshot.js.map +1 -1
  216. package/lib/retryUtils.d.ts.map +1 -1
  217. package/lib/retryUtils.js +9 -5
  218. package/lib/retryUtils.js.map +1 -1
  219. package/lib/zipItDataRepresentationUtils.d.ts +30 -18
  220. package/lib/zipItDataRepresentationUtils.d.ts.map +1 -1
  221. package/lib/zipItDataRepresentationUtils.js +166 -75
  222. package/lib/zipItDataRepresentationUtils.js.map +1 -1
  223. package/package.json +33 -20
  224. package/src/ReadBufferUtils.ts +1 -0
  225. package/src/WriteBufferUtils.ts +67 -58
  226. package/src/compactSnapshotParser.ts +102 -40
  227. package/src/compactSnapshotWriter.ts +25 -17
  228. package/src/contracts.ts +2 -14
  229. package/src/createFile.ts +75 -15
  230. package/src/createNewUtils.ts +2 -4
  231. package/src/createOdspCreateContainerRequest.ts +7 -4
  232. package/src/epochTracker.ts +47 -7
  233. package/src/fetchSnapshot.ts +35 -31
  234. package/src/getFileLink.ts +26 -39
  235. package/src/index.ts +1 -3
  236. package/src/localOdspDriver/localOdspDocumentStorageManager.ts +2 -2
  237. package/src/odspDeltaStorageService.ts +8 -9
  238. package/src/odspDocumentDeltaConnection.ts +3 -5
  239. package/src/odspDocumentService.ts +16 -13
  240. package/src/odspDocumentServiceFactoryCore.ts +40 -4
  241. package/src/odspDocumentStorageManager.ts +64 -50
  242. package/src/odspDocumentStorageServiceBase.ts +4 -2
  243. package/src/odspDriverUrlResolverForShareLink.ts +2 -5
  244. package/src/odspFluidFileLink.ts +1 -1
  245. package/src/odspLocationRedirection.ts +23 -0
  246. package/src/odspSnapshotParser.ts +3 -4
  247. package/src/odspSummaryUploadManager.ts +10 -11
  248. package/src/odspUrlHelper.ts +1 -1
  249. package/src/odspUtils.ts +40 -7
  250. package/src/opsCaching.ts +3 -2
  251. package/src/packageVersion.ts +1 -1
  252. package/src/prefetchLatestSnapshot.ts +6 -4
  253. package/src/retryUtils.ts +8 -5
  254. package/src/zipItDataRepresentationUtils.ts +198 -75
package/src/odspUtils.ts CHANGED
@@ -13,7 +13,7 @@ import {
13
13
  NetworkErrorBasic,
14
14
  } from "@fluidframework/driver-utils";
15
15
  import { assert, performance } from "@fluidframework/common-utils";
16
- import { ChildLogger, PerformanceEvent, wrapError } from "@fluidframework/telemetry-utils";
16
+ import { ChildLogger, PerformanceEvent, TelemetryDataTag, wrapError } from "@fluidframework/telemetry-utils";
17
17
  import {
18
18
  fetchIncorrectResponse,
19
19
  throwOdspNetworkError,
@@ -27,6 +27,7 @@ import {
27
27
  isTokenFromCache,
28
28
  OdspResourceTokenFetchOptions,
29
29
  ShareLinkTypes,
30
+ ISharingLinkKind,
30
31
  TokenFetcher,
31
32
  ICacheEntry,
32
33
  snapshotKey,
@@ -123,7 +124,8 @@ export async function fetchHelper(
123
124
  }, (error) => {
124
125
  const online = isOnline();
125
126
  const errorText = `${error}`;
126
-
127
+ const urlRegex = /((http|https):\/\/(\S*))/i;
128
+ const redactedErrorText = errorText.replace(urlRegex, "REDACTED_URL");
127
129
  // This error is thrown by fetch() when AbortSignal is provided and it gets cancelled
128
130
  if (error.name === "AbortError") {
129
131
  throw new RetryableError(
@@ -135,21 +137,30 @@ export async function fetchHelper(
135
137
  "Fetch Timeout (ETIMEDOUT)", OdspErrorType.fetchTimeout, { driverVersion });
136
138
  }
137
139
 
138
- //
139
140
  // WARNING: Do not log error object itself or any of its properties!
140
141
  // It could contain PII, like URI in message itself, or token in properties.
141
142
  // It is also non-serializable object due to circular references.
142
- //
143
+ // eslint-disable-next-line unicorn/prefer-ternary
143
144
  if (online === OnlineStatus.Offline) {
144
145
  throw new RetryableError(
145
146
  // pre-0.58 error message prefix: Offline
146
- `ODSP fetch failure (Offline): ${errorText}`, DriverErrorType.offlineError, { driverVersion });
147
+ `ODSP fetch failure (Offline): ${redactedErrorText}`,
148
+ DriverErrorType.offlineError,
149
+ {
150
+ driverVersion,
151
+ rawErrorMessage: { value: errorText, tag: TelemetryDataTag.UserData },
152
+ });
147
153
  } else {
148
154
  // It is perhaps still possible that this is due to being offline, the error does not reveal enough
149
155
  // information to conclude. Could also be DNS errors, malformed fetch request, CSP violation, etc.
150
156
  throw new RetryableError(
151
157
  // pre-0.58 error message prefix: Fetch error
152
- `ODSP fetch failure: ${errorText}`, DriverErrorType.fetchFailure, { driverVersion });
158
+ `ODSP fetch failure: ${redactedErrorText}`,
159
+ DriverErrorType.fetchFailure,
160
+ {
161
+ driverVersion,
162
+ rawErrorMessage: { value: errorText, tag: TelemetryDataTag.UserData },
163
+ });
153
164
  }
154
165
  });
155
166
  }
@@ -222,8 +233,10 @@ export interface INewFileInfo {
222
233
  * application can request creation of a share link along with the creation of a new file
223
234
  * by passing in an optional param to specify the kind of sharing link
224
235
  * (at the time of adding this comment Sept/2021), odsp only supports csl
236
+ * ShareLinkTypes will deprecated in future. Use ISharingLinkKind instead which specifies both
237
+ * share link type and the role type.
225
238
  */
226
- createLinkType?: ShareLinkTypes;
239
+ createLinkType?: ShareLinkTypes | ISharingLinkKind;
227
240
  }
228
241
 
229
242
  export function getOdspResolvedUrl(resolvedUrl: IResolvedUrl): IOdspResolvedUrl {
@@ -337,3 +350,23 @@ export function createCacheSnapshotKey(odspResolvedUrl: IOdspResolvedUrl): ICach
337
350
  // 80KB is the max body size that we can put in ump post body for server to be able to accept it.
338
351
  // Keeping it 78KB to be a little cautious. As per the telemetry 99p is less than 78KB.
339
352
  export const maxUmpPostBodySize = 79872;
353
+
354
+ /**
355
+ * Build request parameters to request for the creation of a sharing link along with the creation of the file
356
+ * through the /snapshot api call.
357
+ * @param shareLinkType - Kind of sharing link requested
358
+ * @returns A string of request parameters that can be concatenated with the base URI
359
+ */
360
+ export function buildOdspShareLinkReqParams(shareLinkType: ShareLinkTypes | ISharingLinkKind | undefined) {
361
+ if (!shareLinkType) {
362
+ return;
363
+ }
364
+ const scope = (shareLinkType as ISharingLinkKind).scope;
365
+ if (!scope) {
366
+ return `createLinkType=${shareLinkType}`;
367
+ }
368
+ let shareLinkRequestParams = `createLinkScope=${scope}`;
369
+ const role = (shareLinkType as ISharingLinkKind).role;
370
+ shareLinkRequestParams = role ? `${shareLinkRequestParams}&createLinkRole=${role}` : shareLinkRequestParams;
371
+ return shareLinkRequestParams;
372
+ }
package/src/opsCaching.ts CHANGED
@@ -40,8 +40,9 @@ export class OpsCache {
40
40
  private readonly timerGranularity,
41
41
  private totalOpsToCache,
42
42
  ) {
43
- /** initial batch is a special case because it will never be full - all ops prior (inclusive) to
44
- * startingSequenceNumber are never going to show up (undefined)
43
+ /**
44
+ * Initial batch is a special case because it will never be full - all ops prior (inclusive) to
45
+ * `startingSequenceNumber` are never going to show up (undefined)
45
46
  */
46
47
  const remainingSlots = this.batchSize - this.getPositionInBatchArray(startingSequenceNumber) - 1;
47
48
  if (remainingSlots !== 0) {
@@ -6,4 +6,4 @@
6
6
  */
7
7
 
8
8
  export const pkgName = "@fluidframework/odsp-driver";
9
- export const pkgVersion = "1.2.7";
9
+ export const pkgVersion = "2.0.0-dev.1.3.0.96595";
@@ -28,19 +28,21 @@ import { IVersionedValueWithEpoch } from "./contracts";
28
28
  /**
29
29
  * Function to prefetch the snapshot and cached it in the persistant cache, so that when the container is loaded
30
30
  * the cached latest snapshot could be used and removes the network call from the critical path.
31
+ *
31
32
  * @param resolvedUrl - Resolved url to fetch the snapshot.
32
33
  * @param getStorageToken - function that can provide the storage token for a given site. This is
33
- * is also referred to as the "VROOM" token in SPO.
34
+ * is also referred to as the "VROOM" token in SPO.
34
35
  * @param persistedCache - Cache to store the fetched snapshot.
35
36
  * @param forceAccessTokenViaAuthorizationHeader - whether to force passing given token via authorization header.
36
37
  * @param logger - Logger to have telemetry events.
37
38
  * @param hostSnapshotFetchOptions - Options to fetch the snapshot if any. Otherwise default will be used.
38
39
  * @param enableRedeemFallback - True to have the sharing link redeem fallback in case the Trees Latest/Redeem
39
- * 1RT call fails with redeem error. During fallback it will first redeem the sharing link and then make
40
- * the Trees latest call.
41
- * @deprecated - This will be replaced with snapshotFormatFetchType.
40
+ * 1RT call fails with redeem error. During fallback it will first redeem the sharing link and then make
41
+ * the Trees latest call.
42
+ * Note: this can be considered deprecated - it will be replaced with `snapshotFormatFetchType`.
42
43
  * @param fetchBinarySnapshotFormat - Control if we want to fetch binary format snapshot.
43
44
  * @param snapshotFormatFetchType - Snapshot format to fetch.
45
+ *
44
46
  * @returns - True if the snapshot is cached, false otherwise.
45
47
  */
46
48
  export async function prefetchLatestSnapshot(
package/src/retryUtils.ts CHANGED
@@ -5,7 +5,7 @@
5
5
 
6
6
  import { ITelemetryLogger } from "@fluidframework/common-definitions";
7
7
  import { delay, performance } from "@fluidframework/common-utils";
8
- import { canRetryOnError } from "@fluidframework/driver-utils";
8
+ import { canRetryOnError, getRetryDelayFromError } from "@fluidframework/driver-utils";
9
9
  import { OdspErrorType } from "@fluidframework/odsp-driver-definitions";
10
10
  import { Odsp409Error } from "./epochTracker";
11
11
 
@@ -43,8 +43,10 @@ export async function runWithRetry<T>(
43
43
 
44
44
  const coherencyError = error?.[Odsp409Error] === true;
45
45
  const serviceReadonlyError = error?.errorType === OdspErrorType.serviceReadOnly;
46
- // Retry for retriable 409 coherency errors or serviceReadOnly errors.
47
- if (!(coherencyError || serviceReadonlyError || canRetry)) {
46
+ // Retry for retriable 409 coherency errors or serviceReadOnly errors. These errors are always retriable
47
+ // unless someone specifically set canRetry = false on the error like in fetchSnapshot() flow. So in
48
+ // that case don't retry.
49
+ if (!((coherencyError || serviceReadonlyError) && canRetry)) {
48
50
  throw error;
49
51
  }
50
52
 
@@ -54,8 +56,8 @@ export async function runWithRetry<T>(
54
56
  if (attempts === 5) {
55
57
  logger.sendErrorEvent(
56
58
  {
57
- eventName: coherencyError ?
58
- "CoherencyErrorTooManyRetries" : "ServiceReadonlyErrorTooManyRetries",
59
+ eventName: coherencyError ? "CoherencyErrorTooManyRetries" :
60
+ "ServiceReadonlyErrorTooManyRetries",
59
61
  callName,
60
62
  attempts,
61
63
  duration: performance.now() - start, // record total wait time.
@@ -66,6 +68,7 @@ export async function runWithRetry<T>(
66
68
  throw error;
67
69
  }
68
70
 
71
+ retryAfter = getRetryDelayFromError(error) ?? retryAfter;
69
72
  await delay(Math.floor(retryAfter));
70
73
  retryAfter += retryAfter / 4 * (1 + Math.random());
71
74
  lastError = error;
@@ -8,7 +8,8 @@
8
8
  * https://microsoft.sharepoint-df.com/:w:/t/ODSPFileStore/ER06b64K_XdDjEyAKl-UT60BJiId39SCVkYSyo_2pvH9gQ?e=KYQ0c5
9
9
  */
10
10
 
11
- import { assert, IsoBuffer, Uint8ArrayToArrayBuffer, Uint8ArrayToString } from "@fluidframework/common-utils";
11
+ import { assert, Uint8ArrayToArrayBuffer, Uint8ArrayToString } from "@fluidframework/common-utils";
12
+ import { ITelemetryLogger } from "@fluidframework/common-definitions";
12
13
  import { NonRetryableError } from "@fluidframework/driver-utils";
13
14
  import { DriverErrorType } from "@fluidframework/driver-definitions";
14
15
  import { ReadBuffer } from "./ReadBufferUtils";
@@ -111,19 +112,11 @@ export function getValueSafely(map: { [index: number]: number; }, key: number) {
111
112
  return val;
112
113
  }
113
114
 
114
- export function getAndValidateNodeProps(node: NodeCore, props: string[], enforceAllProps = true) {
115
- const propSet = new Set(props);
115
+ export function getNodeProps(node: NodeCore) {
116
116
  const res: Record<string, NodeTypes> = {};
117
117
  for (const [keyNode, valueNode] of node.iteratePairs()) {
118
- assertBlobCoreInstance(keyNode, "keynode should be a blob");
119
- const keyStr = keyNode.toString();
120
- if (propSet.has(keyStr)) {
121
- propSet.delete(keyStr);
122
- res[keyStr] = valueNode;
123
- }
124
- }
125
- if (enforceAllProps) {
126
- assert(propSet.size === 0, 0x288 /* All properties should exist */);
118
+ const id = getStringInstance(keyNode, "keynode should be a string");
119
+ res[id] = valueNode;
127
120
  }
128
121
  return res;
129
122
  }
@@ -159,20 +152,12 @@ export function iterate<T>(obj: { [Symbol.iterator]: () => IterableIterator<T>;
159
152
  */
160
153
  export abstract class BlobCore {
161
154
  public abstract get buffer(): Uint8Array;
162
- public get arrayBuffer(): ArrayBufferLike {
163
- return Uint8ArrayToArrayBuffer(this.buffer);
164
- }
155
+ public abstract get arrayBuffer(): ArrayBufferLike;
165
156
 
166
157
  /**
167
158
  * Represents a blob.
168
- * @param constString - Whether it contains const string declaration.
169
- * @param useUtf8Code - Represents if the utf8 string marker code should be used when representing.
170
159
  */
171
- constructor(public readonly constString: boolean, public readonly useUtf8Code: boolean = false) {}
172
-
173
- public toString() {
174
- return Uint8ArrayToString(this.buffer, "utf-8");
175
- }
160
+ constructor() {}
176
161
  }
177
162
 
178
163
  /**
@@ -184,24 +169,26 @@ class BlobDeepCopy extends BlobCore {
184
169
  /**
185
170
  * Represents a deep copy of the blob.
186
171
  * @param data - Data array of the blob
187
- * @param constString - Whether it contains const string declaration.
188
- * @param useUtf8Code - Represents if the utf8 string marker code should be used when representing.
189
172
  */
190
- constructor(protected readonly data: Uint8Array, constString: boolean, useUtf8Code: boolean = false) {
191
- super(constString, useUtf8Code);
173
+ constructor(protected readonly data: Uint8Array) {
174
+ super();
192
175
  }
193
176
 
194
177
  public get buffer() {
195
178
  return this.data;
196
179
  }
197
180
 
198
- public static read(buffer: ReadBuffer, lengthLen: number, constString: boolean): BlobCore {
181
+ public get arrayBuffer(): ArrayBufferLike {
182
+ return Uint8ArrayToArrayBuffer(this.buffer);
183
+ }
184
+
185
+ public static read(buffer: ReadBuffer, lengthLen: number): BlobCore {
199
186
  const length = buffer.read(lengthLen);
200
187
  const data = new Uint8Array(length);
201
188
  for (let counter = 0; counter < length; counter++) {
202
189
  data[counter] = buffer.read();
203
190
  }
204
- return new BlobDeepCopy(data, constString);
191
+ return new BlobDeepCopy(data);
205
192
  }
206
193
  }
207
194
 
@@ -215,40 +202,60 @@ class BlobDeepCopy extends BlobCore {
215
202
  * @param data - Data array of the blob
216
203
  * @param start - Start point of the blob in the buffer.
217
204
  * @param end - End point of the blob in the buffer.
218
- * @param constString - Whether it contains const string declaration.
219
205
  */
220
206
  constructor(
221
- protected data: ReadBuffer,
207
+ protected data: Uint8Array,
222
208
  protected start: number,
223
209
  protected end: number,
224
- constString: boolean,
225
210
  ) {
226
- super(constString);
211
+ super();
227
212
  }
228
213
 
229
214
  public get buffer() {
230
- return this.data.buffer.subarray(this.start, this.end);
215
+ return this.data.subarray(this.start, this.end);
216
+ }
217
+
218
+ // Equivalent to Uint8ArrayToArrayBuffer(this.buffer)
219
+ public get arrayBuffer(): ArrayBufferLike {
220
+ const offset = this.data.byteOffset;
221
+ return this.data.buffer.slice(this.start + offset, this.end + offset);
231
222
  }
232
223
 
233
- public static read(buffer: ReadBuffer, lengthLen: number, constString: boolean): BlobCore {
224
+ public static read(buffer: ReadBuffer, lengthLen: number): BlobCore {
234
225
  const length = buffer.read(lengthLen);
235
226
  const pos = buffer.pos;
236
227
  buffer.skip(length);
237
- return new BlobShallowCopy(buffer, pos, pos + length, constString);
228
+ return new BlobShallowCopy(buffer.buffer, pos, pos + length);
238
229
  }
239
230
  }
240
231
 
241
232
  export const addStringProperty =
242
- (node: NodeCore, a: string, b: string, encodeValAsConstString: boolean = false) => {
243
- node.addString(a, true); node.addString(b, encodeValAsConstString);
233
+ (node: NodeCore, a: string, b: string) => {
234
+ node.addDictionaryString(a); node.addString(b);
235
+ };
236
+ export const addDictionaryStringProperty =
237
+ (node: NodeCore, a: string, b: string) => {
238
+ node.addDictionaryString(a); node.addString(b);
244
239
  };
245
240
  export const addNumberProperty = (node: NodeCore, a: string, b: number) => {
246
- node.addString(a, true); node.addNumber(b);
241
+ node.addDictionaryString(a); node.addNumber(b);
247
242
  };
248
243
  export const addBoolProperty = (node: NodeCore, a: string, b: boolean) => {
249
- node.addString(a, true); node.addBool(b);
244
+ node.addDictionaryString(a); node.addBool(b);
250
245
  };
251
246
 
247
+ export interface IStringElement {
248
+ content: string;
249
+ dictionary: boolean;
250
+ _stringElement: true;
251
+ }
252
+
253
+ export interface IStringElementInternal extends Omit<IStringElement, "content"> {
254
+ content?: string;
255
+ startPos: number;
256
+ endPos: number;
257
+ }
258
+
252
259
  /**
253
260
  * Three leaf types supported by tree:
254
261
  * 1. Node (sub-tree)
@@ -256,7 +263,7 @@ export const addBoolProperty = (node: NodeCore, a: string, b: boolean) => {
256
263
  * 3. integer
257
264
  * 4. boolean
258
265
  */
259
- export type NodeTypes = NodeCore | BlobCore | number | boolean;
266
+ export type NodeTypes = NodeCore | BlobCore | number | boolean | IStringElement;
260
267
 
261
268
  export type NodeCoreTypes = "list" | "set";
262
269
 
@@ -287,8 +294,12 @@ export class NodeCore {
287
294
 
288
295
  public getString(index: number): string {
289
296
  const node = this.children[index];
290
- assertBlobCoreInstance(node, "getString should return stringblob");
291
- return node.toString();
297
+ return getStringInstance(node, "getString should return string");
298
+ }
299
+
300
+ public getMaybeString(index: number) {
301
+ const node = this.children[index];
302
+ return getMaybeStringInstance(node);
292
303
  }
293
304
 
294
305
  public getBlob(index: number): BlobCore {
@@ -321,12 +332,24 @@ export class NodeCore {
321
332
  return node;
322
333
  }
323
334
 
324
- public addBlob(blob: Uint8Array, constString: boolean, useUtf8Code: boolean = false) {
325
- this.children.push(new BlobDeepCopy(blob, constString, useUtf8Code));
335
+ public addBlob(blob: Uint8Array) {
336
+ this.children.push(new BlobDeepCopy(blob));
337
+ }
338
+
339
+ public addDictionaryString(payload: string) {
340
+ this.children.push({
341
+ content: payload,
342
+ dictionary: true,
343
+ _stringElement: true,
344
+ });
326
345
  }
327
346
 
328
- public addString(payload: string, constString: boolean) {
329
- this.addBlob(IsoBuffer.from(payload, "utf-8"), constString, true);
347
+ public addString(payload: string) {
348
+ this.children.push({
349
+ content: payload,
350
+ dictionary: false,
351
+ _stringElement: true,
352
+ });
330
353
  }
331
354
 
332
355
  public addNumber(payload: number | undefined) {
@@ -339,59 +362,91 @@ export class NodeCore {
339
362
  this.children.push(payload);
340
363
  }
341
364
 
365
+ // Can we do more efficiently here, without extra objects somehow??
366
+ private static readString(buffer: ReadBuffer, code: number, dictionary: boolean) {
367
+ const lengthLen = getValueSafely(codeToBytesMap, code);
368
+ const length = buffer.read(lengthLen);
369
+ const startPos = buffer.pos;
370
+ buffer.skip(length);
371
+ const result: IStringElementInternal = {
372
+ // Note: Setting here property 'content: undefined' makes code substantially slower!
373
+ dictionary,
374
+ _stringElement: true,
375
+ startPos,
376
+ endPos: buffer.pos,
377
+ };
378
+
379
+ // We are lying here in terms of presence of `content` property.
380
+ // This will be addressed at the bottom of NodeCore.load() by resolving all strings at once!
381
+ // It's equivalent (but much slower!) to do it here via
382
+ // result.content = Uint8ArrayToString(buffer.buffer.subarray(startPos, buffer.pos), "utf-8");
383
+ return result as IStringElementInternal & IStringElement;
384
+ }
385
+
342
386
  /**
343
387
  * Load and parse the buffer into a tree.
344
388
  * @param buffer - buffer to read from.
345
389
  */
346
- protected load(buffer: ReadBuffer, dictionary: BlobCore[]) {
390
+ protected load(buffer: ReadBuffer, logger: ITelemetryLogger) {
391
+ const stack: NodeTypes[][] = [];
392
+ const stringsToResolve: IStringElementInternal[] = [];
393
+ const dictionary: IStringElement[] = [];
394
+
395
+ let children = this.children;
347
396
  for (;!buffer.eof;) {
348
- let childValue: NodeTypes | undefined;
349
397
  const code = buffer.read();
350
398
  switch (code) {
351
399
  case MarkerCodesStart.list:
352
400
  case MarkerCodesStart.set: {
353
- childValue = new NodeCore(code === MarkerCodesStart.set ? "set" : "list");
354
- this.children.push(childValue);
355
- childValue.load(buffer, dictionary);
356
- break;
401
+ const childValue = new NodeCore(code === MarkerCodesStart.set ? "set" : "list");
402
+ children.push(childValue);
403
+ stack.push(children);
404
+ children = childValue.children;
405
+ continue;
357
406
  }
358
407
  case MarkerCodes.ConstStringDeclare:
359
408
  case MarkerCodes.ConstStringDeclareBig:
360
409
  {
361
410
  const stringId = buffer.read(getValueSafely(codeToBytesMap, code));
362
- const constString = BlobShallowCopy.read(buffer, getValueSafely(codeToBytesMap, code), true);
411
+ const constString = NodeCore.readString(buffer, code, true /* dictionary */);
412
+ stringsToResolve.push(constString);
363
413
  dictionary[stringId] = constString;
364
- break;
414
+ continue;
365
415
  }
366
416
  case MarkerCodes.ConstString8Id:
367
417
  case MarkerCodes.ConstString16Id:
368
418
  case MarkerCodes.ConstString32Id:
369
419
  {
370
420
  const stringId = buffer.read(getValueSafely(codeToBytesMap, code));
371
- childValue = dictionary[stringId];
372
- this.children.push(childValue);
373
- break;
421
+ const content = dictionary[stringId];
422
+ assert(content !== undefined, 0x3de /* const string not found */);
423
+ children.push(content);
424
+ continue;
374
425
  }
375
426
  case MarkerCodes.StringEmpty:
376
427
  case MarkerCodes.String8Length:
377
428
  case MarkerCodes.String16Length:
378
429
  case MarkerCodes.String32Length:
430
+ {
431
+ const str = NodeCore.readString(buffer, code, false /* dictionary */);
432
+ stringsToResolve.push(str);
433
+ children.push(str);
434
+ continue;
435
+ }
379
436
  case MarkerCodes.BinaryEmpty:
380
437
  case MarkerCodes.BinarySingle8:
381
438
  case MarkerCodes.BinarySingle16:
382
439
  case MarkerCodes.BinarySingle32:
383
440
  case MarkerCodes.BinarySingle64:
384
441
  {
385
- childValue = BlobShallowCopy.read(buffer, getValueSafely(codeToBytesMap, code), false);
386
- this.children.push(childValue);
387
- break;
442
+ children.push(BlobShallowCopy.read(buffer, getValueSafely(codeToBytesMap, code)));
443
+ continue;
388
444
  }
389
445
  // If integer is 0.
390
446
  case MarkerCodes.Int0:
391
447
  {
392
- childValue = 0;
393
- this.children.push(childValue);
394
- break;
448
+ children.push(0);
449
+ continue;
395
450
  }
396
451
  case MarkerCodes.UInt8:
397
452
  case MarkerCodes.UInt16:
@@ -402,23 +457,72 @@ export class NodeCore {
402
457
  case MarkerCodes.Int32:
403
458
  case MarkerCodes.Int64:
404
459
  {
405
- childValue = buffer.read(getValueSafely(codeToBytesMap, code));
406
- this.children.push(childValue);
407
- break;
460
+ children.push(buffer.read(getValueSafely(codeToBytesMap, code)));
461
+ continue;
408
462
  }
409
463
  case MarkerCodes.BoolTrue:
410
- this.children.push(true);
411
- break;
464
+ children.push(true);
465
+ continue;
412
466
  case MarkerCodes.BoolFalse:
413
- this.children.push(false);
414
- break;
467
+ children.push(false);
468
+ continue;
415
469
  case MarkerCodesEnd.list:
416
470
  case MarkerCodesEnd.set:
417
- return;
471
+ // Note: We are not checking that end marker matches start marker.
472
+ // I.e. that we do not have a case where we start a 'list' but end with a 'set'
473
+ // Checking it would require more state tracking that seems not very useful, given
474
+ // our code does not care.
475
+ children = stack.pop()!;
476
+
477
+ // To my surprise, checking children !== undefined adds measurable cost!
478
+ // We will rely on children.push() crashing in case of mismatch, and check below
479
+ // (outside of the loop)
480
+ continue;
418
481
  default:
419
482
  throw new Error(`Invalid code: ${code}`);
420
483
  }
421
484
  }
485
+
486
+ // This also ensures that stack.length === 0.
487
+ assert(children === this.children, 0x3e7 /* Unpaired start/end list/set markers! */);
488
+
489
+ /**
490
+ * Process all the strings at once!
491
+ */
492
+ let length = 0;
493
+ for (const el of stringsToResolve) {
494
+ length += el.endPos - el.startPos + 1;
495
+ }
496
+ const stringBuffer = new Uint8Array(length);
497
+
498
+ length = 0;
499
+ const input = buffer.buffer;
500
+ assert(input.byteOffset === 0, 0x3e8 /* code below assumes no offset */);
501
+
502
+ for (const el of stringsToResolve) {
503
+ for (let it = el.startPos; it < el.endPos; it++) {
504
+ stringBuffer[length] = input[it];
505
+ length++;
506
+ }
507
+ stringBuffer[length] = 0;
508
+ length++;
509
+ }
510
+
511
+ if (length === stringBuffer.length) {
512
+ // All is good, we expect all the cases to get here
513
+ const result = Uint8ArrayToString(stringBuffer, "utf-8").split(String.fromCharCode(0));
514
+ assert(result.length === stringsToResolve.length + 1, 0x3e9 /* String content has \0 chars! */);
515
+ for (let i = 0; i < stringsToResolve.length; i++) {
516
+ stringsToResolve[i].content = result[i];
517
+ }
518
+ } else {
519
+ // Recovery code
520
+ logger.sendErrorEvent({ eventName: "StringParsingError" });
521
+ for (const el of stringsToResolve) {
522
+ assert(el.content === Uint8ArrayToString(input.subarray(el.startPos, el.endPos), "utf-8"),
523
+ 0x3ea /* test */);
524
+ }
525
+ }
422
526
  }
423
527
  }
424
528
 
@@ -427,15 +531,32 @@ export class NodeCore {
427
531
  * Provides loading and serialization capabilities.
428
532
  */
429
533
  export class TreeBuilder extends NodeCore {
430
- static load(buffer: ReadBuffer): TreeBuilder {
534
+ static load(buffer: ReadBuffer, logger: ITelemetryLogger): TreeBuilder {
431
535
  const builder = new TreeBuilder();
432
- const dictionary = new Array<BlobCore>();
433
- builder.load(buffer, dictionary);
536
+ builder.load(buffer, logger);
434
537
  assert(buffer.eof, 0x233 /* "Unexpected data at the end of buffer" */);
435
538
  return builder;
436
539
  }
437
540
  }
438
541
 
542
+ export function getMaybeStringInstance(node: NodeTypes): string | undefined {
543
+ const maybeString = node as IStringElement;
544
+ if (maybeString._stringElement) {
545
+ return maybeString.content;
546
+ }
547
+ }
548
+
549
+ export function getStringInstance(
550
+ node: NodeTypes,
551
+ message: string,
552
+ ): string {
553
+ const maybeString = node as IStringElement;
554
+ if (maybeString._stringElement) {
555
+ return maybeString.content;
556
+ }
557
+ throwBufferParseException(node, "BlobCore", message);
558
+ }
559
+
439
560
  export function assertBlobCoreInstance(
440
561
  node: NodeTypes,
441
562
  message: string,
@@ -500,8 +621,10 @@ function getNodeType(value: NodeTypes): NodeType {
500
621
  return "NodeCore";
501
622
  } else if (typeof value === "boolean") {
502
623
  return "Boolean";
624
+ } else if (value._stringElement) {
625
+ return "String";
503
626
  }
504
627
  return "UnknownType";
505
628
  }
506
629
 
507
- type NodeType = "Number" | "BlobCore" | "NodeCore" | "Boolean" | "UnknownType";
630
+ type NodeType = "Number" | "BlobCore" | "NodeCore" | "Boolean" | "UnknownType" | "String";