@fluidframework/container-runtime 2.0.0-internal.2.1.1 → 2.0.0-internal.2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (161) hide show
  1. package/.eslintrc.js +1 -1
  2. package/dist/blobManager.d.ts +20 -5
  3. package/dist/blobManager.d.ts.map +1 -1
  4. package/dist/blobManager.js +57 -15
  5. package/dist/blobManager.js.map +1 -1
  6. package/dist/containerRuntime.d.ts +39 -40
  7. package/dist/containerRuntime.d.ts.map +1 -1
  8. package/dist/containerRuntime.js +115 -278
  9. package/dist/containerRuntime.js.map +1 -1
  10. package/dist/dataStoreContext.d.ts +3 -1
  11. package/dist/dataStoreContext.d.ts.map +1 -1
  12. package/dist/dataStoreContext.js +21 -3
  13. package/dist/dataStoreContext.js.map +1 -1
  14. package/dist/dataStores.d.ts +8 -5
  15. package/dist/dataStores.d.ts.map +1 -1
  16. package/dist/dataStores.js +26 -13
  17. package/dist/dataStores.js.map +1 -1
  18. package/dist/garbageCollection.d.ts +15 -17
  19. package/dist/garbageCollection.d.ts.map +1 -1
  20. package/dist/garbageCollection.js +92 -106
  21. package/dist/garbageCollection.js.map +1 -1
  22. package/dist/garbageCollectionConstants.d.ts +19 -0
  23. package/dist/garbageCollectionConstants.d.ts.map +1 -0
  24. package/dist/garbageCollectionConstants.js +34 -0
  25. package/dist/garbageCollectionConstants.js.map +1 -0
  26. package/dist/gcSweepReadyUsageDetection.js +2 -2
  27. package/dist/gcSweepReadyUsageDetection.js.map +1 -1
  28. package/dist/index.d.ts +4 -2
  29. package/dist/index.d.ts.map +1 -1
  30. package/dist/index.js +8 -6
  31. package/dist/index.js.map +1 -1
  32. package/dist/opLifecycle/batchManager.d.ts +30 -0
  33. package/dist/opLifecycle/batchManager.d.ts.map +1 -0
  34. package/dist/{batchManager.js → opLifecycle/batchManager.js} +25 -10
  35. package/dist/opLifecycle/batchManager.js.map +1 -0
  36. package/dist/opLifecycle/definitions.d.ts +40 -0
  37. package/dist/opLifecycle/definitions.d.ts.map +1 -0
  38. package/dist/opLifecycle/definitions.js +7 -0
  39. package/dist/opLifecycle/definitions.js.map +1 -0
  40. package/dist/opLifecycle/index.d.ts +12 -0
  41. package/dist/opLifecycle/index.d.ts.map +1 -0
  42. package/dist/opLifecycle/index.js +21 -0
  43. package/dist/opLifecycle/index.js.map +1 -0
  44. package/dist/opLifecycle/opCompressor.d.ts +18 -0
  45. package/dist/opLifecycle/opCompressor.d.ts.map +1 -0
  46. package/dist/opLifecycle/opCompressor.js +53 -0
  47. package/dist/opLifecycle/opCompressor.js.map +1 -0
  48. package/dist/opLifecycle/opDecompressor.d.ts +20 -0
  49. package/dist/opLifecycle/opDecompressor.d.ts.map +1 -0
  50. package/dist/opLifecycle/opDecompressor.js +72 -0
  51. package/dist/opLifecycle/opDecompressor.js.map +1 -0
  52. package/dist/opLifecycle/opSplitter.d.ts +17 -0
  53. package/dist/opLifecycle/opSplitter.d.ts.map +1 -0
  54. package/dist/opLifecycle/opSplitter.js +61 -0
  55. package/dist/opLifecycle/opSplitter.js.map +1 -0
  56. package/dist/opLifecycle/outbox.d.ts +47 -0
  57. package/dist/opLifecycle/outbox.d.ts.map +1 -0
  58. package/dist/opLifecycle/outbox.js +153 -0
  59. package/dist/opLifecycle/outbox.js.map +1 -0
  60. package/dist/opLifecycle/remoteMessageProcessor.d.ts +26 -0
  61. package/dist/opLifecycle/remoteMessageProcessor.d.ts.map +1 -0
  62. package/dist/opLifecycle/remoteMessageProcessor.js +81 -0
  63. package/dist/opLifecycle/remoteMessageProcessor.js.map +1 -0
  64. package/dist/packageVersion.d.ts +1 -1
  65. package/dist/packageVersion.js +1 -1
  66. package/dist/packageVersion.js.map +1 -1
  67. package/dist/summaryFormat.js +2 -2
  68. package/dist/summaryFormat.js.map +1 -1
  69. package/lib/blobManager.d.ts +20 -5
  70. package/lib/blobManager.d.ts.map +1 -1
  71. package/lib/blobManager.js +59 -17
  72. package/lib/blobManager.js.map +1 -1
  73. package/lib/containerRuntime.d.ts +39 -40
  74. package/lib/containerRuntime.d.ts.map +1 -1
  75. package/lib/containerRuntime.js +113 -275
  76. package/lib/containerRuntime.js.map +1 -1
  77. package/lib/dataStoreContext.d.ts +3 -1
  78. package/lib/dataStoreContext.d.ts.map +1 -1
  79. package/lib/dataStoreContext.js +23 -5
  80. package/lib/dataStoreContext.js.map +1 -1
  81. package/lib/dataStores.d.ts +8 -5
  82. package/lib/dataStores.d.ts.map +1 -1
  83. package/lib/dataStores.js +28 -15
  84. package/lib/dataStores.js.map +1 -1
  85. package/lib/garbageCollection.d.ts +15 -17
  86. package/lib/garbageCollection.d.ts.map +1 -1
  87. package/lib/garbageCollection.js +72 -86
  88. package/lib/garbageCollection.js.map +1 -1
  89. package/lib/garbageCollectionConstants.d.ts +19 -0
  90. package/lib/garbageCollectionConstants.d.ts.map +1 -0
  91. package/lib/garbageCollectionConstants.js +31 -0
  92. package/lib/garbageCollectionConstants.js.map +1 -0
  93. package/lib/gcSweepReadyUsageDetection.js +1 -1
  94. package/lib/gcSweepReadyUsageDetection.js.map +1 -1
  95. package/lib/index.d.ts +4 -2
  96. package/lib/index.d.ts.map +1 -1
  97. package/lib/index.js +3 -2
  98. package/lib/index.js.map +1 -1
  99. package/lib/opLifecycle/batchManager.d.ts +30 -0
  100. package/lib/opLifecycle/batchManager.d.ts.map +1 -0
  101. package/lib/{batchManager.js → opLifecycle/batchManager.js} +25 -10
  102. package/lib/opLifecycle/batchManager.js.map +1 -0
  103. package/lib/opLifecycle/definitions.d.ts +40 -0
  104. package/lib/opLifecycle/definitions.d.ts.map +1 -0
  105. package/lib/opLifecycle/definitions.js +6 -0
  106. package/lib/opLifecycle/definitions.js.map +1 -0
  107. package/lib/opLifecycle/index.d.ts +12 -0
  108. package/lib/opLifecycle/index.d.ts.map +1 -0
  109. package/lib/opLifecycle/index.js +11 -0
  110. package/lib/opLifecycle/index.js.map +1 -0
  111. package/lib/opLifecycle/opCompressor.d.ts +18 -0
  112. package/lib/opLifecycle/opCompressor.d.ts.map +1 -0
  113. package/lib/opLifecycle/opCompressor.js +49 -0
  114. package/lib/opLifecycle/opCompressor.js.map +1 -0
  115. package/lib/opLifecycle/opDecompressor.d.ts +20 -0
  116. package/lib/opLifecycle/opDecompressor.d.ts.map +1 -0
  117. package/lib/opLifecycle/opDecompressor.js +68 -0
  118. package/lib/opLifecycle/opDecompressor.js.map +1 -0
  119. package/lib/opLifecycle/opSplitter.d.ts +17 -0
  120. package/lib/opLifecycle/opSplitter.d.ts.map +1 -0
  121. package/lib/opLifecycle/opSplitter.js +57 -0
  122. package/lib/opLifecycle/opSplitter.js.map +1 -0
  123. package/lib/opLifecycle/outbox.d.ts +47 -0
  124. package/lib/opLifecycle/outbox.d.ts.map +1 -0
  125. package/lib/opLifecycle/outbox.js +149 -0
  126. package/lib/opLifecycle/outbox.js.map +1 -0
  127. package/lib/opLifecycle/remoteMessageProcessor.d.ts +26 -0
  128. package/lib/opLifecycle/remoteMessageProcessor.d.ts.map +1 -0
  129. package/lib/opLifecycle/remoteMessageProcessor.js +76 -0
  130. package/lib/opLifecycle/remoteMessageProcessor.js.map +1 -0
  131. package/lib/packageVersion.d.ts +1 -1
  132. package/lib/packageVersion.js +1 -1
  133. package/lib/packageVersion.js.map +1 -1
  134. package/lib/summaryFormat.js +1 -1
  135. package/lib/summaryFormat.js.map +1 -1
  136. package/package.json +35 -21
  137. package/prettier.config.cjs +8 -0
  138. package/src/blobManager.ts +74 -19
  139. package/src/containerRuntime.ts +144 -341
  140. package/src/dataStoreContext.ts +33 -5
  141. package/src/dataStores.ts +32 -16
  142. package/src/garbageCollection.ts +106 -82
  143. package/src/garbageCollectionConstants.ts +35 -0
  144. package/src/gcSweepReadyUsageDetection.ts +1 -1
  145. package/src/index.ts +6 -4
  146. package/src/{batchManager.ts → opLifecycle/batchManager.ts} +41 -23
  147. package/src/opLifecycle/definitions.ts +44 -0
  148. package/src/opLifecycle/index.ts +17 -0
  149. package/src/opLifecycle/opCompressor.ts +64 -0
  150. package/src/opLifecycle/opDecompressor.ts +84 -0
  151. package/src/opLifecycle/opSplitter.ts +78 -0
  152. package/src/opLifecycle/outbox.ts +204 -0
  153. package/src/opLifecycle/remoteMessageProcessor.ts +90 -0
  154. package/src/packageVersion.ts +1 -1
  155. package/src/summaryFormat.ts +1 -1
  156. package/dist/batchManager.d.ts +0 -36
  157. package/dist/batchManager.d.ts.map +0 -1
  158. package/dist/batchManager.js.map +0 -1
  159. package/lib/batchManager.d.ts +0 -36
  160. package/lib/batchManager.d.ts.map +0 -1
  161. package/lib/batchManager.js.map +0 -1
@@ -6,7 +6,7 @@
6
6
  Object.defineProperty(exports, "__esModule", { value: true });
7
7
  exports.SweepReadyUsageDetectionHandler = exports.SweepReadyUsageError = exports.closuresMapLocalStorageKey = exports.skipClosureForXDaysKey = void 0;
8
8
  const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
9
- const garbageCollection_1 = require("./garbageCollection");
9
+ const garbageCollectionConstants_1 = require("./garbageCollectionConstants");
10
10
  /**
11
11
  * Feature Gate Key -
12
12
  * How many days between closing the container from this error (avoids locking user out of their file altogether)
@@ -104,7 +104,7 @@ class SweepReadyUsageDetectionHandler {
104
104
  }
105
105
  lastCloseTime = (_a = pastClosuresMap[this.uniqueContainerKey]) === null || _a === void 0 ? void 0 : _a.lastCloseTime;
106
106
  // Don't close if we did already within the Skip Closure Period
107
- if (lastCloseTime !== undefined && Date.now() < lastCloseTime + skipClosureForXDays * garbageCollection_1.oneDayMs) {
107
+ if (lastCloseTime !== undefined && Date.now() < lastCloseTime + skipClosureForXDays * garbageCollectionConstants_1.oneDayMs) {
108
108
  shouldClose = false;
109
109
  }
110
110
  }
@@ -1 +1 @@
1
- {"version":3,"file":"gcSweepReadyUsageDetection.js","sourceRoot":"","sources":["../src/gcSweepReadyUsageDetection.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAIH,qEAKyC;AACzC,2DAA+C;AAE/C;;;GAGG;AACU,QAAA,sBAAsB,GAAG,8EAA8E,CAAC;AAErH;;;GAGG;AACU,QAAA,0BAA0B,GAAG,mEAAmE,CAAC;AAE9G;;;GAGG;AACH,MAAM,+BAA+B,GAAG;IACpC,IAAI,CAAC,MAAuB;QACxB,MAAM,2BAA2B,GAAG,0DAA0D,CAAC;QAC/F,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,2BAA2B,CAAC,CAAC;QAC5D,IAAI,KAAK,KAAK,SAAS,EAAE;YACrB,OAAO,EAAE,iBAAiB,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;SAC1D;QACD,OAAO;YACH,iBAAiB,EAAE,KAAK,CAAC,QAAQ,CAAC,mBAAmB,CAAC;YACtD,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC;SAC3C,CAAC;IACN,CAAC;CACJ,CAAC;AAEF;;;;;;;GAOG;AACH,MAAa,oBAAqB,SAAQ,8BAAY;IAAtD;;QACI,oHAAoH;QAC7G,cAAS,GAAW,6CAA6C,CAAC;IAC7E,CAAC;CAAA;AAHD,oDAGC;AAED;;;;;;;GAOG;AACH,MAAa,+BAA+B;IAGxC,YACqB,kBAA0B,EAC1B,EAAqB,EACrB,OAAkD,EACnE,oBAA2D;;QAH1C,uBAAkB,GAAlB,kBAAkB,CAAQ;QAC1B,OAAE,GAAF,EAAE,CAAmB;QACrB,YAAO,GAAP,OAAO,CAA2C;QAGnE,MAAM,WAAW,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,GAAE,CAAC,EAAE,CAAC;QAC/D,0FAA0F;QAC1F,IAAI,CAAC,YAAY,GAAG,MAAA,oBAAoB,aAApB,oBAAoB,cAApB,oBAAoB,GAAI,UAAU,CAAC,YAAY,mCAAI,WAAW,CAAC;QAEnF,IAAI,IAAI,CAAC,YAAY,KAAK,WAAW,EAAE;YACnC,0DAA0D;YAC1D,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,kBAAkB,CAAC,EAAE,SAAS,EAAE,4CAA4C,EAAE,CAAC,CAAC;SAClG;IACL,CAAC;IAED;;;;;;OAMG;IACI,gCAAgC,CAAC,UAAgC;;QACpE,IAAI,CAAC,+BAA+B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,iBAAiB,EAAE;YACzE,OAAO;SACV;QAED,iGAAiG;QACjG,gGAAgG;QAChG,oGAAoG;QACpG,IAAI,WAAW,GAAY,IAAI,CAAC;QAChC,IAAI,eAAe,GAA2D,EAAE,CAAC;QACjF,IAAI,aAAiC,CAAC;QACtC,MAAM,mBAAmB,GAAG,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,8BAAsB,CAAC,CAAC;QAC7E,IAAI,mBAAmB,KAAK,SAAS,EAAE;YACnC,qFAAqF;YACrF,IAAI;gBACA,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,kCAA0B,CAAC,CAAC;gBACvE,MAAM,WAAW,GAAG,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBAClE,IAAI,OAAO,WAAW,KAAK,QAAQ,EAAE;oBACjC,eAAe,GAAG,WAAW,CAAC;iBACjC;aACJ;YAAC,OAAO,CAAC,EAAE;aACX;YACD,aAAa,GAAG,MAAA,eAAe,CAAC,IAAI,CAAC,kBAAkB,CAAC,0CAAE,aAAa,CAAC;YAExE,+DAA+D;YAC/D,IAAI,aAAa,KAAK,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,aAAa,GAAG,mBAAmB,GAAG,4BAAQ,EAAE;gBAC5F,WAAW,GAAG,KAAK,CAAC;aACvB;SACJ;QAED,MAAM,KAAK,GAAG,IAAI,oBAAoB,CAClC,iDAAiD,EACjD,EAAE,YAAY,EAAE,IAAI,CAAC,SAAS,iCAAM,UAAU,KAAE,aAAa,EAAE,mBAAmB,IAAG,EAAE,CAC1F,CAAC;QACF,IAAI,WAAW,EAAE;YACb,qDAAqD;YACrD,8FAA8F;YAC9F,mGAAmG;YACnG,eAAe,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;YACzE,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,kCAA0B,EAAE,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC,CAAC;YAEvF,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;SACvB;aAAM;YACH,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,EAAE,+BAA+B,EAAE,EAAE,KAAK,CAAC,CAAC;SACxF;IACL,CAAC;CACJ;AAxED,0EAwEC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryProperties } from \"@fluidframework/common-definitions\";\nimport { ICriticalContainerError } from \"@fluidframework/container-definitions\";\nimport {\n IConfigProvider,\n IFluidErrorBase,\n LoggingError,\n MonitoringContext,\n} from \"@fluidframework/telemetry-utils\";\nimport { oneDayMs } from \"./garbageCollection\";\n\n/**\n * Feature Gate Key -\n * How many days between closing the container from this error (avoids locking user out of their file altogether)\n */\nexport const skipClosureForXDaysKey = \"Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection.SkipClosureForXDays\";\n\n/**\n * LocalStorage key (NOT via feature gate / monitoring context)\n * A map from docId to info about the last time we closed due to this error\n */\nexport const closuresMapLocalStorageKey = \"Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection.Closures\";\n\n/**\n * Feature gate key to enable closing the container if SweepReady objects are used.\n * Value should contain keywords \"interactiveClient\" and/or \"summarizer\" to enable detection in each container type\n */\nconst sweepReadyUsageDetectionSetting = {\n read(config: IConfigProvider) {\n const sweepReadyUsageDetectionKey = \"Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection\";\n const value = config.getString(sweepReadyUsageDetectionKey);\n if (value === undefined) {\n return { interactiveClient: false, summarizer: false };\n }\n return {\n interactiveClient: value.includes(\"interactiveClient\"),\n summarizer: value.includes(\"summarizer\"),\n };\n },\n};\n\n/**\n * Error class raised when a SweepReady object is used, indicating a bug in how\n * references are managed in the container by the application, or a bug in how\n * GC tracks those references.\n *\n * There's a chance for false positives when this error is raised by an Interactive Container,\n * since only the Summarizer has the latest truth about unreferenced node tracking\n */\nexport class SweepReadyUsageError extends LoggingError implements IFluidErrorBase {\n /** This errorType will be in temporary use (until Sweep is fully implemented) so don't add to any errorType type */\n public errorType: string = \"unreferencedObjectUsedAfterGarbageCollected\";\n}\n\n/**\n * This class encapsulates the logic around what to do when a SweepReady object is used.\n * There are several tactics we plan to use in Dogfood environments to aid diagnosis of these cases:\n * - Closing the interactive container when either the interactive or summarizer client detects this kind of violation\n * (via sweepReadyUsageDetectionSetting above)\n * - Throttling the frequency of these crashes via a \"Skip Closure Period\" per container per device\n * (via skipClosureForXDaysKey above. Uses localStorage and closuresMapLocalStorageKey to implement this behavior)\n */\nexport class SweepReadyUsageDetectionHandler {\n private readonly localStorage: Pick<Storage, \"getItem\" | \"setItem\">;\n\n constructor(\n private readonly uniqueContainerKey: string,\n private readonly mc: MonitoringContext,\n private readonly closeFn: (error?: ICriticalContainerError) => void,\n localStorageOverride?: Pick<Storage, \"getItem\" | \"setItem\">,\n ) {\n const noopStorage = { getItem: () => null, setItem: () => {} };\n // localStorage is not defined in Node environment, so fall back to noopStorage if needed.\n this.localStorage = localStorageOverride ?? globalThis.localStorage ?? noopStorage;\n\n if (this.localStorage === noopStorage) {\n // This means the Skip Closure Period logic will not work.\n this.mc.logger.sendTelemetryEvent({ eventName: \"SweepReadyUsageDetectionHandlerNoopStorage\" });\n }\n }\n\n /**\n * If SweepReady Usage Detection is enabled, close the interactive container.\n * If the SkipClosureForXDays setting is set, don't close the container more than once in that period.\n *\n * Once Sweep is fully implemented, this will be removed since the objects will be gone\n * and errors will arise elsewhere in the runtime\n */\n public usageDetectedInInteractiveClient(errorProps: ITelemetryProperties) {\n if (!sweepReadyUsageDetectionSetting.read(this.mc.config).interactiveClient) {\n return;\n }\n\n // Default stance is we close every time - this reflects the severity of SweepReady Object Usage.\n // However, we may choose to \"throttle\" the closures by setting the SkipClosureForXDays setting,\n // which will only allow the container to close once during that period, to avoid locking users out.\n let shouldClose: boolean = true;\n let pastClosuresMap: Record<string, { lastCloseTime: number; } | undefined> = {};\n let lastCloseTime: number | undefined;\n const skipClosureForXDays = this.mc.config.getNumber(skipClosureForXDaysKey);\n if (skipClosureForXDays !== undefined) {\n // Read pastClosuresMap from localStorage then extract the lastCloseTime from the map\n try {\n const rawValue = this.localStorage.getItem(closuresMapLocalStorageKey);\n const parsedValue = rawValue === null ? {} : JSON.parse(rawValue);\n if (typeof parsedValue === \"object\") {\n pastClosuresMap = parsedValue;\n }\n } catch (e) {\n }\n lastCloseTime = pastClosuresMap[this.uniqueContainerKey]?.lastCloseTime;\n\n // Don't close if we did already within the Skip Closure Period\n if (lastCloseTime !== undefined && Date.now() < lastCloseTime + skipClosureForXDays * oneDayMs) {\n shouldClose = false;\n }\n }\n\n const error = new SweepReadyUsageError(\n \"SweepReady object used in Non-Summarizer Client\",\n { errorDetails: JSON.stringify({ ...errorProps, lastCloseTime, skipClosureForXDays }) },\n );\n if (shouldClose) {\n // Update closures map in localStorage before closing\n // Note there is a race condition between different tabs updating localStorage and overwriting\n // each others' updates. If so, some tab will crash again. Just reload one at a time to get unstuck\n pastClosuresMap[this.uniqueContainerKey] = { lastCloseTime: Date.now() };\n this.localStorage.setItem(closuresMapLocalStorageKey, JSON.stringify(pastClosuresMap));\n\n this.closeFn(error);\n } else {\n this.mc.logger.sendErrorEvent({ eventName: \"SweepReadyObject_UsageAllowed\" }, error);\n }\n }\n}\n"]}
1
+ {"version":3,"file":"gcSweepReadyUsageDetection.js","sourceRoot":"","sources":["../src/gcSweepReadyUsageDetection.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAIH,qEAKyC;AACzC,6EAAwD;AAExD;;;GAGG;AACU,QAAA,sBAAsB,GAAG,8EAA8E,CAAC;AAErH;;;GAGG;AACU,QAAA,0BAA0B,GAAG,mEAAmE,CAAC;AAE9G;;;GAGG;AACH,MAAM,+BAA+B,GAAG;IACpC,IAAI,CAAC,MAAuB;QACxB,MAAM,2BAA2B,GAAG,0DAA0D,CAAC;QAC/F,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,2BAA2B,CAAC,CAAC;QAC5D,IAAI,KAAK,KAAK,SAAS,EAAE;YACrB,OAAO,EAAE,iBAAiB,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;SAC1D;QACD,OAAO;YACH,iBAAiB,EAAE,KAAK,CAAC,QAAQ,CAAC,mBAAmB,CAAC;YACtD,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC;SAC3C,CAAC;IACN,CAAC;CACJ,CAAC;AAEF;;;;;;;GAOG;AACH,MAAa,oBAAqB,SAAQ,8BAAY;IAAtD;;QACI,oHAAoH;QAC7G,cAAS,GAAW,6CAA6C,CAAC;IAC7E,CAAC;CAAA;AAHD,oDAGC;AAED;;;;;;;GAOG;AACH,MAAa,+BAA+B;IAGxC,YACqB,kBAA0B,EAC1B,EAAqB,EACrB,OAAkD,EACnE,oBAA2D;;QAH1C,uBAAkB,GAAlB,kBAAkB,CAAQ;QAC1B,OAAE,GAAF,EAAE,CAAmB;QACrB,YAAO,GAAP,OAAO,CAA2C;QAGnE,MAAM,WAAW,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,GAAE,CAAC,EAAE,CAAC;QAC/D,0FAA0F;QAC1F,IAAI,CAAC,YAAY,GAAG,MAAA,oBAAoB,aAApB,oBAAoB,cAApB,oBAAoB,GAAI,UAAU,CAAC,YAAY,mCAAI,WAAW,CAAC;QAEnF,IAAI,IAAI,CAAC,YAAY,KAAK,WAAW,EAAE;YACnC,0DAA0D;YAC1D,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,kBAAkB,CAAC,EAAE,SAAS,EAAE,4CAA4C,EAAE,CAAC,CAAC;SAClG;IACL,CAAC;IAED;;;;;;OAMG;IACI,gCAAgC,CAAC,UAAgC;;QACpE,IAAI,CAAC,+BAA+B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,iBAAiB,EAAE;YACzE,OAAO;SACV;QAED,iGAAiG;QACjG,gGAAgG;QAChG,oGAAoG;QACpG,IAAI,WAAW,GAAY,IAAI,CAAC;QAChC,IAAI,eAAe,GAA2D,EAAE,CAAC;QACjF,IAAI,aAAiC,CAAC;QACtC,MAAM,mBAAmB,GAAG,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,8BAAsB,CAAC,CAAC;QAC7E,IAAI,mBAAmB,KAAK,SAAS,EAAE;YACnC,qFAAqF;YACrF,IAAI;gBACA,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,kCAA0B,CAAC,CAAC;gBACvE,MAAM,WAAW,GAAG,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBAClE,IAAI,OAAO,WAAW,KAAK,QAAQ,EAAE;oBACjC,eAAe,GAAG,WAAW,CAAC;iBACjC;aACJ;YAAC,OAAO,CAAC,EAAE;aACX;YACD,aAAa,GAAG,MAAA,eAAe,CAAC,IAAI,CAAC,kBAAkB,CAAC,0CAAE,aAAa,CAAC;YAExE,+DAA+D;YAC/D,IAAI,aAAa,KAAK,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,aAAa,GAAG,mBAAmB,GAAG,qCAAQ,EAAE;gBAC5F,WAAW,GAAG,KAAK,CAAC;aACvB;SACJ;QAED,MAAM,KAAK,GAAG,IAAI,oBAAoB,CAClC,iDAAiD,EACjD,EAAE,YAAY,EAAE,IAAI,CAAC,SAAS,iCAAM,UAAU,KAAE,aAAa,EAAE,mBAAmB,IAAG,EAAE,CAC1F,CAAC;QACF,IAAI,WAAW,EAAE;YACb,qDAAqD;YACrD,8FAA8F;YAC9F,mGAAmG;YACnG,eAAe,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;YACzE,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,kCAA0B,EAAE,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC,CAAC;YAEvF,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;SACvB;aAAM;YACH,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,EAAE,+BAA+B,EAAE,EAAE,KAAK,CAAC,CAAC;SACxF;IACL,CAAC;CACJ;AAxED,0EAwEC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryProperties } from \"@fluidframework/common-definitions\";\nimport { ICriticalContainerError } from \"@fluidframework/container-definitions\";\nimport {\n IConfigProvider,\n IFluidErrorBase,\n LoggingError,\n MonitoringContext,\n} from \"@fluidframework/telemetry-utils\";\nimport { oneDayMs } from \"./garbageCollectionConstants\";\n\n/**\n * Feature Gate Key -\n * How many days between closing the container from this error (avoids locking user out of their file altogether)\n */\nexport const skipClosureForXDaysKey = \"Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection.SkipClosureForXDays\";\n\n/**\n * LocalStorage key (NOT via feature gate / monitoring context)\n * A map from docId to info about the last time we closed due to this error\n */\nexport const closuresMapLocalStorageKey = \"Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection.Closures\";\n\n/**\n * Feature gate key to enable closing the container if SweepReady objects are used.\n * Value should contain keywords \"interactiveClient\" and/or \"summarizer\" to enable detection in each container type\n */\nconst sweepReadyUsageDetectionSetting = {\n read(config: IConfigProvider) {\n const sweepReadyUsageDetectionKey = \"Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection\";\n const value = config.getString(sweepReadyUsageDetectionKey);\n if (value === undefined) {\n return { interactiveClient: false, summarizer: false };\n }\n return {\n interactiveClient: value.includes(\"interactiveClient\"),\n summarizer: value.includes(\"summarizer\"),\n };\n },\n};\n\n/**\n * Error class raised when a SweepReady object is used, indicating a bug in how\n * references are managed in the container by the application, or a bug in how\n * GC tracks those references.\n *\n * There's a chance for false positives when this error is raised by an Interactive Container,\n * since only the Summarizer has the latest truth about unreferenced node tracking\n */\nexport class SweepReadyUsageError extends LoggingError implements IFluidErrorBase {\n /** This errorType will be in temporary use (until Sweep is fully implemented) so don't add to any errorType type */\n public errorType: string = \"unreferencedObjectUsedAfterGarbageCollected\";\n}\n\n/**\n * This class encapsulates the logic around what to do when a SweepReady object is used.\n * There are several tactics we plan to use in Dogfood environments to aid diagnosis of these cases:\n * - Closing the interactive container when either the interactive or summarizer client detects this kind of violation\n * (via sweepReadyUsageDetectionSetting above)\n * - Throttling the frequency of these crashes via a \"Skip Closure Period\" per container per device\n * (via skipClosureForXDaysKey above. Uses localStorage and closuresMapLocalStorageKey to implement this behavior)\n */\nexport class SweepReadyUsageDetectionHandler {\n private readonly localStorage: Pick<Storage, \"getItem\" | \"setItem\">;\n\n constructor(\n private readonly uniqueContainerKey: string,\n private readonly mc: MonitoringContext,\n private readonly closeFn: (error?: ICriticalContainerError) => void,\n localStorageOverride?: Pick<Storage, \"getItem\" | \"setItem\">,\n ) {\n const noopStorage = { getItem: () => null, setItem: () => {} };\n // localStorage is not defined in Node environment, so fall back to noopStorage if needed.\n this.localStorage = localStorageOverride ?? globalThis.localStorage ?? noopStorage;\n\n if (this.localStorage === noopStorage) {\n // This means the Skip Closure Period logic will not work.\n this.mc.logger.sendTelemetryEvent({ eventName: \"SweepReadyUsageDetectionHandlerNoopStorage\" });\n }\n }\n\n /**\n * If SweepReady Usage Detection is enabled, close the interactive container.\n * If the SkipClosureForXDays setting is set, don't close the container more than once in that period.\n *\n * Once Sweep is fully implemented, this will be removed since the objects will be gone\n * and errors will arise elsewhere in the runtime\n */\n public usageDetectedInInteractiveClient(errorProps: ITelemetryProperties) {\n if (!sweepReadyUsageDetectionSetting.read(this.mc.config).interactiveClient) {\n return;\n }\n\n // Default stance is we close every time - this reflects the severity of SweepReady Object Usage.\n // However, we may choose to \"throttle\" the closures by setting the SkipClosureForXDays setting,\n // which will only allow the container to close once during that period, to avoid locking users out.\n let shouldClose: boolean = true;\n let pastClosuresMap: Record<string, { lastCloseTime: number; } | undefined> = {};\n let lastCloseTime: number | undefined;\n const skipClosureForXDays = this.mc.config.getNumber(skipClosureForXDaysKey);\n if (skipClosureForXDays !== undefined) {\n // Read pastClosuresMap from localStorage then extract the lastCloseTime from the map\n try {\n const rawValue = this.localStorage.getItem(closuresMapLocalStorageKey);\n const parsedValue = rawValue === null ? {} : JSON.parse(rawValue);\n if (typeof parsedValue === \"object\") {\n pastClosuresMap = parsedValue;\n }\n } catch (e) {\n }\n lastCloseTime = pastClosuresMap[this.uniqueContainerKey]?.lastCloseTime;\n\n // Don't close if we did already within the Skip Closure Period\n if (lastCloseTime !== undefined && Date.now() < lastCloseTime + skipClosureForXDays * oneDayMs) {\n shouldClose = false;\n }\n }\n\n const error = new SweepReadyUsageError(\n \"SweepReady object used in Non-Summarizer Client\",\n { errorDetails: JSON.stringify({ ...errorProps, lastCloseTime, skipClosureForXDays }) },\n );\n if (shouldClose) {\n // Update closures map in localStorage before closing\n // Note there is a race condition between different tabs updating localStorage and overwriting\n // each others' updates. If so, some tab will crash again. Just reload one at a time to get unstuck\n pastClosuresMap[this.uniqueContainerKey] = { lastCloseTime: Date.now() };\n this.localStorage.setItem(closuresMapLocalStorageKey, JSON.stringify(pastClosuresMap));\n\n this.closeFn(error);\n } else {\n this.mc.logger.sendErrorEvent({ eventName: \"SweepReadyObject_UsageAllowed\" }, error);\n }\n }\n}\n"]}
package/dist/index.d.ts CHANGED
@@ -2,12 +2,14 @@
2
2
  * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
3
3
  * Licensed under the MIT License.
4
4
  */
5
- export { ContainerMessageType, IChunkedOp, ContainerRuntimeMessage, IGCRuntimeOptions, ISummaryRuntimeOptions, ISummaryBaseConfiguration, ISummaryConfigurationHeuristics, ISummaryConfigurationDisableSummarizer, ISummaryConfigurationDisableHeuristics, IContainerRuntimeOptions, IRootSummaryTreeWithStats, isRuntimeMessage, RuntimeMessage, unpackRuntimeMessage, agentSchedulerId, ContainerRuntime, RuntimeHeaders, ISummaryConfiguration, DefaultSummaryConfiguration, ICompressionRuntimeOptions, } from "./containerRuntime";
5
+ export { ContainerMessageType, ContainerRuntimeMessage, IGCRuntimeOptions, ISummaryRuntimeOptions, ISummaryBaseConfiguration, ISummaryConfigurationHeuristics, ISummaryConfigurationDisableSummarizer, ISummaryConfigurationDisableHeuristics, IContainerRuntimeOptions, IRootSummaryTreeWithStats, isRuntimeMessage, RuntimeMessage, agentSchedulerId, ContainerRuntime, RuntimeHeaders, ISummaryConfiguration, DefaultSummaryConfiguration, ICompressionRuntimeOptions, CompressionAlgorithms, } from "./containerRuntime";
6
6
  export { FluidDataStoreRegistry } from "./dataStoreRegistry";
7
- export { gcBlobPrefix, gcTombstoneBlobKey, gcTreeKey, IGCStats, } from "./garbageCollection";
7
+ export { IGCStats, } from "./garbageCollection";
8
+ export { gcBlobPrefix, gcTombstoneBlobKey, gcTreeKey, } from "./garbageCollectionConstants";
8
9
  export { IPendingFlush, IPendingLocalState, IPendingMessage, IPendingState, } from "./pendingStateManager";
9
10
  export { Summarizer } from "./summarizer";
10
11
  export { EnqueueSummarizeResult, IAckSummaryResult, IBaseSummarizeResult, IBroadcastSummaryResult, ICancellationToken, IConnectableRuntime, IEnqueueSummarizeOptions, IGenerateSummaryTreeResult, IGeneratedSummaryStats, INackSummaryResult, IOnDemandSummarizeOptions, IProvideSummarizer, IRefreshSummaryAckOptions, ISubmitSummaryOpResult, ISubmitSummaryOptions, ISummarizeOptions, ISummarizeResults, ISummarizer, ISummarizerEvents, ISummarizerInternalsProvider, ISummarizerRuntime, ISummarizingWarning, ISummaryCancellationToken, IUploadSummaryResult, SubmitSummaryResult, SummarizeResultPart, SummarizerStopReason, } from "./summarizerTypes";
11
12
  export { IAckedSummary, IClientSummaryWatcher, ISummary, ISummaryCollectionOpEvents, ISummaryAckMessage, ISummaryNackMessage, ISummaryOpMessage, OpActionEventListener, OpActionEventName, SummaryCollection, } from "./summaryCollection";
12
13
  export { ICancellableSummarizerController, neverCancelledSummaryToken } from "./runWhileConnectedCoordinator";
14
+ export { IChunkedOp, unpackRuntimeMessage } from "./opLifecycle";
13
15
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACH,oBAAoB,EACpB,UAAU,EACV,uBAAuB,EACvB,iBAAiB,EACjB,sBAAsB,EACtB,yBAAyB,EACzB,+BAA+B,EAC/B,sCAAsC,EACtC,sCAAsC,EACtC,wBAAwB,EACxB,yBAAyB,EACzB,gBAAgB,EAChB,cAAc,EACd,oBAAoB,EACpB,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,qBAAqB,EACrB,2BAA2B,EAC3B,0BAA0B,GAC7B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EACH,YAAY,EACZ,kBAAkB,EAClB,SAAS,EACT,QAAQ,GACX,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACH,aAAa,EACb,kBAAkB,EAClB,eAAe,EACf,aAAa,GAChB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EACH,sBAAsB,EACtB,iBAAiB,EACjB,oBAAoB,EACpB,uBAAuB,EACvB,kBAAkB,EAClB,mBAAmB,EACnB,wBAAwB,EACxB,0BAA0B,EAC1B,sBAAsB,EACtB,kBAAkB,EAClB,yBAAyB,EACzB,kBAAkB,EAClB,yBAAyB,EACzB,sBAAsB,EACtB,qBAAqB,EACrB,iBAAiB,EACjB,iBAAiB,EACjB,WAAW,EACX,iBAAiB,EACjB,4BAA4B,EAC5B,kBAAkB,EAClB,mBAAmB,EACnB,yBAAyB,EACzB,oBAAoB,EACpB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,GACvB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACH,aAAa,EACb,qBAAqB,EACrB,QAAQ,EACR,0BAA0B,EAC1B,kBAAkB,EAClB,mBAAmB,EACnB,iBAAiB,EACjB,qBAAqB,EACrB,iBAAiB,EACjB,iBAAiB,GACpB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,gCAAgC,EAAE,0BAA0B,EAAE,MAAM,gCAAgC,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACH,oBAAoB,EACpB,uBAAuB,EACvB,iBAAiB,EACjB,sBAAsB,EACtB,yBAAyB,EACzB,+BAA+B,EAC/B,sCAAsC,EACtC,sCAAsC,EACtC,wBAAwB,EACxB,yBAAyB,EACzB,gBAAgB,EAChB,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,qBAAqB,EACrB,2BAA2B,EAC3B,0BAA0B,EAC1B,qBAAqB,GACxB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EACH,QAAQ,GACX,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACH,YAAY,EACZ,kBAAkB,EAClB,SAAS,GACZ,MAAM,8BAA8B,CAAC;AACtC,OAAO,EACH,aAAa,EACb,kBAAkB,EAClB,eAAe,EACf,aAAa,GAChB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EACH,sBAAsB,EACtB,iBAAiB,EACjB,oBAAoB,EACpB,uBAAuB,EACvB,kBAAkB,EAClB,mBAAmB,EACnB,wBAAwB,EACxB,0BAA0B,EAC1B,sBAAsB,EACtB,kBAAkB,EAClB,yBAAyB,EACzB,kBAAkB,EAClB,yBAAyB,EACzB,sBAAsB,EACtB,qBAAqB,EACrB,iBAAiB,EACjB,iBAAiB,EACjB,WAAW,EACX,iBAAiB,EACjB,4BAA4B,EAC5B,kBAAkB,EAClB,mBAAmB,EACnB,yBAAyB,EACzB,oBAAoB,EACpB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,GACvB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACH,aAAa,EACb,qBAAqB,EACrB,QAAQ,EACR,0BAA0B,EAC1B,kBAAkB,EAClB,mBAAmB,EACnB,iBAAiB,EACjB,qBAAqB,EACrB,iBAAiB,EACjB,iBAAiB,GACpB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,gCAAgC,EAAE,0BAA0B,EAAE,MAAM,gCAAgC,CAAC;AAC9G,OAAO,EAAE,UAAU,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC"}
package/dist/index.js CHANGED
@@ -4,22 +4,22 @@
4
4
  * Licensed under the MIT License.
5
5
  */
6
6
  Object.defineProperty(exports, "__esModule", { value: true });
7
- exports.neverCancelledSummaryToken = exports.SummaryCollection = exports.ISummarizer = exports.Summarizer = exports.gcTreeKey = exports.gcTombstoneBlobKey = exports.gcBlobPrefix = exports.FluidDataStoreRegistry = exports.DefaultSummaryConfiguration = exports.RuntimeHeaders = exports.ContainerRuntime = exports.agentSchedulerId = exports.unpackRuntimeMessage = exports.RuntimeMessage = exports.isRuntimeMessage = exports.ContainerMessageType = void 0;
7
+ exports.unpackRuntimeMessage = exports.neverCancelledSummaryToken = exports.SummaryCollection = exports.ISummarizer = exports.Summarizer = exports.gcTreeKey = exports.gcTombstoneBlobKey = exports.gcBlobPrefix = exports.FluidDataStoreRegistry = exports.CompressionAlgorithms = exports.DefaultSummaryConfiguration = exports.RuntimeHeaders = exports.ContainerRuntime = exports.agentSchedulerId = exports.RuntimeMessage = exports.isRuntimeMessage = exports.ContainerMessageType = void 0;
8
8
  var containerRuntime_1 = require("./containerRuntime");
9
9
  Object.defineProperty(exports, "ContainerMessageType", { enumerable: true, get: function () { return containerRuntime_1.ContainerMessageType; } });
10
10
  Object.defineProperty(exports, "isRuntimeMessage", { enumerable: true, get: function () { return containerRuntime_1.isRuntimeMessage; } });
11
11
  Object.defineProperty(exports, "RuntimeMessage", { enumerable: true, get: function () { return containerRuntime_1.RuntimeMessage; } });
12
- Object.defineProperty(exports, "unpackRuntimeMessage", { enumerable: true, get: function () { return containerRuntime_1.unpackRuntimeMessage; } });
13
12
  Object.defineProperty(exports, "agentSchedulerId", { enumerable: true, get: function () { return containerRuntime_1.agentSchedulerId; } });
14
13
  Object.defineProperty(exports, "ContainerRuntime", { enumerable: true, get: function () { return containerRuntime_1.ContainerRuntime; } });
15
14
  Object.defineProperty(exports, "RuntimeHeaders", { enumerable: true, get: function () { return containerRuntime_1.RuntimeHeaders; } });
16
15
  Object.defineProperty(exports, "DefaultSummaryConfiguration", { enumerable: true, get: function () { return containerRuntime_1.DefaultSummaryConfiguration; } });
16
+ Object.defineProperty(exports, "CompressionAlgorithms", { enumerable: true, get: function () { return containerRuntime_1.CompressionAlgorithms; } });
17
17
  var dataStoreRegistry_1 = require("./dataStoreRegistry");
18
18
  Object.defineProperty(exports, "FluidDataStoreRegistry", { enumerable: true, get: function () { return dataStoreRegistry_1.FluidDataStoreRegistry; } });
19
- var garbageCollection_1 = require("./garbageCollection");
20
- Object.defineProperty(exports, "gcBlobPrefix", { enumerable: true, get: function () { return garbageCollection_1.gcBlobPrefix; } });
21
- Object.defineProperty(exports, "gcTombstoneBlobKey", { enumerable: true, get: function () { return garbageCollection_1.gcTombstoneBlobKey; } });
22
- Object.defineProperty(exports, "gcTreeKey", { enumerable: true, get: function () { return garbageCollection_1.gcTreeKey; } });
19
+ var garbageCollectionConstants_1 = require("./garbageCollectionConstants");
20
+ Object.defineProperty(exports, "gcBlobPrefix", { enumerable: true, get: function () { return garbageCollectionConstants_1.gcBlobPrefix; } });
21
+ Object.defineProperty(exports, "gcTombstoneBlobKey", { enumerable: true, get: function () { return garbageCollectionConstants_1.gcTombstoneBlobKey; } });
22
+ Object.defineProperty(exports, "gcTreeKey", { enumerable: true, get: function () { return garbageCollectionConstants_1.gcTreeKey; } });
23
23
  var summarizer_1 = require("./summarizer");
24
24
  Object.defineProperty(exports, "Summarizer", { enumerable: true, get: function () { return summarizer_1.Summarizer; } });
25
25
  var summarizerTypes_1 = require("./summarizerTypes");
@@ -28,4 +28,6 @@ var summaryCollection_1 = require("./summaryCollection");
28
28
  Object.defineProperty(exports, "SummaryCollection", { enumerable: true, get: function () { return summaryCollection_1.SummaryCollection; } });
29
29
  var runWhileConnectedCoordinator_1 = require("./runWhileConnectedCoordinator");
30
30
  Object.defineProperty(exports, "neverCancelledSummaryToken", { enumerable: true, get: function () { return runWhileConnectedCoordinator_1.neverCancelledSummaryToken; } });
31
+ var opLifecycle_1 = require("./opLifecycle");
32
+ Object.defineProperty(exports, "unpackRuntimeMessage", { enumerable: true, get: function () { return opLifecycle_1.unpackRuntimeMessage; } });
31
33
  //# sourceMappingURL=index.js.map
package/dist/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,uDAqB4B;AApBxB,wHAAA,oBAAoB,OAAA;AAWpB,oHAAA,gBAAgB,OAAA;AAChB,kHAAA,cAAc,OAAA;AACd,wHAAA,oBAAoB,OAAA;AACpB,oHAAA,gBAAgB,OAAA;AAChB,oHAAA,gBAAgB,OAAA;AAChB,kHAAA,cAAc,OAAA;AAEd,+HAAA,2BAA2B,OAAA;AAG/B,yDAA6D;AAApD,2HAAA,sBAAsB,OAAA;AAC/B,yDAK6B;AAJzB,iHAAA,YAAY,OAAA;AACZ,uHAAA,kBAAkB,OAAA;AAClB,8GAAA,SAAS,OAAA;AASb,2CAA0C;AAAjC,wGAAA,UAAU,OAAA;AACnB,qDA4B2B;AAVvB,8GAAA,WAAW,OAAA;AAWf,yDAW6B;AADzB,sHAAA,iBAAiB,OAAA;AAErB,+EAA8G;AAAnE,0IAAA,0BAA0B,OAAA","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nexport {\n ContainerMessageType,\n IChunkedOp,\n ContainerRuntimeMessage,\n IGCRuntimeOptions,\n ISummaryRuntimeOptions,\n ISummaryBaseConfiguration,\n ISummaryConfigurationHeuristics,\n ISummaryConfigurationDisableSummarizer,\n ISummaryConfigurationDisableHeuristics,\n IContainerRuntimeOptions,\n IRootSummaryTreeWithStats,\n isRuntimeMessage,\n RuntimeMessage,\n unpackRuntimeMessage,\n agentSchedulerId,\n ContainerRuntime,\n RuntimeHeaders,\n ISummaryConfiguration,\n DefaultSummaryConfiguration,\n ICompressionRuntimeOptions,\n} from \"./containerRuntime\";\nexport { FluidDataStoreRegistry } from \"./dataStoreRegistry\";\nexport {\n gcBlobPrefix,\n gcTombstoneBlobKey,\n gcTreeKey,\n IGCStats,\n} from \"./garbageCollection\";\nexport {\n IPendingFlush,\n IPendingLocalState,\n IPendingMessage,\n IPendingState,\n} from \"./pendingStateManager\";\nexport { Summarizer } from \"./summarizer\";\nexport {\n EnqueueSummarizeResult,\n IAckSummaryResult,\n IBaseSummarizeResult,\n IBroadcastSummaryResult,\n ICancellationToken,\n IConnectableRuntime,\n IEnqueueSummarizeOptions,\n IGenerateSummaryTreeResult,\n IGeneratedSummaryStats,\n INackSummaryResult,\n IOnDemandSummarizeOptions,\n IProvideSummarizer,\n IRefreshSummaryAckOptions,\n ISubmitSummaryOpResult,\n ISubmitSummaryOptions,\n ISummarizeOptions,\n ISummarizeResults,\n ISummarizer,\n ISummarizerEvents,\n ISummarizerInternalsProvider,\n ISummarizerRuntime,\n ISummarizingWarning,\n ISummaryCancellationToken,\n IUploadSummaryResult,\n SubmitSummaryResult,\n SummarizeResultPart,\n SummarizerStopReason,\n} from \"./summarizerTypes\";\nexport {\n IAckedSummary,\n IClientSummaryWatcher,\n ISummary,\n ISummaryCollectionOpEvents,\n ISummaryAckMessage,\n ISummaryNackMessage,\n ISummaryOpMessage,\n OpActionEventListener,\n OpActionEventName,\n SummaryCollection,\n} from \"./summaryCollection\";\nexport { ICancellableSummarizerController, neverCancelledSummaryToken } from \"./runWhileConnectedCoordinator\";\n"]}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,uDAoB4B;AAnBxB,wHAAA,oBAAoB,OAAA;AAUpB,oHAAA,gBAAgB,OAAA;AAChB,kHAAA,cAAc,OAAA;AACd,oHAAA,gBAAgB,OAAA;AAChB,oHAAA,gBAAgB,OAAA;AAChB,kHAAA,cAAc,OAAA;AAEd,+HAAA,2BAA2B,OAAA;AAE3B,yHAAA,qBAAqB,OAAA;AAEzB,yDAA6D;AAApD,2HAAA,sBAAsB,OAAA;AAI/B,2EAIsC;AAHlC,0HAAA,YAAY,OAAA;AACZ,gIAAA,kBAAkB,OAAA;AAClB,uHAAA,SAAS,OAAA;AAQb,2CAA0C;AAAjC,wGAAA,UAAU,OAAA;AACnB,qDA4B2B;AAVvB,8GAAA,WAAW,OAAA;AAWf,yDAW6B;AADzB,sHAAA,iBAAiB,OAAA;AAErB,+EAA8G;AAAnE,0IAAA,0BAA0B,OAAA;AACrE,6CAAiE;AAA5C,mHAAA,oBAAoB,OAAA","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nexport {\n ContainerMessageType,\n ContainerRuntimeMessage,\n IGCRuntimeOptions,\n ISummaryRuntimeOptions,\n ISummaryBaseConfiguration,\n ISummaryConfigurationHeuristics,\n ISummaryConfigurationDisableSummarizer,\n ISummaryConfigurationDisableHeuristics,\n IContainerRuntimeOptions,\n IRootSummaryTreeWithStats,\n isRuntimeMessage,\n RuntimeMessage,\n agentSchedulerId,\n ContainerRuntime,\n RuntimeHeaders,\n ISummaryConfiguration,\n DefaultSummaryConfiguration,\n ICompressionRuntimeOptions,\n CompressionAlgorithms,\n} from \"./containerRuntime\";\nexport { FluidDataStoreRegistry } from \"./dataStoreRegistry\";\nexport {\n IGCStats,\n} from \"./garbageCollection\";\nexport {\n gcBlobPrefix,\n gcTombstoneBlobKey,\n gcTreeKey,\n} from \"./garbageCollectionConstants\";\nexport {\n IPendingFlush,\n IPendingLocalState,\n IPendingMessage,\n IPendingState,\n} from \"./pendingStateManager\";\nexport { Summarizer } from \"./summarizer\";\nexport {\n EnqueueSummarizeResult,\n IAckSummaryResult,\n IBaseSummarizeResult,\n IBroadcastSummaryResult,\n ICancellationToken,\n IConnectableRuntime,\n IEnqueueSummarizeOptions,\n IGenerateSummaryTreeResult,\n IGeneratedSummaryStats,\n INackSummaryResult,\n IOnDemandSummarizeOptions,\n IProvideSummarizer,\n IRefreshSummaryAckOptions,\n ISubmitSummaryOpResult,\n ISubmitSummaryOptions,\n ISummarizeOptions,\n ISummarizeResults,\n ISummarizer,\n ISummarizerEvents,\n ISummarizerInternalsProvider,\n ISummarizerRuntime,\n ISummarizingWarning,\n ISummaryCancellationToken,\n IUploadSummaryResult,\n SubmitSummaryResult,\n SummarizeResultPart,\n SummarizerStopReason,\n} from \"./summarizerTypes\";\nexport {\n IAckedSummary,\n IClientSummaryWatcher,\n ISummary,\n ISummaryCollectionOpEvents,\n ISummaryAckMessage,\n ISummaryNackMessage,\n ISummaryOpMessage,\n OpActionEventListener,\n OpActionEventName,\n SummaryCollection,\n} from \"./summaryCollection\";\nexport { ICancellableSummarizerController, neverCancelledSummaryToken } from \"./runWhileConnectedCoordinator\";\nexport { IChunkedOp, unpackRuntimeMessage } from \"./opLifecycle\";\n"]}
@@ -0,0 +1,30 @@
1
+ /*!
2
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
3
+ * Licensed under the MIT License.
4
+ */
5
+ import { ICompressionRuntimeOptions } from "../containerRuntime";
6
+ import { BatchMessage, IBatch, IBatchCheckpoint } from "./definitions";
7
+ export interface IBatchManagerOptions {
8
+ readonly hardLimit: number;
9
+ readonly softLimit?: number;
10
+ readonly compressionOptions?: ICompressionRuntimeOptions;
11
+ }
12
+ /**
13
+ * Helper class that manages partial batch & rollback.
14
+ */
15
+ export declare class BatchManager {
16
+ readonly options: IBatchManagerOptions;
17
+ private pendingBatch;
18
+ private batchContentSize;
19
+ get length(): number;
20
+ get contentSizeInBytes(): number;
21
+ constructor(options: IBatchManagerOptions);
22
+ push(message: BatchMessage): boolean;
23
+ get empty(): boolean;
24
+ popBatch(): IBatch;
25
+ /**
26
+ * Capture the pending state at this point
27
+ */
28
+ checkpoint(): IBatchCheckpoint;
29
+ }
30
+ //# sourceMappingURL=batchManager.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"batchManager.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/batchManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,0BAA0B,EAAE,MAAM,qBAAqB,CAAC;AACjE,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAEvE,MAAM,WAAW,oBAAoB;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,kBAAkB,CAAC,EAAE,0BAA0B,CAAC;CAC5D;AAED;;GAEG;AACH,qBAAa,YAAY;aAOO,OAAO,EAAE,oBAAoB;IANzD,OAAO,CAAC,YAAY,CAAsB;IAC1C,OAAO,CAAC,gBAAgB,CAAK;IAE7B,IAAW,MAAM,WAAuC;IACxD,IAAW,kBAAkB,WAAoC;gBAErC,OAAO,EAAE,oBAAoB;IAElD,IAAI,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO;IA+B3C,IAAW,KAAK,YAA6C;IAEtD,QAAQ,IAAI,MAAM;IAYzB;;OAEG;IACI,UAAU,IAAI,gBAAgB;CAexC"}
@@ -9,16 +9,16 @@ exports.BatchManager = void 0;
9
9
  * Helper class that manages partial batch & rollback.
10
10
  */
11
11
  class BatchManager {
12
- constructor(hardLimit, softLimit) {
13
- this.hardLimit = hardLimit;
14
- this.softLimit = softLimit;
12
+ constructor(options) {
13
+ this.options = options;
15
14
  this.pendingBatch = [];
16
15
  this.batchContentSize = 0;
17
16
  }
18
17
  get length() { return this.pendingBatch.length; }
19
- get limit() { return this.hardLimit; }
18
+ get contentSizeInBytes() { return this.batchContentSize; }
20
19
  push(message) {
21
- const contentSize = this.batchContentSize + message.contents.length;
20
+ var _a, _b;
21
+ const contentSize = this.batchContentSize + ((_b = (_a = message.contents) === null || _a === void 0 ? void 0 : _a.length) !== null && _b !== void 0 ? _b : 0);
22
22
  const opCount = this.pendingBatch.length;
23
23
  // Attempt to estimate batch size, aka socket message size.
24
24
  // Each op has pretty large envelope, estimating to be 200 bytes.
@@ -29,10 +29,14 @@ class BatchManager {
29
29
  // If we were provided soft limit, check for exceeding it.
30
30
  // But only if we have any ops, as the intention here is to flush existing ops (on exceeding this limit)
31
31
  // and start over. That's not an option if we have no ops.
32
- if (this.softLimit !== undefined && this.length > 0 && socketMessageSize >= this.softLimit) {
32
+ // If compression is enabled, the soft and hard limit are ignored and the message will be pushed anyways.
33
+ // Cases where the message is still too large will be handled by the maxConsecutiveReconnects path.
34
+ if (this.options.softLimit !== undefined
35
+ && this.length > 0
36
+ && socketMessageSize >= this.options.softLimit) {
33
37
  return false;
34
38
  }
35
- if (socketMessageSize >= this.limit) {
39
+ if (socketMessageSize >= this.options.hardLimit) {
36
40
  return false;
37
41
  }
38
42
  this.batchContentSize = contentSize;
@@ -41,10 +45,13 @@ class BatchManager {
41
45
  }
42
46
  get empty() { return this.pendingBatch.length === 0; }
43
47
  popBatch() {
44
- const batch = this.pendingBatch;
48
+ const batch = {
49
+ content: this.pendingBatch,
50
+ contentSizeInBytes: this.batchContentSize,
51
+ };
45
52
  this.pendingBatch = [];
46
53
  this.batchContentSize = 0;
47
- return batch;
54
+ return addBatchMetadata(batch);
48
55
  }
49
56
  /**
50
57
  * Capture the pending state at this point
@@ -53,10 +60,11 @@ class BatchManager {
53
60
  const startPoint = this.pendingBatch.length;
54
61
  return {
55
62
  rollback: (process) => {
63
+ var _a, _b;
56
64
  for (let i = this.pendingBatch.length; i > startPoint;) {
57
65
  i--;
58
66
  const message = this.pendingBatch[i];
59
- this.batchContentSize -= message.contents.length;
67
+ this.batchContentSize -= (_b = (_a = message.contents) === null || _a === void 0 ? void 0 : _a.length) !== null && _b !== void 0 ? _b : 0;
60
68
  process(message);
61
69
  }
62
70
  this.pendingBatch.length = startPoint;
@@ -65,4 +73,11 @@ class BatchManager {
65
73
  }
66
74
  }
67
75
  exports.BatchManager = BatchManager;
76
+ const addBatchMetadata = (batch) => {
77
+ if (batch.content.length > 1) {
78
+ batch.content[0].metadata = Object.assign(Object.assign({}, batch.content[0].metadata), { batch: true });
79
+ batch.content[batch.content.length - 1].metadata = Object.assign(Object.assign({}, batch.content[batch.content.length - 1].metadata), { batch: false });
80
+ }
81
+ return batch;
82
+ };
68
83
  //# sourceMappingURL=batchManager.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"batchManager.js","sourceRoot":"","sources":["../../src/opLifecycle/batchManager.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAWH;;GAEG;AACH,MAAa,YAAY;IAOrB,YAA4B,OAA6B;QAA7B,YAAO,GAAP,OAAO,CAAsB;QANjD,iBAAY,GAAmB,EAAE,CAAC;QAClC,qBAAgB,GAAG,CAAC,CAAC;IAKgC,CAAC;IAH9D,IAAW,MAAM,KAAK,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;IACxD,IAAW,kBAAkB,KAAK,OAAO,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC;IAI1D,IAAI,CAAC,OAAqB;;QAC7B,MAAM,WAAW,GAAG,IAAI,CAAC,gBAAgB,GAAG,CAAC,MAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,MAAM,mCAAI,CAAC,CAAC,CAAC;QAC5E,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;QAEzC,2DAA2D;QAC3D,iEAAiE;QACjE,sGAAsG;QACtG,iGAAiG;QACjG,gFAAgF;QAChF,MAAM,iBAAiB,GAAG,WAAW,GAAG,GAAG,GAAG,OAAO,CAAC;QAEtD,0DAA0D;QAC1D,wGAAwG;QACxG,0DAA0D;QAC1D,yGAAyG;QACzG,mGAAmG;QACnG,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS;eACjC,IAAI,CAAC,MAAM,GAAG,CAAC;eACf,iBAAiB,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YAChD,OAAO,KAAK,CAAC;SAChB;QAED,IAAI,iBAAiB,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YAC7C,OAAO,KAAK,CAAC;SAChB;QAED,IAAI,CAAC,gBAAgB,GAAG,WAAW,CAAC;QACpC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAChC,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,IAAW,KAAK,KAAK,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC;IAEtD,QAAQ;QACX,MAAM,KAAK,GAAW;YAClB,OAAO,EAAE,IAAI,CAAC,YAAY;YAC1B,kBAAkB,EAAE,IAAI,CAAC,gBAAgB;SAC5C,CAAC;QAEF,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC;QAE1B,OAAO,gBAAgB,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC;IAED;;OAEG;IACI,UAAU;QACb,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;QAC5C,OAAO;YACH,QAAQ,EAAE,CAAC,OAAwC,EAAE,EAAE;;gBACnD,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,GAAG,UAAU,GAAG;oBACpD,CAAC,EAAE,CAAC;oBACJ,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;oBACrC,IAAI,CAAC,gBAAgB,IAAI,MAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,MAAM,mCAAI,CAAC,CAAC;oBACvD,OAAO,CAAC,OAAO,CAAC,CAAC;iBACpB;gBAED,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,UAAU,CAAC;YAC1C,CAAC;SACJ,CAAC;IACN,CAAC;CACJ;AAxED,oCAwEC;AAED,MAAM,gBAAgB,GAAG,CAAC,KAAa,EAAU,EAAE;IAC/C,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;QAC1B,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,mCAClB,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,KAC5B,KAAK,EAAE,IAAI,GACd,CAAC;QACF,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,QAAQ,mCACzC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,QAAQ,KACnD,KAAK,EAAE,KAAK,GACf,CAAC;KACL;IAED,OAAO,KAAK,CAAC;AACjB,CAAC,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ICompressionRuntimeOptions } from \"../containerRuntime\";\nimport { BatchMessage, IBatch, IBatchCheckpoint } from \"./definitions\";\n\nexport interface IBatchManagerOptions {\n readonly hardLimit: number;\n readonly softLimit?: number;\n readonly compressionOptions?: ICompressionRuntimeOptions;\n}\n\n/**\n * Helper class that manages partial batch & rollback.\n */\nexport class BatchManager {\n private pendingBatch: BatchMessage[] = [];\n private batchContentSize = 0;\n\n public get length() { return this.pendingBatch.length; }\n public get contentSizeInBytes() { return this.batchContentSize; }\n\n constructor(public readonly options: IBatchManagerOptions) { }\n\n public push(message: BatchMessage): boolean {\n const contentSize = this.batchContentSize + (message.contents?.length ?? 0);\n const opCount = this.pendingBatch.length;\n\n // Attempt to estimate batch size, aka socket message size.\n // Each op has pretty large envelope, estimating to be 200 bytes.\n // Also content will be strigified, and that adds a lot of overhead due to a lot of escape characters.\n // Not taking it into account, as compression work should help there - compressed payload will be\n // initially stored as base64, and that requires only 2 extra escape characters.\n const socketMessageSize = contentSize + 200 * opCount;\n\n // If we were provided soft limit, check for exceeding it.\n // But only if we have any ops, as the intention here is to flush existing ops (on exceeding this limit)\n // and start over. That's not an option if we have no ops.\n // If compression is enabled, the soft and hard limit are ignored and the message will be pushed anyways.\n // Cases where the message is still too large will be handled by the maxConsecutiveReconnects path.\n if (this.options.softLimit !== undefined\n && this.length > 0\n && socketMessageSize >= this.options.softLimit) {\n return false;\n }\n\n if (socketMessageSize >= this.options.hardLimit) {\n return false;\n }\n\n this.batchContentSize = contentSize;\n this.pendingBatch.push(message);\n return true;\n }\n\n public get empty() { return this.pendingBatch.length === 0; }\n\n public popBatch(): IBatch {\n const batch: IBatch = {\n content: this.pendingBatch,\n contentSizeInBytes: this.batchContentSize,\n };\n\n this.pendingBatch = [];\n this.batchContentSize = 0;\n\n return addBatchMetadata(batch);\n }\n\n /**\n * Capture the pending state at this point\n */\n public checkpoint(): IBatchCheckpoint {\n const startPoint = this.pendingBatch.length;\n return {\n rollback: (process: (message: BatchMessage) => void) => {\n for (let i = this.pendingBatch.length; i > startPoint;) {\n i--;\n const message = this.pendingBatch[i];\n this.batchContentSize -= message.contents?.length ?? 0;\n process(message);\n }\n\n this.pendingBatch.length = startPoint;\n },\n };\n }\n}\n\nconst addBatchMetadata = (batch: IBatch): IBatch => {\n if (batch.content.length > 1) {\n batch.content[0].metadata = {\n ...batch.content[0].metadata,\n batch: true\n };\n batch.content[batch.content.length - 1].metadata = {\n ...batch.content[batch.content.length - 1].metadata,\n batch: false\n };\n }\n\n return batch;\n};\n"]}
@@ -0,0 +1,40 @@
1
+ /*!
2
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
3
+ * Licensed under the MIT License.
4
+ */
5
+ import { IBatchMessage } from "@fluidframework/container-definitions";
6
+ import { MessageType } from "@fluidframework/protocol-definitions";
7
+ import { CompressionAlgorithms, ContainerMessageType, ContainerRuntimeMessage } from "..";
8
+ /**
9
+ * Batch message type used internally by the runtime
10
+ */
11
+ export declare type BatchMessage = IBatchMessage & {
12
+ localOpMetadata: unknown;
13
+ deserializedContent: ContainerRuntimeMessage;
14
+ referenceSequenceNumber: number;
15
+ compression?: CompressionAlgorithms;
16
+ };
17
+ /**
18
+ * Batch interface used internally by the runtime.
19
+ */
20
+ export interface IBatch {
21
+ /**
22
+ * Sum of the in-memory content sizes of all messages in the batch.
23
+ * If the batch is compressed, this number reflects the post-compression size.
24
+ */
25
+ readonly contentSizeInBytes: number;
26
+ /**
27
+ * All the messages in the batch
28
+ */
29
+ readonly content: BatchMessage[];
30
+ }
31
+ export interface IBatchCheckpoint {
32
+ rollback: (action: (message: BatchMessage) => void) => void;
33
+ }
34
+ export interface IChunkedOp {
35
+ chunkId: number;
36
+ totalChunks: number;
37
+ contents: string;
38
+ originalType: MessageType | ContainerMessageType;
39
+ }
40
+ //# sourceMappingURL=definitions.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"definitions.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/definitions.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,uCAAuC,CAAC;AACtE,OAAO,EAAE,WAAW,EAAE,MAAM,sCAAsC,CAAC;AACnE,OAAO,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,uBAAuB,EAAE,MAAM,IAAI,CAAC;AAE1F;;GAEG;AACH,oBAAY,YAAY,GAAG,aAAa,GAAG;IACvC,eAAe,EAAE,OAAO,CAAC;IACzB,mBAAmB,EAAE,uBAAuB,CAAC;IAC7C,uBAAuB,EAAE,MAAM,CAAC;IAChC,WAAW,CAAC,EAAE,qBAAqB,CAAC;CACvC,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,MAAM;IACnB;;;OAGG;IACH,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,YAAY,EAAE,CAAC;CACpC;AAED,MAAM,WAAW,gBAAgB;IAC7B,QAAQ,EAAE,CAAC,MAAM,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,IAAI,KAAK,IAAI,CAAC;CAC/D;AAED,MAAM,WAAW,UAAU;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,WAAW,GAAG,oBAAoB,CAAC;CACpD"}
@@ -0,0 +1,7 @@
1
+ "use strict";
2
+ /*!
3
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
4
+ * Licensed under the MIT License.
5
+ */
6
+ Object.defineProperty(exports, "__esModule", { value: true });
7
+ //# sourceMappingURL=definitions.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"definitions.js","sourceRoot":"","sources":["../../src/opLifecycle/definitions.ts"],"names":[],"mappings":";AAAA;;;GAGG","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { IBatchMessage } from \"@fluidframework/container-definitions\";\nimport { MessageType } from \"@fluidframework/protocol-definitions\";\nimport { CompressionAlgorithms, ContainerMessageType, ContainerRuntimeMessage } from \"..\";\n\n/**\n * Batch message type used internally by the runtime\n */\nexport type BatchMessage = IBatchMessage & {\n localOpMetadata: unknown;\n deserializedContent: ContainerRuntimeMessage;\n referenceSequenceNumber: number;\n compression?: CompressionAlgorithms;\n};\n\n/**\n * Batch interface used internally by the runtime.\n */\nexport interface IBatch {\n /**\n * Sum of the in-memory content sizes of all messages in the batch.\n * If the batch is compressed, this number reflects the post-compression size.\n */\n readonly contentSizeInBytes: number;\n /**\n * All the messages in the batch\n */\n readonly content: BatchMessage[];\n}\n\nexport interface IBatchCheckpoint {\n rollback: (action: (message: BatchMessage) => void) => void;\n}\n\nexport interface IChunkedOp {\n chunkId: number;\n totalChunks: number;\n contents: string;\n originalType: MessageType | ContainerMessageType;\n}\n"]}
@@ -0,0 +1,12 @@
1
+ /*!
2
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
3
+ * Licensed under the MIT License.
4
+ */
5
+ export { BatchManager } from "./batchManager";
6
+ export { BatchMessage, IBatch, IBatchCheckpoint, IChunkedOp, } from "./definitions";
7
+ export { Outbox } from "./outbox";
8
+ export { OpCompressor } from "./opCompressor";
9
+ export { OpDecompressor } from "./opDecompressor";
10
+ export { OpSplitter } from "./opSplitter";
11
+ export { RemoteMessageProcessor, unpackRuntimeMessage } from "./remoteMessageProcessor";
12
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EACH,YAAY,EACZ,MAAM,EACN,gBAAgB,EAChB,UAAU,GACb,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAClC,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,sBAAsB,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC"}
@@ -0,0 +1,21 @@
1
+ "use strict";
2
+ /*!
3
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
4
+ * Licensed under the MIT License.
5
+ */
6
+ Object.defineProperty(exports, "__esModule", { value: true });
7
+ exports.unpackRuntimeMessage = exports.RemoteMessageProcessor = exports.OpSplitter = exports.OpDecompressor = exports.OpCompressor = exports.Outbox = exports.BatchManager = void 0;
8
+ var batchManager_1 = require("./batchManager");
9
+ Object.defineProperty(exports, "BatchManager", { enumerable: true, get: function () { return batchManager_1.BatchManager; } });
10
+ var outbox_1 = require("./outbox");
11
+ Object.defineProperty(exports, "Outbox", { enumerable: true, get: function () { return outbox_1.Outbox; } });
12
+ var opCompressor_1 = require("./opCompressor");
13
+ Object.defineProperty(exports, "OpCompressor", { enumerable: true, get: function () { return opCompressor_1.OpCompressor; } });
14
+ var opDecompressor_1 = require("./opDecompressor");
15
+ Object.defineProperty(exports, "OpDecompressor", { enumerable: true, get: function () { return opDecompressor_1.OpDecompressor; } });
16
+ var opSplitter_1 = require("./opSplitter");
17
+ Object.defineProperty(exports, "OpSplitter", { enumerable: true, get: function () { return opSplitter_1.OpSplitter; } });
18
+ var remoteMessageProcessor_1 = require("./remoteMessageProcessor");
19
+ Object.defineProperty(exports, "RemoteMessageProcessor", { enumerable: true, get: function () { return remoteMessageProcessor_1.RemoteMessageProcessor; } });
20
+ Object.defineProperty(exports, "unpackRuntimeMessage", { enumerable: true, get: function () { return remoteMessageProcessor_1.unpackRuntimeMessage; } });
21
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/opLifecycle/index.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,+CAA8C;AAArC,4GAAA,YAAY,OAAA;AAOrB,mCAAkC;AAAzB,gGAAA,MAAM,OAAA;AACf,+CAA8C;AAArC,4GAAA,YAAY,OAAA;AACrB,mDAAkD;AAAzC,gHAAA,cAAc,OAAA;AACvB,2CAA0C;AAAjC,wGAAA,UAAU,OAAA;AACnB,mEAAwF;AAA/E,gIAAA,sBAAsB,OAAA;AAAE,8HAAA,oBAAoB,OAAA","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nexport { BatchManager } from \"./batchManager\";\nexport {\n BatchMessage,\n IBatch,\n IBatchCheckpoint,\n IChunkedOp,\n} from \"./definitions\";\nexport { Outbox } from \"./outbox\";\nexport { OpCompressor } from \"./opCompressor\";\nexport { OpDecompressor } from \"./opDecompressor\";\nexport { OpSplitter } from \"./opSplitter\";\nexport { RemoteMessageProcessor, unpackRuntimeMessage } from \"./remoteMessageProcessor\";\n"]}
@@ -0,0 +1,18 @@
1
+ /*!
2
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
3
+ * Licensed under the MIT License.
4
+ */
5
+ import { ITelemetryLogger } from "@fluidframework/common-definitions";
6
+ import { IBatch } from "./definitions";
7
+ /**
8
+ * Compresses batches of ops. It generates a single compressed op that contains
9
+ * the contents of each op in the batch. It then submits empty ops for each original
10
+ * op to reserve sequence numbers.
11
+ */
12
+ export declare class OpCompressor {
13
+ private readonly logger;
14
+ private compressedBatchCount;
15
+ constructor(logger: ITelemetryLogger);
16
+ compressBatch(batch: IBatch): IBatch;
17
+ }
18
+ //# sourceMappingURL=opCompressor.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"opCompressor.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/opCompressor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAKtE,OAAO,EAAE,MAAM,EAAgB,MAAM,eAAe,CAAC;AAErD;;;;GAIG;AACH,qBAAa,YAAY;IACrB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;IACxB,OAAO,CAAC,oBAAoB,CAAK;gBAErB,MAAM,EAAE,gBAAgB;IAI7B,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;CAsC9C"}
@@ -0,0 +1,53 @@
1
+ "use strict";
2
+ /*!
3
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
4
+ * Licensed under the MIT License.
5
+ */
6
+ Object.defineProperty(exports, "__esModule", { value: true });
7
+ exports.OpCompressor = void 0;
8
+ const common_utils_1 = require("@fluidframework/common-utils");
9
+ const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
10
+ const lz4js_1 = require("lz4js");
11
+ const containerRuntime_1 = require("../containerRuntime");
12
+ /**
13
+ * Compresses batches of ops. It generates a single compressed op that contains
14
+ * the contents of each op in the batch. It then submits empty ops for each original
15
+ * op to reserve sequence numbers.
16
+ */
17
+ class OpCompressor {
18
+ constructor(logger) {
19
+ this.compressedBatchCount = 0;
20
+ this.logger = telemetry_utils_1.ChildLogger.create(logger, "OpCompressor");
21
+ }
22
+ compressBatch(batch) {
23
+ const messages = [];
24
+ this.compressedBatchCount++;
25
+ const contentToCompress = [];
26
+ for (const message of batch.content) {
27
+ contentToCompress.push(message.deserializedContent);
28
+ }
29
+ const compressionStart = Date.now();
30
+ const contentsAsBuffer = new TextEncoder().encode(JSON.stringify(contentToCompress));
31
+ const compressedContents = (0, lz4js_1.compress)(contentsAsBuffer);
32
+ const compressedContent = common_utils_1.IsoBuffer.from(compressedContents).toString("base64");
33
+ const duration = Date.now() - compressionStart;
34
+ if (batch.contentSizeInBytes > 200000 || this.compressedBatchCount % 25) {
35
+ this.logger.sendPerformanceEvent({
36
+ eventName: "CompressedBatch",
37
+ duration,
38
+ sizeBeforeCompression: batch.contentSizeInBytes,
39
+ sizeAfterCompression: compressedContent.length,
40
+ });
41
+ }
42
+ messages.push(Object.assign(Object.assign({}, batch.content[0]), { contents: JSON.stringify({ packedContents: compressedContent }), metadata: Object.assign(Object.assign({}, batch.content[0].metadata), { compressed: true }), compression: containerRuntime_1.CompressionAlgorithms.lz4 }));
43
+ for (const message of batch.content.slice(1)) {
44
+ messages.push(Object.assign(Object.assign({}, message), { contents: undefined }));
45
+ }
46
+ return {
47
+ contentSizeInBytes: compressedContent.length,
48
+ content: messages,
49
+ };
50
+ }
51
+ }
52
+ exports.OpCompressor = OpCompressor;
53
+ //# sourceMappingURL=opCompressor.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"opCompressor.js","sourceRoot":"","sources":["../../src/opLifecycle/opCompressor.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAGH,+DAAyD;AACzD,qEAA8D;AAC9D,iCAAiC;AACjC,0DAAqF;AAGrF;;;;GAIG;AACH,MAAa,YAAY;IAIrB,YAAY,MAAwB;QAF5B,yBAAoB,GAAG,CAAC,CAAC;QAG7B,IAAI,CAAC,MAAM,GAAG,6BAAW,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IAC7D,CAAC;IAEM,aAAa,CAAC,KAAa;QAC9B,MAAM,QAAQ,GAAmB,EAAE,CAAC;QACpC,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAC5B,MAAM,iBAAiB,GAA8B,EAAE,CAAC;QACxD,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,OAAO,EAAE;YACjC,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;SACvD;QAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACpC,MAAM,gBAAgB,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC,CAAC;QACrF,MAAM,kBAAkB,GAAG,IAAA,gBAAQ,EAAC,gBAAgB,CAAC,CAAC;QACtD,MAAM,iBAAiB,GAAG,wBAAS,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAChF,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,gBAAgB,CAAC;QAE/C,IAAI,KAAK,CAAC,kBAAkB,GAAG,MAAM,IAAI,IAAI,CAAC,oBAAoB,GAAG,EAAE,EAAE;YACrE,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;gBAC7B,SAAS,EAAE,iBAAiB;gBAC5B,QAAQ;gBACR,qBAAqB,EAAE,KAAK,CAAC,kBAAkB;gBAC/C,oBAAoB,EAAE,iBAAiB,CAAC,MAAM;aACjD,CAAC,CAAC;SACN;QAED,QAAQ,CAAC,IAAI,iCACN,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAE,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,cAAc,EAAE,iBAAiB,EAAE,CAAC,EACpF,QAAQ,kCAAO,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,KAAE,UAAU,EAAE,IAAI,KAC1D,WAAW,EAAE,wCAAqB,CAAC,GAAG,IACxC,CAAC;QAEH,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;YAC1C,QAAQ,CAAC,IAAI,iCAAM,OAAO,KAAE,QAAQ,EAAE,SAAS,IAAG,CAAC;SACtD;QAED,OAAO;YACH,kBAAkB,EAAE,iBAAiB,CAAC,MAAM;YAC5C,OAAO,EAAE,QAAQ;SACpB,CAAC;IACN,CAAC;CACJ;AA9CD,oCA8CC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { IsoBuffer } from \"@fluidframework/common-utils\";\nimport { ChildLogger } from \"@fluidframework/telemetry-utils\";\nimport { compress } from \"lz4js\";\nimport { CompressionAlgorithms, ContainerRuntimeMessage } from \"../containerRuntime\";\nimport { IBatch, BatchMessage } from \"./definitions\";\n\n/**\n * Compresses batches of ops. It generates a single compressed op that contains\n * the contents of each op in the batch. It then submits empty ops for each original\n * op to reserve sequence numbers.\n */\nexport class OpCompressor {\n private readonly logger;\n private compressedBatchCount = 0;\n\n constructor(logger: ITelemetryLogger) {\n this.logger = ChildLogger.create(logger, \"OpCompressor\");\n }\n\n public compressBatch(batch: IBatch): IBatch {\n const messages: BatchMessage[] = [];\n this.compressedBatchCount++;\n const contentToCompress: ContainerRuntimeMessage[] = [];\n for (const message of batch.content) {\n contentToCompress.push(message.deserializedContent);\n }\n\n const compressionStart = Date.now();\n const contentsAsBuffer = new TextEncoder().encode(JSON.stringify(contentToCompress));\n const compressedContents = compress(contentsAsBuffer);\n const compressedContent = IsoBuffer.from(compressedContents).toString(\"base64\");\n const duration = Date.now() - compressionStart;\n\n if (batch.contentSizeInBytes > 200000 || this.compressedBatchCount % 25) {\n this.logger.sendPerformanceEvent({\n eventName: \"CompressedBatch\",\n duration,\n sizeBeforeCompression: batch.contentSizeInBytes,\n sizeAfterCompression: compressedContent.length,\n });\n }\n\n messages.push({\n ...batch.content[0], contents: JSON.stringify({ packedContents: compressedContent }),\n metadata: { ...batch.content[0].metadata, compressed: true },\n compression: CompressionAlgorithms.lz4,\n });\n\n for (const message of batch.content.slice(1)) {\n messages.push({ ...message, contents: undefined });\n }\n\n return {\n contentSizeInBytes: compressedContent.length,\n content: messages,\n };\n }\n}\n"]}
@@ -0,0 +1,20 @@
1
+ /*!
2
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
3
+ * Licensed under the MIT License.
4
+ */
5
+ import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
6
+ /**
7
+ * State machine that "unrolls" contents of compressed batches of ops after decompressing them.
8
+ * This class relies on some implicit contracts defined below:
9
+ * 1. A compressed batch will have its first message with batch metadata set to true and compressed set to true
10
+ * 2. Messages in the middle of a compressed batch will have neither batch metadata nor the compression property set
11
+ * 3. The final message of a batch will have batch metadata set to false
12
+ * 4. An individually compressed op will have undefined batch metadata and compression set to true
13
+ */
14
+ export declare class OpDecompressor {
15
+ private activeBatch;
16
+ private rootMessageContents;
17
+ private processedCount;
18
+ processMessage(message: ISequencedDocumentMessage): ISequencedDocumentMessage;
19
+ }
20
+ //# sourceMappingURL=opDecompressor.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"opDecompressor.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/opDecompressor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,yBAAyB,EAAE,MAAM,sCAAsC,CAAC;AAIjF;;;;;;;GAOG;AACH,qBAAa,cAAc;IACvB,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,mBAAmB,CAAkB;IAC7C,OAAO,CAAC,cAAc,CAAK;IAEpB,cAAc,CAAC,OAAO,EAAE,yBAAyB,GAAG,yBAAyB;CA4DvF"}
@@ -0,0 +1,72 @@
1
+ "use strict";
2
+ /*!
3
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
4
+ * Licensed under the MIT License.
5
+ */
6
+ Object.defineProperty(exports, "__esModule", { value: true });
7
+ exports.OpDecompressor = void 0;
8
+ const lz4js_1 = require("lz4js");
9
+ const common_utils_1 = require("@fluidframework/common-utils");
10
+ const containerRuntime_1 = require("../containerRuntime");
11
+ /**
12
+ * State machine that "unrolls" contents of compressed batches of ops after decompressing them.
13
+ * This class relies on some implicit contracts defined below:
14
+ * 1. A compressed batch will have its first message with batch metadata set to true and compressed set to true
15
+ * 2. Messages in the middle of a compressed batch will have neither batch metadata nor the compression property set
16
+ * 3. The final message of a batch will have batch metadata set to false
17
+ * 4. An individually compressed op will have undefined batch metadata and compression set to true
18
+ */
19
+ class OpDecompressor {
20
+ constructor() {
21
+ this.activeBatch = false;
22
+ this.processedCount = 0;
23
+ }
24
+ processMessage(message) {
25
+ var _a, _b, _c, _d, _e, _f;
26
+ // We're checking for compression = true or top level compression property so
27
+ // that we can enable compression without waiting on all ordering services
28
+ // to pick up protocol change. Eventually only the top level property should
29
+ // be used.
30
+ if (((_a = message.metadata) === null || _a === void 0 ? void 0 : _a.batch) === true
31
+ && (((_b = message.metadata) === null || _b === void 0 ? void 0 : _b.compressed) || message.compression !== undefined)) {
32
+ // Beginning of a compressed batch
33
+ (0, common_utils_1.assert)(this.activeBatch === false, 0x4b8 /* shouldn't have multiple active batches */);
34
+ if (message.compression) {
35
+ // lz4 is the only supported compression algorithm for now
36
+ (0, common_utils_1.assert)(message.compression === containerRuntime_1.CompressionAlgorithms.lz4, 0x4b9 /* lz4 is currently the only supported compression algorithm */);
37
+ }
38
+ this.activeBatch = true;
39
+ const contents = common_utils_1.IsoBuffer.from(message.contents.packedContents, "base64");
40
+ const decompressedMessage = (0, lz4js_1.decompress)(contents);
41
+ const intoString = (0, common_utils_1.Uint8ArrayToString)(decompressedMessage);
42
+ const asObj = JSON.parse(intoString);
43
+ this.rootMessageContents = asObj;
44
+ return Object.assign(Object.assign({}, message), { contents: this.rootMessageContents[this.processedCount++] });
45
+ }
46
+ if (this.rootMessageContents !== undefined && ((_c = message.metadata) === null || _c === void 0 ? void 0 : _c.batch) === undefined && this.activeBatch) {
47
+ // Continuation of compressed batch
48
+ return Object.assign(Object.assign({}, message), { contents: this.rootMessageContents[this.processedCount++] });
49
+ }
50
+ if (this.rootMessageContents !== undefined && ((_d = message.metadata) === null || _d === void 0 ? void 0 : _d.batch) === false) {
51
+ // End of compressed batch
52
+ const returnMessage = Object.assign(Object.assign({}, message), { contents: this.rootMessageContents[this.processedCount++] });
53
+ this.activeBatch = false;
54
+ this.rootMessageContents = undefined;
55
+ this.processedCount = 0;
56
+ return returnMessage;
57
+ }
58
+ if (((_e = message.metadata) === null || _e === void 0 ? void 0 : _e.batch) === undefined &&
59
+ (((_f = message.metadata) === null || _f === void 0 ? void 0 : _f.compressed) || message.compression === containerRuntime_1.CompressionAlgorithms.lz4)) {
60
+ // Single compressed message
61
+ (0, common_utils_1.assert)(this.activeBatch === false, 0x4ba /* shouldn't receive compressed message in middle of a batch */);
62
+ const contents = common_utils_1.IsoBuffer.from(message.contents.packedContents, "base64");
63
+ const decompressedMessage = (0, lz4js_1.decompress)(contents);
64
+ const intoString = new TextDecoder().decode(decompressedMessage);
65
+ const asObj = JSON.parse(intoString);
66
+ return Object.assign(Object.assign({}, message), { contents: asObj[0] });
67
+ }
68
+ return message;
69
+ }
70
+ }
71
+ exports.OpDecompressor = OpDecompressor;
72
+ //# sourceMappingURL=opDecompressor.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"opDecompressor.js","sourceRoot":"","sources":["../../src/opLifecycle/opDecompressor.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,iCAAmC;AAEnC,+DAAqF;AACrF,0DAA4D;AAE5D;;;;;;;GAOG;AACH,MAAa,cAAc;IAA3B;QACY,gBAAW,GAAG,KAAK,CAAC;QAEpB,mBAAc,GAAG,CAAC,CAAC;IA8D/B,CAAC;IA5DU,cAAc,CAAC,OAAkC;;QACpD,6EAA6E;QAC7E,0EAA0E;QAC1E,4EAA4E;QAC5E,WAAW;QACX,IAAI,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,KAAK,MAAK,IAAI;eAC7B,CAAC,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,UAAU,KAAI,OAAO,CAAC,WAAW,KAAK,SAAS,CAAC,EAAE;YACxE,kCAAkC;YAClC,IAAA,qBAAM,EAAC,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,KAAK,CAAC,4CAA4C,CAAC,CAAC;YACvF,IAAI,OAAO,CAAC,WAAW,EAAE;gBACrB,0DAA0D;gBAC1D,IAAA,qBAAM,EAAC,OAAO,CAAC,WAAW,KAAK,wCAAqB,CAAC,GAAG,EACpD,KAAK,CAAC,+DAA+D,CAAC,CAAC;aAC9E;YAED,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YAExB,MAAM,QAAQ,GAAG,wBAAS,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;YAC3E,MAAM,mBAAmB,GAAG,IAAA,kBAAU,EAAC,QAAQ,CAAC,CAAC;YACjD,MAAM,UAAU,GAAG,IAAA,iCAAkB,EAAC,mBAAmB,CAAC,CAAC;YAC3D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACrC,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;YAEjC,uCAAY,OAAO,KAAE,QAAQ,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,IAAG;SACpF;QAED,IAAI,IAAI,CAAC,mBAAmB,KAAK,SAAS,IAAI,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,KAAK,MAAK,SAAS,IAAI,IAAI,CAAC,WAAW,EAAE;YACrG,mCAAmC;YACnC,uCAAY,OAAO,KAAE,QAAQ,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,IAAG;SACpF;QAED,IAAI,IAAI,CAAC,mBAAmB,KAAK,SAAS,IAAI,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,KAAK,MAAK,KAAK,EAAE;YAC7E,0BAA0B;YAC1B,MAAM,aAAa,mCACZ,OAAO,KACV,QAAQ,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,GAC5D,CAAC;YAEF,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;YACzB,IAAI,CAAC,mBAAmB,GAAG,SAAS,CAAC;YACrC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;YAExB,OAAO,aAAa,CAAC;SACxB;QAED,IAAI,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,KAAK,MAAK,SAAS;YACrC,CAAC,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,UAAU,KAAI,OAAO,CAAC,WAAW,KAAK,wCAAqB,CAAC,GAAG,CAAC,EAAE;YACrF,4BAA4B;YAC5B,IAAA,qBAAM,EAAC,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,KAAK,CAAC,+DAA+D,CAAC,CAAC;YAE1G,MAAM,QAAQ,GAAG,wBAAS,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;YAC3E,MAAM,mBAAmB,GAAG,IAAA,kBAAU,EAAC,QAAQ,CAAC,CAAC;YACjD,MAAM,UAAU,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC;YACjE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAErC,uCAAY,OAAO,KAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,IAAG;SAC7C;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;CACJ;AAjED,wCAiEC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { decompress } from \"lz4js\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/protocol-definitions\";\nimport { assert, IsoBuffer, Uint8ArrayToString } from \"@fluidframework/common-utils\";\nimport { CompressionAlgorithms } from \"../containerRuntime\";\n\n/**\n * State machine that \"unrolls\" contents of compressed batches of ops after decompressing them.\n * This class relies on some implicit contracts defined below:\n * 1. A compressed batch will have its first message with batch metadata set to true and compressed set to true\n * 2. Messages in the middle of a compressed batch will have neither batch metadata nor the compression property set\n * 3. The final message of a batch will have batch metadata set to false\n * 4. An individually compressed op will have undefined batch metadata and compression set to true\n */\nexport class OpDecompressor {\n private activeBatch = false;\n private rootMessageContents: any | undefined;\n private processedCount = 0;\n\n public processMessage(message: ISequencedDocumentMessage): ISequencedDocumentMessage {\n // We're checking for compression = true or top level compression property so\n // that we can enable compression without waiting on all ordering services\n // to pick up protocol change. Eventually only the top level property should\n // be used.\n if (message.metadata?.batch === true\n && (message.metadata?.compressed || message.compression !== undefined)) {\n // Beginning of a compressed batch\n assert(this.activeBatch === false, 0x4b8 /* shouldn't have multiple active batches */);\n if (message.compression) {\n // lz4 is the only supported compression algorithm for now\n assert(message.compression === CompressionAlgorithms.lz4,\n 0x4b9 /* lz4 is currently the only supported compression algorithm */);\n }\n\n this.activeBatch = true;\n\n const contents = IsoBuffer.from(message.contents.packedContents, \"base64\");\n const decompressedMessage = decompress(contents);\n const intoString = Uint8ArrayToString(decompressedMessage);\n const asObj = JSON.parse(intoString);\n this.rootMessageContents = asObj;\n\n return { ...message, contents: this.rootMessageContents[this.processedCount++] };\n }\n\n if (this.rootMessageContents !== undefined && message.metadata?.batch === undefined && this.activeBatch) {\n // Continuation of compressed batch\n return { ...message, contents: this.rootMessageContents[this.processedCount++] };\n }\n\n if (this.rootMessageContents !== undefined && message.metadata?.batch === false) {\n // End of compressed batch\n const returnMessage = {\n ...message,\n contents: this.rootMessageContents[this.processedCount++]\n };\n\n this.activeBatch = false;\n this.rootMessageContents = undefined;\n this.processedCount = 0;\n\n return returnMessage;\n }\n\n if (message.metadata?.batch === undefined &&\n (message.metadata?.compressed || message.compression === CompressionAlgorithms.lz4)) {\n // Single compressed message\n assert(this.activeBatch === false, 0x4ba /* shouldn't receive compressed message in middle of a batch */);\n\n const contents = IsoBuffer.from(message.contents.packedContents, \"base64\");\n const decompressedMessage = decompress(contents);\n const intoString = new TextDecoder().decode(decompressedMessage);\n const asObj = JSON.parse(intoString);\n\n return { ...message, contents: asObj[0] };\n }\n\n return message;\n }\n}\n"]}