loro-repo 0.16.7 → 0.16.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -1,5 +1,6 @@
1
1
  const require_chunk = require('./chunk.cjs');
2
2
  const require_logging = require('./logging.cjs');
3
+ const require_meta_snapshot = require('./meta-snapshot.cjs');
3
4
  let __loro_dev_flock_wasm = require("@loro-dev/flock-wasm");
4
5
  __loro_dev_flock_wasm = require_chunk.__toESM(__loro_dev_flock_wasm);
5
6
  let loro_crdt = require("loro-crdt");
@@ -1025,20 +1026,36 @@ var SyncRunner = class {
1025
1026
  //#endregion
1026
1027
  //#region src/internal/meta-persister.ts
1027
1028
  const DEFAULT_META_PERSIST_DEBOUNCE_MS = 5e3;
1029
+ const DEFAULT_META_COMPACTION_DEBOUNCE_MS = 6e4;
1030
+ const DEFAULT_META_COMPACTION_UPDATE_THRESHOLD = 0;
1031
+ const DEFAULT_META_COMPACTION_BYTE_THRESHOLD = 100 * 1024;
1028
1032
  var MetaPersister = class {
1029
1033
  getMetaFlock;
1030
1034
  storage;
1031
1035
  debounceMs;
1036
+ compactionDebounceMs;
1037
+ compactionUpdateThreshold;
1038
+ compactionByteThreshold;
1032
1039
  lastPersistedVersion;
1033
1040
  unsubscribe;
1034
- flushPromise = Promise.resolve();
1041
+ operationPromise = Promise.resolve();
1035
1042
  flushTimer;
1043
+ compactionTimer;
1044
+ compactionQueued = false;
1045
+ incrementalUpdatesSinceCompaction = 0;
1046
+ incrementalUpdateBytesSinceCompaction = 0;
1036
1047
  destroyed = false;
1037
1048
  constructor(options) {
1038
1049
  this.getMetaFlock = options.getMetaFlock;
1039
1050
  this.storage = options.storage;
1040
1051
  const configuredDebounce = options.debounceMs;
1041
1052
  this.debounceMs = typeof configuredDebounce === "number" && Number.isFinite(configuredDebounce) && configuredDebounce >= 0 ? configuredDebounce : DEFAULT_META_PERSIST_DEBOUNCE_MS;
1053
+ const configuredCompactionDebounce = options.compactionDebounceMs;
1054
+ this.compactionDebounceMs = typeof configuredCompactionDebounce === "number" && Number.isFinite(configuredCompactionDebounce) && configuredCompactionDebounce >= 0 ? configuredCompactionDebounce : DEFAULT_META_COMPACTION_DEBOUNCE_MS;
1055
+ const configuredCompactionThreshold = options.compactionUpdateThreshold;
1056
+ this.compactionUpdateThreshold = typeof configuredCompactionThreshold === "number" && Number.isFinite(configuredCompactionThreshold) && configuredCompactionThreshold >= 0 ? Math.floor(configuredCompactionThreshold) : DEFAULT_META_COMPACTION_UPDATE_THRESHOLD;
1057
+ const configuredCompactionByteThreshold = options.compactionByteThreshold;
1058
+ this.compactionByteThreshold = typeof configuredCompactionByteThreshold === "number" && Number.isFinite(configuredCompactionByteThreshold) && configuredCompactionByteThreshold >= 0 ? Math.floor(configuredCompactionByteThreshold) : DEFAULT_META_COMPACTION_BYTE_THRESHOLD;
1042
1059
  }
1043
1060
  start(initialVersion) {
1044
1061
  this.lastPersistedVersion = initialVersion;
@@ -1053,6 +1070,10 @@ var MetaPersister = class {
1053
1070
  clearTimeout(this.flushTimer);
1054
1071
  this.flushTimer = void 0;
1055
1072
  }
1073
+ if (this.compactionTimer) {
1074
+ clearTimeout(this.compactionTimer);
1075
+ this.compactionTimer = void 0;
1076
+ }
1056
1077
  if (this.unsubscribe) {
1057
1078
  this.unsubscribe();
1058
1079
  this.unsubscribe = void 0;
@@ -1069,34 +1090,82 @@ var MetaPersister = class {
1069
1090
  scheduleFlush() {
1070
1091
  if (this.destroyed) return;
1071
1092
  if (this.debounceMs === 0) {
1072
- this.flush();
1093
+ this.flush().catch(require_logging.logAsyncError("metadata flush"));
1073
1094
  return;
1074
1095
  }
1075
1096
  if (this.flushTimer) clearTimeout(this.flushTimer);
1076
1097
  this.flushTimer = setTimeout(() => {
1077
1098
  this.flushTimer = void 0;
1078
- this.flush();
1099
+ this.flush().catch(require_logging.logAsyncError("metadata flush"));
1079
1100
  }, this.debounceMs);
1080
1101
  }
1081
1102
  async flush() {
1082
- const run = this.flushPromise.catch(() => {}).then(() => this.flushInternal());
1083
- this.flushPromise = run;
1084
- await run;
1103
+ await this.enqueueOperation(() => this.flushInternal());
1085
1104
  }
1086
1105
  async flushInternal() {
1087
1106
  const flock = this.metaFlock;
1088
1107
  const currentVersion = flock.version();
1089
1108
  if (this.lastPersistedVersion && this.versionsEqual(currentVersion, this.lastPersistedVersion)) return;
1109
+ const previousVersion = this.lastPersistedVersion;
1090
1110
  if (!this.storage) {
1091
1111
  this.lastPersistedVersion = currentVersion;
1092
1112
  return;
1093
1113
  }
1094
- const encoded = flock.exportFile();
1095
- await this.storage.save({
1096
- type: "meta",
1097
- update: encoded
1098
- });
1114
+ if (!previousVersion || this.isEmptyVersion(previousVersion)) {
1115
+ const snapshot = flock.exportFile();
1116
+ await this.storage.save({
1117
+ type: "meta-snapshot",
1118
+ snapshot
1119
+ });
1120
+ this.incrementalUpdatesSinceCompaction = 0;
1121
+ this.incrementalUpdateBytesSinceCompaction = 0;
1122
+ } else {
1123
+ const update = require_meta_snapshot.encodeMetaJsonUpdate(await flock.exportJson({ from: previousVersion }));
1124
+ await this.storage.save({
1125
+ type: "meta-update",
1126
+ update
1127
+ });
1128
+ this.incrementalUpdatesSinceCompaction += 1;
1129
+ this.incrementalUpdateBytesSinceCompaction += update.byteLength;
1130
+ }
1099
1131
  this.lastPersistedVersion = currentVersion;
1132
+ this.scheduleMetaCompaction();
1133
+ }
1134
+ enqueueOperation(operation) {
1135
+ const run = this.operationPromise.catch(() => {}).then(operation);
1136
+ this.operationPromise = run;
1137
+ return run;
1138
+ }
1139
+ scheduleMetaCompaction() {
1140
+ if (this.destroyed || !this.storage?.compactMeta) return;
1141
+ if (this.incrementalUpdatesSinceCompaction === 0) return;
1142
+ if (this.hasReachedCompactionUpdateThreshold() || this.hasReachedCompactionByteThreshold()) {
1143
+ if (this.compactionTimer) {
1144
+ clearTimeout(this.compactionTimer);
1145
+ this.compactionTimer = void 0;
1146
+ }
1147
+ this.queueMetaCompaction();
1148
+ return;
1149
+ }
1150
+ if (this.compactionTimer) clearTimeout(this.compactionTimer);
1151
+ this.compactionTimer = setTimeout(() => {
1152
+ this.compactionTimer = void 0;
1153
+ this.queueMetaCompaction();
1154
+ }, this.compactionDebounceMs);
1155
+ this.unrefTimer(this.compactionTimer);
1156
+ }
1157
+ queueMetaCompaction() {
1158
+ if (this.destroyed || this.compactionQueued || !this.storage?.compactMeta) return;
1159
+ this.compactionQueued = true;
1160
+ this.enqueueOperation(async () => {
1161
+ try {
1162
+ await this.storage?.compactMeta?.();
1163
+ this.incrementalUpdatesSinceCompaction = 0;
1164
+ this.incrementalUpdateBytesSinceCompaction = 0;
1165
+ } finally {
1166
+ this.compactionQueued = false;
1167
+ }
1168
+ }).catch(require_logging.logAsyncError("metadata compaction"));
1100
1169
  }
1101
1170
  get metaFlock() {
1102
1171
  return this.getMetaFlock();
@@ -1114,6 +1183,18 @@ var MetaPersister = class {
1114
1183
  }
1115
1184
  return true;
1116
1185
  }
1186
+ isEmptyVersion(version) {
1187
+ return Object.keys(version).length === 0;
1188
+ }
1189
+ hasReachedCompactionUpdateThreshold() {
1190
+ return this.compactionUpdateThreshold > 0 && this.incrementalUpdatesSinceCompaction >= this.compactionUpdateThreshold;
1191
+ }
1192
+ hasReachedCompactionByteThreshold() {
1193
+ return this.compactionByteThreshold > 0 && this.incrementalUpdateBytesSinceCompaction >= this.compactionByteThreshold;
1194
+ }
1195
+ unrefTimer(timer) {
1196
+ timer.unref?.();
1197
+ }
1117
1198
  };
1118
1199
 
1119
1200
  //#endregion
@@ -1150,7 +1231,10 @@ var LoroRepo = class LoroRepo {
1150
1231
  if (this.storage) this.metaPersister = new MetaPersister({
1151
1232
  getMetaFlock,
1152
1233
  storage: this.storage,
1153
- debounceMs: options.metaPersistDebounceMs
1234
+ debounceMs: options.metaPersistDebounceMs,
1235
+ compactionDebounceMs: options.metaCompactionDebounceMs,
1236
+ compactionUpdateThreshold: options.metaCompactionUpdateThreshold,
1237
+ compactionByteThreshold: options.metaCompactionByteThreshold
1154
1238
  });
1155
1239
  this.flockHydrator = new FlockHydrator({
1156
1240
  getMetaFlock,