@peerbit/log 6.1.1 → 6.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/dist/benchmark/native-graph.js +447 -5
  2. package/dist/benchmark/native-graph.js.map +1 -1
  3. package/dist/src/clock.d.ts +1 -0
  4. package/dist/src/clock.d.ts.map +1 -1
  5. package/dist/src/clock.js +32 -0
  6. package/dist/src/clock.js.map +1 -1
  7. package/dist/src/entry-create.d.ts +1 -0
  8. package/dist/src/entry-create.d.ts.map +1 -1
  9. package/dist/src/entry-create.js.map +1 -1
  10. package/dist/src/entry-index.d.ts +288 -12
  11. package/dist/src/entry-index.d.ts.map +1 -1
  12. package/dist/src/entry-index.js +1478 -33
  13. package/dist/src/entry-index.js.map +1 -1
  14. package/dist/src/entry-v0.d.ts +184 -1
  15. package/dist/src/entry-v0.d.ts.map +1 -1
  16. package/dist/src/entry-v0.js +1021 -12
  17. package/dist/src/entry-v0.js.map +1 -1
  18. package/dist/src/entry.d.ts +73 -1
  19. package/dist/src/entry.d.ts.map +1 -1
  20. package/dist/src/entry.js +124 -1
  21. package/dist/src/entry.js.map +1 -1
  22. package/dist/src/index.d.ts +1 -1
  23. package/dist/src/index.d.ts.map +1 -1
  24. package/dist/src/index.js +1 -1
  25. package/dist/src/index.js.map +1 -1
  26. package/dist/src/log.d.ts +62 -13
  27. package/dist/src/log.d.ts.map +1 -1
  28. package/dist/src/log.js +2282 -100
  29. package/dist/src/log.js.map +1 -1
  30. package/dist/src/runtime.d.ts +2 -0
  31. package/dist/src/runtime.d.ts.map +1 -0
  32. package/dist/src/runtime.js +10 -0
  33. package/dist/src/runtime.js.map +1 -0
  34. package/dist/src/trim.d.ts +14 -2
  35. package/dist/src/trim.d.ts.map +1 -1
  36. package/dist/src/trim.js +92 -4
  37. package/dist/src/trim.js.map +1 -1
  38. package/package.json +18 -15
  39. package/src/clock.ts +34 -0
  40. package/src/entry-create.ts +1 -0
  41. package/src/entry-index.ts +2232 -68
  42. package/src/entry-v0.ts +1488 -13
  43. package/src/entry.ts +235 -1
  44. package/src/index.ts +7 -1
  45. package/src/log.ts +3246 -206
  46. package/src/runtime.ts +16 -0
  47. package/src/trim.ts +133 -7
package/dist/src/log.js CHANGED
@@ -35,23 +35,47 @@ var __runInitializers = (this && this.__runInitializers) || function (thisArg, i
35
35
  import { deserialize, field, fixedArray, variant } from "@dao-xyz/borsh";
36
36
  import {} from "@peerbit/any-store";
37
37
  import { cidifyString, } from "@peerbit/blocks-interface";
38
- import { SignatureWithKey, X25519Keypair, randomBytes, sha256Base64Sync, } from "@peerbit/crypto";
38
+ import { Ed25519Keypair, SignatureWithKey, X25519Keypair, randomBytes, sha256Base64Sync, } from "@peerbit/crypto";
39
39
  import {} from "@peerbit/indexer-interface";
40
40
  import {} from "@peerbit/keychain";
41
41
  import {} from "./change.js";
42
42
  import { LamportClock as Clock, HLC, LamportClock, Timestamp, } from "./clock.js";
43
43
  import { NO_ENCODING } from "./encoding.js";
44
44
  import { EntryIndex, } from "./entry-index.js";
45
- import { ShallowEntry } from "./entry-shallow.js";
45
+ import { ShallowEntry, ShallowMeta } from "./entry-shallow.js";
46
46
  import { EntryType } from "./entry-type.js";
47
47
  import { EntryV0 } from "./entry-v0.js";
48
48
  import {} from "./entry-with-refs.js";
49
- import { Entry } from "./entry.js";
49
+ import { Entry, } from "./entry.js";
50
50
  import { findUniques } from "./find-uniques.js";
51
+ import { logger as baseLogger } from "./logger.js";
51
52
  import * as LogError from "./log-errors.js";
52
53
  import * as Sorting from "./log-sorting.js";
54
+ import { canUseOptionalNativeModuleImports } from "./runtime.js";
53
55
  import { Trim } from "./trim.js";
54
56
  const { LastWriteWins } = Sorting;
57
+ const warn = baseLogger.newScope("warn");
58
+ const hasPutMany = (storage) => typeof storage.putMany === "function";
59
+ const hasPutKnown = (storage) => typeof storage.putKnown === "function";
60
+ const hasPutKnownMany = (storage) => typeof storage.putKnownMany === "function";
61
+ const hasPutKnownManyColumns = (storage) => typeof storage.putKnownManyColumns ===
62
+ "function";
63
+ const internalProfileNow = () => globalThis.performance?.now?.() ?? Date.now();
64
+ const internalProfileStart = (sink) => sink ? internalProfileNow() : 0;
65
+ const emitInternalProfileDuration = (sink, startedAt, event) => {
66
+ if (!sink) {
67
+ return;
68
+ }
69
+ sink({
70
+ ...event,
71
+ durationMs: internalProfileNow() - startedAt,
72
+ });
73
+ };
74
+ const EMPTY_NEXT_HASHES = [];
75
+ const EMPTY_NEXT_ENTRIES = [];
76
+ const normalizedUniqueStrings = (values) => values.length <= 1 ? values : [...new Set(values)];
77
+ const isPromiseLike = (value) => typeof value?.then === "function";
78
+ const mapMaybePromise = (value, fn) => (isPromiseLike(value) ? value.then(fn) : fn(value));
55
79
  const getErrorName = (error) => typeof error?.name === "string"
56
80
  ? error.name
57
81
  : undefined;
@@ -80,6 +104,14 @@ const createDefaultIndexer = async () => {
80
104
  }
81
105
  return sqliteCreate();
82
106
  };
107
+ const canAppendAlreadyValidated = (options) => options
108
+ ?.__peerbitCanAppendAlreadyValidated === true;
109
+ const withCanAppendAlreadyValidated = (options = {}) => canAppendAlreadyValidated(options)
110
+ ? options
111
+ : {
112
+ ...options,
113
+ __peerbitCanAppendAlreadyValidated: true,
114
+ };
83
115
  export const ENTRY_JOIN_SHAPE = {
84
116
  hash: true,
85
117
  meta: { type: true, next: true, gid: true, clock: true },
@@ -128,8 +160,10 @@ let Log = (() => {
128
160
  _appendDurability;
129
161
  _joining; // entry hashes that are currently joining into this log
130
162
  _sortFn;
163
+ _hasCustomCanAppend = false;
131
164
  constructor(properties) {
132
165
  this._id = properties?.id || randomBytes(32);
166
+ this._closeController = new AbortController();
133
167
  }
134
168
  async open(store, identity, options = {}) {
135
169
  if (store == null) {
@@ -160,22 +194,39 @@ let Log = (() => {
160
194
  if (!id) {
161
195
  throw new Error("Id not set");
162
196
  }
163
- const nativeGraph = options.nativeGraph &&
197
+ const nativeGraphOption = options.nativeGraph;
198
+ const nativeGraph = nativeGraphOption &&
164
199
  (await (async () => {
200
+ const nativeGraphOptions = typeof nativeGraphOption === "object" ? nativeGraphOption : undefined;
201
+ if (nativeGraphOptions?.graph) {
202
+ const headsRequested = nativeGraphOptions.heads !== false;
203
+ return {
204
+ graph: nativeGraphOptions.graph,
205
+ useHeads: headsRequested && this._sortFn === LastWriteWins,
206
+ };
207
+ }
208
+ if (!canUseOptionalNativeModuleImports()) {
209
+ if (nativeGraphOptions?.optional === true) {
210
+ return undefined;
211
+ }
212
+ throw new Error("Log nativeGraph is unavailable in service worker contexts");
213
+ }
165
214
  let createLogGraphIndex;
166
215
  try {
167
- ({ createLogGraphIndex } = (await import(["@peerbit", "log-rust"].join("/"))));
216
+ ({ createLogGraphIndex } = (await import(
217
+ /* @vite-ignore */ ["@peerbit", "log-rust"].join("/"))));
218
+ const headsRequested = nativeGraphOptions?.heads !== false;
219
+ return {
220
+ graph: await createLogGraphIndex(),
221
+ useHeads: headsRequested && this._sortFn === LastWriteWins,
222
+ };
168
223
  }
169
224
  catch {
225
+ if (nativeGraphOptions?.optional === true) {
226
+ return undefined;
227
+ }
170
228
  throw new Error("Log nativeGraph requires @peerbit/log-rust to be installed and built");
171
229
  }
172
- const headsRequested = typeof options.nativeGraph === "object"
173
- ? options.nativeGraph.heads !== false
174
- : true;
175
- return {
176
- graph: await createLogGraphIndex(),
177
- useHeads: headsRequested && this._sortFn === LastWriteWins,
178
- };
179
230
  })());
180
231
  this._entryIndex = new EntryIndex({
181
232
  store: this._storage,
@@ -196,12 +247,46 @@ let Log = (() => {
196
247
  /* this._values = new Values(this._entryIndex, this._sortFn); */
197
248
  this._trim = new Trim({
198
249
  index: this._entryIndex,
199
- deleteNode: async (node) => {
200
- const resolved = await this.get(node.hash);
201
- await this._entryIndex.delete(node.hash);
250
+ deleteNode: async (node, options) => {
251
+ const shouldResolve = options?.resolveDeletedEntry !== false;
252
+ const resolved = shouldResolve
253
+ ? await this.get(node.hash)
254
+ : undefined;
255
+ const deleted = await this._entryIndex.delete(node.hash, node);
202
256
  await this._storage.rm(node.hash);
203
- return resolved;
257
+ if (!deleted) {
258
+ return resolved;
259
+ }
260
+ return shouldResolve ? resolved : deleted;
204
261
  },
262
+ deleteNodes: this._entryIndex.canDeleteMany()
263
+ ? (nodes, options) => {
264
+ if (nodes.length === 0) {
265
+ return [];
266
+ }
267
+ const shouldResolve = options?.resolveDeletedEntry !== false;
268
+ if (!shouldResolve) {
269
+ return this._entryIndex.deleteManyMaybe(nodes, {
270
+ skipNextHeadUpdates: options?.skipNextHeadUpdates,
271
+ });
272
+ }
273
+ return (async () => {
274
+ const resolvedByHash = new Map();
275
+ const resolved = await this._entryIndex.getMany(nodes.map((node) => node.hash), { type: "full", ignoreMissing: true });
276
+ for (const entry of resolved) {
277
+ if (entry) {
278
+ resolvedByHash.set(entry.hash, entry);
279
+ }
280
+ }
281
+ const deleted = await this._entryIndex.deleteMany(nodes, {
282
+ skipNextHeadUpdates: options?.skipNextHeadUpdates,
283
+ });
284
+ return deleted
285
+ .map((node) => resolvedByHash.get(node.hash))
286
+ .filter((entry) => !!entry);
287
+ })();
288
+ }
289
+ : undefined,
205
290
  sortFn: this._sortFn,
206
291
  getLength: () => this.length,
207
292
  }, trim);
@@ -213,6 +298,7 @@ let Log = (() => {
213
298
  }
214
299
  return true;
215
300
  };
301
+ this._hasCustomCanAppend = !!options?.canAppend;
216
302
  this._onChange = options?.onChange;
217
303
  this._closed = false;
218
304
  this._closeController = new AbortController();
@@ -257,6 +343,9 @@ let Log = (() => {
257
343
  has(cid) {
258
344
  return this._entryIndex.has(cid);
259
345
  }
346
+ hasMany(cids) {
347
+ return this._entryIndex.hasMany(cids);
348
+ }
260
349
  /**
261
350
  * Get all entries sorted. Don't use this method anywhere where performance matters
262
351
  */
@@ -393,35 +482,1409 @@ let Log = (() => {
393
482
  }
394
483
  }
395
484
  }
396
- const ret = [];
397
- for (const hash of hashes) {
398
- const entry = await this.get(hash);
399
- if (entry) {
400
- ret.push(entry);
401
- }
485
+ const ret = [];
486
+ for (const hash of hashes) {
487
+ const entry = await this.get(hash);
488
+ if (entry) {
489
+ ret.push(entry);
490
+ }
491
+ }
492
+ return ret;
493
+ }
494
+ /**
495
+ * Append an entry to the log.
496
+ * @param {T} data The data to be appended
497
+ * @param {AppendOptions} [options] The options for the append
498
+ * @returns {{ entry: Entry<T>; removed: ShallowEntry[] }} The appended entry and an array of removed entries
499
+ */
500
+ async append(data, options = {}) {
501
+ const nexts = await this.getNextsForAppend(options);
502
+ const deferBlockStore = hasPutMany(this._storage);
503
+ const nativeAppendChain = this.entryIndex.properties.nativeGraph
504
+ ? await this.createNativePlainAppendChain([data], options, nexts, deferBlockStore)
505
+ : undefined;
506
+ let entry;
507
+ if (nativeAppendChain) {
508
+ entry = nativeAppendChain.entries[0];
509
+ try {
510
+ await this.joinMissingNexts(entry, nexts);
511
+ if (deferBlockStore && !nativeAppendChain.nativeBlocksCommitted) {
512
+ await this.putAppendEntryBlocks([entry], nativeAppendChain.blocks);
513
+ }
514
+ await this.putAppendEntries([entry], options, nexts.map((next) => next.hash), nativeAppendChain);
515
+ }
516
+ catch (error) {
517
+ if (nativeAppendChain.nativeGraphUpdated) {
518
+ this.rollbackNativeAppendGraph([entry]);
519
+ }
520
+ if (nativeAppendChain.nativeBlocksCommitted) {
521
+ await this.rollbackNativeAppendBlocks([entry]);
522
+ }
523
+ throw error;
524
+ }
525
+ }
526
+ else {
527
+ entry = await this.createAppendEntry(data, options, nexts);
528
+ await this.joinMissingNexts(entry, nexts);
529
+ await this.putAppendEntry(entry, options);
530
+ }
531
+ const pendingDeletes = await this.processEntry(entry);
532
+ entry.init({ encoding: this._encoding, keychain: this._keychain });
533
+ const trimmed = await this.trimIfConfigured(options?.trim);
534
+ if (trimmed) {
535
+ for (const entry of trimmed) {
536
+ pendingDeletes.push({ entry, fn: undefined });
537
+ }
538
+ }
539
+ const removed = pendingDeletes.map((x) => x.entry);
540
+ const changes = {
541
+ added: [{ head: true, entry }],
542
+ removed,
543
+ };
544
+ await (options?.onChange || this._onChange)?.(changes);
545
+ await Promise.all(pendingDeletes.map((x) => x.fn?.()));
546
+ return { entry, removed };
547
+ }
548
+ // Internal trusted local append path for callers that already handled validation
549
+ // and want to apply change observers themselves.
550
+ async appendLocallyPrepared(data, options = {}, properties) {
551
+ if (options.canAppend ||
552
+ options.onChange ||
553
+ options.meta?.type === EntryType.CUT) {
554
+ throw new Error("appendLocallyPrepared only supports trusted plain local appends");
555
+ }
556
+ const appendOptions = withCanAppendAlreadyValidated(options);
557
+ const nextsResult = this.getNextsForAppend(appendOptions);
558
+ const nexts = isPromiseLike(nextsResult) ? await nextsResult : nextsResult;
559
+ const deferBlockStore = hasPutMany(this._storage);
560
+ const nativeAppendChain = this.entryIndex.properties.nativeGraph
561
+ ? await this.createNativePlainAppendChain([data], appendOptions, nexts, deferBlockStore, properties?.payloadData ? [properties.payloadData] : undefined)
562
+ : undefined;
563
+ let entry;
564
+ if (nativeAppendChain) {
565
+ entry = nativeAppendChain.entries[0];
566
+ try {
567
+ if (!properties?.skipMissingNextJoin) {
568
+ await this.joinMissingNexts(entry, nexts);
569
+ }
570
+ if (deferBlockStore && !nativeAppendChain.nativeBlocksCommitted) {
571
+ await this.putAppendEntryBlocks([entry], nativeAppendChain.blocks);
572
+ }
573
+ await this.putAppendEntries([entry], appendOptions, nexts.map((next) => next.hash), nativeAppendChain);
574
+ }
575
+ catch (error) {
576
+ if (nativeAppendChain.nativeGraphUpdated) {
577
+ this.rollbackNativeAppendGraph([entry]);
578
+ }
579
+ if (nativeAppendChain.nativeBlocksCommitted) {
580
+ await this.rollbackNativeAppendBlocks([entry]);
581
+ }
582
+ throw error;
583
+ }
584
+ }
585
+ else {
586
+ if (data == null && properties?.payloadData) {
587
+ throw new Error("appendLocallyPrepared payload-only path requires native append support");
588
+ }
589
+ entry = await this.createAppendEntry(data, appendOptions, nexts);
590
+ if (!properties?.skipMissingNextJoin) {
591
+ await this.joinMissingNexts(entry, nexts);
592
+ }
593
+ await this.putAppendEntry(entry, appendOptions);
594
+ }
595
+ entry.init({ encoding: this._encoding, keychain: this._keychain });
596
+ const trimmed = await this.trimIfConfigured(appendOptions.trim, {
597
+ resolveDeletedEntries: properties?.resolveTrimmedEntries,
598
+ });
599
+ const removed = trimmed ?? [];
600
+ const change = {
601
+ added: [{ head: true, entry }],
602
+ removed,
603
+ };
604
+ const appendFacts = this.createPreparedAppendFacts([entry], nativeAppendChain)[0];
605
+ return { entry, removed, change, appendFacts };
606
+ }
607
+ // Internal trusted local append path for callers that can consume compact
608
+ // append facts before a public Entry object is needed.
609
+ appendLocallyPreparedCommitOnly(data, options = {}, properties) {
610
+ if (options.canAppend ||
611
+ options.onChange ||
612
+ options.meta?.type === EntryType.CUT) {
613
+ throw new Error("appendLocallyPreparedCommitOnly only supports trusted plain local appends");
614
+ }
615
+ const appendOptions = withCanAppendAlreadyValidated(options);
616
+ const nextsResult = this.getNextsForAppend(appendOptions);
617
+ return mapMaybePromise(nextsResult, (nexts) => this.appendLocallyPreparedCommitOnlyWithNexts(data, appendOptions, nexts, properties, this.getNativeCommitOnlyTrimLengthTo(appendOptions.trim, properties?.resolveTrimmedEntries)));
618
+ }
619
+ appendLocallyPreparedNativeNoNextCommitOnly(data, options = {}, properties, prepare) {
620
+ const directResult = this.appendLocallyPreparedNativeKnownNoNextCommitOnly(data, options, properties, prepare);
621
+ if (directResult !== undefined) {
622
+ return directResult;
623
+ }
624
+ return this.appendLocallyPreparedNativeCommitOnly(data, options, properties, prepare);
625
+ }
626
+ appendLocallyPreparedNativeKnownNoNextCommitOnly(data, options = {}, properties, prepare) {
627
+ if (options.meta?.next == null || options.meta.next.length !== 0) {
628
+ return undefined;
629
+ }
630
+ const resolvedTrim = options.trim ?? this._trim.options;
631
+ const supportsNativeTrim = !resolvedTrim ||
632
+ (resolvedTrim.type === "length" &&
633
+ !resolvedTrim.filter?.canTrim &&
634
+ properties.resolveTrimmedEntries === false);
635
+ if (!supportsNativeTrim) {
636
+ return undefined;
637
+ }
638
+ const nativeTrimLengthTo = this.getNativeCommitOnlyTrimLengthTo(options.trim, properties.resolveTrimmedEntries);
639
+ if (!options.meta?.gidSeed) {
640
+ const directResult = this.appendLocallyPreparedNativeKnownNoNextDirectCommitOnly(data, options, properties, prepare, nativeTrimLengthTo);
641
+ if (directResult !== undefined) {
642
+ return directResult;
643
+ }
644
+ }
645
+ return this.appendLocallyPreparedNativeCommitOnly(data, options, properties, prepare, true);
646
+ }
647
+ appendLocallyPreparedNativeKnownNoNextDirectCommitOnly(data, options = {}, properties, prepare, nativeTrimLengthTo) {
648
+ if (options.canAppend ||
649
+ options.onChange ||
650
+ options.encryption ||
651
+ options.signers ||
652
+ options.identity ||
653
+ options.meta?.timestamp ||
654
+ options.meta?.type === EntryType.CUT ||
655
+ options.meta?.next == null ||
656
+ options.meta.next.length !== 0 ||
657
+ options.meta?.gidSeed ||
658
+ (this._hasCustomCanAppend && !canAppendAlreadyValidated(options))) {
659
+ return undefined;
660
+ }
661
+ const identity = this._identity;
662
+ if (!(identity instanceof Ed25519Keypair)) {
663
+ return undefined;
664
+ }
665
+ const payloadData = properties.payloadData ??
666
+ (data == null ? undefined : this._encoding.encoder(data));
667
+ if (!payloadData || !hasPutMany(this._storage)) {
668
+ return undefined;
669
+ }
670
+ const resolvedGid = EntryV0.createGid();
671
+ const timestamp = this._hlc.now();
672
+ const entryType = options.meta?.type ?? EntryType.APPEND;
673
+ const preparedValue = prepare({
674
+ clockId: identity.publicKey.bytes,
675
+ privateKey: identity.privateKey.privateKey,
676
+ publicKey: identity.publicKey.publicKey,
677
+ wallTime: timestamp.wallTime,
678
+ logical: timestamp.logical,
679
+ gid: resolvedGid,
680
+ type: entryType,
681
+ metaData: options.meta?.data,
682
+ payloadData,
683
+ resolveTrimmedEntries: properties.resolveTrimmedEntries,
684
+ trimLengthTo: nativeTrimLengthTo,
685
+ });
686
+ return mapMaybePromise(preparedValue, (prepared) => {
687
+ if (!prepared) {
688
+ return undefined;
689
+ }
690
+ const hash = prepared.cid ?? prepared.hash;
691
+ if (!hash) {
692
+ return undefined;
693
+ }
694
+ const shouldRetainMaterializationBytes = properties.retainMaterializationBytes === true ||
695
+ !!(options.trim ?? this._trim.options);
696
+ let retainedMaterializationBytes;
697
+ const retainMaterializationBytes = () => {
698
+ if (retainedMaterializationBytes ||
699
+ !shouldRetainMaterializationBytes ||
700
+ prepared.bytes) {
701
+ return;
702
+ }
703
+ const bytes = prepared.getBytes?.(hash) ??
704
+ this._storage.get(hash);
705
+ if (bytes &&
706
+ typeof bytes.then !== "function") {
707
+ retainedMaterializationBytes = bytes;
708
+ }
709
+ };
710
+ const effectiveNextHashes = prepared.next ?? EMPTY_NEXT_HASHES;
711
+ const effectiveGid = prepared.gid ?? resolvedGid;
712
+ let clock;
713
+ const getClock = () => (clock ??= new Clock({
714
+ id: identity.publicKey.bytes,
715
+ timestamp,
716
+ }));
717
+ let shallowEntry;
718
+ const getShallowEntry = () => (shallowEntry ??= new ShallowEntry({
719
+ hash,
720
+ payloadSize: payloadData.byteLength,
721
+ head: true,
722
+ meta: new ShallowMeta({
723
+ gid: effectiveGid,
724
+ data: options.meta?.data,
725
+ clock: getClock(),
726
+ next: effectiveNextHashes,
727
+ type: entryType,
728
+ }),
729
+ }));
730
+ const appendFacts = {
731
+ hash,
732
+ gid: effectiveGid,
733
+ next: effectiveNextHashes,
734
+ wallTime: timestamp.wallTime,
735
+ logical: timestamp.logical,
736
+ clockId: identity.publicKey.bytes,
737
+ type: entryType,
738
+ metaData: options.meta?.data,
739
+ payloadSize: payloadData.byteLength,
740
+ metaBytes: prepared.metaBytes,
741
+ hashDigestBytes: prepared.hashDigestBytes,
742
+ };
743
+ let materializedEntry;
744
+ const materializeEntry = () => {
745
+ if (materializedEntry) {
746
+ return materializedEntry;
747
+ }
748
+ const bytes = prepared.bytes ??
749
+ retainedMaterializationBytes ??
750
+ prepared.getBytes?.(hash) ??
751
+ this._storage.get(hash);
752
+ if (!bytes ||
753
+ typeof bytes.then === "function") {
754
+ throw new Error("Missing synchronous native append block bytes");
755
+ }
756
+ const entry = deserialize(bytes, Entry);
757
+ entry.hash = hash;
758
+ entry.size = prepared.byteLength;
759
+ entry.createdLocally = true;
760
+ Entry.prepareShallowEntry(entry, getShallowEntry());
761
+ entry.init({ encoding: this._encoding, keychain: this._keychain });
762
+ materializedEntry = entry;
763
+ return entry;
764
+ };
765
+ const finish = () => {
766
+ retainMaterializationBytes();
767
+ return {
768
+ get entry() {
769
+ return materializeEntry();
770
+ },
771
+ materializeEntry,
772
+ removed: [],
773
+ appendFacts,
774
+ get shallowEntry() {
775
+ return getShallowEntry();
776
+ },
777
+ documentTrimmedHeadsProcessed: prepared.documentTrimmedHeadsProcessed,
778
+ documentPreviousContext: prepared.documentPreviousContext,
779
+ };
780
+ };
781
+ const finishTrim = () => {
782
+ retainMaterializationBytes();
783
+ if (prepared.trimmedEntryHashes) {
784
+ if (prepared.trimmedEntryHashes.length === 0) {
785
+ return finish();
786
+ }
787
+ if (properties.resolveTrimmedEntries === false ||
788
+ prepared.documentTrimmedHeadsProcessed === true) {
789
+ const trimmedEntryHashes = prepared.trimmedEntryHashes.length === 1
790
+ ? prepared.trimmedEntryHashes
791
+ : [...new Set(prepared.trimmedEntryHashes)];
792
+ const consumedNoReturn = this.entryIndex.consumeNativeTrimmedEntryHashesNoReturnMaybe(trimmedEntryHashes, {
793
+ skipNextHeadUpdates: true,
794
+ deleteBlocks: false,
795
+ });
796
+ if (consumedNoReturn !== undefined) {
797
+ return mapMaybePromise(consumedNoReturn, () => ({
798
+ get entry() {
799
+ return materializeEntry();
800
+ },
801
+ materializeEntry,
802
+ removed: [],
803
+ removedHashes: trimmedEntryHashes,
804
+ appendFacts,
805
+ get shallowEntry() {
806
+ return getShallowEntry();
807
+ },
808
+ documentTrimmedHeadsProcessed: prepared.documentTrimmedHeadsProcessed,
809
+ documentPreviousContext: prepared.documentPreviousContext,
810
+ }));
811
+ }
812
+ }
813
+ const consumedResult = this.entryIndex.consumeNativeTrimmedEntryHashesMaybe(prepared.trimmedEntryHashes, {
814
+ skipNextHeadUpdates: true,
815
+ deleteBlocks: false,
816
+ });
817
+ return mapMaybePromise(consumedResult, (removed) => ({
818
+ get entry() {
819
+ return materializeEntry();
820
+ },
821
+ materializeEntry,
822
+ removed,
823
+ removedHashes: prepared.trimmedEntryHashes,
824
+ appendFacts,
825
+ get shallowEntry() {
826
+ return getShallowEntry();
827
+ },
828
+ documentTrimmedHeadsProcessed: prepared.documentTrimmedHeadsProcessed,
829
+ documentPreviousContext: prepared.documentPreviousContext,
830
+ }));
831
+ }
832
+ if (!prepared.trimmedEntries) {
833
+ return finish();
834
+ }
835
+ const trimmedEntries = this.entryIndex.nativeLogEntriesToShallowEntries(prepared.trimmedEntries);
836
+ const consumedResult = this.entryIndex.consumeNativeTrimmedEntriesMaybe(trimmedEntries, {
837
+ skipNextHeadUpdates: true,
838
+ deleteBlocks: false,
839
+ });
840
+ return mapMaybePromise(consumedResult, (removed) => ({
841
+ get entry() {
842
+ return materializeEntry();
843
+ },
844
+ materializeEntry,
845
+ removed,
846
+ appendFacts,
847
+ get shallowEntry() {
848
+ return getShallowEntry();
849
+ },
850
+ documentTrimmedHeadsProcessed: prepared.documentTrimmedHeadsProcessed,
851
+ documentPreviousContext: prepared.documentPreviousContext,
852
+ }));
853
+ };
854
+ const finishBlocks = () => {
855
+ if (!prepared.bytes) {
856
+ return finishTrim();
857
+ }
858
+ return mapMaybePromise(this.putPreparedAppendBlocks([
859
+ Entry.preparedBlockFromBytes(prepared.bytes, hash),
860
+ ]), finishTrim);
861
+ };
862
+ const rollback = (error) => {
863
+ this.rollbackNativeAppendGraphHashes([hash]);
864
+ return this.rollbackNativeAppendBlocksHashes([hash]).then(() => {
865
+ throw error;
866
+ });
867
+ };
868
+ try {
869
+ const putResult = this.entryIndex.putNativeCommittedAppendFacts({
870
+ hash,
871
+ unique: true,
872
+ externalNextHashes: effectiveNextHashes,
873
+ getShallowEntry,
874
+ isHead: true,
875
+ });
876
+ const result = mapMaybePromise(putResult, finishBlocks);
877
+ return isPromiseLike(result) ? result.catch(rollback) : result;
878
+ }
879
+ catch (error) {
880
+ return rollback(error);
881
+ }
882
+ });
883
+ }
884
+ appendLocallyPreparedNativeCommitOnly(data, options = {}, properties, prepare, knownNoNext = false) {
885
+ const resolvedTrim = options.trim ?? this._trim.options;
886
+ const supportsNativeTrim = !resolvedTrim ||
887
+ (resolvedTrim.type === "length" &&
888
+ !resolvedTrim.filter?.canTrim &&
889
+ properties.resolveTrimmedEntries === false);
890
+ const nativeTrimLengthTo = this.getNativeCommitOnlyTrimLengthTo(options.trim, properties.resolveTrimmedEntries);
891
+ if (options.canAppend ||
892
+ options.onChange ||
893
+ options.encryption ||
894
+ options.signers ||
895
+ options.identity ||
896
+ options.meta?.timestamp ||
897
+ !supportsNativeTrim ||
898
+ (this._hasCustomCanAppend && !canAppendAlreadyValidated(options))) {
899
+ return undefined;
900
+ }
901
+ const identity = this._identity;
902
+ if (!(identity instanceof Ed25519Keypair)) {
903
+ return undefined;
904
+ }
905
+ const payloadData = properties.payloadData ??
906
+ (data == null ? undefined : this._encoding.encoder(data));
907
+ if (!payloadData || !hasPutMany(this._storage)) {
908
+ return undefined;
909
+ }
910
+ const appendOptions = withCanAppendAlreadyValidated(options);
911
+ const knownNoNextAppend = knownNoNext || properties.knownNoNext === true;
912
+ const nextsResult = knownNoNextAppend
913
+ ? EMPTY_NEXT_ENTRIES
914
+ : this.getNextsForAppend(appendOptions);
915
+ return mapMaybePromise(nextsResult, (nexts) => {
916
+ if (nexts.length > 0 && properties.skipMissingNextJoin !== true) {
917
+ return undefined;
918
+ }
919
+ const nextHashes = knownNoNextAppend ? EMPTY_NEXT_HASHES : [];
920
+ let nextGid;
921
+ if (nexts.length > 0) {
922
+ if (appendOptions.meta?.gid) {
923
+ return undefined;
924
+ }
925
+ for (const next of nexts) {
926
+ if (!next.hash) {
927
+ return undefined;
928
+ }
929
+ nextHashes.push(next.hash);
930
+ nextGid =
931
+ nextGid == null || next.meta.gid < nextGid
932
+ ? next.meta.gid
933
+ : nextGid;
934
+ }
935
+ }
936
+ const gid = nextGid ?? EntryV0.createGid(appendOptions.meta?.gidSeed);
937
+ return mapMaybePromise(gid, (resolvedGid) => {
938
+ const clock = new Clock({
939
+ id: identity.publicKey.bytes,
940
+ timestamp: this._hlc.now(),
941
+ });
942
+ const entryType = appendOptions.meta?.type ?? EntryType.APPEND;
943
+ const preparedValue = prepare({
944
+ clockId: identity.publicKey.bytes,
945
+ privateKey: identity.privateKey.privateKey,
946
+ publicKey: identity.publicKey.publicKey,
947
+ wallTime: clock.timestamp.wallTime,
948
+ logical: clock.timestamp.logical,
949
+ gid: resolvedGid,
950
+ next: nextHashes,
951
+ type: entryType,
952
+ metaData: appendOptions.meta?.data,
953
+ payloadData,
954
+ resolveTrimmedEntries: properties.resolveTrimmedEntries,
955
+ trimLengthTo: nativeTrimLengthTo,
956
+ });
957
+ return mapMaybePromise(preparedValue, (prepared) => {
958
+ if (!prepared) {
959
+ return undefined;
960
+ }
961
+ const hash = prepared.cid ?? prepared.hash;
962
+ if (!hash) {
963
+ return undefined;
964
+ }
965
+ const shouldRetainMaterializationBytes = properties.retainMaterializationBytes === true || !!resolvedTrim;
966
+ let retainedMaterializationBytes;
967
+ const retainMaterializationBytes = () => {
968
+ if (retainedMaterializationBytes ||
969
+ !shouldRetainMaterializationBytes ||
970
+ prepared.bytes) {
971
+ return;
972
+ }
973
+ const bytes = prepared.getBytes?.(hash) ??
974
+ this._storage.get(hash);
975
+ if (bytes &&
976
+ typeof bytes.then !== "function") {
977
+ retainedMaterializationBytes = bytes;
978
+ }
979
+ };
980
+ const effectiveNextHashes = prepared.next ?? nextHashes;
981
+ const effectiveGid = prepared.gid ?? resolvedGid;
982
+ const shallowEntry = new ShallowEntry({
983
+ hash,
984
+ payloadSize: payloadData.byteLength,
985
+ head: true,
986
+ meta: new ShallowMeta({
987
+ gid: effectiveGid,
988
+ data: appendOptions.meta?.data,
989
+ clock,
990
+ next: effectiveNextHashes,
991
+ type: entryType,
992
+ }),
993
+ });
994
+ const appendFacts = {
995
+ hash,
996
+ gid: effectiveGid,
997
+ next: effectiveNextHashes,
998
+ wallTime: clock.timestamp.wallTime,
999
+ logical: clock.timestamp.logical,
1000
+ clockId: clock.id,
1001
+ type: entryType,
1002
+ metaData: appendOptions.meta?.data,
1003
+ payloadSize: payloadData.byteLength,
1004
+ metaBytes: prepared.metaBytes,
1005
+ hashDigestBytes: prepared.hashDigestBytes,
1006
+ };
1007
+ let materializedEntry;
1008
+ const materializeEntry = () => {
1009
+ if (materializedEntry) {
1010
+ return materializedEntry;
1011
+ }
1012
+ const bytes = prepared.bytes ??
1013
+ retainedMaterializationBytes ??
1014
+ prepared.getBytes?.(hash) ??
1015
+ this._storage.get(hash);
1016
+ if (!bytes ||
1017
+ typeof bytes.then === "function") {
1018
+ throw new Error("Missing synchronous native append block bytes");
1019
+ }
1020
+ const entry = deserialize(bytes, Entry);
1021
+ entry.hash = hash;
1022
+ entry.size = prepared.byteLength;
1023
+ entry.createdLocally = true;
1024
+ Entry.prepareShallowEntry(entry, shallowEntry);
1025
+ entry.init({ encoding: this._encoding, keychain: this._keychain });
1026
+ materializedEntry = entry;
1027
+ return entry;
1028
+ };
1029
+ const finish = () => {
1030
+ retainMaterializationBytes();
1031
+ return {
1032
+ get entry() {
1033
+ return materializeEntry();
1034
+ },
1035
+ materializeEntry,
1036
+ removed: [],
1037
+ appendFacts,
1038
+ shallowEntry,
1039
+ documentTrimmedHeadsProcessed: prepared.documentTrimmedHeadsProcessed,
1040
+ documentPreviousContext: prepared.documentPreviousContext,
1041
+ };
1042
+ };
1043
+ const finishBlocks = () => {
1044
+ if (!prepared.bytes) {
1045
+ return finishTrim();
1046
+ }
1047
+ return mapMaybePromise(this.putPreparedAppendBlocks([
1048
+ Entry.preparedBlockFromBytes(prepared.bytes, hash),
1049
+ ]), finishTrim);
1050
+ };
1051
+ const finishTrim = () => {
1052
+ retainMaterializationBytes();
1053
+ if (prepared.trimmedEntryHashes) {
1054
+ if (prepared.trimmedEntryHashes.length === 0) {
1055
+ return finish();
1056
+ }
1057
+ if (properties.resolveTrimmedEntries === false ||
1058
+ prepared.documentTrimmedHeadsProcessed === true) {
1059
+ const trimmedEntryHashes = prepared.trimmedEntryHashes.length === 1
1060
+ ? prepared.trimmedEntryHashes
1061
+ : [...new Set(prepared.trimmedEntryHashes)];
1062
+ const consumedNoReturn = this.entryIndex.consumeNativeTrimmedEntryHashesNoReturnMaybe(trimmedEntryHashes, {
1063
+ skipNextHeadUpdates: true,
1064
+ deleteBlocks: false,
1065
+ });
1066
+ if (consumedNoReturn !== undefined) {
1067
+ return mapMaybePromise(consumedNoReturn, () => ({
1068
+ get entry() {
1069
+ return materializeEntry();
1070
+ },
1071
+ materializeEntry,
1072
+ removed: [],
1073
+ removedHashes: trimmedEntryHashes,
1074
+ appendFacts,
1075
+ shallowEntry,
1076
+ documentTrimmedHeadsProcessed: prepared.documentTrimmedHeadsProcessed,
1077
+ documentPreviousContext: prepared.documentPreviousContext,
1078
+ }));
1079
+ }
1080
+ }
1081
+ const consumedResult = this.entryIndex.consumeNativeTrimmedEntryHashesMaybe(prepared.trimmedEntryHashes, {
1082
+ skipNextHeadUpdates: true,
1083
+ deleteBlocks: false,
1084
+ });
1085
+ return mapMaybePromise(consumedResult, (removed) => ({
1086
+ get entry() {
1087
+ return materializeEntry();
1088
+ },
1089
+ materializeEntry,
1090
+ removed,
1091
+ appendFacts,
1092
+ shallowEntry,
1093
+ documentTrimmedHeadsProcessed: prepared.documentTrimmedHeadsProcessed,
1094
+ documentPreviousContext: prepared.documentPreviousContext,
1095
+ }));
1096
+ }
1097
+ if (!prepared.trimmedEntries) {
1098
+ return finish();
1099
+ }
1100
+ const trimmedEntries = this.entryIndex.nativeLogEntriesToShallowEntries(prepared.trimmedEntries);
1101
+ const consumedResult = this.entryIndex.consumeNativeTrimmedEntriesMaybe(trimmedEntries, {
1102
+ skipNextHeadUpdates: true,
1103
+ deleteBlocks: false,
1104
+ });
1105
+ return mapMaybePromise(consumedResult, (removed) => ({
1106
+ get entry() {
1107
+ return materializeEntry();
1108
+ },
1109
+ materializeEntry,
1110
+ removed,
1111
+ appendFacts,
1112
+ shallowEntry,
1113
+ documentTrimmedHeadsProcessed: prepared.documentTrimmedHeadsProcessed,
1114
+ documentPreviousContext: prepared.documentPreviousContext,
1115
+ }));
1116
+ };
1117
+ const rollback = (error) => {
1118
+ this.rollbackNativeAppendGraphHashes([hash]);
1119
+ return this.rollbackNativeAppendBlocksHashes([hash]).then(() => {
1120
+ throw error;
1121
+ });
1122
+ };
1123
+ try {
1124
+ const putResult = this.entryIndex.putNativeCommittedAppendFacts({
1125
+ hash,
1126
+ unique: true,
1127
+ externalNextHashes: nextHashes,
1128
+ shallowEntry,
1129
+ isHead: true,
1130
+ });
1131
+ const result = mapMaybePromise(putResult, finishBlocks);
1132
+ return isPromiseLike(result) ? result.catch(rollback) : result;
1133
+ }
1134
+ catch (error) {
1135
+ return rollback(error);
1136
+ }
1137
+ });
1138
+ });
1139
+ });
1140
+ }
1141
+ appendLocallyPreparedCommitOnlyWithNexts(data, appendOptions, nexts, properties, nativeTrimLengthTo) {
1142
+ const deferBlockStore = hasPutMany(this._storage);
1143
+ const nativeAppendChainResult = this.createNativePlainAppendCommitOnly([data], appendOptions, nexts, deferBlockStore, properties?.payloadData ? [properties.payloadData] : undefined, properties?.includeMaterializationBytes, properties?.includeAppendFactsBytes, nativeTrimLengthTo);
1144
+ return mapMaybePromise(nativeAppendChainResult, (nativeAppendChain) => this.finishLocallyPreparedCommitOnlyAppend(nativeAppendChain, appendOptions, nexts, deferBlockStore, properties));
1145
+ }
1146
+ finishLocallyPreparedCommitOnlyAppend(nativeAppendChain, appendOptions, nexts, deferBlockStore, properties) {
1147
+ if (!nativeAppendChain) {
1148
+ return undefined;
1149
+ }
1150
+ const appendFacts = nativeAppendChain.appendFacts[0];
1151
+ const shallowEntry = nativeAppendChain.shallowEntries[0];
1152
+ let materializedEntry;
1153
+ const materializeEntry = () => {
1154
+ const entry = materializedEntry ?? nativeAppendChain.materializeEntry(0);
1155
+ entry.init({ encoding: this._encoding, keychain: this._keychain });
1156
+ materializedEntry = entry;
1157
+ return entry;
1158
+ };
1159
+ const finishTrim = () => {
1160
+ if (nativeAppendChain.trimmedNativeEntryHashes) {
1161
+ const trimmedEntryHashes = [
1162
+ ...new Set(nativeAppendChain.trimmedNativeEntryHashes),
1163
+ ];
1164
+ if (properties?.resolveTrimmedEntries === false &&
1165
+ nativeAppendChain.trimmedNativeBlocksDeleted === true) {
1166
+ const consumedNoReturn = this.entryIndex.consumeNativeTrimmedEntryHashesNoReturnMaybe(trimmedEntryHashes, {
1167
+ skipNextHeadUpdates: true,
1168
+ deleteBlocks: false,
1169
+ });
1170
+ if (consumedNoReturn !== undefined) {
1171
+ return mapMaybePromise(consumedNoReturn, () => ({
1172
+ get entry() {
1173
+ return materializeEntry();
1174
+ },
1175
+ materializeEntry,
1176
+ removed: [],
1177
+ removedHashes: trimmedEntryHashes,
1178
+ appendFacts,
1179
+ shallowEntry,
1180
+ }));
1181
+ }
1182
+ }
1183
+ const consumedResult = this.entryIndex.consumeNativeTrimmedEntryHashesMaybe(trimmedEntryHashes, {
1184
+ skipNextHeadUpdates: true,
1185
+ deleteBlocks: nativeAppendChain.trimmedNativeBlocksDeleted !== true,
1186
+ });
1187
+ return mapMaybePromise(consumedResult, (removed) => ({
1188
+ get entry() {
1189
+ return materializeEntry();
1190
+ },
1191
+ materializeEntry,
1192
+ removed,
1193
+ removedHashes: trimmedEntryHashes,
1194
+ appendFacts,
1195
+ shallowEntry,
1196
+ }));
1197
+ }
1198
+ if (nativeAppendChain.trimmedNativeEntries) {
1199
+ const trimmedEntries = this.entryIndex.nativeLogEntriesToShallowEntries(nativeAppendChain.trimmedNativeEntries);
1200
+ const consumedResult = this.entryIndex.consumeNativeTrimmedEntriesMaybe(trimmedEntries, {
1201
+ skipNextHeadUpdates: true,
1202
+ deleteBlocks: nativeAppendChain.trimmedNativeBlocksDeleted !== true,
1203
+ });
1204
+ return mapMaybePromise(consumedResult, (removed) => ({
1205
+ get entry() {
1206
+ return materializeEntry();
1207
+ },
1208
+ materializeEntry,
1209
+ removed,
1210
+ appendFacts,
1211
+ shallowEntry,
1212
+ }));
1213
+ }
1214
+ const trimmedResult = this.trimIfConfigured(appendOptions.trim, {
1215
+ resolveDeletedEntries: properties?.resolveTrimmedEntries,
1216
+ });
1217
+ return mapMaybePromise(trimmedResult, (trimmed) => {
1218
+ const removed = trimmed ?? [];
1219
+ return {
1220
+ get entry() {
1221
+ return materializeEntry();
1222
+ },
1223
+ materializeEntry,
1224
+ removed,
1225
+ appendFacts,
1226
+ shallowEntry,
1227
+ };
1228
+ });
1229
+ };
1230
+ const finishFacts = () => {
1231
+ const putFactsResult = this.entryIndex.putNativeCommittedAppendFacts({
1232
+ hash: appendFacts.hash,
1233
+ unique: true,
1234
+ externalNextHashes: nexts.map((next) => next.hash),
1235
+ shallowEntry,
1236
+ isHead: true,
1237
+ });
1238
+ return mapMaybePromise(putFactsResult, finishTrim);
1239
+ };
1240
+ const finishBlocks = () => {
1241
+ if (deferBlockStore && !nativeAppendChain.nativeBlocksCommitted) {
1242
+ return mapMaybePromise(this.putPreparedAppendBlocks(nativeAppendChain.blocks), finishFacts);
1243
+ }
1244
+ return finishFacts();
1245
+ };
1246
+ const rollback = (error) => {
1247
+ if (nativeAppendChain.nativeGraphUpdated) {
1248
+ this.rollbackNativeAppendGraphHashes([appendFacts.hash]);
1249
+ }
1250
+ if (nativeAppendChain.nativeBlocksCommitted) {
1251
+ return this.rollbackNativeAppendBlocksHashes([appendFacts.hash]).then(() => {
1252
+ throw error;
1253
+ });
1254
+ }
1255
+ throw error;
1256
+ };
1257
+ try {
1258
+ let result;
1259
+ if (!properties?.skipMissingNextJoin && nexts.length > 0) {
1260
+ result = mapMaybePromise(this.joinMissingNexts(materializeEntry(), nexts), finishBlocks);
1261
+ }
1262
+ else {
1263
+ result = finishBlocks();
1264
+ }
1265
+ return isPromiseLike(result) ? result.catch(rollback) : result;
1266
+ }
1267
+ catch (error) {
1268
+ return rollback(error);
1269
+ }
1270
+ }
1271
+ async appendLocallyPreparedManyIndependent(data, options = {}, properties) {
1272
+ if (data.length === 0) {
1273
+ return {
1274
+ entries: [],
1275
+ removed: [],
1276
+ change: { added: [], removed: [] },
1277
+ appendFacts: [],
1278
+ };
1279
+ }
1280
+ if (options.canAppend ||
1281
+ options.onChange ||
1282
+ options.meta?.type === EntryType.CUT) {
1283
+ throw new Error("appendLocallyPreparedManyIndependent only supports trusted plain local appends");
1284
+ }
1285
+ const appendOptions = withCanAppendAlreadyValidated(options);
1286
+ const deferBlockStore = hasPutMany(this._storage);
1287
+ const nativeAppendBatch = await this.createNativePlainAppendEntriesBatch(data, appendOptions, deferBlockStore, properties?.payloadDatas, properties?.nexts);
1288
+ if (!nativeAppendBatch) {
1289
+ return undefined;
1290
+ }
1291
+ const entries = nativeAppendBatch.entries;
1292
+ const externalNextHashes = properties?.nexts?.flatMap((nexts) => nexts.map((next) => next.hash)) ??
1293
+ [];
1294
+ try {
1295
+ if (deferBlockStore && !nativeAppendBatch.nativeBlocksCommitted) {
1296
+ await this.putAppendEntryBlocks(entries, nativeAppendBatch.blocks);
1297
+ }
1298
+ await this.putAppendEntries(entries, appendOptions, externalNextHashes, nativeAppendBatch, entries.map(() => true));
1299
+ }
1300
+ catch (error) {
1301
+ if (nativeAppendBatch.nativeGraphUpdated) {
1302
+ this.rollbackNativeAppendGraph(entries);
1303
+ }
1304
+ if (nativeAppendBatch.nativeBlocksCommitted) {
1305
+ await this.rollbackNativeAppendBlocks(entries);
1306
+ }
1307
+ throw error;
1308
+ }
1309
+ for (const entry of entries) {
1310
+ entry.init({ encoding: this._encoding, keychain: this._keychain });
1311
+ }
1312
+ const trimmed = await this.trimIfConfigured(appendOptions.trim, {
1313
+ resolveDeletedEntries: properties?.resolveTrimmedEntries,
1314
+ });
1315
+ const removed = trimmed ?? [];
1316
+ const change = {
1317
+ added: entries.map((entry) => ({ head: true, entry })),
1318
+ removed,
1319
+ };
1320
+ const appendFacts = this.createPreparedAppendFacts(entries, nativeAppendBatch);
1321
+ return { entries, removed, change, appendFacts };
1322
+ }
1323
+ appendLocallyPreparedNativeKnownNoNextCommitOnlyBatch(data, options = {}, properties, prepare) {
1324
+ if (data.length === 0) {
1325
+ return {
1326
+ entries: [],
1327
+ materializeEntries: [],
1328
+ removed: [],
1329
+ appendFacts: [],
1330
+ };
1331
+ }
1332
+ if (data.length !== properties.payloadDatas.length) {
1333
+ throw new Error("Mismatched native batch payload count");
1334
+ }
1335
+ const resolvedTrim = options.trim ?? this._trim.options;
1336
+ const supportsNativeTrim = !resolvedTrim ||
1337
+ (resolvedTrim.type === "length" &&
1338
+ !resolvedTrim.filter?.canTrim &&
1339
+ properties.resolveTrimmedEntries === false);
1340
+ if (options.canAppend ||
1341
+ options.onChange ||
1342
+ options.encryption ||
1343
+ options.signers ||
1344
+ options.identity ||
1345
+ options.meta?.timestamp ||
1346
+ options.meta?.type === EntryType.CUT ||
1347
+ (options.meta?.next != null && options.meta.next.length !== 0) ||
1348
+ options.meta?.gidSeed ||
1349
+ !supportsNativeTrim ||
1350
+ (this._hasCustomCanAppend && !canAppendAlreadyValidated(options))) {
1351
+ return undefined;
1352
+ }
1353
+ const identity = this._identity;
1354
+ if (!(identity instanceof Ed25519Keypair) || !hasPutMany(this._storage)) {
1355
+ return undefined;
1356
+ }
1357
+ const nativeTrimLengthTo = this.getNativeCommitOnlyTrimLengthTo(options.trim, properties.resolveTrimmedEntries);
1358
+ const entryType = options.meta?.type ?? EntryType.APPEND;
1359
+ const rows = properties.payloadDatas.map((payloadData) => {
1360
+ const gid = EntryV0.createGid();
1361
+ const timestamp = this._hlc.now();
1362
+ return {
1363
+ gid,
1364
+ timestamp,
1365
+ payloadData,
1366
+ input: {
1367
+ clockId: identity.publicKey.bytes,
1368
+ privateKey: identity.privateKey.privateKey,
1369
+ publicKey: identity.publicKey.publicKey,
1370
+ wallTime: timestamp.wallTime,
1371
+ logical: timestamp.logical,
1372
+ gid,
1373
+ type: entryType,
1374
+ metaData: options.meta?.data,
1375
+ payloadData,
1376
+ resolveTrimmedEntries: properties.resolveTrimmedEntries,
1377
+ trimLengthTo: nativeTrimLengthTo,
1378
+ },
1379
+ };
1380
+ });
1381
+ const preparedValue = prepare(rows.map((row) => row.input));
1382
+ return mapMaybePromise(preparedValue, (preparedRows) => {
1383
+ if (!preparedRows || preparedRows.length !== rows.length) {
1384
+ return undefined;
1385
+ }
1386
+ const appendFacts = [];
1387
+ const materializeEntries = [];
1388
+ const retainMaterializationBytesFns = [];
1389
+ const indexRows = [];
1390
+ const trimmedEntryHashes = [];
1391
+ const documentTrimmedHeadsProcessed = [];
1392
+ for (let index = 0; index < rows.length; index++) {
1393
+ const row = rows[index];
1394
+ const prepared = preparedRows[index];
1395
+ if (!prepared) {
1396
+ return undefined;
1397
+ }
1398
+ const hash = prepared.cid ?? prepared.hash;
1399
+ if (!hash) {
1400
+ return undefined;
1401
+ }
1402
+ const shouldRetainMaterializationBytes = properties.retainMaterializationBytes === true || !!resolvedTrim;
1403
+ let retainedMaterializationBytes;
1404
+ const retainMaterializationBytes = () => {
1405
+ if (retainedMaterializationBytes ||
1406
+ !shouldRetainMaterializationBytes ||
1407
+ prepared.bytes) {
1408
+ return;
1409
+ }
1410
+ const bytes = prepared.getBytes?.(hash) ??
1411
+ this._storage.get(hash);
1412
+ if (bytes &&
1413
+ typeof bytes.then !== "function") {
1414
+ retainedMaterializationBytes = bytes;
1415
+ }
1416
+ };
1417
+ const effectiveNextHashes = prepared.next ?? EMPTY_NEXT_HASHES;
1418
+ if (effectiveNextHashes.length !== 0 &&
1419
+ properties.allowPreparedNexts !== true) {
1420
+ return undefined;
1421
+ }
1422
+ const effectiveGid = prepared.gid ?? row.gid;
1423
+ let clock;
1424
+ const getClock = () => (clock ??= new Clock({
1425
+ id: identity.publicKey.bytes,
1426
+ timestamp: row.timestamp,
1427
+ }));
1428
+ let shallowEntry;
1429
+ const getShallowEntry = () => (shallowEntry ??= new ShallowEntry({
1430
+ hash,
1431
+ payloadSize: row.payloadData.byteLength,
1432
+ head: true,
1433
+ meta: new ShallowMeta({
1434
+ gid: effectiveGid,
1435
+ data: options.meta?.data,
1436
+ clock: getClock(),
1437
+ next: effectiveNextHashes,
1438
+ type: entryType,
1439
+ }),
1440
+ }));
1441
+ const facts = {
1442
+ hash,
1443
+ gid: effectiveGid,
1444
+ next: effectiveNextHashes,
1445
+ wallTime: row.timestamp.wallTime,
1446
+ logical: row.timestamp.logical,
1447
+ clockId: identity.publicKey.bytes,
1448
+ type: entryType,
1449
+ metaData: options.meta?.data,
1450
+ payloadSize: row.payloadData.byteLength,
1451
+ metaBytes: prepared.metaBytes,
1452
+ hashDigestBytes: prepared.hashDigestBytes,
1453
+ };
1454
+ let materializedEntry;
1455
+ const materializeEntry = () => {
1456
+ if (materializedEntry) {
1457
+ return materializedEntry;
1458
+ }
1459
+ const bytes = prepared.bytes ??
1460
+ retainedMaterializationBytes ??
1461
+ prepared.getBytes?.(hash) ??
1462
+ this._storage.get(hash);
1463
+ if (!bytes ||
1464
+ typeof bytes.then === "function") {
1465
+ throw new Error("Missing synchronous native append block bytes");
1466
+ }
1467
+ const entry = deserialize(bytes, Entry);
1468
+ entry.hash = hash;
1469
+ entry.size = prepared.byteLength;
1470
+ entry.createdLocally = true;
1471
+ Entry.prepareShallowEntry(entry, getShallowEntry());
1472
+ entry.init({ encoding: this._encoding, keychain: this._keychain });
1473
+ materializedEntry = entry;
1474
+ return entry;
1475
+ };
1476
+ appendFacts.push(facts);
1477
+ materializeEntries.push(materializeEntry);
1478
+ retainMaterializationBytesFns.push(retainMaterializationBytes);
1479
+ indexRows.push({
1480
+ hash,
1481
+ unique: true,
1482
+ externalNextHashes: effectiveNextHashes,
1483
+ getShallowEntry,
1484
+ isHead: true,
1485
+ });
1486
+ if (prepared.trimmedEntryHashes?.length) {
1487
+ trimmedEntryHashes.push(...prepared.trimmedEntryHashes);
1488
+ }
1489
+ documentTrimmedHeadsProcessed.push(prepared.documentTrimmedHeadsProcessed === true);
1490
+ }
1491
+ const rollback = (error) => {
1492
+ this.rollbackNativeAppendGraphHashes(appendFacts.map((facts) => facts.hash));
1493
+ return this.rollbackNativeAppendBlocksHashes(appendFacts.map((facts) => facts.hash)).then(() => {
1494
+ throw error;
1495
+ });
1496
+ };
1497
+ const finish = () => {
1498
+ for (const retainMaterializationBytes of retainMaterializationBytesFns) {
1499
+ retainMaterializationBytes();
1500
+ }
1501
+ let entries;
1502
+ return {
1503
+ get entries() {
1504
+ return (entries ??= materializeEntries.map((materializeEntry) => materializeEntry()));
1505
+ },
1506
+ materializeEntries,
1507
+ removed: [],
1508
+ removedHashes: trimmedEntryHashes.length > 0
1509
+ ? normalizedUniqueStrings(trimmedEntryHashes)
1510
+ : undefined,
1511
+ appendFacts,
1512
+ documentTrimmedHeadsProcessed,
1513
+ };
1514
+ };
1515
+ const finishTrim = () => {
1516
+ if (trimmedEntryHashes.length === 0) {
1517
+ return finish();
1518
+ }
1519
+ if (properties.resolveTrimmedEntries !== false &&
1520
+ !documentTrimmedHeadsProcessed.every(Boolean)) {
1521
+ return undefined;
1522
+ }
1523
+ const uniqueTrimmedHashes = normalizedUniqueStrings(trimmedEntryHashes);
1524
+ const consumedNoReturn = this.entryIndex.consumeNativeTrimmedEntryHashesNoReturnMaybe(uniqueTrimmedHashes, {
1525
+ skipNextHeadUpdates: true,
1526
+ deleteBlocks: false,
1527
+ });
1528
+ if (consumedNoReturn === undefined) {
1529
+ return undefined;
1530
+ }
1531
+ return mapMaybePromise(consumedNoReturn, finish);
1532
+ };
1533
+ try {
1534
+ const putResult = this.entryIndex.putNativeCommittedAppendFactsBatch(indexRows);
1535
+ const result = mapMaybePromise(putResult, finishTrim);
1536
+ return isPromiseLike(result) ? result.catch(rollback) : result;
1537
+ }
1538
+ catch (error) {
1539
+ return rollback(error);
1540
+ }
1541
+ });
1542
+ }
1543
+ createPreparedAppendFacts(entries, prepared) {
1544
+ if (prepared?.appendFacts?.length === entries.length) {
1545
+ return prepared.appendFacts;
1546
+ }
1547
+ return entries.map((entry, index) => {
1548
+ const shallowEntry = prepared?.shallowEntries[index];
1549
+ if (shallowEntry) {
1550
+ return {
1551
+ hash: shallowEntry.hash,
1552
+ gid: shallowEntry.meta.gid,
1553
+ next: shallowEntry.meta.next,
1554
+ wallTime: shallowEntry.meta.clock.timestamp.wallTime,
1555
+ logical: shallowEntry.meta.clock.timestamp.logical,
1556
+ clockId: shallowEntry.meta.clock.id,
1557
+ type: shallowEntry.meta.type,
1558
+ metaData: shallowEntry.meta.data,
1559
+ payloadSize: shallowEntry.payloadSize,
1560
+ metaBytes: entry.getMetaBytes?.(),
1561
+ hashDigestBytes: entry.getHashDigestBytes?.(),
1562
+ };
1563
+ }
1564
+ return {
1565
+ hash: entry.hash,
1566
+ gid: entry.meta.gid,
1567
+ next: entry.meta.next,
1568
+ wallTime: entry.meta.clock.timestamp.wallTime,
1569
+ logical: entry.meta.clock.timestamp.logical,
1570
+ clockId: entry.meta.clock.id,
1571
+ type: entry.meta.type,
1572
+ metaData: entry.meta.data,
1573
+ payloadSize: entry.payload.byteLength,
1574
+ metaBytes: entry.getMetaBytes?.(),
1575
+ hashDigestBytes: entry.getHashDigestBytes?.(),
1576
+ };
1577
+ });
1578
+ }
1579
+ async appendMany(data, options = {}) {
1580
+ if (data.length === 0) {
1581
+ return { entries: [], removed: [] };
1582
+ }
1583
+ if (options.meta?.type === EntryType.CUT) {
1584
+ throw new Error("appendMany does not support CUT entries");
1585
+ }
1586
+ let nexts = await this.getNextsForAppend(options);
1587
+ const initialNexts = nexts;
1588
+ const entries = [];
1589
+ const pendingDeletes = [];
1590
+ const deferBlockStore = hasPutMany(this._storage);
1591
+ const nativeAppendChain = await this.createNativePlainAppendChain(data, options, nexts, deferBlockStore);
1592
+ if (nativeAppendChain) {
1593
+ entries.push(...nativeAppendChain.entries);
1594
+ nexts = [entries[entries.length - 1]];
1595
+ }
1596
+ else {
1597
+ for (const item of data) {
1598
+ const entry = await this.createAppendEntry(item, options, nexts, {
1599
+ deferStore: deferBlockStore,
1600
+ });
1601
+ entries.push(entry);
1602
+ nexts = [entry];
1603
+ }
1604
+ }
1605
+ try {
1606
+ await this.joinMissingNexts(entries[0], initialNexts);
1607
+ if (deferBlockStore && !nativeAppendChain?.nativeBlocksCommitted) {
1608
+ await this.putAppendEntryBlocks(entries, nativeAppendChain?.blocks);
1609
+ }
1610
+ await this.putAppendEntries(entries, options, initialNexts.map((entry) => entry.hash), nativeAppendChain);
1611
+ }
1612
+ catch (error) {
1613
+ if (nativeAppendChain?.nativeGraphUpdated) {
1614
+ this.rollbackNativeAppendGraph(entries);
1615
+ }
1616
+ if (nativeAppendChain?.nativeBlocksCommitted) {
1617
+ await this.rollbackNativeAppendBlocks(entries);
1618
+ }
1619
+ throw error;
1620
+ }
1621
+ for (const entry of entries) {
1622
+ entry.init({ encoding: this._encoding, keychain: this._keychain });
1623
+ }
1624
+ const trimmed = await this.trimIfConfigured(options?.trim);
1625
+ if (trimmed) {
1626
+ for (const entry of trimmed) {
1627
+ pendingDeletes.push({ entry, fn: undefined });
1628
+ }
1629
+ }
1630
+ const removed = pendingDeletes.map((x) => x.entry);
1631
+ const changes = {
1632
+ added: entries.map((entry, index) => ({
1633
+ head: index === entries.length - 1,
1634
+ entry,
1635
+ })),
1636
+ removed,
1637
+ };
1638
+ await (options?.onChange || this._onChange)?.(changes);
1639
+ await Promise.all(pendingDeletes.map((x) => x.fn?.()));
1640
+ return { entries, removed };
1641
+ }
1642
+ createNativePlainAppendChain(data, options, nexts, deferBlockStore, payloadDatas) {
1643
+ const canAppendAlreadyValidatedForOptions = canAppendAlreadyValidated(options);
1644
+ if (!deferBlockStore ||
1645
+ options.encryption ||
1646
+ options.signers ||
1647
+ options.canAppend ||
1648
+ (this._hasCustomCanAppend && !canAppendAlreadyValidatedForOptions) ||
1649
+ options.meta?.timestamp ||
1650
+ options.meta?.type === EntryType.CUT) {
1651
+ return Promise.resolve(undefined);
1652
+ }
1653
+ const nativeGraph = !this.entryIndex.properties.onGidRemoved &&
1654
+ (this.entryIndex.properties.nativeGraph?.graph
1655
+ .prepareEntryV0PlainChainCommit ||
1656
+ this.entryIndex.properties.nativeGraph?.graph
1657
+ .prepareEntryV0PlainEntryCommit ||
1658
+ this.entryIndex.properties.nativeGraph?.graph
1659
+ .prepareEntryV0PlainChainAndPut ||
1660
+ this.entryIndex.properties.nativeGraph?.graph
1661
+ .prepareEntryV0PlainEntryAndPut)
1662
+ ? this.entryIndex.properties.nativeGraph.graph
1663
+ : undefined;
1664
+ return EntryV0.createPlainAppendChainBatch({
1665
+ data,
1666
+ meta: {
1667
+ clocks: () => data.map(() => new Clock({
1668
+ id: this._identity.publicKey.bytes,
1669
+ timestamp: this._hlc.now(),
1670
+ })),
1671
+ type: options.meta?.type,
1672
+ gidSeed: options.meta?.gidSeed,
1673
+ data: options.meta?.data,
1674
+ next: nexts,
1675
+ },
1676
+ encoding: this._encoding,
1677
+ payloadDatas,
1678
+ identity: options.identity || this._identity,
1679
+ deferStore: deferBlockStore,
1680
+ cachePreparedEntries: false,
1681
+ nativeGraph,
1682
+ nativeBlockStore: this._storage,
1683
+ });
1684
+ }
1685
+ createNativePlainAppendCommitOnly(data, options, nexts, deferBlockStore, payloadDatas, includeMaterializationBytes, includeAppendFactsBytes, nativeTrimLengthTo) {
1686
+ const canAppendAlreadyValidatedForOptions = canAppendAlreadyValidated(options);
1687
+ if (data.length !== 1 ||
1688
+ !deferBlockStore ||
1689
+ options.encryption ||
1690
+ options.signers ||
1691
+ options.canAppend ||
1692
+ (this._hasCustomCanAppend && !canAppendAlreadyValidatedForOptions) ||
1693
+ options.meta?.timestamp ||
1694
+ options.meta?.type === EntryType.CUT) {
1695
+ return undefined;
1696
+ }
1697
+ const nativeGraph = !this.entryIndex.properties.onGidRemoved &&
1698
+ (this.entryIndex.properties.nativeGraph?.graph
1699
+ .prepareEntryV0PlainEntryCommit ||
1700
+ this.entryIndex.properties.nativeGraph?.graph
1701
+ .prepareEntryV0PlainEntryAndPut)
1702
+ ? this.entryIndex.properties.nativeGraph.graph
1703
+ : undefined;
1704
+ if (!nativeGraph) {
1705
+ return undefined;
1706
+ }
1707
+ return EntryV0.createPlainAppendChainCommitOnly({
1708
+ data,
1709
+ meta: {
1710
+ clocks: () => [
1711
+ new Clock({
1712
+ id: this._identity.publicKey.bytes,
1713
+ timestamp: this._hlc.now(),
1714
+ }),
1715
+ ],
1716
+ type: options.meta?.type,
1717
+ gidSeed: options.meta?.gidSeed,
1718
+ data: options.meta?.data,
1719
+ next: nexts,
1720
+ },
1721
+ encoding: this._encoding,
1722
+ payloadDatas,
1723
+ identity: options.identity || this._identity,
1724
+ deferStore: deferBlockStore,
1725
+ nativeGraph,
1726
+ nativeBlockStore: this._storage,
1727
+ includeMaterializationBytes,
1728
+ includeAppendFactsBytes,
1729
+ nativeTrimLengthTo,
1730
+ });
1731
+ }
1732
+ async createNativePlainAppendEntriesBatch(data, options, deferBlockStore, payloadDatas, nexts) {
1733
+ const canAppendAlreadyValidatedForOptions = canAppendAlreadyValidated(options);
1734
+ if (!deferBlockStore ||
1735
+ data.length === 0 ||
1736
+ options.encryption ||
1737
+ options.signers ||
1738
+ options.canAppend ||
1739
+ (this._hasCustomCanAppend && !canAppendAlreadyValidatedForOptions) ||
1740
+ options.meta?.timestamp ||
1741
+ options.meta?.type === EntryType.CUT ||
1742
+ options.meta?.gidSeed ||
1743
+ options.meta?.next ||
1744
+ (nexts && nexts.length !== data.length) ||
1745
+ (payloadDatas && payloadDatas.length !== data.length)) {
1746
+ return undefined;
1747
+ }
1748
+ const nativeGraph = !this.entryIndex.properties.onGidRemoved &&
1749
+ this.entryIndex.properties.nativeGraph?.graph
1750
+ ? this.entryIndex.properties.nativeGraph.graph
1751
+ : undefined;
1752
+ const generatedGids = EntryV0.createGids(data.length);
1753
+ const gids = generatedGids.map((generatedGid, index) => {
1754
+ const entryNexts = nexts?.[index];
1755
+ if (!entryNexts || entryNexts.length === 0) {
1756
+ return generatedGid;
1757
+ }
1758
+ let gid = entryNexts[0].meta.gid;
1759
+ for (let i = 1; i < entryNexts.length; i++) {
1760
+ const nextGid = entryNexts[i].meta.gid;
1761
+ if (nextGid < gid) {
1762
+ gid = nextGid;
1763
+ }
1764
+ }
1765
+ return gid;
1766
+ });
1767
+ const clockId = this._identity.publicKey.bytes;
1768
+ const clocks = this._hlc.nowBatch(data.length).map((timestamp) => new Clock({
1769
+ id: clockId,
1770
+ timestamp,
1771
+ }));
1772
+ const metaDatas = data.map(() => options.meta?.data);
1773
+ const directBatch = nativeGraph?.prepareEntryV0PlainEntriesCommit
1774
+ ? await EntryV0.createPlainAppendEntriesBatch({
1775
+ data,
1776
+ payloadDatas,
1777
+ meta: {
1778
+ clocks: () => clocks,
1779
+ gids,
1780
+ nexts,
1781
+ type: options.meta?.type,
1782
+ datas: metaDatas,
1783
+ },
1784
+ encoding: this._encoding,
1785
+ identity: options.identity || this._identity,
1786
+ deferStore: deferBlockStore,
1787
+ cachePreparedEntries: false,
1788
+ nativeGraph,
1789
+ nativeBlockStore: this._storage,
1790
+ })
1791
+ : undefined;
1792
+ if (directBatch) {
1793
+ return directBatch;
1794
+ }
1795
+ const entries = [];
1796
+ const blocks = [];
1797
+ const shallowEntries = [];
1798
+ const nativeEntries = [];
1799
+ let nativeGraphUpdated = false;
1800
+ let nativeBlocksCommitted = true;
1801
+ for (let i = 0; i < data.length; i++) {
1802
+ const entryNexts = nexts?.[i] ?? [];
1803
+ const prepared = await EntryV0.createPlainAppendChainBatch({
1804
+ data: [data[i]],
1805
+ payloadDatas: payloadDatas ? [payloadDatas[i]] : undefined,
1806
+ meta: {
1807
+ clocks: () => [clocks[i]],
1808
+ gid: entryNexts.length === 0 ? gids[i] : undefined,
1809
+ type: options.meta?.type,
1810
+ data: metaDatas[i],
1811
+ next: entryNexts,
1812
+ },
1813
+ encoding: this._encoding,
1814
+ identity: options.identity || this._identity,
1815
+ deferStore: deferBlockStore,
1816
+ cachePreparedEntries: false,
1817
+ nativeGraph,
1818
+ nativeBlockStore: this._storage,
1819
+ });
1820
+ if (!prepared) {
1821
+ return undefined;
1822
+ }
1823
+ entries.push(prepared.entries[0]);
1824
+ if (prepared.blocks) {
1825
+ blocks.push(...prepared.blocks);
1826
+ }
1827
+ shallowEntries.push(...prepared.shallowEntries);
1828
+ if (prepared.nativeEntries) {
1829
+ nativeEntries.push(...prepared.nativeEntries);
1830
+ }
1831
+ nativeGraphUpdated ||= prepared.nativeGraphUpdated === true;
1832
+ nativeBlocksCommitted &&= prepared.nativeBlocksCommitted === true;
1833
+ }
1834
+ if (!nativeBlocksCommitted && blocks.length !== entries.length) {
1835
+ return undefined;
1836
+ }
1837
+ return {
1838
+ entries,
1839
+ blocks: blocks.length > 0 ? blocks : undefined,
1840
+ shallowEntries,
1841
+ nativeEntries,
1842
+ nativeGraphUpdated,
1843
+ nativeBlocksCommitted,
1844
+ };
1845
+ }
1846
+ rollbackNativeAppendGraph(entries) {
1847
+ this.rollbackNativeAppendGraphHashes(entries.map((entry) => entry.hash));
1848
+ }
1849
+ rollbackNativeAppendGraphHashes(hashes) {
1850
+ const graph = this.entryIndex.properties.nativeGraph?.graph;
1851
+ if (!graph) {
1852
+ return;
1853
+ }
1854
+ for (let i = hashes.length - 1; i >= 0; i--) {
1855
+ graph.delete(hashes[i]);
1856
+ }
1857
+ }
1858
+ async rollbackNativeAppendBlocks(entries) {
1859
+ await this.rollbackNativeAppendBlocksHashes(entries.map((entry) => entry.hash));
1860
+ }
1861
+ async rollbackNativeAppendBlocksHashes(hashes) {
1862
+ const storage = this._storage;
1863
+ if (typeof storage.rmMany === "function") {
1864
+ await storage.rmMany(hashes);
1865
+ return;
402
1866
  }
403
- return ret;
1867
+ await Promise.all(hashes.map((hash) => this._storage.rm(hash)));
404
1868
  }
405
- /**
406
- * Append an entry to the log.
407
- * @param {T} data The data to be appended
408
- * @param {AppendOptions} [options] The options for the append
409
- * @returns {{ entry: Entry<T>; removed: ShallowEntry[] }} The appended entry and an array of removed entries
410
- */
411
- async append(data, options = {}) {
412
- // Update the clock (find the latest clock)
413
- if (options.meta?.next) {
414
- for (const n of options.meta.next) {
415
- if (!n.hash)
416
- throw new Error("Expecting nexts to already be saved. missing hash for one or more entries");
1869
+ validateExplicitNexts(options) {
1870
+ if (!options.meta?.next) {
1871
+ return;
1872
+ }
1873
+ for (const n of options.meta.next) {
1874
+ if (!n.hash) {
1875
+ throw new Error("Expecting nexts to already be saved. missing hash for one or more entries");
417
1876
  }
418
1877
  }
419
- /* await this.load({ reload: false }); */
420
- const nexts = options.meta?.next ||
421
- (await this.entryIndex
1878
+ }
1879
+ getNextsForAppend(options) {
1880
+ this.validateExplicitNexts(options);
1881
+ return (options.meta?.next ||
1882
+ this.entryIndex.getHeadsForAppend() ||
1883
+ this.entryIndex
422
1884
  .getHeads(undefined, { type: "shape", shape: Sorting.ENTRY_SORT_SHAPE })
423
1885
  .all());
424
- // Calculate max time for log/graph
1886
+ }
1887
+ async createAppendEntry(data, options, nexts, storeOptions) {
425
1888
  const clock = new Clock({
426
1889
  id: this._identity.publicKey.bytes,
427
1890
  timestamp: options?.meta?.timestamp || this._hlc.now(),
@@ -447,31 +1910,41 @@ let Log = (() => {
447
1910
  },
448
1911
  }
449
1912
  : undefined,
450
- canAppend: options.canAppend || this._canAppend,
1913
+ canAppend: canAppendAlreadyValidated(options)
1914
+ ? undefined
1915
+ : options.canAppend ||
1916
+ (this._hasCustomCanAppend ? this._canAppend : undefined),
1917
+ deferStore: storeOptions?.deferStore,
451
1918
  });
452
1919
  if (!entry.hash) {
453
1920
  throw new Error("Unexpected");
454
1921
  }
455
- if (entry.meta.type !== EntryType.CUT) {
456
- for (const e of nexts) {
457
- if (!(await this.has(e.hash))) {
458
- let entry;
459
- if (e instanceof Entry) {
460
- entry = e;
461
- }
462
- else {
463
- let resolved = await this.entryIndex.get(e.hash);
464
- if (!resolved) {
465
- // eslint-disable-next-line no-console
466
- console.warn("Unexpected missing entry when joining", e.hash);
467
- continue;
468
- }
469
- entry = resolved;
470
- }
471
- await this.join([entry]);
1922
+ return entry;
1923
+ }
1924
+ async joinMissingNexts(entry, nexts) {
1925
+ if (entry.meta.type === EntryType.CUT) {
1926
+ return;
1927
+ }
1928
+ for (const e of nexts) {
1929
+ if (await this.has(e.hash)) {
1930
+ continue;
1931
+ }
1932
+ let nextEntry;
1933
+ if (e instanceof Entry) {
1934
+ nextEntry = e;
1935
+ }
1936
+ else {
1937
+ const resolved = await this.entryIndex.get(e.hash);
1938
+ if (!resolved) {
1939
+ warn("Unexpected missing entry when joining", e.hash);
1940
+ continue;
472
1941
  }
1942
+ nextEntry = resolved;
473
1943
  }
1944
+ await this.join([nextEntry]);
474
1945
  }
1946
+ }
1947
+ async putAppendEntry(entry, options) {
475
1948
  await this.entryIndex.put(entry, {
476
1949
  unique: true,
477
1950
  isHead: true,
@@ -481,22 +1954,214 @@ let Log = (() => {
481
1954
  ? options.durability === "buffered"
482
1955
  : this._appendDurability === "buffered"),
483
1956
  });
484
- const pendingDeletes = await this.processEntry(entry);
485
- entry.init({ encoding: this._encoding, keychain: this._keychain });
486
- const trimmed = await this.trim(options?.trim);
487
- if (trimmed) {
488
- for (const entry of trimmed) {
489
- pendingDeletes.push({ entry, fn: undefined });
1957
+ }
1958
+ async putAppendEntries(entries, options, externalNextHashes, preparedAppendChain, heads) {
1959
+ const prepared = preparedAppendChain &&
1960
+ entries.length === preparedAppendChain.entries.length
1961
+ ? {
1962
+ shallowEntries: preparedAppendChain.shallowEntries,
1963
+ nativeEntries: preparedAppendChain.nativeEntries,
1964
+ nativeGraphUpdated: preparedAppendChain.nativeGraphUpdated,
1965
+ nativeBlocksCommitted: preparedAppendChain.nativeBlocksCommitted,
490
1966
  }
1967
+ : undefined;
1968
+ if (entries.length === 1 &&
1969
+ prepared?.nativeGraphUpdated === true &&
1970
+ !this.entryIndex.properties.onGidRemoved) {
1971
+ await this.entryIndex.putNativeCommittedAppend(entries[0], {
1972
+ unique: true,
1973
+ externalNextHashes,
1974
+ shallowEntry: prepared.shallowEntries[0],
1975
+ isHead: heads?.[0] ?? true,
1976
+ });
1977
+ return;
491
1978
  }
492
- const removed = pendingDeletes.map((x) => x.entry);
493
- const changes = {
494
- added: [{ head: true, entry }],
495
- removed,
1979
+ await this.entryIndex.putAppendBatch(entries, {
1980
+ unique: true,
1981
+ externalNextHashes,
1982
+ heads,
1983
+ prepared,
1984
+ deferIndexWrite: options.deferIndexWrite ??
1985
+ (options.durability
1986
+ ? options.durability === "buffered"
1987
+ : this._appendDurability === "buffered"),
1988
+ });
1989
+ }
1990
+ async putAppendEntryBlocks(entries, preparedBlocks) {
1991
+ const blocks = preparedBlocks && preparedBlocks.length === entries.length
1992
+ ? preparedBlocks
1993
+ : entries.map((entry) => {
1994
+ const prepared = Entry.takePreparedBlock(entry);
1995
+ if (!prepared) {
1996
+ throw new Error("Missing prepared entry block");
1997
+ }
1998
+ return prepared;
1999
+ });
2000
+ if (blocks.length === 1 && hasPutKnown(this._storage)) {
2001
+ const block = blocks[0];
2002
+ const cidResult = this._storage.putKnown(block.cid, block.block.bytes);
2003
+ const cid = isPromiseLike(cidResult) ? await cidResult : cidResult;
2004
+ if (cid !== block.cid) {
2005
+ throw new Error("Unexpected block cid");
2006
+ }
2007
+ return;
2008
+ }
2009
+ if (hasPutKnownManyColumns(this._storage)) {
2010
+ const cids = new Array(blocks.length);
2011
+ const bytes = new Array(blocks.length);
2012
+ for (let i = 0; i < blocks.length; i++) {
2013
+ const block = blocks[i];
2014
+ cids[i] = block.cid;
2015
+ bytes[i] = block.block.bytes;
2016
+ }
2017
+ const cidsResult = this._storage.putKnownManyColumns(cids, bytes);
2018
+ const result = isPromiseLike(cidsResult) ? await cidsResult : cidsResult;
2019
+ if (result.length !== blocks.length) {
2020
+ throw new Error("Unexpected block batch result length");
2021
+ }
2022
+ for (let i = 0; i < result.length; i++) {
2023
+ if (result[i] !== cids[i]) {
2024
+ throw new Error("Unexpected block batch cid");
2025
+ }
2026
+ }
2027
+ return;
2028
+ }
2029
+ if (hasPutKnownMany(this._storage)) {
2030
+ const cidsResult = this._storage.putKnownMany(blocks.map((block) => [block.cid, block.block.bytes]));
2031
+ const cids = isPromiseLike(cidsResult) ? await cidsResult : cidsResult;
2032
+ if (cids.length !== blocks.length) {
2033
+ throw new Error("Unexpected block batch result length");
2034
+ }
2035
+ for (let i = 0; i < cids.length; i++) {
2036
+ if (cids[i] !== blocks[i].cid) {
2037
+ throw new Error("Unexpected block batch cid");
2038
+ }
2039
+ }
2040
+ return;
2041
+ }
2042
+ const cids = await this._storage.putMany(blocks);
2043
+ if (cids.length !== blocks.length) {
2044
+ throw new Error("Unexpected block batch result length");
2045
+ }
2046
+ for (let i = 0; i < cids.length; i++) {
2047
+ if (cids[i] !== blocks[i].cid) {
2048
+ throw new Error("Unexpected block batch cid");
2049
+ }
2050
+ }
2051
+ }
2052
+ async putKnownEntryBytesBatch(blocks) {
2053
+ if (blocks.length === 0) {
2054
+ return;
2055
+ }
2056
+ if (blocks.length === 1 && hasPutKnown(this._storage)) {
2057
+ const block = blocks[0];
2058
+ const cidResult = this._storage.putKnown(block.cid, block.bytes);
2059
+ const cid = isPromiseLike(cidResult) ? await cidResult : cidResult;
2060
+ if (cid !== block.cid) {
2061
+ throw new Error("Unexpected block cid");
2062
+ }
2063
+ return;
2064
+ }
2065
+ if (hasPutKnownManyColumns(this._storage)) {
2066
+ const cids = new Array(blocks.length);
2067
+ const bytes = new Array(blocks.length);
2068
+ for (let i = 0; i < blocks.length; i++) {
2069
+ const block = blocks[i];
2070
+ cids[i] = block.cid;
2071
+ bytes[i] = block.bytes;
2072
+ }
2073
+ const cidsResult = this._storage.putKnownManyColumns(cids, bytes);
2074
+ const result = isPromiseLike(cidsResult) ? await cidsResult : cidsResult;
2075
+ if (result.length !== blocks.length) {
2076
+ throw new Error("Unexpected block batch result length");
2077
+ }
2078
+ for (let i = 0; i < result.length; i++) {
2079
+ if (result[i] !== cids[i]) {
2080
+ throw new Error("Unexpected block batch cid");
2081
+ }
2082
+ }
2083
+ return;
2084
+ }
2085
+ if (hasPutKnownMany(this._storage)) {
2086
+ const cidsResult = this._storage.putKnownMany(blocks.map((block) => [block.cid, block.bytes]));
2087
+ const cids = isPromiseLike(cidsResult) ? await cidsResult : cidsResult;
2088
+ if (cids.length !== blocks.length) {
2089
+ throw new Error("Unexpected block batch result length");
2090
+ }
2091
+ for (let i = 0; i < cids.length; i++) {
2092
+ if (cids[i] !== blocks[i].cid) {
2093
+ throw new Error("Unexpected block batch cid");
2094
+ }
2095
+ }
2096
+ return;
2097
+ }
2098
+ const preparedBlocks = blocks.map((block) => Entry.preparedBlockFromBytes(block.bytes, block.cid));
2099
+ const cids = await this._storage.putMany(preparedBlocks);
2100
+ if (cids.length !== blocks.length) {
2101
+ throw new Error("Unexpected block batch result length");
2102
+ }
2103
+ for (let i = 0; i < cids.length; i++) {
2104
+ if (cids[i] !== blocks[i].cid) {
2105
+ throw new Error("Unexpected block batch cid");
2106
+ }
2107
+ }
2108
+ }
2109
+ putPreparedAppendBlocks(preparedBlocks) {
2110
+ if (!preparedBlocks || preparedBlocks.length === 0) {
2111
+ throw new Error("Missing prepared entry block");
2112
+ }
2113
+ if (preparedBlocks.length === 1 && hasPutKnown(this._storage)) {
2114
+ const block = preparedBlocks[0];
2115
+ const cidResult = this._storage.putKnown(block.cid, block.block.bytes);
2116
+ const checkCid = (cid) => {
2117
+ if (cid !== block.cid) {
2118
+ throw new Error("Unexpected block cid");
2119
+ }
2120
+ };
2121
+ if (isPromiseLike(cidResult)) {
2122
+ return cidResult.then(checkCid);
2123
+ }
2124
+ checkCid(cidResult);
2125
+ return;
2126
+ }
2127
+ const checkCids = (cids) => {
2128
+ if (cids.length !== preparedBlocks.length) {
2129
+ throw new Error("Unexpected block batch result length");
2130
+ }
2131
+ for (let i = 0; i < cids.length; i++) {
2132
+ if (cids[i] !== preparedBlocks[i].cid) {
2133
+ throw new Error("Unexpected block batch cid");
2134
+ }
2135
+ }
496
2136
  };
497
- await (options?.onChange || this._onChange)?.(changes);
498
- await Promise.all(pendingDeletes.map((x) => x.fn?.()));
499
- return { entry, removed };
2137
+ if (hasPutKnownManyColumns(this._storage)) {
2138
+ const cids = new Array(preparedBlocks.length);
2139
+ const bytes = new Array(preparedBlocks.length);
2140
+ for (let i = 0; i < preparedBlocks.length; i++) {
2141
+ const block = preparedBlocks[i];
2142
+ cids[i] = block.cid;
2143
+ bytes[i] = block.block.bytes;
2144
+ }
2145
+ const cidsResult = this._storage.putKnownManyColumns(cids, bytes);
2146
+ if (isPromiseLike(cidsResult)) {
2147
+ return cidsResult.then(checkCids);
2148
+ }
2149
+ checkCids(cidsResult);
2150
+ return;
2151
+ }
2152
+ if (hasPutKnownMany(this._storage)) {
2153
+ const cidsResult = this._storage.putKnownMany(preparedBlocks.map((block) => [block.cid, block.block.bytes]));
2154
+ if (isPromiseLike(cidsResult)) {
2155
+ return cidsResult.then(checkCids);
2156
+ }
2157
+ checkCids(cidsResult);
2158
+ return;
2159
+ }
2160
+ const cidsResult = this._storage.putMany(preparedBlocks);
2161
+ if (isPromiseLike(cidsResult)) {
2162
+ return cidsResult.then(checkCids);
2163
+ }
2164
+ checkCids(cidsResult);
500
2165
  }
501
2166
  async remove(entry, options) {
502
2167
  /* await this.load({ reload: false }); */
@@ -528,8 +2193,26 @@ let Log = (() => {
528
2193
  await Promise.all(removed.map((x) => x.fn()));
529
2194
  return change;
530
2195
  }
531
- async trim(option = this._trim.options) {
532
- return this._trim.trim(option);
2196
+ async trim(option = this._trim.options, properties) {
2197
+ return this._trim.trim(option, properties);
2198
+ }
2199
+ trimIfConfigured(option, properties) {
2200
+ const resolved = option ?? this._trim.options;
2201
+ return resolved ? this.trim(resolved, properties) : undefined;
2202
+ }
2203
+ getNativeCommitOnlyTrimLengthTo(option, resolveDeletedEntries) {
2204
+ const resolved = option ?? this._trim.options;
2205
+ if (!resolved ||
2206
+ resolved.type !== "length" ||
2207
+ resolved.filter?.canTrim ||
2208
+ resolveDeletedEntries !== false) {
2209
+ return;
2210
+ }
2211
+ const from = resolved.from ?? resolved.to;
2212
+ if (this.length + 1 < from) {
2213
+ return;
2214
+ }
2215
+ return resolved.to;
533
2216
  }
534
2217
  async join(entriesOrLog, options) {
535
2218
  let entries;
@@ -566,7 +2249,7 @@ let Log = (() => {
566
2249
  else if (Array.isArray(entriesOrLog)) {
567
2250
  if (entriesOrLog.length === 0)
568
2251
  return;
569
- const existingHashes = options?.reset
2252
+ const existingHashes = options?.reset || options?.__peerbitEntriesAlreadyMissing === true
570
2253
  ? new Set()
571
2254
  : await this.entryIndex.hasMany(entriesOrLog.map((element) => typeof element === "string"
572
2255
  ? element
@@ -648,14 +2331,33 @@ let Log = (() => {
648
2331
  return;
649
2332
  entries = all;
650
2333
  }
2334
+ const profile = options?.__peerbitProfile;
2335
+ const headsStartedAt = internalProfileStart(profile);
651
2336
  const heads = new Map();
652
2337
  for (const entry of entries) {
653
2338
  if (heads.has(entry.hash))
654
2339
  continue;
655
2340
  heads.set(entry.hash, true);
656
- for (const next of await entry.getNext())
2341
+ const nexts = options?.__peerbitBatchIndependent === true
2342
+ ? entry.meta.next
2343
+ : await entry.getNext();
2344
+ for (const next of nexts)
657
2345
  heads.set(next, false);
658
2346
  }
2347
+ emitInternalProfileDuration(profile, headsStartedAt, {
2348
+ name: "log.join.prepareHeads",
2349
+ component: "log",
2350
+ entries: entries.length,
2351
+ count: heads.size,
2352
+ messages: 1,
2353
+ details: {
2354
+ batchIndependent: options?.__peerbitBatchIndependent === true,
2355
+ },
2356
+ });
2357
+ if (options?.__peerbitBatchIndependent === true &&
2358
+ (await this.tryJoinIndependentAppendBatch(entries, heads, options))) {
2359
+ return;
2360
+ }
659
2361
  for (const entry of entries) {
660
2362
  const isHead = heads.get(entry.hash);
661
2363
  const prev = this._joining.get(entry.hash);
@@ -680,6 +2382,451 @@ let Log = (() => {
680
2382
  await p;
681
2383
  }
682
2384
  }
2385
+ // Internal trusted receive path for callers that can supply prepared append facts.
2386
+ async joinPreparedAppendFactsBatch(entries, options) {
2387
+ if (entries.length === 0 ||
2388
+ !canAppendAlreadyValidated(options) ||
2389
+ entries.some((entry) => this._joining.has(entry.hash))) {
2390
+ return false;
2391
+ }
2392
+ const resolvedOptions = options;
2393
+ const profile = resolvedOptions.__peerbitProfile;
2394
+ const prepareStartedAt = internalProfileStart(profile);
2395
+ const entryHashes = new Array(entries.length);
2396
+ let hasAnyNext = false;
2397
+ for (let i = 0; i < entries.length; i++) {
2398
+ const entry = entries[i];
2399
+ // byteLength stands in for the bytes presence check: prepared facts
2400
+ // carry both, and probing `bytes` would force stash-backed heads to
2401
+ // materialize block bytes the native commit never reads.
2402
+ if (!entry.hash ||
2403
+ entry.byteLength == null ||
2404
+ entry.meta.type !== EntryType.APPEND) {
2405
+ return false;
2406
+ }
2407
+ entryHashes[i] = entry.hash;
2408
+ if (entry.meta.next.length > 0) {
2409
+ hasAnyNext = true;
2410
+ }
2411
+ }
2412
+ const batchHashes = new Set(entryHashes);
2413
+ let heads;
2414
+ if (hasAnyNext) {
2415
+ heads = new Map();
2416
+ for (const entry of entries) {
2417
+ if (heads.has(entry.hash)) {
2418
+ continue;
2419
+ }
2420
+ heads.set(entry.hash, true);
2421
+ for (const next of entry.meta.next) {
2422
+ heads.set(next, false);
2423
+ }
2424
+ }
2425
+ }
2426
+ emitInternalProfileDuration(profile, prepareStartedAt, {
2427
+ name: "log.joinPreparedFacts.prepare",
2428
+ component: "log",
2429
+ entries: entries.length,
2430
+ count: heads?.size ?? entries.length,
2431
+ messages: 1,
2432
+ });
2433
+ const nativeCommitValidatesPlan = resolvedOptions.__peerbitNativePreparedJoinCommitValidatesPlan === true &&
2434
+ !!resolvedOptions.__peerbitNativePreparedJoinCommit;
2435
+ const headFlags = [];
2436
+ const headFlagsBytes = new Uint8Array(entries.length);
2437
+ const pushHeadFlag = (index, isHead) => {
2438
+ headFlags.push(isHead);
2439
+ headFlagsBytes[index] = isHead ? 1 : 0;
2440
+ };
2441
+ if (nativeCommitValidatesPlan) {
2442
+ for (let i = 0; i < entries.length; i++) {
2443
+ const entry = entries[i];
2444
+ pushHeadFlag(i, heads?.get(entry.hash) ?? true);
2445
+ }
2446
+ emitInternalProfileDuration(profile, internalProfileStart(profile), {
2447
+ name: "log.joinPreparedFacts.plan",
2448
+ component: "log",
2449
+ entries: entries.length,
2450
+ messages: 1,
2451
+ details: { nativeCommitValidatesPlan: true },
2452
+ });
2453
+ emitInternalProfileDuration(profile, internalProfileStart(profile), {
2454
+ name: "log.joinPreparedFacts.validatePlan",
2455
+ component: "log",
2456
+ entries: entries.length,
2457
+ messages: 1,
2458
+ details: { nativeCommitValidatesPlan: true },
2459
+ });
2460
+ }
2461
+ else {
2462
+ const planStartedAt = internalProfileStart(profile);
2463
+ const joinPlans = await this.entryIndex.planJoinBatch(entries, false, profile);
2464
+ emitInternalProfileDuration(profile, planStartedAt, {
2465
+ name: "log.joinPreparedFacts.plan",
2466
+ component: "log",
2467
+ entries: entries.length,
2468
+ messages: 1,
2469
+ });
2470
+ const validatePlanStartedAt = internalProfileStart(profile);
2471
+ for (let i = 0; i < entries.length; i++) {
2472
+ const entry = entries[i];
2473
+ const joinPlan = joinPlans[i];
2474
+ if (joinPlan.skip ||
2475
+ joinPlan.coveredByCut ||
2476
+ !joinPlan.cutChecked ||
2477
+ joinPlan.missingParents.some((hash) => !batchHashes.has(hash))) {
2478
+ return false;
2479
+ }
2480
+ pushHeadFlag(i, heads?.get(entry.hash) ?? true);
2481
+ }
2482
+ emitInternalProfileDuration(profile, validatePlanStartedAt, {
2483
+ name: "log.joinPreparedFacts.validatePlan",
2484
+ component: "log",
2485
+ entries: entries.length,
2486
+ messages: 1,
2487
+ });
2488
+ }
2489
+ let nativeValidatedCommitRejected = false;
2490
+ const batchPromise = (async () => {
2491
+ const clockStartedAt = internalProfileStart(profile);
2492
+ for (const entry of entries) {
2493
+ this._hlc.update(entry.meta.clock.timestamp);
2494
+ }
2495
+ emitInternalProfileDuration(profile, clockStartedAt, {
2496
+ name: "log.joinPreparedFacts.clock",
2497
+ component: "log",
2498
+ entries: entries.length,
2499
+ messages: 1,
2500
+ });
2501
+ const trustedMissing = resolvedOptions.__peerbitEntriesAlreadyMissing === true &&
2502
+ batchHashes.size === entries.length;
2503
+ let nativePreparedCommitted = false;
2504
+ if (resolvedOptions.__peerbitNativePreparedJoinCommit) {
2505
+ const nativeCommitStartedAt = internalProfileStart(profile);
2506
+ nativePreparedCommitted =
2507
+ (await resolvedOptions.__peerbitNativePreparedJoinCommit({
2508
+ entries,
2509
+ hashes: entryHashes,
2510
+ headFlags,
2511
+ headFlagsBytes,
2512
+ trustedMissing,
2513
+ validatePlan: nativeCommitValidatesPlan,
2514
+ })) === true;
2515
+ emitInternalProfileDuration(profile, nativeCommitStartedAt, {
2516
+ name: "log.joinPreparedFacts.nativePreparedCommit",
2517
+ component: "log",
2518
+ entries: entries.length,
2519
+ messages: 1,
2520
+ details: { nativePreparedCommitted },
2521
+ });
2522
+ }
2523
+ if (nativeCommitValidatesPlan && !nativePreparedCommitted) {
2524
+ nativeValidatedCommitRejected = true;
2525
+ return;
2526
+ }
2527
+ const blocksStartedAt = internalProfileStart(profile);
2528
+ if (!nativePreparedCommitted) {
2529
+ await this.putKnownEntryBytesBatch(entries.map((entry) => ({
2530
+ cid: entry.hash,
2531
+ bytes: entry.bytes,
2532
+ })));
2533
+ }
2534
+ emitInternalProfileDuration(profile, blocksStartedAt, {
2535
+ name: "log.joinPreparedFacts.blocks",
2536
+ component: "log",
2537
+ entries: entries.length,
2538
+ bytes: entries.reduce((sum, entry) => sum + entry.byteLength, 0),
2539
+ messages: 1,
2540
+ details: { nativePreparedCommitted },
2541
+ });
2542
+ const indexStartedAt = internalProfileStart(profile);
2543
+ let nativeCommittedFactsIndexed = false;
2544
+ if (nativePreparedCommitted &&
2545
+ resolvedOptions.__peerbitDeferIndexWrite === true) {
2546
+ const indexBatchHashes = entries.length > 1 ? batchHashes : undefined;
2547
+ const indexRows = entries.map((entry, index) => {
2548
+ const isHead = headFlags[index] ?? true;
2549
+ const externalNextHashes = indexBatchHashes
2550
+ ? entry.meta.next.filter((next) => !indexBatchHashes.has(next))
2551
+ : entry.meta.next;
2552
+ return {
2553
+ hash: entry.hash,
2554
+ unique: trustedMissing,
2555
+ externalNextHashes,
2556
+ getShallowEntry: () => {
2557
+ const shallowEntry = entry.shallowEntry ?? entry.getShallowEntry?.(isHead);
2558
+ if (!shallowEntry) {
2559
+ throw new Error("Missing prepared append shallow entry");
2560
+ }
2561
+ shallowEntry.head = isHead;
2562
+ entry.shallowEntry = shallowEntry;
2563
+ return shallowEntry;
2564
+ },
2565
+ isHead,
2566
+ };
2567
+ });
2568
+ await this.entryIndex.putNativeCommittedAppendFactsBatch(indexRows);
2569
+ nativeCommittedFactsIndexed = true;
2570
+ }
2571
+ else {
2572
+ const externalNextHashes = entries.length === 1 ? entries[0].meta.next : undefined;
2573
+ await this.entryIndex.putAppendFactsBatch(entries, {
2574
+ unique: trustedMissing,
2575
+ externalNextHashes,
2576
+ heads: headFlags,
2577
+ deferIndexWrite: resolvedOptions.__peerbitDeferIndexWrite,
2578
+ nativeGraphUpdated: nativePreparedCommitted,
2579
+ profile,
2580
+ });
2581
+ }
2582
+ emitInternalProfileDuration(profile, indexStartedAt, {
2583
+ name: "log.joinPreparedFacts.entryIndex",
2584
+ component: "log",
2585
+ entries: entries.length,
2586
+ messages: 1,
2587
+ details: { trustedMissing, nativeCommittedFactsIndexed },
2588
+ });
2589
+ if (resolvedOptions.__peerbitOnPreparedJoinCommitted) {
2590
+ const committedStartedAt = internalProfileStart(profile);
2591
+ await resolvedOptions.__peerbitOnPreparedJoinCommitted({
2592
+ entries,
2593
+ hashes: entryHashes,
2594
+ headFlags,
2595
+ nativePreparedCommitted,
2596
+ });
2597
+ emitInternalProfileDuration(profile, committedStartedAt, {
2598
+ name: "log.joinPreparedFacts.committed",
2599
+ component: "log",
2600
+ entries: entries.length,
2601
+ messages: 1,
2602
+ details: { nativePreparedCommitted },
2603
+ });
2604
+ }
2605
+ const changeStartedAt = internalProfileStart(profile);
2606
+ if (resolvedOptions.__peerbitOnAppendHashes) {
2607
+ await resolvedOptions.__peerbitOnAppendHashes(entries.map((entry) => entry.hash));
2608
+ }
2609
+ else {
2610
+ const change = {
2611
+ added: entries.map((entry, index) => {
2612
+ const materializeEntry = entry.materializeEntry;
2613
+ if (!materializeEntry) {
2614
+ throw new Error("Missing prepared append materializer");
2615
+ }
2616
+ return {
2617
+ head: headFlags[index],
2618
+ entry: materializeEntry(),
2619
+ };
2620
+ }),
2621
+ removed: [],
2622
+ };
2623
+ await this._onChange?.(change);
2624
+ }
2625
+ emitInternalProfileDuration(profile, changeStartedAt, {
2626
+ name: "log.joinPreparedFacts.change",
2627
+ component: "log",
2628
+ entries: entries.length,
2629
+ messages: 1,
2630
+ details: { hashOnly: !!resolvedOptions.__peerbitOnAppendHashes },
2631
+ });
2632
+ })().finally(() => {
2633
+ for (const entry of entries) {
2634
+ this._joining.delete(entry.hash);
2635
+ }
2636
+ });
2637
+ for (const entry of entries) {
2638
+ this._joining.set(entry.hash, batchPromise);
2639
+ }
2640
+ await batchPromise;
2641
+ if (nativeValidatedCommitRejected) {
2642
+ return false;
2643
+ }
2644
+ return true;
2645
+ }
2646
+ async tryJoinIndependentAppendBatch(entries, heads, options) {
2647
+ if (entries.length < 2 ||
2648
+ options.reset ||
2649
+ options.trim ||
2650
+ options.verifySignatures ||
2651
+ entries.some((entry) => this._joining.has(entry.hash))) {
2652
+ return false;
2653
+ }
2654
+ const profile = options.__peerbitProfile;
2655
+ const prepareStartedAt = internalProfileStart(profile);
2656
+ const batchHashes = new Set(entries.map((entry) => entry.hash));
2657
+ const headFlags = [];
2658
+ for (const entry of entries) {
2659
+ if (!entry.hash || !Entry.hasPreparedBlock(entry)) {
2660
+ return false;
2661
+ }
2662
+ entry.init(this);
2663
+ if (entry.meta.type !== EntryType.APPEND) {
2664
+ return false;
2665
+ }
2666
+ }
2667
+ emitInternalProfileDuration(profile, prepareStartedAt, {
2668
+ name: "log.joinIndependent.prepare",
2669
+ component: "log",
2670
+ entries: entries.length,
2671
+ messages: 1,
2672
+ });
2673
+ const planStartedAt = internalProfileStart(profile);
2674
+ const joinPlans = await this.entryIndex.planJoinBatch(entries, false, profile);
2675
+ emitInternalProfileDuration(profile, planStartedAt, {
2676
+ name: "log.joinIndependent.plan",
2677
+ component: "log",
2678
+ entries: entries.length,
2679
+ messages: 1,
2680
+ });
2681
+ const validatePlanStartedAt = internalProfileStart(profile);
2682
+ for (let i = 0; i < entries.length; i++) {
2683
+ const entry = entries[i];
2684
+ const joinPlan = joinPlans[i];
2685
+ if (joinPlan.skip ||
2686
+ joinPlan.coveredByCut ||
2687
+ !joinPlan.cutChecked ||
2688
+ joinPlan.missingParents.some((hash) => !batchHashes.has(hash))) {
2689
+ return false;
2690
+ }
2691
+ headFlags.push(heads.get(entry.hash) ?? true);
2692
+ }
2693
+ emitInternalProfileDuration(profile, validatePlanStartedAt, {
2694
+ name: "log.joinIndependent.validatePlan",
2695
+ component: "log",
2696
+ entries: entries.length,
2697
+ messages: 1,
2698
+ });
2699
+ if (!canAppendAlreadyValidated(options)) {
2700
+ const canAppendStartedAt = internalProfileStart(profile);
2701
+ for (const entry of entries) {
2702
+ if (this._canAppend && !(await this._canAppend(entry))) {
2703
+ return false;
2704
+ }
2705
+ }
2706
+ emitInternalProfileDuration(profile, canAppendStartedAt, {
2707
+ name: "log.joinIndependent.canAppend",
2708
+ component: "log",
2709
+ entries: entries.length,
2710
+ messages: 1,
2711
+ });
2712
+ }
2713
+ const preparedBatch = this.takePreparedIndependentAppendBatch(entries, headFlags);
2714
+ if (!preparedBatch) {
2715
+ return false;
2716
+ }
2717
+ const batchPromise = (async () => {
2718
+ const clockStartedAt = internalProfileStart(profile);
2719
+ for (const entry of entries) {
2720
+ this._hlc.update(entry.meta.clock.timestamp);
2721
+ }
2722
+ emitInternalProfileDuration(profile, clockStartedAt, {
2723
+ name: "log.joinIndependent.clock",
2724
+ component: "log",
2725
+ entries: entries.length,
2726
+ messages: 1,
2727
+ });
2728
+ const blocksStartedAt = internalProfileStart(profile);
2729
+ await this.putAppendEntryBlocks(entries, preparedBatch.blocks);
2730
+ emitInternalProfileDuration(profile, blocksStartedAt, {
2731
+ name: "log.joinIndependent.blocks",
2732
+ component: "log",
2733
+ entries: entries.length,
2734
+ bytes: entries.reduce((sum, entry) => sum + (entry.size ?? 0), 0),
2735
+ messages: 1,
2736
+ });
2737
+ const indexStartedAt = internalProfileStart(profile);
2738
+ const trustedMissing = options.__peerbitEntriesAlreadyMissing === true &&
2739
+ batchHashes.size === entries.length;
2740
+ await this.entryIndex.putAppendBatch(entries, {
2741
+ unique: trustedMissing,
2742
+ heads: headFlags,
2743
+ prepared: preparedBatch.prepared,
2744
+ deferIndexWrite: options.__peerbitDeferIndexWrite,
2745
+ profile,
2746
+ });
2747
+ emitInternalProfileDuration(profile, indexStartedAt, {
2748
+ name: "log.joinIndependent.entryIndex",
2749
+ component: "log",
2750
+ entries: entries.length,
2751
+ messages: 1,
2752
+ details: { trustedMissing },
2753
+ });
2754
+ const changeStartedAt = internalProfileStart(profile);
2755
+ if (options.__peerbitOnAppendHashes && !options.onChange) {
2756
+ await options.__peerbitOnAppendHashes(entries.map((entry) => entry.hash));
2757
+ }
2758
+ else {
2759
+ const change = {
2760
+ added: entries.map((entry, index) => ({
2761
+ head: headFlags[index],
2762
+ entry,
2763
+ })),
2764
+ removed: [],
2765
+ };
2766
+ await options.onChange?.(change);
2767
+ await this._onChange?.(change);
2768
+ }
2769
+ emitInternalProfileDuration(profile, changeStartedAt, {
2770
+ name: "log.joinIndependent.change",
2771
+ component: "log",
2772
+ entries: entries.length,
2773
+ messages: 1,
2774
+ details: {
2775
+ hashOnly: !!options.__peerbitOnAppendHashes && !options.onChange,
2776
+ },
2777
+ });
2778
+ })().finally(() => {
2779
+ for (const entry of entries) {
2780
+ this._joining.delete(entry.hash);
2781
+ }
2782
+ });
2783
+ for (const entry of entries) {
2784
+ this._joining.set(entry.hash, batchPromise);
2785
+ }
2786
+ await batchPromise;
2787
+ return true;
2788
+ }
2789
+ takePreparedIndependentAppendBatch(entries, headFlags) {
2790
+ if (!entries.every((entry) => Entry.hasPreparedBlock(entry))) {
2791
+ return;
2792
+ }
2793
+ const hasPreparedShallowEntries = entries.every((entry) => Entry.hasPreparedShallowEntry(entry));
2794
+ const hasPreparedNativeEntries = hasPreparedShallowEntries &&
2795
+ entries.every((entry) => Entry.hasPreparedNativeLogEntry(entry));
2796
+ const blocks = entries.map((entry) => {
2797
+ const prepared = Entry.takePreparedBlock(entry);
2798
+ if (!prepared) {
2799
+ throw new Error("Missing prepared entry block");
2800
+ }
2801
+ return prepared;
2802
+ });
2803
+ if (!hasPreparedShallowEntries) {
2804
+ return { blocks };
2805
+ }
2806
+ const shallowEntries = entries.map((entry, index) => {
2807
+ const shallowEntry = Entry.takePreparedShallowEntry(entry, headFlags[index] ?? true);
2808
+ if (!shallowEntry) {
2809
+ throw new Error("Missing prepared shallow entry");
2810
+ }
2811
+ return shallowEntry;
2812
+ });
2813
+ const nativeEntries = hasPreparedNativeEntries
2814
+ ? entries.map((entry, index) => {
2815
+ const nativeEntry = Entry.takePreparedNativeLogEntry(entry, headFlags[index] ?? true);
2816
+ if (!nativeEntry) {
2817
+ throw new Error("Missing prepared native log entry");
2818
+ }
2819
+ return nativeEntry;
2820
+ })
2821
+ : undefined;
2822
+ return {
2823
+ blocks,
2824
+ prepared: {
2825
+ shallowEntries,
2826
+ nativeEntries,
2827
+ },
2828
+ };
2829
+ }
683
2830
  /**
684
2831
  * Bottom up join of entries into the log
685
2832
  * @param entry
@@ -693,8 +2840,8 @@ let Log = (() => {
693
2840
  if (!entry.hash) {
694
2841
  throw new Error("Unexpected");
695
2842
  }
696
- if ((await this.entryIndex.getShallow(entry.hash)) != null &&
697
- !options.reset) {
2843
+ const joinPlan = await this.entryIndex.planJoin(entry, options.reset);
2844
+ if (joinPlan.skip) {
698
2845
  return false;
699
2846
  }
700
2847
  entry.init(this);
@@ -703,10 +2850,11 @@ let Log = (() => {
703
2850
  throw new Error(`Invalid signature entry with hash "${entry.hash}"`);
704
2851
  }
705
2852
  }
706
- const headsWithGid = await this.entryIndex
707
- .getHeads(entry.meta.gid, { type: "shape", shape: ENTRY_JOIN_SHAPE })
708
- .all();
709
- if (headsWithGid) {
2853
+ if (joinPlan.coveredByCut) {
2854
+ return false;
2855
+ }
2856
+ if (!joinPlan.cutChecked) {
2857
+ const headsWithGid = await this.entryIndex.getJoinHeads(entry.meta.gid);
710
2858
  for (const v of headsWithGid) {
711
2859
  // TODO second argument should be a time compare instead? what about next nexts?
712
2860
  // and check the cut entry is newer than the current 'entry'
@@ -721,27 +2869,51 @@ let Log = (() => {
721
2869
  const remote = options.remote && typeof options.remote === "object"
722
2870
  ? options.remote
723
2871
  : undefined;
724
- for (const a of entry.meta.next) {
2872
+ const parents = [];
2873
+ const unresolvedParentHashes = [];
2874
+ for (const a of joinPlan.missingParents) {
725
2875
  const prev = this._joining.get(a);
726
2876
  if (prev) {
727
2877
  await prev;
728
2878
  continue;
729
2879
  }
730
- if ((await this.entryIndex.getShallow(a)) != null && !options.reset) {
2880
+ const referenced = options.references?.get(a);
2881
+ parents.push({ hash: a, entry: referenced });
2882
+ if (!referenced) {
2883
+ unresolvedParentHashes.push(a);
2884
+ }
2885
+ }
2886
+ const localParents = unresolvedParentHashes.length > 0
2887
+ ? await this.entryIndex.getMany(unresolvedParentHashes, {
2888
+ type: "full",
2889
+ ignoreMissing: true,
2890
+ })
2891
+ : [];
2892
+ const localParentByHash = new Map();
2893
+ for (const parent of localParents) {
2894
+ if (parent) {
2895
+ localParentByHash.set(parent.hash, parent);
2896
+ }
2897
+ }
2898
+ for (const parent of parents) {
2899
+ const a = parent.hash;
2900
+ const prev = this._joining.get(a);
2901
+ if (prev) {
2902
+ await prev;
731
2903
  continue;
732
2904
  }
733
- const from = await options.resolveRemoteFrom?.(a, remote?.signal);
734
- let nested;
2905
+ let nested = parent.entry ?? localParentByHash.get(a);
735
2906
  try {
736
- nested =
737
- options.references?.get(a) ??
738
- (await Entry.fromMultihash(this._storage, a, {
739
- remote: {
740
- timeout: remote?.timeout,
741
- signal: remote?.signal,
742
- ...(from && from.length > 0 ? { from } : {}),
743
- },
744
- }));
2907
+ if (!nested) {
2908
+ const from = await options.resolveRemoteFrom?.(a, remote?.signal);
2909
+ nested = await Entry.fromMultihash(this._storage, a, {
2910
+ remote: {
2911
+ timeout: remote?.timeout,
2912
+ signal: remote?.signal,
2913
+ ...(from && from.length > 0 ? { from } : {}),
2914
+ },
2915
+ });
2916
+ }
745
2917
  }
746
2918
  catch (error) {
747
2919
  if (isRecoverableJoinResolveError(error)) {
@@ -768,7 +2940,7 @@ let Log = (() => {
768
2940
  toMultiHash: true,
769
2941
  });
770
2942
  const pendingDeletes = await this.processEntry(entry);
771
- const trimmed = await this.trim(options.trim);
2943
+ const trimmed = await this.trimIfConfigured(options.trim);
772
2944
  if (trimmed) {
773
2945
  for (const removedEntry of trimmed) {
774
2946
  pendingDeletes.push({ entry: removedEntry, fn: undefined });
@@ -805,7 +2977,18 @@ let Log = (() => {
805
2977
  }
806
2978
  /// TODO simplify methods below
807
2979
  async prepareDeleteRecursively(from, skipFirst = false) {
808
- const stack = Array.isArray(from) ? [...from] : [from];
2980
+ const entries = Array.isArray(from) ? [...from] : [from];
2981
+ const nativeDeletePlan = this.entryIndex.planDeleteRecursively(entries, skipFirst);
2982
+ if (nativeDeletePlan) {
2983
+ const toDelete = [];
2984
+ for (const hash of nativeDeletePlan) {
2985
+ const deleteFn = await this.prepareDelete(hash);
2986
+ deleteFn.entry &&
2987
+ toDelete.push({ entry: deleteFn.entry, fn: deleteFn.fn });
2988
+ }
2989
+ return toDelete;
2990
+ }
2991
+ const stack = entries;
809
2992
  const promises = [];
810
2993
  let counter = 0;
811
2994
  const toDelete = [];
@@ -818,8 +3001,7 @@ let Log = (() => {
818
3001
  toDelete.push({ entry: deleteFn.entry, fn: deleteFn.fn });
819
3002
  }
820
3003
  for (const next of entry.meta.next) {
821
- const nextFromNext = this.entryIndex.getHasNext(next);
822
- const entriesThatHasNext = await nextFromNext.all();
3004
+ const entriesThatHasNext = await this.entryIndex.getJoinChildren(next);
823
3005
  // if there are no entries which is not of "CUT" type, we can safely delete the next entry
824
3006
  // figureately speaking, these means where are cutting all branches to a stem, so we can delete the stem as well
825
3007
  let hasAlternativeNext = !!entriesThatHasNext.find((x) => x.meta.type !== EntryType.CUT && x.hash !== entry.hash);
@@ -878,8 +3060,8 @@ let Log = (() => {
878
3060
  async close() {
879
3061
  // Don't return early here if closed = true, because "load" might create processes that needs to be closed
880
3062
  this._closed = true; // closed = true before doing below, else we might try to open the headsIndex cache because it is closed as we assume log is still open
881
- this._closeController.abort();
882
- await this._entryIndex.flushPendingWrites();
3063
+ this._closeController?.abort();
3064
+ await this._entryIndex?.flushPendingWrites();
883
3065
  await this._indexer?.stop?.();
884
3066
  this._indexer = undefined;
885
3067
  this._loadedOnce = false;
@@ -888,7 +3070,7 @@ let Log = (() => {
888
3070
  // Don't return early here if closed = true, because "load" might create processes that needs to be closed
889
3071
  this._closed = true; // closed = true before doing below, else we might try to open the headsIndex cache because it is closed as we assume log is still open
890
3072
  this._closeController.abort();
891
- await this.entryIndex?.clear();
3073
+ await this._entryIndex?.clear();
892
3074
  await this._indexer?.drop();
893
3075
  await this._indexer?.stop?.();
894
3076
  }