@peerbit/log 6.1.0 → 6.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/dist/benchmark/native-graph.js +447 -5
  2. package/dist/benchmark/native-graph.js.map +1 -1
  3. package/dist/src/clock.d.ts +1 -0
  4. package/dist/src/clock.d.ts.map +1 -1
  5. package/dist/src/clock.js +32 -0
  6. package/dist/src/clock.js.map +1 -1
  7. package/dist/src/entry-create.d.ts +1 -0
  8. package/dist/src/entry-create.d.ts.map +1 -1
  9. package/dist/src/entry-create.js.map +1 -1
  10. package/dist/src/entry-index.d.ts +288 -12
  11. package/dist/src/entry-index.d.ts.map +1 -1
  12. package/dist/src/entry-index.js +1478 -33
  13. package/dist/src/entry-index.js.map +1 -1
  14. package/dist/src/entry-v0.d.ts +184 -1
  15. package/dist/src/entry-v0.d.ts.map +1 -1
  16. package/dist/src/entry-v0.js +1021 -12
  17. package/dist/src/entry-v0.js.map +1 -1
  18. package/dist/src/entry.d.ts +73 -1
  19. package/dist/src/entry.d.ts.map +1 -1
  20. package/dist/src/entry.js +124 -1
  21. package/dist/src/entry.js.map +1 -1
  22. package/dist/src/index.d.ts +1 -1
  23. package/dist/src/index.d.ts.map +1 -1
  24. package/dist/src/index.js +1 -1
  25. package/dist/src/index.js.map +1 -1
  26. package/dist/src/log.d.ts +62 -13
  27. package/dist/src/log.d.ts.map +1 -1
  28. package/dist/src/log.js +2282 -100
  29. package/dist/src/log.js.map +1 -1
  30. package/dist/src/runtime.d.ts +2 -0
  31. package/dist/src/runtime.d.ts.map +1 -0
  32. package/dist/src/runtime.js +10 -0
  33. package/dist/src/runtime.js.map +1 -0
  34. package/dist/src/trim.d.ts +14 -2
  35. package/dist/src/trim.d.ts.map +1 -1
  36. package/dist/src/trim.js +92 -4
  37. package/dist/src/trim.js.map +1 -1
  38. package/package.json +18 -15
  39. package/src/clock.ts +34 -0
  40. package/src/entry-create.ts +1 -0
  41. package/src/entry-index.ts +2232 -68
  42. package/src/entry-v0.ts +1488 -13
  43. package/src/entry.ts +235 -1
  44. package/src/index.ts +7 -1
  45. package/src/log.ts +3246 -206
  46. package/src/runtime.ts +16 -0
  47. package/src/trim.ts +133 -7
package/src/log.ts CHANGED
@@ -6,6 +6,7 @@ import {
6
6
  cidifyString,
7
7
  } from "@peerbit/blocks-interface";
8
8
  import {
9
+ Ed25519Keypair,
9
10
  type Identity,
10
11
  SignatureWithKey,
11
12
  X25519Keypair,
@@ -26,21 +27,206 @@ import {
26
27
  EntryIndex,
27
28
  type MaybeResolveOptions,
28
29
  type NativeLogGraph,
30
+ type PreparedAppendIndexFacts,
29
31
  type ResultsIterator,
30
32
  type ReturnTypeFromResolveOptions,
31
33
  } from "./entry-index.js";
32
- import { ShallowEntry } from "./entry-shallow.js";
34
+ import { ShallowEntry, ShallowMeta } from "./entry-shallow.js";
33
35
  import { EntryType } from "./entry-type.js";
34
36
  import { type EncryptionTemplateMaybeEncrypted, EntryV0 } from "./entry-v0.js";
35
37
  import { type EntryWithRefs } from "./entry-with-refs.js";
36
- import { type CanAppend, Entry, type ShallowOrFullEntry } from "./entry.js";
38
+ import {
39
+ type CanAppend,
40
+ Entry,
41
+ type PreparedAppendChain,
42
+ type PreparedAppendCommitOnlyChain,
43
+ type PreparedAppendFacts,
44
+ type PreparedEntryBlock,
45
+ type PreparedNativeLogEntry,
46
+ type ShallowOrFullEntry,
47
+ } from "./entry.js";
37
48
  import { findUniques } from "./find-uniques.js";
49
+ import { logger as baseLogger } from "./logger.js";
38
50
  import * as LogError from "./log-errors.js";
39
51
  import * as Sorting from "./log-sorting.js";
40
52
  import type { Payload } from "./payload.js";
53
+ import { canUseOptionalNativeModuleImports } from "./runtime.js";
41
54
  import { Trim, type TrimOptions } from "./trim.js";
42
55
 
43
56
  const { LastWriteWins } = Sorting;
57
+ const warn = baseLogger.newScope("warn");
58
+
59
+ type BlocksWithPutMany = Blocks & {
60
+ putMany?: (blocks: PreparedEntryBlock[]) => Promise<string[]> | string[];
61
+ rmMany?: (cids: string[]) => Promise<number | void> | number | void;
62
+ };
63
+
64
+ type BlocksWithPutKnownMany = Blocks & {
65
+ putKnownMany: (
66
+ blocks: Array<readonly [cid: string, bytes: Uint8Array]>,
67
+ ) => Promise<string[]> | string[];
68
+ };
69
+
70
+ type BlocksWithPutKnownManyColumns = Blocks & {
71
+ putKnownManyColumns: (
72
+ cids: string[],
73
+ bytes: Uint8Array[],
74
+ ) => Promise<string[]> | string[];
75
+ };
76
+
77
+ type BlocksWithPutKnown = Blocks & {
78
+ putKnown: (cid: string, bytes: Uint8Array) => Promise<string> | string;
79
+ };
80
+
81
+ const hasPutMany = (storage: Blocks): storage is BlocksWithPutMany =>
82
+ typeof (storage as BlocksWithPutMany).putMany === "function";
83
+
84
+ const hasPutKnown = (storage: Blocks): storage is BlocksWithPutKnown =>
85
+ typeof (storage as BlocksWithPutKnown).putKnown === "function";
86
+
87
+ const hasPutKnownMany = (storage: Blocks): storage is BlocksWithPutKnownMany =>
88
+ typeof (storage as BlocksWithPutKnownMany).putKnownMany === "function";
89
+
90
+ const hasPutKnownManyColumns = (
91
+ storage: Blocks,
92
+ ): storage is BlocksWithPutKnownManyColumns =>
93
+ typeof (storage as BlocksWithPutKnownManyColumns).putKnownManyColumns ===
94
+ "function";
95
+
96
+ type MaybePromise<T> = T | Promise<T>;
97
+
98
+ type InternalProfileValue = string | number | boolean | undefined;
99
+ type InternalProfileEvent = {
100
+ name: string;
101
+ component?: string;
102
+ durationMs?: number;
103
+ entries?: number;
104
+ bytes?: number;
105
+ messages?: number;
106
+ count?: number;
107
+ details?: Record<string, InternalProfileValue>;
108
+ };
109
+ type InternalProfileSink = (event: InternalProfileEvent) => void;
110
+ type InternalAppendHashesSink = (hashes: string[]) => void | Promise<void>;
111
+
112
+ const internalProfileNow = () => globalThis.performance?.now?.() ?? Date.now();
113
+ const internalProfileStart = (sink: InternalProfileSink | undefined) =>
114
+ sink ? internalProfileNow() : 0;
115
+ const emitInternalProfileDuration = (
116
+ sink: InternalProfileSink | undefined,
117
+ startedAt: number,
118
+ event: Omit<InternalProfileEvent, "durationMs">,
119
+ ) => {
120
+ if (!sink) {
121
+ return;
122
+ }
123
+ sink({
124
+ ...event,
125
+ durationMs: internalProfileNow() - startedAt,
126
+ });
127
+ };
128
+ const EMPTY_NEXT_HASHES: string[] = [];
129
+ const EMPTY_NEXT_ENTRIES: Sorting.SortableEntry[] = [];
130
+ const normalizedUniqueStrings = (values: string[]): string[] =>
131
+ values.length <= 1 ? values : [...new Set(values)];
132
+
133
+ type PreparedCommitOnlyAppendResult<T> = {
134
+ entry: Entry<T>;
135
+ materializeEntry: () => Entry<T>;
136
+ removed: ShallowOrFullEntry<T>[];
137
+ removedHashes?: string[];
138
+ appendFacts: PreparedAppendFacts;
139
+ shallowEntry: ShallowEntry;
140
+ documentTrimmedHeadsProcessed?: boolean;
141
+ documentPreviousContext?: {
142
+ created: bigint;
143
+ modified: bigint;
144
+ head: string;
145
+ gid: string;
146
+ size: number;
147
+ };
148
+ };
149
+
150
+ type PreparedCommitOnlyAppendBatchResult<T> = {
151
+ entries: Entry<T>[];
152
+ materializeEntries: Array<() => Entry<T>>;
153
+ removed: ShallowOrFullEntry<T>[];
154
+ removedHashes?: string[];
155
+ appendFacts: PreparedAppendFacts[];
156
+ documentTrimmedHeadsProcessed?: boolean[];
157
+ };
158
+
159
+ type PreparedIndependentAppendBatch = {
160
+ blocks: PreparedEntryBlock[];
161
+ prepared?: {
162
+ shallowEntries: ShallowEntry[];
163
+ nativeEntries?: PreparedNativeLogEntry[];
164
+ };
165
+ };
166
+
167
+ type PreparedJoinNativeCommitInput = {
168
+ entries: PreparedAppendJoinFacts[];
169
+ hashes: string[];
170
+ headFlags: boolean[];
171
+ headFlagsBytes: Uint8Array;
172
+ trustedMissing: boolean;
173
+ validatePlan?: boolean;
174
+ };
175
+
176
+ type PreparedJoinCommittedInput = {
177
+ entries: PreparedAppendJoinFacts[];
178
+ hashes: string[];
179
+ headFlags: boolean[];
180
+ nativePreparedCommitted: boolean;
181
+ };
182
+
183
+ export type PreparedAppendJoinFacts = PreparedAppendIndexFacts & {
184
+ bytes: Uint8Array;
185
+ byteLength: number;
186
+ materializeEntry?: () => Entry<any>;
187
+ };
188
+
189
+ type NativePreparedNoNextCommit = {
190
+ bytes?: Uint8Array;
191
+ getBytes?: (hash: string) => Uint8Array | undefined;
192
+ cid?: string;
193
+ hash?: string;
194
+ gid?: string;
195
+ next?: string[];
196
+ byteLength: number;
197
+ metaBytes?: Uint8Array;
198
+ hashDigestBytes?: Uint8Array;
199
+ trimmedEntries?: PreparedNativeLogEntry[];
200
+ trimmedEntryHashes?: string[];
201
+ documentTrimmedHeadsProcessed?: boolean;
202
+ documentPreviousContext?: PreparedCommitOnlyAppendResult<unknown>["documentPreviousContext"];
203
+ };
204
+
205
+ type NativeNoNextCommitInput = {
206
+ clockId: Uint8Array;
207
+ privateKey: Uint8Array;
208
+ publicKey: Uint8Array;
209
+ wallTime: bigint;
210
+ logical: number;
211
+ gid: string;
212
+ type: EntryType;
213
+ metaData?: Uint8Array;
214
+ payloadData: Uint8Array;
215
+ resolveTrimmedEntries?: boolean;
216
+ trimLengthTo?: number;
217
+ };
218
+
219
+ type NativeCommitInput = NativeNoNextCommitInput & {
220
+ next: string[];
221
+ };
222
+
223
+ const isPromiseLike = <T>(value: Promise<T> | T): value is Promise<T> =>
224
+ typeof (value as { then?: unknown })?.then === "function";
225
+
226
+ const mapMaybePromise = <T, R>(
227
+ value: MaybePromise<T>,
228
+ fn: (value: T) => MaybePromise<R>,
229
+ ): MaybePromise<R> => (isPromiseLike(value) ? value.then(fn) : fn(value));
44
230
 
45
231
  const getErrorName = (error: unknown) =>
46
232
  typeof (error as { name?: unknown })?.name === "string"
@@ -89,12 +275,18 @@ export type MemoryProperties = {
89
275
  indexer?: Indices;
90
276
  };
91
277
 
278
+ export type NativeGraphOptions = {
279
+ heads?: boolean;
280
+ optional?: boolean;
281
+ graph?: NativeLogGraph;
282
+ };
283
+
92
284
  export type LogProperties<T> = {
93
285
  keychain?: CryptoKeychain;
94
286
  encoding?: Encoding<T>;
95
287
  clock?: LamportClock;
96
288
  appendDurability?: AppendDurability;
97
- nativeGraph?: boolean | { heads?: boolean };
289
+ nativeGraph?: boolean | NativeGraphOptions;
98
290
  sortFn?: Sorting.SortFn;
99
291
  trim?: TrimOptions;
100
292
  canAppend?: CanAppend<T>;
@@ -133,11 +325,63 @@ export type AppendOptions<T> = {
133
325
  canAppend?: CanAppend<T>;
134
326
  };
135
327
 
328
+ type TrustedAppendOptions<T> = AppendOptions<T> & {
329
+ __peerbitCanAppendAlreadyValidated?: boolean;
330
+ };
331
+
332
+ const canAppendAlreadyValidated = (options?: unknown): boolean =>
333
+ (options as { __peerbitCanAppendAlreadyValidated?: unknown } | undefined)
334
+ ?.__peerbitCanAppendAlreadyValidated === true;
335
+
336
+ const withCanAppendAlreadyValidated = <T>(
337
+ options: AppendOptions<T> = {},
338
+ ): TrustedAppendOptions<T> =>
339
+ canAppendAlreadyValidated(options)
340
+ ? (options as TrustedAppendOptions<T>)
341
+ : {
342
+ ...options,
343
+ __peerbitCanAppendAlreadyValidated: true,
344
+ };
345
+
346
+ export type { PreparedAppendFacts } from "./entry.js";
347
+
136
348
  type OnChange<T> = (
137
349
  change: Change<T>,
138
350
  reference?: undefined,
139
351
  ) => void | Promise<void>;
140
352
 
353
+ export type JoinOptions<T> = {
354
+ verifySignatures?: boolean;
355
+ trim?: TrimOptions;
356
+ timeout?: number;
357
+ onChange?: OnChange<T>;
358
+ reset?: boolean;
359
+ };
360
+
361
+ type TrustedJoinOptions<T> = JoinOptions<T> & {
362
+ __peerbitBatchIndependent?: boolean;
363
+ __peerbitEntriesAlreadyMissing?: boolean;
364
+ __peerbitCanAppendAlreadyValidated?: boolean;
365
+ __peerbitOnAppendHashes?: InternalAppendHashesSink;
366
+ __peerbitDeferIndexWrite?: boolean;
367
+ __peerbitProfile?: InternalProfileSink;
368
+ };
369
+
370
+ type TrustedPreparedAppendFactsBatchJoinOptions = {
371
+ __peerbitEntriesAlreadyMissing?: boolean;
372
+ __peerbitCanAppendAlreadyValidated?: boolean;
373
+ __peerbitOnAppendHashes?: InternalAppendHashesSink;
374
+ __peerbitDeferIndexWrite?: boolean;
375
+ __peerbitProfile?: InternalProfileSink;
376
+ __peerbitNativePreparedJoinCommit?: (
377
+ input: PreparedJoinNativeCommitInput,
378
+ ) => MaybePromise<boolean>;
379
+ __peerbitNativePreparedJoinCommitValidatesPlan?: boolean;
380
+ __peerbitOnPreparedJoinCommitted?: (
381
+ input: PreparedJoinCommittedInput,
382
+ ) => MaybePromise<void>;
383
+ };
384
+
141
385
  export type JoinableEntry = {
142
386
  meta: {
143
387
  clock: {
@@ -160,6 +404,11 @@ type PendingDelete<T> = {
160
404
  fn: () => Promise<ShallowEntry | undefined>;
161
405
  };
162
406
 
407
+ type EntryWithMetaBytes = {
408
+ getMetaBytes?: () => Uint8Array | undefined;
409
+ getHashDigestBytes?: () => Uint8Array | undefined;
410
+ };
411
+
163
412
  @variant(0)
164
413
  export class Log<T> {
165
414
  @field({ type: fixedArray("u8", 32) })
@@ -191,9 +440,11 @@ export class Log<T> {
191
440
  private _appendDurability!: AppendDurability;
192
441
  private _joining!: Map<string, Promise<any>>; // entry hashes that are currently joining into this log
193
442
  private _sortFn!: Sorting.SortFn;
443
+ private _hasCustomCanAppend = false;
194
444
 
195
445
  constructor(properties?: { id?: Uint8Array }) {
196
446
  this._id = properties?.id || randomBytes(32);
447
+ this._closeController = new AbortController();
197
448
  }
198
449
 
199
450
  async open(store: Blocks, identity: Identity, options: LogOptions<T> = {}) {
@@ -245,29 +496,47 @@ export class Log<T> {
245
496
  throw new Error("Id not set");
246
497
  }
247
498
 
499
+ const nativeGraphOption = options.nativeGraph;
248
500
  const nativeGraph =
249
- options.nativeGraph &&
501
+ nativeGraphOption &&
250
502
  (await (async () => {
503
+ const nativeGraphOptions =
504
+ typeof nativeGraphOption === "object" ? nativeGraphOption : undefined;
505
+ if (nativeGraphOptions?.graph) {
506
+ const headsRequested = nativeGraphOptions.heads !== false;
507
+ return {
508
+ graph: nativeGraphOptions.graph,
509
+ useHeads: headsRequested && this._sortFn === LastWriteWins,
510
+ };
511
+ }
512
+ if (!canUseOptionalNativeModuleImports()) {
513
+ if (nativeGraphOptions?.optional === true) {
514
+ return undefined;
515
+ }
516
+ throw new Error(
517
+ "Log nativeGraph is unavailable in service worker contexts",
518
+ );
519
+ }
251
520
  let createLogGraphIndex: () => Promise<NativeLogGraph>;
252
521
  try {
253
522
  ({ createLogGraphIndex } = (await import(
254
- ["@peerbit", "log-rust"].join("/")
523
+ /* @vite-ignore */ ["@peerbit", "log-rust"].join("/")
255
524
  )) as {
256
525
  createLogGraphIndex: () => Promise<NativeLogGraph>;
257
526
  });
527
+ const headsRequested = nativeGraphOptions?.heads !== false;
528
+ return {
529
+ graph: await createLogGraphIndex(),
530
+ useHeads: headsRequested && this._sortFn === LastWriteWins,
531
+ };
258
532
  } catch {
533
+ if (nativeGraphOptions?.optional === true) {
534
+ return undefined;
535
+ }
259
536
  throw new Error(
260
537
  "Log nativeGraph requires @peerbit/log-rust to be installed and built",
261
538
  );
262
539
  }
263
- const headsRequested =
264
- typeof options.nativeGraph === "object"
265
- ? options.nativeGraph.heads !== false
266
- : true;
267
- return {
268
- graph: await createLogGraphIndex(),
269
- useHeads: headsRequested && this._sortFn === LastWriteWins,
270
- };
271
540
  })());
272
541
 
273
542
  this._entryIndex = new EntryIndex({
@@ -293,12 +562,58 @@ export class Log<T> {
293
562
  this._trim = new Trim(
294
563
  {
295
564
  index: this._entryIndex,
296
- deleteNode: async (node: ShallowEntry) => {
297
- const resolved = await this.get(node.hash);
298
- await this._entryIndex.delete(node.hash);
565
+ deleteNode: async (
566
+ node: ShallowEntry,
567
+ options?: { resolveDeletedEntry?: boolean },
568
+ ) => {
569
+ const shouldResolve = options?.resolveDeletedEntry !== false;
570
+ const resolved = shouldResolve
571
+ ? await this.get(node.hash)
572
+ : undefined;
573
+ const deleted = await this._entryIndex.delete(node.hash, node);
299
574
  await this._storage.rm(node.hash);
300
- return resolved;
575
+ if (!deleted) {
576
+ return resolved;
577
+ }
578
+ return shouldResolve ? resolved : deleted;
301
579
  },
580
+ deleteNodes: this._entryIndex.canDeleteMany()
581
+ ? (
582
+ nodes: ShallowEntry[],
583
+ options?: {
584
+ resolveDeletedEntry?: boolean;
585
+ skipNextHeadUpdates?: boolean;
586
+ },
587
+ ): MaybePromise<(Entry<T> | ShallowEntry)[]> => {
588
+ if (nodes.length === 0) {
589
+ return [];
590
+ }
591
+ const shouldResolve = options?.resolveDeletedEntry !== false;
592
+ if (!shouldResolve) {
593
+ return this._entryIndex.deleteManyMaybe(nodes, {
594
+ skipNextHeadUpdates: options?.skipNextHeadUpdates,
595
+ });
596
+ }
597
+ return (async () => {
598
+ const resolvedByHash = new Map<string, Entry<T>>();
599
+ const resolved = await this._entryIndex.getMany(
600
+ nodes.map((node) => node.hash),
601
+ { type: "full", ignoreMissing: true },
602
+ );
603
+ for (const entry of resolved) {
604
+ if (entry) {
605
+ resolvedByHash.set(entry.hash, entry);
606
+ }
607
+ }
608
+ const deleted = await this._entryIndex.deleteMany(nodes, {
609
+ skipNextHeadUpdates: options?.skipNextHeadUpdates,
610
+ });
611
+ return deleted
612
+ .map((node) => resolvedByHash.get(node.hash))
613
+ .filter((entry): entry is Entry<T> => !!entry);
614
+ })();
615
+ }
616
+ : undefined,
302
617
  sortFn: this._sortFn,
303
618
  getLength: () => this.length,
304
619
  },
@@ -313,6 +628,7 @@ export class Log<T> {
313
628
  }
314
629
  return true;
315
630
  };
631
+ this._hasCustomCanAppend = !!options?.canAppend;
316
632
 
317
633
  this._onChange = options?.onChange;
318
634
  this._closed = false;
@@ -366,6 +682,11 @@ export class Log<T> {
366
682
  has(cid: string) {
367
683
  return this._entryIndex.has(cid);
368
684
  }
685
+
686
+ hasMany(cids: Iterable<string>) {
687
+ return this._entryIndex.hasMany(cids);
688
+ }
689
+
369
690
  /**
370
691
  * Get all entries sorted. Don't use this method anywhere where performance matters
371
692
  */
@@ -546,98 +867,53 @@ export class Log<T> {
546
867
  data: T,
547
868
  options: AppendOptions<T> = {},
548
869
  ): Promise<{ entry: Entry<T>; removed: ShallowOrFullEntry<T>[] }> {
549
- // Update the clock (find the latest clock)
550
- if (options.meta?.next) {
551
- for (const n of options.meta.next) {
552
- if (!n.hash)
553
- throw new Error(
554
- "Expecting nexts to already be saved. missing hash for one or more entries",
555
- );
556
- }
557
- }
558
-
559
- /* await this.load({ reload: false }); */
560
-
561
- const nexts: Sorting.SortableEntry[] =
562
- options.meta?.next ||
563
- (await this.entryIndex
564
- .getHeads(undefined, { type: "shape", shape: Sorting.ENTRY_SORT_SHAPE })
565
- .all());
566
-
567
- // Calculate max time for log/graph
568
- const clock = new Clock({
569
- id: this._identity.publicKey.bytes,
570
- timestamp: options?.meta?.timestamp || this._hlc.now(),
571
- });
572
-
573
- const entry = await EntryV0.create<T>({
574
- store: this._storage,
575
- identity: options.identity || this._identity,
576
- signers: options.signers,
577
- data,
578
- meta: {
579
- clock,
580
- type: options.meta?.type,
581
- gidSeed: options.meta?.gidSeed,
582
- data: options.meta?.data,
583
- next: nexts,
584
- },
585
-
586
- encoding: this._encoding,
587
- encryption: options.encryption
588
- ? {
589
- keypair: options.encryption.keypair,
590
- receiver: {
591
- ...options.encryption.receiver,
592
- },
593
- }
594
- : undefined,
595
- canAppend: options.canAppend || this._canAppend,
596
- });
597
-
598
- if (!entry.hash) {
599
- throw new Error("Unexpected");
600
- }
601
-
602
- if (entry.meta.type !== EntryType.CUT) {
603
- for (const e of nexts) {
604
- if (!(await this.has(e.hash))) {
605
- let entry: Entry<any>;
606
- if (e instanceof Entry) {
607
- entry = e;
608
- } else {
609
- let resolved = await this.entryIndex.get(e.hash);
610
- if (!resolved) {
611
- // eslint-disable-next-line no-console
612
- console.warn("Unexpected missing entry when joining", e.hash);
613
- continue;
614
- }
615
- entry = resolved;
616
- }
617
- await this.join([entry]);
870
+ const nexts = await this.getNextsForAppend(options);
871
+ const deferBlockStore = hasPutMany(this._storage);
872
+ const nativeAppendChain = this.entryIndex.properties.nativeGraph
873
+ ? await this.createNativePlainAppendChain(
874
+ [data],
875
+ options,
876
+ nexts,
877
+ deferBlockStore,
878
+ )
879
+ : undefined;
880
+ let entry: Entry<T>;
881
+ if (nativeAppendChain) {
882
+ entry = nativeAppendChain.entries[0]!;
883
+ try {
884
+ await this.joinMissingNexts(entry, nexts);
885
+ if (deferBlockStore && !nativeAppendChain.nativeBlocksCommitted) {
886
+ await this.putAppendEntryBlocks([entry], nativeAppendChain.blocks);
887
+ }
888
+ await this.putAppendEntries(
889
+ [entry],
890
+ options,
891
+ nexts.map((next) => next.hash),
892
+ nativeAppendChain,
893
+ );
894
+ } catch (error) {
895
+ if (nativeAppendChain.nativeGraphUpdated) {
896
+ this.rollbackNativeAppendGraph([entry]);
618
897
  }
898
+ if (nativeAppendChain.nativeBlocksCommitted) {
899
+ await this.rollbackNativeAppendBlocks([entry]);
900
+ }
901
+ throw error;
619
902
  }
903
+ } else {
904
+ entry = await this.createAppendEntry(data, options, nexts);
905
+ await this.joinMissingNexts(entry, nexts);
906
+ await this.putAppendEntry(entry, options);
620
907
  }
621
908
 
622
- await this.entryIndex.put(entry, {
623
- unique: true,
624
- isHead: true,
625
- toMultiHash: false,
626
- deferIndexWrite:
627
- options.deferIndexWrite ??
628
- (options.durability
629
- ? options.durability === "buffered"
630
- : this._appendDurability === "buffered"),
631
- });
632
-
633
909
  const pendingDeletes: (
634
910
  | PendingDelete<T>
635
- | { entry: Entry<T>; fn: undefined }
911
+ | { entry: ShallowOrFullEntry<T>; fn: undefined }
636
912
  )[] = await this.processEntry(entry);
637
913
 
638
914
  entry.init({ encoding: this._encoding, keychain: this._keychain });
639
915
 
640
- const trimmed = await this.trim(options?.trim);
916
+ const trimmed = await this.trimIfConfigured(options?.trim);
641
917
 
642
918
  if (trimmed) {
643
919
  for (const entry of trimmed) {
@@ -655,113 +931,2294 @@ export class Log<T> {
655
931
  return { entry, removed };
656
932
  }
657
933
 
658
- async remove(
659
- entry:
660
- | { hash: string; meta: { next: string[] } }
661
- | { hash: string; meta: { next: string[] } }[],
662
- options?: { recursively?: boolean },
663
- ): Promise<Change<T>> {
664
- /* await this.load({ reload: false }); */
665
- const entries = Array.isArray(entry) ? entry : [entry];
666
-
667
- if (entries.length === 0) {
668
- return {
669
- added: [],
670
- removed: [],
671
- };
934
+ // Internal trusted local append path for callers that already handled validation
935
+ // and want to apply change observers themselves.
936
+ private async appendLocallyPrepared(
937
+ data: T,
938
+ options: AppendOptions<T> = {},
939
+ properties?: {
940
+ skipMissingNextJoin?: boolean;
941
+ resolveTrimmedEntries?: boolean;
942
+ payloadData?: Uint8Array;
943
+ includeMaterializationBytes?: boolean;
944
+ includeAppendFactsBytes?: boolean;
945
+ },
946
+ ): Promise<{
947
+ entry: Entry<T>;
948
+ removed: ShallowOrFullEntry<T>[];
949
+ change: Change<T>;
950
+ appendFacts: PreparedAppendFacts;
951
+ }> {
952
+ if (
953
+ options.canAppend ||
954
+ options.onChange ||
955
+ options.meta?.type === EntryType.CUT
956
+ ) {
957
+ throw new Error(
958
+ "appendLocallyPrepared only supports trusted plain local appends",
959
+ );
672
960
  }
673
961
 
674
- let removed: {
675
- entry: ShallowOrFullEntry<T>;
676
- fn: () => Promise<ShallowEntry | undefined>;
677
- }[];
678
-
679
- if (options?.recursively) {
680
- removed = await this.prepareDeleteRecursively(entry);
962
+ const appendOptions = withCanAppendAlreadyValidated(options);
963
+ const nextsResult = this.getNextsForAppend(appendOptions);
964
+ const nexts = isPromiseLike(nextsResult) ? await nextsResult : nextsResult;
965
+ const deferBlockStore = hasPutMany(this._storage);
966
+ const nativeAppendChain = this.entryIndex.properties.nativeGraph
967
+ ? await this.createNativePlainAppendChain(
968
+ [data],
969
+ appendOptions,
970
+ nexts,
971
+ deferBlockStore,
972
+ properties?.payloadData ? [properties.payloadData] : undefined,
973
+ )
974
+ : undefined;
975
+ let entry: Entry<T>;
976
+ if (nativeAppendChain) {
977
+ entry = nativeAppendChain.entries[0]!;
978
+ try {
979
+ if (!properties?.skipMissingNextJoin) {
980
+ await this.joinMissingNexts(entry, nexts);
981
+ }
982
+ if (deferBlockStore && !nativeAppendChain.nativeBlocksCommitted) {
983
+ await this.putAppendEntryBlocks([entry], nativeAppendChain.blocks);
984
+ }
985
+ await this.putAppendEntries(
986
+ [entry],
987
+ appendOptions,
988
+ nexts.map((next) => next.hash),
989
+ nativeAppendChain,
990
+ );
991
+ } catch (error) {
992
+ if (nativeAppendChain.nativeGraphUpdated) {
993
+ this.rollbackNativeAppendGraph([entry]);
994
+ }
995
+ if (nativeAppendChain.nativeBlocksCommitted) {
996
+ await this.rollbackNativeAppendBlocks([entry]);
997
+ }
998
+ throw error;
999
+ }
681
1000
  } else {
682
- removed = [];
683
- for (const entry of entries) {
684
- const deleteFn = await this.prepareDelete(entry.hash);
685
- deleteFn.entry &&
686
- removed.push({ entry: deleteFn.entry, fn: deleteFn.fn });
1001
+ if (data == null && properties?.payloadData) {
1002
+ throw new Error(
1003
+ "appendLocallyPrepared payload-only path requires native append support",
1004
+ );
1005
+ }
1006
+ entry = await this.createAppendEntry(data, appendOptions, nexts);
1007
+ if (!properties?.skipMissingNextJoin) {
1008
+ await this.joinMissingNexts(entry, nexts);
687
1009
  }
1010
+ await this.putAppendEntry(entry, appendOptions);
688
1011
  }
689
1012
 
1013
+ entry.init({ encoding: this._encoding, keychain: this._keychain });
1014
+
1015
+ const trimmed = await this.trimIfConfigured(appendOptions.trim, {
1016
+ resolveDeletedEntries: properties?.resolveTrimmedEntries,
1017
+ });
1018
+ const removed = trimmed ?? [];
690
1019
  const change: Change<T> = {
691
- added: [],
692
- removed: removed.map((x) => x.entry),
1020
+ added: [{ head: true, entry }],
1021
+ removed,
693
1022
  };
1023
+ const appendFacts = this.createPreparedAppendFacts(
1024
+ [entry],
1025
+ nativeAppendChain,
1026
+ )[0]!;
694
1027
 
695
- await this._onChange?.(change);
1028
+ return { entry, removed, change, appendFacts };
1029
+ }
696
1030
 
697
- // invoke deletions
698
- await Promise.all(removed.map((x) => x.fn()));
1031
+ // Internal trusted local append path for callers that can consume compact
1032
+ // append facts before a public Entry object is needed.
1033
+ private appendLocallyPreparedCommitOnly(
1034
+ data: T,
1035
+ options: AppendOptions<T> = {},
1036
+ properties?: {
1037
+ skipMissingNextJoin?: boolean;
1038
+ resolveTrimmedEntries?: boolean;
1039
+ payloadData?: Uint8Array;
1040
+ includeMaterializationBytes?: boolean;
1041
+ includeAppendFactsBytes?: boolean;
1042
+ },
1043
+ ): MaybePromise<PreparedCommitOnlyAppendResult<T> | undefined> {
1044
+ if (
1045
+ options.canAppend ||
1046
+ options.onChange ||
1047
+ options.meta?.type === EntryType.CUT
1048
+ ) {
1049
+ throw new Error(
1050
+ "appendLocallyPreparedCommitOnly only supports trusted plain local appends",
1051
+ );
1052
+ }
699
1053
 
700
- return change;
1054
+ const appendOptions = withCanAppendAlreadyValidated(options);
1055
+ const nextsResult = this.getNextsForAppend(appendOptions);
1056
+ return mapMaybePromise(nextsResult, (nexts) =>
1057
+ this.appendLocallyPreparedCommitOnlyWithNexts(
1058
+ data,
1059
+ appendOptions,
1060
+ nexts,
1061
+ properties,
1062
+ this.getNativeCommitOnlyTrimLengthTo(
1063
+ appendOptions.trim,
1064
+ properties?.resolveTrimmedEntries,
1065
+ ),
1066
+ ),
1067
+ );
701
1068
  }
702
1069
 
703
- async trim(option: TrimOptions | undefined = this._trim.options) {
704
- return this._trim.trim(option);
1070
+ private appendLocallyPreparedNativeNoNextCommitOnly(
1071
+ data: T,
1072
+ options: AppendOptions<T> = {},
1073
+ properties: {
1074
+ payloadData?: Uint8Array;
1075
+ resolveTrimmedEntries?: boolean;
1076
+ skipMissingNextJoin?: boolean;
1077
+ retainMaterializationBytes?: boolean;
1078
+ },
1079
+ prepare: (
1080
+ input: NativeNoNextCommitInput,
1081
+ ) => MaybePromise<NativePreparedNoNextCommit | undefined>,
1082
+ ): MaybePromise<PreparedCommitOnlyAppendResult<T> | undefined> {
1083
+ const directResult = this.appendLocallyPreparedNativeKnownNoNextCommitOnly(
1084
+ data,
1085
+ options,
1086
+ properties,
1087
+ prepare,
1088
+ );
1089
+ if (directResult !== undefined) {
1090
+ return directResult;
1091
+ }
1092
+ return this.appendLocallyPreparedNativeCommitOnly(
1093
+ data,
1094
+ options,
1095
+ properties,
1096
+ prepare,
1097
+ );
705
1098
  }
706
1099
 
707
- async join(
708
- entriesOrLog:
709
- | (string | Entry<T> | ShallowEntry | EntryWithRefs<T>)[]
710
- | Log<T>
711
- | ResultsIterator<Entry<any>>,
712
- options?: {
713
- verifySignatures?: boolean;
714
- trim?: TrimOptions;
715
- timeout?: number;
716
- onChange?: OnChange<T>;
717
- reset?: boolean;
1100
+ private appendLocallyPreparedNativeKnownNoNextCommitOnly(
1101
+ data: T,
1102
+ options: AppendOptions<T> = {},
1103
+ properties: {
1104
+ payloadData?: Uint8Array;
1105
+ resolveTrimmedEntries?: boolean;
1106
+ skipMissingNextJoin?: boolean;
1107
+ retainMaterializationBytes?: boolean;
718
1108
  },
719
- ): Promise<void> {
720
- let entries: Entry<T>[];
721
- const references: Map<string, Entry<T>> = new Map();
722
-
723
- const fromCache = new Map<string, string[] | null>();
724
- const resolveRemoteFrom = async (hash: string, signal?: AbortSignal) => {
725
- const cached = fromCache.get(hash);
726
- if (cached !== undefined) return cached === null ? undefined : cached;
727
-
728
- let from: string[] | undefined;
729
- try {
730
- from = await this.entryIndex.properties.resolveRemotePeers?.(hash, {
731
- signal,
732
- });
733
- } catch {
734
- from = undefined;
1109
+ prepare: (
1110
+ input: NativeNoNextCommitInput,
1111
+ ) => MaybePromise<NativePreparedNoNextCommit | undefined>,
1112
+ ): MaybePromise<PreparedCommitOnlyAppendResult<T> | undefined> {
1113
+ if (options.meta?.next == null || options.meta.next.length !== 0) {
1114
+ return undefined;
1115
+ }
1116
+ const resolvedTrim = options.trim ?? this._trim.options;
1117
+ const supportsNativeTrim =
1118
+ !resolvedTrim ||
1119
+ (resolvedTrim.type === "length" &&
1120
+ !resolvedTrim.filter?.canTrim &&
1121
+ properties.resolveTrimmedEntries === false);
1122
+ if (!supportsNativeTrim) {
1123
+ return undefined;
1124
+ }
1125
+ const nativeTrimLengthTo = this.getNativeCommitOnlyTrimLengthTo(
1126
+ options.trim,
1127
+ properties.resolveTrimmedEntries,
1128
+ );
1129
+ if (!options.meta?.gidSeed) {
1130
+ const directResult =
1131
+ this.appendLocallyPreparedNativeKnownNoNextDirectCommitOnly(
1132
+ data,
1133
+ options,
1134
+ properties,
1135
+ prepare,
1136
+ nativeTrimLengthTo,
1137
+ );
1138
+ if (directResult !== undefined) {
1139
+ return directResult;
735
1140
  }
736
- const normalized = from && from.length > 0 ? from : undefined;
737
- fromCache.set(hash, normalized ?? null);
738
- return normalized;
739
- };
1141
+ }
1142
+ return this.appendLocallyPreparedNativeCommitOnly(
1143
+ data,
1144
+ options,
1145
+ properties,
1146
+ prepare,
1147
+ true,
1148
+ );
1149
+ }
740
1150
 
741
- const remote: NonNullable<Exclude<GetOptions["remote"], boolean>> = {
742
- timeout: options?.timeout,
743
- signal: this._closeController.signal,
744
- };
1151
+ private appendLocallyPreparedNativeKnownNoNextDirectCommitOnly(
1152
+ data: T,
1153
+ options: AppendOptions<T> = {},
1154
+ properties: {
1155
+ payloadData?: Uint8Array;
1156
+ resolveTrimmedEntries?: boolean;
1157
+ retainMaterializationBytes?: boolean;
1158
+ },
1159
+ prepare: (
1160
+ input: NativeNoNextCommitInput,
1161
+ ) => MaybePromise<NativePreparedNoNextCommit | undefined>,
1162
+ nativeTrimLengthTo?: number,
1163
+ ): MaybePromise<PreparedCommitOnlyAppendResult<T> | undefined> {
1164
+ if (
1165
+ options.canAppend ||
1166
+ options.onChange ||
1167
+ options.encryption ||
1168
+ options.signers ||
1169
+ options.identity ||
1170
+ options.meta?.timestamp ||
1171
+ options.meta?.type === EntryType.CUT ||
1172
+ options.meta?.next == null ||
1173
+ options.meta.next.length !== 0 ||
1174
+ options.meta?.gidSeed ||
1175
+ (this._hasCustomCanAppend && !canAppendAlreadyValidated(options))
1176
+ ) {
1177
+ return undefined;
1178
+ }
1179
+ const identity = this._identity;
1180
+ if (!(identity instanceof Ed25519Keypair)) {
1181
+ return undefined;
1182
+ }
1183
+ const payloadData =
1184
+ properties.payloadData ??
1185
+ (data == null ? undefined : this._encoding.encoder(data));
1186
+ if (!payloadData || !hasPutMany(this._storage)) {
1187
+ return undefined;
1188
+ }
745
1189
 
746
- if (entriesOrLog instanceof Log) {
1190
+ const resolvedGid = EntryV0.createGid() as string;
1191
+ const timestamp = this._hlc.now();
1192
+ const entryType = options.meta?.type ?? EntryType.APPEND;
1193
+ const preparedValue = prepare({
1194
+ clockId: identity.publicKey.bytes,
1195
+ privateKey: identity.privateKey.privateKey,
1196
+ publicKey: identity.publicKey.publicKey,
1197
+ wallTime: timestamp.wallTime,
1198
+ logical: timestamp.logical,
1199
+ gid: resolvedGid,
1200
+ type: entryType,
1201
+ metaData: options.meta?.data,
1202
+ payloadData,
1203
+ resolveTrimmedEntries: properties.resolveTrimmedEntries,
1204
+ trimLengthTo: nativeTrimLengthTo,
1205
+ });
1206
+ return mapMaybePromise(preparedValue, (prepared) => {
1207
+ if (!prepared) {
1208
+ return undefined;
1209
+ }
1210
+ const hash = prepared.cid ?? prepared.hash;
1211
+ if (!hash) {
1212
+ return undefined;
1213
+ }
1214
+ const shouldRetainMaterializationBytes =
1215
+ properties.retainMaterializationBytes === true ||
1216
+ !!(options.trim ?? this._trim.options);
1217
+ let retainedMaterializationBytes: Uint8Array | undefined;
1218
+ const retainMaterializationBytes = () => {
1219
+ if (
1220
+ retainedMaterializationBytes ||
1221
+ !shouldRetainMaterializationBytes ||
1222
+ prepared.bytes
1223
+ ) {
1224
+ return;
1225
+ }
1226
+ const bytes =
1227
+ prepared.getBytes?.(hash) ??
1228
+ (this._storage.get(hash) as Uint8Array | undefined);
1229
+ if (
1230
+ bytes &&
1231
+ typeof (bytes as { then?: unknown }).then !== "function"
1232
+ ) {
1233
+ retainedMaterializationBytes = bytes;
1234
+ }
1235
+ };
1236
+ const effectiveNextHashes = prepared.next ?? EMPTY_NEXT_HASHES;
1237
+ const effectiveGid = prepared.gid ?? resolvedGid;
1238
+ let clock: Clock | undefined;
1239
+ const getClock = () =>
1240
+ (clock ??= new Clock({
1241
+ id: identity.publicKey.bytes,
1242
+ timestamp,
1243
+ }));
1244
+ let shallowEntry: ShallowEntry | undefined;
1245
+ const getShallowEntry = () =>
1246
+ (shallowEntry ??= new ShallowEntry({
1247
+ hash,
1248
+ payloadSize: payloadData.byteLength,
1249
+ head: true,
1250
+ meta: new ShallowMeta({
1251
+ gid: effectiveGid,
1252
+ data: options.meta?.data,
1253
+ clock: getClock(),
1254
+ next: effectiveNextHashes,
1255
+ type: entryType,
1256
+ }),
1257
+ }));
1258
+ const appendFacts: PreparedAppendFacts = {
1259
+ hash,
1260
+ gid: effectiveGid,
1261
+ next: effectiveNextHashes,
1262
+ wallTime: timestamp.wallTime,
1263
+ logical: timestamp.logical,
1264
+ clockId: identity.publicKey.bytes,
1265
+ type: entryType,
1266
+ metaData: options.meta?.data,
1267
+ payloadSize: payloadData.byteLength,
1268
+ metaBytes: prepared.metaBytes,
1269
+ hashDigestBytes: prepared.hashDigestBytes,
1270
+ };
1271
+ let materializedEntry: Entry<T> | undefined;
1272
+ const materializeEntry = () => {
1273
+ if (materializedEntry) {
1274
+ return materializedEntry;
1275
+ }
1276
+ const bytes =
1277
+ prepared.bytes ??
1278
+ retainedMaterializationBytes ??
1279
+ prepared.getBytes?.(hash) ??
1280
+ (this._storage.get(hash) as Uint8Array | undefined);
1281
+ if (
1282
+ !bytes ||
1283
+ typeof (bytes as { then?: unknown }).then === "function"
1284
+ ) {
1285
+ throw new Error("Missing synchronous native append block bytes");
1286
+ }
1287
+ const entry = deserialize(bytes, Entry) as Entry<T>;
1288
+ entry.hash = hash;
1289
+ entry.size = prepared.byteLength;
1290
+ entry.createdLocally = true;
1291
+ Entry.prepareShallowEntry(entry, getShallowEntry());
1292
+ entry.init({ encoding: this._encoding, keychain: this._keychain });
1293
+ materializedEntry = entry;
1294
+ return entry;
1295
+ };
1296
+ const finish = (): PreparedCommitOnlyAppendResult<T> => {
1297
+ retainMaterializationBytes();
1298
+ return {
1299
+ get entry() {
1300
+ return materializeEntry();
1301
+ },
1302
+ materializeEntry,
1303
+ removed: [],
1304
+ appendFacts,
1305
+ get shallowEntry() {
1306
+ return getShallowEntry();
1307
+ },
1308
+ documentTrimmedHeadsProcessed: prepared.documentTrimmedHeadsProcessed,
1309
+ documentPreviousContext: prepared.documentPreviousContext,
1310
+ };
1311
+ };
1312
+ const finishTrim = ():
1313
+ | PreparedCommitOnlyAppendResult<T>
1314
+ | Promise<PreparedCommitOnlyAppendResult<T>> => {
1315
+ retainMaterializationBytes();
1316
+ if (prepared.trimmedEntryHashes) {
1317
+ if (prepared.trimmedEntryHashes.length === 0) {
1318
+ return finish();
1319
+ }
1320
+ if (
1321
+ properties.resolveTrimmedEntries === false ||
1322
+ prepared.documentTrimmedHeadsProcessed === true
1323
+ ) {
1324
+ const trimmedEntryHashes =
1325
+ prepared.trimmedEntryHashes.length === 1
1326
+ ? prepared.trimmedEntryHashes
1327
+ : [...new Set(prepared.trimmedEntryHashes)];
1328
+ const consumedNoReturn =
1329
+ this.entryIndex.consumeNativeTrimmedEntryHashesNoReturnMaybe(
1330
+ trimmedEntryHashes,
1331
+ {
1332
+ skipNextHeadUpdates: true,
1333
+ deleteBlocks: false,
1334
+ },
1335
+ );
1336
+ if (consumedNoReturn !== undefined) {
1337
+ return mapMaybePromise(consumedNoReturn, () => ({
1338
+ get entry() {
1339
+ return materializeEntry();
1340
+ },
1341
+ materializeEntry,
1342
+ removed: [],
1343
+ removedHashes: trimmedEntryHashes,
1344
+ appendFacts,
1345
+ get shallowEntry() {
1346
+ return getShallowEntry();
1347
+ },
1348
+ documentTrimmedHeadsProcessed:
1349
+ prepared.documentTrimmedHeadsProcessed,
1350
+ documentPreviousContext: prepared.documentPreviousContext,
1351
+ }));
1352
+ }
1353
+ }
1354
+ const consumedResult =
1355
+ this.entryIndex.consumeNativeTrimmedEntryHashesMaybe(
1356
+ prepared.trimmedEntryHashes,
1357
+ {
1358
+ skipNextHeadUpdates: true,
1359
+ deleteBlocks: false,
1360
+ },
1361
+ );
1362
+ return mapMaybePromise(consumedResult, (removed) => ({
1363
+ get entry() {
1364
+ return materializeEntry();
1365
+ },
1366
+ materializeEntry,
1367
+ removed,
1368
+ removedHashes: prepared.trimmedEntryHashes,
1369
+ appendFacts,
1370
+ get shallowEntry() {
1371
+ return getShallowEntry();
1372
+ },
1373
+ documentTrimmedHeadsProcessed:
1374
+ prepared.documentTrimmedHeadsProcessed,
1375
+ documentPreviousContext: prepared.documentPreviousContext,
1376
+ }));
1377
+ }
1378
+ if (!prepared.trimmedEntries) {
1379
+ return finish();
1380
+ }
1381
+ const trimmedEntries = this.entryIndex.nativeLogEntriesToShallowEntries(
1382
+ prepared.trimmedEntries,
1383
+ );
1384
+ const consumedResult = this.entryIndex.consumeNativeTrimmedEntriesMaybe(
1385
+ trimmedEntries,
1386
+ {
1387
+ skipNextHeadUpdates: true,
1388
+ deleteBlocks: false,
1389
+ },
1390
+ );
1391
+ return mapMaybePromise(consumedResult, (removed) => ({
1392
+ get entry() {
1393
+ return materializeEntry();
1394
+ },
1395
+ materializeEntry,
1396
+ removed,
1397
+ appendFacts,
1398
+ get shallowEntry() {
1399
+ return getShallowEntry();
1400
+ },
1401
+ documentTrimmedHeadsProcessed: prepared.documentTrimmedHeadsProcessed,
1402
+ documentPreviousContext: prepared.documentPreviousContext,
1403
+ }));
1404
+ };
1405
+ const finishBlocks = ():
1406
+ | PreparedCommitOnlyAppendResult<T>
1407
+ | Promise<PreparedCommitOnlyAppendResult<T>> => {
1408
+ if (!prepared.bytes) {
1409
+ return finishTrim();
1410
+ }
1411
+ return mapMaybePromise(
1412
+ this.putPreparedAppendBlocks([
1413
+ Entry.preparedBlockFromBytes(prepared.bytes, hash),
1414
+ ]),
1415
+ finishTrim,
1416
+ );
1417
+ };
1418
+ const rollback = (error: unknown): never | Promise<never> => {
1419
+ this.rollbackNativeAppendGraphHashes([hash]);
1420
+ return this.rollbackNativeAppendBlocksHashes([hash]).then(() => {
1421
+ throw error;
1422
+ });
1423
+ };
1424
+ try {
1425
+ const putResult = this.entryIndex.putNativeCommittedAppendFacts({
1426
+ hash,
1427
+ unique: true,
1428
+ externalNextHashes: effectiveNextHashes,
1429
+ getShallowEntry,
1430
+ isHead: true,
1431
+ });
1432
+ const result = mapMaybePromise(putResult, finishBlocks);
1433
+ return isPromiseLike(result) ? result.catch(rollback) : result;
1434
+ } catch (error) {
1435
+ return rollback(error);
1436
+ }
1437
+ });
1438
+ }
1439
+
1440
+ private appendLocallyPreparedNativeCommitOnly(
1441
+ data: T,
1442
+ options: AppendOptions<T> = {},
1443
+ properties: {
1444
+ payloadData?: Uint8Array;
1445
+ resolveTrimmedEntries?: boolean;
1446
+ skipMissingNextJoin?: boolean;
1447
+ knownNoNext?: boolean;
1448
+ retainMaterializationBytes?: boolean;
1449
+ },
1450
+ prepare: (
1451
+ input: NativeCommitInput,
1452
+ ) => MaybePromise<NativePreparedNoNextCommit | undefined>,
1453
+ knownNoNext = false,
1454
+ ): MaybePromise<PreparedCommitOnlyAppendResult<T> | undefined> {
1455
+ const resolvedTrim = options.trim ?? this._trim.options;
1456
+ const supportsNativeTrim =
1457
+ !resolvedTrim ||
1458
+ (resolvedTrim.type === "length" &&
1459
+ !resolvedTrim.filter?.canTrim &&
1460
+ properties.resolveTrimmedEntries === false);
1461
+ const nativeTrimLengthTo = this.getNativeCommitOnlyTrimLengthTo(
1462
+ options.trim,
1463
+ properties.resolveTrimmedEntries,
1464
+ );
1465
+ if (
1466
+ options.canAppend ||
1467
+ options.onChange ||
1468
+ options.encryption ||
1469
+ options.signers ||
1470
+ options.identity ||
1471
+ options.meta?.timestamp ||
1472
+ !supportsNativeTrim ||
1473
+ (this._hasCustomCanAppend && !canAppendAlreadyValidated(options))
1474
+ ) {
1475
+ return undefined;
1476
+ }
1477
+ const identity = this._identity;
1478
+ if (!(identity instanceof Ed25519Keypair)) {
1479
+ return undefined;
1480
+ }
1481
+ const payloadData =
1482
+ properties.payloadData ??
1483
+ (data == null ? undefined : this._encoding.encoder(data));
1484
+ if (!payloadData || !hasPutMany(this._storage)) {
1485
+ return undefined;
1486
+ }
1487
+
1488
+ const appendOptions = withCanAppendAlreadyValidated(options);
1489
+ const knownNoNextAppend = knownNoNext || properties.knownNoNext === true;
1490
+ const nextsResult = knownNoNextAppend
1491
+ ? EMPTY_NEXT_ENTRIES
1492
+ : this.getNextsForAppend(appendOptions);
1493
+ return mapMaybePromise(nextsResult, (nexts) => {
1494
+ if (nexts.length > 0 && properties.skipMissingNextJoin !== true) {
1495
+ return undefined;
1496
+ }
1497
+
1498
+ const nextHashes: string[] = knownNoNextAppend ? EMPTY_NEXT_HASHES : [];
1499
+ let nextGid: string | undefined;
1500
+ if (nexts.length > 0) {
1501
+ if ((appendOptions.meta as { gid?: string } | undefined)?.gid) {
1502
+ return undefined;
1503
+ }
1504
+ for (const next of nexts) {
1505
+ if (!next.hash) {
1506
+ return undefined;
1507
+ }
1508
+ nextHashes.push(next.hash);
1509
+ nextGid =
1510
+ nextGid == null || next.meta.gid < nextGid
1511
+ ? next.meta.gid
1512
+ : nextGid;
1513
+ }
1514
+ }
1515
+
1516
+ const gid = nextGid ?? EntryV0.createGid(appendOptions.meta?.gidSeed);
1517
+ return mapMaybePromise(gid, (resolvedGid) => {
1518
+ const clock = new Clock({
1519
+ id: identity.publicKey.bytes,
1520
+ timestamp: this._hlc.now(),
1521
+ });
1522
+ const entryType = appendOptions.meta?.type ?? EntryType.APPEND;
1523
+ const preparedValue = prepare({
1524
+ clockId: identity.publicKey.bytes,
1525
+ privateKey: identity.privateKey.privateKey,
1526
+ publicKey: identity.publicKey.publicKey,
1527
+ wallTime: clock.timestamp.wallTime,
1528
+ logical: clock.timestamp.logical,
1529
+ gid: resolvedGid,
1530
+ next: nextHashes,
1531
+ type: entryType,
1532
+ metaData: appendOptions.meta?.data,
1533
+ payloadData,
1534
+ resolveTrimmedEntries: properties.resolveTrimmedEntries,
1535
+ trimLengthTo: nativeTrimLengthTo,
1536
+ });
1537
+ return mapMaybePromise(preparedValue, (prepared) => {
1538
+ if (!prepared) {
1539
+ return undefined;
1540
+ }
1541
+ const hash = prepared.cid ?? prepared.hash;
1542
+ if (!hash) {
1543
+ return undefined;
1544
+ }
1545
+ const shouldRetainMaterializationBytes =
1546
+ properties.retainMaterializationBytes === true || !!resolvedTrim;
1547
+ let retainedMaterializationBytes: Uint8Array | undefined;
1548
+ const retainMaterializationBytes = () => {
1549
+ if (
1550
+ retainedMaterializationBytes ||
1551
+ !shouldRetainMaterializationBytes ||
1552
+ prepared.bytes
1553
+ ) {
1554
+ return;
1555
+ }
1556
+ const bytes =
1557
+ prepared.getBytes?.(hash) ??
1558
+ (this._storage.get(hash) as Uint8Array | undefined);
1559
+ if (
1560
+ bytes &&
1561
+ typeof (bytes as { then?: unknown }).then !== "function"
1562
+ ) {
1563
+ retainedMaterializationBytes = bytes;
1564
+ }
1565
+ };
1566
+ const effectiveNextHashes = prepared.next ?? nextHashes;
1567
+ const effectiveGid = prepared.gid ?? resolvedGid;
1568
+ const shallowEntry = new ShallowEntry({
1569
+ hash,
1570
+ payloadSize: payloadData.byteLength,
1571
+ head: true,
1572
+ meta: new ShallowMeta({
1573
+ gid: effectiveGid,
1574
+ data: appendOptions.meta?.data,
1575
+ clock,
1576
+ next: effectiveNextHashes,
1577
+ type: entryType,
1578
+ }),
1579
+ });
1580
+ const appendFacts: PreparedAppendFacts = {
1581
+ hash,
1582
+ gid: effectiveGid,
1583
+ next: effectiveNextHashes,
1584
+ wallTime: clock.timestamp.wallTime,
1585
+ logical: clock.timestamp.logical,
1586
+ clockId: clock.id,
1587
+ type: entryType,
1588
+ metaData: appendOptions.meta?.data,
1589
+ payloadSize: payloadData.byteLength,
1590
+ metaBytes: prepared.metaBytes,
1591
+ hashDigestBytes: prepared.hashDigestBytes,
1592
+ };
1593
+ let materializedEntry: Entry<T> | undefined;
1594
+ const materializeEntry = () => {
1595
+ if (materializedEntry) {
1596
+ return materializedEntry;
1597
+ }
1598
+ const bytes =
1599
+ prepared.bytes ??
1600
+ retainedMaterializationBytes ??
1601
+ prepared.getBytes?.(hash) ??
1602
+ (this._storage.get(hash) as Uint8Array | undefined);
1603
+ if (
1604
+ !bytes ||
1605
+ typeof (bytes as { then?: unknown }).then === "function"
1606
+ ) {
1607
+ throw new Error("Missing synchronous native append block bytes");
1608
+ }
1609
+ const entry = deserialize(bytes, Entry) as Entry<T>;
1610
+ entry.hash = hash;
1611
+ entry.size = prepared.byteLength;
1612
+ entry.createdLocally = true;
1613
+ Entry.prepareShallowEntry(entry, shallowEntry);
1614
+ entry.init({ encoding: this._encoding, keychain: this._keychain });
1615
+ materializedEntry = entry;
1616
+ return entry;
1617
+ };
1618
+ const finish = (): PreparedCommitOnlyAppendResult<T> => {
1619
+ retainMaterializationBytes();
1620
+ return {
1621
+ get entry() {
1622
+ return materializeEntry();
1623
+ },
1624
+ materializeEntry,
1625
+ removed: [],
1626
+ appendFacts,
1627
+ shallowEntry,
1628
+ documentTrimmedHeadsProcessed:
1629
+ prepared.documentTrimmedHeadsProcessed,
1630
+ documentPreviousContext: prepared.documentPreviousContext,
1631
+ };
1632
+ };
1633
+ const finishBlocks = ():
1634
+ | PreparedCommitOnlyAppendResult<T>
1635
+ | Promise<PreparedCommitOnlyAppendResult<T>> => {
1636
+ if (!prepared.bytes) {
1637
+ return finishTrim();
1638
+ }
1639
+ return mapMaybePromise(
1640
+ this.putPreparedAppendBlocks([
1641
+ Entry.preparedBlockFromBytes(prepared.bytes, hash),
1642
+ ]),
1643
+ finishTrim,
1644
+ );
1645
+ };
1646
+ const finishTrim = ():
1647
+ | PreparedCommitOnlyAppendResult<T>
1648
+ | Promise<PreparedCommitOnlyAppendResult<T>> => {
1649
+ retainMaterializationBytes();
1650
+ if (prepared.trimmedEntryHashes) {
1651
+ if (prepared.trimmedEntryHashes.length === 0) {
1652
+ return finish();
1653
+ }
1654
+ if (
1655
+ properties.resolveTrimmedEntries === false ||
1656
+ prepared.documentTrimmedHeadsProcessed === true
1657
+ ) {
1658
+ const trimmedEntryHashes =
1659
+ prepared.trimmedEntryHashes.length === 1
1660
+ ? prepared.trimmedEntryHashes
1661
+ : [...new Set(prepared.trimmedEntryHashes)];
1662
+ const consumedNoReturn =
1663
+ this.entryIndex.consumeNativeTrimmedEntryHashesNoReturnMaybe(
1664
+ trimmedEntryHashes,
1665
+ {
1666
+ skipNextHeadUpdates: true,
1667
+ deleteBlocks: false,
1668
+ },
1669
+ );
1670
+ if (consumedNoReturn !== undefined) {
1671
+ return mapMaybePromise(consumedNoReturn, () => ({
1672
+ get entry() {
1673
+ return materializeEntry();
1674
+ },
1675
+ materializeEntry,
1676
+ removed: [],
1677
+ removedHashes: trimmedEntryHashes,
1678
+ appendFacts,
1679
+ shallowEntry,
1680
+ documentTrimmedHeadsProcessed:
1681
+ prepared.documentTrimmedHeadsProcessed,
1682
+ documentPreviousContext: prepared.documentPreviousContext,
1683
+ }));
1684
+ }
1685
+ }
1686
+ const consumedResult =
1687
+ this.entryIndex.consumeNativeTrimmedEntryHashesMaybe(
1688
+ prepared.trimmedEntryHashes,
1689
+ {
1690
+ skipNextHeadUpdates: true,
1691
+ deleteBlocks: false,
1692
+ },
1693
+ );
1694
+ return mapMaybePromise(consumedResult, (removed) => ({
1695
+ get entry() {
1696
+ return materializeEntry();
1697
+ },
1698
+ materializeEntry,
1699
+ removed,
1700
+ appendFacts,
1701
+ shallowEntry,
1702
+ documentTrimmedHeadsProcessed:
1703
+ prepared.documentTrimmedHeadsProcessed,
1704
+ documentPreviousContext: prepared.documentPreviousContext,
1705
+ }));
1706
+ }
1707
+ if (!prepared.trimmedEntries) {
1708
+ return finish();
1709
+ }
1710
+ const trimmedEntries =
1711
+ this.entryIndex.nativeLogEntriesToShallowEntries(
1712
+ prepared.trimmedEntries,
1713
+ );
1714
+ const consumedResult =
1715
+ this.entryIndex.consumeNativeTrimmedEntriesMaybe(trimmedEntries, {
1716
+ skipNextHeadUpdates: true,
1717
+ deleteBlocks: false,
1718
+ });
1719
+ return mapMaybePromise(consumedResult, (removed) => ({
1720
+ get entry() {
1721
+ return materializeEntry();
1722
+ },
1723
+ materializeEntry,
1724
+ removed,
1725
+ appendFacts,
1726
+ shallowEntry,
1727
+ documentTrimmedHeadsProcessed:
1728
+ prepared.documentTrimmedHeadsProcessed,
1729
+ documentPreviousContext: prepared.documentPreviousContext,
1730
+ }));
1731
+ };
1732
+ const rollback = (error: unknown): never | Promise<never> => {
1733
+ this.rollbackNativeAppendGraphHashes([hash]);
1734
+ return this.rollbackNativeAppendBlocksHashes([hash]).then(() => {
1735
+ throw error;
1736
+ });
1737
+ };
1738
+ try {
1739
+ const putResult = this.entryIndex.putNativeCommittedAppendFacts({
1740
+ hash,
1741
+ unique: true,
1742
+ externalNextHashes: nextHashes,
1743
+ shallowEntry,
1744
+ isHead: true,
1745
+ });
1746
+ const result = mapMaybePromise(putResult, finishBlocks);
1747
+ return isPromiseLike(result) ? result.catch(rollback) : result;
1748
+ } catch (error) {
1749
+ return rollback(error);
1750
+ }
1751
+ });
1752
+ });
1753
+ });
1754
+ }
1755
+
1756
+ private appendLocallyPreparedCommitOnlyWithNexts(
1757
+ data: T,
1758
+ appendOptions: AppendOptions<T>,
1759
+ nexts: Sorting.SortableEntry[],
1760
+ properties?: {
1761
+ skipMissingNextJoin?: boolean;
1762
+ resolveTrimmedEntries?: boolean;
1763
+ payloadData?: Uint8Array;
1764
+ includeMaterializationBytes?: boolean;
1765
+ includeAppendFactsBytes?: boolean;
1766
+ },
1767
+ nativeTrimLengthTo?: number,
1768
+ ): MaybePromise<PreparedCommitOnlyAppendResult<T> | undefined> {
1769
+ const deferBlockStore = hasPutMany(this._storage);
1770
+ const nativeAppendChainResult = this.createNativePlainAppendCommitOnly(
1771
+ [data],
1772
+ appendOptions,
1773
+ nexts,
1774
+ deferBlockStore,
1775
+ properties?.payloadData ? [properties.payloadData] : undefined,
1776
+ properties?.includeMaterializationBytes,
1777
+ properties?.includeAppendFactsBytes,
1778
+ nativeTrimLengthTo,
1779
+ );
1780
+ return mapMaybePromise(nativeAppendChainResult, (nativeAppendChain) =>
1781
+ this.finishLocallyPreparedCommitOnlyAppend(
1782
+ nativeAppendChain,
1783
+ appendOptions,
1784
+ nexts,
1785
+ deferBlockStore,
1786
+ properties,
1787
+ ),
1788
+ );
1789
+ }
1790
+
1791
+ private finishLocallyPreparedCommitOnlyAppend(
1792
+ nativeAppendChain: PreparedAppendCommitOnlyChain<T> | undefined,
1793
+ appendOptions: AppendOptions<T>,
1794
+ nexts: Sorting.SortableEntry[],
1795
+ deferBlockStore: boolean,
1796
+ properties?: {
1797
+ skipMissingNextJoin?: boolean;
1798
+ resolveTrimmedEntries?: boolean;
1799
+ },
1800
+ ): MaybePromise<PreparedCommitOnlyAppendResult<T> | undefined> {
1801
+ if (!nativeAppendChain) {
1802
+ return undefined;
1803
+ }
1804
+
1805
+ const appendFacts = nativeAppendChain.appendFacts[0]!;
1806
+ const shallowEntry = nativeAppendChain.shallowEntries[0]!;
1807
+ let materializedEntry: Entry<T> | undefined;
1808
+ const materializeEntry = () => {
1809
+ const entry = materializedEntry ?? nativeAppendChain.materializeEntry(0);
1810
+ entry.init({ encoding: this._encoding, keychain: this._keychain });
1811
+ materializedEntry = entry;
1812
+ return entry;
1813
+ };
1814
+ const finishTrim = (): MaybePromise<PreparedCommitOnlyAppendResult<T>> => {
1815
+ if (nativeAppendChain.trimmedNativeEntryHashes) {
1816
+ const trimmedEntryHashes = [
1817
+ ...new Set(nativeAppendChain.trimmedNativeEntryHashes),
1818
+ ];
1819
+ if (
1820
+ properties?.resolveTrimmedEntries === false &&
1821
+ nativeAppendChain.trimmedNativeBlocksDeleted === true
1822
+ ) {
1823
+ const consumedNoReturn =
1824
+ this.entryIndex.consumeNativeTrimmedEntryHashesNoReturnMaybe(
1825
+ trimmedEntryHashes,
1826
+ {
1827
+ skipNextHeadUpdates: true,
1828
+ deleteBlocks: false,
1829
+ },
1830
+ );
1831
+ if (consumedNoReturn !== undefined) {
1832
+ return mapMaybePromise(consumedNoReturn, () => ({
1833
+ get entry() {
1834
+ return materializeEntry();
1835
+ },
1836
+ materializeEntry,
1837
+ removed: [],
1838
+ removedHashes: trimmedEntryHashes,
1839
+ appendFacts,
1840
+ shallowEntry,
1841
+ }));
1842
+ }
1843
+ }
1844
+ const consumedResult =
1845
+ this.entryIndex.consumeNativeTrimmedEntryHashesMaybe(
1846
+ trimmedEntryHashes,
1847
+ {
1848
+ skipNextHeadUpdates: true,
1849
+ deleteBlocks:
1850
+ nativeAppendChain.trimmedNativeBlocksDeleted !== true,
1851
+ },
1852
+ );
1853
+ return mapMaybePromise(consumedResult, (removed) => ({
1854
+ get entry() {
1855
+ return materializeEntry();
1856
+ },
1857
+ materializeEntry,
1858
+ removed,
1859
+ removedHashes: trimmedEntryHashes,
1860
+ appendFacts,
1861
+ shallowEntry,
1862
+ }));
1863
+ }
1864
+ if (nativeAppendChain.trimmedNativeEntries) {
1865
+ const trimmedEntries = this.entryIndex.nativeLogEntriesToShallowEntries(
1866
+ nativeAppendChain.trimmedNativeEntries,
1867
+ );
1868
+ const consumedResult = this.entryIndex.consumeNativeTrimmedEntriesMaybe(
1869
+ trimmedEntries,
1870
+ {
1871
+ skipNextHeadUpdates: true,
1872
+ deleteBlocks: nativeAppendChain.trimmedNativeBlocksDeleted !== true,
1873
+ },
1874
+ );
1875
+ return mapMaybePromise(consumedResult, (removed) => ({
1876
+ get entry() {
1877
+ return materializeEntry();
1878
+ },
1879
+ materializeEntry,
1880
+ removed,
1881
+ appendFacts,
1882
+ shallowEntry,
1883
+ }));
1884
+ }
1885
+ const trimmedResult = this.trimIfConfigured(appendOptions.trim, {
1886
+ resolveDeletedEntries: properties?.resolveTrimmedEntries,
1887
+ });
1888
+ return mapMaybePromise(trimmedResult, (trimmed) => {
1889
+ const removed = trimmed ?? [];
1890
+ return {
1891
+ get entry() {
1892
+ return materializeEntry();
1893
+ },
1894
+ materializeEntry,
1895
+ removed,
1896
+ appendFacts,
1897
+ shallowEntry,
1898
+ };
1899
+ });
1900
+ };
1901
+ const finishFacts = (): MaybePromise<PreparedCommitOnlyAppendResult<T>> => {
1902
+ const putFactsResult = this.entryIndex.putNativeCommittedAppendFacts({
1903
+ hash: appendFacts.hash,
1904
+ unique: true,
1905
+ externalNextHashes: nexts.map((next) => next.hash),
1906
+ shallowEntry,
1907
+ isHead: true,
1908
+ });
1909
+ return mapMaybePromise(putFactsResult, finishTrim);
1910
+ };
1911
+ const finishBlocks = (): MaybePromise<
1912
+ PreparedCommitOnlyAppendResult<T>
1913
+ > => {
1914
+ if (deferBlockStore && !nativeAppendChain.nativeBlocksCommitted) {
1915
+ return mapMaybePromise(
1916
+ this.putPreparedAppendBlocks(nativeAppendChain.blocks),
1917
+ finishFacts,
1918
+ );
1919
+ }
1920
+ return finishFacts();
1921
+ };
1922
+ const rollback = (error: unknown): never | Promise<never> => {
1923
+ if (nativeAppendChain.nativeGraphUpdated) {
1924
+ this.rollbackNativeAppendGraphHashes([appendFacts.hash]);
1925
+ }
1926
+ if (nativeAppendChain.nativeBlocksCommitted) {
1927
+ return this.rollbackNativeAppendBlocksHashes([appendFacts.hash]).then(
1928
+ () => {
1929
+ throw error;
1930
+ },
1931
+ );
1932
+ }
1933
+ throw error;
1934
+ };
1935
+ try {
1936
+ let result: MaybePromise<PreparedCommitOnlyAppendResult<T>>;
1937
+ if (!properties?.skipMissingNextJoin && nexts.length > 0) {
1938
+ result = mapMaybePromise(
1939
+ this.joinMissingNexts(materializeEntry(), nexts),
1940
+ finishBlocks,
1941
+ );
1942
+ } else {
1943
+ result = finishBlocks();
1944
+ }
1945
+ return isPromiseLike(result) ? result.catch(rollback) : result;
1946
+ } catch (error) {
1947
+ return rollback(error);
1948
+ }
1949
+ }
1950
+
1951
+ private async appendLocallyPreparedManyIndependent(
1952
+ data: T[],
1953
+ options: AppendOptions<T> = {},
1954
+ properties?: {
1955
+ resolveTrimmedEntries?: boolean;
1956
+ payloadDatas?: Uint8Array[];
1957
+ nexts?: Sorting.SortableEntry[][];
1958
+ },
1959
+ ): Promise<
1960
+ | {
1961
+ entries: Entry<T>[];
1962
+ removed: ShallowOrFullEntry<T>[];
1963
+ change: Change<T>;
1964
+ appendFacts: PreparedAppendFacts[];
1965
+ }
1966
+ | undefined
1967
+ > {
1968
+ if (data.length === 0) {
1969
+ return {
1970
+ entries: [],
1971
+ removed: [],
1972
+ change: { added: [], removed: [] },
1973
+ appendFacts: [],
1974
+ };
1975
+ }
1976
+ if (
1977
+ options.canAppend ||
1978
+ options.onChange ||
1979
+ options.meta?.type === EntryType.CUT
1980
+ ) {
1981
+ throw new Error(
1982
+ "appendLocallyPreparedManyIndependent only supports trusted plain local appends",
1983
+ );
1984
+ }
1985
+
1986
+ const appendOptions = withCanAppendAlreadyValidated(options);
1987
+ const deferBlockStore = hasPutMany(this._storage);
1988
+ const nativeAppendBatch = await this.createNativePlainAppendEntriesBatch(
1989
+ data,
1990
+ appendOptions,
1991
+ deferBlockStore,
1992
+ properties?.payloadDatas,
1993
+ properties?.nexts,
1994
+ );
1995
+ if (!nativeAppendBatch) {
1996
+ return undefined;
1997
+ }
1998
+
1999
+ const entries = nativeAppendBatch.entries;
2000
+ const externalNextHashes =
2001
+ properties?.nexts?.flatMap((nexts) => nexts.map((next) => next.hash)) ??
2002
+ [];
2003
+ try {
2004
+ if (deferBlockStore && !nativeAppendBatch.nativeBlocksCommitted) {
2005
+ await this.putAppendEntryBlocks(entries, nativeAppendBatch.blocks);
2006
+ }
2007
+ await this.putAppendEntries(
2008
+ entries,
2009
+ appendOptions,
2010
+ externalNextHashes,
2011
+ nativeAppendBatch,
2012
+ entries.map(() => true),
2013
+ );
2014
+ } catch (error) {
2015
+ if (nativeAppendBatch.nativeGraphUpdated) {
2016
+ this.rollbackNativeAppendGraph(entries);
2017
+ }
2018
+ if (nativeAppendBatch.nativeBlocksCommitted) {
2019
+ await this.rollbackNativeAppendBlocks(entries);
2020
+ }
2021
+ throw error;
2022
+ }
2023
+
2024
+ for (const entry of entries) {
2025
+ entry.init({ encoding: this._encoding, keychain: this._keychain });
2026
+ }
2027
+
2028
+ const trimmed = await this.trimIfConfigured(appendOptions.trim, {
2029
+ resolveDeletedEntries: properties?.resolveTrimmedEntries,
2030
+ });
2031
+ const removed = trimmed ?? [];
2032
+ const change: Change<T> = {
2033
+ added: entries.map((entry) => ({ head: true, entry })),
2034
+ removed,
2035
+ };
2036
+ const appendFacts = this.createPreparedAppendFacts(
2037
+ entries,
2038
+ nativeAppendBatch,
2039
+ );
2040
+
2041
+ return { entries, removed, change, appendFacts };
2042
+ }
2043
+
2044
+ private appendLocallyPreparedNativeKnownNoNextCommitOnlyBatch(
2045
+ data: T[],
2046
+ options: AppendOptions<T> = {},
2047
+ properties: {
2048
+ payloadDatas: Uint8Array[];
2049
+ resolveTrimmedEntries?: boolean;
2050
+ allowPreparedNexts?: boolean;
2051
+ retainMaterializationBytes?: boolean;
2052
+ },
2053
+ prepare: (
2054
+ inputs: NativeNoNextCommitInput[],
2055
+ ) => MaybePromise<Array<NativePreparedNoNextCommit | undefined> | undefined>,
2056
+ ): MaybePromise<PreparedCommitOnlyAppendBatchResult<T> | undefined> {
2057
+ if (data.length === 0) {
2058
+ return {
2059
+ entries: [],
2060
+ materializeEntries: [],
2061
+ removed: [],
2062
+ appendFacts: [],
2063
+ };
2064
+ }
2065
+ if (data.length !== properties.payloadDatas.length) {
2066
+ throw new Error("Mismatched native batch payload count");
2067
+ }
2068
+ const resolvedTrim = options.trim ?? this._trim.options;
2069
+ const supportsNativeTrim =
2070
+ !resolvedTrim ||
2071
+ (resolvedTrim.type === "length" &&
2072
+ !resolvedTrim.filter?.canTrim &&
2073
+ properties.resolveTrimmedEntries === false);
2074
+ if (
2075
+ options.canAppend ||
2076
+ options.onChange ||
2077
+ options.encryption ||
2078
+ options.signers ||
2079
+ options.identity ||
2080
+ options.meta?.timestamp ||
2081
+ options.meta?.type === EntryType.CUT ||
2082
+ (options.meta?.next != null && options.meta.next.length !== 0) ||
2083
+ options.meta?.gidSeed ||
2084
+ !supportsNativeTrim ||
2085
+ (this._hasCustomCanAppend && !canAppendAlreadyValidated(options))
2086
+ ) {
2087
+ return undefined;
2088
+ }
2089
+ const identity = this._identity;
2090
+ if (!(identity instanceof Ed25519Keypair) || !hasPutMany(this._storage)) {
2091
+ return undefined;
2092
+ }
2093
+ const nativeTrimLengthTo = this.getNativeCommitOnlyTrimLengthTo(
2094
+ options.trim,
2095
+ properties.resolveTrimmedEntries,
2096
+ );
2097
+ const entryType = options.meta?.type ?? EntryType.APPEND;
2098
+ const rows = properties.payloadDatas.map((payloadData) => {
2099
+ const gid = EntryV0.createGid() as string;
2100
+ const timestamp = this._hlc.now();
2101
+ return {
2102
+ gid,
2103
+ timestamp,
2104
+ payloadData,
2105
+ input: {
2106
+ clockId: identity.publicKey.bytes,
2107
+ privateKey: identity.privateKey.privateKey,
2108
+ publicKey: identity.publicKey.publicKey,
2109
+ wallTime: timestamp.wallTime,
2110
+ logical: timestamp.logical,
2111
+ gid,
2112
+ type: entryType,
2113
+ metaData: options.meta?.data,
2114
+ payloadData,
2115
+ resolveTrimmedEntries: properties.resolveTrimmedEntries,
2116
+ trimLengthTo: nativeTrimLengthTo,
2117
+ } satisfies NativeNoNextCommitInput,
2118
+ };
2119
+ });
2120
+ const preparedValue = prepare(rows.map((row) => row.input));
2121
+ return mapMaybePromise(preparedValue, (preparedRows) => {
2122
+ if (!preparedRows || preparedRows.length !== rows.length) {
2123
+ return undefined;
2124
+ }
2125
+ const appendFacts: PreparedAppendFacts[] = [];
2126
+ const materializeEntries: Array<() => Entry<T>> = [];
2127
+ const retainMaterializationBytesFns: Array<() => void> = [];
2128
+ const indexRows: Parameters<
2129
+ EntryIndex<T>["putNativeCommittedAppendFactsBatch"]
2130
+ >[0] = [];
2131
+ const trimmedEntryHashes: string[] = [];
2132
+ const documentTrimmedHeadsProcessed: boolean[] = [];
2133
+ for (let index = 0; index < rows.length; index++) {
2134
+ const row = rows[index]!;
2135
+ const prepared = preparedRows[index];
2136
+ if (!prepared) {
2137
+ return undefined;
2138
+ }
2139
+ const hash = prepared.cid ?? prepared.hash;
2140
+ if (!hash) {
2141
+ return undefined;
2142
+ }
2143
+ const shouldRetainMaterializationBytes =
2144
+ properties.retainMaterializationBytes === true || !!resolvedTrim;
2145
+ let retainedMaterializationBytes: Uint8Array | undefined;
2146
+ const retainMaterializationBytes = () => {
2147
+ if (
2148
+ retainedMaterializationBytes ||
2149
+ !shouldRetainMaterializationBytes ||
2150
+ prepared.bytes
2151
+ ) {
2152
+ return;
2153
+ }
2154
+ const bytes =
2155
+ prepared.getBytes?.(hash) ??
2156
+ (this._storage.get(hash) as Uint8Array | undefined);
2157
+ if (
2158
+ bytes &&
2159
+ typeof (bytes as { then?: unknown }).then !== "function"
2160
+ ) {
2161
+ retainedMaterializationBytes = bytes;
2162
+ }
2163
+ };
2164
+ const effectiveNextHashes = prepared.next ?? EMPTY_NEXT_HASHES;
2165
+ if (
2166
+ effectiveNextHashes.length !== 0 &&
2167
+ properties.allowPreparedNexts !== true
2168
+ ) {
2169
+ return undefined;
2170
+ }
2171
+ const effectiveGid = prepared.gid ?? row.gid;
2172
+ let clock: Clock | undefined;
2173
+ const getClock = () =>
2174
+ (clock ??= new Clock({
2175
+ id: identity.publicKey.bytes,
2176
+ timestamp: row.timestamp,
2177
+ }));
2178
+ let shallowEntry: ShallowEntry | undefined;
2179
+ const getShallowEntry = () =>
2180
+ (shallowEntry ??= new ShallowEntry({
2181
+ hash,
2182
+ payloadSize: row.payloadData.byteLength,
2183
+ head: true,
2184
+ meta: new ShallowMeta({
2185
+ gid: effectiveGid,
2186
+ data: options.meta?.data,
2187
+ clock: getClock(),
2188
+ next: effectiveNextHashes,
2189
+ type: entryType,
2190
+ }),
2191
+ }));
2192
+ const facts: PreparedAppendFacts = {
2193
+ hash,
2194
+ gid: effectiveGid,
2195
+ next: effectiveNextHashes,
2196
+ wallTime: row.timestamp.wallTime,
2197
+ logical: row.timestamp.logical,
2198
+ clockId: identity.publicKey.bytes,
2199
+ type: entryType,
2200
+ metaData: options.meta?.data,
2201
+ payloadSize: row.payloadData.byteLength,
2202
+ metaBytes: prepared.metaBytes,
2203
+ hashDigestBytes: prepared.hashDigestBytes,
2204
+ };
2205
+ let materializedEntry: Entry<T> | undefined;
2206
+ const materializeEntry = () => {
2207
+ if (materializedEntry) {
2208
+ return materializedEntry;
2209
+ }
2210
+ const bytes =
2211
+ prepared.bytes ??
2212
+ retainedMaterializationBytes ??
2213
+ prepared.getBytes?.(hash) ??
2214
+ (this._storage.get(hash) as Uint8Array | undefined);
2215
+ if (
2216
+ !bytes ||
2217
+ typeof (bytes as { then?: unknown }).then === "function"
2218
+ ) {
2219
+ throw new Error("Missing synchronous native append block bytes");
2220
+ }
2221
+ const entry = deserialize(bytes, Entry) as Entry<T>;
2222
+ entry.hash = hash;
2223
+ entry.size = prepared.byteLength;
2224
+ entry.createdLocally = true;
2225
+ Entry.prepareShallowEntry(entry, getShallowEntry());
2226
+ entry.init({ encoding: this._encoding, keychain: this._keychain });
2227
+ materializedEntry = entry;
2228
+ return entry;
2229
+ };
2230
+ appendFacts.push(facts);
2231
+ materializeEntries.push(materializeEntry);
2232
+ retainMaterializationBytesFns.push(retainMaterializationBytes);
2233
+ indexRows.push({
2234
+ hash,
2235
+ unique: true,
2236
+ externalNextHashes: effectiveNextHashes,
2237
+ getShallowEntry,
2238
+ isHead: true,
2239
+ });
2240
+ if (prepared.trimmedEntryHashes?.length) {
2241
+ trimmedEntryHashes.push(...prepared.trimmedEntryHashes);
2242
+ }
2243
+ documentTrimmedHeadsProcessed.push(
2244
+ prepared.documentTrimmedHeadsProcessed === true,
2245
+ );
2246
+ }
2247
+ const rollback = (error: unknown): never | Promise<never> => {
2248
+ this.rollbackNativeAppendGraphHashes(
2249
+ appendFacts.map((facts) => facts.hash),
2250
+ );
2251
+ return this.rollbackNativeAppendBlocksHashes(
2252
+ appendFacts.map((facts) => facts.hash),
2253
+ ).then(() => {
2254
+ throw error;
2255
+ });
2256
+ };
2257
+ const finish = (): PreparedCommitOnlyAppendBatchResult<T> => {
2258
+ for (const retainMaterializationBytes of retainMaterializationBytesFns) {
2259
+ retainMaterializationBytes();
2260
+ }
2261
+ let entries: Entry<T>[] | undefined;
2262
+ return {
2263
+ get entries() {
2264
+ return (entries ??= materializeEntries.map((materializeEntry) =>
2265
+ materializeEntry(),
2266
+ ));
2267
+ },
2268
+ materializeEntries,
2269
+ removed: [],
2270
+ removedHashes:
2271
+ trimmedEntryHashes.length > 0
2272
+ ? normalizedUniqueStrings(trimmedEntryHashes)
2273
+ : undefined,
2274
+ appendFacts,
2275
+ documentTrimmedHeadsProcessed,
2276
+ };
2277
+ };
2278
+ const finishTrim = ():
2279
+ | PreparedCommitOnlyAppendBatchResult<T>
2280
+ | Promise<PreparedCommitOnlyAppendBatchResult<T>>
2281
+ | undefined => {
2282
+ if (trimmedEntryHashes.length === 0) {
2283
+ return finish();
2284
+ }
2285
+ if (
2286
+ properties.resolveTrimmedEntries !== false &&
2287
+ !documentTrimmedHeadsProcessed.every(Boolean)
2288
+ ) {
2289
+ return undefined;
2290
+ }
2291
+ const uniqueTrimmedHashes = normalizedUniqueStrings(trimmedEntryHashes);
2292
+ const consumedNoReturn =
2293
+ this.entryIndex.consumeNativeTrimmedEntryHashesNoReturnMaybe(
2294
+ uniqueTrimmedHashes,
2295
+ {
2296
+ skipNextHeadUpdates: true,
2297
+ deleteBlocks: false,
2298
+ },
2299
+ );
2300
+ if (consumedNoReturn === undefined) {
2301
+ return undefined;
2302
+ }
2303
+ return mapMaybePromise(consumedNoReturn, finish);
2304
+ };
2305
+ try {
2306
+ const putResult =
2307
+ this.entryIndex.putNativeCommittedAppendFactsBatch(indexRows);
2308
+ const result = mapMaybePromise(putResult, finishTrim);
2309
+ return isPromiseLike(result) ? result.catch(rollback) : result;
2310
+ } catch (error) {
2311
+ return rollback(error);
2312
+ }
2313
+ });
2314
+ }
2315
+
2316
+ private createPreparedAppendFacts(
2317
+ entries: Entry<T>[],
2318
+ prepared?: PreparedAppendChain<T>,
2319
+ ): PreparedAppendFacts[] {
2320
+ if (prepared?.appendFacts?.length === entries.length) {
2321
+ return prepared.appendFacts;
2322
+ }
2323
+ return entries.map((entry, index) => {
2324
+ const shallowEntry = prepared?.shallowEntries[index];
2325
+ if (shallowEntry) {
2326
+ return {
2327
+ hash: shallowEntry.hash,
2328
+ gid: shallowEntry.meta.gid,
2329
+ next: shallowEntry.meta.next,
2330
+ wallTime: shallowEntry.meta.clock.timestamp.wallTime,
2331
+ logical: shallowEntry.meta.clock.timestamp.logical,
2332
+ clockId: shallowEntry.meta.clock.id,
2333
+ type: shallowEntry.meta.type,
2334
+ metaData: shallowEntry.meta.data,
2335
+ payloadSize: shallowEntry.payloadSize,
2336
+ metaBytes: (entry as EntryWithMetaBytes).getMetaBytes?.(),
2337
+ hashDigestBytes: (entry as EntryWithMetaBytes).getHashDigestBytes?.(),
2338
+ };
2339
+ }
2340
+ return {
2341
+ hash: entry.hash,
2342
+ gid: entry.meta.gid,
2343
+ next: entry.meta.next,
2344
+ wallTime: entry.meta.clock.timestamp.wallTime,
2345
+ logical: entry.meta.clock.timestamp.logical,
2346
+ clockId: entry.meta.clock.id,
2347
+ type: entry.meta.type,
2348
+ metaData: entry.meta.data,
2349
+ payloadSize: entry.payload.byteLength,
2350
+ metaBytes: (entry as EntryWithMetaBytes).getMetaBytes?.(),
2351
+ hashDigestBytes: (entry as EntryWithMetaBytes).getHashDigestBytes?.(),
2352
+ };
2353
+ });
2354
+ }
2355
+
2356
+ async appendMany(
2357
+ data: T[],
2358
+ options: AppendOptions<T> = {},
2359
+ ): Promise<{ entries: Entry<T>[]; removed: ShallowOrFullEntry<T>[] }> {
2360
+ if (data.length === 0) {
2361
+ return { entries: [], removed: [] };
2362
+ }
2363
+ if (options.meta?.type === EntryType.CUT) {
2364
+ throw new Error("appendMany does not support CUT entries");
2365
+ }
2366
+
2367
+ let nexts = await this.getNextsForAppend(options);
2368
+ const initialNexts = nexts;
2369
+ const entries: Entry<T>[] = [];
2370
+ const pendingDeletes: (
2371
+ | PendingDelete<T>
2372
+ | { entry: ShallowOrFullEntry<T>; fn: undefined }
2373
+ )[] = [];
2374
+ const deferBlockStore = hasPutMany(this._storage);
2375
+ const nativeAppendChain = await this.createNativePlainAppendChain(
2376
+ data,
2377
+ options,
2378
+ nexts,
2379
+ deferBlockStore,
2380
+ );
2381
+
2382
+ if (nativeAppendChain) {
2383
+ entries.push(...nativeAppendChain.entries);
2384
+ nexts = [entries[entries.length - 1]!];
2385
+ } else {
2386
+ for (const item of data) {
2387
+ const entry = await this.createAppendEntry(item, options, nexts, {
2388
+ deferStore: deferBlockStore,
2389
+ });
2390
+ entries.push(entry);
2391
+ nexts = [entry];
2392
+ }
2393
+ }
2394
+
2395
+ try {
2396
+ await this.joinMissingNexts(entries[0]!, initialNexts);
2397
+ if (deferBlockStore && !nativeAppendChain?.nativeBlocksCommitted) {
2398
+ await this.putAppendEntryBlocks(entries, nativeAppendChain?.blocks);
2399
+ }
2400
+ await this.putAppendEntries(
2401
+ entries,
2402
+ options,
2403
+ initialNexts.map((entry) => entry.hash),
2404
+ nativeAppendChain,
2405
+ );
2406
+ } catch (error) {
2407
+ if (nativeAppendChain?.nativeGraphUpdated) {
2408
+ this.rollbackNativeAppendGraph(entries);
2409
+ }
2410
+ if (nativeAppendChain?.nativeBlocksCommitted) {
2411
+ await this.rollbackNativeAppendBlocks(entries);
2412
+ }
2413
+ throw error;
2414
+ }
2415
+
2416
+ for (const entry of entries) {
2417
+ entry.init({ encoding: this._encoding, keychain: this._keychain });
2418
+ }
2419
+
2420
+ const trimmed = await this.trimIfConfigured(options?.trim);
2421
+ if (trimmed) {
2422
+ for (const entry of trimmed) {
2423
+ pendingDeletes.push({ entry, fn: undefined });
2424
+ }
2425
+ }
2426
+
2427
+ const removed = pendingDeletes.map((x) => x.entry);
2428
+ const changes: Change<T> = {
2429
+ added: entries.map((entry, index) => ({
2430
+ head: index === entries.length - 1,
2431
+ entry,
2432
+ })),
2433
+ removed,
2434
+ };
2435
+
2436
+ await (options?.onChange || this._onChange)?.(changes);
2437
+ await Promise.all(pendingDeletes.map((x) => x.fn?.()));
2438
+ return { entries, removed };
2439
+ }
2440
+
2441
+ private createNativePlainAppendChain(
2442
+ data: T[],
2443
+ options: AppendOptions<T>,
2444
+ nexts: Sorting.SortableEntry[],
2445
+ deferBlockStore: boolean,
2446
+ payloadDatas?: Uint8Array[],
2447
+ ): Promise<PreparedAppendChain<T> | undefined> {
2448
+ const canAppendAlreadyValidatedForOptions =
2449
+ canAppendAlreadyValidated(options);
2450
+ if (
2451
+ !deferBlockStore ||
2452
+ options.encryption ||
2453
+ options.signers ||
2454
+ options.canAppend ||
2455
+ (this._hasCustomCanAppend && !canAppendAlreadyValidatedForOptions) ||
2456
+ options.meta?.timestamp ||
2457
+ options.meta?.type === EntryType.CUT
2458
+ ) {
2459
+ return Promise.resolve(undefined);
2460
+ }
2461
+
2462
+ const nativeGraph =
2463
+ !this.entryIndex.properties.onGidRemoved &&
2464
+ (this.entryIndex.properties.nativeGraph?.graph
2465
+ .prepareEntryV0PlainChainCommit ||
2466
+ this.entryIndex.properties.nativeGraph?.graph
2467
+ .prepareEntryV0PlainEntryCommit ||
2468
+ this.entryIndex.properties.nativeGraph?.graph
2469
+ .prepareEntryV0PlainChainAndPut ||
2470
+ this.entryIndex.properties.nativeGraph?.graph
2471
+ .prepareEntryV0PlainEntryAndPut)
2472
+ ? this.entryIndex.properties.nativeGraph.graph
2473
+ : undefined;
2474
+ return EntryV0.createPlainAppendChainBatch<T>({
2475
+ data,
2476
+ meta: {
2477
+ clocks: () =>
2478
+ data.map(
2479
+ () =>
2480
+ new Clock({
2481
+ id: this._identity.publicKey.bytes,
2482
+ timestamp: this._hlc.now(),
2483
+ }),
2484
+ ),
2485
+ type: options.meta?.type,
2486
+ gidSeed: options.meta?.gidSeed,
2487
+ data: options.meta?.data,
2488
+ next: nexts,
2489
+ },
2490
+ encoding: this._encoding,
2491
+ payloadDatas,
2492
+ identity: options.identity || this._identity,
2493
+ deferStore: deferBlockStore,
2494
+ cachePreparedEntries: false,
2495
+ nativeGraph,
2496
+ nativeBlockStore: this._storage,
2497
+ });
2498
+ }
2499
+
2500
+ private createNativePlainAppendCommitOnly(
2501
+ data: T[],
2502
+ options: AppendOptions<T>,
2503
+ nexts: Sorting.SortableEntry[],
2504
+ deferBlockStore: boolean,
2505
+ payloadDatas?: Uint8Array[],
2506
+ includeMaterializationBytes?: boolean,
2507
+ includeAppendFactsBytes?: boolean,
2508
+ nativeTrimLengthTo?: number,
2509
+ ): MaybePromise<PreparedAppendCommitOnlyChain<T> | undefined> {
2510
+ const canAppendAlreadyValidatedForOptions =
2511
+ canAppendAlreadyValidated(options);
2512
+ if (
2513
+ data.length !== 1 ||
2514
+ !deferBlockStore ||
2515
+ options.encryption ||
2516
+ options.signers ||
2517
+ options.canAppend ||
2518
+ (this._hasCustomCanAppend && !canAppendAlreadyValidatedForOptions) ||
2519
+ options.meta?.timestamp ||
2520
+ options.meta?.type === EntryType.CUT
2521
+ ) {
2522
+ return undefined;
2523
+ }
2524
+
2525
+ const nativeGraph =
2526
+ !this.entryIndex.properties.onGidRemoved &&
2527
+ (this.entryIndex.properties.nativeGraph?.graph
2528
+ .prepareEntryV0PlainEntryCommit ||
2529
+ this.entryIndex.properties.nativeGraph?.graph
2530
+ .prepareEntryV0PlainEntryAndPut)
2531
+ ? this.entryIndex.properties.nativeGraph.graph
2532
+ : undefined;
2533
+ if (!nativeGraph) {
2534
+ return undefined;
2535
+ }
2536
+ return EntryV0.createPlainAppendChainCommitOnly<T>({
2537
+ data,
2538
+ meta: {
2539
+ clocks: () => [
2540
+ new Clock({
2541
+ id: this._identity.publicKey.bytes,
2542
+ timestamp: this._hlc.now(),
2543
+ }),
2544
+ ],
2545
+ type: options.meta?.type,
2546
+ gidSeed: options.meta?.gidSeed,
2547
+ data: options.meta?.data,
2548
+ next: nexts,
2549
+ },
2550
+ encoding: this._encoding,
2551
+ payloadDatas,
2552
+ identity: options.identity || this._identity,
2553
+ deferStore: deferBlockStore,
2554
+ nativeGraph,
2555
+ nativeBlockStore: this._storage,
2556
+ includeMaterializationBytes,
2557
+ includeAppendFactsBytes,
2558
+ nativeTrimLengthTo,
2559
+ });
2560
+ }
2561
+
2562
+ private async createNativePlainAppendEntriesBatch(
2563
+ data: T[],
2564
+ options: AppendOptions<T>,
2565
+ deferBlockStore: boolean,
2566
+ payloadDatas?: Uint8Array[],
2567
+ nexts?: Sorting.SortableEntry[][],
2568
+ ): Promise<PreparedAppendChain<T> | undefined> {
2569
+ const canAppendAlreadyValidatedForOptions =
2570
+ canAppendAlreadyValidated(options);
2571
+ if (
2572
+ !deferBlockStore ||
2573
+ data.length === 0 ||
2574
+ options.encryption ||
2575
+ options.signers ||
2576
+ options.canAppend ||
2577
+ (this._hasCustomCanAppend && !canAppendAlreadyValidatedForOptions) ||
2578
+ options.meta?.timestamp ||
2579
+ options.meta?.type === EntryType.CUT ||
2580
+ options.meta?.gidSeed ||
2581
+ options.meta?.next ||
2582
+ (nexts && nexts.length !== data.length) ||
2583
+ (payloadDatas && payloadDatas.length !== data.length)
2584
+ ) {
2585
+ return undefined;
2586
+ }
2587
+
2588
+ const nativeGraph =
2589
+ !this.entryIndex.properties.onGidRemoved &&
2590
+ this.entryIndex.properties.nativeGraph?.graph
2591
+ ? this.entryIndex.properties.nativeGraph.graph
2592
+ : undefined;
2593
+
2594
+ const generatedGids = EntryV0.createGids(data.length);
2595
+ const gids = generatedGids.map((generatedGid, index) => {
2596
+ const entryNexts = nexts?.[index];
2597
+ if (!entryNexts || entryNexts.length === 0) {
2598
+ return generatedGid;
2599
+ }
2600
+ let gid = entryNexts[0]!.meta.gid;
2601
+ for (let i = 1; i < entryNexts.length; i++) {
2602
+ const nextGid = entryNexts[i]!.meta.gid;
2603
+ if (nextGid < gid) {
2604
+ gid = nextGid;
2605
+ }
2606
+ }
2607
+ return gid;
2608
+ });
2609
+ const clockId = this._identity.publicKey.bytes;
2610
+ const clocks = this._hlc.nowBatch(data.length).map(
2611
+ (timestamp) =>
2612
+ new Clock({
2613
+ id: clockId,
2614
+ timestamp,
2615
+ }),
2616
+ );
2617
+ const metaDatas = data.map(() => options.meta?.data);
2618
+ const directBatch = nativeGraph?.prepareEntryV0PlainEntriesCommit
2619
+ ? await EntryV0.createPlainAppendEntriesBatch<T>({
2620
+ data,
2621
+ payloadDatas,
2622
+ meta: {
2623
+ clocks: () => clocks,
2624
+ gids,
2625
+ nexts,
2626
+ type: options.meta?.type,
2627
+ datas: metaDatas,
2628
+ },
2629
+ encoding: this._encoding,
2630
+ identity: options.identity || this._identity,
2631
+ deferStore: deferBlockStore,
2632
+ cachePreparedEntries: false,
2633
+ nativeGraph,
2634
+ nativeBlockStore: this._storage,
2635
+ })
2636
+ : undefined;
2637
+ if (directBatch) {
2638
+ return directBatch;
2639
+ }
2640
+
2641
+ const entries: Entry<T>[] = [];
2642
+ const blocks: PreparedEntryBlock[] = [];
2643
+ const shallowEntries: PreparedAppendChain<T>["shallowEntries"] = [];
2644
+ const nativeEntries: NonNullable<PreparedAppendChain<T>["nativeEntries"]> =
2645
+ [];
2646
+ let nativeGraphUpdated = false;
2647
+ let nativeBlocksCommitted = true;
2648
+ for (let i = 0; i < data.length; i++) {
2649
+ const entryNexts = nexts?.[i] ?? [];
2650
+ const prepared = await EntryV0.createPlainAppendChainBatch<T>({
2651
+ data: [data[i]!],
2652
+ payloadDatas: payloadDatas ? [payloadDatas[i]!] : undefined,
2653
+ meta: {
2654
+ clocks: () => [clocks[i]!],
2655
+ gid: entryNexts.length === 0 ? gids[i]! : undefined,
2656
+ type: options.meta?.type,
2657
+ data: metaDatas[i],
2658
+ next: entryNexts,
2659
+ },
2660
+ encoding: this._encoding,
2661
+ identity: options.identity || this._identity,
2662
+ deferStore: deferBlockStore,
2663
+ cachePreparedEntries: false,
2664
+ nativeGraph,
2665
+ nativeBlockStore: this._storage,
2666
+ });
2667
+ if (!prepared) {
2668
+ return undefined;
2669
+ }
2670
+ entries.push(prepared.entries[0]!);
2671
+ if (prepared.blocks) {
2672
+ blocks.push(...prepared.blocks);
2673
+ }
2674
+ shallowEntries.push(...prepared.shallowEntries);
2675
+ if (prepared.nativeEntries) {
2676
+ nativeEntries.push(...prepared.nativeEntries);
2677
+ }
2678
+ nativeGraphUpdated ||= prepared.nativeGraphUpdated === true;
2679
+ nativeBlocksCommitted &&= prepared.nativeBlocksCommitted === true;
2680
+ }
2681
+
2682
+ if (!nativeBlocksCommitted && blocks.length !== entries.length) {
2683
+ return undefined;
2684
+ }
2685
+ return {
2686
+ entries,
2687
+ blocks: blocks.length > 0 ? blocks : undefined,
2688
+ shallowEntries,
2689
+ nativeEntries,
2690
+ nativeGraphUpdated,
2691
+ nativeBlocksCommitted,
2692
+ };
2693
+ }
2694
+
2695
+ private rollbackNativeAppendGraph(entries: Entry<T>[]) {
2696
+ this.rollbackNativeAppendGraphHashes(entries.map((entry) => entry.hash));
2697
+ }
2698
+
2699
+ private rollbackNativeAppendGraphHashes(hashes: string[]) {
2700
+ const graph = this.entryIndex.properties.nativeGraph?.graph;
2701
+ if (!graph) {
2702
+ return;
2703
+ }
2704
+ for (let i = hashes.length - 1; i >= 0; i--) {
2705
+ graph.delete(hashes[i]!);
2706
+ }
2707
+ }
2708
+
2709
+ private async rollbackNativeAppendBlocks(entries: Entry<T>[]) {
2710
+ await this.rollbackNativeAppendBlocksHashes(
2711
+ entries.map((entry) => entry.hash),
2712
+ );
2713
+ }
2714
+
2715
+ private async rollbackNativeAppendBlocksHashes(hashes: string[]) {
2716
+ const storage = this._storage as BlocksWithPutMany;
2717
+ if (typeof storage.rmMany === "function") {
2718
+ await storage.rmMany(hashes);
2719
+ return;
2720
+ }
2721
+ await Promise.all(hashes.map((hash) => this._storage.rm(hash)));
2722
+ }
2723
+
2724
+ private validateExplicitNexts(options: AppendOptions<T>) {
2725
+ if (!options.meta?.next) {
2726
+ return;
2727
+ }
2728
+ for (const n of options.meta.next) {
2729
+ if (!n.hash) {
2730
+ throw new Error(
2731
+ "Expecting nexts to already be saved. missing hash for one or more entries",
2732
+ );
2733
+ }
2734
+ }
2735
+ }
2736
+
2737
+ private getNextsForAppend(
2738
+ options: AppendOptions<T>,
2739
+ ): MaybePromise<Sorting.SortableEntry[]> {
2740
+ this.validateExplicitNexts(options);
2741
+ return (
2742
+ options.meta?.next ||
2743
+ this.entryIndex.getHeadsForAppend() ||
2744
+ this.entryIndex
2745
+ .getHeads(undefined, { type: "shape", shape: Sorting.ENTRY_SORT_SHAPE })
2746
+ .all()
2747
+ );
2748
+ }
2749
+
2750
+ private async createAppendEntry(
2751
+ data: T,
2752
+ options: AppendOptions<T>,
2753
+ nexts: Sorting.SortableEntry[],
2754
+ storeOptions?: {
2755
+ deferStore?: boolean;
2756
+ },
2757
+ ): Promise<Entry<T>> {
2758
+ const clock = new Clock({
2759
+ id: this._identity.publicKey.bytes,
2760
+ timestamp: options?.meta?.timestamp || this._hlc.now(),
2761
+ });
2762
+
2763
+ const entry = await EntryV0.create<T>({
2764
+ store: this._storage,
2765
+ identity: options.identity || this._identity,
2766
+ signers: options.signers,
2767
+ data,
2768
+ meta: {
2769
+ clock,
2770
+ type: options.meta?.type,
2771
+ gidSeed: options.meta?.gidSeed,
2772
+ data: options.meta?.data,
2773
+ next: nexts,
2774
+ },
2775
+ encoding: this._encoding,
2776
+ encryption: options.encryption
2777
+ ? {
2778
+ keypair: options.encryption.keypair,
2779
+ receiver: {
2780
+ ...options.encryption.receiver,
2781
+ },
2782
+ }
2783
+ : undefined,
2784
+ canAppend:
2785
+ canAppendAlreadyValidated(options)
2786
+ ? undefined
2787
+ : options.canAppend ||
2788
+ (this._hasCustomCanAppend ? this._canAppend : undefined),
2789
+ deferStore: storeOptions?.deferStore,
2790
+ });
2791
+
2792
+ if (!entry.hash) {
2793
+ throw new Error("Unexpected");
2794
+ }
2795
+ return entry;
2796
+ }
2797
+
2798
+ private async joinMissingNexts(
2799
+ entry: Entry<T>,
2800
+ nexts: Sorting.SortableEntry[],
2801
+ ) {
2802
+ if (entry.meta.type === EntryType.CUT) {
2803
+ return;
2804
+ }
2805
+ for (const e of nexts) {
2806
+ if (await this.has(e.hash)) {
2807
+ continue;
2808
+ }
2809
+ let nextEntry: Entry<any>;
2810
+ if (e instanceof Entry) {
2811
+ nextEntry = e;
2812
+ } else {
2813
+ const resolved = await this.entryIndex.get(e.hash);
2814
+ if (!resolved) {
2815
+ warn("Unexpected missing entry when joining", e.hash);
2816
+ continue;
2817
+ }
2818
+ nextEntry = resolved;
2819
+ }
2820
+ await this.join([nextEntry]);
2821
+ }
2822
+ }
2823
+
2824
+ private async putAppendEntry(entry: Entry<T>, options: AppendOptions<T>) {
2825
+ await this.entryIndex.put(entry, {
2826
+ unique: true,
2827
+ isHead: true,
2828
+ toMultiHash: false,
2829
+ deferIndexWrite:
2830
+ options.deferIndexWrite ??
2831
+ (options.durability
2832
+ ? options.durability === "buffered"
2833
+ : this._appendDurability === "buffered"),
2834
+ });
2835
+ }
2836
+
2837
+ private async putAppendEntries(
2838
+ entries: Entry<T>[],
2839
+ options: AppendOptions<T>,
2840
+ externalNextHashes: string[],
2841
+ preparedAppendChain?: PreparedAppendChain<T>,
2842
+ heads?: boolean[],
2843
+ ) {
2844
+ const prepared =
2845
+ preparedAppendChain &&
2846
+ entries.length === preparedAppendChain.entries.length
2847
+ ? {
2848
+ shallowEntries: preparedAppendChain.shallowEntries,
2849
+ nativeEntries: preparedAppendChain.nativeEntries,
2850
+ nativeGraphUpdated: preparedAppendChain.nativeGraphUpdated,
2851
+ nativeBlocksCommitted: preparedAppendChain.nativeBlocksCommitted,
2852
+ }
2853
+ : undefined;
2854
+ if (
2855
+ entries.length === 1 &&
2856
+ prepared?.nativeGraphUpdated === true &&
2857
+ !this.entryIndex.properties.onGidRemoved
2858
+ ) {
2859
+ await this.entryIndex.putNativeCommittedAppend(entries[0]!, {
2860
+ unique: true,
2861
+ externalNextHashes,
2862
+ shallowEntry: prepared.shallowEntries[0],
2863
+ isHead: heads?.[0] ?? true,
2864
+ });
2865
+ return;
2866
+ }
2867
+
2868
+ await this.entryIndex.putAppendBatch(entries, {
2869
+ unique: true,
2870
+ externalNextHashes,
2871
+ heads,
2872
+ prepared,
2873
+ deferIndexWrite:
2874
+ options.deferIndexWrite ??
2875
+ (options.durability
2876
+ ? options.durability === "buffered"
2877
+ : this._appendDurability === "buffered"),
2878
+ });
2879
+ }
2880
+
2881
+ private async putAppendEntryBlocks(
2882
+ entries: Entry<T>[],
2883
+ preparedBlocks?: PreparedEntryBlock[],
2884
+ ) {
2885
+ const blocks =
2886
+ preparedBlocks && preparedBlocks.length === entries.length
2887
+ ? preparedBlocks
2888
+ : entries.map((entry) => {
2889
+ const prepared = Entry.takePreparedBlock(entry);
2890
+ if (!prepared) {
2891
+ throw new Error("Missing prepared entry block");
2892
+ }
2893
+ return prepared;
2894
+ });
2895
+
2896
+ if (blocks.length === 1 && hasPutKnown(this._storage)) {
2897
+ const block = blocks[0]!;
2898
+ const cidResult = this._storage.putKnown(block.cid, block.block.bytes);
2899
+ const cid = isPromiseLike(cidResult) ? await cidResult : cidResult;
2900
+ if (cid !== block.cid) {
2901
+ throw new Error("Unexpected block cid");
2902
+ }
2903
+ return;
2904
+ }
2905
+ if (hasPutKnownManyColumns(this._storage)) {
2906
+ const cids = new Array<string>(blocks.length);
2907
+ const bytes = new Array<Uint8Array>(blocks.length);
2908
+ for (let i = 0; i < blocks.length; i++) {
2909
+ const block = blocks[i]!;
2910
+ cids[i] = block.cid;
2911
+ bytes[i] = block.block.bytes;
2912
+ }
2913
+ const cidsResult = this._storage.putKnownManyColumns(cids, bytes);
2914
+ const result = isPromiseLike(cidsResult) ? await cidsResult : cidsResult;
2915
+ if (result.length !== blocks.length) {
2916
+ throw new Error("Unexpected block batch result length");
2917
+ }
2918
+ for (let i = 0; i < result.length; i++) {
2919
+ if (result[i] !== cids[i]) {
2920
+ throw new Error("Unexpected block batch cid");
2921
+ }
2922
+ }
2923
+ return;
2924
+ }
2925
+ if (hasPutKnownMany(this._storage)) {
2926
+ const cidsResult = this._storage.putKnownMany(
2927
+ blocks.map((block) => [block.cid, block.block.bytes] as const),
2928
+ );
2929
+ const cids = isPromiseLike(cidsResult) ? await cidsResult : cidsResult;
2930
+ if (cids.length !== blocks.length) {
2931
+ throw new Error("Unexpected block batch result length");
2932
+ }
2933
+ for (let i = 0; i < cids.length; i++) {
2934
+ if (cids[i] !== blocks[i]!.cid) {
2935
+ throw new Error("Unexpected block batch cid");
2936
+ }
2937
+ }
2938
+ return;
2939
+ }
2940
+ const cids = await (this._storage as BlocksWithPutMany).putMany!(blocks);
2941
+ if (cids.length !== blocks.length) {
2942
+ throw new Error("Unexpected block batch result length");
2943
+ }
2944
+ for (let i = 0; i < cids.length; i++) {
2945
+ if (cids[i] !== blocks[i].cid) {
2946
+ throw new Error("Unexpected block batch cid");
2947
+ }
2948
+ }
2949
+ }
2950
+
2951
+ private async putKnownEntryBytesBatch(
2952
+ blocks: Array<{ cid: string; bytes: Uint8Array }>,
2953
+ ) {
2954
+ if (blocks.length === 0) {
2955
+ return;
2956
+ }
2957
+ if (blocks.length === 1 && hasPutKnown(this._storage)) {
2958
+ const block = blocks[0]!;
2959
+ const cidResult = this._storage.putKnown(block.cid, block.bytes);
2960
+ const cid = isPromiseLike(cidResult) ? await cidResult : cidResult;
2961
+ if (cid !== block.cid) {
2962
+ throw new Error("Unexpected block cid");
2963
+ }
2964
+ return;
2965
+ }
2966
+ if (hasPutKnownManyColumns(this._storage)) {
2967
+ const cids = new Array<string>(blocks.length);
2968
+ const bytes = new Array<Uint8Array>(blocks.length);
2969
+ for (let i = 0; i < blocks.length; i++) {
2970
+ const block = blocks[i]!;
2971
+ cids[i] = block.cid;
2972
+ bytes[i] = block.bytes;
2973
+ }
2974
+ const cidsResult = this._storage.putKnownManyColumns(cids, bytes);
2975
+ const result = isPromiseLike(cidsResult) ? await cidsResult : cidsResult;
2976
+ if (result.length !== blocks.length) {
2977
+ throw new Error("Unexpected block batch result length");
2978
+ }
2979
+ for (let i = 0; i < result.length; i++) {
2980
+ if (result[i] !== cids[i]) {
2981
+ throw new Error("Unexpected block batch cid");
2982
+ }
2983
+ }
2984
+ return;
2985
+ }
2986
+ if (hasPutKnownMany(this._storage)) {
2987
+ const cidsResult = this._storage.putKnownMany(
2988
+ blocks.map((block) => [block.cid, block.bytes] as const),
2989
+ );
2990
+ const cids = isPromiseLike(cidsResult) ? await cidsResult : cidsResult;
2991
+ if (cids.length !== blocks.length) {
2992
+ throw new Error("Unexpected block batch result length");
2993
+ }
2994
+ for (let i = 0; i < cids.length; i++) {
2995
+ if (cids[i] !== blocks[i]!.cid) {
2996
+ throw new Error("Unexpected block batch cid");
2997
+ }
2998
+ }
2999
+ return;
3000
+ }
3001
+ const preparedBlocks = blocks.map((block) =>
3002
+ Entry.preparedBlockFromBytes(block.bytes, block.cid),
3003
+ );
3004
+ const cids = await (this._storage as BlocksWithPutMany).putMany!(
3005
+ preparedBlocks,
3006
+ );
3007
+ if (cids.length !== blocks.length) {
3008
+ throw new Error("Unexpected block batch result length");
3009
+ }
3010
+ for (let i = 0; i < cids.length; i++) {
3011
+ if (cids[i] !== blocks[i]!.cid) {
3012
+ throw new Error("Unexpected block batch cid");
3013
+ }
3014
+ }
3015
+ }
3016
+
3017
+ private putPreparedAppendBlocks(
3018
+ preparedBlocks?: PreparedEntryBlock[],
3019
+ ): MaybePromise<void> {
3020
+ if (!preparedBlocks || preparedBlocks.length === 0) {
3021
+ throw new Error("Missing prepared entry block");
3022
+ }
3023
+ if (preparedBlocks.length === 1 && hasPutKnown(this._storage)) {
3024
+ const block = preparedBlocks[0]!;
3025
+ const cidResult = this._storage.putKnown(block.cid, block.block.bytes);
3026
+ const checkCid = (cid: string) => {
3027
+ if (cid !== block.cid) {
3028
+ throw new Error("Unexpected block cid");
3029
+ }
3030
+ };
3031
+ if (isPromiseLike(cidResult)) {
3032
+ return cidResult.then(checkCid);
3033
+ }
3034
+ checkCid(cidResult);
3035
+ return;
3036
+ }
3037
+ const checkCids = (cids: string[]) => {
3038
+ if (cids.length !== preparedBlocks.length) {
3039
+ throw new Error("Unexpected block batch result length");
3040
+ }
3041
+ for (let i = 0; i < cids.length; i++) {
3042
+ if (cids[i] !== preparedBlocks[i]!.cid) {
3043
+ throw new Error("Unexpected block batch cid");
3044
+ }
3045
+ }
3046
+ };
3047
+ if (hasPutKnownManyColumns(this._storage)) {
3048
+ const cids = new Array<string>(preparedBlocks.length);
3049
+ const bytes = new Array<Uint8Array>(preparedBlocks.length);
3050
+ for (let i = 0; i < preparedBlocks.length; i++) {
3051
+ const block = preparedBlocks[i]!;
3052
+ cids[i] = block.cid;
3053
+ bytes[i] = block.block.bytes;
3054
+ }
3055
+ const cidsResult = this._storage.putKnownManyColumns(cids, bytes);
3056
+ if (isPromiseLike(cidsResult)) {
3057
+ return cidsResult.then(checkCids);
3058
+ }
3059
+ checkCids(cidsResult);
3060
+ return;
3061
+ }
3062
+ if (hasPutKnownMany(this._storage)) {
3063
+ const cidsResult = this._storage.putKnownMany(
3064
+ preparedBlocks.map((block) => [block.cid, block.block.bytes] as const),
3065
+ );
3066
+ if (isPromiseLike(cidsResult)) {
3067
+ return cidsResult.then(checkCids);
3068
+ }
3069
+ checkCids(cidsResult);
3070
+ return;
3071
+ }
3072
+ const cidsResult = (this._storage as BlocksWithPutMany).putMany!(
3073
+ preparedBlocks,
3074
+ );
3075
+ if (isPromiseLike(cidsResult)) {
3076
+ return cidsResult.then(checkCids);
3077
+ }
3078
+ checkCids(cidsResult);
3079
+ }
3080
+
3081
+ async remove(
3082
+ entry:
3083
+ | { hash: string; meta: { next: string[] } }
3084
+ | { hash: string; meta: { next: string[] } }[],
3085
+ options?: { recursively?: boolean },
3086
+ ): Promise<Change<T>> {
3087
+ /* await this.load({ reload: false }); */
3088
+ const entries = Array.isArray(entry) ? entry : [entry];
3089
+
3090
+ if (entries.length === 0) {
3091
+ return {
3092
+ added: [],
3093
+ removed: [],
3094
+ };
3095
+ }
3096
+
3097
+ let removed: {
3098
+ entry: ShallowOrFullEntry<T>;
3099
+ fn: () => Promise<ShallowEntry | undefined>;
3100
+ }[];
3101
+
3102
+ if (options?.recursively) {
3103
+ removed = await this.prepareDeleteRecursively(entry);
3104
+ } else {
3105
+ removed = [];
3106
+ for (const entry of entries) {
3107
+ const deleteFn = await this.prepareDelete(entry.hash);
3108
+ deleteFn.entry &&
3109
+ removed.push({ entry: deleteFn.entry, fn: deleteFn.fn });
3110
+ }
3111
+ }
3112
+
3113
+ const change: Change<T> = {
3114
+ added: [],
3115
+ removed: removed.map((x) => x.entry),
3116
+ };
3117
+
3118
+ await this._onChange?.(change);
3119
+
3120
+ // invoke deletions
3121
+ await Promise.all(removed.map((x) => x.fn()));
3122
+
3123
+ return change;
3124
+ }
3125
+
3126
+ async trim(
3127
+ option: TrimOptions | undefined = this._trim.options,
3128
+ properties?: { resolveDeletedEntries?: boolean },
3129
+ ) {
3130
+ return this._trim.trim(option, properties);
3131
+ }
3132
+
3133
+ private trimIfConfigured(
3134
+ option?: TrimOptions,
3135
+ properties?: { resolveDeletedEntries?: boolean },
3136
+ ): MaybePromise<ShallowOrFullEntry<T>[] | undefined> {
3137
+ const resolved = option ?? this._trim.options;
3138
+ return resolved ? this.trim(resolved, properties) : undefined;
3139
+ }
3140
+
3141
+ private getNativeCommitOnlyTrimLengthTo(
3142
+ option: TrimOptions | undefined,
3143
+ resolveDeletedEntries: boolean | undefined,
3144
+ ): number | undefined {
3145
+ const resolved = option ?? this._trim.options;
3146
+ if (
3147
+ !resolved ||
3148
+ resolved.type !== "length" ||
3149
+ resolved.filter?.canTrim ||
3150
+ resolveDeletedEntries !== false
3151
+ ) {
3152
+ return;
3153
+ }
3154
+
3155
+ const from = resolved.from ?? resolved.to;
3156
+ if (this.length + 1 < from) {
3157
+ return;
3158
+ }
3159
+ return resolved.to;
3160
+ }
3161
+
3162
+ async join(
3163
+ entriesOrLog:
3164
+ | (string | Entry<T> | ShallowEntry | EntryWithRefs<T>)[]
3165
+ | Log<T>
3166
+ | ResultsIterator<Entry<any>>,
3167
+ options?: JoinOptions<T>,
3168
+ ): Promise<void>;
3169
+ async join(
3170
+ entriesOrLog:
3171
+ | (string | Entry<T> | ShallowEntry | EntryWithRefs<T>)[]
3172
+ | Log<T>
3173
+ | ResultsIterator<Entry<any>>,
3174
+ options?: TrustedJoinOptions<T>,
3175
+ ): Promise<void> {
3176
+ let entries: Entry<T>[];
3177
+ const references: Map<string, Entry<T>> = new Map();
3178
+
3179
+ const fromCache = new Map<string, string[] | null>();
3180
+ const resolveRemoteFrom = async (hash: string, signal?: AbortSignal) => {
3181
+ const cached = fromCache.get(hash);
3182
+ if (cached !== undefined) return cached === null ? undefined : cached;
3183
+
3184
+ let from: string[] | undefined;
3185
+ try {
3186
+ from = await this.entryIndex.properties.resolveRemotePeers?.(hash, {
3187
+ signal,
3188
+ });
3189
+ } catch {
3190
+ from = undefined;
3191
+ }
3192
+ const normalized = from && from.length > 0 ? from : undefined;
3193
+ fromCache.set(hash, normalized ?? null);
3194
+ return normalized;
3195
+ };
3196
+
3197
+ const remote: NonNullable<Exclude<GetOptions["remote"], boolean>> = {
3198
+ timeout: options?.timeout,
3199
+ signal: this._closeController.signal,
3200
+ };
3201
+
3202
+ if (entriesOrLog instanceof Log) {
747
3203
  if (entriesOrLog.entryIndex.length === 0) return;
748
3204
  entries = await entriesOrLog.toArray();
749
3205
  for (const element of entries) references.set(element.hash, element);
750
3206
  } else if (Array.isArray(entriesOrLog)) {
751
3207
  if (entriesOrLog.length === 0) return;
752
- const existingHashes = options?.reset
753
- ? new Set<string>()
754
- : await this.entryIndex.hasMany(
755
- entriesOrLog.map((element) =>
756
- typeof element === "string"
757
- ? element
758
- : element instanceof Entry
759
- ? element.hash
760
- : element instanceof ShallowEntry
3208
+ const existingHashes =
3209
+ options?.reset || options?.__peerbitEntriesAlreadyMissing === true
3210
+ ? new Set<string>()
3211
+ : await this.entryIndex.hasMany(
3212
+ entriesOrLog.map((element) =>
3213
+ typeof element === "string"
3214
+ ? element
3215
+ : element instanceof Entry
761
3216
  ? element.hash
762
- : element.entry.hash,
763
- ),
764
- );
3217
+ : element instanceof ShallowEntry
3218
+ ? element.hash
3219
+ : element.entry.hash,
3220
+ ),
3221
+ );
765
3222
 
766
3223
  entries = [];
767
3224
  for (const element of entriesOrLog) {
@@ -844,11 +3301,34 @@ export class Log<T> {
844
3301
  entries = all;
845
3302
  }
846
3303
 
3304
+ const profile = options?.__peerbitProfile;
3305
+ const headsStartedAt = internalProfileStart(profile);
847
3306
  const heads: Map<string, boolean> = new Map();
848
3307
  for (const entry of entries) {
849
3308
  if (heads.has(entry.hash)) continue;
850
3309
  heads.set(entry.hash, true);
851
- for (const next of await entry.getNext()) heads.set(next, false);
3310
+ const nexts =
3311
+ options?.__peerbitBatchIndependent === true
3312
+ ? entry.meta.next
3313
+ : await entry.getNext();
3314
+ for (const next of nexts) heads.set(next, false);
3315
+ }
3316
+ emitInternalProfileDuration(profile, headsStartedAt, {
3317
+ name: "log.join.prepareHeads",
3318
+ component: "log",
3319
+ entries: entries.length,
3320
+ count: heads.size,
3321
+ messages: 1,
3322
+ details: {
3323
+ batchIndependent: options?.__peerbitBatchIndependent === true,
3324
+ },
3325
+ });
3326
+
3327
+ if (
3328
+ options?.__peerbitBatchIndependent === true &&
3329
+ (await this.tryJoinIndependentAppendBatch(entries, heads, options))
3330
+ ) {
3331
+ return;
852
3332
  }
853
3333
 
854
3334
  for (const entry of entries) {
@@ -877,6 +3357,522 @@ export class Log<T> {
877
3357
  }
878
3358
  }
879
3359
 
3360
+ // Internal trusted receive path for callers that can supply prepared append facts.
3361
+ private async joinPreparedAppendFactsBatch(
3362
+ entries: PreparedAppendJoinFacts[],
3363
+ options?: TrustedPreparedAppendFactsBatchJoinOptions,
3364
+ ): Promise<boolean> {
3365
+ if (
3366
+ entries.length === 0 ||
3367
+ !canAppendAlreadyValidated(options) ||
3368
+ entries.some((entry) => this._joining.has(entry.hash))
3369
+ ) {
3370
+ return false;
3371
+ }
3372
+
3373
+ const resolvedOptions = options!;
3374
+ const profile = resolvedOptions.__peerbitProfile;
3375
+ const prepareStartedAt = internalProfileStart(profile);
3376
+ const entryHashes = new Array<string>(entries.length);
3377
+ let hasAnyNext = false;
3378
+ for (let i = 0; i < entries.length; i++) {
3379
+ const entry = entries[i]!;
3380
+ // byteLength stands in for the bytes presence check: prepared facts
3381
+ // carry both, and probing `bytes` would force stash-backed heads to
3382
+ // materialize block bytes the native commit never reads.
3383
+ if (
3384
+ !entry.hash ||
3385
+ entry.byteLength == null ||
3386
+ entry.meta.type !== EntryType.APPEND
3387
+ ) {
3388
+ return false;
3389
+ }
3390
+ entryHashes[i] = entry.hash;
3391
+ if (entry.meta.next.length > 0) {
3392
+ hasAnyNext = true;
3393
+ }
3394
+ }
3395
+ const batchHashes = new Set(entryHashes);
3396
+ let heads: Map<string, boolean> | undefined;
3397
+ if (hasAnyNext) {
3398
+ heads = new Map();
3399
+ for (const entry of entries) {
3400
+ if (heads.has(entry.hash)) {
3401
+ continue;
3402
+ }
3403
+ heads.set(entry.hash, true);
3404
+ for (const next of entry.meta.next) {
3405
+ heads.set(next, false);
3406
+ }
3407
+ }
3408
+ }
3409
+ emitInternalProfileDuration(profile, prepareStartedAt, {
3410
+ name: "log.joinPreparedFacts.prepare",
3411
+ component: "log",
3412
+ entries: entries.length,
3413
+ count: heads?.size ?? entries.length,
3414
+ messages: 1,
3415
+ });
3416
+
3417
+ const nativeCommitValidatesPlan =
3418
+ resolvedOptions.__peerbitNativePreparedJoinCommitValidatesPlan === true &&
3419
+ !!resolvedOptions.__peerbitNativePreparedJoinCommit;
3420
+ const headFlags: boolean[] = [];
3421
+ const headFlagsBytes = new Uint8Array(entries.length);
3422
+ const pushHeadFlag = (index: number, isHead: boolean) => {
3423
+ headFlags.push(isHead);
3424
+ headFlagsBytes[index] = isHead ? 1 : 0;
3425
+ };
3426
+ if (nativeCommitValidatesPlan) {
3427
+ for (let i = 0; i < entries.length; i++) {
3428
+ const entry = entries[i]!;
3429
+ pushHeadFlag(i, heads?.get(entry.hash) ?? true);
3430
+ }
3431
+ emitInternalProfileDuration(profile, internalProfileStart(profile), {
3432
+ name: "log.joinPreparedFacts.plan",
3433
+ component: "log",
3434
+ entries: entries.length,
3435
+ messages: 1,
3436
+ details: { nativeCommitValidatesPlan: true },
3437
+ });
3438
+ emitInternalProfileDuration(profile, internalProfileStart(profile), {
3439
+ name: "log.joinPreparedFacts.validatePlan",
3440
+ component: "log",
3441
+ entries: entries.length,
3442
+ messages: 1,
3443
+ details: { nativeCommitValidatesPlan: true },
3444
+ });
3445
+ } else {
3446
+ const planStartedAt = internalProfileStart(profile);
3447
+ const joinPlans = await this.entryIndex.planJoinBatch(
3448
+ entries,
3449
+ false,
3450
+ profile,
3451
+ );
3452
+ emitInternalProfileDuration(profile, planStartedAt, {
3453
+ name: "log.joinPreparedFacts.plan",
3454
+ component: "log",
3455
+ entries: entries.length,
3456
+ messages: 1,
3457
+ });
3458
+ const validatePlanStartedAt = internalProfileStart(profile);
3459
+ for (let i = 0; i < entries.length; i++) {
3460
+ const entry = entries[i]!;
3461
+ const joinPlan = joinPlans[i]!;
3462
+ if (
3463
+ joinPlan.skip ||
3464
+ joinPlan.coveredByCut ||
3465
+ !joinPlan.cutChecked ||
3466
+ joinPlan.missingParents.some((hash) => !batchHashes.has(hash))
3467
+ ) {
3468
+ return false;
3469
+ }
3470
+ pushHeadFlag(i, heads?.get(entry.hash) ?? true);
3471
+ }
3472
+ emitInternalProfileDuration(profile, validatePlanStartedAt, {
3473
+ name: "log.joinPreparedFacts.validatePlan",
3474
+ component: "log",
3475
+ entries: entries.length,
3476
+ messages: 1,
3477
+ });
3478
+ }
3479
+
3480
+ let nativeValidatedCommitRejected = false;
3481
+ const batchPromise = (async () => {
3482
+ const clockStartedAt = internalProfileStart(profile);
3483
+ for (const entry of entries) {
3484
+ this._hlc.update(entry.meta.clock.timestamp);
3485
+ }
3486
+ emitInternalProfileDuration(profile, clockStartedAt, {
3487
+ name: "log.joinPreparedFacts.clock",
3488
+ component: "log",
3489
+ entries: entries.length,
3490
+ messages: 1,
3491
+ });
3492
+
3493
+ const trustedMissing =
3494
+ resolvedOptions.__peerbitEntriesAlreadyMissing === true &&
3495
+ batchHashes.size === entries.length;
3496
+ let nativePreparedCommitted = false;
3497
+ if (resolvedOptions.__peerbitNativePreparedJoinCommit) {
3498
+ const nativeCommitStartedAt = internalProfileStart(profile);
3499
+ nativePreparedCommitted =
3500
+ (await resolvedOptions.__peerbitNativePreparedJoinCommit({
3501
+ entries,
3502
+ hashes: entryHashes,
3503
+ headFlags,
3504
+ headFlagsBytes,
3505
+ trustedMissing,
3506
+ validatePlan: nativeCommitValidatesPlan,
3507
+ })) === true;
3508
+ emitInternalProfileDuration(profile, nativeCommitStartedAt, {
3509
+ name: "log.joinPreparedFacts.nativePreparedCommit",
3510
+ component: "log",
3511
+ entries: entries.length,
3512
+ messages: 1,
3513
+ details: { nativePreparedCommitted },
3514
+ });
3515
+ }
3516
+ if (nativeCommitValidatesPlan && !nativePreparedCommitted) {
3517
+ nativeValidatedCommitRejected = true;
3518
+ return;
3519
+ }
3520
+ const blocksStartedAt = internalProfileStart(profile);
3521
+ if (!nativePreparedCommitted) {
3522
+ await this.putKnownEntryBytesBatch(
3523
+ entries.map((entry) => ({
3524
+ cid: entry.hash,
3525
+ bytes: entry.bytes,
3526
+ })),
3527
+ );
3528
+ }
3529
+ emitInternalProfileDuration(profile, blocksStartedAt, {
3530
+ name: "log.joinPreparedFacts.blocks",
3531
+ component: "log",
3532
+ entries: entries.length,
3533
+ bytes: entries.reduce((sum, entry) => sum + entry.byteLength, 0),
3534
+ messages: 1,
3535
+ details: { nativePreparedCommitted },
3536
+ });
3537
+
3538
+ const indexStartedAt = internalProfileStart(profile);
3539
+ let nativeCommittedFactsIndexed = false;
3540
+ if (
3541
+ nativePreparedCommitted &&
3542
+ resolvedOptions.__peerbitDeferIndexWrite === true
3543
+ ) {
3544
+ const indexBatchHashes = entries.length > 1 ? batchHashes : undefined;
3545
+ const indexRows = entries.map((entry, index) => {
3546
+ const isHead = headFlags[index] ?? true;
3547
+ const externalNextHashes = indexBatchHashes
3548
+ ? entry.meta.next.filter((next) => !indexBatchHashes.has(next))
3549
+ : entry.meta.next;
3550
+ return {
3551
+ hash: entry.hash,
3552
+ unique: trustedMissing,
3553
+ externalNextHashes,
3554
+ getShallowEntry: () => {
3555
+ const shallowEntry =
3556
+ entry.shallowEntry ?? entry.getShallowEntry?.(isHead);
3557
+ if (!shallowEntry) {
3558
+ throw new Error("Missing prepared append shallow entry");
3559
+ }
3560
+ shallowEntry.head = isHead;
3561
+ entry.shallowEntry = shallowEntry;
3562
+ return shallowEntry;
3563
+ },
3564
+ isHead,
3565
+ };
3566
+ });
3567
+ await this.entryIndex.putNativeCommittedAppendFactsBatch(indexRows);
3568
+ nativeCommittedFactsIndexed = true;
3569
+ } else {
3570
+ const externalNextHashes =
3571
+ entries.length === 1 ? entries[0]!.meta.next : undefined;
3572
+ await this.entryIndex.putAppendFactsBatch(entries, {
3573
+ unique: trustedMissing,
3574
+ externalNextHashes,
3575
+ heads: headFlags,
3576
+ deferIndexWrite: resolvedOptions.__peerbitDeferIndexWrite,
3577
+ nativeGraphUpdated: nativePreparedCommitted,
3578
+ profile,
3579
+ });
3580
+ }
3581
+ emitInternalProfileDuration(profile, indexStartedAt, {
3582
+ name: "log.joinPreparedFacts.entryIndex",
3583
+ component: "log",
3584
+ entries: entries.length,
3585
+ messages: 1,
3586
+ details: { trustedMissing, nativeCommittedFactsIndexed },
3587
+ });
3588
+
3589
+ if (resolvedOptions.__peerbitOnPreparedJoinCommitted) {
3590
+ const committedStartedAt = internalProfileStart(profile);
3591
+ await resolvedOptions.__peerbitOnPreparedJoinCommitted({
3592
+ entries,
3593
+ hashes: entryHashes,
3594
+ headFlags,
3595
+ nativePreparedCommitted,
3596
+ });
3597
+ emitInternalProfileDuration(profile, committedStartedAt, {
3598
+ name: "log.joinPreparedFacts.committed",
3599
+ component: "log",
3600
+ entries: entries.length,
3601
+ messages: 1,
3602
+ details: { nativePreparedCommitted },
3603
+ });
3604
+ }
3605
+
3606
+ const changeStartedAt = internalProfileStart(profile);
3607
+ if (resolvedOptions.__peerbitOnAppendHashes) {
3608
+ await resolvedOptions.__peerbitOnAppendHashes(
3609
+ entries.map((entry) => entry.hash),
3610
+ );
3611
+ } else {
3612
+ const change: Change<T> = {
3613
+ added: entries.map((entry, index) => {
3614
+ const materializeEntry = entry.materializeEntry;
3615
+ if (!materializeEntry) {
3616
+ throw new Error("Missing prepared append materializer");
3617
+ }
3618
+ return {
3619
+ head: headFlags[index]!,
3620
+ entry: materializeEntry() as Entry<T>,
3621
+ };
3622
+ }),
3623
+ removed: [],
3624
+ };
3625
+ await this._onChange?.(change);
3626
+ }
3627
+ emitInternalProfileDuration(profile, changeStartedAt, {
3628
+ name: "log.joinPreparedFacts.change",
3629
+ component: "log",
3630
+ entries: entries.length,
3631
+ messages: 1,
3632
+ details: { hashOnly: !!resolvedOptions.__peerbitOnAppendHashes },
3633
+ });
3634
+ })().finally(() => {
3635
+ for (const entry of entries) {
3636
+ this._joining.delete(entry.hash);
3637
+ }
3638
+ });
3639
+
3640
+ for (const entry of entries) {
3641
+ this._joining.set(entry.hash, batchPromise);
3642
+ }
3643
+ await batchPromise;
3644
+ if (nativeValidatedCommitRejected) {
3645
+ return false;
3646
+ }
3647
+ return true;
3648
+ }
3649
+
3650
+ private async tryJoinIndependentAppendBatch(
3651
+ entries: Entry<T>[],
3652
+ heads: Map<string, boolean>,
3653
+ options: TrustedJoinOptions<T>,
3654
+ ): Promise<boolean> {
3655
+ if (
3656
+ entries.length < 2 ||
3657
+ options.reset ||
3658
+ options.trim ||
3659
+ options.verifySignatures ||
3660
+ entries.some((entry) => this._joining.has(entry.hash))
3661
+ ) {
3662
+ return false;
3663
+ }
3664
+
3665
+ const profile = options.__peerbitProfile;
3666
+ const prepareStartedAt = internalProfileStart(profile);
3667
+ const batchHashes = new Set(entries.map((entry) => entry.hash));
3668
+ const headFlags: boolean[] = [];
3669
+ for (const entry of entries) {
3670
+ if (!entry.hash || !Entry.hasPreparedBlock(entry)) {
3671
+ return false;
3672
+ }
3673
+ entry.init(this);
3674
+ if (entry.meta.type !== EntryType.APPEND) {
3675
+ return false;
3676
+ }
3677
+ }
3678
+ emitInternalProfileDuration(profile, prepareStartedAt, {
3679
+ name: "log.joinIndependent.prepare",
3680
+ component: "log",
3681
+ entries: entries.length,
3682
+ messages: 1,
3683
+ });
3684
+
3685
+ const planStartedAt = internalProfileStart(profile);
3686
+ const joinPlans = await this.entryIndex.planJoinBatch(
3687
+ entries,
3688
+ false,
3689
+ profile,
3690
+ );
3691
+ emitInternalProfileDuration(profile, planStartedAt, {
3692
+ name: "log.joinIndependent.plan",
3693
+ component: "log",
3694
+ entries: entries.length,
3695
+ messages: 1,
3696
+ });
3697
+ const validatePlanStartedAt = internalProfileStart(profile);
3698
+ for (let i = 0; i < entries.length; i++) {
3699
+ const entry = entries[i]!;
3700
+ const joinPlan = joinPlans[i]!;
3701
+ if (
3702
+ joinPlan.skip ||
3703
+ joinPlan.coveredByCut ||
3704
+ !joinPlan.cutChecked ||
3705
+ joinPlan.missingParents.some((hash) => !batchHashes.has(hash))
3706
+ ) {
3707
+ return false;
3708
+ }
3709
+ headFlags.push(heads.get(entry.hash) ?? true);
3710
+ }
3711
+ emitInternalProfileDuration(profile, validatePlanStartedAt, {
3712
+ name: "log.joinIndependent.validatePlan",
3713
+ component: "log",
3714
+ entries: entries.length,
3715
+ messages: 1,
3716
+ });
3717
+
3718
+ if (!canAppendAlreadyValidated(options)) {
3719
+ const canAppendStartedAt = internalProfileStart(profile);
3720
+ for (const entry of entries) {
3721
+ if (this._canAppend && !(await this._canAppend(entry))) {
3722
+ return false;
3723
+ }
3724
+ }
3725
+ emitInternalProfileDuration(profile, canAppendStartedAt, {
3726
+ name: "log.joinIndependent.canAppend",
3727
+ component: "log",
3728
+ entries: entries.length,
3729
+ messages: 1,
3730
+ });
3731
+ }
3732
+
3733
+ const preparedBatch = this.takePreparedIndependentAppendBatch(
3734
+ entries,
3735
+ headFlags,
3736
+ );
3737
+ if (!preparedBatch) {
3738
+ return false;
3739
+ }
3740
+
3741
+ const batchPromise = (async () => {
3742
+ const clockStartedAt = internalProfileStart(profile);
3743
+ for (const entry of entries) {
3744
+ this._hlc.update(entry.meta.clock.timestamp);
3745
+ }
3746
+ emitInternalProfileDuration(profile, clockStartedAt, {
3747
+ name: "log.joinIndependent.clock",
3748
+ component: "log",
3749
+ entries: entries.length,
3750
+ messages: 1,
3751
+ });
3752
+
3753
+ const blocksStartedAt = internalProfileStart(profile);
3754
+ await this.putAppendEntryBlocks(entries, preparedBatch.blocks);
3755
+ emitInternalProfileDuration(profile, blocksStartedAt, {
3756
+ name: "log.joinIndependent.blocks",
3757
+ component: "log",
3758
+ entries: entries.length,
3759
+ bytes: entries.reduce((sum, entry) => sum + (entry.size ?? 0), 0),
3760
+ messages: 1,
3761
+ });
3762
+ const indexStartedAt = internalProfileStart(profile);
3763
+ const trustedMissing =
3764
+ options.__peerbitEntriesAlreadyMissing === true &&
3765
+ batchHashes.size === entries.length;
3766
+ await this.entryIndex.putAppendBatch(entries, {
3767
+ unique: trustedMissing,
3768
+ heads: headFlags,
3769
+ prepared: preparedBatch.prepared,
3770
+ deferIndexWrite: options.__peerbitDeferIndexWrite,
3771
+ profile,
3772
+ });
3773
+ emitInternalProfileDuration(profile, indexStartedAt, {
3774
+ name: "log.joinIndependent.entryIndex",
3775
+ component: "log",
3776
+ entries: entries.length,
3777
+ messages: 1,
3778
+ details: { trustedMissing },
3779
+ });
3780
+
3781
+ const changeStartedAt = internalProfileStart(profile);
3782
+ if (options.__peerbitOnAppendHashes && !options.onChange) {
3783
+ await options.__peerbitOnAppendHashes(
3784
+ entries.map((entry) => entry.hash),
3785
+ );
3786
+ } else {
3787
+ const change: Change<T> = {
3788
+ added: entries.map((entry, index) => ({
3789
+ head: headFlags[index]!,
3790
+ entry,
3791
+ })),
3792
+ removed: [],
3793
+ };
3794
+ await options.onChange?.(change);
3795
+ await this._onChange?.(change);
3796
+ }
3797
+ emitInternalProfileDuration(profile, changeStartedAt, {
3798
+ name: "log.joinIndependent.change",
3799
+ component: "log",
3800
+ entries: entries.length,
3801
+ messages: 1,
3802
+ details: {
3803
+ hashOnly: !!options.__peerbitOnAppendHashes && !options.onChange,
3804
+ },
3805
+ });
3806
+ })().finally(() => {
3807
+ for (const entry of entries) {
3808
+ this._joining.delete(entry.hash);
3809
+ }
3810
+ });
3811
+
3812
+ for (const entry of entries) {
3813
+ this._joining.set(entry.hash, batchPromise);
3814
+ }
3815
+ await batchPromise;
3816
+ return true;
3817
+ }
3818
+
3819
+ private takePreparedIndependentAppendBatch(
3820
+ entries: Entry<T>[],
3821
+ headFlags: boolean[],
3822
+ ): PreparedIndependentAppendBatch | undefined {
3823
+ if (!entries.every((entry) => Entry.hasPreparedBlock(entry))) {
3824
+ return;
3825
+ }
3826
+ const hasPreparedShallowEntries = entries.every((entry) =>
3827
+ Entry.hasPreparedShallowEntry(entry),
3828
+ );
3829
+ const hasPreparedNativeEntries =
3830
+ hasPreparedShallowEntries &&
3831
+ entries.every((entry) => Entry.hasPreparedNativeLogEntry(entry));
3832
+
3833
+ const blocks = entries.map((entry) => {
3834
+ const prepared = Entry.takePreparedBlock(entry);
3835
+ if (!prepared) {
3836
+ throw new Error("Missing prepared entry block");
3837
+ }
3838
+ return prepared;
3839
+ });
3840
+ if (!hasPreparedShallowEntries) {
3841
+ return { blocks };
3842
+ }
3843
+
3844
+ const shallowEntries = entries.map((entry, index) => {
3845
+ const shallowEntry = Entry.takePreparedShallowEntry(
3846
+ entry,
3847
+ headFlags[index] ?? true,
3848
+ );
3849
+ if (!shallowEntry) {
3850
+ throw new Error("Missing prepared shallow entry");
3851
+ }
3852
+ return shallowEntry;
3853
+ });
3854
+ const nativeEntries = hasPreparedNativeEntries
3855
+ ? entries.map((entry, index) => {
3856
+ const nativeEntry = Entry.takePreparedNativeLogEntry(
3857
+ entry,
3858
+ headFlags[index] ?? true,
3859
+ );
3860
+ if (!nativeEntry) {
3861
+ throw new Error("Missing prepared native log entry");
3862
+ }
3863
+ return nativeEntry;
3864
+ })
3865
+ : undefined;
3866
+
3867
+ return {
3868
+ blocks,
3869
+ prepared: {
3870
+ shallowEntries,
3871
+ nativeEntries,
3872
+ },
3873
+ };
3874
+ }
3875
+
880
3876
  /**
881
3877
  * Bottom up join of entries into the log
882
3878
  * @param entry
@@ -909,10 +3905,8 @@ export class Log<T> {
909
3905
  throw new Error("Unexpected");
910
3906
  }
911
3907
 
912
- if (
913
- (await this.entryIndex.getShallow(entry.hash)) != null &&
914
- !options.reset
915
- ) {
3908
+ const joinPlan = await this.entryIndex.planJoin(entry, options.reset);
3909
+ if (joinPlan.skip) {
916
3910
  return false;
917
3911
  }
918
3912
 
@@ -924,10 +3918,14 @@ export class Log<T> {
924
3918
  }
925
3919
  }
926
3920
 
927
- const headsWithGid: JoinableEntry[] = await this.entryIndex
928
- .getHeads(entry.meta.gid, { type: "shape", shape: ENTRY_JOIN_SHAPE })
929
- .all();
930
- if (headsWithGid) {
3921
+ if (joinPlan.coveredByCut) {
3922
+ return false;
3923
+ }
3924
+
3925
+ if (!joinPlan.cutChecked) {
3926
+ const headsWithGid: JoinableEntry[] = await this.entryIndex.getJoinHeads(
3927
+ entry.meta.gid,
3928
+ );
931
3929
  for (const v of headsWithGid) {
932
3930
  // TODO second argument should be a time compare instead? what about next nexts?
933
3931
  // and check the cut entry is newer than the current 'entry'
@@ -946,29 +3944,57 @@ export class Log<T> {
946
3944
  options.remote && typeof options.remote === "object"
947
3945
  ? options.remote
948
3946
  : undefined;
3947
+ const parents: Array<{ hash: string; entry?: Entry<T> }> = [];
3948
+ const unresolvedParentHashes: string[] = [];
949
3949
 
950
- for (const a of entry.meta.next) {
3950
+ for (const a of joinPlan.missingParents) {
951
3951
  const prev = this._joining.get(a);
952
3952
  if (prev) {
953
3953
  await prev;
954
3954
  continue;
955
3955
  }
956
- if ((await this.entryIndex.getShallow(a)) != null && !options.reset) {
3956
+
3957
+ const referenced = options.references?.get(a);
3958
+ parents.push({ hash: a, entry: referenced });
3959
+ if (!referenced) {
3960
+ unresolvedParentHashes.push(a);
3961
+ }
3962
+ }
3963
+
3964
+ const localParents =
3965
+ unresolvedParentHashes.length > 0
3966
+ ? await this.entryIndex.getMany(unresolvedParentHashes, {
3967
+ type: "full",
3968
+ ignoreMissing: true,
3969
+ })
3970
+ : [];
3971
+ const localParentByHash = new Map<string, Entry<T>>();
3972
+ for (const parent of localParents) {
3973
+ if (parent) {
3974
+ localParentByHash.set(parent.hash, parent);
3975
+ }
3976
+ }
3977
+
3978
+ for (const parent of parents) {
3979
+ const a = parent.hash;
3980
+ const prev = this._joining.get(a);
3981
+ if (prev) {
3982
+ await prev;
957
3983
  continue;
958
3984
  }
959
3985
 
960
- const from = await options.resolveRemoteFrom?.(a, remote?.signal);
961
- let nested: Entry<T>;
3986
+ let nested = parent.entry ?? localParentByHash.get(a);
962
3987
  try {
963
- nested =
964
- options.references?.get(a) ??
965
- (await Entry.fromMultihash<T>(this._storage, a, {
3988
+ if (!nested) {
3989
+ const from = await options.resolveRemoteFrom?.(a, remote?.signal);
3990
+ nested = await Entry.fromMultihash<T>(this._storage, a, {
966
3991
  remote: {
967
3992
  timeout: remote?.timeout,
968
3993
  signal: remote?.signal,
969
3994
  ...(from && from.length > 0 ? { from } : {}),
970
3995
  },
971
- }));
3996
+ });
3997
+ }
972
3998
  } catch (error) {
973
3999
  if (isRecoverableJoinResolveError(error)) {
974
4000
  return false;
@@ -1003,9 +4029,9 @@ export class Log<T> {
1003
4029
 
1004
4030
  const pendingDeletes: (
1005
4031
  | PendingDelete<T>
1006
- | { entry: Entry<T>; fn: undefined }
4032
+ | { entry: ShallowOrFullEntry<T>; fn: undefined }
1007
4033
  )[] = await this.processEntry(entry);
1008
- const trimmed = await this.trim(options.trim);
4034
+ const trimmed = await this.trimIfConfigured(options.trim);
1009
4035
 
1010
4036
  if (trimmed) {
1011
4037
  for (const removedEntry of trimmed) {
@@ -1064,7 +4090,22 @@ export class Log<T> {
1064
4090
  | { hash: string; meta: { next: string[] } }[],
1065
4091
  skipFirst = false,
1066
4092
  ) {
1067
- const stack = Array.isArray(from) ? [...from] : [from];
4093
+ const entries = Array.isArray(from) ? [...from] : [from];
4094
+ const nativeDeletePlan = this.entryIndex.planDeleteRecursively(
4095
+ entries,
4096
+ skipFirst,
4097
+ );
4098
+ if (nativeDeletePlan) {
4099
+ const toDelete: PendingDelete<T>[] = [];
4100
+ for (const hash of nativeDeletePlan) {
4101
+ const deleteFn = await this.prepareDelete(hash);
4102
+ deleteFn.entry &&
4103
+ toDelete.push({ entry: deleteFn.entry, fn: deleteFn.fn });
4104
+ }
4105
+ return toDelete;
4106
+ }
4107
+
4108
+ const stack = entries;
1068
4109
  const promises: (Promise<void> | void)[] = [];
1069
4110
  let counter = 0;
1070
4111
  const toDelete: PendingDelete<T>[] = [];
@@ -1079,8 +4120,7 @@ export class Log<T> {
1079
4120
  }
1080
4121
 
1081
4122
  for (const next of entry.meta.next) {
1082
- const nextFromNext = this.entryIndex.getHasNext(next);
1083
- const entriesThatHasNext = await nextFromNext.all();
4123
+ const entriesThatHasNext = await this.entryIndex.getJoinChildren(next);
1084
4124
 
1085
4125
  // if there are no entries which is not of "CUT" type, we can safely delete the next entry
1086
4126
  // figureately speaking, these means where are cutting all branches to a stem, so we can delete the stem as well
@@ -1163,8 +4203,8 @@ export class Log<T> {
1163
4203
  async close() {
1164
4204
  // Don't return early here if closed = true, because "load" might create processes that needs to be closed
1165
4205
  this._closed = true; // closed = true before doing below, else we might try to open the headsIndex cache because it is closed as we assume log is still open
1166
- this._closeController.abort();
1167
- await this._entryIndex.flushPendingWrites();
4206
+ this._closeController?.abort();
4207
+ await this._entryIndex?.flushPendingWrites();
1168
4208
  await this._indexer?.stop?.();
1169
4209
  this._indexer = undefined as any;
1170
4210
  this._loadedOnce = false;
@@ -1174,7 +4214,7 @@ export class Log<T> {
1174
4214
  // Don't return early here if closed = true, because "load" might create processes that needs to be closed
1175
4215
  this._closed = true; // closed = true before doing below, else we might try to open the headsIndex cache because it is closed as we assume log is still open
1176
4216
  this._closeController.abort();
1177
- await this.entryIndex?.clear();
4217
+ await this._entryIndex?.clear();
1178
4218
  await this._indexer?.drop();
1179
4219
  await this._indexer?.stop?.();
1180
4220
  }