@peerbit/log 6.1.1 → 6.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/benchmark/native-graph.js +447 -5
- package/dist/benchmark/native-graph.js.map +1 -1
- package/dist/src/clock.d.ts +1 -0
- package/dist/src/clock.d.ts.map +1 -1
- package/dist/src/clock.js +32 -0
- package/dist/src/clock.js.map +1 -1
- package/dist/src/entry-create.d.ts +1 -0
- package/dist/src/entry-create.d.ts.map +1 -1
- package/dist/src/entry-create.js.map +1 -1
- package/dist/src/entry-index.d.ts +288 -12
- package/dist/src/entry-index.d.ts.map +1 -1
- package/dist/src/entry-index.js +1478 -33
- package/dist/src/entry-index.js.map +1 -1
- package/dist/src/entry-v0.d.ts +184 -1
- package/dist/src/entry-v0.d.ts.map +1 -1
- package/dist/src/entry-v0.js +1021 -12
- package/dist/src/entry-v0.js.map +1 -1
- package/dist/src/entry.d.ts +73 -1
- package/dist/src/entry.d.ts.map +1 -1
- package/dist/src/entry.js +124 -1
- package/dist/src/entry.js.map +1 -1
- package/dist/src/index.d.ts +1 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +1 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/log.d.ts +62 -13
- package/dist/src/log.d.ts.map +1 -1
- package/dist/src/log.js +2282 -100
- package/dist/src/log.js.map +1 -1
- package/dist/src/runtime.d.ts +2 -0
- package/dist/src/runtime.d.ts.map +1 -0
- package/dist/src/runtime.js +10 -0
- package/dist/src/runtime.js.map +1 -0
- package/dist/src/trim.d.ts +14 -2
- package/dist/src/trim.d.ts.map +1 -1
- package/dist/src/trim.js +92 -4
- package/dist/src/trim.js.map +1 -1
- package/package.json +18 -15
- package/src/clock.ts +34 -0
- package/src/entry-create.ts +1 -0
- package/src/entry-index.ts +2232 -68
- package/src/entry-v0.ts +1488 -13
- package/src/entry.ts +235 -1
- package/src/index.ts +7 -1
- package/src/log.ts +3246 -206
- package/src/runtime.ts +16 -0
- package/src/trim.ts +133 -7
package/src/log.ts
CHANGED
|
@@ -6,6 +6,7 @@ import {
|
|
|
6
6
|
cidifyString,
|
|
7
7
|
} from "@peerbit/blocks-interface";
|
|
8
8
|
import {
|
|
9
|
+
Ed25519Keypair,
|
|
9
10
|
type Identity,
|
|
10
11
|
SignatureWithKey,
|
|
11
12
|
X25519Keypair,
|
|
@@ -26,21 +27,206 @@ import {
|
|
|
26
27
|
EntryIndex,
|
|
27
28
|
type MaybeResolveOptions,
|
|
28
29
|
type NativeLogGraph,
|
|
30
|
+
type PreparedAppendIndexFacts,
|
|
29
31
|
type ResultsIterator,
|
|
30
32
|
type ReturnTypeFromResolveOptions,
|
|
31
33
|
} from "./entry-index.js";
|
|
32
|
-
import { ShallowEntry } from "./entry-shallow.js";
|
|
34
|
+
import { ShallowEntry, ShallowMeta } from "./entry-shallow.js";
|
|
33
35
|
import { EntryType } from "./entry-type.js";
|
|
34
36
|
import { type EncryptionTemplateMaybeEncrypted, EntryV0 } from "./entry-v0.js";
|
|
35
37
|
import { type EntryWithRefs } from "./entry-with-refs.js";
|
|
36
|
-
import {
|
|
38
|
+
import {
|
|
39
|
+
type CanAppend,
|
|
40
|
+
Entry,
|
|
41
|
+
type PreparedAppendChain,
|
|
42
|
+
type PreparedAppendCommitOnlyChain,
|
|
43
|
+
type PreparedAppendFacts,
|
|
44
|
+
type PreparedEntryBlock,
|
|
45
|
+
type PreparedNativeLogEntry,
|
|
46
|
+
type ShallowOrFullEntry,
|
|
47
|
+
} from "./entry.js";
|
|
37
48
|
import { findUniques } from "./find-uniques.js";
|
|
49
|
+
import { logger as baseLogger } from "./logger.js";
|
|
38
50
|
import * as LogError from "./log-errors.js";
|
|
39
51
|
import * as Sorting from "./log-sorting.js";
|
|
40
52
|
import type { Payload } from "./payload.js";
|
|
53
|
+
import { canUseOptionalNativeModuleImports } from "./runtime.js";
|
|
41
54
|
import { Trim, type TrimOptions } from "./trim.js";
|
|
42
55
|
|
|
43
56
|
const { LastWriteWins } = Sorting;
|
|
57
|
+
const warn = baseLogger.newScope("warn");
|
|
58
|
+
|
|
59
|
+
type BlocksWithPutMany = Blocks & {
|
|
60
|
+
putMany?: (blocks: PreparedEntryBlock[]) => Promise<string[]> | string[];
|
|
61
|
+
rmMany?: (cids: string[]) => Promise<number | void> | number | void;
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
type BlocksWithPutKnownMany = Blocks & {
|
|
65
|
+
putKnownMany: (
|
|
66
|
+
blocks: Array<readonly [cid: string, bytes: Uint8Array]>,
|
|
67
|
+
) => Promise<string[]> | string[];
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
type BlocksWithPutKnownManyColumns = Blocks & {
|
|
71
|
+
putKnownManyColumns: (
|
|
72
|
+
cids: string[],
|
|
73
|
+
bytes: Uint8Array[],
|
|
74
|
+
) => Promise<string[]> | string[];
|
|
75
|
+
};
|
|
76
|
+
|
|
77
|
+
type BlocksWithPutKnown = Blocks & {
|
|
78
|
+
putKnown: (cid: string, bytes: Uint8Array) => Promise<string> | string;
|
|
79
|
+
};
|
|
80
|
+
|
|
81
|
+
const hasPutMany = (storage: Blocks): storage is BlocksWithPutMany =>
|
|
82
|
+
typeof (storage as BlocksWithPutMany).putMany === "function";
|
|
83
|
+
|
|
84
|
+
const hasPutKnown = (storage: Blocks): storage is BlocksWithPutKnown =>
|
|
85
|
+
typeof (storage as BlocksWithPutKnown).putKnown === "function";
|
|
86
|
+
|
|
87
|
+
const hasPutKnownMany = (storage: Blocks): storage is BlocksWithPutKnownMany =>
|
|
88
|
+
typeof (storage as BlocksWithPutKnownMany).putKnownMany === "function";
|
|
89
|
+
|
|
90
|
+
const hasPutKnownManyColumns = (
|
|
91
|
+
storage: Blocks,
|
|
92
|
+
): storage is BlocksWithPutKnownManyColumns =>
|
|
93
|
+
typeof (storage as BlocksWithPutKnownManyColumns).putKnownManyColumns ===
|
|
94
|
+
"function";
|
|
95
|
+
|
|
96
|
+
type MaybePromise<T> = T | Promise<T>;
|
|
97
|
+
|
|
98
|
+
type InternalProfileValue = string | number | boolean | undefined;
|
|
99
|
+
type InternalProfileEvent = {
|
|
100
|
+
name: string;
|
|
101
|
+
component?: string;
|
|
102
|
+
durationMs?: number;
|
|
103
|
+
entries?: number;
|
|
104
|
+
bytes?: number;
|
|
105
|
+
messages?: number;
|
|
106
|
+
count?: number;
|
|
107
|
+
details?: Record<string, InternalProfileValue>;
|
|
108
|
+
};
|
|
109
|
+
type InternalProfileSink = (event: InternalProfileEvent) => void;
|
|
110
|
+
type InternalAppendHashesSink = (hashes: string[]) => void | Promise<void>;
|
|
111
|
+
|
|
112
|
+
const internalProfileNow = () => globalThis.performance?.now?.() ?? Date.now();
|
|
113
|
+
const internalProfileStart = (sink: InternalProfileSink | undefined) =>
|
|
114
|
+
sink ? internalProfileNow() : 0;
|
|
115
|
+
const emitInternalProfileDuration = (
|
|
116
|
+
sink: InternalProfileSink | undefined,
|
|
117
|
+
startedAt: number,
|
|
118
|
+
event: Omit<InternalProfileEvent, "durationMs">,
|
|
119
|
+
) => {
|
|
120
|
+
if (!sink) {
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
sink({
|
|
124
|
+
...event,
|
|
125
|
+
durationMs: internalProfileNow() - startedAt,
|
|
126
|
+
});
|
|
127
|
+
};
|
|
128
|
+
const EMPTY_NEXT_HASHES: string[] = [];
|
|
129
|
+
const EMPTY_NEXT_ENTRIES: Sorting.SortableEntry[] = [];
|
|
130
|
+
const normalizedUniqueStrings = (values: string[]): string[] =>
|
|
131
|
+
values.length <= 1 ? values : [...new Set(values)];
|
|
132
|
+
|
|
133
|
+
type PreparedCommitOnlyAppendResult<T> = {
|
|
134
|
+
entry: Entry<T>;
|
|
135
|
+
materializeEntry: () => Entry<T>;
|
|
136
|
+
removed: ShallowOrFullEntry<T>[];
|
|
137
|
+
removedHashes?: string[];
|
|
138
|
+
appendFacts: PreparedAppendFacts;
|
|
139
|
+
shallowEntry: ShallowEntry;
|
|
140
|
+
documentTrimmedHeadsProcessed?: boolean;
|
|
141
|
+
documentPreviousContext?: {
|
|
142
|
+
created: bigint;
|
|
143
|
+
modified: bigint;
|
|
144
|
+
head: string;
|
|
145
|
+
gid: string;
|
|
146
|
+
size: number;
|
|
147
|
+
};
|
|
148
|
+
};
|
|
149
|
+
|
|
150
|
+
type PreparedCommitOnlyAppendBatchResult<T> = {
|
|
151
|
+
entries: Entry<T>[];
|
|
152
|
+
materializeEntries: Array<() => Entry<T>>;
|
|
153
|
+
removed: ShallowOrFullEntry<T>[];
|
|
154
|
+
removedHashes?: string[];
|
|
155
|
+
appendFacts: PreparedAppendFacts[];
|
|
156
|
+
documentTrimmedHeadsProcessed?: boolean[];
|
|
157
|
+
};
|
|
158
|
+
|
|
159
|
+
type PreparedIndependentAppendBatch = {
|
|
160
|
+
blocks: PreparedEntryBlock[];
|
|
161
|
+
prepared?: {
|
|
162
|
+
shallowEntries: ShallowEntry[];
|
|
163
|
+
nativeEntries?: PreparedNativeLogEntry[];
|
|
164
|
+
};
|
|
165
|
+
};
|
|
166
|
+
|
|
167
|
+
type PreparedJoinNativeCommitInput = {
|
|
168
|
+
entries: PreparedAppendJoinFacts[];
|
|
169
|
+
hashes: string[];
|
|
170
|
+
headFlags: boolean[];
|
|
171
|
+
headFlagsBytes: Uint8Array;
|
|
172
|
+
trustedMissing: boolean;
|
|
173
|
+
validatePlan?: boolean;
|
|
174
|
+
};
|
|
175
|
+
|
|
176
|
+
type PreparedJoinCommittedInput = {
|
|
177
|
+
entries: PreparedAppendJoinFacts[];
|
|
178
|
+
hashes: string[];
|
|
179
|
+
headFlags: boolean[];
|
|
180
|
+
nativePreparedCommitted: boolean;
|
|
181
|
+
};
|
|
182
|
+
|
|
183
|
+
export type PreparedAppendJoinFacts = PreparedAppendIndexFacts & {
|
|
184
|
+
bytes: Uint8Array;
|
|
185
|
+
byteLength: number;
|
|
186
|
+
materializeEntry?: () => Entry<any>;
|
|
187
|
+
};
|
|
188
|
+
|
|
189
|
+
type NativePreparedNoNextCommit = {
|
|
190
|
+
bytes?: Uint8Array;
|
|
191
|
+
getBytes?: (hash: string) => Uint8Array | undefined;
|
|
192
|
+
cid?: string;
|
|
193
|
+
hash?: string;
|
|
194
|
+
gid?: string;
|
|
195
|
+
next?: string[];
|
|
196
|
+
byteLength: number;
|
|
197
|
+
metaBytes?: Uint8Array;
|
|
198
|
+
hashDigestBytes?: Uint8Array;
|
|
199
|
+
trimmedEntries?: PreparedNativeLogEntry[];
|
|
200
|
+
trimmedEntryHashes?: string[];
|
|
201
|
+
documentTrimmedHeadsProcessed?: boolean;
|
|
202
|
+
documentPreviousContext?: PreparedCommitOnlyAppendResult<unknown>["documentPreviousContext"];
|
|
203
|
+
};
|
|
204
|
+
|
|
205
|
+
type NativeNoNextCommitInput = {
|
|
206
|
+
clockId: Uint8Array;
|
|
207
|
+
privateKey: Uint8Array;
|
|
208
|
+
publicKey: Uint8Array;
|
|
209
|
+
wallTime: bigint;
|
|
210
|
+
logical: number;
|
|
211
|
+
gid: string;
|
|
212
|
+
type: EntryType;
|
|
213
|
+
metaData?: Uint8Array;
|
|
214
|
+
payloadData: Uint8Array;
|
|
215
|
+
resolveTrimmedEntries?: boolean;
|
|
216
|
+
trimLengthTo?: number;
|
|
217
|
+
};
|
|
218
|
+
|
|
219
|
+
type NativeCommitInput = NativeNoNextCommitInput & {
|
|
220
|
+
next: string[];
|
|
221
|
+
};
|
|
222
|
+
|
|
223
|
+
const isPromiseLike = <T>(value: Promise<T> | T): value is Promise<T> =>
|
|
224
|
+
typeof (value as { then?: unknown })?.then === "function";
|
|
225
|
+
|
|
226
|
+
const mapMaybePromise = <T, R>(
|
|
227
|
+
value: MaybePromise<T>,
|
|
228
|
+
fn: (value: T) => MaybePromise<R>,
|
|
229
|
+
): MaybePromise<R> => (isPromiseLike(value) ? value.then(fn) : fn(value));
|
|
44
230
|
|
|
45
231
|
const getErrorName = (error: unknown) =>
|
|
46
232
|
typeof (error as { name?: unknown })?.name === "string"
|
|
@@ -89,12 +275,18 @@ export type MemoryProperties = {
|
|
|
89
275
|
indexer?: Indices;
|
|
90
276
|
};
|
|
91
277
|
|
|
278
|
+
export type NativeGraphOptions = {
|
|
279
|
+
heads?: boolean;
|
|
280
|
+
optional?: boolean;
|
|
281
|
+
graph?: NativeLogGraph;
|
|
282
|
+
};
|
|
283
|
+
|
|
92
284
|
export type LogProperties<T> = {
|
|
93
285
|
keychain?: CryptoKeychain;
|
|
94
286
|
encoding?: Encoding<T>;
|
|
95
287
|
clock?: LamportClock;
|
|
96
288
|
appendDurability?: AppendDurability;
|
|
97
|
-
nativeGraph?: boolean |
|
|
289
|
+
nativeGraph?: boolean | NativeGraphOptions;
|
|
98
290
|
sortFn?: Sorting.SortFn;
|
|
99
291
|
trim?: TrimOptions;
|
|
100
292
|
canAppend?: CanAppend<T>;
|
|
@@ -133,11 +325,63 @@ export type AppendOptions<T> = {
|
|
|
133
325
|
canAppend?: CanAppend<T>;
|
|
134
326
|
};
|
|
135
327
|
|
|
328
|
+
type TrustedAppendOptions<T> = AppendOptions<T> & {
|
|
329
|
+
__peerbitCanAppendAlreadyValidated?: boolean;
|
|
330
|
+
};
|
|
331
|
+
|
|
332
|
+
const canAppendAlreadyValidated = (options?: unknown): boolean =>
|
|
333
|
+
(options as { __peerbitCanAppendAlreadyValidated?: unknown } | undefined)
|
|
334
|
+
?.__peerbitCanAppendAlreadyValidated === true;
|
|
335
|
+
|
|
336
|
+
const withCanAppendAlreadyValidated = <T>(
|
|
337
|
+
options: AppendOptions<T> = {},
|
|
338
|
+
): TrustedAppendOptions<T> =>
|
|
339
|
+
canAppendAlreadyValidated(options)
|
|
340
|
+
? (options as TrustedAppendOptions<T>)
|
|
341
|
+
: {
|
|
342
|
+
...options,
|
|
343
|
+
__peerbitCanAppendAlreadyValidated: true,
|
|
344
|
+
};
|
|
345
|
+
|
|
346
|
+
export type { PreparedAppendFacts } from "./entry.js";
|
|
347
|
+
|
|
136
348
|
type OnChange<T> = (
|
|
137
349
|
change: Change<T>,
|
|
138
350
|
reference?: undefined,
|
|
139
351
|
) => void | Promise<void>;
|
|
140
352
|
|
|
353
|
+
export type JoinOptions<T> = {
|
|
354
|
+
verifySignatures?: boolean;
|
|
355
|
+
trim?: TrimOptions;
|
|
356
|
+
timeout?: number;
|
|
357
|
+
onChange?: OnChange<T>;
|
|
358
|
+
reset?: boolean;
|
|
359
|
+
};
|
|
360
|
+
|
|
361
|
+
type TrustedJoinOptions<T> = JoinOptions<T> & {
|
|
362
|
+
__peerbitBatchIndependent?: boolean;
|
|
363
|
+
__peerbitEntriesAlreadyMissing?: boolean;
|
|
364
|
+
__peerbitCanAppendAlreadyValidated?: boolean;
|
|
365
|
+
__peerbitOnAppendHashes?: InternalAppendHashesSink;
|
|
366
|
+
__peerbitDeferIndexWrite?: boolean;
|
|
367
|
+
__peerbitProfile?: InternalProfileSink;
|
|
368
|
+
};
|
|
369
|
+
|
|
370
|
+
type TrustedPreparedAppendFactsBatchJoinOptions = {
|
|
371
|
+
__peerbitEntriesAlreadyMissing?: boolean;
|
|
372
|
+
__peerbitCanAppendAlreadyValidated?: boolean;
|
|
373
|
+
__peerbitOnAppendHashes?: InternalAppendHashesSink;
|
|
374
|
+
__peerbitDeferIndexWrite?: boolean;
|
|
375
|
+
__peerbitProfile?: InternalProfileSink;
|
|
376
|
+
__peerbitNativePreparedJoinCommit?: (
|
|
377
|
+
input: PreparedJoinNativeCommitInput,
|
|
378
|
+
) => MaybePromise<boolean>;
|
|
379
|
+
__peerbitNativePreparedJoinCommitValidatesPlan?: boolean;
|
|
380
|
+
__peerbitOnPreparedJoinCommitted?: (
|
|
381
|
+
input: PreparedJoinCommittedInput,
|
|
382
|
+
) => MaybePromise<void>;
|
|
383
|
+
};
|
|
384
|
+
|
|
141
385
|
export type JoinableEntry = {
|
|
142
386
|
meta: {
|
|
143
387
|
clock: {
|
|
@@ -160,6 +404,11 @@ type PendingDelete<T> = {
|
|
|
160
404
|
fn: () => Promise<ShallowEntry | undefined>;
|
|
161
405
|
};
|
|
162
406
|
|
|
407
|
+
type EntryWithMetaBytes = {
|
|
408
|
+
getMetaBytes?: () => Uint8Array | undefined;
|
|
409
|
+
getHashDigestBytes?: () => Uint8Array | undefined;
|
|
410
|
+
};
|
|
411
|
+
|
|
163
412
|
@variant(0)
|
|
164
413
|
export class Log<T> {
|
|
165
414
|
@field({ type: fixedArray("u8", 32) })
|
|
@@ -191,9 +440,11 @@ export class Log<T> {
|
|
|
191
440
|
private _appendDurability!: AppendDurability;
|
|
192
441
|
private _joining!: Map<string, Promise<any>>; // entry hashes that are currently joining into this log
|
|
193
442
|
private _sortFn!: Sorting.SortFn;
|
|
443
|
+
private _hasCustomCanAppend = false;
|
|
194
444
|
|
|
195
445
|
constructor(properties?: { id?: Uint8Array }) {
|
|
196
446
|
this._id = properties?.id || randomBytes(32);
|
|
447
|
+
this._closeController = new AbortController();
|
|
197
448
|
}
|
|
198
449
|
|
|
199
450
|
async open(store: Blocks, identity: Identity, options: LogOptions<T> = {}) {
|
|
@@ -245,29 +496,47 @@ export class Log<T> {
|
|
|
245
496
|
throw new Error("Id not set");
|
|
246
497
|
}
|
|
247
498
|
|
|
499
|
+
const nativeGraphOption = options.nativeGraph;
|
|
248
500
|
const nativeGraph =
|
|
249
|
-
|
|
501
|
+
nativeGraphOption &&
|
|
250
502
|
(await (async () => {
|
|
503
|
+
const nativeGraphOptions =
|
|
504
|
+
typeof nativeGraphOption === "object" ? nativeGraphOption : undefined;
|
|
505
|
+
if (nativeGraphOptions?.graph) {
|
|
506
|
+
const headsRequested = nativeGraphOptions.heads !== false;
|
|
507
|
+
return {
|
|
508
|
+
graph: nativeGraphOptions.graph,
|
|
509
|
+
useHeads: headsRequested && this._sortFn === LastWriteWins,
|
|
510
|
+
};
|
|
511
|
+
}
|
|
512
|
+
if (!canUseOptionalNativeModuleImports()) {
|
|
513
|
+
if (nativeGraphOptions?.optional === true) {
|
|
514
|
+
return undefined;
|
|
515
|
+
}
|
|
516
|
+
throw new Error(
|
|
517
|
+
"Log nativeGraph is unavailable in service worker contexts",
|
|
518
|
+
);
|
|
519
|
+
}
|
|
251
520
|
let createLogGraphIndex: () => Promise<NativeLogGraph>;
|
|
252
521
|
try {
|
|
253
522
|
({ createLogGraphIndex } = (await import(
|
|
254
|
-
["@peerbit", "log-rust"].join("/")
|
|
523
|
+
/* @vite-ignore */ ["@peerbit", "log-rust"].join("/")
|
|
255
524
|
)) as {
|
|
256
525
|
createLogGraphIndex: () => Promise<NativeLogGraph>;
|
|
257
526
|
});
|
|
527
|
+
const headsRequested = nativeGraphOptions?.heads !== false;
|
|
528
|
+
return {
|
|
529
|
+
graph: await createLogGraphIndex(),
|
|
530
|
+
useHeads: headsRequested && this._sortFn === LastWriteWins,
|
|
531
|
+
};
|
|
258
532
|
} catch {
|
|
533
|
+
if (nativeGraphOptions?.optional === true) {
|
|
534
|
+
return undefined;
|
|
535
|
+
}
|
|
259
536
|
throw new Error(
|
|
260
537
|
"Log nativeGraph requires @peerbit/log-rust to be installed and built",
|
|
261
538
|
);
|
|
262
539
|
}
|
|
263
|
-
const headsRequested =
|
|
264
|
-
typeof options.nativeGraph === "object"
|
|
265
|
-
? options.nativeGraph.heads !== false
|
|
266
|
-
: true;
|
|
267
|
-
return {
|
|
268
|
-
graph: await createLogGraphIndex(),
|
|
269
|
-
useHeads: headsRequested && this._sortFn === LastWriteWins,
|
|
270
|
-
};
|
|
271
540
|
})());
|
|
272
541
|
|
|
273
542
|
this._entryIndex = new EntryIndex({
|
|
@@ -293,12 +562,58 @@ export class Log<T> {
|
|
|
293
562
|
this._trim = new Trim(
|
|
294
563
|
{
|
|
295
564
|
index: this._entryIndex,
|
|
296
|
-
deleteNode: async (
|
|
297
|
-
|
|
298
|
-
|
|
565
|
+
deleteNode: async (
|
|
566
|
+
node: ShallowEntry,
|
|
567
|
+
options?: { resolveDeletedEntry?: boolean },
|
|
568
|
+
) => {
|
|
569
|
+
const shouldResolve = options?.resolveDeletedEntry !== false;
|
|
570
|
+
const resolved = shouldResolve
|
|
571
|
+
? await this.get(node.hash)
|
|
572
|
+
: undefined;
|
|
573
|
+
const deleted = await this._entryIndex.delete(node.hash, node);
|
|
299
574
|
await this._storage.rm(node.hash);
|
|
300
|
-
|
|
575
|
+
if (!deleted) {
|
|
576
|
+
return resolved;
|
|
577
|
+
}
|
|
578
|
+
return shouldResolve ? resolved : deleted;
|
|
301
579
|
},
|
|
580
|
+
deleteNodes: this._entryIndex.canDeleteMany()
|
|
581
|
+
? (
|
|
582
|
+
nodes: ShallowEntry[],
|
|
583
|
+
options?: {
|
|
584
|
+
resolveDeletedEntry?: boolean;
|
|
585
|
+
skipNextHeadUpdates?: boolean;
|
|
586
|
+
},
|
|
587
|
+
): MaybePromise<(Entry<T> | ShallowEntry)[]> => {
|
|
588
|
+
if (nodes.length === 0) {
|
|
589
|
+
return [];
|
|
590
|
+
}
|
|
591
|
+
const shouldResolve = options?.resolveDeletedEntry !== false;
|
|
592
|
+
if (!shouldResolve) {
|
|
593
|
+
return this._entryIndex.deleteManyMaybe(nodes, {
|
|
594
|
+
skipNextHeadUpdates: options?.skipNextHeadUpdates,
|
|
595
|
+
});
|
|
596
|
+
}
|
|
597
|
+
return (async () => {
|
|
598
|
+
const resolvedByHash = new Map<string, Entry<T>>();
|
|
599
|
+
const resolved = await this._entryIndex.getMany(
|
|
600
|
+
nodes.map((node) => node.hash),
|
|
601
|
+
{ type: "full", ignoreMissing: true },
|
|
602
|
+
);
|
|
603
|
+
for (const entry of resolved) {
|
|
604
|
+
if (entry) {
|
|
605
|
+
resolvedByHash.set(entry.hash, entry);
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
const deleted = await this._entryIndex.deleteMany(nodes, {
|
|
609
|
+
skipNextHeadUpdates: options?.skipNextHeadUpdates,
|
|
610
|
+
});
|
|
611
|
+
return deleted
|
|
612
|
+
.map((node) => resolvedByHash.get(node.hash))
|
|
613
|
+
.filter((entry): entry is Entry<T> => !!entry);
|
|
614
|
+
})();
|
|
615
|
+
}
|
|
616
|
+
: undefined,
|
|
302
617
|
sortFn: this._sortFn,
|
|
303
618
|
getLength: () => this.length,
|
|
304
619
|
},
|
|
@@ -313,6 +628,7 @@ export class Log<T> {
|
|
|
313
628
|
}
|
|
314
629
|
return true;
|
|
315
630
|
};
|
|
631
|
+
this._hasCustomCanAppend = !!options?.canAppend;
|
|
316
632
|
|
|
317
633
|
this._onChange = options?.onChange;
|
|
318
634
|
this._closed = false;
|
|
@@ -366,6 +682,11 @@ export class Log<T> {
|
|
|
366
682
|
has(cid: string) {
|
|
367
683
|
return this._entryIndex.has(cid);
|
|
368
684
|
}
|
|
685
|
+
|
|
686
|
+
hasMany(cids: Iterable<string>) {
|
|
687
|
+
return this._entryIndex.hasMany(cids);
|
|
688
|
+
}
|
|
689
|
+
|
|
369
690
|
/**
|
|
370
691
|
* Get all entries sorted. Don't use this method anywhere where performance matters
|
|
371
692
|
*/
|
|
@@ -546,98 +867,53 @@ export class Log<T> {
|
|
|
546
867
|
data: T,
|
|
547
868
|
options: AppendOptions<T> = {},
|
|
548
869
|
): Promise<{ entry: Entry<T>; removed: ShallowOrFullEntry<T>[] }> {
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
signers: options.signers,
|
|
577
|
-
data,
|
|
578
|
-
meta: {
|
|
579
|
-
clock,
|
|
580
|
-
type: options.meta?.type,
|
|
581
|
-
gidSeed: options.meta?.gidSeed,
|
|
582
|
-
data: options.meta?.data,
|
|
583
|
-
next: nexts,
|
|
584
|
-
},
|
|
585
|
-
|
|
586
|
-
encoding: this._encoding,
|
|
587
|
-
encryption: options.encryption
|
|
588
|
-
? {
|
|
589
|
-
keypair: options.encryption.keypair,
|
|
590
|
-
receiver: {
|
|
591
|
-
...options.encryption.receiver,
|
|
592
|
-
},
|
|
593
|
-
}
|
|
594
|
-
: undefined,
|
|
595
|
-
canAppend: options.canAppend || this._canAppend,
|
|
596
|
-
});
|
|
597
|
-
|
|
598
|
-
if (!entry.hash) {
|
|
599
|
-
throw new Error("Unexpected");
|
|
600
|
-
}
|
|
601
|
-
|
|
602
|
-
if (entry.meta.type !== EntryType.CUT) {
|
|
603
|
-
for (const e of nexts) {
|
|
604
|
-
if (!(await this.has(e.hash))) {
|
|
605
|
-
let entry: Entry<any>;
|
|
606
|
-
if (e instanceof Entry) {
|
|
607
|
-
entry = e;
|
|
608
|
-
} else {
|
|
609
|
-
let resolved = await this.entryIndex.get(e.hash);
|
|
610
|
-
if (!resolved) {
|
|
611
|
-
// eslint-disable-next-line no-console
|
|
612
|
-
console.warn("Unexpected missing entry when joining", e.hash);
|
|
613
|
-
continue;
|
|
614
|
-
}
|
|
615
|
-
entry = resolved;
|
|
616
|
-
}
|
|
617
|
-
await this.join([entry]);
|
|
870
|
+
const nexts = await this.getNextsForAppend(options);
|
|
871
|
+
const deferBlockStore = hasPutMany(this._storage);
|
|
872
|
+
const nativeAppendChain = this.entryIndex.properties.nativeGraph
|
|
873
|
+
? await this.createNativePlainAppendChain(
|
|
874
|
+
[data],
|
|
875
|
+
options,
|
|
876
|
+
nexts,
|
|
877
|
+
deferBlockStore,
|
|
878
|
+
)
|
|
879
|
+
: undefined;
|
|
880
|
+
let entry: Entry<T>;
|
|
881
|
+
if (nativeAppendChain) {
|
|
882
|
+
entry = nativeAppendChain.entries[0]!;
|
|
883
|
+
try {
|
|
884
|
+
await this.joinMissingNexts(entry, nexts);
|
|
885
|
+
if (deferBlockStore && !nativeAppendChain.nativeBlocksCommitted) {
|
|
886
|
+
await this.putAppendEntryBlocks([entry], nativeAppendChain.blocks);
|
|
887
|
+
}
|
|
888
|
+
await this.putAppendEntries(
|
|
889
|
+
[entry],
|
|
890
|
+
options,
|
|
891
|
+
nexts.map((next) => next.hash),
|
|
892
|
+
nativeAppendChain,
|
|
893
|
+
);
|
|
894
|
+
} catch (error) {
|
|
895
|
+
if (nativeAppendChain.nativeGraphUpdated) {
|
|
896
|
+
this.rollbackNativeAppendGraph([entry]);
|
|
618
897
|
}
|
|
898
|
+
if (nativeAppendChain.nativeBlocksCommitted) {
|
|
899
|
+
await this.rollbackNativeAppendBlocks([entry]);
|
|
900
|
+
}
|
|
901
|
+
throw error;
|
|
619
902
|
}
|
|
903
|
+
} else {
|
|
904
|
+
entry = await this.createAppendEntry(data, options, nexts);
|
|
905
|
+
await this.joinMissingNexts(entry, nexts);
|
|
906
|
+
await this.putAppendEntry(entry, options);
|
|
620
907
|
}
|
|
621
908
|
|
|
622
|
-
await this.entryIndex.put(entry, {
|
|
623
|
-
unique: true,
|
|
624
|
-
isHead: true,
|
|
625
|
-
toMultiHash: false,
|
|
626
|
-
deferIndexWrite:
|
|
627
|
-
options.deferIndexWrite ??
|
|
628
|
-
(options.durability
|
|
629
|
-
? options.durability === "buffered"
|
|
630
|
-
: this._appendDurability === "buffered"),
|
|
631
|
-
});
|
|
632
|
-
|
|
633
909
|
const pendingDeletes: (
|
|
634
910
|
| PendingDelete<T>
|
|
635
|
-
| { entry:
|
|
911
|
+
| { entry: ShallowOrFullEntry<T>; fn: undefined }
|
|
636
912
|
)[] = await this.processEntry(entry);
|
|
637
913
|
|
|
638
914
|
entry.init({ encoding: this._encoding, keychain: this._keychain });
|
|
639
915
|
|
|
640
|
-
const trimmed = await this.
|
|
916
|
+
const trimmed = await this.trimIfConfigured(options?.trim);
|
|
641
917
|
|
|
642
918
|
if (trimmed) {
|
|
643
919
|
for (const entry of trimmed) {
|
|
@@ -655,113 +931,2294 @@ export class Log<T> {
|
|
|
655
931
|
return { entry, removed };
|
|
656
932
|
}
|
|
657
933
|
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
options
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
934
|
+
// Internal trusted local append path for callers that already handled validation
|
|
935
|
+
// and want to apply change observers themselves.
|
|
936
|
+
private async appendLocallyPrepared(
|
|
937
|
+
data: T,
|
|
938
|
+
options: AppendOptions<T> = {},
|
|
939
|
+
properties?: {
|
|
940
|
+
skipMissingNextJoin?: boolean;
|
|
941
|
+
resolveTrimmedEntries?: boolean;
|
|
942
|
+
payloadData?: Uint8Array;
|
|
943
|
+
includeMaterializationBytes?: boolean;
|
|
944
|
+
includeAppendFactsBytes?: boolean;
|
|
945
|
+
},
|
|
946
|
+
): Promise<{
|
|
947
|
+
entry: Entry<T>;
|
|
948
|
+
removed: ShallowOrFullEntry<T>[];
|
|
949
|
+
change: Change<T>;
|
|
950
|
+
appendFacts: PreparedAppendFacts;
|
|
951
|
+
}> {
|
|
952
|
+
if (
|
|
953
|
+
options.canAppend ||
|
|
954
|
+
options.onChange ||
|
|
955
|
+
options.meta?.type === EntryType.CUT
|
|
956
|
+
) {
|
|
957
|
+
throw new Error(
|
|
958
|
+
"appendLocallyPrepared only supports trusted plain local appends",
|
|
959
|
+
);
|
|
672
960
|
}
|
|
673
961
|
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
962
|
+
const appendOptions = withCanAppendAlreadyValidated(options);
|
|
963
|
+
const nextsResult = this.getNextsForAppend(appendOptions);
|
|
964
|
+
const nexts = isPromiseLike(nextsResult) ? await nextsResult : nextsResult;
|
|
965
|
+
const deferBlockStore = hasPutMany(this._storage);
|
|
966
|
+
const nativeAppendChain = this.entryIndex.properties.nativeGraph
|
|
967
|
+
? await this.createNativePlainAppendChain(
|
|
968
|
+
[data],
|
|
969
|
+
appendOptions,
|
|
970
|
+
nexts,
|
|
971
|
+
deferBlockStore,
|
|
972
|
+
properties?.payloadData ? [properties.payloadData] : undefined,
|
|
973
|
+
)
|
|
974
|
+
: undefined;
|
|
975
|
+
let entry: Entry<T>;
|
|
976
|
+
if (nativeAppendChain) {
|
|
977
|
+
entry = nativeAppendChain.entries[0]!;
|
|
978
|
+
try {
|
|
979
|
+
if (!properties?.skipMissingNextJoin) {
|
|
980
|
+
await this.joinMissingNexts(entry, nexts);
|
|
981
|
+
}
|
|
982
|
+
if (deferBlockStore && !nativeAppendChain.nativeBlocksCommitted) {
|
|
983
|
+
await this.putAppendEntryBlocks([entry], nativeAppendChain.blocks);
|
|
984
|
+
}
|
|
985
|
+
await this.putAppendEntries(
|
|
986
|
+
[entry],
|
|
987
|
+
appendOptions,
|
|
988
|
+
nexts.map((next) => next.hash),
|
|
989
|
+
nativeAppendChain,
|
|
990
|
+
);
|
|
991
|
+
} catch (error) {
|
|
992
|
+
if (nativeAppendChain.nativeGraphUpdated) {
|
|
993
|
+
this.rollbackNativeAppendGraph([entry]);
|
|
994
|
+
}
|
|
995
|
+
if (nativeAppendChain.nativeBlocksCommitted) {
|
|
996
|
+
await this.rollbackNativeAppendBlocks([entry]);
|
|
997
|
+
}
|
|
998
|
+
throw error;
|
|
999
|
+
}
|
|
681
1000
|
} else {
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
1001
|
+
if (data == null && properties?.payloadData) {
|
|
1002
|
+
throw new Error(
|
|
1003
|
+
"appendLocallyPrepared payload-only path requires native append support",
|
|
1004
|
+
);
|
|
1005
|
+
}
|
|
1006
|
+
entry = await this.createAppendEntry(data, appendOptions, nexts);
|
|
1007
|
+
if (!properties?.skipMissingNextJoin) {
|
|
1008
|
+
await this.joinMissingNexts(entry, nexts);
|
|
687
1009
|
}
|
|
1010
|
+
await this.putAppendEntry(entry, appendOptions);
|
|
688
1011
|
}
|
|
689
1012
|
|
|
1013
|
+
entry.init({ encoding: this._encoding, keychain: this._keychain });
|
|
1014
|
+
|
|
1015
|
+
const trimmed = await this.trimIfConfigured(appendOptions.trim, {
|
|
1016
|
+
resolveDeletedEntries: properties?.resolveTrimmedEntries,
|
|
1017
|
+
});
|
|
1018
|
+
const removed = trimmed ?? [];
|
|
690
1019
|
const change: Change<T> = {
|
|
691
|
-
added: [],
|
|
692
|
-
removed
|
|
1020
|
+
added: [{ head: true, entry }],
|
|
1021
|
+
removed,
|
|
693
1022
|
};
|
|
1023
|
+
const appendFacts = this.createPreparedAppendFacts(
|
|
1024
|
+
[entry],
|
|
1025
|
+
nativeAppendChain,
|
|
1026
|
+
)[0]!;
|
|
694
1027
|
|
|
695
|
-
|
|
1028
|
+
return { entry, removed, change, appendFacts };
|
|
1029
|
+
}
|
|
696
1030
|
|
|
697
|
-
|
|
698
|
-
|
|
1031
|
+
// Internal trusted local append path for callers that can consume compact
|
|
1032
|
+
// append facts before a public Entry object is needed.
|
|
1033
|
+
private appendLocallyPreparedCommitOnly(
|
|
1034
|
+
data: T,
|
|
1035
|
+
options: AppendOptions<T> = {},
|
|
1036
|
+
properties?: {
|
|
1037
|
+
skipMissingNextJoin?: boolean;
|
|
1038
|
+
resolveTrimmedEntries?: boolean;
|
|
1039
|
+
payloadData?: Uint8Array;
|
|
1040
|
+
includeMaterializationBytes?: boolean;
|
|
1041
|
+
includeAppendFactsBytes?: boolean;
|
|
1042
|
+
},
|
|
1043
|
+
): MaybePromise<PreparedCommitOnlyAppendResult<T> | undefined> {
|
|
1044
|
+
if (
|
|
1045
|
+
options.canAppend ||
|
|
1046
|
+
options.onChange ||
|
|
1047
|
+
options.meta?.type === EntryType.CUT
|
|
1048
|
+
) {
|
|
1049
|
+
throw new Error(
|
|
1050
|
+
"appendLocallyPreparedCommitOnly only supports trusted plain local appends",
|
|
1051
|
+
);
|
|
1052
|
+
}
|
|
699
1053
|
|
|
700
|
-
|
|
1054
|
+
const appendOptions = withCanAppendAlreadyValidated(options);
|
|
1055
|
+
const nextsResult = this.getNextsForAppend(appendOptions);
|
|
1056
|
+
return mapMaybePromise(nextsResult, (nexts) =>
|
|
1057
|
+
this.appendLocallyPreparedCommitOnlyWithNexts(
|
|
1058
|
+
data,
|
|
1059
|
+
appendOptions,
|
|
1060
|
+
nexts,
|
|
1061
|
+
properties,
|
|
1062
|
+
this.getNativeCommitOnlyTrimLengthTo(
|
|
1063
|
+
appendOptions.trim,
|
|
1064
|
+
properties?.resolveTrimmedEntries,
|
|
1065
|
+
),
|
|
1066
|
+
),
|
|
1067
|
+
);
|
|
701
1068
|
}
|
|
702
1069
|
|
|
703
|
-
|
|
704
|
-
|
|
1070
|
+
private appendLocallyPreparedNativeNoNextCommitOnly(
|
|
1071
|
+
data: T,
|
|
1072
|
+
options: AppendOptions<T> = {},
|
|
1073
|
+
properties: {
|
|
1074
|
+
payloadData?: Uint8Array;
|
|
1075
|
+
resolveTrimmedEntries?: boolean;
|
|
1076
|
+
skipMissingNextJoin?: boolean;
|
|
1077
|
+
retainMaterializationBytes?: boolean;
|
|
1078
|
+
},
|
|
1079
|
+
prepare: (
|
|
1080
|
+
input: NativeNoNextCommitInput,
|
|
1081
|
+
) => MaybePromise<NativePreparedNoNextCommit | undefined>,
|
|
1082
|
+
): MaybePromise<PreparedCommitOnlyAppendResult<T> | undefined> {
|
|
1083
|
+
const directResult = this.appendLocallyPreparedNativeKnownNoNextCommitOnly(
|
|
1084
|
+
data,
|
|
1085
|
+
options,
|
|
1086
|
+
properties,
|
|
1087
|
+
prepare,
|
|
1088
|
+
);
|
|
1089
|
+
if (directResult !== undefined) {
|
|
1090
|
+
return directResult;
|
|
1091
|
+
}
|
|
1092
|
+
return this.appendLocallyPreparedNativeCommitOnly(
|
|
1093
|
+
data,
|
|
1094
|
+
options,
|
|
1095
|
+
properties,
|
|
1096
|
+
prepare,
|
|
1097
|
+
);
|
|
705
1098
|
}
|
|
706
1099
|
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
timeout?: number;
|
|
716
|
-
onChange?: OnChange<T>;
|
|
717
|
-
reset?: boolean;
|
|
1100
|
+
private appendLocallyPreparedNativeKnownNoNextCommitOnly(
|
|
1101
|
+
data: T,
|
|
1102
|
+
options: AppendOptions<T> = {},
|
|
1103
|
+
properties: {
|
|
1104
|
+
payloadData?: Uint8Array;
|
|
1105
|
+
resolveTrimmedEntries?: boolean;
|
|
1106
|
+
skipMissingNextJoin?: boolean;
|
|
1107
|
+
retainMaterializationBytes?: boolean;
|
|
718
1108
|
},
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
1109
|
+
prepare: (
|
|
1110
|
+
input: NativeNoNextCommitInput,
|
|
1111
|
+
) => MaybePromise<NativePreparedNoNextCommit | undefined>,
|
|
1112
|
+
): MaybePromise<PreparedCommitOnlyAppendResult<T> | undefined> {
|
|
1113
|
+
if (options.meta?.next == null || options.meta.next.length !== 0) {
|
|
1114
|
+
return undefined;
|
|
1115
|
+
}
|
|
1116
|
+
const resolvedTrim = options.trim ?? this._trim.options;
|
|
1117
|
+
const supportsNativeTrim =
|
|
1118
|
+
!resolvedTrim ||
|
|
1119
|
+
(resolvedTrim.type === "length" &&
|
|
1120
|
+
!resolvedTrim.filter?.canTrim &&
|
|
1121
|
+
properties.resolveTrimmedEntries === false);
|
|
1122
|
+
if (!supportsNativeTrim) {
|
|
1123
|
+
return undefined;
|
|
1124
|
+
}
|
|
1125
|
+
const nativeTrimLengthTo = this.getNativeCommitOnlyTrimLengthTo(
|
|
1126
|
+
options.trim,
|
|
1127
|
+
properties.resolveTrimmedEntries,
|
|
1128
|
+
);
|
|
1129
|
+
if (!options.meta?.gidSeed) {
|
|
1130
|
+
const directResult =
|
|
1131
|
+
this.appendLocallyPreparedNativeKnownNoNextDirectCommitOnly(
|
|
1132
|
+
data,
|
|
1133
|
+
options,
|
|
1134
|
+
properties,
|
|
1135
|
+
prepare,
|
|
1136
|
+
nativeTrimLengthTo,
|
|
1137
|
+
);
|
|
1138
|
+
if (directResult !== undefined) {
|
|
1139
|
+
return directResult;
|
|
735
1140
|
}
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
1141
|
+
}
|
|
1142
|
+
return this.appendLocallyPreparedNativeCommitOnly(
|
|
1143
|
+
data,
|
|
1144
|
+
options,
|
|
1145
|
+
properties,
|
|
1146
|
+
prepare,
|
|
1147
|
+
true,
|
|
1148
|
+
);
|
|
1149
|
+
}
|
|
740
1150
|
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
1151
|
+
private appendLocallyPreparedNativeKnownNoNextDirectCommitOnly(
|
|
1152
|
+
data: T,
|
|
1153
|
+
options: AppendOptions<T> = {},
|
|
1154
|
+
properties: {
|
|
1155
|
+
payloadData?: Uint8Array;
|
|
1156
|
+
resolveTrimmedEntries?: boolean;
|
|
1157
|
+
retainMaterializationBytes?: boolean;
|
|
1158
|
+
},
|
|
1159
|
+
prepare: (
|
|
1160
|
+
input: NativeNoNextCommitInput,
|
|
1161
|
+
) => MaybePromise<NativePreparedNoNextCommit | undefined>,
|
|
1162
|
+
nativeTrimLengthTo?: number,
|
|
1163
|
+
): MaybePromise<PreparedCommitOnlyAppendResult<T> | undefined> {
|
|
1164
|
+
if (
|
|
1165
|
+
options.canAppend ||
|
|
1166
|
+
options.onChange ||
|
|
1167
|
+
options.encryption ||
|
|
1168
|
+
options.signers ||
|
|
1169
|
+
options.identity ||
|
|
1170
|
+
options.meta?.timestamp ||
|
|
1171
|
+
options.meta?.type === EntryType.CUT ||
|
|
1172
|
+
options.meta?.next == null ||
|
|
1173
|
+
options.meta.next.length !== 0 ||
|
|
1174
|
+
options.meta?.gidSeed ||
|
|
1175
|
+
(this._hasCustomCanAppend && !canAppendAlreadyValidated(options))
|
|
1176
|
+
) {
|
|
1177
|
+
return undefined;
|
|
1178
|
+
}
|
|
1179
|
+
const identity = this._identity;
|
|
1180
|
+
if (!(identity instanceof Ed25519Keypair)) {
|
|
1181
|
+
return undefined;
|
|
1182
|
+
}
|
|
1183
|
+
const payloadData =
|
|
1184
|
+
properties.payloadData ??
|
|
1185
|
+
(data == null ? undefined : this._encoding.encoder(data));
|
|
1186
|
+
if (!payloadData || !hasPutMany(this._storage)) {
|
|
1187
|
+
return undefined;
|
|
1188
|
+
}
|
|
745
1189
|
|
|
746
|
-
|
|
1190
|
+
const resolvedGid = EntryV0.createGid() as string;
|
|
1191
|
+
const timestamp = this._hlc.now();
|
|
1192
|
+
const entryType = options.meta?.type ?? EntryType.APPEND;
|
|
1193
|
+
const preparedValue = prepare({
|
|
1194
|
+
clockId: identity.publicKey.bytes,
|
|
1195
|
+
privateKey: identity.privateKey.privateKey,
|
|
1196
|
+
publicKey: identity.publicKey.publicKey,
|
|
1197
|
+
wallTime: timestamp.wallTime,
|
|
1198
|
+
logical: timestamp.logical,
|
|
1199
|
+
gid: resolvedGid,
|
|
1200
|
+
type: entryType,
|
|
1201
|
+
metaData: options.meta?.data,
|
|
1202
|
+
payloadData,
|
|
1203
|
+
resolveTrimmedEntries: properties.resolveTrimmedEntries,
|
|
1204
|
+
trimLengthTo: nativeTrimLengthTo,
|
|
1205
|
+
});
|
|
1206
|
+
return mapMaybePromise(preparedValue, (prepared) => {
|
|
1207
|
+
if (!prepared) {
|
|
1208
|
+
return undefined;
|
|
1209
|
+
}
|
|
1210
|
+
const hash = prepared.cid ?? prepared.hash;
|
|
1211
|
+
if (!hash) {
|
|
1212
|
+
return undefined;
|
|
1213
|
+
}
|
|
1214
|
+
const shouldRetainMaterializationBytes =
|
|
1215
|
+
properties.retainMaterializationBytes === true ||
|
|
1216
|
+
!!(options.trim ?? this._trim.options);
|
|
1217
|
+
let retainedMaterializationBytes: Uint8Array | undefined;
|
|
1218
|
+
const retainMaterializationBytes = () => {
|
|
1219
|
+
if (
|
|
1220
|
+
retainedMaterializationBytes ||
|
|
1221
|
+
!shouldRetainMaterializationBytes ||
|
|
1222
|
+
prepared.bytes
|
|
1223
|
+
) {
|
|
1224
|
+
return;
|
|
1225
|
+
}
|
|
1226
|
+
const bytes =
|
|
1227
|
+
prepared.getBytes?.(hash) ??
|
|
1228
|
+
(this._storage.get(hash) as Uint8Array | undefined);
|
|
1229
|
+
if (
|
|
1230
|
+
bytes &&
|
|
1231
|
+
typeof (bytes as { then?: unknown }).then !== "function"
|
|
1232
|
+
) {
|
|
1233
|
+
retainedMaterializationBytes = bytes;
|
|
1234
|
+
}
|
|
1235
|
+
};
|
|
1236
|
+
const effectiveNextHashes = prepared.next ?? EMPTY_NEXT_HASHES;
|
|
1237
|
+
const effectiveGid = prepared.gid ?? resolvedGid;
|
|
1238
|
+
let clock: Clock | undefined;
|
|
1239
|
+
const getClock = () =>
|
|
1240
|
+
(clock ??= new Clock({
|
|
1241
|
+
id: identity.publicKey.bytes,
|
|
1242
|
+
timestamp,
|
|
1243
|
+
}));
|
|
1244
|
+
let shallowEntry: ShallowEntry | undefined;
|
|
1245
|
+
const getShallowEntry = () =>
|
|
1246
|
+
(shallowEntry ??= new ShallowEntry({
|
|
1247
|
+
hash,
|
|
1248
|
+
payloadSize: payloadData.byteLength,
|
|
1249
|
+
head: true,
|
|
1250
|
+
meta: new ShallowMeta({
|
|
1251
|
+
gid: effectiveGid,
|
|
1252
|
+
data: options.meta?.data,
|
|
1253
|
+
clock: getClock(),
|
|
1254
|
+
next: effectiveNextHashes,
|
|
1255
|
+
type: entryType,
|
|
1256
|
+
}),
|
|
1257
|
+
}));
|
|
1258
|
+
const appendFacts: PreparedAppendFacts = {
|
|
1259
|
+
hash,
|
|
1260
|
+
gid: effectiveGid,
|
|
1261
|
+
next: effectiveNextHashes,
|
|
1262
|
+
wallTime: timestamp.wallTime,
|
|
1263
|
+
logical: timestamp.logical,
|
|
1264
|
+
clockId: identity.publicKey.bytes,
|
|
1265
|
+
type: entryType,
|
|
1266
|
+
metaData: options.meta?.data,
|
|
1267
|
+
payloadSize: payloadData.byteLength,
|
|
1268
|
+
metaBytes: prepared.metaBytes,
|
|
1269
|
+
hashDigestBytes: prepared.hashDigestBytes,
|
|
1270
|
+
};
|
|
1271
|
+
let materializedEntry: Entry<T> | undefined;
|
|
1272
|
+
const materializeEntry = () => {
|
|
1273
|
+
if (materializedEntry) {
|
|
1274
|
+
return materializedEntry;
|
|
1275
|
+
}
|
|
1276
|
+
const bytes =
|
|
1277
|
+
prepared.bytes ??
|
|
1278
|
+
retainedMaterializationBytes ??
|
|
1279
|
+
prepared.getBytes?.(hash) ??
|
|
1280
|
+
(this._storage.get(hash) as Uint8Array | undefined);
|
|
1281
|
+
if (
|
|
1282
|
+
!bytes ||
|
|
1283
|
+
typeof (bytes as { then?: unknown }).then === "function"
|
|
1284
|
+
) {
|
|
1285
|
+
throw new Error("Missing synchronous native append block bytes");
|
|
1286
|
+
}
|
|
1287
|
+
const entry = deserialize(bytes, Entry) as Entry<T>;
|
|
1288
|
+
entry.hash = hash;
|
|
1289
|
+
entry.size = prepared.byteLength;
|
|
1290
|
+
entry.createdLocally = true;
|
|
1291
|
+
Entry.prepareShallowEntry(entry, getShallowEntry());
|
|
1292
|
+
entry.init({ encoding: this._encoding, keychain: this._keychain });
|
|
1293
|
+
materializedEntry = entry;
|
|
1294
|
+
return entry;
|
|
1295
|
+
};
|
|
1296
|
+
const finish = (): PreparedCommitOnlyAppendResult<T> => {
|
|
1297
|
+
retainMaterializationBytes();
|
|
1298
|
+
return {
|
|
1299
|
+
get entry() {
|
|
1300
|
+
return materializeEntry();
|
|
1301
|
+
},
|
|
1302
|
+
materializeEntry,
|
|
1303
|
+
removed: [],
|
|
1304
|
+
appendFacts,
|
|
1305
|
+
get shallowEntry() {
|
|
1306
|
+
return getShallowEntry();
|
|
1307
|
+
},
|
|
1308
|
+
documentTrimmedHeadsProcessed: prepared.documentTrimmedHeadsProcessed,
|
|
1309
|
+
documentPreviousContext: prepared.documentPreviousContext,
|
|
1310
|
+
};
|
|
1311
|
+
};
|
|
1312
|
+
const finishTrim = ():
|
|
1313
|
+
| PreparedCommitOnlyAppendResult<T>
|
|
1314
|
+
| Promise<PreparedCommitOnlyAppendResult<T>> => {
|
|
1315
|
+
retainMaterializationBytes();
|
|
1316
|
+
if (prepared.trimmedEntryHashes) {
|
|
1317
|
+
if (prepared.trimmedEntryHashes.length === 0) {
|
|
1318
|
+
return finish();
|
|
1319
|
+
}
|
|
1320
|
+
if (
|
|
1321
|
+
properties.resolveTrimmedEntries === false ||
|
|
1322
|
+
prepared.documentTrimmedHeadsProcessed === true
|
|
1323
|
+
) {
|
|
1324
|
+
const trimmedEntryHashes =
|
|
1325
|
+
prepared.trimmedEntryHashes.length === 1
|
|
1326
|
+
? prepared.trimmedEntryHashes
|
|
1327
|
+
: [...new Set(prepared.trimmedEntryHashes)];
|
|
1328
|
+
const consumedNoReturn =
|
|
1329
|
+
this.entryIndex.consumeNativeTrimmedEntryHashesNoReturnMaybe(
|
|
1330
|
+
trimmedEntryHashes,
|
|
1331
|
+
{
|
|
1332
|
+
skipNextHeadUpdates: true,
|
|
1333
|
+
deleteBlocks: false,
|
|
1334
|
+
},
|
|
1335
|
+
);
|
|
1336
|
+
if (consumedNoReturn !== undefined) {
|
|
1337
|
+
return mapMaybePromise(consumedNoReturn, () => ({
|
|
1338
|
+
get entry() {
|
|
1339
|
+
return materializeEntry();
|
|
1340
|
+
},
|
|
1341
|
+
materializeEntry,
|
|
1342
|
+
removed: [],
|
|
1343
|
+
removedHashes: trimmedEntryHashes,
|
|
1344
|
+
appendFacts,
|
|
1345
|
+
get shallowEntry() {
|
|
1346
|
+
return getShallowEntry();
|
|
1347
|
+
},
|
|
1348
|
+
documentTrimmedHeadsProcessed:
|
|
1349
|
+
prepared.documentTrimmedHeadsProcessed,
|
|
1350
|
+
documentPreviousContext: prepared.documentPreviousContext,
|
|
1351
|
+
}));
|
|
1352
|
+
}
|
|
1353
|
+
}
|
|
1354
|
+
const consumedResult =
|
|
1355
|
+
this.entryIndex.consumeNativeTrimmedEntryHashesMaybe(
|
|
1356
|
+
prepared.trimmedEntryHashes,
|
|
1357
|
+
{
|
|
1358
|
+
skipNextHeadUpdates: true,
|
|
1359
|
+
deleteBlocks: false,
|
|
1360
|
+
},
|
|
1361
|
+
);
|
|
1362
|
+
return mapMaybePromise(consumedResult, (removed) => ({
|
|
1363
|
+
get entry() {
|
|
1364
|
+
return materializeEntry();
|
|
1365
|
+
},
|
|
1366
|
+
materializeEntry,
|
|
1367
|
+
removed,
|
|
1368
|
+
removedHashes: prepared.trimmedEntryHashes,
|
|
1369
|
+
appendFacts,
|
|
1370
|
+
get shallowEntry() {
|
|
1371
|
+
return getShallowEntry();
|
|
1372
|
+
},
|
|
1373
|
+
documentTrimmedHeadsProcessed:
|
|
1374
|
+
prepared.documentTrimmedHeadsProcessed,
|
|
1375
|
+
documentPreviousContext: prepared.documentPreviousContext,
|
|
1376
|
+
}));
|
|
1377
|
+
}
|
|
1378
|
+
if (!prepared.trimmedEntries) {
|
|
1379
|
+
return finish();
|
|
1380
|
+
}
|
|
1381
|
+
const trimmedEntries = this.entryIndex.nativeLogEntriesToShallowEntries(
|
|
1382
|
+
prepared.trimmedEntries,
|
|
1383
|
+
);
|
|
1384
|
+
const consumedResult = this.entryIndex.consumeNativeTrimmedEntriesMaybe(
|
|
1385
|
+
trimmedEntries,
|
|
1386
|
+
{
|
|
1387
|
+
skipNextHeadUpdates: true,
|
|
1388
|
+
deleteBlocks: false,
|
|
1389
|
+
},
|
|
1390
|
+
);
|
|
1391
|
+
return mapMaybePromise(consumedResult, (removed) => ({
|
|
1392
|
+
get entry() {
|
|
1393
|
+
return materializeEntry();
|
|
1394
|
+
},
|
|
1395
|
+
materializeEntry,
|
|
1396
|
+
removed,
|
|
1397
|
+
appendFacts,
|
|
1398
|
+
get shallowEntry() {
|
|
1399
|
+
return getShallowEntry();
|
|
1400
|
+
},
|
|
1401
|
+
documentTrimmedHeadsProcessed: prepared.documentTrimmedHeadsProcessed,
|
|
1402
|
+
documentPreviousContext: prepared.documentPreviousContext,
|
|
1403
|
+
}));
|
|
1404
|
+
};
|
|
1405
|
+
const finishBlocks = ():
|
|
1406
|
+
| PreparedCommitOnlyAppendResult<T>
|
|
1407
|
+
| Promise<PreparedCommitOnlyAppendResult<T>> => {
|
|
1408
|
+
if (!prepared.bytes) {
|
|
1409
|
+
return finishTrim();
|
|
1410
|
+
}
|
|
1411
|
+
return mapMaybePromise(
|
|
1412
|
+
this.putPreparedAppendBlocks([
|
|
1413
|
+
Entry.preparedBlockFromBytes(prepared.bytes, hash),
|
|
1414
|
+
]),
|
|
1415
|
+
finishTrim,
|
|
1416
|
+
);
|
|
1417
|
+
};
|
|
1418
|
+
const rollback = (error: unknown): never | Promise<never> => {
|
|
1419
|
+
this.rollbackNativeAppendGraphHashes([hash]);
|
|
1420
|
+
return this.rollbackNativeAppendBlocksHashes([hash]).then(() => {
|
|
1421
|
+
throw error;
|
|
1422
|
+
});
|
|
1423
|
+
};
|
|
1424
|
+
try {
|
|
1425
|
+
const putResult = this.entryIndex.putNativeCommittedAppendFacts({
|
|
1426
|
+
hash,
|
|
1427
|
+
unique: true,
|
|
1428
|
+
externalNextHashes: effectiveNextHashes,
|
|
1429
|
+
getShallowEntry,
|
|
1430
|
+
isHead: true,
|
|
1431
|
+
});
|
|
1432
|
+
const result = mapMaybePromise(putResult, finishBlocks);
|
|
1433
|
+
return isPromiseLike(result) ? result.catch(rollback) : result;
|
|
1434
|
+
} catch (error) {
|
|
1435
|
+
return rollback(error);
|
|
1436
|
+
}
|
|
1437
|
+
});
|
|
1438
|
+
}
|
|
1439
|
+
|
|
1440
|
+
private appendLocallyPreparedNativeCommitOnly(
|
|
1441
|
+
data: T,
|
|
1442
|
+
options: AppendOptions<T> = {},
|
|
1443
|
+
properties: {
|
|
1444
|
+
payloadData?: Uint8Array;
|
|
1445
|
+
resolveTrimmedEntries?: boolean;
|
|
1446
|
+
skipMissingNextJoin?: boolean;
|
|
1447
|
+
knownNoNext?: boolean;
|
|
1448
|
+
retainMaterializationBytes?: boolean;
|
|
1449
|
+
},
|
|
1450
|
+
prepare: (
|
|
1451
|
+
input: NativeCommitInput,
|
|
1452
|
+
) => MaybePromise<NativePreparedNoNextCommit | undefined>,
|
|
1453
|
+
knownNoNext = false,
|
|
1454
|
+
): MaybePromise<PreparedCommitOnlyAppendResult<T> | undefined> {
|
|
1455
|
+
const resolvedTrim = options.trim ?? this._trim.options;
|
|
1456
|
+
const supportsNativeTrim =
|
|
1457
|
+
!resolvedTrim ||
|
|
1458
|
+
(resolvedTrim.type === "length" &&
|
|
1459
|
+
!resolvedTrim.filter?.canTrim &&
|
|
1460
|
+
properties.resolveTrimmedEntries === false);
|
|
1461
|
+
const nativeTrimLengthTo = this.getNativeCommitOnlyTrimLengthTo(
|
|
1462
|
+
options.trim,
|
|
1463
|
+
properties.resolveTrimmedEntries,
|
|
1464
|
+
);
|
|
1465
|
+
if (
|
|
1466
|
+
options.canAppend ||
|
|
1467
|
+
options.onChange ||
|
|
1468
|
+
options.encryption ||
|
|
1469
|
+
options.signers ||
|
|
1470
|
+
options.identity ||
|
|
1471
|
+
options.meta?.timestamp ||
|
|
1472
|
+
!supportsNativeTrim ||
|
|
1473
|
+
(this._hasCustomCanAppend && !canAppendAlreadyValidated(options))
|
|
1474
|
+
) {
|
|
1475
|
+
return undefined;
|
|
1476
|
+
}
|
|
1477
|
+
const identity = this._identity;
|
|
1478
|
+
if (!(identity instanceof Ed25519Keypair)) {
|
|
1479
|
+
return undefined;
|
|
1480
|
+
}
|
|
1481
|
+
const payloadData =
|
|
1482
|
+
properties.payloadData ??
|
|
1483
|
+
(data == null ? undefined : this._encoding.encoder(data));
|
|
1484
|
+
if (!payloadData || !hasPutMany(this._storage)) {
|
|
1485
|
+
return undefined;
|
|
1486
|
+
}
|
|
1487
|
+
|
|
1488
|
+
const appendOptions = withCanAppendAlreadyValidated(options);
|
|
1489
|
+
const knownNoNextAppend = knownNoNext || properties.knownNoNext === true;
|
|
1490
|
+
const nextsResult = knownNoNextAppend
|
|
1491
|
+
? EMPTY_NEXT_ENTRIES
|
|
1492
|
+
: this.getNextsForAppend(appendOptions);
|
|
1493
|
+
return mapMaybePromise(nextsResult, (nexts) => {
|
|
1494
|
+
if (nexts.length > 0 && properties.skipMissingNextJoin !== true) {
|
|
1495
|
+
return undefined;
|
|
1496
|
+
}
|
|
1497
|
+
|
|
1498
|
+
const nextHashes: string[] = knownNoNextAppend ? EMPTY_NEXT_HASHES : [];
|
|
1499
|
+
let nextGid: string | undefined;
|
|
1500
|
+
if (nexts.length > 0) {
|
|
1501
|
+
if ((appendOptions.meta as { gid?: string } | undefined)?.gid) {
|
|
1502
|
+
return undefined;
|
|
1503
|
+
}
|
|
1504
|
+
for (const next of nexts) {
|
|
1505
|
+
if (!next.hash) {
|
|
1506
|
+
return undefined;
|
|
1507
|
+
}
|
|
1508
|
+
nextHashes.push(next.hash);
|
|
1509
|
+
nextGid =
|
|
1510
|
+
nextGid == null || next.meta.gid < nextGid
|
|
1511
|
+
? next.meta.gid
|
|
1512
|
+
: nextGid;
|
|
1513
|
+
}
|
|
1514
|
+
}
|
|
1515
|
+
|
|
1516
|
+
const gid = nextGid ?? EntryV0.createGid(appendOptions.meta?.gidSeed);
|
|
1517
|
+
return mapMaybePromise(gid, (resolvedGid) => {
|
|
1518
|
+
const clock = new Clock({
|
|
1519
|
+
id: identity.publicKey.bytes,
|
|
1520
|
+
timestamp: this._hlc.now(),
|
|
1521
|
+
});
|
|
1522
|
+
const entryType = appendOptions.meta?.type ?? EntryType.APPEND;
|
|
1523
|
+
const preparedValue = prepare({
|
|
1524
|
+
clockId: identity.publicKey.bytes,
|
|
1525
|
+
privateKey: identity.privateKey.privateKey,
|
|
1526
|
+
publicKey: identity.publicKey.publicKey,
|
|
1527
|
+
wallTime: clock.timestamp.wallTime,
|
|
1528
|
+
logical: clock.timestamp.logical,
|
|
1529
|
+
gid: resolvedGid,
|
|
1530
|
+
next: nextHashes,
|
|
1531
|
+
type: entryType,
|
|
1532
|
+
metaData: appendOptions.meta?.data,
|
|
1533
|
+
payloadData,
|
|
1534
|
+
resolveTrimmedEntries: properties.resolveTrimmedEntries,
|
|
1535
|
+
trimLengthTo: nativeTrimLengthTo,
|
|
1536
|
+
});
|
|
1537
|
+
return mapMaybePromise(preparedValue, (prepared) => {
|
|
1538
|
+
if (!prepared) {
|
|
1539
|
+
return undefined;
|
|
1540
|
+
}
|
|
1541
|
+
const hash = prepared.cid ?? prepared.hash;
|
|
1542
|
+
if (!hash) {
|
|
1543
|
+
return undefined;
|
|
1544
|
+
}
|
|
1545
|
+
const shouldRetainMaterializationBytes =
|
|
1546
|
+
properties.retainMaterializationBytes === true || !!resolvedTrim;
|
|
1547
|
+
let retainedMaterializationBytes: Uint8Array | undefined;
|
|
1548
|
+
const retainMaterializationBytes = () => {
|
|
1549
|
+
if (
|
|
1550
|
+
retainedMaterializationBytes ||
|
|
1551
|
+
!shouldRetainMaterializationBytes ||
|
|
1552
|
+
prepared.bytes
|
|
1553
|
+
) {
|
|
1554
|
+
return;
|
|
1555
|
+
}
|
|
1556
|
+
const bytes =
|
|
1557
|
+
prepared.getBytes?.(hash) ??
|
|
1558
|
+
(this._storage.get(hash) as Uint8Array | undefined);
|
|
1559
|
+
if (
|
|
1560
|
+
bytes &&
|
|
1561
|
+
typeof (bytes as { then?: unknown }).then !== "function"
|
|
1562
|
+
) {
|
|
1563
|
+
retainedMaterializationBytes = bytes;
|
|
1564
|
+
}
|
|
1565
|
+
};
|
|
1566
|
+
const effectiveNextHashes = prepared.next ?? nextHashes;
|
|
1567
|
+
const effectiveGid = prepared.gid ?? resolvedGid;
|
|
1568
|
+
const shallowEntry = new ShallowEntry({
|
|
1569
|
+
hash,
|
|
1570
|
+
payloadSize: payloadData.byteLength,
|
|
1571
|
+
head: true,
|
|
1572
|
+
meta: new ShallowMeta({
|
|
1573
|
+
gid: effectiveGid,
|
|
1574
|
+
data: appendOptions.meta?.data,
|
|
1575
|
+
clock,
|
|
1576
|
+
next: effectiveNextHashes,
|
|
1577
|
+
type: entryType,
|
|
1578
|
+
}),
|
|
1579
|
+
});
|
|
1580
|
+
const appendFacts: PreparedAppendFacts = {
|
|
1581
|
+
hash,
|
|
1582
|
+
gid: effectiveGid,
|
|
1583
|
+
next: effectiveNextHashes,
|
|
1584
|
+
wallTime: clock.timestamp.wallTime,
|
|
1585
|
+
logical: clock.timestamp.logical,
|
|
1586
|
+
clockId: clock.id,
|
|
1587
|
+
type: entryType,
|
|
1588
|
+
metaData: appendOptions.meta?.data,
|
|
1589
|
+
payloadSize: payloadData.byteLength,
|
|
1590
|
+
metaBytes: prepared.metaBytes,
|
|
1591
|
+
hashDigestBytes: prepared.hashDigestBytes,
|
|
1592
|
+
};
|
|
1593
|
+
let materializedEntry: Entry<T> | undefined;
|
|
1594
|
+
const materializeEntry = () => {
|
|
1595
|
+
if (materializedEntry) {
|
|
1596
|
+
return materializedEntry;
|
|
1597
|
+
}
|
|
1598
|
+
const bytes =
|
|
1599
|
+
prepared.bytes ??
|
|
1600
|
+
retainedMaterializationBytes ??
|
|
1601
|
+
prepared.getBytes?.(hash) ??
|
|
1602
|
+
(this._storage.get(hash) as Uint8Array | undefined);
|
|
1603
|
+
if (
|
|
1604
|
+
!bytes ||
|
|
1605
|
+
typeof (bytes as { then?: unknown }).then === "function"
|
|
1606
|
+
) {
|
|
1607
|
+
throw new Error("Missing synchronous native append block bytes");
|
|
1608
|
+
}
|
|
1609
|
+
const entry = deserialize(bytes, Entry) as Entry<T>;
|
|
1610
|
+
entry.hash = hash;
|
|
1611
|
+
entry.size = prepared.byteLength;
|
|
1612
|
+
entry.createdLocally = true;
|
|
1613
|
+
Entry.prepareShallowEntry(entry, shallowEntry);
|
|
1614
|
+
entry.init({ encoding: this._encoding, keychain: this._keychain });
|
|
1615
|
+
materializedEntry = entry;
|
|
1616
|
+
return entry;
|
|
1617
|
+
};
|
|
1618
|
+
const finish = (): PreparedCommitOnlyAppendResult<T> => {
|
|
1619
|
+
retainMaterializationBytes();
|
|
1620
|
+
return {
|
|
1621
|
+
get entry() {
|
|
1622
|
+
return materializeEntry();
|
|
1623
|
+
},
|
|
1624
|
+
materializeEntry,
|
|
1625
|
+
removed: [],
|
|
1626
|
+
appendFacts,
|
|
1627
|
+
shallowEntry,
|
|
1628
|
+
documentTrimmedHeadsProcessed:
|
|
1629
|
+
prepared.documentTrimmedHeadsProcessed,
|
|
1630
|
+
documentPreviousContext: prepared.documentPreviousContext,
|
|
1631
|
+
};
|
|
1632
|
+
};
|
|
1633
|
+
const finishBlocks = ():
|
|
1634
|
+
| PreparedCommitOnlyAppendResult<T>
|
|
1635
|
+
| Promise<PreparedCommitOnlyAppendResult<T>> => {
|
|
1636
|
+
if (!prepared.bytes) {
|
|
1637
|
+
return finishTrim();
|
|
1638
|
+
}
|
|
1639
|
+
return mapMaybePromise(
|
|
1640
|
+
this.putPreparedAppendBlocks([
|
|
1641
|
+
Entry.preparedBlockFromBytes(prepared.bytes, hash),
|
|
1642
|
+
]),
|
|
1643
|
+
finishTrim,
|
|
1644
|
+
);
|
|
1645
|
+
};
|
|
1646
|
+
const finishTrim = ():
|
|
1647
|
+
| PreparedCommitOnlyAppendResult<T>
|
|
1648
|
+
| Promise<PreparedCommitOnlyAppendResult<T>> => {
|
|
1649
|
+
retainMaterializationBytes();
|
|
1650
|
+
if (prepared.trimmedEntryHashes) {
|
|
1651
|
+
if (prepared.trimmedEntryHashes.length === 0) {
|
|
1652
|
+
return finish();
|
|
1653
|
+
}
|
|
1654
|
+
if (
|
|
1655
|
+
properties.resolveTrimmedEntries === false ||
|
|
1656
|
+
prepared.documentTrimmedHeadsProcessed === true
|
|
1657
|
+
) {
|
|
1658
|
+
const trimmedEntryHashes =
|
|
1659
|
+
prepared.trimmedEntryHashes.length === 1
|
|
1660
|
+
? prepared.trimmedEntryHashes
|
|
1661
|
+
: [...new Set(prepared.trimmedEntryHashes)];
|
|
1662
|
+
const consumedNoReturn =
|
|
1663
|
+
this.entryIndex.consumeNativeTrimmedEntryHashesNoReturnMaybe(
|
|
1664
|
+
trimmedEntryHashes,
|
|
1665
|
+
{
|
|
1666
|
+
skipNextHeadUpdates: true,
|
|
1667
|
+
deleteBlocks: false,
|
|
1668
|
+
},
|
|
1669
|
+
);
|
|
1670
|
+
if (consumedNoReturn !== undefined) {
|
|
1671
|
+
return mapMaybePromise(consumedNoReturn, () => ({
|
|
1672
|
+
get entry() {
|
|
1673
|
+
return materializeEntry();
|
|
1674
|
+
},
|
|
1675
|
+
materializeEntry,
|
|
1676
|
+
removed: [],
|
|
1677
|
+
removedHashes: trimmedEntryHashes,
|
|
1678
|
+
appendFacts,
|
|
1679
|
+
shallowEntry,
|
|
1680
|
+
documentTrimmedHeadsProcessed:
|
|
1681
|
+
prepared.documentTrimmedHeadsProcessed,
|
|
1682
|
+
documentPreviousContext: prepared.documentPreviousContext,
|
|
1683
|
+
}));
|
|
1684
|
+
}
|
|
1685
|
+
}
|
|
1686
|
+
const consumedResult =
|
|
1687
|
+
this.entryIndex.consumeNativeTrimmedEntryHashesMaybe(
|
|
1688
|
+
prepared.trimmedEntryHashes,
|
|
1689
|
+
{
|
|
1690
|
+
skipNextHeadUpdates: true,
|
|
1691
|
+
deleteBlocks: false,
|
|
1692
|
+
},
|
|
1693
|
+
);
|
|
1694
|
+
return mapMaybePromise(consumedResult, (removed) => ({
|
|
1695
|
+
get entry() {
|
|
1696
|
+
return materializeEntry();
|
|
1697
|
+
},
|
|
1698
|
+
materializeEntry,
|
|
1699
|
+
removed,
|
|
1700
|
+
appendFacts,
|
|
1701
|
+
shallowEntry,
|
|
1702
|
+
documentTrimmedHeadsProcessed:
|
|
1703
|
+
prepared.documentTrimmedHeadsProcessed,
|
|
1704
|
+
documentPreviousContext: prepared.documentPreviousContext,
|
|
1705
|
+
}));
|
|
1706
|
+
}
|
|
1707
|
+
if (!prepared.trimmedEntries) {
|
|
1708
|
+
return finish();
|
|
1709
|
+
}
|
|
1710
|
+
const trimmedEntries =
|
|
1711
|
+
this.entryIndex.nativeLogEntriesToShallowEntries(
|
|
1712
|
+
prepared.trimmedEntries,
|
|
1713
|
+
);
|
|
1714
|
+
const consumedResult =
|
|
1715
|
+
this.entryIndex.consumeNativeTrimmedEntriesMaybe(trimmedEntries, {
|
|
1716
|
+
skipNextHeadUpdates: true,
|
|
1717
|
+
deleteBlocks: false,
|
|
1718
|
+
});
|
|
1719
|
+
return mapMaybePromise(consumedResult, (removed) => ({
|
|
1720
|
+
get entry() {
|
|
1721
|
+
return materializeEntry();
|
|
1722
|
+
},
|
|
1723
|
+
materializeEntry,
|
|
1724
|
+
removed,
|
|
1725
|
+
appendFacts,
|
|
1726
|
+
shallowEntry,
|
|
1727
|
+
documentTrimmedHeadsProcessed:
|
|
1728
|
+
prepared.documentTrimmedHeadsProcessed,
|
|
1729
|
+
documentPreviousContext: prepared.documentPreviousContext,
|
|
1730
|
+
}));
|
|
1731
|
+
};
|
|
1732
|
+
const rollback = (error: unknown): never | Promise<never> => {
|
|
1733
|
+
this.rollbackNativeAppendGraphHashes([hash]);
|
|
1734
|
+
return this.rollbackNativeAppendBlocksHashes([hash]).then(() => {
|
|
1735
|
+
throw error;
|
|
1736
|
+
});
|
|
1737
|
+
};
|
|
1738
|
+
try {
|
|
1739
|
+
const putResult = this.entryIndex.putNativeCommittedAppendFacts({
|
|
1740
|
+
hash,
|
|
1741
|
+
unique: true,
|
|
1742
|
+
externalNextHashes: nextHashes,
|
|
1743
|
+
shallowEntry,
|
|
1744
|
+
isHead: true,
|
|
1745
|
+
});
|
|
1746
|
+
const result = mapMaybePromise(putResult, finishBlocks);
|
|
1747
|
+
return isPromiseLike(result) ? result.catch(rollback) : result;
|
|
1748
|
+
} catch (error) {
|
|
1749
|
+
return rollback(error);
|
|
1750
|
+
}
|
|
1751
|
+
});
|
|
1752
|
+
});
|
|
1753
|
+
});
|
|
1754
|
+
}
|
|
1755
|
+
|
|
1756
|
+
private appendLocallyPreparedCommitOnlyWithNexts(
|
|
1757
|
+
data: T,
|
|
1758
|
+
appendOptions: AppendOptions<T>,
|
|
1759
|
+
nexts: Sorting.SortableEntry[],
|
|
1760
|
+
properties?: {
|
|
1761
|
+
skipMissingNextJoin?: boolean;
|
|
1762
|
+
resolveTrimmedEntries?: boolean;
|
|
1763
|
+
payloadData?: Uint8Array;
|
|
1764
|
+
includeMaterializationBytes?: boolean;
|
|
1765
|
+
includeAppendFactsBytes?: boolean;
|
|
1766
|
+
},
|
|
1767
|
+
nativeTrimLengthTo?: number,
|
|
1768
|
+
): MaybePromise<PreparedCommitOnlyAppendResult<T> | undefined> {
|
|
1769
|
+
const deferBlockStore = hasPutMany(this._storage);
|
|
1770
|
+
const nativeAppendChainResult = this.createNativePlainAppendCommitOnly(
|
|
1771
|
+
[data],
|
|
1772
|
+
appendOptions,
|
|
1773
|
+
nexts,
|
|
1774
|
+
deferBlockStore,
|
|
1775
|
+
properties?.payloadData ? [properties.payloadData] : undefined,
|
|
1776
|
+
properties?.includeMaterializationBytes,
|
|
1777
|
+
properties?.includeAppendFactsBytes,
|
|
1778
|
+
nativeTrimLengthTo,
|
|
1779
|
+
);
|
|
1780
|
+
return mapMaybePromise(nativeAppendChainResult, (nativeAppendChain) =>
|
|
1781
|
+
this.finishLocallyPreparedCommitOnlyAppend(
|
|
1782
|
+
nativeAppendChain,
|
|
1783
|
+
appendOptions,
|
|
1784
|
+
nexts,
|
|
1785
|
+
deferBlockStore,
|
|
1786
|
+
properties,
|
|
1787
|
+
),
|
|
1788
|
+
);
|
|
1789
|
+
}
|
|
1790
|
+
|
|
1791
|
+
private finishLocallyPreparedCommitOnlyAppend(
|
|
1792
|
+
nativeAppendChain: PreparedAppendCommitOnlyChain<T> | undefined,
|
|
1793
|
+
appendOptions: AppendOptions<T>,
|
|
1794
|
+
nexts: Sorting.SortableEntry[],
|
|
1795
|
+
deferBlockStore: boolean,
|
|
1796
|
+
properties?: {
|
|
1797
|
+
skipMissingNextJoin?: boolean;
|
|
1798
|
+
resolveTrimmedEntries?: boolean;
|
|
1799
|
+
},
|
|
1800
|
+
): MaybePromise<PreparedCommitOnlyAppendResult<T> | undefined> {
|
|
1801
|
+
if (!nativeAppendChain) {
|
|
1802
|
+
return undefined;
|
|
1803
|
+
}
|
|
1804
|
+
|
|
1805
|
+
const appendFacts = nativeAppendChain.appendFacts[0]!;
|
|
1806
|
+
const shallowEntry = nativeAppendChain.shallowEntries[0]!;
|
|
1807
|
+
let materializedEntry: Entry<T> | undefined;
|
|
1808
|
+
const materializeEntry = () => {
|
|
1809
|
+
const entry = materializedEntry ?? nativeAppendChain.materializeEntry(0);
|
|
1810
|
+
entry.init({ encoding: this._encoding, keychain: this._keychain });
|
|
1811
|
+
materializedEntry = entry;
|
|
1812
|
+
return entry;
|
|
1813
|
+
};
|
|
1814
|
+
const finishTrim = (): MaybePromise<PreparedCommitOnlyAppendResult<T>> => {
|
|
1815
|
+
if (nativeAppendChain.trimmedNativeEntryHashes) {
|
|
1816
|
+
const trimmedEntryHashes = [
|
|
1817
|
+
...new Set(nativeAppendChain.trimmedNativeEntryHashes),
|
|
1818
|
+
];
|
|
1819
|
+
if (
|
|
1820
|
+
properties?.resolveTrimmedEntries === false &&
|
|
1821
|
+
nativeAppendChain.trimmedNativeBlocksDeleted === true
|
|
1822
|
+
) {
|
|
1823
|
+
const consumedNoReturn =
|
|
1824
|
+
this.entryIndex.consumeNativeTrimmedEntryHashesNoReturnMaybe(
|
|
1825
|
+
trimmedEntryHashes,
|
|
1826
|
+
{
|
|
1827
|
+
skipNextHeadUpdates: true,
|
|
1828
|
+
deleteBlocks: false,
|
|
1829
|
+
},
|
|
1830
|
+
);
|
|
1831
|
+
if (consumedNoReturn !== undefined) {
|
|
1832
|
+
return mapMaybePromise(consumedNoReturn, () => ({
|
|
1833
|
+
get entry() {
|
|
1834
|
+
return materializeEntry();
|
|
1835
|
+
},
|
|
1836
|
+
materializeEntry,
|
|
1837
|
+
removed: [],
|
|
1838
|
+
removedHashes: trimmedEntryHashes,
|
|
1839
|
+
appendFacts,
|
|
1840
|
+
shallowEntry,
|
|
1841
|
+
}));
|
|
1842
|
+
}
|
|
1843
|
+
}
|
|
1844
|
+
const consumedResult =
|
|
1845
|
+
this.entryIndex.consumeNativeTrimmedEntryHashesMaybe(
|
|
1846
|
+
trimmedEntryHashes,
|
|
1847
|
+
{
|
|
1848
|
+
skipNextHeadUpdates: true,
|
|
1849
|
+
deleteBlocks:
|
|
1850
|
+
nativeAppendChain.trimmedNativeBlocksDeleted !== true,
|
|
1851
|
+
},
|
|
1852
|
+
);
|
|
1853
|
+
return mapMaybePromise(consumedResult, (removed) => ({
|
|
1854
|
+
get entry() {
|
|
1855
|
+
return materializeEntry();
|
|
1856
|
+
},
|
|
1857
|
+
materializeEntry,
|
|
1858
|
+
removed,
|
|
1859
|
+
removedHashes: trimmedEntryHashes,
|
|
1860
|
+
appendFacts,
|
|
1861
|
+
shallowEntry,
|
|
1862
|
+
}));
|
|
1863
|
+
}
|
|
1864
|
+
if (nativeAppendChain.trimmedNativeEntries) {
|
|
1865
|
+
const trimmedEntries = this.entryIndex.nativeLogEntriesToShallowEntries(
|
|
1866
|
+
nativeAppendChain.trimmedNativeEntries,
|
|
1867
|
+
);
|
|
1868
|
+
const consumedResult = this.entryIndex.consumeNativeTrimmedEntriesMaybe(
|
|
1869
|
+
trimmedEntries,
|
|
1870
|
+
{
|
|
1871
|
+
skipNextHeadUpdates: true,
|
|
1872
|
+
deleteBlocks: nativeAppendChain.trimmedNativeBlocksDeleted !== true,
|
|
1873
|
+
},
|
|
1874
|
+
);
|
|
1875
|
+
return mapMaybePromise(consumedResult, (removed) => ({
|
|
1876
|
+
get entry() {
|
|
1877
|
+
return materializeEntry();
|
|
1878
|
+
},
|
|
1879
|
+
materializeEntry,
|
|
1880
|
+
removed,
|
|
1881
|
+
appendFacts,
|
|
1882
|
+
shallowEntry,
|
|
1883
|
+
}));
|
|
1884
|
+
}
|
|
1885
|
+
const trimmedResult = this.trimIfConfigured(appendOptions.trim, {
|
|
1886
|
+
resolveDeletedEntries: properties?.resolveTrimmedEntries,
|
|
1887
|
+
});
|
|
1888
|
+
return mapMaybePromise(trimmedResult, (trimmed) => {
|
|
1889
|
+
const removed = trimmed ?? [];
|
|
1890
|
+
return {
|
|
1891
|
+
get entry() {
|
|
1892
|
+
return materializeEntry();
|
|
1893
|
+
},
|
|
1894
|
+
materializeEntry,
|
|
1895
|
+
removed,
|
|
1896
|
+
appendFacts,
|
|
1897
|
+
shallowEntry,
|
|
1898
|
+
};
|
|
1899
|
+
});
|
|
1900
|
+
};
|
|
1901
|
+
const finishFacts = (): MaybePromise<PreparedCommitOnlyAppendResult<T>> => {
|
|
1902
|
+
const putFactsResult = this.entryIndex.putNativeCommittedAppendFacts({
|
|
1903
|
+
hash: appendFacts.hash,
|
|
1904
|
+
unique: true,
|
|
1905
|
+
externalNextHashes: nexts.map((next) => next.hash),
|
|
1906
|
+
shallowEntry,
|
|
1907
|
+
isHead: true,
|
|
1908
|
+
});
|
|
1909
|
+
return mapMaybePromise(putFactsResult, finishTrim);
|
|
1910
|
+
};
|
|
1911
|
+
const finishBlocks = (): MaybePromise<
|
|
1912
|
+
PreparedCommitOnlyAppendResult<T>
|
|
1913
|
+
> => {
|
|
1914
|
+
if (deferBlockStore && !nativeAppendChain.nativeBlocksCommitted) {
|
|
1915
|
+
return mapMaybePromise(
|
|
1916
|
+
this.putPreparedAppendBlocks(nativeAppendChain.blocks),
|
|
1917
|
+
finishFacts,
|
|
1918
|
+
);
|
|
1919
|
+
}
|
|
1920
|
+
return finishFacts();
|
|
1921
|
+
};
|
|
1922
|
+
const rollback = (error: unknown): never | Promise<never> => {
|
|
1923
|
+
if (nativeAppendChain.nativeGraphUpdated) {
|
|
1924
|
+
this.rollbackNativeAppendGraphHashes([appendFacts.hash]);
|
|
1925
|
+
}
|
|
1926
|
+
if (nativeAppendChain.nativeBlocksCommitted) {
|
|
1927
|
+
return this.rollbackNativeAppendBlocksHashes([appendFacts.hash]).then(
|
|
1928
|
+
() => {
|
|
1929
|
+
throw error;
|
|
1930
|
+
},
|
|
1931
|
+
);
|
|
1932
|
+
}
|
|
1933
|
+
throw error;
|
|
1934
|
+
};
|
|
1935
|
+
try {
|
|
1936
|
+
let result: MaybePromise<PreparedCommitOnlyAppendResult<T>>;
|
|
1937
|
+
if (!properties?.skipMissingNextJoin && nexts.length > 0) {
|
|
1938
|
+
result = mapMaybePromise(
|
|
1939
|
+
this.joinMissingNexts(materializeEntry(), nexts),
|
|
1940
|
+
finishBlocks,
|
|
1941
|
+
);
|
|
1942
|
+
} else {
|
|
1943
|
+
result = finishBlocks();
|
|
1944
|
+
}
|
|
1945
|
+
return isPromiseLike(result) ? result.catch(rollback) : result;
|
|
1946
|
+
} catch (error) {
|
|
1947
|
+
return rollback(error);
|
|
1948
|
+
}
|
|
1949
|
+
}
|
|
1950
|
+
|
|
1951
|
+
private async appendLocallyPreparedManyIndependent(
|
|
1952
|
+
data: T[],
|
|
1953
|
+
options: AppendOptions<T> = {},
|
|
1954
|
+
properties?: {
|
|
1955
|
+
resolveTrimmedEntries?: boolean;
|
|
1956
|
+
payloadDatas?: Uint8Array[];
|
|
1957
|
+
nexts?: Sorting.SortableEntry[][];
|
|
1958
|
+
},
|
|
1959
|
+
): Promise<
|
|
1960
|
+
| {
|
|
1961
|
+
entries: Entry<T>[];
|
|
1962
|
+
removed: ShallowOrFullEntry<T>[];
|
|
1963
|
+
change: Change<T>;
|
|
1964
|
+
appendFacts: PreparedAppendFacts[];
|
|
1965
|
+
}
|
|
1966
|
+
| undefined
|
|
1967
|
+
> {
|
|
1968
|
+
if (data.length === 0) {
|
|
1969
|
+
return {
|
|
1970
|
+
entries: [],
|
|
1971
|
+
removed: [],
|
|
1972
|
+
change: { added: [], removed: [] },
|
|
1973
|
+
appendFacts: [],
|
|
1974
|
+
};
|
|
1975
|
+
}
|
|
1976
|
+
if (
|
|
1977
|
+
options.canAppend ||
|
|
1978
|
+
options.onChange ||
|
|
1979
|
+
options.meta?.type === EntryType.CUT
|
|
1980
|
+
) {
|
|
1981
|
+
throw new Error(
|
|
1982
|
+
"appendLocallyPreparedManyIndependent only supports trusted plain local appends",
|
|
1983
|
+
);
|
|
1984
|
+
}
|
|
1985
|
+
|
|
1986
|
+
const appendOptions = withCanAppendAlreadyValidated(options);
|
|
1987
|
+
const deferBlockStore = hasPutMany(this._storage);
|
|
1988
|
+
const nativeAppendBatch = await this.createNativePlainAppendEntriesBatch(
|
|
1989
|
+
data,
|
|
1990
|
+
appendOptions,
|
|
1991
|
+
deferBlockStore,
|
|
1992
|
+
properties?.payloadDatas,
|
|
1993
|
+
properties?.nexts,
|
|
1994
|
+
);
|
|
1995
|
+
if (!nativeAppendBatch) {
|
|
1996
|
+
return undefined;
|
|
1997
|
+
}
|
|
1998
|
+
|
|
1999
|
+
const entries = nativeAppendBatch.entries;
|
|
2000
|
+
const externalNextHashes =
|
|
2001
|
+
properties?.nexts?.flatMap((nexts) => nexts.map((next) => next.hash)) ??
|
|
2002
|
+
[];
|
|
2003
|
+
try {
|
|
2004
|
+
if (deferBlockStore && !nativeAppendBatch.nativeBlocksCommitted) {
|
|
2005
|
+
await this.putAppendEntryBlocks(entries, nativeAppendBatch.blocks);
|
|
2006
|
+
}
|
|
2007
|
+
await this.putAppendEntries(
|
|
2008
|
+
entries,
|
|
2009
|
+
appendOptions,
|
|
2010
|
+
externalNextHashes,
|
|
2011
|
+
nativeAppendBatch,
|
|
2012
|
+
entries.map(() => true),
|
|
2013
|
+
);
|
|
2014
|
+
} catch (error) {
|
|
2015
|
+
if (nativeAppendBatch.nativeGraphUpdated) {
|
|
2016
|
+
this.rollbackNativeAppendGraph(entries);
|
|
2017
|
+
}
|
|
2018
|
+
if (nativeAppendBatch.nativeBlocksCommitted) {
|
|
2019
|
+
await this.rollbackNativeAppendBlocks(entries);
|
|
2020
|
+
}
|
|
2021
|
+
throw error;
|
|
2022
|
+
}
|
|
2023
|
+
|
|
2024
|
+
for (const entry of entries) {
|
|
2025
|
+
entry.init({ encoding: this._encoding, keychain: this._keychain });
|
|
2026
|
+
}
|
|
2027
|
+
|
|
2028
|
+
const trimmed = await this.trimIfConfigured(appendOptions.trim, {
|
|
2029
|
+
resolveDeletedEntries: properties?.resolveTrimmedEntries,
|
|
2030
|
+
});
|
|
2031
|
+
const removed = trimmed ?? [];
|
|
2032
|
+
const change: Change<T> = {
|
|
2033
|
+
added: entries.map((entry) => ({ head: true, entry })),
|
|
2034
|
+
removed,
|
|
2035
|
+
};
|
|
2036
|
+
const appendFacts = this.createPreparedAppendFacts(
|
|
2037
|
+
entries,
|
|
2038
|
+
nativeAppendBatch,
|
|
2039
|
+
);
|
|
2040
|
+
|
|
2041
|
+
return { entries, removed, change, appendFacts };
|
|
2042
|
+
}
|
|
2043
|
+
|
|
2044
|
+
private appendLocallyPreparedNativeKnownNoNextCommitOnlyBatch(
|
|
2045
|
+
data: T[],
|
|
2046
|
+
options: AppendOptions<T> = {},
|
|
2047
|
+
properties: {
|
|
2048
|
+
payloadDatas: Uint8Array[];
|
|
2049
|
+
resolveTrimmedEntries?: boolean;
|
|
2050
|
+
allowPreparedNexts?: boolean;
|
|
2051
|
+
retainMaterializationBytes?: boolean;
|
|
2052
|
+
},
|
|
2053
|
+
prepare: (
|
|
2054
|
+
inputs: NativeNoNextCommitInput[],
|
|
2055
|
+
) => MaybePromise<Array<NativePreparedNoNextCommit | undefined> | undefined>,
|
|
2056
|
+
): MaybePromise<PreparedCommitOnlyAppendBatchResult<T> | undefined> {
|
|
2057
|
+
if (data.length === 0) {
|
|
2058
|
+
return {
|
|
2059
|
+
entries: [],
|
|
2060
|
+
materializeEntries: [],
|
|
2061
|
+
removed: [],
|
|
2062
|
+
appendFacts: [],
|
|
2063
|
+
};
|
|
2064
|
+
}
|
|
2065
|
+
if (data.length !== properties.payloadDatas.length) {
|
|
2066
|
+
throw new Error("Mismatched native batch payload count");
|
|
2067
|
+
}
|
|
2068
|
+
const resolvedTrim = options.trim ?? this._trim.options;
|
|
2069
|
+
const supportsNativeTrim =
|
|
2070
|
+
!resolvedTrim ||
|
|
2071
|
+
(resolvedTrim.type === "length" &&
|
|
2072
|
+
!resolvedTrim.filter?.canTrim &&
|
|
2073
|
+
properties.resolveTrimmedEntries === false);
|
|
2074
|
+
if (
|
|
2075
|
+
options.canAppend ||
|
|
2076
|
+
options.onChange ||
|
|
2077
|
+
options.encryption ||
|
|
2078
|
+
options.signers ||
|
|
2079
|
+
options.identity ||
|
|
2080
|
+
options.meta?.timestamp ||
|
|
2081
|
+
options.meta?.type === EntryType.CUT ||
|
|
2082
|
+
(options.meta?.next != null && options.meta.next.length !== 0) ||
|
|
2083
|
+
options.meta?.gidSeed ||
|
|
2084
|
+
!supportsNativeTrim ||
|
|
2085
|
+
(this._hasCustomCanAppend && !canAppendAlreadyValidated(options))
|
|
2086
|
+
) {
|
|
2087
|
+
return undefined;
|
|
2088
|
+
}
|
|
2089
|
+
const identity = this._identity;
|
|
2090
|
+
if (!(identity instanceof Ed25519Keypair) || !hasPutMany(this._storage)) {
|
|
2091
|
+
return undefined;
|
|
2092
|
+
}
|
|
2093
|
+
const nativeTrimLengthTo = this.getNativeCommitOnlyTrimLengthTo(
|
|
2094
|
+
options.trim,
|
|
2095
|
+
properties.resolveTrimmedEntries,
|
|
2096
|
+
);
|
|
2097
|
+
const entryType = options.meta?.type ?? EntryType.APPEND;
|
|
2098
|
+
const rows = properties.payloadDatas.map((payloadData) => {
|
|
2099
|
+
const gid = EntryV0.createGid() as string;
|
|
2100
|
+
const timestamp = this._hlc.now();
|
|
2101
|
+
return {
|
|
2102
|
+
gid,
|
|
2103
|
+
timestamp,
|
|
2104
|
+
payloadData,
|
|
2105
|
+
input: {
|
|
2106
|
+
clockId: identity.publicKey.bytes,
|
|
2107
|
+
privateKey: identity.privateKey.privateKey,
|
|
2108
|
+
publicKey: identity.publicKey.publicKey,
|
|
2109
|
+
wallTime: timestamp.wallTime,
|
|
2110
|
+
logical: timestamp.logical,
|
|
2111
|
+
gid,
|
|
2112
|
+
type: entryType,
|
|
2113
|
+
metaData: options.meta?.data,
|
|
2114
|
+
payloadData,
|
|
2115
|
+
resolveTrimmedEntries: properties.resolveTrimmedEntries,
|
|
2116
|
+
trimLengthTo: nativeTrimLengthTo,
|
|
2117
|
+
} satisfies NativeNoNextCommitInput,
|
|
2118
|
+
};
|
|
2119
|
+
});
|
|
2120
|
+
const preparedValue = prepare(rows.map((row) => row.input));
|
|
2121
|
+
return mapMaybePromise(preparedValue, (preparedRows) => {
|
|
2122
|
+
if (!preparedRows || preparedRows.length !== rows.length) {
|
|
2123
|
+
return undefined;
|
|
2124
|
+
}
|
|
2125
|
+
const appendFacts: PreparedAppendFacts[] = [];
|
|
2126
|
+
const materializeEntries: Array<() => Entry<T>> = [];
|
|
2127
|
+
const retainMaterializationBytesFns: Array<() => void> = [];
|
|
2128
|
+
const indexRows: Parameters<
|
|
2129
|
+
EntryIndex<T>["putNativeCommittedAppendFactsBatch"]
|
|
2130
|
+
>[0] = [];
|
|
2131
|
+
const trimmedEntryHashes: string[] = [];
|
|
2132
|
+
const documentTrimmedHeadsProcessed: boolean[] = [];
|
|
2133
|
+
for (let index = 0; index < rows.length; index++) {
|
|
2134
|
+
const row = rows[index]!;
|
|
2135
|
+
const prepared = preparedRows[index];
|
|
2136
|
+
if (!prepared) {
|
|
2137
|
+
return undefined;
|
|
2138
|
+
}
|
|
2139
|
+
const hash = prepared.cid ?? prepared.hash;
|
|
2140
|
+
if (!hash) {
|
|
2141
|
+
return undefined;
|
|
2142
|
+
}
|
|
2143
|
+
const shouldRetainMaterializationBytes =
|
|
2144
|
+
properties.retainMaterializationBytes === true || !!resolvedTrim;
|
|
2145
|
+
let retainedMaterializationBytes: Uint8Array | undefined;
|
|
2146
|
+
const retainMaterializationBytes = () => {
|
|
2147
|
+
if (
|
|
2148
|
+
retainedMaterializationBytes ||
|
|
2149
|
+
!shouldRetainMaterializationBytes ||
|
|
2150
|
+
prepared.bytes
|
|
2151
|
+
) {
|
|
2152
|
+
return;
|
|
2153
|
+
}
|
|
2154
|
+
const bytes =
|
|
2155
|
+
prepared.getBytes?.(hash) ??
|
|
2156
|
+
(this._storage.get(hash) as Uint8Array | undefined);
|
|
2157
|
+
if (
|
|
2158
|
+
bytes &&
|
|
2159
|
+
typeof (bytes as { then?: unknown }).then !== "function"
|
|
2160
|
+
) {
|
|
2161
|
+
retainedMaterializationBytes = bytes;
|
|
2162
|
+
}
|
|
2163
|
+
};
|
|
2164
|
+
const effectiveNextHashes = prepared.next ?? EMPTY_NEXT_HASHES;
|
|
2165
|
+
if (
|
|
2166
|
+
effectiveNextHashes.length !== 0 &&
|
|
2167
|
+
properties.allowPreparedNexts !== true
|
|
2168
|
+
) {
|
|
2169
|
+
return undefined;
|
|
2170
|
+
}
|
|
2171
|
+
const effectiveGid = prepared.gid ?? row.gid;
|
|
2172
|
+
let clock: Clock | undefined;
|
|
2173
|
+
const getClock = () =>
|
|
2174
|
+
(clock ??= new Clock({
|
|
2175
|
+
id: identity.publicKey.bytes,
|
|
2176
|
+
timestamp: row.timestamp,
|
|
2177
|
+
}));
|
|
2178
|
+
let shallowEntry: ShallowEntry | undefined;
|
|
2179
|
+
const getShallowEntry = () =>
|
|
2180
|
+
(shallowEntry ??= new ShallowEntry({
|
|
2181
|
+
hash,
|
|
2182
|
+
payloadSize: row.payloadData.byteLength,
|
|
2183
|
+
head: true,
|
|
2184
|
+
meta: new ShallowMeta({
|
|
2185
|
+
gid: effectiveGid,
|
|
2186
|
+
data: options.meta?.data,
|
|
2187
|
+
clock: getClock(),
|
|
2188
|
+
next: effectiveNextHashes,
|
|
2189
|
+
type: entryType,
|
|
2190
|
+
}),
|
|
2191
|
+
}));
|
|
2192
|
+
const facts: PreparedAppendFacts = {
|
|
2193
|
+
hash,
|
|
2194
|
+
gid: effectiveGid,
|
|
2195
|
+
next: effectiveNextHashes,
|
|
2196
|
+
wallTime: row.timestamp.wallTime,
|
|
2197
|
+
logical: row.timestamp.logical,
|
|
2198
|
+
clockId: identity.publicKey.bytes,
|
|
2199
|
+
type: entryType,
|
|
2200
|
+
metaData: options.meta?.data,
|
|
2201
|
+
payloadSize: row.payloadData.byteLength,
|
|
2202
|
+
metaBytes: prepared.metaBytes,
|
|
2203
|
+
hashDigestBytes: prepared.hashDigestBytes,
|
|
2204
|
+
};
|
|
2205
|
+
let materializedEntry: Entry<T> | undefined;
|
|
2206
|
+
const materializeEntry = () => {
|
|
2207
|
+
if (materializedEntry) {
|
|
2208
|
+
return materializedEntry;
|
|
2209
|
+
}
|
|
2210
|
+
const bytes =
|
|
2211
|
+
prepared.bytes ??
|
|
2212
|
+
retainedMaterializationBytes ??
|
|
2213
|
+
prepared.getBytes?.(hash) ??
|
|
2214
|
+
(this._storage.get(hash) as Uint8Array | undefined);
|
|
2215
|
+
if (
|
|
2216
|
+
!bytes ||
|
|
2217
|
+
typeof (bytes as { then?: unknown }).then === "function"
|
|
2218
|
+
) {
|
|
2219
|
+
throw new Error("Missing synchronous native append block bytes");
|
|
2220
|
+
}
|
|
2221
|
+
const entry = deserialize(bytes, Entry) as Entry<T>;
|
|
2222
|
+
entry.hash = hash;
|
|
2223
|
+
entry.size = prepared.byteLength;
|
|
2224
|
+
entry.createdLocally = true;
|
|
2225
|
+
Entry.prepareShallowEntry(entry, getShallowEntry());
|
|
2226
|
+
entry.init({ encoding: this._encoding, keychain: this._keychain });
|
|
2227
|
+
materializedEntry = entry;
|
|
2228
|
+
return entry;
|
|
2229
|
+
};
|
|
2230
|
+
appendFacts.push(facts);
|
|
2231
|
+
materializeEntries.push(materializeEntry);
|
|
2232
|
+
retainMaterializationBytesFns.push(retainMaterializationBytes);
|
|
2233
|
+
indexRows.push({
|
|
2234
|
+
hash,
|
|
2235
|
+
unique: true,
|
|
2236
|
+
externalNextHashes: effectiveNextHashes,
|
|
2237
|
+
getShallowEntry,
|
|
2238
|
+
isHead: true,
|
|
2239
|
+
});
|
|
2240
|
+
if (prepared.trimmedEntryHashes?.length) {
|
|
2241
|
+
trimmedEntryHashes.push(...prepared.trimmedEntryHashes);
|
|
2242
|
+
}
|
|
2243
|
+
documentTrimmedHeadsProcessed.push(
|
|
2244
|
+
prepared.documentTrimmedHeadsProcessed === true,
|
|
2245
|
+
);
|
|
2246
|
+
}
|
|
2247
|
+
const rollback = (error: unknown): never | Promise<never> => {
|
|
2248
|
+
this.rollbackNativeAppendGraphHashes(
|
|
2249
|
+
appendFacts.map((facts) => facts.hash),
|
|
2250
|
+
);
|
|
2251
|
+
return this.rollbackNativeAppendBlocksHashes(
|
|
2252
|
+
appendFacts.map((facts) => facts.hash),
|
|
2253
|
+
).then(() => {
|
|
2254
|
+
throw error;
|
|
2255
|
+
});
|
|
2256
|
+
};
|
|
2257
|
+
const finish = (): PreparedCommitOnlyAppendBatchResult<T> => {
|
|
2258
|
+
for (const retainMaterializationBytes of retainMaterializationBytesFns) {
|
|
2259
|
+
retainMaterializationBytes();
|
|
2260
|
+
}
|
|
2261
|
+
let entries: Entry<T>[] | undefined;
|
|
2262
|
+
return {
|
|
2263
|
+
get entries() {
|
|
2264
|
+
return (entries ??= materializeEntries.map((materializeEntry) =>
|
|
2265
|
+
materializeEntry(),
|
|
2266
|
+
));
|
|
2267
|
+
},
|
|
2268
|
+
materializeEntries,
|
|
2269
|
+
removed: [],
|
|
2270
|
+
removedHashes:
|
|
2271
|
+
trimmedEntryHashes.length > 0
|
|
2272
|
+
? normalizedUniqueStrings(trimmedEntryHashes)
|
|
2273
|
+
: undefined,
|
|
2274
|
+
appendFacts,
|
|
2275
|
+
documentTrimmedHeadsProcessed,
|
|
2276
|
+
};
|
|
2277
|
+
};
|
|
2278
|
+
const finishTrim = ():
|
|
2279
|
+
| PreparedCommitOnlyAppendBatchResult<T>
|
|
2280
|
+
| Promise<PreparedCommitOnlyAppendBatchResult<T>>
|
|
2281
|
+
| undefined => {
|
|
2282
|
+
if (trimmedEntryHashes.length === 0) {
|
|
2283
|
+
return finish();
|
|
2284
|
+
}
|
|
2285
|
+
if (
|
|
2286
|
+
properties.resolveTrimmedEntries !== false &&
|
|
2287
|
+
!documentTrimmedHeadsProcessed.every(Boolean)
|
|
2288
|
+
) {
|
|
2289
|
+
return undefined;
|
|
2290
|
+
}
|
|
2291
|
+
const uniqueTrimmedHashes = normalizedUniqueStrings(trimmedEntryHashes);
|
|
2292
|
+
const consumedNoReturn =
|
|
2293
|
+
this.entryIndex.consumeNativeTrimmedEntryHashesNoReturnMaybe(
|
|
2294
|
+
uniqueTrimmedHashes,
|
|
2295
|
+
{
|
|
2296
|
+
skipNextHeadUpdates: true,
|
|
2297
|
+
deleteBlocks: false,
|
|
2298
|
+
},
|
|
2299
|
+
);
|
|
2300
|
+
if (consumedNoReturn === undefined) {
|
|
2301
|
+
return undefined;
|
|
2302
|
+
}
|
|
2303
|
+
return mapMaybePromise(consumedNoReturn, finish);
|
|
2304
|
+
};
|
|
2305
|
+
try {
|
|
2306
|
+
const putResult =
|
|
2307
|
+
this.entryIndex.putNativeCommittedAppendFactsBatch(indexRows);
|
|
2308
|
+
const result = mapMaybePromise(putResult, finishTrim);
|
|
2309
|
+
return isPromiseLike(result) ? result.catch(rollback) : result;
|
|
2310
|
+
} catch (error) {
|
|
2311
|
+
return rollback(error);
|
|
2312
|
+
}
|
|
2313
|
+
});
|
|
2314
|
+
}
|
|
2315
|
+
|
|
2316
|
+
private createPreparedAppendFacts(
|
|
2317
|
+
entries: Entry<T>[],
|
|
2318
|
+
prepared?: PreparedAppendChain<T>,
|
|
2319
|
+
): PreparedAppendFacts[] {
|
|
2320
|
+
if (prepared?.appendFacts?.length === entries.length) {
|
|
2321
|
+
return prepared.appendFacts;
|
|
2322
|
+
}
|
|
2323
|
+
return entries.map((entry, index) => {
|
|
2324
|
+
const shallowEntry = prepared?.shallowEntries[index];
|
|
2325
|
+
if (shallowEntry) {
|
|
2326
|
+
return {
|
|
2327
|
+
hash: shallowEntry.hash,
|
|
2328
|
+
gid: shallowEntry.meta.gid,
|
|
2329
|
+
next: shallowEntry.meta.next,
|
|
2330
|
+
wallTime: shallowEntry.meta.clock.timestamp.wallTime,
|
|
2331
|
+
logical: shallowEntry.meta.clock.timestamp.logical,
|
|
2332
|
+
clockId: shallowEntry.meta.clock.id,
|
|
2333
|
+
type: shallowEntry.meta.type,
|
|
2334
|
+
metaData: shallowEntry.meta.data,
|
|
2335
|
+
payloadSize: shallowEntry.payloadSize,
|
|
2336
|
+
metaBytes: (entry as EntryWithMetaBytes).getMetaBytes?.(),
|
|
2337
|
+
hashDigestBytes: (entry as EntryWithMetaBytes).getHashDigestBytes?.(),
|
|
2338
|
+
};
|
|
2339
|
+
}
|
|
2340
|
+
return {
|
|
2341
|
+
hash: entry.hash,
|
|
2342
|
+
gid: entry.meta.gid,
|
|
2343
|
+
next: entry.meta.next,
|
|
2344
|
+
wallTime: entry.meta.clock.timestamp.wallTime,
|
|
2345
|
+
logical: entry.meta.clock.timestamp.logical,
|
|
2346
|
+
clockId: entry.meta.clock.id,
|
|
2347
|
+
type: entry.meta.type,
|
|
2348
|
+
metaData: entry.meta.data,
|
|
2349
|
+
payloadSize: entry.payload.byteLength,
|
|
2350
|
+
metaBytes: (entry as EntryWithMetaBytes).getMetaBytes?.(),
|
|
2351
|
+
hashDigestBytes: (entry as EntryWithMetaBytes).getHashDigestBytes?.(),
|
|
2352
|
+
};
|
|
2353
|
+
});
|
|
2354
|
+
}
|
|
2355
|
+
|
|
2356
|
+
async appendMany(
|
|
2357
|
+
data: T[],
|
|
2358
|
+
options: AppendOptions<T> = {},
|
|
2359
|
+
): Promise<{ entries: Entry<T>[]; removed: ShallowOrFullEntry<T>[] }> {
|
|
2360
|
+
if (data.length === 0) {
|
|
2361
|
+
return { entries: [], removed: [] };
|
|
2362
|
+
}
|
|
2363
|
+
if (options.meta?.type === EntryType.CUT) {
|
|
2364
|
+
throw new Error("appendMany does not support CUT entries");
|
|
2365
|
+
}
|
|
2366
|
+
|
|
2367
|
+
let nexts = await this.getNextsForAppend(options);
|
|
2368
|
+
const initialNexts = nexts;
|
|
2369
|
+
const entries: Entry<T>[] = [];
|
|
2370
|
+
const pendingDeletes: (
|
|
2371
|
+
| PendingDelete<T>
|
|
2372
|
+
| { entry: ShallowOrFullEntry<T>; fn: undefined }
|
|
2373
|
+
)[] = [];
|
|
2374
|
+
const deferBlockStore = hasPutMany(this._storage);
|
|
2375
|
+
const nativeAppendChain = await this.createNativePlainAppendChain(
|
|
2376
|
+
data,
|
|
2377
|
+
options,
|
|
2378
|
+
nexts,
|
|
2379
|
+
deferBlockStore,
|
|
2380
|
+
);
|
|
2381
|
+
|
|
2382
|
+
if (nativeAppendChain) {
|
|
2383
|
+
entries.push(...nativeAppendChain.entries);
|
|
2384
|
+
nexts = [entries[entries.length - 1]!];
|
|
2385
|
+
} else {
|
|
2386
|
+
for (const item of data) {
|
|
2387
|
+
const entry = await this.createAppendEntry(item, options, nexts, {
|
|
2388
|
+
deferStore: deferBlockStore,
|
|
2389
|
+
});
|
|
2390
|
+
entries.push(entry);
|
|
2391
|
+
nexts = [entry];
|
|
2392
|
+
}
|
|
2393
|
+
}
|
|
2394
|
+
|
|
2395
|
+
try {
|
|
2396
|
+
await this.joinMissingNexts(entries[0]!, initialNexts);
|
|
2397
|
+
if (deferBlockStore && !nativeAppendChain?.nativeBlocksCommitted) {
|
|
2398
|
+
await this.putAppendEntryBlocks(entries, nativeAppendChain?.blocks);
|
|
2399
|
+
}
|
|
2400
|
+
await this.putAppendEntries(
|
|
2401
|
+
entries,
|
|
2402
|
+
options,
|
|
2403
|
+
initialNexts.map((entry) => entry.hash),
|
|
2404
|
+
nativeAppendChain,
|
|
2405
|
+
);
|
|
2406
|
+
} catch (error) {
|
|
2407
|
+
if (nativeAppendChain?.nativeGraphUpdated) {
|
|
2408
|
+
this.rollbackNativeAppendGraph(entries);
|
|
2409
|
+
}
|
|
2410
|
+
if (nativeAppendChain?.nativeBlocksCommitted) {
|
|
2411
|
+
await this.rollbackNativeAppendBlocks(entries);
|
|
2412
|
+
}
|
|
2413
|
+
throw error;
|
|
2414
|
+
}
|
|
2415
|
+
|
|
2416
|
+
for (const entry of entries) {
|
|
2417
|
+
entry.init({ encoding: this._encoding, keychain: this._keychain });
|
|
2418
|
+
}
|
|
2419
|
+
|
|
2420
|
+
const trimmed = await this.trimIfConfigured(options?.trim);
|
|
2421
|
+
if (trimmed) {
|
|
2422
|
+
for (const entry of trimmed) {
|
|
2423
|
+
pendingDeletes.push({ entry, fn: undefined });
|
|
2424
|
+
}
|
|
2425
|
+
}
|
|
2426
|
+
|
|
2427
|
+
const removed = pendingDeletes.map((x) => x.entry);
|
|
2428
|
+
const changes: Change<T> = {
|
|
2429
|
+
added: entries.map((entry, index) => ({
|
|
2430
|
+
head: index === entries.length - 1,
|
|
2431
|
+
entry,
|
|
2432
|
+
})),
|
|
2433
|
+
removed,
|
|
2434
|
+
};
|
|
2435
|
+
|
|
2436
|
+
await (options?.onChange || this._onChange)?.(changes);
|
|
2437
|
+
await Promise.all(pendingDeletes.map((x) => x.fn?.()));
|
|
2438
|
+
return { entries, removed };
|
|
2439
|
+
}
|
|
2440
|
+
|
|
2441
|
+
private createNativePlainAppendChain(
|
|
2442
|
+
data: T[],
|
|
2443
|
+
options: AppendOptions<T>,
|
|
2444
|
+
nexts: Sorting.SortableEntry[],
|
|
2445
|
+
deferBlockStore: boolean,
|
|
2446
|
+
payloadDatas?: Uint8Array[],
|
|
2447
|
+
): Promise<PreparedAppendChain<T> | undefined> {
|
|
2448
|
+
const canAppendAlreadyValidatedForOptions =
|
|
2449
|
+
canAppendAlreadyValidated(options);
|
|
2450
|
+
if (
|
|
2451
|
+
!deferBlockStore ||
|
|
2452
|
+
options.encryption ||
|
|
2453
|
+
options.signers ||
|
|
2454
|
+
options.canAppend ||
|
|
2455
|
+
(this._hasCustomCanAppend && !canAppendAlreadyValidatedForOptions) ||
|
|
2456
|
+
options.meta?.timestamp ||
|
|
2457
|
+
options.meta?.type === EntryType.CUT
|
|
2458
|
+
) {
|
|
2459
|
+
return Promise.resolve(undefined);
|
|
2460
|
+
}
|
|
2461
|
+
|
|
2462
|
+
const nativeGraph =
|
|
2463
|
+
!this.entryIndex.properties.onGidRemoved &&
|
|
2464
|
+
(this.entryIndex.properties.nativeGraph?.graph
|
|
2465
|
+
.prepareEntryV0PlainChainCommit ||
|
|
2466
|
+
this.entryIndex.properties.nativeGraph?.graph
|
|
2467
|
+
.prepareEntryV0PlainEntryCommit ||
|
|
2468
|
+
this.entryIndex.properties.nativeGraph?.graph
|
|
2469
|
+
.prepareEntryV0PlainChainAndPut ||
|
|
2470
|
+
this.entryIndex.properties.nativeGraph?.graph
|
|
2471
|
+
.prepareEntryV0PlainEntryAndPut)
|
|
2472
|
+
? this.entryIndex.properties.nativeGraph.graph
|
|
2473
|
+
: undefined;
|
|
2474
|
+
return EntryV0.createPlainAppendChainBatch<T>({
|
|
2475
|
+
data,
|
|
2476
|
+
meta: {
|
|
2477
|
+
clocks: () =>
|
|
2478
|
+
data.map(
|
|
2479
|
+
() =>
|
|
2480
|
+
new Clock({
|
|
2481
|
+
id: this._identity.publicKey.bytes,
|
|
2482
|
+
timestamp: this._hlc.now(),
|
|
2483
|
+
}),
|
|
2484
|
+
),
|
|
2485
|
+
type: options.meta?.type,
|
|
2486
|
+
gidSeed: options.meta?.gidSeed,
|
|
2487
|
+
data: options.meta?.data,
|
|
2488
|
+
next: nexts,
|
|
2489
|
+
},
|
|
2490
|
+
encoding: this._encoding,
|
|
2491
|
+
payloadDatas,
|
|
2492
|
+
identity: options.identity || this._identity,
|
|
2493
|
+
deferStore: deferBlockStore,
|
|
2494
|
+
cachePreparedEntries: false,
|
|
2495
|
+
nativeGraph,
|
|
2496
|
+
nativeBlockStore: this._storage,
|
|
2497
|
+
});
|
|
2498
|
+
}
|
|
2499
|
+
|
|
2500
|
+
private createNativePlainAppendCommitOnly(
|
|
2501
|
+
data: T[],
|
|
2502
|
+
options: AppendOptions<T>,
|
|
2503
|
+
nexts: Sorting.SortableEntry[],
|
|
2504
|
+
deferBlockStore: boolean,
|
|
2505
|
+
payloadDatas?: Uint8Array[],
|
|
2506
|
+
includeMaterializationBytes?: boolean,
|
|
2507
|
+
includeAppendFactsBytes?: boolean,
|
|
2508
|
+
nativeTrimLengthTo?: number,
|
|
2509
|
+
): MaybePromise<PreparedAppendCommitOnlyChain<T> | undefined> {
|
|
2510
|
+
const canAppendAlreadyValidatedForOptions =
|
|
2511
|
+
canAppendAlreadyValidated(options);
|
|
2512
|
+
if (
|
|
2513
|
+
data.length !== 1 ||
|
|
2514
|
+
!deferBlockStore ||
|
|
2515
|
+
options.encryption ||
|
|
2516
|
+
options.signers ||
|
|
2517
|
+
options.canAppend ||
|
|
2518
|
+
(this._hasCustomCanAppend && !canAppendAlreadyValidatedForOptions) ||
|
|
2519
|
+
options.meta?.timestamp ||
|
|
2520
|
+
options.meta?.type === EntryType.CUT
|
|
2521
|
+
) {
|
|
2522
|
+
return undefined;
|
|
2523
|
+
}
|
|
2524
|
+
|
|
2525
|
+
const nativeGraph =
|
|
2526
|
+
!this.entryIndex.properties.onGidRemoved &&
|
|
2527
|
+
(this.entryIndex.properties.nativeGraph?.graph
|
|
2528
|
+
.prepareEntryV0PlainEntryCommit ||
|
|
2529
|
+
this.entryIndex.properties.nativeGraph?.graph
|
|
2530
|
+
.prepareEntryV0PlainEntryAndPut)
|
|
2531
|
+
? this.entryIndex.properties.nativeGraph.graph
|
|
2532
|
+
: undefined;
|
|
2533
|
+
if (!nativeGraph) {
|
|
2534
|
+
return undefined;
|
|
2535
|
+
}
|
|
2536
|
+
return EntryV0.createPlainAppendChainCommitOnly<T>({
|
|
2537
|
+
data,
|
|
2538
|
+
meta: {
|
|
2539
|
+
clocks: () => [
|
|
2540
|
+
new Clock({
|
|
2541
|
+
id: this._identity.publicKey.bytes,
|
|
2542
|
+
timestamp: this._hlc.now(),
|
|
2543
|
+
}),
|
|
2544
|
+
],
|
|
2545
|
+
type: options.meta?.type,
|
|
2546
|
+
gidSeed: options.meta?.gidSeed,
|
|
2547
|
+
data: options.meta?.data,
|
|
2548
|
+
next: nexts,
|
|
2549
|
+
},
|
|
2550
|
+
encoding: this._encoding,
|
|
2551
|
+
payloadDatas,
|
|
2552
|
+
identity: options.identity || this._identity,
|
|
2553
|
+
deferStore: deferBlockStore,
|
|
2554
|
+
nativeGraph,
|
|
2555
|
+
nativeBlockStore: this._storage,
|
|
2556
|
+
includeMaterializationBytes,
|
|
2557
|
+
includeAppendFactsBytes,
|
|
2558
|
+
nativeTrimLengthTo,
|
|
2559
|
+
});
|
|
2560
|
+
}
|
|
2561
|
+
|
|
2562
|
+
private async createNativePlainAppendEntriesBatch(
|
|
2563
|
+
data: T[],
|
|
2564
|
+
options: AppendOptions<T>,
|
|
2565
|
+
deferBlockStore: boolean,
|
|
2566
|
+
payloadDatas?: Uint8Array[],
|
|
2567
|
+
nexts?: Sorting.SortableEntry[][],
|
|
2568
|
+
): Promise<PreparedAppendChain<T> | undefined> {
|
|
2569
|
+
const canAppendAlreadyValidatedForOptions =
|
|
2570
|
+
canAppendAlreadyValidated(options);
|
|
2571
|
+
if (
|
|
2572
|
+
!deferBlockStore ||
|
|
2573
|
+
data.length === 0 ||
|
|
2574
|
+
options.encryption ||
|
|
2575
|
+
options.signers ||
|
|
2576
|
+
options.canAppend ||
|
|
2577
|
+
(this._hasCustomCanAppend && !canAppendAlreadyValidatedForOptions) ||
|
|
2578
|
+
options.meta?.timestamp ||
|
|
2579
|
+
options.meta?.type === EntryType.CUT ||
|
|
2580
|
+
options.meta?.gidSeed ||
|
|
2581
|
+
options.meta?.next ||
|
|
2582
|
+
(nexts && nexts.length !== data.length) ||
|
|
2583
|
+
(payloadDatas && payloadDatas.length !== data.length)
|
|
2584
|
+
) {
|
|
2585
|
+
return undefined;
|
|
2586
|
+
}
|
|
2587
|
+
|
|
2588
|
+
const nativeGraph =
|
|
2589
|
+
!this.entryIndex.properties.onGidRemoved &&
|
|
2590
|
+
this.entryIndex.properties.nativeGraph?.graph
|
|
2591
|
+
? this.entryIndex.properties.nativeGraph.graph
|
|
2592
|
+
: undefined;
|
|
2593
|
+
|
|
2594
|
+
const generatedGids = EntryV0.createGids(data.length);
|
|
2595
|
+
const gids = generatedGids.map((generatedGid, index) => {
|
|
2596
|
+
const entryNexts = nexts?.[index];
|
|
2597
|
+
if (!entryNexts || entryNexts.length === 0) {
|
|
2598
|
+
return generatedGid;
|
|
2599
|
+
}
|
|
2600
|
+
let gid = entryNexts[0]!.meta.gid;
|
|
2601
|
+
for (let i = 1; i < entryNexts.length; i++) {
|
|
2602
|
+
const nextGid = entryNexts[i]!.meta.gid;
|
|
2603
|
+
if (nextGid < gid) {
|
|
2604
|
+
gid = nextGid;
|
|
2605
|
+
}
|
|
2606
|
+
}
|
|
2607
|
+
return gid;
|
|
2608
|
+
});
|
|
2609
|
+
const clockId = this._identity.publicKey.bytes;
|
|
2610
|
+
const clocks = this._hlc.nowBatch(data.length).map(
|
|
2611
|
+
(timestamp) =>
|
|
2612
|
+
new Clock({
|
|
2613
|
+
id: clockId,
|
|
2614
|
+
timestamp,
|
|
2615
|
+
}),
|
|
2616
|
+
);
|
|
2617
|
+
const metaDatas = data.map(() => options.meta?.data);
|
|
2618
|
+
const directBatch = nativeGraph?.prepareEntryV0PlainEntriesCommit
|
|
2619
|
+
? await EntryV0.createPlainAppendEntriesBatch<T>({
|
|
2620
|
+
data,
|
|
2621
|
+
payloadDatas,
|
|
2622
|
+
meta: {
|
|
2623
|
+
clocks: () => clocks,
|
|
2624
|
+
gids,
|
|
2625
|
+
nexts,
|
|
2626
|
+
type: options.meta?.type,
|
|
2627
|
+
datas: metaDatas,
|
|
2628
|
+
},
|
|
2629
|
+
encoding: this._encoding,
|
|
2630
|
+
identity: options.identity || this._identity,
|
|
2631
|
+
deferStore: deferBlockStore,
|
|
2632
|
+
cachePreparedEntries: false,
|
|
2633
|
+
nativeGraph,
|
|
2634
|
+
nativeBlockStore: this._storage,
|
|
2635
|
+
})
|
|
2636
|
+
: undefined;
|
|
2637
|
+
if (directBatch) {
|
|
2638
|
+
return directBatch;
|
|
2639
|
+
}
|
|
2640
|
+
|
|
2641
|
+
const entries: Entry<T>[] = [];
|
|
2642
|
+
const blocks: PreparedEntryBlock[] = [];
|
|
2643
|
+
const shallowEntries: PreparedAppendChain<T>["shallowEntries"] = [];
|
|
2644
|
+
const nativeEntries: NonNullable<PreparedAppendChain<T>["nativeEntries"]> =
|
|
2645
|
+
[];
|
|
2646
|
+
let nativeGraphUpdated = false;
|
|
2647
|
+
let nativeBlocksCommitted = true;
|
|
2648
|
+
for (let i = 0; i < data.length; i++) {
|
|
2649
|
+
const entryNexts = nexts?.[i] ?? [];
|
|
2650
|
+
const prepared = await EntryV0.createPlainAppendChainBatch<T>({
|
|
2651
|
+
data: [data[i]!],
|
|
2652
|
+
payloadDatas: payloadDatas ? [payloadDatas[i]!] : undefined,
|
|
2653
|
+
meta: {
|
|
2654
|
+
clocks: () => [clocks[i]!],
|
|
2655
|
+
gid: entryNexts.length === 0 ? gids[i]! : undefined,
|
|
2656
|
+
type: options.meta?.type,
|
|
2657
|
+
data: metaDatas[i],
|
|
2658
|
+
next: entryNexts,
|
|
2659
|
+
},
|
|
2660
|
+
encoding: this._encoding,
|
|
2661
|
+
identity: options.identity || this._identity,
|
|
2662
|
+
deferStore: deferBlockStore,
|
|
2663
|
+
cachePreparedEntries: false,
|
|
2664
|
+
nativeGraph,
|
|
2665
|
+
nativeBlockStore: this._storage,
|
|
2666
|
+
});
|
|
2667
|
+
if (!prepared) {
|
|
2668
|
+
return undefined;
|
|
2669
|
+
}
|
|
2670
|
+
entries.push(prepared.entries[0]!);
|
|
2671
|
+
if (prepared.blocks) {
|
|
2672
|
+
blocks.push(...prepared.blocks);
|
|
2673
|
+
}
|
|
2674
|
+
shallowEntries.push(...prepared.shallowEntries);
|
|
2675
|
+
if (prepared.nativeEntries) {
|
|
2676
|
+
nativeEntries.push(...prepared.nativeEntries);
|
|
2677
|
+
}
|
|
2678
|
+
nativeGraphUpdated ||= prepared.nativeGraphUpdated === true;
|
|
2679
|
+
nativeBlocksCommitted &&= prepared.nativeBlocksCommitted === true;
|
|
2680
|
+
}
|
|
2681
|
+
|
|
2682
|
+
if (!nativeBlocksCommitted && blocks.length !== entries.length) {
|
|
2683
|
+
return undefined;
|
|
2684
|
+
}
|
|
2685
|
+
return {
|
|
2686
|
+
entries,
|
|
2687
|
+
blocks: blocks.length > 0 ? blocks : undefined,
|
|
2688
|
+
shallowEntries,
|
|
2689
|
+
nativeEntries,
|
|
2690
|
+
nativeGraphUpdated,
|
|
2691
|
+
nativeBlocksCommitted,
|
|
2692
|
+
};
|
|
2693
|
+
}
|
|
2694
|
+
|
|
2695
|
+
private rollbackNativeAppendGraph(entries: Entry<T>[]) {
|
|
2696
|
+
this.rollbackNativeAppendGraphHashes(entries.map((entry) => entry.hash));
|
|
2697
|
+
}
|
|
2698
|
+
|
|
2699
|
+
private rollbackNativeAppendGraphHashes(hashes: string[]) {
|
|
2700
|
+
const graph = this.entryIndex.properties.nativeGraph?.graph;
|
|
2701
|
+
if (!graph) {
|
|
2702
|
+
return;
|
|
2703
|
+
}
|
|
2704
|
+
for (let i = hashes.length - 1; i >= 0; i--) {
|
|
2705
|
+
graph.delete(hashes[i]!);
|
|
2706
|
+
}
|
|
2707
|
+
}
|
|
2708
|
+
|
|
2709
|
+
private async rollbackNativeAppendBlocks(entries: Entry<T>[]) {
|
|
2710
|
+
await this.rollbackNativeAppendBlocksHashes(
|
|
2711
|
+
entries.map((entry) => entry.hash),
|
|
2712
|
+
);
|
|
2713
|
+
}
|
|
2714
|
+
|
|
2715
|
+
private async rollbackNativeAppendBlocksHashes(hashes: string[]) {
|
|
2716
|
+
const storage = this._storage as BlocksWithPutMany;
|
|
2717
|
+
if (typeof storage.rmMany === "function") {
|
|
2718
|
+
await storage.rmMany(hashes);
|
|
2719
|
+
return;
|
|
2720
|
+
}
|
|
2721
|
+
await Promise.all(hashes.map((hash) => this._storage.rm(hash)));
|
|
2722
|
+
}
|
|
2723
|
+
|
|
2724
|
+
private validateExplicitNexts(options: AppendOptions<T>) {
|
|
2725
|
+
if (!options.meta?.next) {
|
|
2726
|
+
return;
|
|
2727
|
+
}
|
|
2728
|
+
for (const n of options.meta.next) {
|
|
2729
|
+
if (!n.hash) {
|
|
2730
|
+
throw new Error(
|
|
2731
|
+
"Expecting nexts to already be saved. missing hash for one or more entries",
|
|
2732
|
+
);
|
|
2733
|
+
}
|
|
2734
|
+
}
|
|
2735
|
+
}
|
|
2736
|
+
|
|
2737
|
+
private getNextsForAppend(
|
|
2738
|
+
options: AppendOptions<T>,
|
|
2739
|
+
): MaybePromise<Sorting.SortableEntry[]> {
|
|
2740
|
+
this.validateExplicitNexts(options);
|
|
2741
|
+
return (
|
|
2742
|
+
options.meta?.next ||
|
|
2743
|
+
this.entryIndex.getHeadsForAppend() ||
|
|
2744
|
+
this.entryIndex
|
|
2745
|
+
.getHeads(undefined, { type: "shape", shape: Sorting.ENTRY_SORT_SHAPE })
|
|
2746
|
+
.all()
|
|
2747
|
+
);
|
|
2748
|
+
}
|
|
2749
|
+
|
|
2750
|
+
private async createAppendEntry(
|
|
2751
|
+
data: T,
|
|
2752
|
+
options: AppendOptions<T>,
|
|
2753
|
+
nexts: Sorting.SortableEntry[],
|
|
2754
|
+
storeOptions?: {
|
|
2755
|
+
deferStore?: boolean;
|
|
2756
|
+
},
|
|
2757
|
+
): Promise<Entry<T>> {
|
|
2758
|
+
const clock = new Clock({
|
|
2759
|
+
id: this._identity.publicKey.bytes,
|
|
2760
|
+
timestamp: options?.meta?.timestamp || this._hlc.now(),
|
|
2761
|
+
});
|
|
2762
|
+
|
|
2763
|
+
const entry = await EntryV0.create<T>({
|
|
2764
|
+
store: this._storage,
|
|
2765
|
+
identity: options.identity || this._identity,
|
|
2766
|
+
signers: options.signers,
|
|
2767
|
+
data,
|
|
2768
|
+
meta: {
|
|
2769
|
+
clock,
|
|
2770
|
+
type: options.meta?.type,
|
|
2771
|
+
gidSeed: options.meta?.gidSeed,
|
|
2772
|
+
data: options.meta?.data,
|
|
2773
|
+
next: nexts,
|
|
2774
|
+
},
|
|
2775
|
+
encoding: this._encoding,
|
|
2776
|
+
encryption: options.encryption
|
|
2777
|
+
? {
|
|
2778
|
+
keypair: options.encryption.keypair,
|
|
2779
|
+
receiver: {
|
|
2780
|
+
...options.encryption.receiver,
|
|
2781
|
+
},
|
|
2782
|
+
}
|
|
2783
|
+
: undefined,
|
|
2784
|
+
canAppend:
|
|
2785
|
+
canAppendAlreadyValidated(options)
|
|
2786
|
+
? undefined
|
|
2787
|
+
: options.canAppend ||
|
|
2788
|
+
(this._hasCustomCanAppend ? this._canAppend : undefined),
|
|
2789
|
+
deferStore: storeOptions?.deferStore,
|
|
2790
|
+
});
|
|
2791
|
+
|
|
2792
|
+
if (!entry.hash) {
|
|
2793
|
+
throw new Error("Unexpected");
|
|
2794
|
+
}
|
|
2795
|
+
return entry;
|
|
2796
|
+
}
|
|
2797
|
+
|
|
2798
|
+
private async joinMissingNexts(
|
|
2799
|
+
entry: Entry<T>,
|
|
2800
|
+
nexts: Sorting.SortableEntry[],
|
|
2801
|
+
) {
|
|
2802
|
+
if (entry.meta.type === EntryType.CUT) {
|
|
2803
|
+
return;
|
|
2804
|
+
}
|
|
2805
|
+
for (const e of nexts) {
|
|
2806
|
+
if (await this.has(e.hash)) {
|
|
2807
|
+
continue;
|
|
2808
|
+
}
|
|
2809
|
+
let nextEntry: Entry<any>;
|
|
2810
|
+
if (e instanceof Entry) {
|
|
2811
|
+
nextEntry = e;
|
|
2812
|
+
} else {
|
|
2813
|
+
const resolved = await this.entryIndex.get(e.hash);
|
|
2814
|
+
if (!resolved) {
|
|
2815
|
+
warn("Unexpected missing entry when joining", e.hash);
|
|
2816
|
+
continue;
|
|
2817
|
+
}
|
|
2818
|
+
nextEntry = resolved;
|
|
2819
|
+
}
|
|
2820
|
+
await this.join([nextEntry]);
|
|
2821
|
+
}
|
|
2822
|
+
}
|
|
2823
|
+
|
|
2824
|
+
private async putAppendEntry(entry: Entry<T>, options: AppendOptions<T>) {
|
|
2825
|
+
await this.entryIndex.put(entry, {
|
|
2826
|
+
unique: true,
|
|
2827
|
+
isHead: true,
|
|
2828
|
+
toMultiHash: false,
|
|
2829
|
+
deferIndexWrite:
|
|
2830
|
+
options.deferIndexWrite ??
|
|
2831
|
+
(options.durability
|
|
2832
|
+
? options.durability === "buffered"
|
|
2833
|
+
: this._appendDurability === "buffered"),
|
|
2834
|
+
});
|
|
2835
|
+
}
|
|
2836
|
+
|
|
2837
|
+
private async putAppendEntries(
|
|
2838
|
+
entries: Entry<T>[],
|
|
2839
|
+
options: AppendOptions<T>,
|
|
2840
|
+
externalNextHashes: string[],
|
|
2841
|
+
preparedAppendChain?: PreparedAppendChain<T>,
|
|
2842
|
+
heads?: boolean[],
|
|
2843
|
+
) {
|
|
2844
|
+
const prepared =
|
|
2845
|
+
preparedAppendChain &&
|
|
2846
|
+
entries.length === preparedAppendChain.entries.length
|
|
2847
|
+
? {
|
|
2848
|
+
shallowEntries: preparedAppendChain.shallowEntries,
|
|
2849
|
+
nativeEntries: preparedAppendChain.nativeEntries,
|
|
2850
|
+
nativeGraphUpdated: preparedAppendChain.nativeGraphUpdated,
|
|
2851
|
+
nativeBlocksCommitted: preparedAppendChain.nativeBlocksCommitted,
|
|
2852
|
+
}
|
|
2853
|
+
: undefined;
|
|
2854
|
+
if (
|
|
2855
|
+
entries.length === 1 &&
|
|
2856
|
+
prepared?.nativeGraphUpdated === true &&
|
|
2857
|
+
!this.entryIndex.properties.onGidRemoved
|
|
2858
|
+
) {
|
|
2859
|
+
await this.entryIndex.putNativeCommittedAppend(entries[0]!, {
|
|
2860
|
+
unique: true,
|
|
2861
|
+
externalNextHashes,
|
|
2862
|
+
shallowEntry: prepared.shallowEntries[0],
|
|
2863
|
+
isHead: heads?.[0] ?? true,
|
|
2864
|
+
});
|
|
2865
|
+
return;
|
|
2866
|
+
}
|
|
2867
|
+
|
|
2868
|
+
await this.entryIndex.putAppendBatch(entries, {
|
|
2869
|
+
unique: true,
|
|
2870
|
+
externalNextHashes,
|
|
2871
|
+
heads,
|
|
2872
|
+
prepared,
|
|
2873
|
+
deferIndexWrite:
|
|
2874
|
+
options.deferIndexWrite ??
|
|
2875
|
+
(options.durability
|
|
2876
|
+
? options.durability === "buffered"
|
|
2877
|
+
: this._appendDurability === "buffered"),
|
|
2878
|
+
});
|
|
2879
|
+
}
|
|
2880
|
+
|
|
2881
|
+
private async putAppendEntryBlocks(
|
|
2882
|
+
entries: Entry<T>[],
|
|
2883
|
+
preparedBlocks?: PreparedEntryBlock[],
|
|
2884
|
+
) {
|
|
2885
|
+
const blocks =
|
|
2886
|
+
preparedBlocks && preparedBlocks.length === entries.length
|
|
2887
|
+
? preparedBlocks
|
|
2888
|
+
: entries.map((entry) => {
|
|
2889
|
+
const prepared = Entry.takePreparedBlock(entry);
|
|
2890
|
+
if (!prepared) {
|
|
2891
|
+
throw new Error("Missing prepared entry block");
|
|
2892
|
+
}
|
|
2893
|
+
return prepared;
|
|
2894
|
+
});
|
|
2895
|
+
|
|
2896
|
+
if (blocks.length === 1 && hasPutKnown(this._storage)) {
|
|
2897
|
+
const block = blocks[0]!;
|
|
2898
|
+
const cidResult = this._storage.putKnown(block.cid, block.block.bytes);
|
|
2899
|
+
const cid = isPromiseLike(cidResult) ? await cidResult : cidResult;
|
|
2900
|
+
if (cid !== block.cid) {
|
|
2901
|
+
throw new Error("Unexpected block cid");
|
|
2902
|
+
}
|
|
2903
|
+
return;
|
|
2904
|
+
}
|
|
2905
|
+
if (hasPutKnownManyColumns(this._storage)) {
|
|
2906
|
+
const cids = new Array<string>(blocks.length);
|
|
2907
|
+
const bytes = new Array<Uint8Array>(blocks.length);
|
|
2908
|
+
for (let i = 0; i < blocks.length; i++) {
|
|
2909
|
+
const block = blocks[i]!;
|
|
2910
|
+
cids[i] = block.cid;
|
|
2911
|
+
bytes[i] = block.block.bytes;
|
|
2912
|
+
}
|
|
2913
|
+
const cidsResult = this._storage.putKnownManyColumns(cids, bytes);
|
|
2914
|
+
const result = isPromiseLike(cidsResult) ? await cidsResult : cidsResult;
|
|
2915
|
+
if (result.length !== blocks.length) {
|
|
2916
|
+
throw new Error("Unexpected block batch result length");
|
|
2917
|
+
}
|
|
2918
|
+
for (let i = 0; i < result.length; i++) {
|
|
2919
|
+
if (result[i] !== cids[i]) {
|
|
2920
|
+
throw new Error("Unexpected block batch cid");
|
|
2921
|
+
}
|
|
2922
|
+
}
|
|
2923
|
+
return;
|
|
2924
|
+
}
|
|
2925
|
+
if (hasPutKnownMany(this._storage)) {
|
|
2926
|
+
const cidsResult = this._storage.putKnownMany(
|
|
2927
|
+
blocks.map((block) => [block.cid, block.block.bytes] as const),
|
|
2928
|
+
);
|
|
2929
|
+
const cids = isPromiseLike(cidsResult) ? await cidsResult : cidsResult;
|
|
2930
|
+
if (cids.length !== blocks.length) {
|
|
2931
|
+
throw new Error("Unexpected block batch result length");
|
|
2932
|
+
}
|
|
2933
|
+
for (let i = 0; i < cids.length; i++) {
|
|
2934
|
+
if (cids[i] !== blocks[i]!.cid) {
|
|
2935
|
+
throw new Error("Unexpected block batch cid");
|
|
2936
|
+
}
|
|
2937
|
+
}
|
|
2938
|
+
return;
|
|
2939
|
+
}
|
|
2940
|
+
const cids = await (this._storage as BlocksWithPutMany).putMany!(blocks);
|
|
2941
|
+
if (cids.length !== blocks.length) {
|
|
2942
|
+
throw new Error("Unexpected block batch result length");
|
|
2943
|
+
}
|
|
2944
|
+
for (let i = 0; i < cids.length; i++) {
|
|
2945
|
+
if (cids[i] !== blocks[i].cid) {
|
|
2946
|
+
throw new Error("Unexpected block batch cid");
|
|
2947
|
+
}
|
|
2948
|
+
}
|
|
2949
|
+
}
|
|
2950
|
+
|
|
2951
|
+
private async putKnownEntryBytesBatch(
|
|
2952
|
+
blocks: Array<{ cid: string; bytes: Uint8Array }>,
|
|
2953
|
+
) {
|
|
2954
|
+
if (blocks.length === 0) {
|
|
2955
|
+
return;
|
|
2956
|
+
}
|
|
2957
|
+
if (blocks.length === 1 && hasPutKnown(this._storage)) {
|
|
2958
|
+
const block = blocks[0]!;
|
|
2959
|
+
const cidResult = this._storage.putKnown(block.cid, block.bytes);
|
|
2960
|
+
const cid = isPromiseLike(cidResult) ? await cidResult : cidResult;
|
|
2961
|
+
if (cid !== block.cid) {
|
|
2962
|
+
throw new Error("Unexpected block cid");
|
|
2963
|
+
}
|
|
2964
|
+
return;
|
|
2965
|
+
}
|
|
2966
|
+
if (hasPutKnownManyColumns(this._storage)) {
|
|
2967
|
+
const cids = new Array<string>(blocks.length);
|
|
2968
|
+
const bytes = new Array<Uint8Array>(blocks.length);
|
|
2969
|
+
for (let i = 0; i < blocks.length; i++) {
|
|
2970
|
+
const block = blocks[i]!;
|
|
2971
|
+
cids[i] = block.cid;
|
|
2972
|
+
bytes[i] = block.bytes;
|
|
2973
|
+
}
|
|
2974
|
+
const cidsResult = this._storage.putKnownManyColumns(cids, bytes);
|
|
2975
|
+
const result = isPromiseLike(cidsResult) ? await cidsResult : cidsResult;
|
|
2976
|
+
if (result.length !== blocks.length) {
|
|
2977
|
+
throw new Error("Unexpected block batch result length");
|
|
2978
|
+
}
|
|
2979
|
+
for (let i = 0; i < result.length; i++) {
|
|
2980
|
+
if (result[i] !== cids[i]) {
|
|
2981
|
+
throw new Error("Unexpected block batch cid");
|
|
2982
|
+
}
|
|
2983
|
+
}
|
|
2984
|
+
return;
|
|
2985
|
+
}
|
|
2986
|
+
if (hasPutKnownMany(this._storage)) {
|
|
2987
|
+
const cidsResult = this._storage.putKnownMany(
|
|
2988
|
+
blocks.map((block) => [block.cid, block.bytes] as const),
|
|
2989
|
+
);
|
|
2990
|
+
const cids = isPromiseLike(cidsResult) ? await cidsResult : cidsResult;
|
|
2991
|
+
if (cids.length !== blocks.length) {
|
|
2992
|
+
throw new Error("Unexpected block batch result length");
|
|
2993
|
+
}
|
|
2994
|
+
for (let i = 0; i < cids.length; i++) {
|
|
2995
|
+
if (cids[i] !== blocks[i]!.cid) {
|
|
2996
|
+
throw new Error("Unexpected block batch cid");
|
|
2997
|
+
}
|
|
2998
|
+
}
|
|
2999
|
+
return;
|
|
3000
|
+
}
|
|
3001
|
+
const preparedBlocks = blocks.map((block) =>
|
|
3002
|
+
Entry.preparedBlockFromBytes(block.bytes, block.cid),
|
|
3003
|
+
);
|
|
3004
|
+
const cids = await (this._storage as BlocksWithPutMany).putMany!(
|
|
3005
|
+
preparedBlocks,
|
|
3006
|
+
);
|
|
3007
|
+
if (cids.length !== blocks.length) {
|
|
3008
|
+
throw new Error("Unexpected block batch result length");
|
|
3009
|
+
}
|
|
3010
|
+
for (let i = 0; i < cids.length; i++) {
|
|
3011
|
+
if (cids[i] !== blocks[i]!.cid) {
|
|
3012
|
+
throw new Error("Unexpected block batch cid");
|
|
3013
|
+
}
|
|
3014
|
+
}
|
|
3015
|
+
}
|
|
3016
|
+
|
|
3017
|
+
private putPreparedAppendBlocks(
|
|
3018
|
+
preparedBlocks?: PreparedEntryBlock[],
|
|
3019
|
+
): MaybePromise<void> {
|
|
3020
|
+
if (!preparedBlocks || preparedBlocks.length === 0) {
|
|
3021
|
+
throw new Error("Missing prepared entry block");
|
|
3022
|
+
}
|
|
3023
|
+
if (preparedBlocks.length === 1 && hasPutKnown(this._storage)) {
|
|
3024
|
+
const block = preparedBlocks[0]!;
|
|
3025
|
+
const cidResult = this._storage.putKnown(block.cid, block.block.bytes);
|
|
3026
|
+
const checkCid = (cid: string) => {
|
|
3027
|
+
if (cid !== block.cid) {
|
|
3028
|
+
throw new Error("Unexpected block cid");
|
|
3029
|
+
}
|
|
3030
|
+
};
|
|
3031
|
+
if (isPromiseLike(cidResult)) {
|
|
3032
|
+
return cidResult.then(checkCid);
|
|
3033
|
+
}
|
|
3034
|
+
checkCid(cidResult);
|
|
3035
|
+
return;
|
|
3036
|
+
}
|
|
3037
|
+
const checkCids = (cids: string[]) => {
|
|
3038
|
+
if (cids.length !== preparedBlocks.length) {
|
|
3039
|
+
throw new Error("Unexpected block batch result length");
|
|
3040
|
+
}
|
|
3041
|
+
for (let i = 0; i < cids.length; i++) {
|
|
3042
|
+
if (cids[i] !== preparedBlocks[i]!.cid) {
|
|
3043
|
+
throw new Error("Unexpected block batch cid");
|
|
3044
|
+
}
|
|
3045
|
+
}
|
|
3046
|
+
};
|
|
3047
|
+
if (hasPutKnownManyColumns(this._storage)) {
|
|
3048
|
+
const cids = new Array<string>(preparedBlocks.length);
|
|
3049
|
+
const bytes = new Array<Uint8Array>(preparedBlocks.length);
|
|
3050
|
+
for (let i = 0; i < preparedBlocks.length; i++) {
|
|
3051
|
+
const block = preparedBlocks[i]!;
|
|
3052
|
+
cids[i] = block.cid;
|
|
3053
|
+
bytes[i] = block.block.bytes;
|
|
3054
|
+
}
|
|
3055
|
+
const cidsResult = this._storage.putKnownManyColumns(cids, bytes);
|
|
3056
|
+
if (isPromiseLike(cidsResult)) {
|
|
3057
|
+
return cidsResult.then(checkCids);
|
|
3058
|
+
}
|
|
3059
|
+
checkCids(cidsResult);
|
|
3060
|
+
return;
|
|
3061
|
+
}
|
|
3062
|
+
if (hasPutKnownMany(this._storage)) {
|
|
3063
|
+
const cidsResult = this._storage.putKnownMany(
|
|
3064
|
+
preparedBlocks.map((block) => [block.cid, block.block.bytes] as const),
|
|
3065
|
+
);
|
|
3066
|
+
if (isPromiseLike(cidsResult)) {
|
|
3067
|
+
return cidsResult.then(checkCids);
|
|
3068
|
+
}
|
|
3069
|
+
checkCids(cidsResult);
|
|
3070
|
+
return;
|
|
3071
|
+
}
|
|
3072
|
+
const cidsResult = (this._storage as BlocksWithPutMany).putMany!(
|
|
3073
|
+
preparedBlocks,
|
|
3074
|
+
);
|
|
3075
|
+
if (isPromiseLike(cidsResult)) {
|
|
3076
|
+
return cidsResult.then(checkCids);
|
|
3077
|
+
}
|
|
3078
|
+
checkCids(cidsResult);
|
|
3079
|
+
}
|
|
3080
|
+
|
|
3081
|
+
async remove(
|
|
3082
|
+
entry:
|
|
3083
|
+
| { hash: string; meta: { next: string[] } }
|
|
3084
|
+
| { hash: string; meta: { next: string[] } }[],
|
|
3085
|
+
options?: { recursively?: boolean },
|
|
3086
|
+
): Promise<Change<T>> {
|
|
3087
|
+
/* await this.load({ reload: false }); */
|
|
3088
|
+
const entries = Array.isArray(entry) ? entry : [entry];
|
|
3089
|
+
|
|
3090
|
+
if (entries.length === 0) {
|
|
3091
|
+
return {
|
|
3092
|
+
added: [],
|
|
3093
|
+
removed: [],
|
|
3094
|
+
};
|
|
3095
|
+
}
|
|
3096
|
+
|
|
3097
|
+
let removed: {
|
|
3098
|
+
entry: ShallowOrFullEntry<T>;
|
|
3099
|
+
fn: () => Promise<ShallowEntry | undefined>;
|
|
3100
|
+
}[];
|
|
3101
|
+
|
|
3102
|
+
if (options?.recursively) {
|
|
3103
|
+
removed = await this.prepareDeleteRecursively(entry);
|
|
3104
|
+
} else {
|
|
3105
|
+
removed = [];
|
|
3106
|
+
for (const entry of entries) {
|
|
3107
|
+
const deleteFn = await this.prepareDelete(entry.hash);
|
|
3108
|
+
deleteFn.entry &&
|
|
3109
|
+
removed.push({ entry: deleteFn.entry, fn: deleteFn.fn });
|
|
3110
|
+
}
|
|
3111
|
+
}
|
|
3112
|
+
|
|
3113
|
+
const change: Change<T> = {
|
|
3114
|
+
added: [],
|
|
3115
|
+
removed: removed.map((x) => x.entry),
|
|
3116
|
+
};
|
|
3117
|
+
|
|
3118
|
+
await this._onChange?.(change);
|
|
3119
|
+
|
|
3120
|
+
// invoke deletions
|
|
3121
|
+
await Promise.all(removed.map((x) => x.fn()));
|
|
3122
|
+
|
|
3123
|
+
return change;
|
|
3124
|
+
}
|
|
3125
|
+
|
|
3126
|
+
async trim(
|
|
3127
|
+
option: TrimOptions | undefined = this._trim.options,
|
|
3128
|
+
properties?: { resolveDeletedEntries?: boolean },
|
|
3129
|
+
) {
|
|
3130
|
+
return this._trim.trim(option, properties);
|
|
3131
|
+
}
|
|
3132
|
+
|
|
3133
|
+
private trimIfConfigured(
|
|
3134
|
+
option?: TrimOptions,
|
|
3135
|
+
properties?: { resolveDeletedEntries?: boolean },
|
|
3136
|
+
): MaybePromise<ShallowOrFullEntry<T>[] | undefined> {
|
|
3137
|
+
const resolved = option ?? this._trim.options;
|
|
3138
|
+
return resolved ? this.trim(resolved, properties) : undefined;
|
|
3139
|
+
}
|
|
3140
|
+
|
|
3141
|
+
private getNativeCommitOnlyTrimLengthTo(
|
|
3142
|
+
option: TrimOptions | undefined,
|
|
3143
|
+
resolveDeletedEntries: boolean | undefined,
|
|
3144
|
+
): number | undefined {
|
|
3145
|
+
const resolved = option ?? this._trim.options;
|
|
3146
|
+
if (
|
|
3147
|
+
!resolved ||
|
|
3148
|
+
resolved.type !== "length" ||
|
|
3149
|
+
resolved.filter?.canTrim ||
|
|
3150
|
+
resolveDeletedEntries !== false
|
|
3151
|
+
) {
|
|
3152
|
+
return;
|
|
3153
|
+
}
|
|
3154
|
+
|
|
3155
|
+
const from = resolved.from ?? resolved.to;
|
|
3156
|
+
if (this.length + 1 < from) {
|
|
3157
|
+
return;
|
|
3158
|
+
}
|
|
3159
|
+
return resolved.to;
|
|
3160
|
+
}
|
|
3161
|
+
|
|
3162
|
+
async join(
|
|
3163
|
+
entriesOrLog:
|
|
3164
|
+
| (string | Entry<T> | ShallowEntry | EntryWithRefs<T>)[]
|
|
3165
|
+
| Log<T>
|
|
3166
|
+
| ResultsIterator<Entry<any>>,
|
|
3167
|
+
options?: JoinOptions<T>,
|
|
3168
|
+
): Promise<void>;
|
|
3169
|
+
async join(
|
|
3170
|
+
entriesOrLog:
|
|
3171
|
+
| (string | Entry<T> | ShallowEntry | EntryWithRefs<T>)[]
|
|
3172
|
+
| Log<T>
|
|
3173
|
+
| ResultsIterator<Entry<any>>,
|
|
3174
|
+
options?: TrustedJoinOptions<T>,
|
|
3175
|
+
): Promise<void> {
|
|
3176
|
+
let entries: Entry<T>[];
|
|
3177
|
+
const references: Map<string, Entry<T>> = new Map();
|
|
3178
|
+
|
|
3179
|
+
const fromCache = new Map<string, string[] | null>();
|
|
3180
|
+
const resolveRemoteFrom = async (hash: string, signal?: AbortSignal) => {
|
|
3181
|
+
const cached = fromCache.get(hash);
|
|
3182
|
+
if (cached !== undefined) return cached === null ? undefined : cached;
|
|
3183
|
+
|
|
3184
|
+
let from: string[] | undefined;
|
|
3185
|
+
try {
|
|
3186
|
+
from = await this.entryIndex.properties.resolveRemotePeers?.(hash, {
|
|
3187
|
+
signal,
|
|
3188
|
+
});
|
|
3189
|
+
} catch {
|
|
3190
|
+
from = undefined;
|
|
3191
|
+
}
|
|
3192
|
+
const normalized = from && from.length > 0 ? from : undefined;
|
|
3193
|
+
fromCache.set(hash, normalized ?? null);
|
|
3194
|
+
return normalized;
|
|
3195
|
+
};
|
|
3196
|
+
|
|
3197
|
+
const remote: NonNullable<Exclude<GetOptions["remote"], boolean>> = {
|
|
3198
|
+
timeout: options?.timeout,
|
|
3199
|
+
signal: this._closeController.signal,
|
|
3200
|
+
};
|
|
3201
|
+
|
|
3202
|
+
if (entriesOrLog instanceof Log) {
|
|
747
3203
|
if (entriesOrLog.entryIndex.length === 0) return;
|
|
748
3204
|
entries = await entriesOrLog.toArray();
|
|
749
3205
|
for (const element of entries) references.set(element.hash, element);
|
|
750
3206
|
} else if (Array.isArray(entriesOrLog)) {
|
|
751
3207
|
if (entriesOrLog.length === 0) return;
|
|
752
|
-
const existingHashes =
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
: element instanceof ShallowEntry
|
|
3208
|
+
const existingHashes =
|
|
3209
|
+
options?.reset || options?.__peerbitEntriesAlreadyMissing === true
|
|
3210
|
+
? new Set<string>()
|
|
3211
|
+
: await this.entryIndex.hasMany(
|
|
3212
|
+
entriesOrLog.map((element) =>
|
|
3213
|
+
typeof element === "string"
|
|
3214
|
+
? element
|
|
3215
|
+
: element instanceof Entry
|
|
761
3216
|
? element.hash
|
|
762
|
-
: element
|
|
763
|
-
|
|
764
|
-
|
|
3217
|
+
: element instanceof ShallowEntry
|
|
3218
|
+
? element.hash
|
|
3219
|
+
: element.entry.hash,
|
|
3220
|
+
),
|
|
3221
|
+
);
|
|
765
3222
|
|
|
766
3223
|
entries = [];
|
|
767
3224
|
for (const element of entriesOrLog) {
|
|
@@ -844,11 +3301,34 @@ export class Log<T> {
|
|
|
844
3301
|
entries = all;
|
|
845
3302
|
}
|
|
846
3303
|
|
|
3304
|
+
const profile = options?.__peerbitProfile;
|
|
3305
|
+
const headsStartedAt = internalProfileStart(profile);
|
|
847
3306
|
const heads: Map<string, boolean> = new Map();
|
|
848
3307
|
for (const entry of entries) {
|
|
849
3308
|
if (heads.has(entry.hash)) continue;
|
|
850
3309
|
heads.set(entry.hash, true);
|
|
851
|
-
|
|
3310
|
+
const nexts =
|
|
3311
|
+
options?.__peerbitBatchIndependent === true
|
|
3312
|
+
? entry.meta.next
|
|
3313
|
+
: await entry.getNext();
|
|
3314
|
+
for (const next of nexts) heads.set(next, false);
|
|
3315
|
+
}
|
|
3316
|
+
emitInternalProfileDuration(profile, headsStartedAt, {
|
|
3317
|
+
name: "log.join.prepareHeads",
|
|
3318
|
+
component: "log",
|
|
3319
|
+
entries: entries.length,
|
|
3320
|
+
count: heads.size,
|
|
3321
|
+
messages: 1,
|
|
3322
|
+
details: {
|
|
3323
|
+
batchIndependent: options?.__peerbitBatchIndependent === true,
|
|
3324
|
+
},
|
|
3325
|
+
});
|
|
3326
|
+
|
|
3327
|
+
if (
|
|
3328
|
+
options?.__peerbitBatchIndependent === true &&
|
|
3329
|
+
(await this.tryJoinIndependentAppendBatch(entries, heads, options))
|
|
3330
|
+
) {
|
|
3331
|
+
return;
|
|
852
3332
|
}
|
|
853
3333
|
|
|
854
3334
|
for (const entry of entries) {
|
|
@@ -877,6 +3357,522 @@ export class Log<T> {
|
|
|
877
3357
|
}
|
|
878
3358
|
}
|
|
879
3359
|
|
|
3360
|
+
// Internal trusted receive path for callers that can supply prepared append facts.
|
|
3361
|
+
private async joinPreparedAppendFactsBatch(
|
|
3362
|
+
entries: PreparedAppendJoinFacts[],
|
|
3363
|
+
options?: TrustedPreparedAppendFactsBatchJoinOptions,
|
|
3364
|
+
): Promise<boolean> {
|
|
3365
|
+
if (
|
|
3366
|
+
entries.length === 0 ||
|
|
3367
|
+
!canAppendAlreadyValidated(options) ||
|
|
3368
|
+
entries.some((entry) => this._joining.has(entry.hash))
|
|
3369
|
+
) {
|
|
3370
|
+
return false;
|
|
3371
|
+
}
|
|
3372
|
+
|
|
3373
|
+
const resolvedOptions = options!;
|
|
3374
|
+
const profile = resolvedOptions.__peerbitProfile;
|
|
3375
|
+
const prepareStartedAt = internalProfileStart(profile);
|
|
3376
|
+
const entryHashes = new Array<string>(entries.length);
|
|
3377
|
+
let hasAnyNext = false;
|
|
3378
|
+
for (let i = 0; i < entries.length; i++) {
|
|
3379
|
+
const entry = entries[i]!;
|
|
3380
|
+
// byteLength stands in for the bytes presence check: prepared facts
|
|
3381
|
+
// carry both, and probing `bytes` would force stash-backed heads to
|
|
3382
|
+
// materialize block bytes the native commit never reads.
|
|
3383
|
+
if (
|
|
3384
|
+
!entry.hash ||
|
|
3385
|
+
entry.byteLength == null ||
|
|
3386
|
+
entry.meta.type !== EntryType.APPEND
|
|
3387
|
+
) {
|
|
3388
|
+
return false;
|
|
3389
|
+
}
|
|
3390
|
+
entryHashes[i] = entry.hash;
|
|
3391
|
+
if (entry.meta.next.length > 0) {
|
|
3392
|
+
hasAnyNext = true;
|
|
3393
|
+
}
|
|
3394
|
+
}
|
|
3395
|
+
const batchHashes = new Set(entryHashes);
|
|
3396
|
+
let heads: Map<string, boolean> | undefined;
|
|
3397
|
+
if (hasAnyNext) {
|
|
3398
|
+
heads = new Map();
|
|
3399
|
+
for (const entry of entries) {
|
|
3400
|
+
if (heads.has(entry.hash)) {
|
|
3401
|
+
continue;
|
|
3402
|
+
}
|
|
3403
|
+
heads.set(entry.hash, true);
|
|
3404
|
+
for (const next of entry.meta.next) {
|
|
3405
|
+
heads.set(next, false);
|
|
3406
|
+
}
|
|
3407
|
+
}
|
|
3408
|
+
}
|
|
3409
|
+
emitInternalProfileDuration(profile, prepareStartedAt, {
|
|
3410
|
+
name: "log.joinPreparedFacts.prepare",
|
|
3411
|
+
component: "log",
|
|
3412
|
+
entries: entries.length,
|
|
3413
|
+
count: heads?.size ?? entries.length,
|
|
3414
|
+
messages: 1,
|
|
3415
|
+
});
|
|
3416
|
+
|
|
3417
|
+
const nativeCommitValidatesPlan =
|
|
3418
|
+
resolvedOptions.__peerbitNativePreparedJoinCommitValidatesPlan === true &&
|
|
3419
|
+
!!resolvedOptions.__peerbitNativePreparedJoinCommit;
|
|
3420
|
+
const headFlags: boolean[] = [];
|
|
3421
|
+
const headFlagsBytes = new Uint8Array(entries.length);
|
|
3422
|
+
const pushHeadFlag = (index: number, isHead: boolean) => {
|
|
3423
|
+
headFlags.push(isHead);
|
|
3424
|
+
headFlagsBytes[index] = isHead ? 1 : 0;
|
|
3425
|
+
};
|
|
3426
|
+
if (nativeCommitValidatesPlan) {
|
|
3427
|
+
for (let i = 0; i < entries.length; i++) {
|
|
3428
|
+
const entry = entries[i]!;
|
|
3429
|
+
pushHeadFlag(i, heads?.get(entry.hash) ?? true);
|
|
3430
|
+
}
|
|
3431
|
+
emitInternalProfileDuration(profile, internalProfileStart(profile), {
|
|
3432
|
+
name: "log.joinPreparedFacts.plan",
|
|
3433
|
+
component: "log",
|
|
3434
|
+
entries: entries.length,
|
|
3435
|
+
messages: 1,
|
|
3436
|
+
details: { nativeCommitValidatesPlan: true },
|
|
3437
|
+
});
|
|
3438
|
+
emitInternalProfileDuration(profile, internalProfileStart(profile), {
|
|
3439
|
+
name: "log.joinPreparedFacts.validatePlan",
|
|
3440
|
+
component: "log",
|
|
3441
|
+
entries: entries.length,
|
|
3442
|
+
messages: 1,
|
|
3443
|
+
details: { nativeCommitValidatesPlan: true },
|
|
3444
|
+
});
|
|
3445
|
+
} else {
|
|
3446
|
+
const planStartedAt = internalProfileStart(profile);
|
|
3447
|
+
const joinPlans = await this.entryIndex.planJoinBatch(
|
|
3448
|
+
entries,
|
|
3449
|
+
false,
|
|
3450
|
+
profile,
|
|
3451
|
+
);
|
|
3452
|
+
emitInternalProfileDuration(profile, planStartedAt, {
|
|
3453
|
+
name: "log.joinPreparedFacts.plan",
|
|
3454
|
+
component: "log",
|
|
3455
|
+
entries: entries.length,
|
|
3456
|
+
messages: 1,
|
|
3457
|
+
});
|
|
3458
|
+
const validatePlanStartedAt = internalProfileStart(profile);
|
|
3459
|
+
for (let i = 0; i < entries.length; i++) {
|
|
3460
|
+
const entry = entries[i]!;
|
|
3461
|
+
const joinPlan = joinPlans[i]!;
|
|
3462
|
+
if (
|
|
3463
|
+
joinPlan.skip ||
|
|
3464
|
+
joinPlan.coveredByCut ||
|
|
3465
|
+
!joinPlan.cutChecked ||
|
|
3466
|
+
joinPlan.missingParents.some((hash) => !batchHashes.has(hash))
|
|
3467
|
+
) {
|
|
3468
|
+
return false;
|
|
3469
|
+
}
|
|
3470
|
+
pushHeadFlag(i, heads?.get(entry.hash) ?? true);
|
|
3471
|
+
}
|
|
3472
|
+
emitInternalProfileDuration(profile, validatePlanStartedAt, {
|
|
3473
|
+
name: "log.joinPreparedFacts.validatePlan",
|
|
3474
|
+
component: "log",
|
|
3475
|
+
entries: entries.length,
|
|
3476
|
+
messages: 1,
|
|
3477
|
+
});
|
|
3478
|
+
}
|
|
3479
|
+
|
|
3480
|
+
let nativeValidatedCommitRejected = false;
|
|
3481
|
+
const batchPromise = (async () => {
|
|
3482
|
+
const clockStartedAt = internalProfileStart(profile);
|
|
3483
|
+
for (const entry of entries) {
|
|
3484
|
+
this._hlc.update(entry.meta.clock.timestamp);
|
|
3485
|
+
}
|
|
3486
|
+
emitInternalProfileDuration(profile, clockStartedAt, {
|
|
3487
|
+
name: "log.joinPreparedFacts.clock",
|
|
3488
|
+
component: "log",
|
|
3489
|
+
entries: entries.length,
|
|
3490
|
+
messages: 1,
|
|
3491
|
+
});
|
|
3492
|
+
|
|
3493
|
+
const trustedMissing =
|
|
3494
|
+
resolvedOptions.__peerbitEntriesAlreadyMissing === true &&
|
|
3495
|
+
batchHashes.size === entries.length;
|
|
3496
|
+
let nativePreparedCommitted = false;
|
|
3497
|
+
if (resolvedOptions.__peerbitNativePreparedJoinCommit) {
|
|
3498
|
+
const nativeCommitStartedAt = internalProfileStart(profile);
|
|
3499
|
+
nativePreparedCommitted =
|
|
3500
|
+
(await resolvedOptions.__peerbitNativePreparedJoinCommit({
|
|
3501
|
+
entries,
|
|
3502
|
+
hashes: entryHashes,
|
|
3503
|
+
headFlags,
|
|
3504
|
+
headFlagsBytes,
|
|
3505
|
+
trustedMissing,
|
|
3506
|
+
validatePlan: nativeCommitValidatesPlan,
|
|
3507
|
+
})) === true;
|
|
3508
|
+
emitInternalProfileDuration(profile, nativeCommitStartedAt, {
|
|
3509
|
+
name: "log.joinPreparedFacts.nativePreparedCommit",
|
|
3510
|
+
component: "log",
|
|
3511
|
+
entries: entries.length,
|
|
3512
|
+
messages: 1,
|
|
3513
|
+
details: { nativePreparedCommitted },
|
|
3514
|
+
});
|
|
3515
|
+
}
|
|
3516
|
+
if (nativeCommitValidatesPlan && !nativePreparedCommitted) {
|
|
3517
|
+
nativeValidatedCommitRejected = true;
|
|
3518
|
+
return;
|
|
3519
|
+
}
|
|
3520
|
+
const blocksStartedAt = internalProfileStart(profile);
|
|
3521
|
+
if (!nativePreparedCommitted) {
|
|
3522
|
+
await this.putKnownEntryBytesBatch(
|
|
3523
|
+
entries.map((entry) => ({
|
|
3524
|
+
cid: entry.hash,
|
|
3525
|
+
bytes: entry.bytes,
|
|
3526
|
+
})),
|
|
3527
|
+
);
|
|
3528
|
+
}
|
|
3529
|
+
emitInternalProfileDuration(profile, blocksStartedAt, {
|
|
3530
|
+
name: "log.joinPreparedFacts.blocks",
|
|
3531
|
+
component: "log",
|
|
3532
|
+
entries: entries.length,
|
|
3533
|
+
bytes: entries.reduce((sum, entry) => sum + entry.byteLength, 0),
|
|
3534
|
+
messages: 1,
|
|
3535
|
+
details: { nativePreparedCommitted },
|
|
3536
|
+
});
|
|
3537
|
+
|
|
3538
|
+
const indexStartedAt = internalProfileStart(profile);
|
|
3539
|
+
let nativeCommittedFactsIndexed = false;
|
|
3540
|
+
if (
|
|
3541
|
+
nativePreparedCommitted &&
|
|
3542
|
+
resolvedOptions.__peerbitDeferIndexWrite === true
|
|
3543
|
+
) {
|
|
3544
|
+
const indexBatchHashes = entries.length > 1 ? batchHashes : undefined;
|
|
3545
|
+
const indexRows = entries.map((entry, index) => {
|
|
3546
|
+
const isHead = headFlags[index] ?? true;
|
|
3547
|
+
const externalNextHashes = indexBatchHashes
|
|
3548
|
+
? entry.meta.next.filter((next) => !indexBatchHashes.has(next))
|
|
3549
|
+
: entry.meta.next;
|
|
3550
|
+
return {
|
|
3551
|
+
hash: entry.hash,
|
|
3552
|
+
unique: trustedMissing,
|
|
3553
|
+
externalNextHashes,
|
|
3554
|
+
getShallowEntry: () => {
|
|
3555
|
+
const shallowEntry =
|
|
3556
|
+
entry.shallowEntry ?? entry.getShallowEntry?.(isHead);
|
|
3557
|
+
if (!shallowEntry) {
|
|
3558
|
+
throw new Error("Missing prepared append shallow entry");
|
|
3559
|
+
}
|
|
3560
|
+
shallowEntry.head = isHead;
|
|
3561
|
+
entry.shallowEntry = shallowEntry;
|
|
3562
|
+
return shallowEntry;
|
|
3563
|
+
},
|
|
3564
|
+
isHead,
|
|
3565
|
+
};
|
|
3566
|
+
});
|
|
3567
|
+
await this.entryIndex.putNativeCommittedAppendFactsBatch(indexRows);
|
|
3568
|
+
nativeCommittedFactsIndexed = true;
|
|
3569
|
+
} else {
|
|
3570
|
+
const externalNextHashes =
|
|
3571
|
+
entries.length === 1 ? entries[0]!.meta.next : undefined;
|
|
3572
|
+
await this.entryIndex.putAppendFactsBatch(entries, {
|
|
3573
|
+
unique: trustedMissing,
|
|
3574
|
+
externalNextHashes,
|
|
3575
|
+
heads: headFlags,
|
|
3576
|
+
deferIndexWrite: resolvedOptions.__peerbitDeferIndexWrite,
|
|
3577
|
+
nativeGraphUpdated: nativePreparedCommitted,
|
|
3578
|
+
profile,
|
|
3579
|
+
});
|
|
3580
|
+
}
|
|
3581
|
+
emitInternalProfileDuration(profile, indexStartedAt, {
|
|
3582
|
+
name: "log.joinPreparedFacts.entryIndex",
|
|
3583
|
+
component: "log",
|
|
3584
|
+
entries: entries.length,
|
|
3585
|
+
messages: 1,
|
|
3586
|
+
details: { trustedMissing, nativeCommittedFactsIndexed },
|
|
3587
|
+
});
|
|
3588
|
+
|
|
3589
|
+
if (resolvedOptions.__peerbitOnPreparedJoinCommitted) {
|
|
3590
|
+
const committedStartedAt = internalProfileStart(profile);
|
|
3591
|
+
await resolvedOptions.__peerbitOnPreparedJoinCommitted({
|
|
3592
|
+
entries,
|
|
3593
|
+
hashes: entryHashes,
|
|
3594
|
+
headFlags,
|
|
3595
|
+
nativePreparedCommitted,
|
|
3596
|
+
});
|
|
3597
|
+
emitInternalProfileDuration(profile, committedStartedAt, {
|
|
3598
|
+
name: "log.joinPreparedFacts.committed",
|
|
3599
|
+
component: "log",
|
|
3600
|
+
entries: entries.length,
|
|
3601
|
+
messages: 1,
|
|
3602
|
+
details: { nativePreparedCommitted },
|
|
3603
|
+
});
|
|
3604
|
+
}
|
|
3605
|
+
|
|
3606
|
+
const changeStartedAt = internalProfileStart(profile);
|
|
3607
|
+
if (resolvedOptions.__peerbitOnAppendHashes) {
|
|
3608
|
+
await resolvedOptions.__peerbitOnAppendHashes(
|
|
3609
|
+
entries.map((entry) => entry.hash),
|
|
3610
|
+
);
|
|
3611
|
+
} else {
|
|
3612
|
+
const change: Change<T> = {
|
|
3613
|
+
added: entries.map((entry, index) => {
|
|
3614
|
+
const materializeEntry = entry.materializeEntry;
|
|
3615
|
+
if (!materializeEntry) {
|
|
3616
|
+
throw new Error("Missing prepared append materializer");
|
|
3617
|
+
}
|
|
3618
|
+
return {
|
|
3619
|
+
head: headFlags[index]!,
|
|
3620
|
+
entry: materializeEntry() as Entry<T>,
|
|
3621
|
+
};
|
|
3622
|
+
}),
|
|
3623
|
+
removed: [],
|
|
3624
|
+
};
|
|
3625
|
+
await this._onChange?.(change);
|
|
3626
|
+
}
|
|
3627
|
+
emitInternalProfileDuration(profile, changeStartedAt, {
|
|
3628
|
+
name: "log.joinPreparedFacts.change",
|
|
3629
|
+
component: "log",
|
|
3630
|
+
entries: entries.length,
|
|
3631
|
+
messages: 1,
|
|
3632
|
+
details: { hashOnly: !!resolvedOptions.__peerbitOnAppendHashes },
|
|
3633
|
+
});
|
|
3634
|
+
})().finally(() => {
|
|
3635
|
+
for (const entry of entries) {
|
|
3636
|
+
this._joining.delete(entry.hash);
|
|
3637
|
+
}
|
|
3638
|
+
});
|
|
3639
|
+
|
|
3640
|
+
for (const entry of entries) {
|
|
3641
|
+
this._joining.set(entry.hash, batchPromise);
|
|
3642
|
+
}
|
|
3643
|
+
await batchPromise;
|
|
3644
|
+
if (nativeValidatedCommitRejected) {
|
|
3645
|
+
return false;
|
|
3646
|
+
}
|
|
3647
|
+
return true;
|
|
3648
|
+
}
|
|
3649
|
+
|
|
3650
|
+
private async tryJoinIndependentAppendBatch(
|
|
3651
|
+
entries: Entry<T>[],
|
|
3652
|
+
heads: Map<string, boolean>,
|
|
3653
|
+
options: TrustedJoinOptions<T>,
|
|
3654
|
+
): Promise<boolean> {
|
|
3655
|
+
if (
|
|
3656
|
+
entries.length < 2 ||
|
|
3657
|
+
options.reset ||
|
|
3658
|
+
options.trim ||
|
|
3659
|
+
options.verifySignatures ||
|
|
3660
|
+
entries.some((entry) => this._joining.has(entry.hash))
|
|
3661
|
+
) {
|
|
3662
|
+
return false;
|
|
3663
|
+
}
|
|
3664
|
+
|
|
3665
|
+
const profile = options.__peerbitProfile;
|
|
3666
|
+
const prepareStartedAt = internalProfileStart(profile);
|
|
3667
|
+
const batchHashes = new Set(entries.map((entry) => entry.hash));
|
|
3668
|
+
const headFlags: boolean[] = [];
|
|
3669
|
+
for (const entry of entries) {
|
|
3670
|
+
if (!entry.hash || !Entry.hasPreparedBlock(entry)) {
|
|
3671
|
+
return false;
|
|
3672
|
+
}
|
|
3673
|
+
entry.init(this);
|
|
3674
|
+
if (entry.meta.type !== EntryType.APPEND) {
|
|
3675
|
+
return false;
|
|
3676
|
+
}
|
|
3677
|
+
}
|
|
3678
|
+
emitInternalProfileDuration(profile, prepareStartedAt, {
|
|
3679
|
+
name: "log.joinIndependent.prepare",
|
|
3680
|
+
component: "log",
|
|
3681
|
+
entries: entries.length,
|
|
3682
|
+
messages: 1,
|
|
3683
|
+
});
|
|
3684
|
+
|
|
3685
|
+
const planStartedAt = internalProfileStart(profile);
|
|
3686
|
+
const joinPlans = await this.entryIndex.planJoinBatch(
|
|
3687
|
+
entries,
|
|
3688
|
+
false,
|
|
3689
|
+
profile,
|
|
3690
|
+
);
|
|
3691
|
+
emitInternalProfileDuration(profile, planStartedAt, {
|
|
3692
|
+
name: "log.joinIndependent.plan",
|
|
3693
|
+
component: "log",
|
|
3694
|
+
entries: entries.length,
|
|
3695
|
+
messages: 1,
|
|
3696
|
+
});
|
|
3697
|
+
const validatePlanStartedAt = internalProfileStart(profile);
|
|
3698
|
+
for (let i = 0; i < entries.length; i++) {
|
|
3699
|
+
const entry = entries[i]!;
|
|
3700
|
+
const joinPlan = joinPlans[i]!;
|
|
3701
|
+
if (
|
|
3702
|
+
joinPlan.skip ||
|
|
3703
|
+
joinPlan.coveredByCut ||
|
|
3704
|
+
!joinPlan.cutChecked ||
|
|
3705
|
+
joinPlan.missingParents.some((hash) => !batchHashes.has(hash))
|
|
3706
|
+
) {
|
|
3707
|
+
return false;
|
|
3708
|
+
}
|
|
3709
|
+
headFlags.push(heads.get(entry.hash) ?? true);
|
|
3710
|
+
}
|
|
3711
|
+
emitInternalProfileDuration(profile, validatePlanStartedAt, {
|
|
3712
|
+
name: "log.joinIndependent.validatePlan",
|
|
3713
|
+
component: "log",
|
|
3714
|
+
entries: entries.length,
|
|
3715
|
+
messages: 1,
|
|
3716
|
+
});
|
|
3717
|
+
|
|
3718
|
+
if (!canAppendAlreadyValidated(options)) {
|
|
3719
|
+
const canAppendStartedAt = internalProfileStart(profile);
|
|
3720
|
+
for (const entry of entries) {
|
|
3721
|
+
if (this._canAppend && !(await this._canAppend(entry))) {
|
|
3722
|
+
return false;
|
|
3723
|
+
}
|
|
3724
|
+
}
|
|
3725
|
+
emitInternalProfileDuration(profile, canAppendStartedAt, {
|
|
3726
|
+
name: "log.joinIndependent.canAppend",
|
|
3727
|
+
component: "log",
|
|
3728
|
+
entries: entries.length,
|
|
3729
|
+
messages: 1,
|
|
3730
|
+
});
|
|
3731
|
+
}
|
|
3732
|
+
|
|
3733
|
+
const preparedBatch = this.takePreparedIndependentAppendBatch(
|
|
3734
|
+
entries,
|
|
3735
|
+
headFlags,
|
|
3736
|
+
);
|
|
3737
|
+
if (!preparedBatch) {
|
|
3738
|
+
return false;
|
|
3739
|
+
}
|
|
3740
|
+
|
|
3741
|
+
const batchPromise = (async () => {
|
|
3742
|
+
const clockStartedAt = internalProfileStart(profile);
|
|
3743
|
+
for (const entry of entries) {
|
|
3744
|
+
this._hlc.update(entry.meta.clock.timestamp);
|
|
3745
|
+
}
|
|
3746
|
+
emitInternalProfileDuration(profile, clockStartedAt, {
|
|
3747
|
+
name: "log.joinIndependent.clock",
|
|
3748
|
+
component: "log",
|
|
3749
|
+
entries: entries.length,
|
|
3750
|
+
messages: 1,
|
|
3751
|
+
});
|
|
3752
|
+
|
|
3753
|
+
const blocksStartedAt = internalProfileStart(profile);
|
|
3754
|
+
await this.putAppendEntryBlocks(entries, preparedBatch.blocks);
|
|
3755
|
+
emitInternalProfileDuration(profile, blocksStartedAt, {
|
|
3756
|
+
name: "log.joinIndependent.blocks",
|
|
3757
|
+
component: "log",
|
|
3758
|
+
entries: entries.length,
|
|
3759
|
+
bytes: entries.reduce((sum, entry) => sum + (entry.size ?? 0), 0),
|
|
3760
|
+
messages: 1,
|
|
3761
|
+
});
|
|
3762
|
+
const indexStartedAt = internalProfileStart(profile);
|
|
3763
|
+
const trustedMissing =
|
|
3764
|
+
options.__peerbitEntriesAlreadyMissing === true &&
|
|
3765
|
+
batchHashes.size === entries.length;
|
|
3766
|
+
await this.entryIndex.putAppendBatch(entries, {
|
|
3767
|
+
unique: trustedMissing,
|
|
3768
|
+
heads: headFlags,
|
|
3769
|
+
prepared: preparedBatch.prepared,
|
|
3770
|
+
deferIndexWrite: options.__peerbitDeferIndexWrite,
|
|
3771
|
+
profile,
|
|
3772
|
+
});
|
|
3773
|
+
emitInternalProfileDuration(profile, indexStartedAt, {
|
|
3774
|
+
name: "log.joinIndependent.entryIndex",
|
|
3775
|
+
component: "log",
|
|
3776
|
+
entries: entries.length,
|
|
3777
|
+
messages: 1,
|
|
3778
|
+
details: { trustedMissing },
|
|
3779
|
+
});
|
|
3780
|
+
|
|
3781
|
+
const changeStartedAt = internalProfileStart(profile);
|
|
3782
|
+
if (options.__peerbitOnAppendHashes && !options.onChange) {
|
|
3783
|
+
await options.__peerbitOnAppendHashes(
|
|
3784
|
+
entries.map((entry) => entry.hash),
|
|
3785
|
+
);
|
|
3786
|
+
} else {
|
|
3787
|
+
const change: Change<T> = {
|
|
3788
|
+
added: entries.map((entry, index) => ({
|
|
3789
|
+
head: headFlags[index]!,
|
|
3790
|
+
entry,
|
|
3791
|
+
})),
|
|
3792
|
+
removed: [],
|
|
3793
|
+
};
|
|
3794
|
+
await options.onChange?.(change);
|
|
3795
|
+
await this._onChange?.(change);
|
|
3796
|
+
}
|
|
3797
|
+
emitInternalProfileDuration(profile, changeStartedAt, {
|
|
3798
|
+
name: "log.joinIndependent.change",
|
|
3799
|
+
component: "log",
|
|
3800
|
+
entries: entries.length,
|
|
3801
|
+
messages: 1,
|
|
3802
|
+
details: {
|
|
3803
|
+
hashOnly: !!options.__peerbitOnAppendHashes && !options.onChange,
|
|
3804
|
+
},
|
|
3805
|
+
});
|
|
3806
|
+
})().finally(() => {
|
|
3807
|
+
for (const entry of entries) {
|
|
3808
|
+
this._joining.delete(entry.hash);
|
|
3809
|
+
}
|
|
3810
|
+
});
|
|
3811
|
+
|
|
3812
|
+
for (const entry of entries) {
|
|
3813
|
+
this._joining.set(entry.hash, batchPromise);
|
|
3814
|
+
}
|
|
3815
|
+
await batchPromise;
|
|
3816
|
+
return true;
|
|
3817
|
+
}
|
|
3818
|
+
|
|
3819
|
+
private takePreparedIndependentAppendBatch(
|
|
3820
|
+
entries: Entry<T>[],
|
|
3821
|
+
headFlags: boolean[],
|
|
3822
|
+
): PreparedIndependentAppendBatch | undefined {
|
|
3823
|
+
if (!entries.every((entry) => Entry.hasPreparedBlock(entry))) {
|
|
3824
|
+
return;
|
|
3825
|
+
}
|
|
3826
|
+
const hasPreparedShallowEntries = entries.every((entry) =>
|
|
3827
|
+
Entry.hasPreparedShallowEntry(entry),
|
|
3828
|
+
);
|
|
3829
|
+
const hasPreparedNativeEntries =
|
|
3830
|
+
hasPreparedShallowEntries &&
|
|
3831
|
+
entries.every((entry) => Entry.hasPreparedNativeLogEntry(entry));
|
|
3832
|
+
|
|
3833
|
+
const blocks = entries.map((entry) => {
|
|
3834
|
+
const prepared = Entry.takePreparedBlock(entry);
|
|
3835
|
+
if (!prepared) {
|
|
3836
|
+
throw new Error("Missing prepared entry block");
|
|
3837
|
+
}
|
|
3838
|
+
return prepared;
|
|
3839
|
+
});
|
|
3840
|
+
if (!hasPreparedShallowEntries) {
|
|
3841
|
+
return { blocks };
|
|
3842
|
+
}
|
|
3843
|
+
|
|
3844
|
+
const shallowEntries = entries.map((entry, index) => {
|
|
3845
|
+
const shallowEntry = Entry.takePreparedShallowEntry(
|
|
3846
|
+
entry,
|
|
3847
|
+
headFlags[index] ?? true,
|
|
3848
|
+
);
|
|
3849
|
+
if (!shallowEntry) {
|
|
3850
|
+
throw new Error("Missing prepared shallow entry");
|
|
3851
|
+
}
|
|
3852
|
+
return shallowEntry;
|
|
3853
|
+
});
|
|
3854
|
+
const nativeEntries = hasPreparedNativeEntries
|
|
3855
|
+
? entries.map((entry, index) => {
|
|
3856
|
+
const nativeEntry = Entry.takePreparedNativeLogEntry(
|
|
3857
|
+
entry,
|
|
3858
|
+
headFlags[index] ?? true,
|
|
3859
|
+
);
|
|
3860
|
+
if (!nativeEntry) {
|
|
3861
|
+
throw new Error("Missing prepared native log entry");
|
|
3862
|
+
}
|
|
3863
|
+
return nativeEntry;
|
|
3864
|
+
})
|
|
3865
|
+
: undefined;
|
|
3866
|
+
|
|
3867
|
+
return {
|
|
3868
|
+
blocks,
|
|
3869
|
+
prepared: {
|
|
3870
|
+
shallowEntries,
|
|
3871
|
+
nativeEntries,
|
|
3872
|
+
},
|
|
3873
|
+
};
|
|
3874
|
+
}
|
|
3875
|
+
|
|
880
3876
|
/**
|
|
881
3877
|
* Bottom up join of entries into the log
|
|
882
3878
|
* @param entry
|
|
@@ -909,10 +3905,8 @@ export class Log<T> {
|
|
|
909
3905
|
throw new Error("Unexpected");
|
|
910
3906
|
}
|
|
911
3907
|
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
!options.reset
|
|
915
|
-
) {
|
|
3908
|
+
const joinPlan = await this.entryIndex.planJoin(entry, options.reset);
|
|
3909
|
+
if (joinPlan.skip) {
|
|
916
3910
|
return false;
|
|
917
3911
|
}
|
|
918
3912
|
|
|
@@ -924,10 +3918,14 @@ export class Log<T> {
|
|
|
924
3918
|
}
|
|
925
3919
|
}
|
|
926
3920
|
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
3921
|
+
if (joinPlan.coveredByCut) {
|
|
3922
|
+
return false;
|
|
3923
|
+
}
|
|
3924
|
+
|
|
3925
|
+
if (!joinPlan.cutChecked) {
|
|
3926
|
+
const headsWithGid: JoinableEntry[] = await this.entryIndex.getJoinHeads(
|
|
3927
|
+
entry.meta.gid,
|
|
3928
|
+
);
|
|
931
3929
|
for (const v of headsWithGid) {
|
|
932
3930
|
// TODO second argument should be a time compare instead? what about next nexts?
|
|
933
3931
|
// and check the cut entry is newer than the current 'entry'
|
|
@@ -946,29 +3944,57 @@ export class Log<T> {
|
|
|
946
3944
|
options.remote && typeof options.remote === "object"
|
|
947
3945
|
? options.remote
|
|
948
3946
|
: undefined;
|
|
3947
|
+
const parents: Array<{ hash: string; entry?: Entry<T> }> = [];
|
|
3948
|
+
const unresolvedParentHashes: string[] = [];
|
|
949
3949
|
|
|
950
|
-
for (const a of
|
|
3950
|
+
for (const a of joinPlan.missingParents) {
|
|
951
3951
|
const prev = this._joining.get(a);
|
|
952
3952
|
if (prev) {
|
|
953
3953
|
await prev;
|
|
954
3954
|
continue;
|
|
955
3955
|
}
|
|
956
|
-
|
|
3956
|
+
|
|
3957
|
+
const referenced = options.references?.get(a);
|
|
3958
|
+
parents.push({ hash: a, entry: referenced });
|
|
3959
|
+
if (!referenced) {
|
|
3960
|
+
unresolvedParentHashes.push(a);
|
|
3961
|
+
}
|
|
3962
|
+
}
|
|
3963
|
+
|
|
3964
|
+
const localParents =
|
|
3965
|
+
unresolvedParentHashes.length > 0
|
|
3966
|
+
? await this.entryIndex.getMany(unresolvedParentHashes, {
|
|
3967
|
+
type: "full",
|
|
3968
|
+
ignoreMissing: true,
|
|
3969
|
+
})
|
|
3970
|
+
: [];
|
|
3971
|
+
const localParentByHash = new Map<string, Entry<T>>();
|
|
3972
|
+
for (const parent of localParents) {
|
|
3973
|
+
if (parent) {
|
|
3974
|
+
localParentByHash.set(parent.hash, parent);
|
|
3975
|
+
}
|
|
3976
|
+
}
|
|
3977
|
+
|
|
3978
|
+
for (const parent of parents) {
|
|
3979
|
+
const a = parent.hash;
|
|
3980
|
+
const prev = this._joining.get(a);
|
|
3981
|
+
if (prev) {
|
|
3982
|
+
await prev;
|
|
957
3983
|
continue;
|
|
958
3984
|
}
|
|
959
3985
|
|
|
960
|
-
|
|
961
|
-
let nested: Entry<T>;
|
|
3986
|
+
let nested = parent.entry ?? localParentByHash.get(a);
|
|
962
3987
|
try {
|
|
963
|
-
nested
|
|
964
|
-
options.
|
|
965
|
-
|
|
3988
|
+
if (!nested) {
|
|
3989
|
+
const from = await options.resolveRemoteFrom?.(a, remote?.signal);
|
|
3990
|
+
nested = await Entry.fromMultihash<T>(this._storage, a, {
|
|
966
3991
|
remote: {
|
|
967
3992
|
timeout: remote?.timeout,
|
|
968
3993
|
signal: remote?.signal,
|
|
969
3994
|
...(from && from.length > 0 ? { from } : {}),
|
|
970
3995
|
},
|
|
971
|
-
})
|
|
3996
|
+
});
|
|
3997
|
+
}
|
|
972
3998
|
} catch (error) {
|
|
973
3999
|
if (isRecoverableJoinResolveError(error)) {
|
|
974
4000
|
return false;
|
|
@@ -1003,9 +4029,9 @@ export class Log<T> {
|
|
|
1003
4029
|
|
|
1004
4030
|
const pendingDeletes: (
|
|
1005
4031
|
| PendingDelete<T>
|
|
1006
|
-
| { entry:
|
|
4032
|
+
| { entry: ShallowOrFullEntry<T>; fn: undefined }
|
|
1007
4033
|
)[] = await this.processEntry(entry);
|
|
1008
|
-
const trimmed = await this.
|
|
4034
|
+
const trimmed = await this.trimIfConfigured(options.trim);
|
|
1009
4035
|
|
|
1010
4036
|
if (trimmed) {
|
|
1011
4037
|
for (const removedEntry of trimmed) {
|
|
@@ -1064,7 +4090,22 @@ export class Log<T> {
|
|
|
1064
4090
|
| { hash: string; meta: { next: string[] } }[],
|
|
1065
4091
|
skipFirst = false,
|
|
1066
4092
|
) {
|
|
1067
|
-
const
|
|
4093
|
+
const entries = Array.isArray(from) ? [...from] : [from];
|
|
4094
|
+
const nativeDeletePlan = this.entryIndex.planDeleteRecursively(
|
|
4095
|
+
entries,
|
|
4096
|
+
skipFirst,
|
|
4097
|
+
);
|
|
4098
|
+
if (nativeDeletePlan) {
|
|
4099
|
+
const toDelete: PendingDelete<T>[] = [];
|
|
4100
|
+
for (const hash of nativeDeletePlan) {
|
|
4101
|
+
const deleteFn = await this.prepareDelete(hash);
|
|
4102
|
+
deleteFn.entry &&
|
|
4103
|
+
toDelete.push({ entry: deleteFn.entry, fn: deleteFn.fn });
|
|
4104
|
+
}
|
|
4105
|
+
return toDelete;
|
|
4106
|
+
}
|
|
4107
|
+
|
|
4108
|
+
const stack = entries;
|
|
1068
4109
|
const promises: (Promise<void> | void)[] = [];
|
|
1069
4110
|
let counter = 0;
|
|
1070
4111
|
const toDelete: PendingDelete<T>[] = [];
|
|
@@ -1079,8 +4120,7 @@ export class Log<T> {
|
|
|
1079
4120
|
}
|
|
1080
4121
|
|
|
1081
4122
|
for (const next of entry.meta.next) {
|
|
1082
|
-
const
|
|
1083
|
-
const entriesThatHasNext = await nextFromNext.all();
|
|
4123
|
+
const entriesThatHasNext = await this.entryIndex.getJoinChildren(next);
|
|
1084
4124
|
|
|
1085
4125
|
// if there are no entries which is not of "CUT" type, we can safely delete the next entry
|
|
1086
4126
|
// figureately speaking, these means where are cutting all branches to a stem, so we can delete the stem as well
|
|
@@ -1163,8 +4203,8 @@ export class Log<T> {
|
|
|
1163
4203
|
async close() {
|
|
1164
4204
|
// Don't return early here if closed = true, because "load" might create processes that needs to be closed
|
|
1165
4205
|
this._closed = true; // closed = true before doing below, else we might try to open the headsIndex cache because it is closed as we assume log is still open
|
|
1166
|
-
this._closeController
|
|
1167
|
-
await this._entryIndex
|
|
4206
|
+
this._closeController?.abort();
|
|
4207
|
+
await this._entryIndex?.flushPendingWrites();
|
|
1168
4208
|
await this._indexer?.stop?.();
|
|
1169
4209
|
this._indexer = undefined as any;
|
|
1170
4210
|
this._loadedOnce = false;
|
|
@@ -1174,7 +4214,7 @@ export class Log<T> {
|
|
|
1174
4214
|
// Don't return early here if closed = true, because "load" might create processes that needs to be closed
|
|
1175
4215
|
this._closed = true; // closed = true before doing below, else we might try to open the headsIndex cache because it is closed as we assume log is still open
|
|
1176
4216
|
this._closeController.abort();
|
|
1177
|
-
await this.
|
|
4217
|
+
await this._entryIndex?.clear();
|
|
1178
4218
|
await this._indexer?.drop();
|
|
1179
4219
|
await this._indexer?.stop?.();
|
|
1180
4220
|
}
|