@fireproof/core-base 0.22.0-keybag → 0.23.1-dev-issue-1057
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +197 -228
- package/README.md +269 -0
- package/apply-head-queue.js.map +1 -1
- package/bundle-not-impl.js.map +1 -1
- package/compact-strategies.js +3 -0
- package/compact-strategies.js.map +1 -1
- package/crdt-clock.d.ts +1 -1
- package/crdt-clock.js +2 -0
- package/crdt-clock.js.map +1 -1
- package/crdt-helpers.js.map +1 -1
- package/crdt.js.map +1 -1
- package/database.js.map +1 -1
- package/index.js.map +1 -1
- package/indexer-helpers.js.map +1 -1
- package/indexer.js +1 -0
- package/indexer.js.map +1 -1
- package/ledger.js.map +1 -1
- package/package.json +11 -11
- package/version.js.map +1 -1
- package/write-queue.js.map +1 -1
- package/apply-head-queue.ts +0 -72
- package/bundle-not-impl.ts +0 -4
- package/compact-strategies.ts +0 -95
- package/crdt-clock.ts +0 -192
- package/crdt-helpers.ts +0 -408
- package/crdt.ts +0 -275
- package/database.ts +0 -200
- package/index.ts +0 -15
- package/indexer-helpers.ts +0 -263
- package/indexer.ts +0 -360
- package/ledger.ts +0 -345
- package/tsconfig.json +0 -18
- package/version.ts +0 -3
- package/write-queue.ts +0 -93
package/indexer.ts
DELETED
|
@@ -1,360 +0,0 @@
|
|
|
1
|
-
/// <reference types="@fireproof/core-types-base/prolly-trees.d.ts" />
|
|
2
|
-
|
|
3
|
-
import {
|
|
4
|
-
type ClockHead,
|
|
5
|
-
type DocUpdate,
|
|
6
|
-
type MapFn,
|
|
7
|
-
type IndexUpdate,
|
|
8
|
-
type QueryOpts,
|
|
9
|
-
type IdxMeta,
|
|
10
|
-
type DocFragment,
|
|
11
|
-
type IdxMetaMap,
|
|
12
|
-
type IndexKeyType,
|
|
13
|
-
type IndexRows,
|
|
14
|
-
type DocTypes,
|
|
15
|
-
type IndexUpdateString,
|
|
16
|
-
throwFalsy,
|
|
17
|
-
type IndexTransactionMeta,
|
|
18
|
-
type SuperThis,
|
|
19
|
-
type BaseBlockstore,
|
|
20
|
-
type CRDT,
|
|
21
|
-
type HasCRDT,
|
|
22
|
-
type HasLogger,
|
|
23
|
-
type HasSuperThis,
|
|
24
|
-
type RefLedger,
|
|
25
|
-
type DocWithId,
|
|
26
|
-
IndexIf,
|
|
27
|
-
IndexTree,
|
|
28
|
-
} from "@fireproof/core-types-base";
|
|
29
|
-
// import { BaseBlockstore } from "./blockstore/index.js";
|
|
30
|
-
|
|
31
|
-
import {
|
|
32
|
-
bulkIndex,
|
|
33
|
-
indexEntriesForChanges,
|
|
34
|
-
byIdOpts,
|
|
35
|
-
byKeyOpts,
|
|
36
|
-
applyQuery,
|
|
37
|
-
encodeRange,
|
|
38
|
-
encodeKey,
|
|
39
|
-
loadIndex,
|
|
40
|
-
IndexDocString,
|
|
41
|
-
CompareKey,
|
|
42
|
-
} from "./indexer-helpers.js";
|
|
43
|
-
import { ensureLogger } from "@fireproof/core-runtime";
|
|
44
|
-
import { Logger } from "@adviser/cement";
|
|
45
|
-
|
|
46
|
-
// import { ProllyNode } from "prolly-trees/base";
|
|
47
|
-
|
|
48
|
-
function refLedger(u: HasCRDT | RefLedger): u is RefLedger {
|
|
49
|
-
return !!(u as RefLedger).ledger;
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
export function index<T extends DocTypes = DocTypes, K extends IndexKeyType = string, R extends DocFragment = T>(
|
|
53
|
-
refDb: HasLogger & HasSuperThis & (HasCRDT | RefLedger),
|
|
54
|
-
name: string,
|
|
55
|
-
mapFn?: MapFn<T>,
|
|
56
|
-
meta?: IdxMeta,
|
|
57
|
-
): Index<T, K, R> {
|
|
58
|
-
const crdt = refLedger(refDb) ? refDb.ledger.crdt : refDb.crdt;
|
|
59
|
-
|
|
60
|
-
if (mapFn && meta) throw refDb.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
|
|
61
|
-
if (mapFn && mapFn.constructor.name !== "Function") throw refDb.logger.Error().Msg("mapFn must be a function").AsError();
|
|
62
|
-
if (crdt.indexers.has(name)) {
|
|
63
|
-
const idx = crdt.indexers.get(name) as unknown as Index<T, K>;
|
|
64
|
-
idx.applyMapFn(name, mapFn, meta);
|
|
65
|
-
} else {
|
|
66
|
-
const idx = new Index<T, K>(refDb.sthis, crdt, name, mapFn, meta);
|
|
67
|
-
crdt.indexers.set(name, idx as unknown as Index<DocTypes, K, DocTypes>);
|
|
68
|
-
}
|
|
69
|
-
return crdt.indexers.get(name) as unknown as Index<T, K, R>;
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
// interface ByIdIndexIten<K extends IndexKeyType> {
|
|
73
|
-
// readonly key: K;
|
|
74
|
-
// readonly value: [K, K];
|
|
75
|
-
// }
|
|
76
|
-
|
|
77
|
-
export class Index<T extends DocTypes, K extends IndexKeyType = string, R extends DocFragment = T> implements IndexIf<T, K, R> {
|
|
78
|
-
readonly blockstore: BaseBlockstore;
|
|
79
|
-
readonly crdt: CRDT;
|
|
80
|
-
readonly name: string;
|
|
81
|
-
mapFn?: MapFn<T>;
|
|
82
|
-
mapFnString = "";
|
|
83
|
-
byKey: IndexTree<K, R> = {};
|
|
84
|
-
byId: IndexTree<K, R> = {};
|
|
85
|
-
indexHead?: ClockHead;
|
|
86
|
-
|
|
87
|
-
initError?: Error;
|
|
88
|
-
|
|
89
|
-
ready(): Promise<void> {
|
|
90
|
-
return Promise.all([this.blockstore.ready(), this.crdt.ready()]).then(() => {
|
|
91
|
-
/* noop */
|
|
92
|
-
});
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
// close(): Promise<void> {
|
|
96
|
-
// return Promise.all([this.blockstore.close(), this.crdt.close()]).then(() => {
|
|
97
|
-
// /* noop */
|
|
98
|
-
// });
|
|
99
|
-
// }
|
|
100
|
-
// destroy(): Promise<void> {
|
|
101
|
-
// return Promise.all([this.blockstore.destroy(), this.crdt.destroy()]).then(() => {
|
|
102
|
-
// /* noop */
|
|
103
|
-
// });
|
|
104
|
-
// }
|
|
105
|
-
|
|
106
|
-
readonly logger: Logger;
|
|
107
|
-
|
|
108
|
-
constructor(sthis: SuperThis, crdt: CRDT, name: string, mapFn?: MapFn<T>, meta?: IdxMeta) {
|
|
109
|
-
this.logger = ensureLogger(sthis, "Index");
|
|
110
|
-
if (!crdt.indexBlockstore) {
|
|
111
|
-
throw sthis.logger.Error().Msg("indexBlockstore not set").AsError();
|
|
112
|
-
}
|
|
113
|
-
this.blockstore = crdt.indexBlockstore;
|
|
114
|
-
this.crdt = crdt as CRDT;
|
|
115
|
-
this.applyMapFn(name, mapFn, meta);
|
|
116
|
-
this.name = name;
|
|
117
|
-
if (!(this.mapFnString || this.initError)) throw this.logger.Error().Msg("missing mapFnString").AsError();
|
|
118
|
-
// this.ready = this.blockstore.ready.then(() => {
|
|
119
|
-
// return;
|
|
120
|
-
// });
|
|
121
|
-
// .then((header: IdxCarHeader) => {
|
|
122
|
-
// // @ts-ignore
|
|
123
|
-
// if (header.head) throw new Error('cannot have head in idx header')
|
|
124
|
-
// if (header.indexes === undefined) throw new Error('missing indexes in idx header')
|
|
125
|
-
// // for (const [name, idx] of Object.entries(header.indexes)) {
|
|
126
|
-
// // index({ _crdt: crdt }, name, undefined, idx as IdxMeta)
|
|
127
|
-
// // }
|
|
128
|
-
// })
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
applyMapFn(name: string, mapFn?: MapFn<T>, meta?: IdxMeta) {
|
|
132
|
-
if (mapFn && meta) throw this.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
|
|
133
|
-
if (this.name && this.name !== name) throw this.logger.Error().Msg("cannot change name").AsError();
|
|
134
|
-
// this.name = name;
|
|
135
|
-
try {
|
|
136
|
-
let mapFnChanged = false;
|
|
137
|
-
|
|
138
|
-
if (meta) {
|
|
139
|
-
// hydrating from header
|
|
140
|
-
if (this.indexHead && this.indexHead.map((c) => c.toString()).join() !== meta.head.map((c) => c.toString()).join()) {
|
|
141
|
-
throw this.logger.Error().Msg("cannot apply different head meta").AsError();
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
if (this.mapFnString) {
|
|
145
|
-
// we already initialized from application code
|
|
146
|
-
if (this.mapFnString !== meta.map) {
|
|
147
|
-
this.mapFnString = meta.map;
|
|
148
|
-
mapFnChanged = true;
|
|
149
|
-
}
|
|
150
|
-
// Always apply the metadata
|
|
151
|
-
this.byId.cid = meta.byId;
|
|
152
|
-
this.byKey.cid = meta.byKey;
|
|
153
|
-
this.indexHead = meta.head;
|
|
154
|
-
} else {
|
|
155
|
-
// we are first
|
|
156
|
-
this.mapFnString = meta.map;
|
|
157
|
-
this.byId.cid = meta.byId;
|
|
158
|
-
this.byKey.cid = meta.byKey;
|
|
159
|
-
this.indexHead = meta.head;
|
|
160
|
-
}
|
|
161
|
-
} else {
|
|
162
|
-
if (this.mapFn) {
|
|
163
|
-
// we already initialized from application code
|
|
164
|
-
if (mapFn) {
|
|
165
|
-
if (this.mapFn.toString() !== mapFn.toString()) {
|
|
166
|
-
this.mapFn = mapFn;
|
|
167
|
-
this.mapFnString = mapFn.toString();
|
|
168
|
-
mapFnChanged = true;
|
|
169
|
-
}
|
|
170
|
-
}
|
|
171
|
-
} else {
|
|
172
|
-
// application code is creating an index
|
|
173
|
-
if (!mapFn) {
|
|
174
|
-
mapFn = ((doc) => (doc as unknown as Record<string, unknown>)[name] ?? undefined) as MapFn<T>;
|
|
175
|
-
}
|
|
176
|
-
if (this.mapFnString) {
|
|
177
|
-
// we already loaded from a header
|
|
178
|
-
if (this.mapFnString !== mapFn.toString()) {
|
|
179
|
-
mapFnChanged = true;
|
|
180
|
-
}
|
|
181
|
-
} else {
|
|
182
|
-
// we are first
|
|
183
|
-
this.mapFnString = mapFn.toString();
|
|
184
|
-
}
|
|
185
|
-
this.mapFn = mapFn;
|
|
186
|
-
}
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
// If the map function changed, reset the index for correctness
|
|
190
|
-
if (mapFnChanged) {
|
|
191
|
-
this._resetIndex();
|
|
192
|
-
}
|
|
193
|
-
} catch (e) {
|
|
194
|
-
this.initError = e as Error;
|
|
195
|
-
}
|
|
196
|
-
}
|
|
197
|
-
|
|
198
|
-
async query(opts: QueryOpts<K> = {}): Promise<IndexRows<T, K, R>> {
|
|
199
|
-
this.logger.Debug().Msg("enter query");
|
|
200
|
-
await this.ready();
|
|
201
|
-
// this._resetIndex();
|
|
202
|
-
this.logger.Debug().Msg("post ready query");
|
|
203
|
-
await this._updateIndex();
|
|
204
|
-
this.logger.Debug().Msg("post _updateIndex query");
|
|
205
|
-
await this._hydrateIndex();
|
|
206
|
-
this.logger.Debug().Msg("post _hydrateIndex query");
|
|
207
|
-
if (!this.byKey.root) {
|
|
208
|
-
return await applyQuery<T, K, R>(this.crdt, { result: [] }, opts);
|
|
209
|
-
}
|
|
210
|
-
if (opts.includeDocs === undefined) opts.includeDocs = true;
|
|
211
|
-
if (opts.range) {
|
|
212
|
-
const eRange = encodeRange(opts.range);
|
|
213
|
-
return await applyQuery<T, K, R>(this.crdt, await throwFalsy(this.byKey.root).range(eRange[0], eRange[1]), opts);
|
|
214
|
-
}
|
|
215
|
-
if (typeof opts.key === "boolean" || opts.key) {
|
|
216
|
-
const encodedKey = encodeKey(opts.key);
|
|
217
|
-
return await applyQuery<T, K, R>(this.crdt, await throwFalsy(this.byKey.root).get(encodedKey), opts);
|
|
218
|
-
}
|
|
219
|
-
if (Array.isArray(opts.keys)) {
|
|
220
|
-
// Create a new options object without the limit to avoid limiting individual key results
|
|
221
|
-
const optsWithoutLimit = { ...opts };
|
|
222
|
-
delete optsWithoutLimit.limit;
|
|
223
|
-
|
|
224
|
-
// Process each key separately but don't apply limit yet
|
|
225
|
-
const results = await Promise.all(
|
|
226
|
-
opts.keys.map(async (key: DocFragment) => {
|
|
227
|
-
const encodedKey = encodeKey(key);
|
|
228
|
-
return (await applyQuery<T, K, R>(this.crdt, await throwFalsy(this.byKey.root).get(encodedKey), optsWithoutLimit)).rows;
|
|
229
|
-
}),
|
|
230
|
-
);
|
|
231
|
-
|
|
232
|
-
// Flatten all results into a single array
|
|
233
|
-
let flattenedRows = results.flat();
|
|
234
|
-
|
|
235
|
-
// Apply the original limit to the combined results if it was specified
|
|
236
|
-
if (opts) {
|
|
237
|
-
flattenedRows = flattenedRows.slice(0, opts.limit);
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
return {
|
|
241
|
-
rows: flattenedRows,
|
|
242
|
-
docs: flattenedRows.map((r) => r.doc).filter((r): r is DocWithId<T> => !!r),
|
|
243
|
-
};
|
|
244
|
-
}
|
|
245
|
-
if (opts.prefix) {
|
|
246
|
-
if (!Array.isArray(opts.prefix)) opts.prefix = [opts.prefix];
|
|
247
|
-
// prefix should be always an array
|
|
248
|
-
const start = [...opts.prefix, NaN];
|
|
249
|
-
const end = [...opts.prefix, Infinity];
|
|
250
|
-
const encodedR = encodeRange([start, end]);
|
|
251
|
-
return await applyQuery<T, K, R>(this.crdt, await this.byKey.root.range(...encodedR), opts);
|
|
252
|
-
}
|
|
253
|
-
const all = await this.byKey.root.getAllEntries(); // funky return type
|
|
254
|
-
return await applyQuery<T, K, R>(
|
|
255
|
-
this.crdt,
|
|
256
|
-
{
|
|
257
|
-
// getAllEntries returns a different type than range
|
|
258
|
-
result: all.result.map(({ key: [k, id], value }) => ({
|
|
259
|
-
key: k as [K, string],
|
|
260
|
-
id,
|
|
261
|
-
value,
|
|
262
|
-
})),
|
|
263
|
-
},
|
|
264
|
-
opts,
|
|
265
|
-
);
|
|
266
|
-
}
|
|
267
|
-
|
|
268
|
-
_resetIndex() {
|
|
269
|
-
this.byId = {};
|
|
270
|
-
this.byKey = {};
|
|
271
|
-
this.indexHead = undefined;
|
|
272
|
-
}
|
|
273
|
-
|
|
274
|
-
async _hydrateIndex() {
|
|
275
|
-
if (this.byId.root && this.byKey.root) return;
|
|
276
|
-
if (!this.byId.cid || !this.byKey.cid) return;
|
|
277
|
-
this.byId.root = await loadIndex<K, R, string | number>(this.blockstore, this.byId.cid, byIdOpts);
|
|
278
|
-
this.byKey.root = await loadIndex<K, R, CompareKey>(this.blockstore, this.byKey.cid, byKeyOpts);
|
|
279
|
-
}
|
|
280
|
-
|
|
281
|
-
async _updateIndex(): Promise<IndexTransactionMeta> {
|
|
282
|
-
await this.ready();
|
|
283
|
-
this.logger.Debug().Msg("enter _updateIndex");
|
|
284
|
-
if (this.initError) throw this.initError;
|
|
285
|
-
if (!this.mapFn) throw this.logger.Error().Msg("No map function defined").AsError();
|
|
286
|
-
let result: DocUpdate<T>[], head: ClockHead;
|
|
287
|
-
if (!this.indexHead || this.indexHead.length === 0) {
|
|
288
|
-
({ result, head } = await this.crdt.allDocs<T>());
|
|
289
|
-
this.logger.Debug().Msg("enter crdt.allDocs");
|
|
290
|
-
} else {
|
|
291
|
-
({ result, head } = await this.crdt.changes<T>(this.indexHead));
|
|
292
|
-
this.logger.Debug().Msg("enter crdt.changes");
|
|
293
|
-
}
|
|
294
|
-
if (result.length === 0) {
|
|
295
|
-
this.indexHead = head;
|
|
296
|
-
// return { byId: this.byId, byKey: this.byKey } as IndexTransactionMeta;
|
|
297
|
-
}
|
|
298
|
-
let staleKeyIndexEntries: IndexUpdate<K>[] = [];
|
|
299
|
-
let removeIdIndexEntries: IndexUpdateString[] = [];
|
|
300
|
-
if (this.byId.root) {
|
|
301
|
-
const removeIds = result.map(({ id: key }) => key);
|
|
302
|
-
const { result: oldChangeEntries } = await this.byId.root.getMany(removeIds);
|
|
303
|
-
staleKeyIndexEntries = oldChangeEntries.map((key) => ({ key, del: true }));
|
|
304
|
-
removeIdIndexEntries = oldChangeEntries.map((key) => ({ key: key[1], del: true }));
|
|
305
|
-
}
|
|
306
|
-
const indexEntries = indexEntriesForChanges<T, K>(result, this.mapFn); // use a getter to translate from string
|
|
307
|
-
const byIdIndexEntries: IndexDocString[] = indexEntries.map(({ key }) => ({
|
|
308
|
-
key: key[1],
|
|
309
|
-
value: key,
|
|
310
|
-
}));
|
|
311
|
-
const indexerMeta: IdxMetaMap = { indexes: new Map() };
|
|
312
|
-
|
|
313
|
-
for (const [name, indexer] of this.crdt.indexers) {
|
|
314
|
-
if (indexer.indexHead) {
|
|
315
|
-
indexerMeta.indexes?.set(name, {
|
|
316
|
-
byId: indexer.byId.cid,
|
|
317
|
-
byKey: indexer.byKey.cid,
|
|
318
|
-
head: indexer.indexHead,
|
|
319
|
-
map: indexer.mapFnString,
|
|
320
|
-
name: indexer.name,
|
|
321
|
-
} as IdxMeta);
|
|
322
|
-
}
|
|
323
|
-
}
|
|
324
|
-
if (result.length === 0) {
|
|
325
|
-
return indexerMeta as unknown as IndexTransactionMeta;
|
|
326
|
-
}
|
|
327
|
-
this.logger.Debug().Msg("pre this.blockstore.transaction");
|
|
328
|
-
const { meta } = await this.blockstore.transaction<IndexTransactionMeta>(async (tblocks): Promise<IndexTransactionMeta> => {
|
|
329
|
-
this.byId = await bulkIndex<K, R, string | number>(
|
|
330
|
-
this.logger,
|
|
331
|
-
tblocks,
|
|
332
|
-
this.byId,
|
|
333
|
-
removeIdIndexEntries.concat(byIdIndexEntries),
|
|
334
|
-
byIdOpts,
|
|
335
|
-
);
|
|
336
|
-
this.byKey = await bulkIndex<K, R, CompareKey>(
|
|
337
|
-
this.logger,
|
|
338
|
-
tblocks,
|
|
339
|
-
this.byKey,
|
|
340
|
-
staleKeyIndexEntries.concat(indexEntries),
|
|
341
|
-
byKeyOpts,
|
|
342
|
-
);
|
|
343
|
-
this.indexHead = head;
|
|
344
|
-
if (this.byId.cid && this.byKey.cid) {
|
|
345
|
-
const idxMeta = {
|
|
346
|
-
byId: this.byId.cid,
|
|
347
|
-
byKey: this.byKey.cid,
|
|
348
|
-
head,
|
|
349
|
-
map: this.mapFnString,
|
|
350
|
-
name: this.name,
|
|
351
|
-
} as IdxMeta;
|
|
352
|
-
indexerMeta.indexes?.set(this.name, idxMeta);
|
|
353
|
-
}
|
|
354
|
-
this.logger.Debug().Any("indexerMeta", new Array(indexerMeta.indexes?.entries())).Msg("exit this.blockstore.transaction fn");
|
|
355
|
-
return indexerMeta as unknown as IndexTransactionMeta;
|
|
356
|
-
});
|
|
357
|
-
this.logger.Debug().Msg("post this.blockstore.transaction");
|
|
358
|
-
return meta;
|
|
359
|
-
}
|
|
360
|
-
}
|
package/ledger.ts
DELETED
|
@@ -1,345 +0,0 @@
|
|
|
1
|
-
import { BuildURI, KeyedResolvOnce, Logger, ResolveOnce, URI, AppContext } from "@adviser/cement";
|
|
2
|
-
|
|
3
|
-
import { writeQueue } from "./write-queue.js";
|
|
4
|
-
import {
|
|
5
|
-
DocUpdate,
|
|
6
|
-
ConfigOpts,
|
|
7
|
-
defaultWriteQueueOpts,
|
|
8
|
-
DocWithId,
|
|
9
|
-
ListenerFn,
|
|
10
|
-
DocTypes,
|
|
11
|
-
SuperThis,
|
|
12
|
-
Database,
|
|
13
|
-
Ledger,
|
|
14
|
-
WriteQueue,
|
|
15
|
-
CRDT,
|
|
16
|
-
LedgerOpts,
|
|
17
|
-
Attachable,
|
|
18
|
-
Attached,
|
|
19
|
-
LedgerOptsOptionalTracer,
|
|
20
|
-
PARAM,
|
|
21
|
-
} from "@fireproof/core-types-base";
|
|
22
|
-
import { StoreURIRuntime, StoreUrlsOpts } from "@fireproof/core-types-blockstore";
|
|
23
|
-
import { decodeFile, encodeFile, ensureLogger, ensureSuperThis, ensureURIDefaults, hashObject } from "@fireproof/core-runtime";
|
|
24
|
-
|
|
25
|
-
import { DatabaseImpl } from "./database.js";
|
|
26
|
-
import { CRDTImpl } from "./crdt.js";
|
|
27
|
-
import { toSortedArray } from "@adviser/cement/utils";
|
|
28
|
-
import { getDefaultURI } from "@fireproof/core-blockstore";
|
|
29
|
-
import { defaultKeyBagOpts } from "@fireproof/core-keybag";
|
|
30
|
-
|
|
31
|
-
const ledgers = new KeyedResolvOnce<Ledger>();
|
|
32
|
-
|
|
33
|
-
export function keyConfigOpts(sthis: SuperThis, name: string, opts?: ConfigOpts): string {
|
|
34
|
-
return JSON.stringify(
|
|
35
|
-
toSortedArray({
|
|
36
|
-
name,
|
|
37
|
-
stores: toSortedArray(JSON.parse(JSON.stringify(toStoreURIRuntime(sthis, name, opts?.storeUrls)))),
|
|
38
|
-
}),
|
|
39
|
-
);
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
export function isLedger(db: unknown): db is Ledger {
|
|
43
|
-
return db instanceof LedgerImpl || db instanceof LedgerShell;
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
export function LedgerFactory(name: string, opts?: ConfigOpts): Ledger {
|
|
47
|
-
const sthis = ensureSuperThis(opts);
|
|
48
|
-
const key = keyConfigOpts(sthis, name, opts);
|
|
49
|
-
const item = ledgers.get(key);
|
|
50
|
-
// if (!item.ready) {
|
|
51
|
-
// console.log("LedgerFactory", key);
|
|
52
|
-
// }
|
|
53
|
-
return new LedgerShell(
|
|
54
|
-
item.once((key) => {
|
|
55
|
-
// console.log("once-LedgerFactory", key);
|
|
56
|
-
|
|
57
|
-
const db = new LedgerImpl(sthis, {
|
|
58
|
-
name,
|
|
59
|
-
meta: opts?.meta,
|
|
60
|
-
keyBag: defaultKeyBagOpts(sthis, opts?.keyBag),
|
|
61
|
-
storeUrls: toStoreURIRuntime(sthis, name, opts?.storeUrls),
|
|
62
|
-
gatewayInterceptor: opts?.gatewayInterceptor,
|
|
63
|
-
compactStrategy: opts?.compactStrategy,
|
|
64
|
-
writeQueue: defaultWriteQueueOpts(opts?.writeQueue),
|
|
65
|
-
ctx: opts?.ctx ?? new AppContext(),
|
|
66
|
-
storeEnDe: {
|
|
67
|
-
encodeFile,
|
|
68
|
-
decodeFile,
|
|
69
|
-
...opts?.storeEnDe,
|
|
70
|
-
},
|
|
71
|
-
tracer:
|
|
72
|
-
opts?.tracer ??
|
|
73
|
-
(() => {
|
|
74
|
-
/* noop */
|
|
75
|
-
}),
|
|
76
|
-
});
|
|
77
|
-
db.onClosed(() => {
|
|
78
|
-
ledgers.unget(key);
|
|
79
|
-
});
|
|
80
|
-
return db;
|
|
81
|
-
}),
|
|
82
|
-
);
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
export class LedgerShell implements Ledger {
|
|
86
|
-
readonly ref: LedgerImpl;
|
|
87
|
-
readonly writeQueue: WriteQueue<DocUpdate<DocTypes>>;
|
|
88
|
-
readonly name: string;
|
|
89
|
-
|
|
90
|
-
constructor(ref: LedgerImpl) {
|
|
91
|
-
this.ref = ref;
|
|
92
|
-
this.writeQueue = ref.writeQueue;
|
|
93
|
-
this.name = ref.name;
|
|
94
|
-
ref.addShell(this);
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
attach(a: Attachable): Promise<Attached> {
|
|
98
|
-
return this.ref.attach(a);
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
get opts(): LedgerOpts {
|
|
102
|
-
return this.ref.opts;
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
get ctx(): AppContext {
|
|
106
|
-
return this.ref.ctx;
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
refId(): Promise<string> {
|
|
110
|
-
return this.ref.refId();
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
// get id(): string {
|
|
114
|
-
// return this.ref.id;
|
|
115
|
-
// }
|
|
116
|
-
get logger(): Logger {
|
|
117
|
-
return this.ref.logger;
|
|
118
|
-
}
|
|
119
|
-
get sthis(): SuperThis {
|
|
120
|
-
return this.ref.sthis;
|
|
121
|
-
}
|
|
122
|
-
get crdt(): CRDT {
|
|
123
|
-
return this.ref.crdt;
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
onClosed(fn: () => void): () => void {
|
|
127
|
-
return this.ref.onClosed(fn);
|
|
128
|
-
}
|
|
129
|
-
close(): Promise<void> {
|
|
130
|
-
return this.ref.shellClose(this);
|
|
131
|
-
}
|
|
132
|
-
destroy(): Promise<void> {
|
|
133
|
-
return this.ref.destroy();
|
|
134
|
-
}
|
|
135
|
-
ready(): Promise<void> {
|
|
136
|
-
return this.ref.ready();
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
// asDB(): Database {
|
|
140
|
-
// return this.ref.asDB();
|
|
141
|
-
// }
|
|
142
|
-
|
|
143
|
-
subscribe<T extends DocTypes>(listener: ListenerFn<T>, updates?: boolean): () => void {
|
|
144
|
-
return this.ref.subscribe(listener, updates);
|
|
145
|
-
}
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
class LedgerImpl implements Ledger {
|
|
149
|
-
// readonly name: string;
|
|
150
|
-
readonly opts: LedgerOpts;
|
|
151
|
-
|
|
152
|
-
_listening = false;
|
|
153
|
-
readonly _listeners = new Set<ListenerFn<DocTypes>>();
|
|
154
|
-
readonly _noupdate_listeners = new Set<ListenerFn<DocTypes>>();
|
|
155
|
-
readonly crdt: CRDT;
|
|
156
|
-
readonly writeQueue: WriteQueue<DocUpdate<DocTypes>>;
|
|
157
|
-
// readonly blockstore: BaseBlockstore;
|
|
158
|
-
|
|
159
|
-
readonly shells: Set<LedgerShell> = new Set<LedgerShell>();
|
|
160
|
-
|
|
161
|
-
readonly ctx: AppContext;
|
|
162
|
-
|
|
163
|
-
get name(): string {
|
|
164
|
-
return this.opts.name;
|
|
165
|
-
// this.opts.storeUrls.data.data.getParam(PARAM.NAME) ?? "default";
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
addShell(shell: LedgerShell) {
|
|
169
|
-
this.shells.add(shell);
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
readonly _refid = new ResolveOnce<string>();
|
|
173
|
-
refId(): Promise<string> {
|
|
174
|
-
return this._refid.once(() => hashObject(this.opts.storeUrls));
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
readonly _onClosedFns = new Map<string, () => void>();
|
|
178
|
-
onClosed(fn: () => void): () => void {
|
|
179
|
-
const id = this.sthis.nextId().str;
|
|
180
|
-
this._onClosedFns.set(id, fn);
|
|
181
|
-
return () => {
|
|
182
|
-
this._onClosedFns.delete(id);
|
|
183
|
-
};
|
|
184
|
-
}
|
|
185
|
-
async close() {
|
|
186
|
-
throw this.logger.Error().Str("db", this.name).Msg(`use shellClose`).AsError();
|
|
187
|
-
}
|
|
188
|
-
async shellClose(db: LedgerShell) {
|
|
189
|
-
if (!this.shells.has(db)) {
|
|
190
|
-
throw this.logger.Error().Str("db", this.name).Msg(`LedgerShell mismatch`).AsError();
|
|
191
|
-
}
|
|
192
|
-
this.shells.delete(db);
|
|
193
|
-
if (this.shells.size === 0) {
|
|
194
|
-
await this.ready();
|
|
195
|
-
await this.crdt.close();
|
|
196
|
-
await this.writeQueue.close();
|
|
197
|
-
this._onClosedFns.forEach((fn) => fn());
|
|
198
|
-
}
|
|
199
|
-
// await this.blockstore.close();
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
async destroy() {
|
|
203
|
-
await this.ready();
|
|
204
|
-
await this.crdt.destroy();
|
|
205
|
-
// await this.blockstore.destroy();
|
|
206
|
-
}
|
|
207
|
-
|
|
208
|
-
readonly _ready: ResolveOnce<void> = new ResolveOnce<void>();
|
|
209
|
-
async ready(): Promise<void> {
|
|
210
|
-
const ret = await this._ready.once(async () => {
|
|
211
|
-
await this.sthis.start();
|
|
212
|
-
await this.crdt.ready();
|
|
213
|
-
// await this.blockstore.ready();
|
|
214
|
-
});
|
|
215
|
-
return ret;
|
|
216
|
-
}
|
|
217
|
-
|
|
218
|
-
readonly logger: Logger;
|
|
219
|
-
readonly sthis: SuperThis;
|
|
220
|
-
// readonly id: string;
|
|
221
|
-
|
|
222
|
-
constructor(sthis: SuperThis, opts: LedgerOptsOptionalTracer) {
|
|
223
|
-
this.opts = {
|
|
224
|
-
tracer: () => {
|
|
225
|
-
/* noop */
|
|
226
|
-
},
|
|
227
|
-
...opts,
|
|
228
|
-
}; // || this.opts;
|
|
229
|
-
// this.name = opts.storeUrls.data.data.getParam(PARAM.NAME) || "default";
|
|
230
|
-
this.sthis = sthis;
|
|
231
|
-
this.ctx = opts.ctx;
|
|
232
|
-
// this.id = sthis.timeOrderedNextId().str;
|
|
233
|
-
this.logger = ensureLogger(this.sthis, "Ledger");
|
|
234
|
-
this.crdt = new CRDTImpl(this.sthis, this.opts, this);
|
|
235
|
-
this.writeQueue = writeQueue(
|
|
236
|
-
this.sthis,
|
|
237
|
-
async (updates: DocUpdate<DocTypes>[]) => this.crdt.bulk(updates),
|
|
238
|
-
this.opts.writeQueue,
|
|
239
|
-
);
|
|
240
|
-
this.crdt.clock.onTock(() => this._no_update_notify());
|
|
241
|
-
}
|
|
242
|
-
|
|
243
|
-
async attach(a: Attachable): Promise<Attached> {
|
|
244
|
-
await this.ready();
|
|
245
|
-
return this.crdt.blockstore.loader.attach(a);
|
|
246
|
-
}
|
|
247
|
-
|
|
248
|
-
// readonly _asDb = new ResolveOnce<Database>();
|
|
249
|
-
// asDB(): Database {
|
|
250
|
-
// return this._asDb.once(() => new DatabaseImpl(this));
|
|
251
|
-
// }
|
|
252
|
-
|
|
253
|
-
subscribe<T extends DocTypes>(listener: ListenerFn<T>, updates?: boolean): () => void {
|
|
254
|
-
this.ready();
|
|
255
|
-
this.logger.Debug().Bool("updates", updates).Msg("subscribe");
|
|
256
|
-
if (updates) {
|
|
257
|
-
if (!this._listening) {
|
|
258
|
-
this._listening = true;
|
|
259
|
-
this.crdt.clock.onTick((updates: DocUpdate<NonNullable<unknown>>[]) => {
|
|
260
|
-
void this._notify(updates);
|
|
261
|
-
});
|
|
262
|
-
}
|
|
263
|
-
this._listeners.add(listener as ListenerFn<NonNullable<unknown>>);
|
|
264
|
-
return () => {
|
|
265
|
-
this._listeners.delete(listener as ListenerFn<NonNullable<unknown>>);
|
|
266
|
-
};
|
|
267
|
-
} else {
|
|
268
|
-
this._noupdate_listeners.add(listener as ListenerFn<NonNullable<unknown>>);
|
|
269
|
-
return () => {
|
|
270
|
-
this._noupdate_listeners.delete(listener as ListenerFn<NonNullable<unknown>>);
|
|
271
|
-
};
|
|
272
|
-
}
|
|
273
|
-
}
|
|
274
|
-
|
|
275
|
-
private async _notify(updates: DocUpdate<DocTypes>[]) {
|
|
276
|
-
await this.ready();
|
|
277
|
-
if (this._listeners.size) {
|
|
278
|
-
const docs: DocWithId<DocTypes>[] = updates.map(({ id, value }) => ({ ...value, _id: id }));
|
|
279
|
-
for (const listener of this._listeners) {
|
|
280
|
-
await (async () => await listener(docs as DocWithId<DocTypes>[]))().catch((e: Error) => {
|
|
281
|
-
this.logger.Error().Err(e).Msg("subscriber error");
|
|
282
|
-
});
|
|
283
|
-
}
|
|
284
|
-
}
|
|
285
|
-
}
|
|
286
|
-
|
|
287
|
-
private async _no_update_notify() {
|
|
288
|
-
await this.ready();
|
|
289
|
-
if (this._noupdate_listeners.size) {
|
|
290
|
-
for (const listener of this._noupdate_listeners) {
|
|
291
|
-
await (async () => await listener([]))().catch((e: Error) => {
|
|
292
|
-
this.logger.Error().Err(e).Msg("subscriber error");
|
|
293
|
-
});
|
|
294
|
-
}
|
|
295
|
-
}
|
|
296
|
-
}
|
|
297
|
-
}
|
|
298
|
-
|
|
299
|
-
export function toStoreURIRuntime(sthis: SuperThis, name: string, sopts?: StoreUrlsOpts): StoreURIRuntime {
|
|
300
|
-
sopts = sopts || {};
|
|
301
|
-
if (!sopts.base) {
|
|
302
|
-
const fp_env = sthis.env.get("FP_STORAGE_URL");
|
|
303
|
-
if (fp_env) {
|
|
304
|
-
sopts = { ...sopts, base: BuildURI.from(fp_env).setParam(PARAM.URL_GEN, "fromEnv") };
|
|
305
|
-
} else {
|
|
306
|
-
sopts = { ...sopts, base: getDefaultURI(sthis).build().setParam(PARAM.URL_GEN, "default") };
|
|
307
|
-
}
|
|
308
|
-
}
|
|
309
|
-
const base = URI.from(sopts.base);
|
|
310
|
-
// bbase.setParam(PARAM.NAME, name);
|
|
311
|
-
// const base = bbase.URI();
|
|
312
|
-
// readonly public?: boolean;
|
|
313
|
-
// readonly meta?: DbMeta;
|
|
314
|
-
// readonly persistIndexes?: boolean;
|
|
315
|
-
// readonly autoCompact?: number;
|
|
316
|
-
// readonly threshold?: number;
|
|
317
|
-
return {
|
|
318
|
-
idx: {
|
|
319
|
-
car: ensureURIDefaults(sthis, { name }, sopts.idx?.car ?? sopts.data?.car, base, "car", { idx: true }),
|
|
320
|
-
file: ensureURIDefaults(
|
|
321
|
-
sthis,
|
|
322
|
-
{ name },
|
|
323
|
-
sopts.idx?.file ?? sopts.idx?.car ?? sopts.data?.file ?? sopts.data?.car,
|
|
324
|
-
base,
|
|
325
|
-
"file",
|
|
326
|
-
{
|
|
327
|
-
file: true,
|
|
328
|
-
idx: true,
|
|
329
|
-
},
|
|
330
|
-
),
|
|
331
|
-
meta: ensureURIDefaults(sthis, { name }, sopts.idx?.meta ?? sopts.data?.meta, base, "meta", { idx: true }),
|
|
332
|
-
wal: ensureURIDefaults(sthis, { name }, sopts.idx?.wal ?? sopts.data?.wal, base, "wal", { idx: true }),
|
|
333
|
-
},
|
|
334
|
-
data: {
|
|
335
|
-
car: ensureURIDefaults(sthis, { name }, sopts.data?.car, base, "car"),
|
|
336
|
-
file: ensureURIDefaults(sthis, { name }, sopts.data?.file ?? sopts.data?.car, base, "file", { file: true }),
|
|
337
|
-
meta: ensureURIDefaults(sthis, { name }, sopts.data?.meta, base, "meta"),
|
|
338
|
-
wal: ensureURIDefaults(sthis, { name }, sopts.data?.wal, base, "wal"),
|
|
339
|
-
},
|
|
340
|
-
};
|
|
341
|
-
}
|
|
342
|
-
|
|
343
|
-
export function fireproof(name: string, opts?: ConfigOpts): Database {
|
|
344
|
-
return new DatabaseImpl(LedgerFactory(name, opts));
|
|
345
|
-
}
|