@fireproof/core 0.3.22 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/dist/blockstore.js +242 -0
  2. package/dist/clock.js +355 -0
  3. package/dist/crypto.js +59 -0
  4. package/dist/database.js +308 -0
  5. package/dist/db-index.js +314 -0
  6. package/dist/fireproof.js +83 -0
  7. package/dist/hooks/use-fireproof.js +100 -0
  8. package/dist/listener.js +110 -0
  9. package/dist/prolly.js +316 -0
  10. package/dist/sha1.js +74 -0
  11. package/dist/src/blockstore.js +242 -0
  12. package/dist/src/clock.js +355 -0
  13. package/dist/src/crypto.js +59 -0
  14. package/dist/src/database.js +312 -0
  15. package/dist/src/db-index.js +314 -0
  16. package/dist/src/fireproof.d.ts +319 -0
  17. package/dist/src/fireproof.js +38976 -0
  18. package/dist/src/fireproof.js.map +1 -0
  19. package/dist/src/fireproof.mjs +38972 -0
  20. package/dist/src/fireproof.mjs.map +1 -0
  21. package/dist/src/index.d.ts +1 -1
  22. package/dist/src/index.js +19 -14
  23. package/dist/src/index.js.map +1 -1
  24. package/dist/src/index.mjs +19 -14
  25. package/dist/src/index.mjs.map +1 -1
  26. package/dist/src/listener.js +108 -0
  27. package/dist/src/prolly.js +319 -0
  28. package/dist/src/sha1.js +74 -0
  29. package/dist/src/utils.js +16 -0
  30. package/dist/src/valet.js +262 -0
  31. package/dist/test/block.js +57 -0
  32. package/dist/test/clock.test.js +556 -0
  33. package/dist/test/db-index.test.js +231 -0
  34. package/dist/test/fireproof.test.js +444 -0
  35. package/dist/test/fulltext.test.js +61 -0
  36. package/dist/test/helpers.js +39 -0
  37. package/dist/test/hydrator.test.js +142 -0
  38. package/dist/test/listener.test.js +103 -0
  39. package/dist/test/prolly.test.js +162 -0
  40. package/dist/test/proofs.test.js +45 -0
  41. package/dist/test/reproduce-fixture-bug.test.js +57 -0
  42. package/dist/test/valet.test.js +56 -0
  43. package/dist/utils.js +16 -0
  44. package/dist/valet.js +262 -0
  45. package/hooks/use-fireproof.js +38 -63
  46. package/package.json +13 -14
  47. package/src/blockstore.js +8 -4
  48. package/src/database.js +338 -0
  49. package/src/db-index.js +3 -3
  50. package/src/fireproof.js +65 -322
  51. package/src/listener.js +10 -8
  52. package/src/prolly.js +10 -6
  53. package/src/utils.js +16 -0
  54. package/src/hydrator.js +0 -54
  55. package/src/index.js +0 -6
@@ -0,0 +1,308 @@
1
+ // @ts-nocheck
2
+ import { visMerkleClock, visMerkleTree, vis, put, get, getAll, eventsSince } from './prolly.js';
3
+ import { doTransaction } from './blockstore.js';
4
+ import charwise from 'charwise';
5
+ import { localSet } from './utils.js';
6
+ // TypeScript Types
7
+ // eslint-disable-next-line no-unused-vars
8
+ // import { CID } from 'multiformats/dist/types/src/cid.js'
9
+ // eslint-disable-next-line no-unused-vars
10
+ class Proof {
11
+ }
12
+ /**
13
+ * @class Fireproof
14
+ * @classdesc Fireproof stores data in IndexedDB and provides a Merkle clock.
15
+ * This is the main class for saving and loading JSON and other documents with the database. You can find additional examples and
16
+ * usage guides in the repository README.
17
+ *
18
+ * @param {import('./blockstore.js').TransactionBlockstore} blocks - The block storage instance to use documents and indexes
19
+ * @param {CID[]} clock - The Merkle clock head to use for the Fireproof instance.
20
+ * @param {object} [config] - Optional configuration options for the Fireproof instance.
21
+ * @param {object} [authCtx] - Optional authorization context object to use for any authentication checks.
22
+ *
23
+ */
24
+ export class Database {
25
+ listeners = new Set();
26
+ // todo refactor this for the next version
27
+ constructor(blocks, clock, config = {}) {
28
+ this.name = config.name;
29
+ this.instanceId = `fp.${this.name}.${Math.random().toString(36).substring(2, 7)}`;
30
+ this.blocks = blocks;
31
+ this.clock = clock;
32
+ this.config = config;
33
+ this.indexes = new Map();
34
+ }
35
+ /**
36
+ * Renders the Fireproof instance as a JSON object.
37
+ * @returns {Object} - The JSON representation of the Fireproof instance. Includes clock heads for the database and its indexes.
38
+ * @memberof Fireproof
39
+ * @instance
40
+ */
41
+ toJSON() {
42
+ // todo this also needs to return the index roots...
43
+ return {
44
+ clock: this.clockToJSON(),
45
+ name: this.name,
46
+ key: this.blocks.valet?.getKeyMaterial(),
47
+ indexes: [...this.indexes.values()].map(index => index.toJSON())
48
+ };
49
+ }
50
+ /**
51
+ * Returns the Merkle clock heads for the Fireproof instance.
52
+ * @returns {string[]} - The Merkle clock heads for the Fireproof instance.
53
+ * @memberof Fireproof
54
+ * @instance
55
+ */
56
+ clockToJSON() {
57
+ return this.clock.map(cid => cid.toString());
58
+ }
59
+ hydrate({ clock, name, key }) {
60
+ this.name = name;
61
+ this.clock = clock;
62
+ this.blocks.valet?.setKeyMaterial(key);
63
+ this.indexBlocks = null;
64
+ }
65
+ maybeSaveClock() {
66
+ if (this.name && this.blocks.valet) {
67
+ localSet('fp.' + this.name, JSON.stringify(this));
68
+ }
69
+ }
70
+ /**
71
+ * Triggers a notification to all listeners
72
+ * of the Fireproof instance so they can repaint UI, etc.
73
+ * @returns {Promise<void>}
74
+ * @memberof Fireproof
75
+ * @instance
76
+ */
77
+ async notifyReset() {
78
+ await this.notifyListeners({ _reset: true, _clock: this.clockToJSON() });
79
+ }
80
+ // used be indexes etc to notify database listeners of new availability
81
+ async notifyExternal(source = 'unknown') {
82
+ await this.notifyListeners({ _external: source, _clock: this.clockToJSON() });
83
+ }
84
+ /**
85
+ * Returns the changes made to the Fireproof instance since the specified event.
86
+ * @function changesSince
87
+ * @param {CID[]} [event] - The clock head to retrieve changes since. If null or undefined, retrieves all changes.
88
+ * @returns {Promise<{rows : Object[], clock: CID[], proof: {}}>} An object containing the rows and the head of the instance's clock.
89
+ * @memberof Fireproof
90
+ * @instance
91
+ */
92
+ async changesSince(event) {
93
+ // console.log('changesSince', this.instanceId, event, this.clock)
94
+ let rows, dataCIDs, clockCIDs;
95
+ // if (!event) event = []
96
+ if (event) {
97
+ const resp = await eventsSince(this.blocks, this.clock, event);
98
+ const docsMap = new Map();
99
+ for (const { key, type, value } of resp.result.map(decodeEvent)) {
100
+ if (type === 'del') {
101
+ docsMap.set(key, { key, del: true });
102
+ }
103
+ else {
104
+ docsMap.set(key, { key, value });
105
+ }
106
+ }
107
+ rows = Array.from(docsMap.values());
108
+ clockCIDs = resp.clockCIDs;
109
+ // console.log('change rows', this.instanceId, rows)
110
+ }
111
+ else {
112
+ const allResp = await getAll(this.blocks, this.clock);
113
+ rows = allResp.result.map(({ key, value }) => (decodeEvent({ key, value })));
114
+ dataCIDs = allResp.cids;
115
+ // console.log('dbdoc rows', this.instanceId, rows)
116
+ }
117
+ return {
118
+ rows,
119
+ clock: this.clockToJSON(),
120
+ proof: { data: await cidsToProof(dataCIDs), clock: await cidsToProof(clockCIDs) }
121
+ };
122
+ }
123
+ async allDocuments() {
124
+ const allResp = await getAll(this.blocks, this.clock);
125
+ const rows = allResp.result.map(({ key, value }) => (decodeEvent({ key, value }))).map(({ key, value }) => ({ key, value: { _id: key, ...value } }));
126
+ return {
127
+ rows,
128
+ clock: this.clockToJSON(),
129
+ proof: await cidsToProof(allResp.cids)
130
+ };
131
+ }
132
+ /**
133
+ * Runs validation on the specified document using the Fireproof instance's configuration. Throws an error if the document is invalid.
134
+ *
135
+ * @param {Object} doc - The document to validate.
136
+ * @returns {Promise<void>}
137
+ * @throws {Error} - Throws an error if the document is invalid.
138
+ * @memberof Fireproof
139
+ * @instance
140
+ */
141
+ async runValidation(doc) {
142
+ if (this.config && this.config.validateChange) {
143
+ const oldDoc = await this.get(doc._id)
144
+ .then((doc) => doc)
145
+ .catch(() => ({}));
146
+ this.config.validateChange(doc, oldDoc, this.authCtx);
147
+ }
148
+ }
149
+ /**
150
+ * Retrieves the document with the specified ID from the database
151
+ *
152
+ * @param {string} key - the ID of the document to retrieve
153
+ * @param {Object} [opts] - options
154
+ * @returns {Promise<{_id: string}>} - the document with the specified ID
155
+ * @memberof Fireproof
156
+ * @instance
157
+ */
158
+ async get(key, opts = {}) {
159
+ const clock = opts.clock || this.clock;
160
+ const resp = await get(this.blocks, clock, charwise.encode(key));
161
+ // this tombstone is temporary until we can get the prolly tree to delete
162
+ if (!resp || resp.result === null) {
163
+ throw new Error('Not found');
164
+ }
165
+ const doc = resp.result;
166
+ if (opts.mvcc === true) {
167
+ doc._clock = this.clockToJSON();
168
+ }
169
+ doc._proof = {
170
+ data: await cidsToProof(resp.cids),
171
+ clock: this.clockToJSON()
172
+ };
173
+ doc._id = key;
174
+ return doc;
175
+ }
176
+ /**
177
+ * Adds a new document to the database, or updates an existing document. Returns the ID of the document and the new clock head.
178
+ *
179
+ * @param {Object} doc - the document to be added
180
+ * @param {string} doc._id - the document ID. If not provided, a random ID will be generated.
181
+ * @param {CID[]} doc._clock - the document ID. If not provided, a random ID will be generated.
182
+ * @param {Proof} doc._proof - CIDs referenced by the update
183
+ * @returns {Promise<{ id: string, clock: CID[] }>} - The result of adding the document to the database
184
+ * @memberof Fireproof
185
+ * @instance
186
+ */
187
+ async put({ _id, _proof, ...doc }) {
188
+ const id = _id || 'f' + Math.random().toString(36).slice(2);
189
+ await this.runValidation({ _id: id, ...doc });
190
+ return await this.putToProllyTree({ key: id, value: doc }, doc._clock);
191
+ }
192
+ /**
193
+ * Deletes a document from the database
194
+ * @param {string | any} docOrId - the document ID
195
+ * @returns {Promise<{ id: string, clock: CID[] }>} - The result of deleting the document from the database
196
+ * @memberof Fireproof
197
+ * @instance
198
+ */
199
+ async del(docOrId) {
200
+ let id;
201
+ let clock = null;
202
+ if (docOrId._id) {
203
+ id = docOrId._id;
204
+ clock = docOrId._clock;
205
+ }
206
+ else {
207
+ id = docOrId;
208
+ }
209
+ await this.runValidation({ _id: id, _deleted: true });
210
+ return await this.putToProllyTree({ key: id, del: true }, clock); // not working at prolly tree layer?
211
+ // this tombstone is temporary until we can get the prolly tree to delete
212
+ // return await this.putToProllyTree({ key: id, value: null }, clock)
213
+ }
214
+ /**
215
+ * Updates the underlying storage with the specified event.
216
+ * @private
217
+ * @param {{del?: true, key : string, value?: any}} decodedEvent - the event to add
218
+ * @returns {Promise<{ proof:{}, id: string, clock: CID[] }>} - The result of adding the event to storage
219
+ */
220
+ async putToProllyTree(decodedEvent, clock = null) {
221
+ const event = encodeEvent(decodedEvent);
222
+ if (clock && JSON.stringify(clock) !== JSON.stringify(this.clockToJSON())) {
223
+ // we need to check and see what version of the document exists at the clock specified
224
+ // if it is the same as the one we are trying to put, then we can proceed
225
+ const resp = await eventsSince(this.blocks, this.clock, event.value._clock);
226
+ const missedChange = resp.result.find(({ key }) => key === event.key);
227
+ if (missedChange) {
228
+ throw new Error('MVCC conflict, document is changed, please reload the document and try again.');
229
+ }
230
+ }
231
+ const result = await doTransaction('putToProllyTree', this.blocks, async (blocks) => await put(blocks, this.clock, event));
232
+ if (!result) {
233
+ console.error('failed', event);
234
+ throw new Error('failed to put at storage layer');
235
+ }
236
+ // console.log('new clock head', this.instanceId, result.head.toString())
237
+ this.clock = result.head; // do we want to do this as a finally block
238
+ await this.notifyListeners([decodedEvent]); // this type is odd
239
+ return {
240
+ id: decodedEvent.key,
241
+ clock: this.clockToJSON(),
242
+ proof: { data: await cidsToProof(result.cids), clock: await cidsToProof(result.clockCIDs) }
243
+ };
244
+ // todo should include additions (or split clock)
245
+ }
246
+ // /**
247
+ // * Advances the clock to the specified event and updates the root CID
248
+ // * Will be used by replication
249
+ // */
250
+ // async advance (event) {
251
+ // this.clock = await advance(this.blocks, this.clock, event)
252
+ // this.rootCid = await root(this.blocks, this.clock)
253
+ // return this.clock
254
+ // }
255
+ async *vis() {
256
+ return yield* vis(this.blocks, this.clock);
257
+ }
258
+ async visTree() {
259
+ return await visMerkleTree(this.blocks, this.clock);
260
+ }
261
+ async visClock() {
262
+ return await visMerkleClock(this.blocks, this.clock);
263
+ }
264
+ /**
265
+ * Registers a Listener to be called when the Fireproof instance's clock is updated.
266
+ * Recieves live changes from the database after they are committed.
267
+ * @param {Function} listener - The listener to be called when the clock is updated.
268
+ * @returns {Function} - A function that can be called to unregister the listener.
269
+ * @memberof Fireproof
270
+ */
271
+ registerListener(listener) {
272
+ this.listeners.add(listener);
273
+ return () => {
274
+ this.listeners.delete(listener);
275
+ };
276
+ }
277
+ async notifyListeners(changes) {
278
+ // await sleep(10)
279
+ await this.maybeSaveClock();
280
+ for (const listener of this.listeners) {
281
+ await listener(changes);
282
+ }
283
+ }
284
+ setCarUploader(carUploaderFn) {
285
+ // console.log('registering car uploader')
286
+ // https://en.wikipedia.org/wiki/Law_of_Demeter - this is a violation of the law of demeter
287
+ this.blocks.valet.uploadFunction = carUploaderFn;
288
+ }
289
+ setRemoteBlockReader(remoteBlockReaderFn) {
290
+ this.blocks.remoteBlockFunction = remoteBlockReaderFn;
291
+ }
292
+ }
293
+ export async function cidsToProof(cids) {
294
+ if (!cids || !cids.all)
295
+ return [];
296
+ const all = await cids.all();
297
+ return [...all].map((cid) => cid.toString());
298
+ }
299
+ function decodeEvent(event) {
300
+ const decodedKey = charwise.decode(event.key);
301
+ return { ...event, key: decodedKey };
302
+ }
303
+ function encodeEvent(event) {
304
+ if (!(event && event.key))
305
+ return;
306
+ const encodedKey = charwise.encode(event.key);
307
+ return { ...event, key: encodedKey };
308
+ }
@@ -0,0 +1,314 @@
1
+ // @ts-ignore
2
+ import { create, load } from 'prolly-trees/db-index';
3
+ // import { create, load } from '../../../../prolly-trees/src/db-index.js'
4
+ import { sha256 as hasher } from 'multiformats/hashes/sha2';
5
+ // @ts-ignore
6
+ import { nocache as cache } from 'prolly-trees/cache';
7
+ // @ts-ignore
8
+ import { bf, simpleCompare } from 'prolly-trees/utils';
9
+ import { makeGetBlock } from './prolly.js';
10
+ // eslint-disable-next-line no-unused-vars
11
+ import { Database, cidsToProof } from './database.js';
12
+ import * as codec from '@ipld/dag-cbor';
13
+ // import { create as createBlock } from 'multiformats/block'
14
+ import { TransactionBlockstore, doTransaction } from './blockstore.js';
15
+ // @ts-ignore
16
+ import charwise from 'charwise';
17
+ const ALWAYS_REBUILD = false; // todo: make false
18
+ const compare = (a, b) => {
19
+ const [aKey, aRef] = a;
20
+ const [bKey, bRef] = b;
21
+ const comp = simpleCompare(aKey, bKey);
22
+ if (comp !== 0)
23
+ return comp;
24
+ return refCompare(aRef, bRef);
25
+ };
26
+ const refCompare = (aRef, bRef) => {
27
+ if (Number.isNaN(aRef))
28
+ return -1;
29
+ if (Number.isNaN(bRef))
30
+ throw new Error('ref may not be Infinity or NaN');
31
+ if (aRef === Infinity)
32
+ return 1; // need to test this on equal docids!
33
+ // if (!Number.isFinite(bRef)) throw new Error('ref may not be Infinity or NaN')
34
+ return simpleCompare(aRef, bRef);
35
+ };
36
+ const dbIndexOpts = { cache, chunker: bf(3), codec, hasher, compare };
37
+ const idIndexOpts = { cache, chunker: bf(3), codec, hasher, compare: simpleCompare };
38
+ const makeDoc = ({ key, value }) => ({ _id: key, ...value });
39
+ /**
40
+ * JDoc for the result row type.
41
+ * @typedef {Object} ChangeEvent
42
+ * @property {string} key - The key of the document.
43
+ * @property {Object} value - The new value of the document.
44
+ * @property {boolean} [del] - Is the row deleted?
45
+ * @memberof DbIndex
46
+ */
47
+ /**
48
+ * JDoc for the result row type.
49
+ * @typedef {Object} DbIndexEntry
50
+ * @property {string[]} key - The key for the DbIndex entry.
51
+ * @property {Object} value - The value of the document.
52
+ * @property {boolean} [del] - Is the row deleted?
53
+ * @memberof DbIndex
54
+ */
55
+ /**
56
+ * Transforms a set of changes to DbIndex entries using a map function.
57
+ *
58
+ * @param {ChangeEvent[]} changes
59
+ * @param {Function} mapFn
60
+ * @returns {DbIndexEntry[]} The DbIndex entries generated by the map function.
61
+ * @private
62
+ * @memberof DbIndex
63
+ */
64
+ const indexEntriesForChanges = (changes, mapFn) => {
65
+ const indexEntries = [];
66
+ changes.forEach(({ key, value, del }) => {
67
+ if (del || !value)
68
+ return;
69
+ mapFn(makeDoc({ key, value }), (k, v) => {
70
+ if (typeof v === 'undefined' || typeof k === 'undefined')
71
+ return;
72
+ indexEntries.push({
73
+ key: [charwise.encode(k), key],
74
+ value: v
75
+ });
76
+ });
77
+ });
78
+ return indexEntries;
79
+ };
80
+ /**
81
+ * Represents an DbIndex for a Fireproof database.
82
+ *
83
+ * @class DbIndex
84
+ * @classdesc An DbIndex can be used to order and filter the documents in a Fireproof database.
85
+ *
86
+ * @param {Database} database - The Fireproof database instance to DbIndex.
87
+ * @param {Function} mapFn - The map function to apply to each entry in the database.
88
+ *
89
+ */
90
+ export class DbIndex {
91
+ constructor(database, mapFn, clock, opts = {}) {
92
+ this.database = database;
93
+ if (!database.indexBlocks) {
94
+ database.indexBlocks = new TransactionBlockstore(database?.name + '.indexes', database.blocks.valet?.getKeyMaterial());
95
+ }
96
+ /**
97
+ * The map function to apply to each entry in the database.
98
+ * @type {Function}
99
+ */
100
+ if (typeof mapFn === 'string') {
101
+ this.mapFnString = mapFn;
102
+ }
103
+ else {
104
+ this.mapFn = mapFn;
105
+ this.mapFnString = mapFn.toString();
106
+ }
107
+ this.name = opts.name || this.makeName();
108
+ this.indexById = { root: null, cid: null };
109
+ this.indexByKey = { root: null, cid: null };
110
+ this.dbHead = null;
111
+ if (clock) {
112
+ this.indexById.cid = clock.byId;
113
+ this.indexByKey.cid = clock.byKey;
114
+ this.dbHead = clock.db;
115
+ }
116
+ this.instanceId = this.database.instanceId + `.DbIndex.${Math.random().toString(36).substring(2, 7)}`;
117
+ this.updateIndexPromise = null;
118
+ if (!opts.temporary) {
119
+ DbIndex.registerWithDatabase(this, this.database);
120
+ }
121
+ }
122
+ makeName() {
123
+ const regex = /\(([^,()]+,\s*[^,()]+|\[[^\]]+\],\s*[^,()]+)\)/g;
124
+ const matches = Array.from(this.mapFnString.matchAll(regex), match => match[1].trim());
125
+ return matches[1];
126
+ }
127
+ static registerWithDatabase(inIndex, database) {
128
+ if (!database.indexes.has(inIndex.mapFnString)) {
129
+ database.indexes.set(inIndex.mapFnString, inIndex);
130
+ }
131
+ else {
132
+ // merge our inIndex code with the inIndex clock or vice versa
133
+ const existingIndex = database.indexes.get(inIndex.mapFnString);
134
+ // keep the code instance, discard the clock instance
135
+ if (existingIndex.mapFn) { // this one also has other config
136
+ existingIndex.dbHead = inIndex.dbHead;
137
+ existingIndex.indexById.cid = inIndex.indexById.cid;
138
+ existingIndex.indexByKey.cid = inIndex.indexByKey.cid;
139
+ }
140
+ else {
141
+ inIndex.dbHead = existingIndex.dbHead;
142
+ inIndex.indexById.cid = existingIndex.indexById.cid;
143
+ inIndex.indexByKey.cid = existingIndex.indexByKey.cid;
144
+ database.indexes.set(inIndex.mapFnString, inIndex);
145
+ }
146
+ }
147
+ }
148
+ toJSON() {
149
+ const indexJson = { name: this.name, code: this.mapFnString, clock: { db: null, byId: null, byKey: null } };
150
+ indexJson.clock.db = this.dbHead?.map(cid => cid.toString());
151
+ indexJson.clock.byId = this.indexById.cid?.toString();
152
+ indexJson.clock.byKey = this.indexByKey.cid?.toString();
153
+ return indexJson;
154
+ }
155
+ static fromJSON(database, { code, clock, name }) {
156
+ // console.log('DbIndex.fromJSON', database.constructor.name, code, clock)
157
+ return new DbIndex(database, code, clock, { name });
158
+ }
159
+ /**
160
+ * JSDoc for Query type.
161
+ * @typedef {Object} DbQuery
162
+ * @property {string[]} [range] - The range to query.
163
+ * @memberof DbIndex
164
+ */
165
+ /**
166
+ * Query object can have {range}
167
+ * @param {DbQuery} query - the query range to use
168
+ * @returns {Promise<{proof: {}, rows: Array<{id: string, key: string, value: any}>}>}
169
+ * @memberof DbIndex
170
+ * @instance
171
+ */
172
+ async query(query, update = true) {
173
+ // const callId = Math.random().toString(36).substring(2, 7)
174
+ // todo pass a root to query a snapshot
175
+ // console.time(callId + '.updateIndex')
176
+ update && await this.updateIndex(this.database.indexBlocks);
177
+ // console.timeEnd(callId + '.updateIndex')
178
+ // console.time(callId + '.doIndexQuery')
179
+ // console.log('query', query)
180
+ const response = await doIndexQuery(this.database.indexBlocks, this.indexByKey, query);
181
+ // console.timeEnd(callId + '.doIndexQuery')
182
+ return {
183
+ proof: { index: await cidsToProof(response.cids) },
184
+ rows: response.result.map(({ id, key, row }) => {
185
+ return ({ id, key: charwise.decode(key), value: row });
186
+ })
187
+ };
188
+ }
189
+ /**
190
+ * Update the DbIndex with the latest changes
191
+ * @private
192
+ * @returns {Promise<void>}
193
+ */
194
+ async updateIndex(blocks) {
195
+ // todo this could enqueue the request and give fresh ones to all second comers -- right now it gives out stale promises while working
196
+ // what would it do in a world where all indexes provide a database snapshot to query?
197
+ if (this.updateIndexPromise)
198
+ return this.updateIndexPromise;
199
+ this.updateIndexPromise = this.innerUpdateIndex(blocks);
200
+ this.updateIndexPromise.finally(() => { this.updateIndexPromise = null; });
201
+ return this.updateIndexPromise;
202
+ }
203
+ async innerUpdateIndex(inBlocks) {
204
+ // const callTag = Math.random().toString(36).substring(4)
205
+ // console.log(`updateIndex ${callTag} >`, this.instanceId, this.dbHead?.toString(), this.indexByKey.cid?.toString(), this.indexById.cid?.toString())
206
+ // todo remove this hack
207
+ if (ALWAYS_REBUILD) {
208
+ this.indexById = { root: null, cid: null };
209
+ this.indexByKey = { root: null, cid: null };
210
+ this.dbHead = null;
211
+ }
212
+ // console.log('dbHead', this.dbHead)
213
+ // console.time(callTag + '.changesSince')
214
+ const result = await this.database.changesSince(this.dbHead); // {key, value, del}
215
+ // console.timeEnd(callTag + '.changesSince')
216
+ // console.log('result.rows.length', result.rows.length)
217
+ // console.time(callTag + '.doTransactionupdateIndex')
218
+ // console.log('updateIndex changes length', result.rows.length)
219
+ if (result.rows.length === 0) {
220
+ // console.log('updateIndex < no changes', result.clock)
221
+ this.dbHead = result.clock;
222
+ return;
223
+ }
224
+ await doTransaction('updateIndex', inBlocks, async (blocks) => {
225
+ let oldIndexEntries = [];
226
+ let removeByIdIndexEntries = [];
227
+ await loadIndex(blocks, this.indexById, idIndexOpts);
228
+ await loadIndex(blocks, this.indexByKey, dbIndexOpts);
229
+ if (this.dbHead) {
230
+ const oldChangeEntries = await this.indexById.root.getMany(result.rows.map(({ key }) => key));
231
+ oldIndexEntries = oldChangeEntries.result.map((key) => ({ key, del: true }));
232
+ removeByIdIndexEntries = oldIndexEntries.map(({ key }) => ({ key: key[1], del: true }));
233
+ }
234
+ if (!this.mapFn) {
235
+ throw new Error('No live map function installed for index, cannot update. Make sure your index definition runs before any queries.' + (this.mapFnString ? ' Your code should match the stored map function source:\n' + this.mapFnString : ''));
236
+ }
237
+ const indexEntries = indexEntriesForChanges(result.rows, this.mapFn);
238
+ const byIdIndexEntries = indexEntries.map(({ key }) => ({ key: key[1], value: key }));
239
+ this.indexById = await bulkIndex(blocks, this.indexById, removeByIdIndexEntries.concat(byIdIndexEntries), idIndexOpts);
240
+ this.indexByKey = await bulkIndex(blocks, this.indexByKey, oldIndexEntries.concat(indexEntries), dbIndexOpts);
241
+ this.dbHead = result.clock;
242
+ });
243
+ this.database.notifyExternal('dbIndex');
244
+ // console.timeEnd(callTag + '.doTransactionupdateIndex')
245
+ // console.log(`updateIndex ${callTag} <`, this.instanceId, this.dbHead?.toString(), this.indexByKey.cid?.toString(), this.indexById.cid?.toString())
246
+ }
247
+ }
248
+ /**
249
+ * Update the DbIndex with the given entries
250
+ * @param {import('./blockstore.js').Blockstore} blocks
251
+ * @param {{root, cid}} inIndex
252
+ * @param {DbIndexEntry[]} indexEntries
253
+ * @private
254
+ */
255
+ async function bulkIndex(blocks, inIndex, indexEntries, opts) {
256
+ if (!indexEntries.length)
257
+ return inIndex;
258
+ const putBlock = blocks.put.bind(blocks);
259
+ const { getBlock } = makeGetBlock(blocks);
260
+ let returnRootBlock;
261
+ let returnNode;
262
+ if (!inIndex.root) {
263
+ const cid = inIndex.cid;
264
+ if (!cid) {
265
+ for await (const node of await create({ get: getBlock, list: indexEntries, ...opts })) {
266
+ const block = await node.block;
267
+ await putBlock(block.cid, block.bytes);
268
+ returnRootBlock = block;
269
+ returnNode = node;
270
+ }
271
+ return { root: returnNode, cid: returnRootBlock.cid };
272
+ }
273
+ inIndex.root = await load({ cid, get: getBlock, ...dbIndexOpts });
274
+ }
275
+ const { root, blocks: newBlocks } = await inIndex.root.bulk(indexEntries);
276
+ returnRootBlock = await root.block;
277
+ returnNode = root;
278
+ for await (const block of newBlocks) {
279
+ await putBlock(block.cid, block.bytes);
280
+ }
281
+ await putBlock(returnRootBlock.cid, returnRootBlock.bytes);
282
+ return { root: returnNode, cid: returnRootBlock.cid };
283
+ }
284
+ async function loadIndex(blocks, index, indexOpts) {
285
+ if (!index.root) {
286
+ const cid = index.cid;
287
+ if (!cid)
288
+ return;
289
+ const { getBlock } = makeGetBlock(blocks);
290
+ index.root = await load({ cid, get: getBlock, ...indexOpts });
291
+ }
292
+ return index.root;
293
+ }
294
+ async function applyLimit(results, limit) {
295
+ results.result = results.result.slice(0, limit);
296
+ return results;
297
+ }
298
+ async function doIndexQuery(blocks, indexByKey, query = {}) {
299
+ await loadIndex(blocks, indexByKey, dbIndexOpts);
300
+ if (!indexByKey.root)
301
+ return { result: [] };
302
+ if (query.range) {
303
+ const encodedRange = query.range.map((key) => charwise.encode(key));
304
+ return applyLimit(await indexByKey.root.range(...encodedRange), query.limit);
305
+ }
306
+ else if (query.key) {
307
+ const encodedKey = charwise.encode(query.key);
308
+ return indexByKey.root.get(encodedKey);
309
+ }
310
+ else {
311
+ const { result, ...all } = await indexByKey.root.getAllEntries();
312
+ return applyLimit({ result: result.map(({ key: [k, id], value }) => ({ key: k, id, row: value })), ...all }, query.limit);
313
+ }
314
+ }