@fireproof/core 0.3.21 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blockstore.js +242 -0
- package/dist/clock.js +355 -0
- package/dist/crypto.js +59 -0
- package/dist/database.js +308 -0
- package/dist/db-index.js +314 -0
- package/dist/fireproof.js +83 -0
- package/dist/hooks/use-fireproof.js +100 -0
- package/dist/listener.js +110 -0
- package/dist/prolly.js +316 -0
- package/dist/sha1.js +74 -0
- package/dist/src/blockstore.js +242 -0
- package/dist/src/clock.js +355 -0
- package/dist/src/crypto.js +59 -0
- package/dist/src/database.js +312 -0
- package/dist/src/db-index.js +314 -0
- package/dist/src/fireproof.d.ts +319 -0
- package/dist/src/fireproof.js +38976 -0
- package/dist/src/fireproof.js.map +1 -0
- package/dist/src/fireproof.mjs +38972 -0
- package/dist/src/fireproof.mjs.map +1 -0
- package/dist/src/index.d.ts +1 -1
- package/dist/src/index.js +21 -16
- package/dist/src/index.js.map +1 -1
- package/dist/src/index.mjs +21 -16
- package/dist/src/index.mjs.map +1 -1
- package/dist/src/listener.js +108 -0
- package/dist/src/prolly.js +319 -0
- package/dist/src/sha1.js +74 -0
- package/dist/src/utils.js +16 -0
- package/dist/src/valet.js +262 -0
- package/dist/test/block.js +57 -0
- package/dist/test/clock.test.js +556 -0
- package/dist/test/db-index.test.js +231 -0
- package/dist/test/fireproof.test.js +444 -0
- package/dist/test/fulltext.test.js +61 -0
- package/dist/test/helpers.js +39 -0
- package/dist/test/hydrator.test.js +142 -0
- package/dist/test/listener.test.js +103 -0
- package/dist/test/prolly.test.js +162 -0
- package/dist/test/proofs.test.js +45 -0
- package/dist/test/reproduce-fixture-bug.test.js +57 -0
- package/dist/test/valet.test.js +56 -0
- package/dist/utils.js +16 -0
- package/dist/valet.js +262 -0
- package/hooks/use-fireproof.js +38 -63
- package/package.json +13 -14
- package/src/blockstore.js +8 -4
- package/src/database.js +338 -0
- package/src/db-index.js +3 -3
- package/src/fireproof.js +65 -322
- package/src/listener.js +10 -8
- package/src/prolly.js +10 -6
- package/src/utils.js +16 -0
- package/src/valet.js +2 -2
- package/src/hydrator.js +0 -54
- package/src/index.js +0 -6
package/dist/database.js
ADDED
@@ -0,0 +1,308 @@
|
|
1
|
+
// @ts-nocheck
|
2
|
+
import { visMerkleClock, visMerkleTree, vis, put, get, getAll, eventsSince } from './prolly.js';
|
3
|
+
import { doTransaction } from './blockstore.js';
|
4
|
+
import charwise from 'charwise';
|
5
|
+
import { localSet } from './utils.js';
|
6
|
+
// TypeScript Types
|
7
|
+
// eslint-disable-next-line no-unused-vars
|
8
|
+
// import { CID } from 'multiformats/dist/types/src/cid.js'
|
9
|
+
// eslint-disable-next-line no-unused-vars
|
10
|
+
class Proof {
|
11
|
+
}
|
12
|
+
/**
|
13
|
+
* @class Fireproof
|
14
|
+
* @classdesc Fireproof stores data in IndexedDB and provides a Merkle clock.
|
15
|
+
* This is the main class for saving and loading JSON and other documents with the database. You can find additional examples and
|
16
|
+
* usage guides in the repository README.
|
17
|
+
*
|
18
|
+
* @param {import('./blockstore.js').TransactionBlockstore} blocks - The block storage instance to use documents and indexes
|
19
|
+
* @param {CID[]} clock - The Merkle clock head to use for the Fireproof instance.
|
20
|
+
* @param {object} [config] - Optional configuration options for the Fireproof instance.
|
21
|
+
* @param {object} [authCtx] - Optional authorization context object to use for any authentication checks.
|
22
|
+
*
|
23
|
+
*/
|
24
|
+
export class Database {
|
25
|
+
listeners = new Set();
|
26
|
+
// todo refactor this for the next version
|
27
|
+
constructor(blocks, clock, config = {}) {
|
28
|
+
this.name = config.name;
|
29
|
+
this.instanceId = `fp.${this.name}.${Math.random().toString(36).substring(2, 7)}`;
|
30
|
+
this.blocks = blocks;
|
31
|
+
this.clock = clock;
|
32
|
+
this.config = config;
|
33
|
+
this.indexes = new Map();
|
34
|
+
}
|
35
|
+
/**
|
36
|
+
* Renders the Fireproof instance as a JSON object.
|
37
|
+
* @returns {Object} - The JSON representation of the Fireproof instance. Includes clock heads for the database and its indexes.
|
38
|
+
* @memberof Fireproof
|
39
|
+
* @instance
|
40
|
+
*/
|
41
|
+
toJSON() {
|
42
|
+
// todo this also needs to return the index roots...
|
43
|
+
return {
|
44
|
+
clock: this.clockToJSON(),
|
45
|
+
name: this.name,
|
46
|
+
key: this.blocks.valet?.getKeyMaterial(),
|
47
|
+
indexes: [...this.indexes.values()].map(index => index.toJSON())
|
48
|
+
};
|
49
|
+
}
|
50
|
+
/**
|
51
|
+
* Returns the Merkle clock heads for the Fireproof instance.
|
52
|
+
* @returns {string[]} - The Merkle clock heads for the Fireproof instance.
|
53
|
+
* @memberof Fireproof
|
54
|
+
* @instance
|
55
|
+
*/
|
56
|
+
clockToJSON() {
|
57
|
+
return this.clock.map(cid => cid.toString());
|
58
|
+
}
|
59
|
+
hydrate({ clock, name, key }) {
|
60
|
+
this.name = name;
|
61
|
+
this.clock = clock;
|
62
|
+
this.blocks.valet?.setKeyMaterial(key);
|
63
|
+
this.indexBlocks = null;
|
64
|
+
}
|
65
|
+
maybeSaveClock() {
|
66
|
+
if (this.name && this.blocks.valet) {
|
67
|
+
localSet('fp.' + this.name, JSON.stringify(this));
|
68
|
+
}
|
69
|
+
}
|
70
|
+
/**
|
71
|
+
* Triggers a notification to all listeners
|
72
|
+
* of the Fireproof instance so they can repaint UI, etc.
|
73
|
+
* @returns {Promise<void>}
|
74
|
+
* @memberof Fireproof
|
75
|
+
* @instance
|
76
|
+
*/
|
77
|
+
async notifyReset() {
|
78
|
+
await this.notifyListeners({ _reset: true, _clock: this.clockToJSON() });
|
79
|
+
}
|
80
|
+
// used be indexes etc to notify database listeners of new availability
|
81
|
+
async notifyExternal(source = 'unknown') {
|
82
|
+
await this.notifyListeners({ _external: source, _clock: this.clockToJSON() });
|
83
|
+
}
|
84
|
+
/**
|
85
|
+
* Returns the changes made to the Fireproof instance since the specified event.
|
86
|
+
* @function changesSince
|
87
|
+
* @param {CID[]} [event] - The clock head to retrieve changes since. If null or undefined, retrieves all changes.
|
88
|
+
* @returns {Promise<{rows : Object[], clock: CID[], proof: {}}>} An object containing the rows and the head of the instance's clock.
|
89
|
+
* @memberof Fireproof
|
90
|
+
* @instance
|
91
|
+
*/
|
92
|
+
async changesSince(event) {
|
93
|
+
// console.log('changesSince', this.instanceId, event, this.clock)
|
94
|
+
let rows, dataCIDs, clockCIDs;
|
95
|
+
// if (!event) event = []
|
96
|
+
if (event) {
|
97
|
+
const resp = await eventsSince(this.blocks, this.clock, event);
|
98
|
+
const docsMap = new Map();
|
99
|
+
for (const { key, type, value } of resp.result.map(decodeEvent)) {
|
100
|
+
if (type === 'del') {
|
101
|
+
docsMap.set(key, { key, del: true });
|
102
|
+
}
|
103
|
+
else {
|
104
|
+
docsMap.set(key, { key, value });
|
105
|
+
}
|
106
|
+
}
|
107
|
+
rows = Array.from(docsMap.values());
|
108
|
+
clockCIDs = resp.clockCIDs;
|
109
|
+
// console.log('change rows', this.instanceId, rows)
|
110
|
+
}
|
111
|
+
else {
|
112
|
+
const allResp = await getAll(this.blocks, this.clock);
|
113
|
+
rows = allResp.result.map(({ key, value }) => (decodeEvent({ key, value })));
|
114
|
+
dataCIDs = allResp.cids;
|
115
|
+
// console.log('dbdoc rows', this.instanceId, rows)
|
116
|
+
}
|
117
|
+
return {
|
118
|
+
rows,
|
119
|
+
clock: this.clockToJSON(),
|
120
|
+
proof: { data: await cidsToProof(dataCIDs), clock: await cidsToProof(clockCIDs) }
|
121
|
+
};
|
122
|
+
}
|
123
|
+
async allDocuments() {
|
124
|
+
const allResp = await getAll(this.blocks, this.clock);
|
125
|
+
const rows = allResp.result.map(({ key, value }) => (decodeEvent({ key, value }))).map(({ key, value }) => ({ key, value: { _id: key, ...value } }));
|
126
|
+
return {
|
127
|
+
rows,
|
128
|
+
clock: this.clockToJSON(),
|
129
|
+
proof: await cidsToProof(allResp.cids)
|
130
|
+
};
|
131
|
+
}
|
132
|
+
/**
|
133
|
+
* Runs validation on the specified document using the Fireproof instance's configuration. Throws an error if the document is invalid.
|
134
|
+
*
|
135
|
+
* @param {Object} doc - The document to validate.
|
136
|
+
* @returns {Promise<void>}
|
137
|
+
* @throws {Error} - Throws an error if the document is invalid.
|
138
|
+
* @memberof Fireproof
|
139
|
+
* @instance
|
140
|
+
*/
|
141
|
+
async runValidation(doc) {
|
142
|
+
if (this.config && this.config.validateChange) {
|
143
|
+
const oldDoc = await this.get(doc._id)
|
144
|
+
.then((doc) => doc)
|
145
|
+
.catch(() => ({}));
|
146
|
+
this.config.validateChange(doc, oldDoc, this.authCtx);
|
147
|
+
}
|
148
|
+
}
|
149
|
+
/**
|
150
|
+
* Retrieves the document with the specified ID from the database
|
151
|
+
*
|
152
|
+
* @param {string} key - the ID of the document to retrieve
|
153
|
+
* @param {Object} [opts] - options
|
154
|
+
* @returns {Promise<{_id: string}>} - the document with the specified ID
|
155
|
+
* @memberof Fireproof
|
156
|
+
* @instance
|
157
|
+
*/
|
158
|
+
async get(key, opts = {}) {
|
159
|
+
const clock = opts.clock || this.clock;
|
160
|
+
const resp = await get(this.blocks, clock, charwise.encode(key));
|
161
|
+
// this tombstone is temporary until we can get the prolly tree to delete
|
162
|
+
if (!resp || resp.result === null) {
|
163
|
+
throw new Error('Not found');
|
164
|
+
}
|
165
|
+
const doc = resp.result;
|
166
|
+
if (opts.mvcc === true) {
|
167
|
+
doc._clock = this.clockToJSON();
|
168
|
+
}
|
169
|
+
doc._proof = {
|
170
|
+
data: await cidsToProof(resp.cids),
|
171
|
+
clock: this.clockToJSON()
|
172
|
+
};
|
173
|
+
doc._id = key;
|
174
|
+
return doc;
|
175
|
+
}
|
176
|
+
/**
|
177
|
+
* Adds a new document to the database, or updates an existing document. Returns the ID of the document and the new clock head.
|
178
|
+
*
|
179
|
+
* @param {Object} doc - the document to be added
|
180
|
+
* @param {string} doc._id - the document ID. If not provided, a random ID will be generated.
|
181
|
+
* @param {CID[]} doc._clock - the document ID. If not provided, a random ID will be generated.
|
182
|
+
* @param {Proof} doc._proof - CIDs referenced by the update
|
183
|
+
* @returns {Promise<{ id: string, clock: CID[] }>} - The result of adding the document to the database
|
184
|
+
* @memberof Fireproof
|
185
|
+
* @instance
|
186
|
+
*/
|
187
|
+
async put({ _id, _proof, ...doc }) {
|
188
|
+
const id = _id || 'f' + Math.random().toString(36).slice(2);
|
189
|
+
await this.runValidation({ _id: id, ...doc });
|
190
|
+
return await this.putToProllyTree({ key: id, value: doc }, doc._clock);
|
191
|
+
}
|
192
|
+
/**
|
193
|
+
* Deletes a document from the database
|
194
|
+
* @param {string | any} docOrId - the document ID
|
195
|
+
* @returns {Promise<{ id: string, clock: CID[] }>} - The result of deleting the document from the database
|
196
|
+
* @memberof Fireproof
|
197
|
+
* @instance
|
198
|
+
*/
|
199
|
+
async del(docOrId) {
|
200
|
+
let id;
|
201
|
+
let clock = null;
|
202
|
+
if (docOrId._id) {
|
203
|
+
id = docOrId._id;
|
204
|
+
clock = docOrId._clock;
|
205
|
+
}
|
206
|
+
else {
|
207
|
+
id = docOrId;
|
208
|
+
}
|
209
|
+
await this.runValidation({ _id: id, _deleted: true });
|
210
|
+
return await this.putToProllyTree({ key: id, del: true }, clock); // not working at prolly tree layer?
|
211
|
+
// this tombstone is temporary until we can get the prolly tree to delete
|
212
|
+
// return await this.putToProllyTree({ key: id, value: null }, clock)
|
213
|
+
}
|
214
|
+
/**
|
215
|
+
* Updates the underlying storage with the specified event.
|
216
|
+
* @private
|
217
|
+
* @param {{del?: true, key : string, value?: any}} decodedEvent - the event to add
|
218
|
+
* @returns {Promise<{ proof:{}, id: string, clock: CID[] }>} - The result of adding the event to storage
|
219
|
+
*/
|
220
|
+
async putToProllyTree(decodedEvent, clock = null) {
|
221
|
+
const event = encodeEvent(decodedEvent);
|
222
|
+
if (clock && JSON.stringify(clock) !== JSON.stringify(this.clockToJSON())) {
|
223
|
+
// we need to check and see what version of the document exists at the clock specified
|
224
|
+
// if it is the same as the one we are trying to put, then we can proceed
|
225
|
+
const resp = await eventsSince(this.blocks, this.clock, event.value._clock);
|
226
|
+
const missedChange = resp.result.find(({ key }) => key === event.key);
|
227
|
+
if (missedChange) {
|
228
|
+
throw new Error('MVCC conflict, document is changed, please reload the document and try again.');
|
229
|
+
}
|
230
|
+
}
|
231
|
+
const result = await doTransaction('putToProllyTree', this.blocks, async (blocks) => await put(blocks, this.clock, event));
|
232
|
+
if (!result) {
|
233
|
+
console.error('failed', event);
|
234
|
+
throw new Error('failed to put at storage layer');
|
235
|
+
}
|
236
|
+
// console.log('new clock head', this.instanceId, result.head.toString())
|
237
|
+
this.clock = result.head; // do we want to do this as a finally block
|
238
|
+
await this.notifyListeners([decodedEvent]); // this type is odd
|
239
|
+
return {
|
240
|
+
id: decodedEvent.key,
|
241
|
+
clock: this.clockToJSON(),
|
242
|
+
proof: { data: await cidsToProof(result.cids), clock: await cidsToProof(result.clockCIDs) }
|
243
|
+
};
|
244
|
+
// todo should include additions (or split clock)
|
245
|
+
}
|
246
|
+
// /**
|
247
|
+
// * Advances the clock to the specified event and updates the root CID
|
248
|
+
// * Will be used by replication
|
249
|
+
// */
|
250
|
+
// async advance (event) {
|
251
|
+
// this.clock = await advance(this.blocks, this.clock, event)
|
252
|
+
// this.rootCid = await root(this.blocks, this.clock)
|
253
|
+
// return this.clock
|
254
|
+
// }
|
255
|
+
async *vis() {
|
256
|
+
return yield* vis(this.blocks, this.clock);
|
257
|
+
}
|
258
|
+
async visTree() {
|
259
|
+
return await visMerkleTree(this.blocks, this.clock);
|
260
|
+
}
|
261
|
+
async visClock() {
|
262
|
+
return await visMerkleClock(this.blocks, this.clock);
|
263
|
+
}
|
264
|
+
/**
|
265
|
+
* Registers a Listener to be called when the Fireproof instance's clock is updated.
|
266
|
+
* Recieves live changes from the database after they are committed.
|
267
|
+
* @param {Function} listener - The listener to be called when the clock is updated.
|
268
|
+
* @returns {Function} - A function that can be called to unregister the listener.
|
269
|
+
* @memberof Fireproof
|
270
|
+
*/
|
271
|
+
registerListener(listener) {
|
272
|
+
this.listeners.add(listener);
|
273
|
+
return () => {
|
274
|
+
this.listeners.delete(listener);
|
275
|
+
};
|
276
|
+
}
|
277
|
+
async notifyListeners(changes) {
|
278
|
+
// await sleep(10)
|
279
|
+
await this.maybeSaveClock();
|
280
|
+
for (const listener of this.listeners) {
|
281
|
+
await listener(changes);
|
282
|
+
}
|
283
|
+
}
|
284
|
+
setCarUploader(carUploaderFn) {
|
285
|
+
// console.log('registering car uploader')
|
286
|
+
// https://en.wikipedia.org/wiki/Law_of_Demeter - this is a violation of the law of demeter
|
287
|
+
this.blocks.valet.uploadFunction = carUploaderFn;
|
288
|
+
}
|
289
|
+
setRemoteBlockReader(remoteBlockReaderFn) {
|
290
|
+
this.blocks.remoteBlockFunction = remoteBlockReaderFn;
|
291
|
+
}
|
292
|
+
}
|
293
|
+
export async function cidsToProof(cids) {
|
294
|
+
if (!cids || !cids.all)
|
295
|
+
return [];
|
296
|
+
const all = await cids.all();
|
297
|
+
return [...all].map((cid) => cid.toString());
|
298
|
+
}
|
299
|
+
function decodeEvent(event) {
|
300
|
+
const decodedKey = charwise.decode(event.key);
|
301
|
+
return { ...event, key: decodedKey };
|
302
|
+
}
|
303
|
+
function encodeEvent(event) {
|
304
|
+
if (!(event && event.key))
|
305
|
+
return;
|
306
|
+
const encodedKey = charwise.encode(event.key);
|
307
|
+
return { ...event, key: encodedKey };
|
308
|
+
}
|
package/dist/db-index.js
ADDED
@@ -0,0 +1,314 @@
|
|
1
|
+
// @ts-ignore
|
2
|
+
import { create, load } from 'prolly-trees/db-index';
|
3
|
+
// import { create, load } from '../../../../prolly-trees/src/db-index.js'
|
4
|
+
import { sha256 as hasher } from 'multiformats/hashes/sha2';
|
5
|
+
// @ts-ignore
|
6
|
+
import { nocache as cache } from 'prolly-trees/cache';
|
7
|
+
// @ts-ignore
|
8
|
+
import { bf, simpleCompare } from 'prolly-trees/utils';
|
9
|
+
import { makeGetBlock } from './prolly.js';
|
10
|
+
// eslint-disable-next-line no-unused-vars
|
11
|
+
import { Database, cidsToProof } from './database.js';
|
12
|
+
import * as codec from '@ipld/dag-cbor';
|
13
|
+
// import { create as createBlock } from 'multiformats/block'
|
14
|
+
import { TransactionBlockstore, doTransaction } from './blockstore.js';
|
15
|
+
// @ts-ignore
|
16
|
+
import charwise from 'charwise';
|
17
|
+
const ALWAYS_REBUILD = false; // todo: make false
|
18
|
+
const compare = (a, b) => {
|
19
|
+
const [aKey, aRef] = a;
|
20
|
+
const [bKey, bRef] = b;
|
21
|
+
const comp = simpleCompare(aKey, bKey);
|
22
|
+
if (comp !== 0)
|
23
|
+
return comp;
|
24
|
+
return refCompare(aRef, bRef);
|
25
|
+
};
|
26
|
+
const refCompare = (aRef, bRef) => {
|
27
|
+
if (Number.isNaN(aRef))
|
28
|
+
return -1;
|
29
|
+
if (Number.isNaN(bRef))
|
30
|
+
throw new Error('ref may not be Infinity or NaN');
|
31
|
+
if (aRef === Infinity)
|
32
|
+
return 1; // need to test this on equal docids!
|
33
|
+
// if (!Number.isFinite(bRef)) throw new Error('ref may not be Infinity or NaN')
|
34
|
+
return simpleCompare(aRef, bRef);
|
35
|
+
};
|
36
|
+
const dbIndexOpts = { cache, chunker: bf(3), codec, hasher, compare };
|
37
|
+
const idIndexOpts = { cache, chunker: bf(3), codec, hasher, compare: simpleCompare };
|
38
|
+
const makeDoc = ({ key, value }) => ({ _id: key, ...value });
|
39
|
+
/**
|
40
|
+
* JDoc for the result row type.
|
41
|
+
* @typedef {Object} ChangeEvent
|
42
|
+
* @property {string} key - The key of the document.
|
43
|
+
* @property {Object} value - The new value of the document.
|
44
|
+
* @property {boolean} [del] - Is the row deleted?
|
45
|
+
* @memberof DbIndex
|
46
|
+
*/
|
47
|
+
/**
|
48
|
+
* JDoc for the result row type.
|
49
|
+
* @typedef {Object} DbIndexEntry
|
50
|
+
* @property {string[]} key - The key for the DbIndex entry.
|
51
|
+
* @property {Object} value - The value of the document.
|
52
|
+
* @property {boolean} [del] - Is the row deleted?
|
53
|
+
* @memberof DbIndex
|
54
|
+
*/
|
55
|
+
/**
|
56
|
+
* Transforms a set of changes to DbIndex entries using a map function.
|
57
|
+
*
|
58
|
+
* @param {ChangeEvent[]} changes
|
59
|
+
* @param {Function} mapFn
|
60
|
+
* @returns {DbIndexEntry[]} The DbIndex entries generated by the map function.
|
61
|
+
* @private
|
62
|
+
* @memberof DbIndex
|
63
|
+
*/
|
64
|
+
const indexEntriesForChanges = (changes, mapFn) => {
|
65
|
+
const indexEntries = [];
|
66
|
+
changes.forEach(({ key, value, del }) => {
|
67
|
+
if (del || !value)
|
68
|
+
return;
|
69
|
+
mapFn(makeDoc({ key, value }), (k, v) => {
|
70
|
+
if (typeof v === 'undefined' || typeof k === 'undefined')
|
71
|
+
return;
|
72
|
+
indexEntries.push({
|
73
|
+
key: [charwise.encode(k), key],
|
74
|
+
value: v
|
75
|
+
});
|
76
|
+
});
|
77
|
+
});
|
78
|
+
return indexEntries;
|
79
|
+
};
|
80
|
+
/**
|
81
|
+
* Represents an DbIndex for a Fireproof database.
|
82
|
+
*
|
83
|
+
* @class DbIndex
|
84
|
+
* @classdesc An DbIndex can be used to order and filter the documents in a Fireproof database.
|
85
|
+
*
|
86
|
+
* @param {Database} database - The Fireproof database instance to DbIndex.
|
87
|
+
* @param {Function} mapFn - The map function to apply to each entry in the database.
|
88
|
+
*
|
89
|
+
*/
|
90
|
+
export class DbIndex {
|
91
|
+
constructor(database, mapFn, clock, opts = {}) {
|
92
|
+
this.database = database;
|
93
|
+
if (!database.indexBlocks) {
|
94
|
+
database.indexBlocks = new TransactionBlockstore(database?.name + '.indexes', database.blocks.valet?.getKeyMaterial());
|
95
|
+
}
|
96
|
+
/**
|
97
|
+
* The map function to apply to each entry in the database.
|
98
|
+
* @type {Function}
|
99
|
+
*/
|
100
|
+
if (typeof mapFn === 'string') {
|
101
|
+
this.mapFnString = mapFn;
|
102
|
+
}
|
103
|
+
else {
|
104
|
+
this.mapFn = mapFn;
|
105
|
+
this.mapFnString = mapFn.toString();
|
106
|
+
}
|
107
|
+
this.name = opts.name || this.makeName();
|
108
|
+
this.indexById = { root: null, cid: null };
|
109
|
+
this.indexByKey = { root: null, cid: null };
|
110
|
+
this.dbHead = null;
|
111
|
+
if (clock) {
|
112
|
+
this.indexById.cid = clock.byId;
|
113
|
+
this.indexByKey.cid = clock.byKey;
|
114
|
+
this.dbHead = clock.db;
|
115
|
+
}
|
116
|
+
this.instanceId = this.database.instanceId + `.DbIndex.${Math.random().toString(36).substring(2, 7)}`;
|
117
|
+
this.updateIndexPromise = null;
|
118
|
+
if (!opts.temporary) {
|
119
|
+
DbIndex.registerWithDatabase(this, this.database);
|
120
|
+
}
|
121
|
+
}
|
122
|
+
makeName() {
|
123
|
+
const regex = /\(([^,()]+,\s*[^,()]+|\[[^\]]+\],\s*[^,()]+)\)/g;
|
124
|
+
const matches = Array.from(this.mapFnString.matchAll(regex), match => match[1].trim());
|
125
|
+
return matches[1];
|
126
|
+
}
|
127
|
+
static registerWithDatabase(inIndex, database) {
|
128
|
+
if (!database.indexes.has(inIndex.mapFnString)) {
|
129
|
+
database.indexes.set(inIndex.mapFnString, inIndex);
|
130
|
+
}
|
131
|
+
else {
|
132
|
+
// merge our inIndex code with the inIndex clock or vice versa
|
133
|
+
const existingIndex = database.indexes.get(inIndex.mapFnString);
|
134
|
+
// keep the code instance, discard the clock instance
|
135
|
+
if (existingIndex.mapFn) { // this one also has other config
|
136
|
+
existingIndex.dbHead = inIndex.dbHead;
|
137
|
+
existingIndex.indexById.cid = inIndex.indexById.cid;
|
138
|
+
existingIndex.indexByKey.cid = inIndex.indexByKey.cid;
|
139
|
+
}
|
140
|
+
else {
|
141
|
+
inIndex.dbHead = existingIndex.dbHead;
|
142
|
+
inIndex.indexById.cid = existingIndex.indexById.cid;
|
143
|
+
inIndex.indexByKey.cid = existingIndex.indexByKey.cid;
|
144
|
+
database.indexes.set(inIndex.mapFnString, inIndex);
|
145
|
+
}
|
146
|
+
}
|
147
|
+
}
|
148
|
+
toJSON() {
|
149
|
+
const indexJson = { name: this.name, code: this.mapFnString, clock: { db: null, byId: null, byKey: null } };
|
150
|
+
indexJson.clock.db = this.dbHead?.map(cid => cid.toString());
|
151
|
+
indexJson.clock.byId = this.indexById.cid?.toString();
|
152
|
+
indexJson.clock.byKey = this.indexByKey.cid?.toString();
|
153
|
+
return indexJson;
|
154
|
+
}
|
155
|
+
static fromJSON(database, { code, clock, name }) {
|
156
|
+
// console.log('DbIndex.fromJSON', database.constructor.name, code, clock)
|
157
|
+
return new DbIndex(database, code, clock, { name });
|
158
|
+
}
|
159
|
+
/**
|
160
|
+
* JSDoc for Query type.
|
161
|
+
* @typedef {Object} DbQuery
|
162
|
+
* @property {string[]} [range] - The range to query.
|
163
|
+
* @memberof DbIndex
|
164
|
+
*/
|
165
|
+
/**
|
166
|
+
* Query object can have {range}
|
167
|
+
* @param {DbQuery} query - the query range to use
|
168
|
+
* @returns {Promise<{proof: {}, rows: Array<{id: string, key: string, value: any}>}>}
|
169
|
+
* @memberof DbIndex
|
170
|
+
* @instance
|
171
|
+
*/
|
172
|
+
async query(query, update = true) {
|
173
|
+
// const callId = Math.random().toString(36).substring(2, 7)
|
174
|
+
// todo pass a root to query a snapshot
|
175
|
+
// console.time(callId + '.updateIndex')
|
176
|
+
update && await this.updateIndex(this.database.indexBlocks);
|
177
|
+
// console.timeEnd(callId + '.updateIndex')
|
178
|
+
// console.time(callId + '.doIndexQuery')
|
179
|
+
// console.log('query', query)
|
180
|
+
const response = await doIndexQuery(this.database.indexBlocks, this.indexByKey, query);
|
181
|
+
// console.timeEnd(callId + '.doIndexQuery')
|
182
|
+
return {
|
183
|
+
proof: { index: await cidsToProof(response.cids) },
|
184
|
+
rows: response.result.map(({ id, key, row }) => {
|
185
|
+
return ({ id, key: charwise.decode(key), value: row });
|
186
|
+
})
|
187
|
+
};
|
188
|
+
}
|
189
|
+
/**
|
190
|
+
* Update the DbIndex with the latest changes
|
191
|
+
* @private
|
192
|
+
* @returns {Promise<void>}
|
193
|
+
*/
|
194
|
+
async updateIndex(blocks) {
|
195
|
+
// todo this could enqueue the request and give fresh ones to all second comers -- right now it gives out stale promises while working
|
196
|
+
// what would it do in a world where all indexes provide a database snapshot to query?
|
197
|
+
if (this.updateIndexPromise)
|
198
|
+
return this.updateIndexPromise;
|
199
|
+
this.updateIndexPromise = this.innerUpdateIndex(blocks);
|
200
|
+
this.updateIndexPromise.finally(() => { this.updateIndexPromise = null; });
|
201
|
+
return this.updateIndexPromise;
|
202
|
+
}
|
203
|
+
async innerUpdateIndex(inBlocks) {
|
204
|
+
// const callTag = Math.random().toString(36).substring(4)
|
205
|
+
// console.log(`updateIndex ${callTag} >`, this.instanceId, this.dbHead?.toString(), this.indexByKey.cid?.toString(), this.indexById.cid?.toString())
|
206
|
+
// todo remove this hack
|
207
|
+
if (ALWAYS_REBUILD) {
|
208
|
+
this.indexById = { root: null, cid: null };
|
209
|
+
this.indexByKey = { root: null, cid: null };
|
210
|
+
this.dbHead = null;
|
211
|
+
}
|
212
|
+
// console.log('dbHead', this.dbHead)
|
213
|
+
// console.time(callTag + '.changesSince')
|
214
|
+
const result = await this.database.changesSince(this.dbHead); // {key, value, del}
|
215
|
+
// console.timeEnd(callTag + '.changesSince')
|
216
|
+
// console.log('result.rows.length', result.rows.length)
|
217
|
+
// console.time(callTag + '.doTransactionupdateIndex')
|
218
|
+
// console.log('updateIndex changes length', result.rows.length)
|
219
|
+
if (result.rows.length === 0) {
|
220
|
+
// console.log('updateIndex < no changes', result.clock)
|
221
|
+
this.dbHead = result.clock;
|
222
|
+
return;
|
223
|
+
}
|
224
|
+
await doTransaction('updateIndex', inBlocks, async (blocks) => {
|
225
|
+
let oldIndexEntries = [];
|
226
|
+
let removeByIdIndexEntries = [];
|
227
|
+
await loadIndex(blocks, this.indexById, idIndexOpts);
|
228
|
+
await loadIndex(blocks, this.indexByKey, dbIndexOpts);
|
229
|
+
if (this.dbHead) {
|
230
|
+
const oldChangeEntries = await this.indexById.root.getMany(result.rows.map(({ key }) => key));
|
231
|
+
oldIndexEntries = oldChangeEntries.result.map((key) => ({ key, del: true }));
|
232
|
+
removeByIdIndexEntries = oldIndexEntries.map(({ key }) => ({ key: key[1], del: true }));
|
233
|
+
}
|
234
|
+
if (!this.mapFn) {
|
235
|
+
throw new Error('No live map function installed for index, cannot update. Make sure your index definition runs before any queries.' + (this.mapFnString ? ' Your code should match the stored map function source:\n' + this.mapFnString : ''));
|
236
|
+
}
|
237
|
+
const indexEntries = indexEntriesForChanges(result.rows, this.mapFn);
|
238
|
+
const byIdIndexEntries = indexEntries.map(({ key }) => ({ key: key[1], value: key }));
|
239
|
+
this.indexById = await bulkIndex(blocks, this.indexById, removeByIdIndexEntries.concat(byIdIndexEntries), idIndexOpts);
|
240
|
+
this.indexByKey = await bulkIndex(blocks, this.indexByKey, oldIndexEntries.concat(indexEntries), dbIndexOpts);
|
241
|
+
this.dbHead = result.clock;
|
242
|
+
});
|
243
|
+
this.database.notifyExternal('dbIndex');
|
244
|
+
// console.timeEnd(callTag + '.doTransactionupdateIndex')
|
245
|
+
// console.log(`updateIndex ${callTag} <`, this.instanceId, this.dbHead?.toString(), this.indexByKey.cid?.toString(), this.indexById.cid?.toString())
|
246
|
+
}
|
247
|
+
}
|
248
|
+
/**
|
249
|
+
* Update the DbIndex with the given entries
|
250
|
+
* @param {import('./blockstore.js').Blockstore} blocks
|
251
|
+
* @param {{root, cid}} inIndex
|
252
|
+
* @param {DbIndexEntry[]} indexEntries
|
253
|
+
* @private
|
254
|
+
*/
|
255
|
+
async function bulkIndex(blocks, inIndex, indexEntries, opts) {
|
256
|
+
if (!indexEntries.length)
|
257
|
+
return inIndex;
|
258
|
+
const putBlock = blocks.put.bind(blocks);
|
259
|
+
const { getBlock } = makeGetBlock(blocks);
|
260
|
+
let returnRootBlock;
|
261
|
+
let returnNode;
|
262
|
+
if (!inIndex.root) {
|
263
|
+
const cid = inIndex.cid;
|
264
|
+
if (!cid) {
|
265
|
+
for await (const node of await create({ get: getBlock, list: indexEntries, ...opts })) {
|
266
|
+
const block = await node.block;
|
267
|
+
await putBlock(block.cid, block.bytes);
|
268
|
+
returnRootBlock = block;
|
269
|
+
returnNode = node;
|
270
|
+
}
|
271
|
+
return { root: returnNode, cid: returnRootBlock.cid };
|
272
|
+
}
|
273
|
+
inIndex.root = await load({ cid, get: getBlock, ...dbIndexOpts });
|
274
|
+
}
|
275
|
+
const { root, blocks: newBlocks } = await inIndex.root.bulk(indexEntries);
|
276
|
+
returnRootBlock = await root.block;
|
277
|
+
returnNode = root;
|
278
|
+
for await (const block of newBlocks) {
|
279
|
+
await putBlock(block.cid, block.bytes);
|
280
|
+
}
|
281
|
+
await putBlock(returnRootBlock.cid, returnRootBlock.bytes);
|
282
|
+
return { root: returnNode, cid: returnRootBlock.cid };
|
283
|
+
}
|
284
|
+
async function loadIndex(blocks, index, indexOpts) {
|
285
|
+
if (!index.root) {
|
286
|
+
const cid = index.cid;
|
287
|
+
if (!cid)
|
288
|
+
return;
|
289
|
+
const { getBlock } = makeGetBlock(blocks);
|
290
|
+
index.root = await load({ cid, get: getBlock, ...indexOpts });
|
291
|
+
}
|
292
|
+
return index.root;
|
293
|
+
}
|
294
|
+
async function applyLimit(results, limit) {
|
295
|
+
results.result = results.result.slice(0, limit);
|
296
|
+
return results;
|
297
|
+
}
|
298
|
+
async function doIndexQuery(blocks, indexByKey, query = {}) {
|
299
|
+
await loadIndex(blocks, indexByKey, dbIndexOpts);
|
300
|
+
if (!indexByKey.root)
|
301
|
+
return { result: [] };
|
302
|
+
if (query.range) {
|
303
|
+
const encodedRange = query.range.map((key) => charwise.encode(key));
|
304
|
+
return applyLimit(await indexByKey.root.range(...encodedRange), query.limit);
|
305
|
+
}
|
306
|
+
else if (query.key) {
|
307
|
+
const encodedKey = charwise.encode(query.key);
|
308
|
+
return indexByKey.root.get(encodedKey);
|
309
|
+
}
|
310
|
+
else {
|
311
|
+
const { result, ...all } = await indexByKey.root.getAllEntries();
|
312
|
+
return applyLimit({ result: result.map(({ key: [k, id], value }) => ({ key: k, id, row: value })), ...all }, query.limit);
|
313
|
+
}
|
314
|
+
}
|