@fireproof/core 0.5.9 → 0.5.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,387 @@
1
+ // @ts-ignore
2
+ import { create, load } from 'prolly-trees/db-index';
3
+ // import { create, load } from '../../../../prolly-trees/src/db-index.js'
4
+ import { sha256 as hasher } from 'multiformats/hashes/sha2';
5
+ // @ts-ignore
6
+ import { nocache as cache } from 'prolly-trees/cache';
7
+ // @ts-ignore
8
+ import { bf, simpleCompare } from 'prolly-trees/utils';
9
+ import { makeGetBlock, visMerkleTree } from './prolly.js';
10
+ // eslint-disable-next-line no-unused-vars
11
+ import { cidsToProof } from './database.js';
12
+ import * as codec from '@ipld/dag-cbor';
13
+ // import { create as createBlock } from 'multiformats/block'
14
+ import { TransactionBlockstore, doTransaction } from './blockstore.js';
15
+ // @ts-ignore
16
+ import charwise from 'charwise';
17
+ const ALWAYS_REBUILD = false; // todo: remove
18
+ const compare = (a, b) => {
19
+ const [aKey, aRef] = a;
20
+ const [bKey, bRef] = b;
21
+ const comp = simpleCompare(aKey, bKey);
22
+ if (comp !== 0)
23
+ return comp;
24
+ return refCompare(aRef, bRef);
25
+ };
26
+ const refCompare = (aRef, bRef) => {
27
+ if (Number.isNaN(aRef))
28
+ return -1;
29
+ if (Number.isNaN(bRef))
30
+ throw new Error('ref may not be Infinity or NaN');
31
+ if (aRef === Infinity)
32
+ return 1; // need to test this on equal docids!
33
+ // if (!Number.isFinite(bRef)) throw new Error('ref may not be Infinity or NaN')
34
+ return simpleCompare(aRef, bRef);
35
+ };
36
+ const dbIndexOpts = { cache, chunker: bf(30), codec, hasher, compare };
37
+ const idIndexOpts = { cache, chunker: bf(30), codec, hasher, compare: simpleCompare };
38
+ const makeDoc = ({ key, value }) => ({ _id: key, ...value });
39
+ /**
40
+ * JDoc for the result row type.
41
+ * @typedef {Object} ChangeEvent
42
+ * @property {string} key - The key of the document.
43
+ * @property {Object} value - The new value of the document.
44
+ * @property {boolean} [del] - Is the row deleted?
45
+ * @memberof DbIndex
46
+ */
47
+ /**
48
+ * JDoc for the result row type.
49
+ * @typedef {Object} DbIndexEntry
50
+ * @property {string[]} key - The key for the DbIndex entry.
51
+ * @property {Object} value - The value of the document.
52
+ * @property {boolean} [del] - Is the row deleted?
53
+ * @memberof DbIndex
54
+ */
55
+ /**
56
+ * Transforms a set of changes to DbIndex entries using a map function.
57
+ *
58
+ * @param {ChangeEvent[]} changes
59
+ * @param {Function} mapFn
60
+ * @returns {DbIndexEntry[]} The DbIndex entries generated by the map function.
61
+ * @private
62
+ * @memberof DbIndex
63
+ */
64
+ const indexEntriesForChanges = (changes, mapFn) => {
65
+ const indexEntries = [];
66
+ changes.forEach(({ key, value, del }) => {
67
+ // key is _id, value is the document
68
+ if (del || !value)
69
+ return;
70
+ let mapCalled = false;
71
+ const mapReturn = mapFn(makeDoc({ key, value }), (k, v) => {
72
+ mapCalled = true;
73
+ if (typeof k === 'undefined')
74
+ return;
75
+ indexEntries.push({
76
+ key: [charwise.encode(k), key],
77
+ value: v || null
78
+ });
79
+ });
80
+ if (!mapCalled && mapReturn) {
81
+ indexEntries.push({
82
+ key: [charwise.encode(mapReturn), key],
83
+ value: null
84
+ });
85
+ }
86
+ });
87
+ return indexEntries;
88
+ };
89
+ /**
90
+ * Represents an DbIndex for a Fireproof database.
91
+ *
92
+ * @class DbIndex
93
+ * @classdesc An DbIndex can be used to order and filter the documents in a Fireproof database.
94
+ *
95
+ * @param {Database} database - The Fireproof database instance to DbIndex.
96
+ * @param {Function} mapFn - The map function to apply to each entry in the database.
97
+ *
98
+ */
99
+ export class DbIndex {
100
+ /**
101
+ * @param {Database} database
102
+ */
103
+ constructor(database, name, mapFn, clock = null, opts = {}) {
104
+ this.database = database;
105
+ if (!database.indexBlocks) {
106
+ database.indexBlocks = new TransactionBlockstore(database?.name + '.indexes', database.blocks.valet?.getKeyMaterial());
107
+ }
108
+ if (typeof name === 'function') {
109
+ // app is using deprecated API, remove in 0.7
110
+ opts = clock || {};
111
+ clock = mapFn || null;
112
+ mapFn = name;
113
+ name = null;
114
+ }
115
+ this.applyMapFn(mapFn, name);
116
+ this.indexById = { root: null, cid: null };
117
+ this.indexByKey = { root: null, cid: null };
118
+ this.dbHead = null;
119
+ if (clock) {
120
+ this.indexById.cid = clock.byId;
121
+ this.indexByKey.cid = clock.byKey;
122
+ this.dbHead = clock.db;
123
+ }
124
+ this.instanceId = this.database.instanceId + `.DbIndex.${Math.random().toString(36).substring(2, 7)}`;
125
+ this.updateIndexPromise = null;
126
+ if (!opts.temporary) {
127
+ DbIndex.registerWithDatabase(this, this.database);
128
+ }
129
+ }
130
+ applyMapFn(mapFn, name) {
131
+ if (typeof mapFn === 'string') {
132
+ this.mapFnString = mapFn;
133
+ }
134
+ else {
135
+ this.mapFn = mapFn;
136
+ this.mapFnString = mapFn.toString();
137
+ }
138
+ this.name = name || this.makeName();
139
+ }
140
+ makeName() {
141
+ const regex = /\(([^,()]+,\s*[^,()]+|\[[^\]]+\],\s*[^,()]+)\)/g;
142
+ let matches = Array.from(this.mapFnString.matchAll(regex), match => match[1].trim());
143
+ if (matches.length === 0) {
144
+ matches = /=>\s*(.*)/.exec(this.mapFnString);
145
+ }
146
+ if (matches.length === 0) {
147
+ return this.mapFnString;
148
+ }
149
+ else {
150
+ // it's a consise arrow function, match everythign after the arrow
151
+ this.includeDocsDefault = true;
152
+ return matches[1];
153
+ }
154
+ }
155
+ static registerWithDatabase(inIndex, database) {
156
+ if (!database.indexes.has(inIndex.mapFnString)) {
157
+ database.indexes.set(inIndex.mapFnString, inIndex);
158
+ }
159
+ else {
160
+ // merge our inIndex code with the inIndex clock or vice versa
161
+ const existingIndex = database.indexes.get(inIndex.mapFnString);
162
+ // keep the code instance, discard the clock instance
163
+ if (existingIndex.mapFn) {
164
+ // this one also has other config
165
+ existingIndex.dbHead = inIndex.dbHead;
166
+ existingIndex.indexById.cid = inIndex.indexById.cid;
167
+ existingIndex.indexByKey.cid = inIndex.indexByKey.cid;
168
+ }
169
+ else {
170
+ inIndex.dbHead = existingIndex.dbHead;
171
+ inIndex.indexById.cid = existingIndex.indexById.cid;
172
+ inIndex.indexByKey.cid = existingIndex.indexByKey.cid;
173
+ database.indexes.set(inIndex.mapFnString, inIndex);
174
+ }
175
+ }
176
+ }
177
+ toJSON() {
178
+ const indexJson = { name: this.name, code: this.mapFnString, clock: { db: null, byId: null, byKey: null } };
179
+ indexJson.clock.db = this.dbHead?.map(cid => cid.toString());
180
+ indexJson.clock.byId = this.indexById.cid?.toString();
181
+ indexJson.clock.byKey = this.indexByKey.cid?.toString();
182
+ return indexJson;
183
+ }
184
+ static fromJSON(database, { code, clock, name }) {
185
+ // console.log('DbIndex.fromJSON', database.constructor.name, code, clock)
186
+ return new DbIndex(database, name, code, clock);
187
+ }
188
+ async visKeyTree() {
189
+ return await visMerkleTree(this.database.indexBlocks, this.indexById.cid);
190
+ }
191
+ async visIdTree() {
192
+ return await visMerkleTree(this.database.indexBlocks, this.indexByKey.cid);
193
+ }
194
+ /**
195
+ * JSDoc for Query type.
196
+ * @typedef {Object} DbQuery
197
+ * @property {string[]} [range] - The range to query.
198
+ * @memberof DbIndex
199
+ */
200
+ /**
201
+ * Query object can have {range}
202
+ * @param {DbQuery} query - the query range to use
203
+ * @returns {Promise<{proof: {}, rows: Array<{id: string, key: string, value: any}>}>}
204
+ * @memberof DbIndex
205
+ * @instance
206
+ */
207
+ async query(query = {}, update = true) {
208
+ // const callId = Math.random().toString(36).substring(2, 7)
209
+ // todo pass a root to query a snapshot
210
+ // console.time(callId + '.updateIndex')
211
+ update && (await this.updateIndex(this.database.indexBlocks));
212
+ // console.timeEnd(callId + '.updateIndex')
213
+ // console.time(callId + '.doIndexQuery')
214
+ // console.log('query', query)
215
+ const response = await this.doIndexQuery(query);
216
+ // console.timeEnd(callId + '.doIndexQuery')
217
+ return {
218
+ proof: { index: await cidsToProof(response.cids) },
219
+ rows: response.result.map(({ id, key, row, doc }) => {
220
+ return { id, key: charwise.decode(key), value: row, doc };
221
+ })
222
+ };
223
+ }
224
+ /**
225
+ *
226
+ * @param {any} resp
227
+ * @param {any} query
228
+ * @returns
229
+ */
230
+ async applyQuery(resp, query) {
231
+ if (query.descending) {
232
+ resp.result = resp.result.reverse();
233
+ }
234
+ if (query.limit) {
235
+ resp.result = resp.result.slice(0, query.limit);
236
+ }
237
+ if (query.includeDocs) {
238
+ resp.result = await Promise.all(resp.result.map(async (row) => {
239
+ const doc = await this.database.get(row.id);
240
+ return { ...row, doc };
241
+ }));
242
+ }
243
+ return resp;
244
+ }
245
+ async doIndexQuery(query = {}) {
246
+ await loadIndex(this.database.indexBlocks, this.indexByKey, dbIndexOpts);
247
+ if (!this.indexByKey.root)
248
+ return { result: [] };
249
+ if (query.includeDocs === undefined)
250
+ query.includeDocs = this.includeDocsDefault;
251
+ if (query.range) {
252
+ const encodedRange = query.range.map(key => charwise.encode(key));
253
+ return await this.applyQuery(await this.indexByKey.root.range(...encodedRange), query);
254
+ }
255
+ else if (query.key) {
256
+ const encodedKey = charwise.encode(query.key);
257
+ return await this.applyQuery(this.indexByKey.root.get(encodedKey), query);
258
+ }
259
+ else {
260
+ const { result, ...all } = await this.indexByKey.root.getAllEntries();
261
+ return await this.applyQuery({ result: result.map(({ key: [k, id], value }) => ({ key: k, id, row: value })), ...all }, query);
262
+ }
263
+ }
264
+ /**
265
+ * Update the DbIndex with the latest changes
266
+ * @private
267
+ * @returns {Promise<void>}
268
+ */
269
+ async updateIndex(blocks) {
270
+ // todo this could enqueue the request and give fresh ones to all second comers -- right now it gives out stale promises while working
271
+ // what would it do in a world where all indexes provide a database snapshot to query?
272
+ if (this.updateIndexPromise) {
273
+ return this.updateIndexPromise.then(() => {
274
+ this.updateIndexPromise = null;
275
+ return this.updateIndex(blocks);
276
+ });
277
+ }
278
+ this.updateIndexPromise = this.innerUpdateIndex(blocks);
279
+ this.updateIndexPromise.finally(() => {
280
+ this.updateIndexPromise = null;
281
+ });
282
+ return this.updateIndexPromise;
283
+ }
284
+ async innerUpdateIndex(inBlocks) {
285
+ const callTag = Math.random().toString(36).substring(4);
286
+ console.log(`updateIndex ${callTag} >`, this.instanceId, this.dbHead?.toString(), this.indexByKey.cid?.toString(), this.indexById.cid?.toString());
287
+ // todo remove this hack
288
+ if (ALWAYS_REBUILD) {
289
+ this.indexById = { root: null, cid: null };
290
+ this.indexByKey = { root: null, cid: null };
291
+ this.dbHead = null;
292
+ }
293
+ // console.log('dbHead', this.dbHead)
294
+ // console.time(callTag + '.changesSince')
295
+ const result = await this.database.changesSince(this.dbHead); // {key, value, del}
296
+ // console.timeEnd(callTag + '.changesSince')
297
+ // console.log('result.rows.length', result.rows.length)
298
+ // console.time(callTag + '.doTransactionupdateIndex')
299
+ // console.log('updateIndex changes length', result.rows.length)
300
+ if (result.rows.length === 0) {
301
+ // console.log('updateIndex < no changes', result.clock)
302
+ this.dbHead = result.clock;
303
+ return;
304
+ }
305
+ const didT = await doTransaction('updateIndex', inBlocks, async (blocks) => {
306
+ let oldIndexEntries = [];
307
+ let removeByIdIndexEntries = [];
308
+ await loadIndex(blocks, this.indexById, idIndexOpts);
309
+ await loadIndex(blocks, this.indexByKey, dbIndexOpts);
310
+ // console.log('head', this.dbHead, this.indexById)
311
+ if (this.indexById.root) {
312
+ const oldChangeEntries = await this.indexById.root.getMany(result.rows.map(({ key }) => key));
313
+ oldIndexEntries = oldChangeEntries.result.map(key => ({ key, del: true }));
314
+ removeByIdIndexEntries = oldIndexEntries.map(({ key }) => ({ key: key[1], del: true }));
315
+ }
316
+ if (!this.mapFn) {
317
+ throw new Error('No live map function installed for index, cannot update. Make sure your index definition runs before any queries.' +
318
+ (this.mapFnString ? ' Your code should match the stored map function source:\n' + this.mapFnString : ''));
319
+ }
320
+ const indexEntries = indexEntriesForChanges(result.rows, this.mapFn);
321
+ const byIdIndexEntries = indexEntries.map(({ key }) => ({ key: key[1], value: key }));
322
+ this.indexById = await bulkIndex(blocks, this.indexById, removeByIdIndexEntries.concat(byIdIndexEntries), idIndexOpts);
323
+ this.indexByKey = await bulkIndex(blocks, this.indexByKey, oldIndexEntries.concat(indexEntries), dbIndexOpts);
324
+ this.dbHead = result.clock;
325
+ });
326
+ // todo index subscriptions
327
+ // this.database.notifyExternal('dbIndex')
328
+ // console.timeEnd(callTag + '.doTransactionupdateIndex')
329
+ // console.log(`updateIndex ${callTag} <`, this.instanceId, this.dbHead?.toString(), this.indexByKey.cid?.toString(), this.indexById.cid?.toString())
330
+ return didT;
331
+ }
332
+ }
333
+ /**
334
+ * Update the DbIndex with the given entries
335
+ * @param {import('./blockstore.js').Blockstore} blocks
336
+ * @param {{root, cid}} inIndex
337
+ * @param {DbIndexEntry[]} indexEntries
338
+ * @private
339
+ */
340
+ async function bulkIndex(blocks, inIndex, indexEntries, opts) {
341
+ if (!indexEntries.length)
342
+ return inIndex;
343
+ const putBlock = blocks.put.bind(blocks);
344
+ const { getBlock } = makeGetBlock(blocks);
345
+ let returnRootBlock;
346
+ let returnNode;
347
+ if (!inIndex.root) {
348
+ const cid = inIndex.cid;
349
+ if (!cid) {
350
+ for await (const node of await create({ get: getBlock, list: indexEntries, ...opts })) {
351
+ const block = await node.block;
352
+ await putBlock(block.cid, block.bytes);
353
+ returnRootBlock = block;
354
+ returnNode = node;
355
+ }
356
+ return { root: returnNode, cid: returnRootBlock.cid };
357
+ }
358
+ inIndex.root = await load({ cid, get: getBlock, ...dbIndexOpts });
359
+ }
360
+ const { root, blocks: newBlocks } = await inIndex.root.bulk(indexEntries);
361
+ if (root) {
362
+ returnRootBlock = await root.block;
363
+ returnNode = root;
364
+ for await (const block of newBlocks) {
365
+ await putBlock(block.cid, block.bytes);
366
+ }
367
+ await putBlock(returnRootBlock.cid, returnRootBlock.bytes);
368
+ return { root: returnNode, cid: returnRootBlock.cid };
369
+ }
370
+ else {
371
+ // throw new Error('test for index after delete')
372
+ return { root: null, cid: null };
373
+ }
374
+ }
375
+ async function loadIndex(blocks, index, indexOpts) {
376
+ if (!index.root) {
377
+ const cid = index.cid;
378
+ if (!cid) {
379
+ // console.log('no cid', index)
380
+ // throw new Error('cannot load index')
381
+ return null;
382
+ }
383
+ const { getBlock } = makeGetBlock(blocks);
384
+ index.root = await load({ cid, get: getBlock, ...indexOpts });
385
+ }
386
+ return index.root;
387
+ }
@@ -53,7 +53,6 @@ declare class Database$1 {
53
53
  * @instance
54
54
  */
55
55
  notifyReset(): Promise<void>;
56
- notifyExternal(source?: string): Promise<void>;
57
56
  /**
58
57
  * Returns the changes made to the Fireproof instance since the specified event.
59
58
  * @function changesSince
@@ -185,6 +184,13 @@ declare class Database$1 {
185
184
  * @returns {Function} - A function that can be called to unregister the listener.
186
185
  * @memberof Fireproof
187
186
  */
187
+ subscribe(listener: Function): Function;
188
+ /**
189
+ * @deprecated 0.7.0 - renamed subscribe(listener)
190
+ * @param {Function} listener - The listener to be called when the clock is updated.
191
+ * @returns {Function} - A function that can be called to unregister the listener.
192
+ * @memberof Fireproof
193
+ */
188
194
  registerListener(listener: Function): Function;
189
195
  notifyListeners(changes: any): Promise<void>;
190
196
  setCarUploader(carUploaderFn: any): void;
@@ -213,9 +219,6 @@ declare class DbIndex {
213
219
  */
214
220
  constructor(database: Database$1, name: any, mapFn: any, clock?: any, opts?: {});
215
221
  database: Database$1;
216
- mapFnString: any;
217
- mapFn: any;
218
- name: any;
219
222
  indexById: {
220
223
  root: any;
221
224
  cid: any;
@@ -227,7 +230,12 @@ declare class DbIndex {
227
230
  dbHead: any;
228
231
  instanceId: string;
229
232
  updateIndexPromise: Promise<any>;
233
+ applyMapFn(mapFn: any, name: any): void;
234
+ mapFnString: any;
235
+ mapFn: any;
236
+ name: any;
230
237
  makeName(): any;
238
+ includeDocsDefault: boolean;
231
239
  toJSON(): {
232
240
  name: any;
233
241
  code: any;
@@ -254,7 +262,7 @@ declare class DbIndex {
254
262
  /**
255
263
  * Query object can have {range}
256
264
  * @param {DbQuery} query - the query range to use
257
- * @returns {Promise<{proof: {}, rows: Array<{id: string, key: string, value: any}>}>}
265
+ * @returns {Promise<{proof: {}, rows: Array<{id: string, key: string, value: any, doc?: any}>}>}
258
266
  * @memberof DbIndex
259
267
  * @instance
260
268
  */
@@ -269,8 +277,17 @@ declare class DbIndex {
269
277
  id: string;
270
278
  key: string;
271
279
  value: any;
280
+ doc?: any;
272
281
  }>;
273
282
  }>;
283
+ /**
284
+ *
285
+ * @param {any} resp
286
+ * @param {any} query
287
+ * @returns
288
+ */
289
+ applyQuery(resp: any, query: any): Promise<any>;
290
+ doIndexQuery(query?: {}): Promise<any>;
274
291
  /**
275
292
  * Update the DbIndex with the latest changes
276
293
  * @private
@@ -306,6 +323,9 @@ type ChangeEvent = {
306
323
  * @param {import('./database.js').Database} database - The Database database instance to index.
307
324
  * @param {Function} routingFn - The routing function to apply to each entry in the database.
308
325
  */
326
+ /**
327
+ * @deprecated since version 0.7.0
328
+ */
309
329
  declare class Listener {
310
330
  /**
311
331
  * @param {import('./database.js').Database} database