@peerbit/document 1.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,800 @@
1
+ var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
2
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
3
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
4
+ else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
5
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
6
+ };
7
+ var __metadata = (this && this.__metadata) || function (k, v) {
8
+ if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
9
+ };
10
+ import { field, serialize, variant } from "@dao-xyz/borsh";
11
+ import { asString } from "./utils.js";
12
+ import { BORSH_ENCODING } from "@peerbit/log";
13
+ import { equals } from "@peerbit/uint8arrays";
14
+ import { ComposableProgram } from "@peerbit/program";
15
+ import { IntegerCompare, ByteMatchQuery, StringMatch, ResultWithSource, StateFieldQuery, compare, MissingField, StringMatchMethod, LogicalQuery, And, Or, BoolQuery, CollectNextRequest, AbstractSearchRequest, SearchRequest, SortDirection, CloseIteratorRequest, } from "./query.js";
16
+ import { RPC, queryAll, MissingResponsesError, } from "@peerbit/rpc";
17
+ import { Results } from "./query.js";
18
+ import { logger as loggerFn } from "@peerbit/logger";
19
+ import { Cache } from "@peerbit/cache";
20
+ const logger = loggerFn({ module: "document-index" });
21
+ const stringArraysEquals = (a, b) => {
22
+ if (a.length !== b.length) {
23
+ return false;
24
+ }
25
+ for (let i = 0; i < a.length; i++) {
26
+ if (a[i] !== b[i]) {
27
+ return false;
28
+ }
29
+ }
30
+ return true;
31
+ };
32
+ export let Operation = class Operation {
33
+ };
34
+ Operation = __decorate([
35
+ variant(0)
36
+ ], Operation);
37
+ export const BORSH_ENCODING_OPERATION = BORSH_ENCODING(Operation);
38
+ export let PutOperation = class PutOperation extends Operation {
39
+ key;
40
+ data;
41
+ _value;
42
+ constructor(props) {
43
+ super();
44
+ if (props) {
45
+ this.key = props.key;
46
+ this.data = props.data;
47
+ this._value = props.value;
48
+ }
49
+ }
50
+ get value() {
51
+ if (!this._value) {
52
+ throw new Error("Value not decoded, invoke getValue(...) once");
53
+ }
54
+ return this._value;
55
+ }
56
+ getValue(encoding) {
57
+ if (this._value) {
58
+ return this._value;
59
+ }
60
+ this._value = encoding.decoder(this.data);
61
+ return this._value;
62
+ }
63
+ };
64
+ __decorate([
65
+ field({ type: "string" }),
66
+ __metadata("design:type", String)
67
+ ], PutOperation.prototype, "key", void 0);
68
+ __decorate([
69
+ field({ type: Uint8Array }),
70
+ __metadata("design:type", Uint8Array)
71
+ ], PutOperation.prototype, "data", void 0);
72
+ PutOperation = __decorate([
73
+ variant(0),
74
+ __metadata("design:paramtypes", [Object])
75
+ ], PutOperation);
76
+ /* @variant(1)
77
+ export class PutAllOperation<T> extends Operation<T> {
78
+ @field({ type: vec(PutOperation) })
79
+ docs: PutOperation<T>[];
80
+
81
+ constructor(props?: { docs: PutOperation<T>[] }) {
82
+ super();
83
+ if (props) {
84
+ this.docs = props.docs;
85
+ }
86
+ }
87
+ }
88
+ */
89
+ export let DeleteOperation = class DeleteOperation extends Operation {
90
+ key;
91
+ constructor(props) {
92
+ super();
93
+ if (props) {
94
+ this.key = props.key;
95
+ }
96
+ }
97
+ };
98
+ __decorate([
99
+ field({ type: "string" }),
100
+ __metadata("design:type", String)
101
+ ], DeleteOperation.prototype, "key", void 0);
102
+ DeleteOperation = __decorate([
103
+ variant(2),
104
+ __metadata("design:paramtypes", [Object])
105
+ ], DeleteOperation);
106
+ const extractFieldValue = (doc, path) => {
107
+ for (let i = 0; i < path.length; i++) {
108
+ doc = doc[path[i]];
109
+ }
110
+ return doc;
111
+ };
112
+ const sortCompare = (av, bv) => {
113
+ if (typeof av === "string" && typeof bv === "string") {
114
+ return av.localeCompare(bv);
115
+ }
116
+ if (av < bv) {
117
+ return -1;
118
+ }
119
+ else if (av > bv) {
120
+ return 1;
121
+ }
122
+ return 0;
123
+ };
124
+ const extractSortCompare = (a, b, sorts) => {
125
+ for (const sort of sorts) {
126
+ const av = extractFieldValue(a, sort.key);
127
+ const bv = extractFieldValue(b, sort.key);
128
+ const cmp = sortCompare(av, bv);
129
+ if (cmp != 0) {
130
+ if (sort.direction === SortDirection.ASC) {
131
+ return cmp;
132
+ }
133
+ else {
134
+ return -cmp;
135
+ }
136
+ }
137
+ }
138
+ return 0;
139
+ };
140
+ const resolvedSort = async (arr, index, sorts) => {
141
+ await Promise.all(arr.map(async (result) => (result[SORT_TMP_KEY] = await index(result.value, result.context))));
142
+ arr.sort((a, b) => extractSortCompare(a[SORT_TMP_KEY], b[SORT_TMP_KEY], sorts));
143
+ return arr;
144
+ };
145
+ /*
146
+ const sortValueWithContext = async<T>(arr: {
147
+ value: T;
148
+ context: Context;
149
+ }[], index: Indexable<T>) => {
150
+
151
+
152
+ }
153
+ */
154
+ const SORT_TMP_KEY = "__sort_ref";
155
+ const introduceEntries = async (responses, type, sync, options) => {
156
+ return Promise.all(responses.map(async (x) => {
157
+ x.response.results.forEach((r) => r.init(type));
158
+ if (typeof options?.remote !== "boolean" && options?.remote?.sync) {
159
+ await sync(x.response);
160
+ }
161
+ if (!x.from) {
162
+ logger.error("Missing from for response");
163
+ }
164
+ options?.onResponse && options.onResponse(x.response, x.from);
165
+ return x;
166
+ }));
167
+ };
168
+ const dedup = (allResult, dedupBy) => {
169
+ const unique = new Set();
170
+ const dedup = [];
171
+ for (const result of allResult) {
172
+ const key = asString(dedupBy(result));
173
+ if (unique.has(key)) {
174
+ continue;
175
+ }
176
+ unique.add(key);
177
+ dedup.push(result);
178
+ }
179
+ return dedup;
180
+ };
181
+ const DEFAULT_INDEX_BY = "id";
182
+ export let DocumentIndex = class DocumentIndex extends ComposableProgram {
183
+ _query;
184
+ type;
185
+ // Index key
186
+ _indexBy;
187
+ _indexByArr;
188
+ // Resolve doc value by index key
189
+ indexByResolver;
190
+ // Indexed (transforms an docuemnt into an obj with fields that ought to be indexed)
191
+ _toIndex;
192
+ _valueEncoding;
193
+ _sync;
194
+ _index;
195
+ _resultsCollectQueue;
196
+ _log;
197
+ constructor(properties) {
198
+ super();
199
+ this._query = properties?.query || new RPC();
200
+ }
201
+ get index() {
202
+ return this._index;
203
+ }
204
+ get valueEncoding() {
205
+ return this._valueEncoding;
206
+ }
207
+ get toIndex() {
208
+ return this._toIndex;
209
+ }
210
+ async open(properties) {
211
+ this._index = new Map();
212
+ this._log = properties.log;
213
+ this.type = properties.type;
214
+ this._sync = properties.sync;
215
+ this._toIndex = properties.fields;
216
+ this._indexBy = properties.indexBy || DEFAULT_INDEX_BY;
217
+ this._indexByArr = Array.isArray(this._indexBy)
218
+ ? this._indexBy
219
+ : [this._indexBy];
220
+ this.indexByResolver =
221
+ typeof this._indexBy === "string"
222
+ ? (obj) => obj[this._indexBy]
223
+ : (obj) => extractFieldValue(obj, this._indexBy);
224
+ this._valueEncoding = BORSH_ENCODING(this.type);
225
+ this._resultsCollectQueue = new Cache({ max: 10000 }); // TODO choose limit better
226
+ await this._query.open({
227
+ topic: this._log.log.idString + "/document",
228
+ canRead: properties.canRead,
229
+ responseHandler: async (query) => {
230
+ if (query instanceof CloseIteratorRequest) {
231
+ this.processCloseIteratorRequest(query);
232
+ }
233
+ else {
234
+ const results = await this.processFetchRequest(query);
235
+ return new Results({
236
+ // Even if results might have length 0, respond, because then we now at least there are no matching results
237
+ results: results.results.map((r) => new ResultWithSource({
238
+ source: serialize(r.value),
239
+ context: r.context,
240
+ })),
241
+ kept: BigInt(results.kept),
242
+ });
243
+ }
244
+ },
245
+ responseType: Results,
246
+ queryType: AbstractSearchRequest,
247
+ });
248
+ }
249
+ async get(key, options) {
250
+ return (await this.getDetailed(key, options))?.[0]?.results[0]?.value;
251
+ }
252
+ async getDetailed(key, options) {
253
+ let results;
254
+ if (key instanceof Uint8Array) {
255
+ results = await this.queryDetailed(new SearchRequest({
256
+ query: [new ByteMatchQuery({ key: this._indexByArr, value: key })],
257
+ }), options);
258
+ }
259
+ else {
260
+ const stringValue = asString(key);
261
+ results = await this.queryDetailed(new SearchRequest({
262
+ query: [
263
+ new StringMatch({
264
+ key: this._indexByArr,
265
+ value: stringValue,
266
+ }),
267
+ ],
268
+ }), options);
269
+ }
270
+ return results;
271
+ }
272
+ get size() {
273
+ return this._index.size;
274
+ }
275
+ async getDocument(value) {
276
+ const payloadValue = await (await this._log.log.get(value.context.head)).getPayloadValue();
277
+ if (payloadValue instanceof PutOperation) {
278
+ return payloadValue.getValue(this.valueEncoding);
279
+ }
280
+ throw new Error("Unexpected");
281
+ }
282
+ async _queryDocuments(filter) {
283
+ // Whether we return the full operation data or just the db value
284
+ const results = [];
285
+ for (const value of this._index.values()) {
286
+ if (filter(value)) {
287
+ results.push({
288
+ context: value.context,
289
+ value: await this.getDocument(value),
290
+ });
291
+ }
292
+ }
293
+ return results;
294
+ }
295
+ async processFetchRequest(query) {
296
+ // We do special case for querying the id as we can do it faster than iterating
297
+ if (query instanceof SearchRequest) {
298
+ if (query.query.length === 1 &&
299
+ (query.query[0] instanceof ByteMatchQuery ||
300
+ query.query[0] instanceof StringMatch) &&
301
+ stringArraysEquals(query.query[0].key, this._indexByArr)) {
302
+ const firstQuery = query.query[0];
303
+ if (firstQuery instanceof ByteMatchQuery) {
304
+ const doc = this._index.get(asString(firstQuery.value)); // TODO could there be a issue with types here?
305
+ return doc
306
+ ? {
307
+ results: [
308
+ {
309
+ value: await this.getDocument(doc),
310
+ context: doc.context,
311
+ },
312
+ ],
313
+ kept: 0,
314
+ }
315
+ : { results: [], kept: 0 };
316
+ }
317
+ else if (firstQuery instanceof StringMatch &&
318
+ firstQuery.method === StringMatchMethod.exact &&
319
+ firstQuery.caseInsensitive === false) {
320
+ const doc = this._index.get(firstQuery.value); // TODO could there be a issue with types here?
321
+ return doc
322
+ ? {
323
+ results: [
324
+ {
325
+ value: await this.getDocument(doc),
326
+ context: doc.context,
327
+ },
328
+ ],
329
+ kept: 0,
330
+ }
331
+ : { results: [], kept: 0 };
332
+ }
333
+ }
334
+ const results = await this._queryDocuments((doc) => {
335
+ for (const f of query.query) {
336
+ if (!this.handleQueryObject(f, doc)) {
337
+ return false;
338
+ }
339
+ }
340
+ return true;
341
+ });
342
+ // Sort
343
+ await resolvedSort(results, this._toIndex, query.sort);
344
+ const batch = results.splice(0, query.fetch);
345
+ if (results.length > 0) {
346
+ this._resultsCollectQueue.add(query.idString, results); // cache resulst not returned
347
+ }
348
+ return { results: batch, kept: results.length }; // Only return 1 result since we are doing distributed sort, TODO buffer more initially
349
+ }
350
+ else if (query instanceof CollectNextRequest) {
351
+ const results = this._resultsCollectQueue.get(query.idString);
352
+ if (!results) {
353
+ return {
354
+ results: [],
355
+ kept: 0,
356
+ };
357
+ }
358
+ const batch = results.splice(0, query.amount);
359
+ if (results.length === 0) {
360
+ this._resultsCollectQueue.del(query.idString); // TODO add tests for proper cleanup/timeouts
361
+ }
362
+ return { results: batch, kept: results.length };
363
+ }
364
+ throw new Error("Unsupported");
365
+ }
366
+ async processCloseIteratorRequest(query) {
367
+ this._resultsCollectQueue.del(query.idString);
368
+ }
369
+ handleQueryObject(f, doc) {
370
+ if (f instanceof StateFieldQuery) {
371
+ const fv = extractFieldValue(doc.value, f.key);
372
+ if (f instanceof StringMatch) {
373
+ let compare = f.value;
374
+ if (f.caseInsensitive) {
375
+ compare = compare.toLowerCase();
376
+ }
377
+ if (Array.isArray(fv)) {
378
+ for (const string of fv) {
379
+ if (this.handleStringMatch(f, compare, string)) {
380
+ return true;
381
+ }
382
+ }
383
+ return false;
384
+ }
385
+ else {
386
+ if (this.handleStringMatch(f, compare, fv)) {
387
+ return true;
388
+ }
389
+ return false;
390
+ }
391
+ }
392
+ else if (f instanceof ByteMatchQuery) {
393
+ if (fv instanceof Uint8Array === false) {
394
+ if (stringArraysEquals(f.key, this._indexByArr)) {
395
+ return f.valueString === fv;
396
+ }
397
+ return false;
398
+ }
399
+ return equals(fv, f.value);
400
+ }
401
+ else if (f instanceof IntegerCompare) {
402
+ const value = fv;
403
+ if (typeof value !== "bigint" && typeof value !== "number") {
404
+ return false;
405
+ }
406
+ return compare(value, f.compare, f.value.value);
407
+ }
408
+ else if (f instanceof MissingField) {
409
+ return fv == null; // null or undefined
410
+ }
411
+ else if (f instanceof BoolQuery) {
412
+ return fv === f.value; // true/false
413
+ }
414
+ }
415
+ else if (f instanceof LogicalQuery) {
416
+ if (f instanceof And) {
417
+ for (const and of f.and) {
418
+ if (!this.handleQueryObject(and, doc)) {
419
+ return false;
420
+ }
421
+ }
422
+ return true;
423
+ }
424
+ if (f instanceof Or) {
425
+ for (const or of f.or) {
426
+ if (this.handleQueryObject(or, doc)) {
427
+ return true;
428
+ }
429
+ }
430
+ return false;
431
+ }
432
+ return false;
433
+ }
434
+ logger.info("Unsupported query type: " + f.constructor.name);
435
+ return false;
436
+ }
437
+ handleStringMatch(f, compare, fv) {
438
+ if (typeof fv !== "string") {
439
+ return false;
440
+ }
441
+ if (f.caseInsensitive) {
442
+ fv = fv.toLowerCase();
443
+ }
444
+ if (f.method === StringMatchMethod.exact) {
445
+ return fv === compare;
446
+ }
447
+ if (f.method === StringMatchMethod.prefix) {
448
+ return fv.startsWith(compare);
449
+ }
450
+ if (f.method === StringMatchMethod.contains) {
451
+ return fv.includes(compare);
452
+ }
453
+ throw new Error("Unsupported");
454
+ }
455
+ /**
456
+ * Query and retrieve results with most details
457
+ * @param queryRequest
458
+ * @param options
459
+ * @returns
460
+ */
461
+ async queryDetailed(queryRequest, options) {
462
+ const local = typeof options?.local == "boolean" ? options?.local : true;
463
+ let remote = undefined;
464
+ if (typeof options?.remote === "boolean") {
465
+ if (options?.remote) {
466
+ remote = {};
467
+ }
468
+ else {
469
+ remote = undefined;
470
+ }
471
+ }
472
+ else {
473
+ remote = options?.remote || {};
474
+ }
475
+ const promises = [];
476
+ if (!local && !remote) {
477
+ throw new Error("Expecting either 'options.remote' or 'options.local' to be true");
478
+ }
479
+ const allResults = [];
480
+ if (local) {
481
+ const results = await this.processFetchRequest(queryRequest);
482
+ if (results.results.length > 0) {
483
+ const resultsObject = new Results({
484
+ results: await Promise.all(results.results.map(async (r) => {
485
+ const payloadValue = await (await this._log.log.get(r.context.head))?.getPayloadValue();
486
+ if (payloadValue instanceof PutOperation) {
487
+ return new ResultWithSource({
488
+ context: r.context,
489
+ value: r.value,
490
+ source: payloadValue.data,
491
+ });
492
+ }
493
+ throw new Error("Unexpected");
494
+ })),
495
+ kept: BigInt(results.kept),
496
+ });
497
+ options?.onResponse &&
498
+ options.onResponse(resultsObject, this.node.identity.publicKey);
499
+ allResults.push(resultsObject);
500
+ }
501
+ }
502
+ if (remote) {
503
+ const replicatorGroups = await this._log.replicators?.();
504
+ if (replicatorGroups) {
505
+ const fn = async () => {
506
+ const rs = [];
507
+ const responseHandler = async (results) => {
508
+ await introduceEntries(results, this.type, this._sync, options).then((x) => x.forEach((y) => rs.push(y.response)));
509
+ };
510
+ try {
511
+ if (queryRequest instanceof CloseIteratorRequest) {
512
+ // don't wait for responses
513
+ await this._query.request(queryRequest, { to: remote.to });
514
+ }
515
+ else {
516
+ await queryAll(this._query, replicatorGroups, queryRequest, responseHandler, remote);
517
+ }
518
+ }
519
+ catch (error) {
520
+ if (error instanceof MissingResponsesError) {
521
+ logger.error("Did not reciveve responses from all shard");
522
+ }
523
+ }
524
+ return rs;
525
+ };
526
+ promises.push(fn());
527
+ }
528
+ else {
529
+ // TODO send without direction out to the world? or just assume we can insert?
530
+ /* promises.push(
531
+ this._query
532
+ .request(queryRequest, remote)
533
+ .then((results) => introduceEntries(results, this.type, this._sync, options).then(x => x.map(y => y.response)))
534
+ ); */
535
+ /* throw new Error(
536
+ "Missing remote replicator info for performing distributed document query"
537
+ ); */
538
+ }
539
+ }
540
+ const resolved = await Promise.all(promises);
541
+ for (const r of resolved) {
542
+ if (r) {
543
+ if (r instanceof Array) {
544
+ allResults.push(...r);
545
+ }
546
+ else {
547
+ allResults.push(r);
548
+ }
549
+ }
550
+ }
551
+ return allResults;
552
+ }
553
+ /**
554
+ * Query and retrieve results
555
+ * @param queryRequest
556
+ * @param options
557
+ * @returns
558
+ */
559
+ async search(queryRequest, options) {
560
+ // Set fetch to search size, or max value (default to max u32 (4294967295))
561
+ queryRequest.fetch = options?.size ?? 0xffffffff;
562
+ // So that the iterator is pre-fetching the right amount of entries
563
+ const iterator = this.iterate(queryRequest, options);
564
+ // So that this call will not do any remote requests
565
+ const allResult = await iterator.next(queryRequest.fetch);
566
+ await iterator.close();
567
+ //s Deduplicate and return values directly
568
+ return dedup(allResult, this.indexByResolver);
569
+ }
570
+ /**
571
+ * Query and retrieve documents in a iterator
572
+ * @param queryRequest
573
+ * @param options
574
+ * @returns
575
+ */
576
+ iterate(queryRequest, options) {
577
+ let fetchPromise = undefined;
578
+ const peerBufferMap = new Map();
579
+ const visited = new Set();
580
+ let done = false;
581
+ let first = false;
582
+ // TODO handle join/leave while iterating
583
+ let stopperFns = [];
584
+ const peerBuffers = () => {
585
+ return [...peerBufferMap.values()].map((x) => x.buffer).flat();
586
+ };
587
+ const fetchFirst = async (n) => {
588
+ done = true; // Assume we are donne
589
+ queryRequest.fetch = n;
590
+ await this.queryDetailed(queryRequest, {
591
+ ...options,
592
+ onResponse: (response, from) => {
593
+ if (!from) {
594
+ logger.error("Missing response from");
595
+ return;
596
+ }
597
+ if (response.kept === 0n && response.results.length === 0) {
598
+ return;
599
+ }
600
+ if (response.kept > 0n) {
601
+ done = false; // we have more to do later!
602
+ }
603
+ peerBufferMap.set(from.hashcode(), {
604
+ buffer: response.results
605
+ .filter((x) => !visited.has(asString(this.indexByResolver(x.value))))
606
+ .map((x) => {
607
+ visited.add(asString(this.indexByResolver(x.value)));
608
+ return { from, value: x.value, context: x.context };
609
+ }),
610
+ kept: Number(response.kept),
611
+ });
612
+ },
613
+ });
614
+ return done;
615
+ };
616
+ const fetchAtLeast = async (n) => {
617
+ if (done && first) {
618
+ return;
619
+ }
620
+ await fetchPromise;
621
+ if (!first) {
622
+ first = true;
623
+ fetchPromise = fetchFirst(n);
624
+ return fetchPromise;
625
+ }
626
+ const promises = [];
627
+ stopperFns = [];
628
+ let resultsLeft = 0;
629
+ for (const [peer, buffer] of peerBufferMap) {
630
+ if (buffer.buffer.length < n) {
631
+ if (buffer.kept === 0) {
632
+ if (peerBufferMap.get(peer)?.buffer.length === 0) {
633
+ peerBufferMap.delete(peer); // No more results
634
+ }
635
+ continue;
636
+ }
637
+ // TODO buffer more than deleted?
638
+ // TODO batch to multiple 'to's
639
+ const collectRequest = new CollectNextRequest({
640
+ id: queryRequest.id,
641
+ amount: 10, //n - buffer.buffer.length,
642
+ });
643
+ // Fetch locally?
644
+ if (peer === this.node.identity.publicKey.hashcode()) {
645
+ promises.push(this.processFetchRequest(collectRequest)
646
+ .then((results) => {
647
+ resultsLeft += results.kept;
648
+ if (results.results.length === 0) {
649
+ if (peerBufferMap.get(peer)?.buffer.length === 0) {
650
+ peerBufferMap.delete(peer); // No more results
651
+ }
652
+ }
653
+ else {
654
+ const peerBuffer = peerBufferMap.get(peer);
655
+ if (!peerBuffer) {
656
+ return;
657
+ }
658
+ peerBuffer.kept = results.kept;
659
+ peerBuffer.buffer.push(...results.results
660
+ .filter((x) => !visited.has(asString(this.indexByResolver(x.value))))
661
+ .map((x) => {
662
+ visited.add(asString(this.indexByResolver(x.value)));
663
+ return {
664
+ value: x.value,
665
+ context: x.context,
666
+ from: this.node.identity.publicKey,
667
+ };
668
+ }));
669
+ }
670
+ })
671
+ .catch((e) => {
672
+ logger.error("Failed to collect sorted results self. " + e?.message);
673
+ peerBufferMap.delete(peer);
674
+ }));
675
+ }
676
+ else {
677
+ // Fetch remotely
678
+ promises.push(this._query
679
+ .request(collectRequest, {
680
+ ...options,
681
+ stopper: (fn) => stopperFns.push(fn),
682
+ to: [peer],
683
+ })
684
+ .then((response) => introduceEntries(response, this.type, this._sync, options)
685
+ .then((responses) => {
686
+ responses.map((response) => {
687
+ resultsLeft += Number(response.response.kept);
688
+ if (!response.from) {
689
+ logger.error("Missing from for sorted query");
690
+ return;
691
+ }
692
+ if (response.response.results.length === 0) {
693
+ if (peerBufferMap.get(peer)?.buffer.length === 0) {
694
+ peerBufferMap.delete(peer); // No more results
695
+ }
696
+ }
697
+ else {
698
+ const peerBuffer = peerBufferMap.get(peer);
699
+ if (!peerBuffer) {
700
+ return;
701
+ }
702
+ peerBuffer.kept = Number(response.response.kept);
703
+ peerBuffer.buffer.push(...response.response.results
704
+ .filter((x) => !visited.has(asString(this.indexByResolver(x.value))))
705
+ .map((x) => {
706
+ visited.add(asString(this.indexByResolver(x.value)));
707
+ return {
708
+ value: x.value,
709
+ context: x.context,
710
+ from: response.from,
711
+ };
712
+ }));
713
+ }
714
+ });
715
+ })
716
+ .catch((e) => {
717
+ logger.error("Failed to collect sorted results from: " +
718
+ peer +
719
+ ". " +
720
+ e?.message);
721
+ peerBufferMap.delete(peer);
722
+ })));
723
+ }
724
+ }
725
+ else {
726
+ resultsLeft += peerBufferMap.get(peer)?.kept || 0;
727
+ }
728
+ }
729
+ return (fetchPromise = Promise.all(promises).then(() => {
730
+ return resultsLeft === 0; // 0 results left to fetch and 0 pending results
731
+ }));
732
+ };
733
+ const next = async (n) => {
734
+ if (n < 0) {
735
+ throw new Error("Expecting to fetch a positive amount of element");
736
+ }
737
+ if (n === 0) {
738
+ return [];
739
+ }
740
+ // TODO everything below is not very optimized
741
+ const fetchedAll = await fetchAtLeast(n);
742
+ // get n next top entries, shift and pull more results
743
+ const results = await resolvedSort(peerBuffers(), this._toIndex, queryRequest.sort);
744
+ const pendingMoreResults = n < results.length;
745
+ const batch = results.splice(0, n);
746
+ for (const result of batch) {
747
+ const arr = peerBufferMap.get(result.from.hashcode());
748
+ if (!arr) {
749
+ logger.error("Unexpected empty result buffer");
750
+ continue;
751
+ }
752
+ const idx = arr.buffer.findIndex((x) => x == result);
753
+ if (idx >= 0) {
754
+ arr.buffer.splice(idx, 1);
755
+ }
756
+ }
757
+ done = fetchedAll && !pendingMoreResults;
758
+ return dedup(batch.map((x) => x.value), this.indexByResolver);
759
+ };
760
+ const close = async () => {
761
+ for (const fn of stopperFns) {
762
+ fn();
763
+ }
764
+ const closeRequest = new CloseIteratorRequest({ id: queryRequest.id });
765
+ const promises = [];
766
+ for (const [peer, buffer] of peerBufferMap) {
767
+ if (buffer.kept === 0) {
768
+ peerBufferMap.delete(peer);
769
+ continue;
770
+ }
771
+ // Fetch locally?
772
+ if (peer === this.node.identity.publicKey.hashcode()) {
773
+ promises.push(this.processCloseIteratorRequest(closeRequest));
774
+ }
775
+ else {
776
+ // Fetch remotely
777
+ promises.push(this._query.send(closeRequest, {
778
+ ...options,
779
+ to: [peer],
780
+ }));
781
+ }
782
+ }
783
+ await Promise.all(promises);
784
+ };
785
+ return {
786
+ close,
787
+ next,
788
+ done: () => done,
789
+ };
790
+ }
791
+ };
792
+ __decorate([
793
+ field({ type: RPC }),
794
+ __metadata("design:type", RPC)
795
+ ], DocumentIndex.prototype, "_query", void 0);
796
+ DocumentIndex = __decorate([
797
+ variant("documents_index"),
798
+ __metadata("design:paramtypes", [Object])
799
+ ], DocumentIndex);
800
+ //# sourceMappingURL=document-index.js.map