@peerbit/document 13.0.34 → 13.0.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/program.d.ts.map +1 -1
- package/dist/src/program.js +5 -5
- package/dist/src/program.js.map +1 -1
- package/dist/src/result-shape.d.ts +7 -0
- package/dist/src/result-shape.d.ts.map +1 -0
- package/dist/src/result-shape.js +20 -0
- package/dist/src/result-shape.js.map +1 -0
- package/dist/src/search.d.ts.map +1 -1
- package/dist/src/search.js +49 -47
- package/dist/src/search.js.map +1 -1
- package/package.json +11 -11
- package/src/program.ts +14 -12
- package/src/result-shape.ts +47 -0
- package/src/search.ts +294 -277
package/src/search.ts
CHANGED
|
@@ -50,6 +50,12 @@ import MostCommonQueryPredictor, {
|
|
|
50
50
|
import { type Operation, isPutOperation } from "./operation.js";
|
|
51
51
|
import { Prefetch } from "./prefetch.js";
|
|
52
52
|
import type { ExtractArgs } from "./program.js";
|
|
53
|
+
import {
|
|
54
|
+
initializeResultType,
|
|
55
|
+
isResultIndexedValue,
|
|
56
|
+
isResultValue,
|
|
57
|
+
isResults,
|
|
58
|
+
} from "./result-shape.js";
|
|
53
59
|
import { ResumableIterators } from "./resumable-iterator.js";
|
|
54
60
|
|
|
55
61
|
const WARNING_WHEN_ITERATING_FOR_MORE_THAN = 1e5;
|
|
@@ -411,19 +417,20 @@ const introduceEntries = async <
|
|
|
411
417
|
logger.error("Missing from for response");
|
|
412
418
|
}
|
|
413
419
|
|
|
414
|
-
if (response.response
|
|
420
|
+
if (isResults(response.response)) {
|
|
415
421
|
response.response.results.forEach((r) =>
|
|
416
|
-
r
|
|
417
|
-
? r.init(documentType)
|
|
418
|
-
: r.init(indexedType),
|
|
422
|
+
initializeResultType(r, documentType, indexedType),
|
|
419
423
|
);
|
|
420
424
|
if (typeof options?.remote !== "boolean" && options?.remote?.replicate) {
|
|
421
425
|
const uniqueResults = response.response.results.filter((result) => {
|
|
426
|
+
const resultWithContext = result as
|
|
427
|
+
| types.ResultValue<T>
|
|
428
|
+
| types.ResultIndexedValue<I>;
|
|
422
429
|
const head =
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
?
|
|
426
|
-
:
|
|
430
|
+
isResultIndexedValue(resultWithContext) &&
|
|
431
|
+
resultWithContext.entries.length > 0
|
|
432
|
+
? resultWithContext.entries[0]!.hash
|
|
433
|
+
: resultWithContext.context.head;
|
|
427
434
|
if (replicatedHeads.has(head)) {
|
|
428
435
|
return false;
|
|
429
436
|
}
|
|
@@ -1491,24 +1498,24 @@ export class DocumentIndex<
|
|
|
1491
1498
|
options?: Options,
|
|
1492
1499
|
): Promise<WithContext<I>>;
|
|
1493
1500
|
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1501
|
+
public async get<
|
|
1502
|
+
Options extends GetOptions<T, I, D, Resolve>,
|
|
1503
|
+
Resolve extends boolean | undefined = ExtractResolveFromOptions<Options>,
|
|
1504
|
+
>(key: indexerTypes.Ideable | indexerTypes.IdKey, options?: Options) {
|
|
1505
|
+
let deferred:
|
|
1506
|
+
| DeferredPromise<WithIndexedContext<T, I> | WithContext<I>>
|
|
1507
|
+
| undefined;
|
|
1508
|
+
let baseRemote:
|
|
1509
|
+
| RemoteQueryOptions<
|
|
1510
|
+
types.AbstractSearchRequest,
|
|
1511
|
+
types.AbstractSearchResult,
|
|
1512
|
+
D
|
|
1513
|
+
>
|
|
1514
|
+
| undefined;
|
|
1515
|
+
|
|
1516
|
+
// Normalize the id key early so listeners can use it
|
|
1517
|
+
let idKey =
|
|
1518
|
+
key instanceof indexerTypes.IdKey ? key : indexerTypes.toId(key);
|
|
1512
1519
|
|
|
1513
1520
|
if (options?.waitFor) {
|
|
1514
1521
|
// add change listener before query because we might get a concurrent change that matches the query,
|
|
@@ -1541,16 +1548,16 @@ export class DocumentIndex<
|
|
|
1541
1548
|
|
|
1542
1549
|
let timeout = setTimeout(resolveUndefined, options.waitFor);
|
|
1543
1550
|
this.events.addEventListener("close", resolveUndefined);
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1551
|
+
this.documentEvents.addEventListener("change", listener);
|
|
1552
|
+
deferred.promise.then(cleanup);
|
|
1553
|
+
|
|
1554
|
+
// Prepare remote options without mutating caller options
|
|
1555
|
+
baseRemote =
|
|
1556
|
+
options?.remote === false
|
|
1557
|
+
? undefined
|
|
1558
|
+
: typeof options?.remote === "object"
|
|
1559
|
+
? { ...options.remote }
|
|
1560
|
+
: {};
|
|
1554
1561
|
if (baseRemote) {
|
|
1555
1562
|
const waitPolicy = baseRemote.wait;
|
|
1556
1563
|
if (
|
|
@@ -1586,20 +1593,20 @@ export class DocumentIndex<
|
|
|
1586
1593
|
deferred!.resolve(first.value as any);
|
|
1587
1594
|
}
|
|
1588
1595
|
},
|
|
1589
|
-
|
|
1590
|
-
}
|
|
1596
|
+
});
|
|
1591
1597
|
}
|
|
1598
|
+
}
|
|
1592
1599
|
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1600
|
+
const initialOptions = baseRemote
|
|
1601
|
+
? ({ ...(options as any), remote: baseRemote } as Options)
|
|
1602
|
+
: options;
|
|
1603
|
+
const result = (await this.getDetailed(idKey, initialOptions))?.[0]
|
|
1604
|
+
?.results[0];
|
|
1598
1605
|
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1606
|
+
// if no results, and we have remote joining options, we wait for the timout and if there are joining peers we re-query
|
|
1607
|
+
if (!result) {
|
|
1608
|
+
return deferred?.promise;
|
|
1609
|
+
} else if (deferred) {
|
|
1603
1610
|
deferred.resolve(undefined);
|
|
1604
1611
|
}
|
|
1605
1612
|
return result?.value;
|
|
@@ -1822,15 +1829,14 @@ export class DocumentIndex<
|
|
|
1822
1829
|
let value = set.results[i];
|
|
1823
1830
|
let resolved: T | undefined;
|
|
1824
1831
|
if (shouldResolve) {
|
|
1825
|
-
resolved =
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
: value.value;
|
|
1832
|
+
resolved = isResultIndexedValue(value)
|
|
1833
|
+
? (
|
|
1834
|
+
await this.resolveDocument({
|
|
1835
|
+
indexed: value.value,
|
|
1836
|
+
head: value.context.head,
|
|
1837
|
+
})
|
|
1838
|
+
)?.value
|
|
1839
|
+
: value.value;
|
|
1834
1840
|
} else {
|
|
1835
1841
|
resolved = value.value as T;
|
|
1836
1842
|
}
|
|
@@ -2001,26 +2007,26 @@ export class DocumentIndex<
|
|
|
2001
2007
|
const resolveFlag = resolvesDocuments(
|
|
2002
2008
|
(fromQuery || query) as AnyIterationRequest,
|
|
2003
2009
|
);
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2010
|
+
prevQueued = {
|
|
2011
|
+
from,
|
|
2012
|
+
queue: [],
|
|
2013
|
+
timeout: setTimeout(() => {
|
|
2014
|
+
this._resultQueue.delete(query.idString);
|
|
2015
|
+
}, 6e4),
|
|
2016
|
+
keptInIndex: kept,
|
|
2017
|
+
fromQuery: (fromQuery || query) as
|
|
2018
|
+
| types.SearchRequest
|
|
2019
|
+
| types.SearchRequestIndexed
|
|
2020
|
+
| types.IterationRequest,
|
|
2021
|
+
resolveResults: resolveFlag,
|
|
2022
|
+
};
|
|
2023
|
+
// Don't keep Node alive just to GC old remote iterator state.
|
|
2024
|
+
prevQueued.timeout.unref?.();
|
|
2025
|
+
if (
|
|
2026
|
+
fromQuery instanceof types.IterationRequest &&
|
|
2027
|
+
fromQuery.pushUpdates
|
|
2028
|
+
) {
|
|
2029
|
+
prevQueued.pushMode = fromQuery.pushUpdates;
|
|
2024
2030
|
}
|
|
2025
2031
|
this._resultQueue.set(query.idString, prevQueued);
|
|
2026
2032
|
}
|
|
@@ -2118,19 +2124,19 @@ export class DocumentIndex<
|
|
|
2118
2124
|
string,
|
|
2119
2125
|
ReturnType<typeof setTimeout>
|
|
2120
2126
|
>());
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
|
|
2127
|
+
const timer = setTimeout(() => {
|
|
2128
|
+
timers.delete(idString);
|
|
2129
|
+
const queued = this._resultQueue.get(idString);
|
|
2130
|
+
if (queued) {
|
|
2131
|
+
clearTimeout(queued.timeout);
|
|
2132
|
+
this._resultQueue.delete(idString);
|
|
2133
|
+
}
|
|
2134
|
+
this._resumableIterators.close({ idString });
|
|
2135
|
+
}, delay);
|
|
2136
|
+
// This is a best-effort cleanup timer; it should not keep Node alive.
|
|
2137
|
+
timer.unref?.();
|
|
2138
|
+
timers.set(idString, timer);
|
|
2139
|
+
}
|
|
2134
2140
|
|
|
2135
2141
|
private cancelIteratorKeepAlive(idString: string) {
|
|
2136
2142
|
const timers = this.iteratorKeepAliveTimers;
|
|
@@ -2325,16 +2331,17 @@ export class DocumentIndex<
|
|
|
2325
2331
|
const onQueryJoin = (e: { detail: PublicSignKey }) => {
|
|
2326
2332
|
void handlePeer(e.detail);
|
|
2327
2333
|
};
|
|
2328
|
-
const onReplicatorEvent = (e: {
|
|
2329
|
-
detail: { publicKey: PublicSignKey };
|
|
2330
|
-
}) => {
|
|
2334
|
+
const onReplicatorEvent = (e: { detail: { publicKey: PublicSignKey } }) => {
|
|
2331
2335
|
void handlePeer(e.detail.publicKey);
|
|
2332
2336
|
};
|
|
2333
2337
|
|
|
2334
2338
|
this._query.events.addEventListener("join", onQueryJoin);
|
|
2335
2339
|
this._log?.events?.addEventListener("replicator:join", onReplicatorEvent);
|
|
2336
2340
|
this._log?.events?.addEventListener("replicator:mature", onReplicatorEvent);
|
|
2337
|
-
this._log?.events?.addEventListener(
|
|
2341
|
+
this._log?.events?.addEventListener(
|
|
2342
|
+
"replication:change",
|
|
2343
|
+
onReplicatorEvent,
|
|
2344
|
+
);
|
|
2338
2345
|
|
|
2339
2346
|
return () => {
|
|
2340
2347
|
this._query.events.removeEventListener("join", onQueryJoin);
|
|
@@ -2389,20 +2396,20 @@ export class DocumentIndex<
|
|
|
2389
2396
|
queryRequest: R,
|
|
2390
2397
|
options?: QueryDetailedOptions<T, I, D, boolean | undefined>,
|
|
2391
2398
|
fetchFirstForRemote?: Set<string>,
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
2399
|
+
): Promise<types.Results<RT>[]> {
|
|
2400
|
+
const local = typeof options?.local === "boolean" ? options?.local : true;
|
|
2401
|
+
let remote:
|
|
2402
|
+
| RemoteQueryOptions<
|
|
2403
|
+
types.AbstractSearchRequest,
|
|
2404
|
+
types.AbstractSearchResult,
|
|
2405
|
+
D
|
|
2406
|
+
>
|
|
2407
|
+
| undefined = undefined;
|
|
2408
|
+
if (typeof options?.remote === "boolean") {
|
|
2409
|
+
remote = options.remote ? {} : undefined;
|
|
2410
|
+
} else {
|
|
2411
|
+
remote = options?.remote || {};
|
|
2412
|
+
}
|
|
2406
2413
|
if (remote && remote.priority == null) {
|
|
2407
2414
|
// give queries higher priority than other "normal" data activities
|
|
2408
2415
|
// without this, we might have a scenario that a peer joina network with large amount of data to be synced, but can not query anything before that is done
|
|
@@ -2447,58 +2454,61 @@ export class DocumentIndex<
|
|
|
2447
2454
|
throw new Error("Unexpected");
|
|
2448
2455
|
}
|
|
2449
2456
|
|
|
2450
|
-
|
|
2451
|
-
|
|
2452
|
-
|
|
2453
|
-
|
|
2454
|
-
|
|
2455
|
-
|
|
2456
|
-
|
|
2457
|
-
|
|
2458
|
-
|
|
2459
|
-
|
|
2460
|
-
|
|
2461
|
-
|
|
2462
|
-
|
|
2463
|
-
|
|
2457
|
+
const coverProps = remote.domain ?? { args: undefined };
|
|
2458
|
+
const isDefaultDomainArgs =
|
|
2459
|
+
!("range" in coverProps) &&
|
|
2460
|
+
(!("args" in coverProps) || (coverProps as any).args == null);
|
|
2461
|
+
const remoteWasExplicit = options?.remote != null;
|
|
2462
|
+
|
|
2463
|
+
let replicatorGroups = options?.remote?.from
|
|
2464
|
+
? options?.remote?.from
|
|
2465
|
+
: await this._log.getCover(coverProps, {
|
|
2466
|
+
roleAge: remote.minAge,
|
|
2467
|
+
eager: remote.reach?.eager,
|
|
2468
|
+
reachableOnly: !!remote.wait, // when we want to merge joining we can ignore pending to be online peers and instead consider them once they become online
|
|
2469
|
+
signal: options?.signal,
|
|
2470
|
+
});
|
|
2464
2471
|
|
|
2465
|
-
|
|
2466
|
-
|
|
2467
|
-
|
|
2468
|
-
|
|
2469
|
-
|
|
2470
|
-
|
|
2471
|
-
|
|
2472
|
-
|
|
2473
|
-
|
|
2474
|
-
|
|
2475
|
-
|
|
2476
|
-
|
|
2477
|
-
|
|
2478
|
-
|
|
2479
|
-
|
|
2480
|
-
|
|
2481
|
-
|
|
2482
|
-
|
|
2483
|
-
|
|
2484
|
-
|
|
2485
|
-
|
|
2486
|
-
|
|
2487
|
-
|
|
2488
|
-
|
|
2489
|
-
|
|
2490
|
-
|
|
2491
|
-
|
|
2492
|
-
|
|
2493
|
-
|
|
2472
|
+
// Cold start: cover can be temporarily self-only while replication metadata
|
|
2473
|
+
// converges. For explicit remote searches, query bounded connected peers
|
|
2474
|
+
// instead of waiting for replicator metadata to catch up.
|
|
2475
|
+
if (!options?.remote?.from && isDefaultDomainArgs && remoteWasExplicit) {
|
|
2476
|
+
const selfHash = this.node.identity.publicKey.hashcode();
|
|
2477
|
+
const remoteCount = replicatorGroups.filter(
|
|
2478
|
+
(h) => h !== selfHash,
|
|
2479
|
+
).length;
|
|
2480
|
+
if (remoteCount === 0) {
|
|
2481
|
+
const waitEnabled = Boolean(remote.wait);
|
|
2482
|
+
const coverIsSelfOnly =
|
|
2483
|
+
replicatorGroups.length === 1 && replicatorGroups[0] === selfHash;
|
|
2484
|
+
|
|
2485
|
+
// If the cover is explicitly empty (no shards), don't override it unless
|
|
2486
|
+
// the caller requested waiting for joins (e.g. get(waitFor)).
|
|
2487
|
+
if (waitEnabled || coverIsSelfOnly) {
|
|
2488
|
+
const peerMap: Map<string, unknown> | undefined = (
|
|
2489
|
+
this.node.services.pubsub as any
|
|
2490
|
+
)?.peers;
|
|
2491
|
+
if (peerMap?.keys) {
|
|
2492
|
+
const extra: string[] = [];
|
|
2493
|
+
for (const hash of peerMap.keys()) {
|
|
2494
|
+
if (!hash || hash === selfHash) continue;
|
|
2495
|
+
extra.push(hash);
|
|
2496
|
+
if (extra.length >= 8) break;
|
|
2497
|
+
}
|
|
2498
|
+
if (extra.length > 0) {
|
|
2499
|
+
replicatorGroups = [
|
|
2500
|
+
...new Set([...replicatorGroups, ...extra]),
|
|
2501
|
+
];
|
|
2494
2502
|
}
|
|
2495
2503
|
}
|
|
2496
2504
|
}
|
|
2505
|
+
}
|
|
2506
|
+
}
|
|
2497
2507
|
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
2508
|
+
if (replicatorGroups) {
|
|
2509
|
+
const responseHandler = async (
|
|
2510
|
+
results: {
|
|
2511
|
+
response: types.AbstractSearchResult;
|
|
2502
2512
|
from?: PublicSignKey;
|
|
2503
2513
|
}[],
|
|
2504
2514
|
) => {
|
|
@@ -2734,7 +2744,7 @@ export class DocumentIndex<
|
|
|
2734
2744
|
result: types.ResultValue<T> | types.ResultIndexedValue<I>,
|
|
2735
2745
|
results: types.ResultTypeFromRequest<R, T, I>[],
|
|
2736
2746
|
) {
|
|
2737
|
-
if (result
|
|
2747
|
+
if (isResultIndexedValue(result)) {
|
|
2738
2748
|
return coerceWithContext(result.value as I, result.context);
|
|
2739
2749
|
}
|
|
2740
2750
|
|
|
@@ -2752,7 +2762,7 @@ export class DocumentIndex<
|
|
|
2752
2762
|
) => {
|
|
2753
2763
|
// look through the search results and see if we can find the indexed representation
|
|
2754
2764
|
for (const otherResult of results) {
|
|
2755
|
-
if (otherResult
|
|
2765
|
+
if (isResultIndexedValue(otherResult)) {
|
|
2756
2766
|
if (otherResult.context.head === result.context.head) {
|
|
2757
2767
|
otherResult.init(this.indexedType);
|
|
2758
2768
|
return coerceWithContext(
|
|
@@ -2991,7 +3001,7 @@ export class DocumentIndex<
|
|
|
2991
3001
|
}
|
|
2992
3002
|
const retryMissingResponseGroups =
|
|
2993
3003
|
typeof options?.remote === "object"
|
|
2994
|
-
? options.remote.retryMissingResponses ?? true
|
|
3004
|
+
? (options.remote.retryMissingResponses ?? true)
|
|
2995
3005
|
: true;
|
|
2996
3006
|
|
|
2997
3007
|
indexIteratorLogger.trace("Iterate with options", {
|
|
@@ -3243,7 +3253,7 @@ export class DocumentIndex<
|
|
|
3243
3253
|
if (response instanceof types.NoAccess) {
|
|
3244
3254
|
logger.error("Dont have access");
|
|
3245
3255
|
return;
|
|
3246
|
-
} else if (response
|
|
3256
|
+
} else if (isResults(response)) {
|
|
3247
3257
|
const results = response as types.Results<
|
|
3248
3258
|
types.ResultTypeFromRequest<R, T, I>
|
|
3249
3259
|
>;
|
|
@@ -3280,7 +3290,7 @@ export class DocumentIndex<
|
|
|
3280
3290
|
const indexKey = indexerTypes.toId(
|
|
3281
3291
|
this.indexByResolver(result.value),
|
|
3282
3292
|
).primitive;
|
|
3283
|
-
if (result
|
|
3293
|
+
if (isResultValue(result)) {
|
|
3284
3294
|
const existingIndexed = indexedPlaceholders?.get(indexKey);
|
|
3285
3295
|
if (existingIndexed) {
|
|
3286
3296
|
existingIndexed.value =
|
|
@@ -3309,6 +3319,8 @@ export class DocumentIndex<
|
|
|
3309
3319
|
),
|
|
3310
3320
|
});
|
|
3311
3321
|
} else {
|
|
3322
|
+
const indexedResult =
|
|
3323
|
+
result as unknown as types.ResultIndexedValue<I>;
|
|
3312
3324
|
if (
|
|
3313
3325
|
visited.has(indexKey) &&
|
|
3314
3326
|
!indexedPlaceholders?.has(indexKey)
|
|
@@ -3317,12 +3329,12 @@ export class DocumentIndex<
|
|
|
3317
3329
|
}
|
|
3318
3330
|
visited.add(indexKey);
|
|
3319
3331
|
const indexed = coerceWithContext(
|
|
3320
|
-
|
|
3321
|
-
|
|
3332
|
+
indexedResult.indexed || indexedResult.value,
|
|
3333
|
+
indexedResult.context,
|
|
3322
3334
|
);
|
|
3323
3335
|
const placeholder = {
|
|
3324
|
-
value:
|
|
3325
|
-
context:
|
|
3336
|
+
value: indexedResult.value,
|
|
3337
|
+
context: indexedResult.context,
|
|
3326
3338
|
from,
|
|
3327
3339
|
indexed,
|
|
3328
3340
|
};
|
|
@@ -3346,9 +3358,11 @@ export class DocumentIndex<
|
|
|
3346
3358
|
if (!retryMissingResponseGroups) {
|
|
3347
3359
|
return;
|
|
3348
3360
|
}
|
|
3349
|
-
const missingGroups = (
|
|
3350
|
-
|
|
3351
|
-
|
|
3361
|
+
const missingGroups = (
|
|
3362
|
+
error as MissingResponsesError & {
|
|
3363
|
+
missingGroups?: string[][];
|
|
3364
|
+
}
|
|
3365
|
+
).missingGroups;
|
|
3352
3366
|
if (!missingGroups?.length) {
|
|
3353
3367
|
return;
|
|
3354
3368
|
}
|
|
@@ -3472,7 +3486,7 @@ export class DocumentIndex<
|
|
|
3472
3486
|
const keyPrimitive = indexerTypes.toId(
|
|
3473
3487
|
this.indexByResolver(result.value),
|
|
3474
3488
|
).primitive;
|
|
3475
|
-
if (result
|
|
3489
|
+
if (isResultValue(result)) {
|
|
3476
3490
|
const existingIndexed =
|
|
3477
3491
|
indexedPlaceholders?.get(keyPrimitive);
|
|
3478
3492
|
if (existingIndexed) {
|
|
@@ -3519,6 +3533,8 @@ export class DocumentIndex<
|
|
|
3519
3533
|
indexed,
|
|
3520
3534
|
});
|
|
3521
3535
|
} else {
|
|
3536
|
+
const indexedResult =
|
|
3537
|
+
result as unknown as types.ResultIndexedValue<I>;
|
|
3522
3538
|
if (
|
|
3523
3539
|
visited.has(keyPrimitive) &&
|
|
3524
3540
|
!indexedPlaceholders?.has(keyPrimitive)
|
|
@@ -3527,12 +3543,12 @@ export class DocumentIndex<
|
|
|
3527
3543
|
}
|
|
3528
3544
|
visited.add(keyPrimitive);
|
|
3529
3545
|
const indexed = coerceWithContext(
|
|
3530
|
-
|
|
3531
|
-
|
|
3546
|
+
indexedResult.indexed || indexedResult.value,
|
|
3547
|
+
indexedResult.context,
|
|
3532
3548
|
);
|
|
3533
3549
|
const placeholder = {
|
|
3534
|
-
value:
|
|
3535
|
-
context:
|
|
3550
|
+
value: indexedResult.value,
|
|
3551
|
+
context: indexedResult.context,
|
|
3536
3552
|
from: this.node.identity.publicKey,
|
|
3537
3553
|
indexed,
|
|
3538
3554
|
};
|
|
@@ -3621,7 +3637,7 @@ export class DocumentIndex<
|
|
|
3621
3637
|
const indexKey = indexerTypes.toId(
|
|
3622
3638
|
this.indexByResolver(result.value),
|
|
3623
3639
|
).primitive;
|
|
3624
|
-
if (result
|
|
3640
|
+
if (isResultValue(result)) {
|
|
3625
3641
|
const existingIndexed =
|
|
3626
3642
|
indexedPlaceholders?.get(indexKey);
|
|
3627
3643
|
if (existingIndexed) {
|
|
@@ -3672,6 +3688,8 @@ export class DocumentIndex<
|
|
|
3672
3688
|
indexed,
|
|
3673
3689
|
});
|
|
3674
3690
|
} else {
|
|
3691
|
+
const indexedResult =
|
|
3692
|
+
result as unknown as types.ResultIndexedValue<I>;
|
|
3675
3693
|
if (
|
|
3676
3694
|
visited.has(indexKey) &&
|
|
3677
3695
|
!indexedPlaceholders?.has(indexKey)
|
|
@@ -3680,12 +3698,12 @@ export class DocumentIndex<
|
|
|
3680
3698
|
}
|
|
3681
3699
|
visited.add(indexKey);
|
|
3682
3700
|
const indexed = coerceWithContext(
|
|
3683
|
-
|
|
3684
|
-
|
|
3701
|
+
indexedResult.value,
|
|
3702
|
+
indexedResult.context,
|
|
3685
3703
|
);
|
|
3686
3704
|
const placeholder = {
|
|
3687
|
-
value:
|
|
3688
|
-
context:
|
|
3705
|
+
value: indexedResult.value,
|
|
3706
|
+
context: indexedResult.context,
|
|
3689
3707
|
from: from!,
|
|
3690
3708
|
indexed,
|
|
3691
3709
|
};
|
|
@@ -3764,9 +3782,7 @@ export class DocumentIndex<
|
|
|
3764
3782
|
// blocking on an eager remote top-up for the full requested batch size.
|
|
3765
3783
|
// This keeps `next(n)` responsive when a late push/update arrives while a
|
|
3766
3784
|
// remote iterator has no more immediate items to collect.
|
|
3767
|
-
const fetchedAll = preferBufferedResults
|
|
3768
|
-
? false
|
|
3769
|
-
: await fetchAtLeast(n);
|
|
3785
|
+
const fetchedAll = preferBufferedResults ? false : await fetchAtLeast(n);
|
|
3770
3786
|
|
|
3771
3787
|
// get n next top entries, shift and pull more results
|
|
3772
3788
|
const peerBuffersArr = peerBuffers();
|
|
@@ -3854,32 +3870,30 @@ export class DocumentIndex<
|
|
|
3854
3870
|
done = true;
|
|
3855
3871
|
};
|
|
3856
3872
|
|
|
3857
|
-
|
|
3858
|
-
|
|
3873
|
+
let close = async () => {
|
|
3874
|
+
cleanupAndDone();
|
|
3859
3875
|
|
|
3860
|
-
|
|
3861
|
-
|
|
3862
|
-
|
|
3863
|
-
|
|
3864
|
-
|
|
3865
|
-
|
|
3866
|
-
|
|
3867
|
-
|
|
3868
|
-
|
|
3869
|
-
|
|
3870
|
-
|
|
3871
|
-
|
|
3872
|
-
|
|
3873
|
-
|
|
3874
|
-
|
|
3875
|
-
|
|
3876
|
-
|
|
3877
|
-
|
|
3878
|
-
|
|
3879
|
-
|
|
3880
|
-
|
|
3881
|
-
);
|
|
3882
|
-
};
|
|
3876
|
+
// Keep-open iterators can still have active remote state even when
|
|
3877
|
+
// their pending count has already drained to zero.
|
|
3878
|
+
const closeRequest = new types.CloseIteratorRequest({
|
|
3879
|
+
id: queryRequestCoerced.id,
|
|
3880
|
+
});
|
|
3881
|
+
const selfHash = this.node.identity.publicKey.hashcode();
|
|
3882
|
+
const remotePeers = keepRemoteAlive
|
|
3883
|
+
? [...peerBufferMap.keys()].filter((peer) => peer !== selfHash)
|
|
3884
|
+
: [...peerBufferMap.entries()]
|
|
3885
|
+
.filter(([peer, buffer]) => peer !== selfHash && buffer.kept > 0)
|
|
3886
|
+
.map(([peer]) => peer);
|
|
3887
|
+
peerBufferMap.clear();
|
|
3888
|
+
await Promise.allSettled(
|
|
3889
|
+
remotePeers.map((peer) =>
|
|
3890
|
+
this._query.send(closeRequest, {
|
|
3891
|
+
...options,
|
|
3892
|
+
mode: new SilentDelivery({ to: [peer], redundancy: 1 }),
|
|
3893
|
+
}),
|
|
3894
|
+
),
|
|
3895
|
+
);
|
|
3896
|
+
};
|
|
3883
3897
|
options?.signal && options.signal.addEventListener("abort", close);
|
|
3884
3898
|
|
|
3885
3899
|
let doneFn = () => {
|
|
@@ -4122,7 +4136,7 @@ export class DocumentIndex<
|
|
|
4122
4136
|
const indexKey = indexerTypes.toId(
|
|
4123
4137
|
this.indexByResolver(result.value),
|
|
4124
4138
|
).primitive;
|
|
4125
|
-
if (result
|
|
4139
|
+
if (isResultValue(result)) {
|
|
4126
4140
|
const existingIndexed = indexedPlaceholders?.get(indexKey);
|
|
4127
4141
|
if (existingIndexed) {
|
|
4128
4142
|
existingIndexed.value =
|
|
@@ -4165,18 +4179,20 @@ export class DocumentIndex<
|
|
|
4165
4179
|
indexed,
|
|
4166
4180
|
});
|
|
4167
4181
|
} else {
|
|
4182
|
+
const indexedResult =
|
|
4183
|
+
result as unknown as types.ResultIndexedValue<I>;
|
|
4168
4184
|
const indexed = coerceWithContext(
|
|
4169
|
-
|
|
4170
|
-
|
|
4185
|
+
indexedResult.indexed || indexedResult.value,
|
|
4186
|
+
indexedResult.context,
|
|
4171
4187
|
);
|
|
4172
4188
|
const late = isLateResult(indexed);
|
|
4173
4189
|
if (late) {
|
|
4174
4190
|
lateCount++;
|
|
4175
4191
|
lateResults?.push({
|
|
4176
4192
|
indexed,
|
|
4177
|
-
context:
|
|
4193
|
+
context: indexedResult.context,
|
|
4178
4194
|
from: from!,
|
|
4179
|
-
value:
|
|
4195
|
+
value: indexedResult.value,
|
|
4180
4196
|
});
|
|
4181
4197
|
if (outOfOrderMode === "drop") {
|
|
4182
4198
|
visited.add(indexKey);
|
|
@@ -4188,8 +4204,8 @@ export class DocumentIndex<
|
|
|
4188
4204
|
}
|
|
4189
4205
|
visited.add(indexKey);
|
|
4190
4206
|
const placeholder = {
|
|
4191
|
-
value:
|
|
4192
|
-
context:
|
|
4207
|
+
value: indexedResult.value,
|
|
4208
|
+
context: indexedResult.context,
|
|
4193
4209
|
from,
|
|
4194
4210
|
indexed,
|
|
4195
4211
|
};
|
|
@@ -4240,7 +4256,7 @@ export class DocumentIndex<
|
|
|
4240
4256
|
continue;
|
|
4241
4257
|
}
|
|
4242
4258
|
const payload = response.response;
|
|
4243
|
-
if (!(payload
|
|
4259
|
+
if (!isResults(payload)) {
|
|
4244
4260
|
continue;
|
|
4245
4261
|
}
|
|
4246
4262
|
await mergePrefetchedResults(
|
|
@@ -4477,7 +4493,9 @@ export class DocumentIndex<
|
|
|
4477
4493
|
}
|
|
4478
4494
|
|
|
4479
4495
|
const remoteConfig =
|
|
4480
|
-
options && typeof options.remote === "object"
|
|
4496
|
+
options && typeof options.remote === "object"
|
|
4497
|
+
? options.remote
|
|
4498
|
+
: undefined;
|
|
4481
4499
|
const remoteWaitPolicy =
|
|
4482
4500
|
remoteConfig && typeof remoteConfig.wait === "object"
|
|
4483
4501
|
? remoteConfig.wait
|
|
@@ -4485,63 +4503,62 @@ export class DocumentIndex<
|
|
|
4485
4503
|
const remoteWaitBehavior: WaitBehavior =
|
|
4486
4504
|
remoteWaitPolicy?.behavior ?? "keep-open";
|
|
4487
4505
|
const keepRemoteWaitOpen =
|
|
4488
|
-
!!remoteConfig?.wait &&
|
|
4489
|
-
remoteWaitBehavior === "keep-open";
|
|
4506
|
+
!!remoteConfig?.wait && remoteWaitBehavior === "keep-open";
|
|
4490
4507
|
let fetchLateJoinPeers = async (
|
|
4491
4508
|
_candidateHashes?: Iterable<string>,
|
|
4492
4509
|
_candidateKeys?: Map<string, PublicSignKey>,
|
|
4493
4510
|
) => false;
|
|
4494
4511
|
|
|
4495
|
-
|
|
4496
|
-
|
|
4497
|
-
|
|
4498
|
-
updateDeferred = pDefer<void>();
|
|
4499
|
-
const lateJoinFetchesInFlight = new Set<string>();
|
|
4500
|
-
const shouldIgnoreLateJoinFetchError = (error: unknown) => {
|
|
4501
|
-
if (
|
|
4502
|
-
this.closed ||
|
|
4503
|
-
ensureController().signal.aborted ||
|
|
4504
|
-
error instanceof ClosedError ||
|
|
4505
|
-
error instanceof AbortError
|
|
4506
|
-
) {
|
|
4507
|
-
return true;
|
|
4508
|
-
}
|
|
4509
|
-
return (
|
|
4510
|
-
error instanceof Error &&
|
|
4511
|
-
error.message.trim().toLowerCase() === "closed"
|
|
4512
|
-
);
|
|
4513
|
-
};
|
|
4512
|
+
if (keepRemoteWaitOpen) {
|
|
4513
|
+
// was used to account for missed results when a peer joins; omitted in this minimal handler
|
|
4514
4514
|
|
|
4515
|
-
|
|
4516
|
-
|
|
4517
|
-
|
|
4518
|
-
|
|
4519
|
-
|
|
4520
|
-
|
|
4521
|
-
|
|
4522
|
-
|
|
4523
|
-
|
|
4524
|
-
|
|
4515
|
+
updateDeferred = pDefer<void>();
|
|
4516
|
+
const lateJoinFetchesInFlight = new Set<string>();
|
|
4517
|
+
const shouldIgnoreLateJoinFetchError = (error: unknown) => {
|
|
4518
|
+
if (
|
|
4519
|
+
this.closed ||
|
|
4520
|
+
ensureController().signal.aborted ||
|
|
4521
|
+
error instanceof ClosedError ||
|
|
4522
|
+
error instanceof AbortError
|
|
4523
|
+
) {
|
|
4524
|
+
return true;
|
|
4525
|
+
}
|
|
4526
|
+
return (
|
|
4527
|
+
error instanceof Error &&
|
|
4528
|
+
error.message.trim().toLowerCase() === "closed"
|
|
4529
|
+
);
|
|
4530
|
+
};
|
|
4525
4531
|
|
|
4526
|
-
|
|
4527
|
-
|
|
4528
|
-
|
|
4532
|
+
fetchLateJoinPeers = async (
|
|
4533
|
+
candidateHashes?: Iterable<string>,
|
|
4534
|
+
candidateKeys?: Map<string, PublicSignKey>,
|
|
4535
|
+
) => {
|
|
4536
|
+
if (totalFetchedCounter === 0) {
|
|
4537
|
+
return false;
|
|
4538
|
+
}
|
|
4539
|
+
if (this.closed || ensureController().signal.aborted) {
|
|
4540
|
+
return false;
|
|
4541
|
+
}
|
|
4529
4542
|
|
|
4530
|
-
|
|
4531
|
-
|
|
4532
|
-
|
|
4533
|
-
|
|
4534
|
-
|
|
4535
|
-
|
|
4536
|
-
|
|
4537
|
-
|
|
4538
|
-
|
|
4539
|
-
|
|
4540
|
-
|
|
4541
|
-
|
|
4542
|
-
|
|
4543
|
-
|
|
4543
|
+
if (done) {
|
|
4544
|
+
unsetDone();
|
|
4545
|
+
}
|
|
4546
|
+
|
|
4547
|
+
const selfHash = this.node.identity.publicKey.hashcode();
|
|
4548
|
+
const knownCandidateKeys = candidateKeys
|
|
4549
|
+
? new Map(candidateKeys)
|
|
4550
|
+
: new Map<string, PublicSignKey>();
|
|
4551
|
+
let hashes: string[];
|
|
4552
|
+
try {
|
|
4553
|
+
hashes = candidateHashes
|
|
4554
|
+
? [...candidateHashes]
|
|
4555
|
+
: [...(await this._log.getReplicators()).keys()];
|
|
4556
|
+
} catch (error) {
|
|
4557
|
+
if (shouldIgnoreLateJoinFetchError(error)) {
|
|
4558
|
+
return false;
|
|
4544
4559
|
}
|
|
4560
|
+
throw error;
|
|
4561
|
+
}
|
|
4545
4562
|
let missing = hashes.filter((hash) => {
|
|
4546
4563
|
if (hash === selfHash) return false;
|
|
4547
4564
|
if (peerBufferMap.has(hash)) return false;
|
|
@@ -4604,23 +4621,23 @@ export class DocumentIndex<
|
|
|
4604
4621
|
return false;
|
|
4605
4622
|
}
|
|
4606
4623
|
|
|
4607
|
-
|
|
4608
|
-
|
|
4609
|
-
|
|
4610
|
-
|
|
4611
|
-
|
|
4612
|
-
|
|
4613
|
-
|
|
4614
|
-
|
|
4615
|
-
|
|
4616
|
-
|
|
4617
|
-
|
|
4618
|
-
}
|
|
4619
|
-
throw error;
|
|
4624
|
+
const lateJoinFetchPromise = trackFetch(
|
|
4625
|
+
fetchFirst(totalFetchedCounter, {
|
|
4626
|
+
from: unresolved,
|
|
4627
|
+
fetchedFirstForRemote,
|
|
4628
|
+
}),
|
|
4629
|
+
);
|
|
4630
|
+
try {
|
|
4631
|
+
await lateJoinFetchPromise;
|
|
4632
|
+
} catch (error) {
|
|
4633
|
+
if (shouldIgnoreLateJoinFetchError(error)) {
|
|
4634
|
+
return false;
|
|
4620
4635
|
}
|
|
4621
|
-
|
|
4622
|
-
|
|
4623
|
-
|
|
4636
|
+
throw error;
|
|
4637
|
+
}
|
|
4638
|
+
for (const hash of unresolved) {
|
|
4639
|
+
if (!peerBufferMap.has(hash)) {
|
|
4640
|
+
fetchedFirstForRemote?.delete(hash);
|
|
4624
4641
|
}
|
|
4625
4642
|
}
|
|
4626
4643
|
|