@peerbit/document 9.4.3-fb47029 → 9.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/program.d.ts +2 -7
- package/dist/src/program.d.ts.map +1 -1
- package/dist/src/program.js +1 -2
- package/dist/src/program.js.map +1 -1
- package/dist/src/resumable-iterator.d.ts +1 -1
- package/dist/src/resumable-iterator.d.ts.map +1 -1
- package/dist/src/resumable-iterator.js +2 -10
- package/dist/src/resumable-iterator.js.map +1 -1
- package/dist/src/search.d.ts +3 -24
- package/dist/src/search.d.ts.map +1 -1
- package/dist/src/search.js +46 -209
- package/dist/src/search.js.map +1 -1
- package/package.json +74 -75
- package/src/operation.ts +9 -9
- package/src/program.ts +2 -9
- package/src/resumable-iterator.ts +2 -10
- package/src/search.ts +68 -343
- package/dist/src/most-common-query-predictor.d.ts +0 -38
- package/dist/src/most-common-query-predictor.d.ts.map +0 -1
- package/dist/src/most-common-query-predictor.js +0 -115
- package/dist/src/most-common-query-predictor.js.map +0 -1
- package/dist/src/prefetch.d.ts +0 -22
- package/dist/src/prefetch.d.ts.map +0 -1
- package/dist/src/prefetch.js +0 -47
- package/dist/src/prefetch.js.map +0 -1
- package/src/most-common-query-predictor.ts +0 -161
- package/src/prefetch.ts +0 -80
|
@@ -1,115 +0,0 @@
|
|
|
1
|
-
import { deserialize, serialize } from "@dao-xyz/borsh";
|
|
2
|
-
import { randomBytes, toBase64 } from "@peerbit/crypto";
|
|
3
|
-
/* ───────────────────── helpers ───────────────────── */
|
|
4
|
-
const nullifyQuery = (query) => {
|
|
5
|
-
const cloned = deserialize(serialize(query), query.constructor);
|
|
6
|
-
cloned.id = new Uint8Array(32);
|
|
7
|
-
return cloned;
|
|
8
|
-
};
|
|
9
|
-
export const idAgnosticQueryKey = (query) => toBase64(serialize(nullifyQuery(query)));
|
|
10
|
-
/**
|
|
11
|
-
* Learns the most common recent queries and predicts the most frequent one.
|
|
12
|
-
* If we just pre-empted a peer with some query, the *first* matching request
|
|
13
|
-
* that arrives from that peer within `ignoreWindow` ms is ignored.
|
|
14
|
-
*/
|
|
15
|
-
export default class MostCommonQueryPredictor {
|
|
16
|
-
threshold;
|
|
17
|
-
ttl;
|
|
18
|
-
ignoreWindow;
|
|
19
|
-
queries = new Map();
|
|
20
|
-
/**
|
|
21
|
-
* predicted:
|
|
22
|
-
* requestKey → Map<peerHash, timestamp>
|
|
23
|
-
*/
|
|
24
|
-
predicted = new Map();
|
|
25
|
-
constructor(threshold, ttl = 10 * 60 * 1000, // 10 min
|
|
26
|
-
ignoreWindow = 5_000) {
|
|
27
|
-
this.threshold = threshold;
|
|
28
|
-
this.ttl = ttl;
|
|
29
|
-
this.ignoreWindow = ignoreWindow;
|
|
30
|
-
}
|
|
31
|
-
/* ───────── housekeeping ───────── */
|
|
32
|
-
cleanupQueries(now) {
|
|
33
|
-
for (const [key, stats] of this.queries) {
|
|
34
|
-
if (now - stats.lastSeen > this.ttl) {
|
|
35
|
-
this.queries.delete(key);
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
}
|
|
39
|
-
cleanupPredictions(now) {
|
|
40
|
-
for (const [key, peerMap] of this.predicted) {
|
|
41
|
-
for (const [peer, ts] of peerMap) {
|
|
42
|
-
if (now - ts > this.ignoreWindow) {
|
|
43
|
-
peerMap.delete(peer);
|
|
44
|
-
}
|
|
45
|
-
}
|
|
46
|
-
if (peerMap.size === 0) {
|
|
47
|
-
this.predicted.delete(key);
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
/* ───────── public API ───────── */
|
|
52
|
-
onRequest(request, { from }) {
|
|
53
|
-
const now = Date.now();
|
|
54
|
-
const peerHash = from.hashcode();
|
|
55
|
-
const key = idAgnosticQueryKey(request);
|
|
56
|
-
/* — 1. Ignore if this (key, peer) pair was just predicted — */
|
|
57
|
-
const peerMap = this.predicted.get(key);
|
|
58
|
-
const ts = peerMap?.get(peerHash);
|
|
59
|
-
let ignore = false;
|
|
60
|
-
if (ts !== undefined && now - ts <= this.ignoreWindow) {
|
|
61
|
-
peerMap.delete(peerHash); // one-shot
|
|
62
|
-
if (peerMap.size === 0) {
|
|
63
|
-
this.predicted.delete(key);
|
|
64
|
-
}
|
|
65
|
-
ignore = true;
|
|
66
|
-
}
|
|
67
|
-
/* — 2. Learn from the request — */
|
|
68
|
-
const stats = this.queries.get(key);
|
|
69
|
-
if (stats) {
|
|
70
|
-
stats.count += 1;
|
|
71
|
-
stats.lastSeen = now;
|
|
72
|
-
}
|
|
73
|
-
else {
|
|
74
|
-
this.queries.set(key, {
|
|
75
|
-
queryBytes: serialize(request),
|
|
76
|
-
queryClazz: request.constructor,
|
|
77
|
-
count: 1,
|
|
78
|
-
lastSeen: now,
|
|
79
|
-
});
|
|
80
|
-
}
|
|
81
|
-
/* — 3. Maintenance — */
|
|
82
|
-
this.cleanupQueries(now);
|
|
83
|
-
this.cleanupPredictions(now);
|
|
84
|
-
return { ignore };
|
|
85
|
-
}
|
|
86
|
-
predictedQuery(from) {
|
|
87
|
-
const now = Date.now();
|
|
88
|
-
this.cleanupQueries(now);
|
|
89
|
-
this.cleanupPredictions(now);
|
|
90
|
-
/* pick the most frequent query meeting the threshold */
|
|
91
|
-
let winnerKey;
|
|
92
|
-
let winnerCount = 0;
|
|
93
|
-
for (const [key, { count }] of this.queries) {
|
|
94
|
-
if (count > winnerCount) {
|
|
95
|
-
winnerKey = key;
|
|
96
|
-
winnerCount = count;
|
|
97
|
-
}
|
|
98
|
-
}
|
|
99
|
-
if (!winnerKey || winnerCount < this.threshold)
|
|
100
|
-
return undefined;
|
|
101
|
-
const winner = this.queries.get(winnerKey);
|
|
102
|
-
const cloned = deserialize(winner.queryBytes, winner.queryClazz);
|
|
103
|
-
cloned.id = randomBytes(32);
|
|
104
|
-
/* remember that we pre-empted `from` with this query */
|
|
105
|
-
const peerHash = from.hashcode();
|
|
106
|
-
let peerMap = this.predicted.get(winnerKey);
|
|
107
|
-
if (!peerMap) {
|
|
108
|
-
peerMap = new Map();
|
|
109
|
-
this.predicted.set(winnerKey, peerMap);
|
|
110
|
-
}
|
|
111
|
-
peerMap.set(peerHash, now);
|
|
112
|
-
return cloned;
|
|
113
|
-
}
|
|
114
|
-
}
|
|
115
|
-
//# sourceMappingURL=most-common-query-predictor.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"most-common-query-predictor.js","sourceRoot":"","sources":["../../src/most-common-query-predictor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqB,WAAW,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3E,OAAO,EAAsB,WAAW,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAG5E,yDAAyD;AAEzD,MAAM,YAAY,GAAG,CACpB,KAAuD,EACtD,EAAE;IACH,MAAM,MAAM,GAAG,WAAW,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,WAAW,CAEjC,CAAC;IAC9B,MAAM,CAAC,EAAE,GAAG,IAAI,UAAU,CAAC,EAAE,CAAC,CAAC;IAC/B,OAAO,MAAM,CAAC;AACf,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAG,CACjC,KAAuD,EACtD,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAsB9C;;;;GAIG;AACH,MAAM,CAAC,OAAO,OAAO,wBAAwB;IAU1B;IACA;IACA;IAXD,OAAO,GAAG,IAAI,GAAG,EAAsB,CAAC;IAEzD;;;OAGG;IACc,SAAS,GAAG,IAAI,GAAG,EAA+B,CAAC;IAEpE,YACkB,SAAiB,EACjB,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,SAAS;IAC/B,eAAe,KAAK;QAFpB,cAAS,GAAT,SAAS,CAAQ;QACjB,QAAG,GAAH,GAAG,CAAiB;QACpB,iBAAY,GAAZ,YAAY,CAAQ;IACnC,CAAC;IAEJ,sCAAsC;IAC9B,cAAc,CAAC,GAAW;QACjC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACzC,IAAI,GAAG,GAAG,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACrC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC1B,CAAC;QACF,CAAC;IACF,CAAC;IAEO,kBAAkB,CAAC,GAAW;QACrC,KAAK,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YAC7C,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,OAAO,EAAE,CAAC;gBAClC,IAAI,GAAG,GAAG,EAAE,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACtB,CAAC;YACF,CAAC;YACD,IAAI,OAAO,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;gBACxB,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC5B,CAAC;QACF,CAAC;IACF,CAAC;IAED,oCAAoC;IAEpC,SAAS,CACR,OAAyD,EACzD,EAAE,IAAI,EAA2B;QAEjC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAExC,+DAA+D;QAC/D,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,EAAE,GAAG,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;QAClC,IAAI,MAAM,GAAG,KAAK,CAAC;QACnB,IAAI,EAAE,KAAK,SAAS,IAAI,GAAG,GAAG,EAAE,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACvD,OAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW;YACtC,IAAI,OAAQ,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;gBACzB,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC5B,CAAC;YACD,MAAM,GAAG,IAAI,CAAC;QACf,CAAC;QAED,mCAAmC;QACnC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,KAAK,EAAE,CAAC;YACX,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC;YACjB,KAAK,CAAC,QAAQ,GAAG,GAAG,CAAC;QACtB,CAAC;aAAM,CAAC;YACP,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE;gBACrB,UAAU,EAAE,SAAS,CAAC,OAAO,CAAC;gBAC9B,UAAU,EAAE,OAAO,CAAC,WAAW;gBAC/B,KAAK,EAAE,CAAC;gBACR,QAAQ,EAAE,GAAG;aACb,CAAC,CAAC;QACJ,CAAC;QAED,wBAAwB;QACxB,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;QAE7B,OAAO,EAAE,MAAM,EAAE,CAAC;IACnB,CAAC;IAED,cAAc,CACb,IAAmB;QAEnB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;QAE7B,wDAAwD;QACxD,IAAI,SAA6B,CAAC;QAClC,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,KAAK,MAAM,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAC7C,IAAI,KAAK,GAAG,WAAW,EAAE,CAAC;gBACzB,SAAS,GAAG,GAAG,CAAC;gBAChB,WAAW,GAAG,KAAK,CAAC;YACrB,CAAC;QACF,CAAC;QACD,IAAI,CAAC,SAAS,IAAI,WAAW,GAAG,IAAI,CAAC,SAAS;YAAE,OAAO,SAAS,CAAC;QAEjE,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAE,CAAC;QAC5C,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,UAAU,CAElC,CAAC;QAC9B,MAAM,CAAC,EAAE,GAAG,WAAW,CAAC,EAAE,CAAC,CAAC;QAE5B,wDAAwD;QACxD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QACjC,IAAI,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC5C,IAAI,CAAC,OAAO,EAAE,CAAC;YACd,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;YACpC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACxC,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAE3B,OAAO,MAAM,CAAC;IACf,CAAC;CACD"}
|
package/dist/src/prefetch.d.ts
DELETED
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
import { TypedEventEmitter } from "@libp2p/interface";
|
|
2
|
-
import { Cache } from "@peerbit/cache";
|
|
3
|
-
import type * as types from "@peerbit/document-interface";
|
|
4
|
-
import type { RPCResponse } from "@peerbit/rpc/dist/src";
|
|
5
|
-
type AddEvent = {
|
|
6
|
-
consumable: RPCResponse<types.PredictedSearchRequest<any>>;
|
|
7
|
-
};
|
|
8
|
-
export declare class Prefetch extends TypedEventEmitter<{
|
|
9
|
-
add: CustomEvent<AddEvent>;
|
|
10
|
-
}> {
|
|
11
|
-
private prefetch;
|
|
12
|
-
private searchIdTranslationMap;
|
|
13
|
-
constructor(prefetch?: Cache<RPCResponse<types.PredictedSearchRequest<any>>>, searchIdTranslationMap?: Map<string, Map<string, Uint8Array>>);
|
|
14
|
-
/** Store the prediction **and** notify listeners */
|
|
15
|
-
add(request: RPCResponse<types.PredictedSearchRequest<any>>, keyHash: string): void;
|
|
16
|
-
consume(request: types.SearchRequest | types.SearchRequestIndexed, keyHash: string): RPCResponse<types.PredictedSearchRequest<any>> | undefined;
|
|
17
|
-
clear(request: types.SearchRequest | types.SearchRequestIndexed): void;
|
|
18
|
-
getTranslationMap(request: types.SearchRequest | types.SearchRequestIndexed): Map<string, Uint8Array> | undefined;
|
|
19
|
-
get size(): number;
|
|
20
|
-
}
|
|
21
|
-
export {};
|
|
22
|
-
//# sourceMappingURL=prefetch.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"prefetch.d.ts","sourceRoot":"","sources":["../../src/prefetch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,KAAK,KAAK,KAAK,MAAM,6BAA6B,CAAC;AAC1D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAIzD,KAAK,QAAQ,GAAG;IACf,UAAU,EAAE,WAAW,CAAC,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC;CAC3D,CAAC;AAQF,qBAAa,QAAS,SAAQ,iBAAiB,CAAC;IAC/C,GAAG,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC;CAC3B,CAAC;IAEA,OAAO,CAAC,QAAQ;IAMhB,OAAO,CAAC,sBAAsB;gBANtB,QAAQ,GAAE,KAAK,CACtB,WAAW,CAAC,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAI7C,EACM,sBAAsB,GAAE,GAAG,CAClC,MAAM,EACN,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CACX;IAKd,oDAAoD;IAC7C,GAAG,CACT,OAAO,EAAE,WAAW,CAAC,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,EACvD,OAAO,EAAE,MAAM,GACb,IAAI;IAQA,OAAO,CACb,OAAO,EAAE,KAAK,CAAC,aAAa,GAAG,KAAK,CAAC,oBAAoB,EACzD,OAAO,EAAE,MAAM,GACb,WAAW,CAAC,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS;IAgB7D,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,aAAa,GAAG,KAAK,CAAC,oBAAoB;IAI/D,iBAAiB,CAChB,OAAO,EAAE,KAAK,CAAC,aAAa,GAAG,KAAK,CAAC,oBAAoB,GACvD,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,GAAG,SAAS;IAItC,IAAI,IAAI,WAEP;CACD"}
|
package/dist/src/prefetch.js
DELETED
|
@@ -1,47 +0,0 @@
|
|
|
1
|
-
import { TypedEventEmitter } from "@libp2p/interface";
|
|
2
|
-
import { Cache } from "@peerbit/cache";
|
|
3
|
-
import { idAgnosticQueryKey } from "./most-common-query-predictor";
|
|
4
|
-
const prefetchKey = (request, keyHash) => `${idAgnosticQueryKey(request)} - ${keyHash}`;
|
|
5
|
-
// --------------------------------------------------------------------------
|
|
6
|
-
export class Prefetch extends TypedEventEmitter {
|
|
7
|
-
prefetch;
|
|
8
|
-
searchIdTranslationMap;
|
|
9
|
-
constructor(prefetch = new Cache({
|
|
10
|
-
max: 100,
|
|
11
|
-
ttl: 1e4,
|
|
12
|
-
}), searchIdTranslationMap = new Map()) {
|
|
13
|
-
super();
|
|
14
|
-
this.prefetch = prefetch;
|
|
15
|
-
this.searchIdTranslationMap = searchIdTranslationMap;
|
|
16
|
-
}
|
|
17
|
-
/** Store the prediction **and** notify listeners */
|
|
18
|
-
add(request, keyHash) {
|
|
19
|
-
const key = prefetchKey(request.response.request, keyHash);
|
|
20
|
-
this.prefetch.add(key, request);
|
|
21
|
-
this.dispatchEvent(new CustomEvent("add", { detail: { consumable: request } }));
|
|
22
|
-
}
|
|
23
|
-
consume(request, keyHash) {
|
|
24
|
-
const key = prefetchKey(request, keyHash);
|
|
25
|
-
const pre = this.prefetch.get(key);
|
|
26
|
-
if (!pre)
|
|
27
|
-
return;
|
|
28
|
-
this.prefetch.del(key);
|
|
29
|
-
let peerMap = this.searchIdTranslationMap.get(request.idString);
|
|
30
|
-
if (!peerMap) {
|
|
31
|
-
peerMap = new Map();
|
|
32
|
-
this.searchIdTranslationMap.set(request.idString, peerMap);
|
|
33
|
-
}
|
|
34
|
-
peerMap.set(keyHash, pre.response.request.id);
|
|
35
|
-
return pre;
|
|
36
|
-
}
|
|
37
|
-
clear(request) {
|
|
38
|
-
this.searchIdTranslationMap.delete(request.idString);
|
|
39
|
-
}
|
|
40
|
-
getTranslationMap(request) {
|
|
41
|
-
return this.searchIdTranslationMap.get(request.idString);
|
|
42
|
-
}
|
|
43
|
-
get size() {
|
|
44
|
-
return this.prefetch.size;
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
//# sourceMappingURL=prefetch.js.map
|
package/dist/src/prefetch.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"prefetch.js","sourceRoot":"","sources":["../../src/prefetch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAGvC,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AAOnE,MAAM,WAAW,GAAG,CACnB,OAAyD,EACzD,OAAe,EACd,EAAE,CAAC,GAAG,kBAAkB,CAAC,OAAO,CAAC,MAAM,OAAO,EAAE,CAAC;AAEnD,6EAA6E;AAC7E,MAAM,OAAO,QAAS,SAAQ,iBAE5B;IAEQ;IAMA;IAPT,YACS,WAEJ,IAAI,KAAK,CAAC;QACb,GAAG,EAAE,GAAG;QACR,GAAG,EAAE,GAAG;KACR,CAAC,EACM,yBAGJ,IAAI,GAAG,EAAE;QAEb,KAAK,EAAE,CAAC;QAXA,aAAQ,GAAR,QAAQ,CAKd;QACM,2BAAsB,GAAtB,sBAAsB,CAGjB;IAGd,CAAC;IAED,oDAAoD;IAC7C,GAAG,CACT,OAAuD,EACvD,OAAe;QAEf,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC3D,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAChC,IAAI,CAAC,aAAa,CACjB,IAAI,WAAW,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC,CAC3D,CAAC;IACH,CAAC;IAEM,OAAO,CACb,OAAyD,EACzD,OAAe;QAEf,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC1C,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,CAAC,GAAG;YAAE,OAAO;QAEjB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEvB,IAAI,OAAO,GAAG,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAChE,IAAI,CAAC,OAAO,EAAE,CAAC;YACd,OAAO,GAAG,IAAI,GAAG,EAAE,CAAC;YACpB,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC5D,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC9C,OAAO,GAAG,CAAC;IACZ,CAAC;IAED,KAAK,CAAC,OAAyD;QAC9D,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACtD,CAAC;IAED,iBAAiB,CAChB,OAAyD;QAEzD,OAAO,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC1D,CAAC;IAED,IAAI,IAAI;QACP,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;IAC3B,CAAC;CACD"}
|
|
@@ -1,161 +0,0 @@
|
|
|
1
|
-
import { type AbstractType, deserialize, serialize } from "@dao-xyz/borsh";
|
|
2
|
-
import { type PublicSignKey, randomBytes, toBase64 } from "@peerbit/crypto";
|
|
3
|
-
import type * as types from "@peerbit/document-interface";
|
|
4
|
-
|
|
5
|
-
/* ───────────────────── helpers ───────────────────── */
|
|
6
|
-
|
|
7
|
-
const nullifyQuery = (
|
|
8
|
-
query: types.SearchRequest | types.SearchRequestIndexed,
|
|
9
|
-
) => {
|
|
10
|
-
const cloned = deserialize(serialize(query), query.constructor) as
|
|
11
|
-
| types.SearchRequest
|
|
12
|
-
| types.SearchRequestIndexed;
|
|
13
|
-
cloned.id = new Uint8Array(32);
|
|
14
|
-
return cloned;
|
|
15
|
-
};
|
|
16
|
-
|
|
17
|
-
export const idAgnosticQueryKey = (
|
|
18
|
-
query: types.SearchRequest | types.SearchRequestIndexed,
|
|
19
|
-
) => toBase64(serialize(nullifyQuery(query)));
|
|
20
|
-
|
|
21
|
-
/* ───────────────────── predictor ───────────────────── */
|
|
22
|
-
|
|
23
|
-
export interface QueryPredictor {
|
|
24
|
-
onRequest: (
|
|
25
|
-
request: types.SearchRequest | types.SearchRequestIndexed,
|
|
26
|
-
ctx: { from: PublicSignKey },
|
|
27
|
-
) => { ignore: boolean };
|
|
28
|
-
|
|
29
|
-
predictedQuery: (
|
|
30
|
-
from: PublicSignKey,
|
|
31
|
-
) => types.SearchRequest | types.SearchRequestIndexed | undefined;
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
interface QueryStats {
|
|
35
|
-
count: number;
|
|
36
|
-
lastSeen: number;
|
|
37
|
-
queryBytes: Uint8Array;
|
|
38
|
-
queryClazz: AbstractType<types.SearchRequest | types.SearchRequestIndexed>;
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
/**
|
|
42
|
-
* Learns the most common recent queries and predicts the most frequent one.
|
|
43
|
-
* If we just pre-empted a peer with some query, the *first* matching request
|
|
44
|
-
* that arrives from that peer within `ignoreWindow` ms is ignored.
|
|
45
|
-
*/
|
|
46
|
-
export default class MostCommonQueryPredictor implements QueryPredictor {
|
|
47
|
-
private readonly queries = new Map<string, QueryStats>();
|
|
48
|
-
|
|
49
|
-
/**
|
|
50
|
-
* predicted:
|
|
51
|
-
* requestKey → Map<peerHash, timestamp>
|
|
52
|
-
*/
|
|
53
|
-
private readonly predicted = new Map<string, Map<string, number>>();
|
|
54
|
-
|
|
55
|
-
constructor(
|
|
56
|
-
private readonly threshold: number,
|
|
57
|
-
private readonly ttl = 10 * 60 * 1000, // 10 min
|
|
58
|
-
private readonly ignoreWindow = 5_000, // 5 s
|
|
59
|
-
) {}
|
|
60
|
-
|
|
61
|
-
/* ───────── housekeeping ───────── */
|
|
62
|
-
private cleanupQueries(now: number) {
|
|
63
|
-
for (const [key, stats] of this.queries) {
|
|
64
|
-
if (now - stats.lastSeen > this.ttl) {
|
|
65
|
-
this.queries.delete(key);
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
private cleanupPredictions(now: number) {
|
|
71
|
-
for (const [key, peerMap] of this.predicted) {
|
|
72
|
-
for (const [peer, ts] of peerMap) {
|
|
73
|
-
if (now - ts > this.ignoreWindow) {
|
|
74
|
-
peerMap.delete(peer);
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
if (peerMap.size === 0) {
|
|
78
|
-
this.predicted.delete(key);
|
|
79
|
-
}
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
/* ───────── public API ───────── */
|
|
84
|
-
|
|
85
|
-
onRequest(
|
|
86
|
-
request: types.SearchRequest | types.SearchRequestIndexed,
|
|
87
|
-
{ from }: { from: PublicSignKey },
|
|
88
|
-
): { ignore: boolean } {
|
|
89
|
-
const now = Date.now();
|
|
90
|
-
const peerHash = from.hashcode();
|
|
91
|
-
const key = idAgnosticQueryKey(request);
|
|
92
|
-
|
|
93
|
-
/* — 1. Ignore if this (key, peer) pair was just predicted — */
|
|
94
|
-
const peerMap = this.predicted.get(key);
|
|
95
|
-
const ts = peerMap?.get(peerHash);
|
|
96
|
-
let ignore = false;
|
|
97
|
-
if (ts !== undefined && now - ts <= this.ignoreWindow) {
|
|
98
|
-
peerMap!.delete(peerHash); // one-shot
|
|
99
|
-
if (peerMap!.size === 0) {
|
|
100
|
-
this.predicted.delete(key);
|
|
101
|
-
}
|
|
102
|
-
ignore = true;
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
/* — 2. Learn from the request — */
|
|
106
|
-
const stats = this.queries.get(key);
|
|
107
|
-
if (stats) {
|
|
108
|
-
stats.count += 1;
|
|
109
|
-
stats.lastSeen = now;
|
|
110
|
-
} else {
|
|
111
|
-
this.queries.set(key, {
|
|
112
|
-
queryBytes: serialize(request),
|
|
113
|
-
queryClazz: request.constructor,
|
|
114
|
-
count: 1,
|
|
115
|
-
lastSeen: now,
|
|
116
|
-
});
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
/* — 3. Maintenance — */
|
|
120
|
-
this.cleanupQueries(now);
|
|
121
|
-
this.cleanupPredictions(now);
|
|
122
|
-
|
|
123
|
-
return { ignore };
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
predictedQuery(
|
|
127
|
-
from: PublicSignKey,
|
|
128
|
-
): types.SearchRequest | types.SearchRequestIndexed | undefined {
|
|
129
|
-
const now = Date.now();
|
|
130
|
-
this.cleanupQueries(now);
|
|
131
|
-
this.cleanupPredictions(now);
|
|
132
|
-
|
|
133
|
-
/* pick the most frequent query meeting the threshold */
|
|
134
|
-
let winnerKey: string | undefined;
|
|
135
|
-
let winnerCount = 0;
|
|
136
|
-
for (const [key, { count }] of this.queries) {
|
|
137
|
-
if (count > winnerCount) {
|
|
138
|
-
winnerKey = key;
|
|
139
|
-
winnerCount = count;
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
if (!winnerKey || winnerCount < this.threshold) return undefined;
|
|
143
|
-
|
|
144
|
-
const winner = this.queries.get(winnerKey)!;
|
|
145
|
-
const cloned = deserialize(winner.queryBytes, winner.queryClazz) as
|
|
146
|
-
| types.SearchRequest
|
|
147
|
-
| types.SearchRequestIndexed;
|
|
148
|
-
cloned.id = randomBytes(32);
|
|
149
|
-
|
|
150
|
-
/* remember that we pre-empted `from` with this query */
|
|
151
|
-
const peerHash = from.hashcode();
|
|
152
|
-
let peerMap = this.predicted.get(winnerKey);
|
|
153
|
-
if (!peerMap) {
|
|
154
|
-
peerMap = new Map<string, number>();
|
|
155
|
-
this.predicted.set(winnerKey, peerMap);
|
|
156
|
-
}
|
|
157
|
-
peerMap.set(peerHash, now);
|
|
158
|
-
|
|
159
|
-
return cloned;
|
|
160
|
-
}
|
|
161
|
-
}
|
package/src/prefetch.ts
DELETED
|
@@ -1,80 +0,0 @@
|
|
|
1
|
-
import { TypedEventEmitter } from "@libp2p/interface";
|
|
2
|
-
import { Cache } from "@peerbit/cache";
|
|
3
|
-
import type * as types from "@peerbit/document-interface";
|
|
4
|
-
import type { RPCResponse } from "@peerbit/rpc/dist/src";
|
|
5
|
-
import { idAgnosticQueryKey } from "./most-common-query-predictor";
|
|
6
|
-
|
|
7
|
-
// --- typed helper ---------------------------------------------------------
|
|
8
|
-
type AddEvent = {
|
|
9
|
-
consumable: RPCResponse<types.PredictedSearchRequest<any>>;
|
|
10
|
-
};
|
|
11
|
-
|
|
12
|
-
const prefetchKey = (
|
|
13
|
-
request: types.SearchRequest | types.SearchRequestIndexed,
|
|
14
|
-
keyHash: string,
|
|
15
|
-
) => `${idAgnosticQueryKey(request)} - ${keyHash}`;
|
|
16
|
-
|
|
17
|
-
// --------------------------------------------------------------------------
|
|
18
|
-
export class Prefetch extends TypedEventEmitter<{
|
|
19
|
-
add: CustomEvent<AddEvent>;
|
|
20
|
-
}> {
|
|
21
|
-
constructor(
|
|
22
|
-
private prefetch: Cache<
|
|
23
|
-
RPCResponse<types.PredictedSearchRequest<any>>
|
|
24
|
-
> = new Cache({
|
|
25
|
-
max: 100,
|
|
26
|
-
ttl: 1e4,
|
|
27
|
-
}),
|
|
28
|
-
private searchIdTranslationMap: Map<
|
|
29
|
-
string,
|
|
30
|
-
Map<string, Uint8Array>
|
|
31
|
-
> = new Map(),
|
|
32
|
-
) {
|
|
33
|
-
super();
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
/** Store the prediction **and** notify listeners */
|
|
37
|
-
public add(
|
|
38
|
-
request: RPCResponse<types.PredictedSearchRequest<any>>,
|
|
39
|
-
keyHash: string,
|
|
40
|
-
): void {
|
|
41
|
-
const key = prefetchKey(request.response.request, keyHash);
|
|
42
|
-
this.prefetch.add(key, request);
|
|
43
|
-
this.dispatchEvent(
|
|
44
|
-
new CustomEvent("add", { detail: { consumable: request } }),
|
|
45
|
-
);
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
public consume(
|
|
49
|
-
request: types.SearchRequest | types.SearchRequestIndexed,
|
|
50
|
-
keyHash: string,
|
|
51
|
-
): RPCResponse<types.PredictedSearchRequest<any>> | undefined {
|
|
52
|
-
const key = prefetchKey(request, keyHash);
|
|
53
|
-
const pre = this.prefetch.get(key);
|
|
54
|
-
if (!pre) return;
|
|
55
|
-
|
|
56
|
-
this.prefetch.del(key);
|
|
57
|
-
|
|
58
|
-
let peerMap = this.searchIdTranslationMap.get(request.idString);
|
|
59
|
-
if (!peerMap) {
|
|
60
|
-
peerMap = new Map();
|
|
61
|
-
this.searchIdTranslationMap.set(request.idString, peerMap);
|
|
62
|
-
}
|
|
63
|
-
peerMap.set(keyHash, pre.response.request.id);
|
|
64
|
-
return pre;
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
clear(request: types.SearchRequest | types.SearchRequestIndexed) {
|
|
68
|
-
this.searchIdTranslationMap.delete(request.idString);
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
getTranslationMap(
|
|
72
|
-
request: types.SearchRequest | types.SearchRequestIndexed,
|
|
73
|
-
): Map<string, Uint8Array> | undefined {
|
|
74
|
-
return this.searchIdTranslationMap.get(request.idString);
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
get size() {
|
|
78
|
-
return this.prefetch.size;
|
|
79
|
-
}
|
|
80
|
-
}
|